# coding=latin-1
#
# File: scribpro.py
# by Justin Mangue
#
# Description:
# This is a script that enables use of a Prolog natural language parsing component to control a Scribbler II robot
# over bluetooth. Use in sync with nlp.pl.
#
# Dependencies:
# Python 2.7.4 http://www.python.org/getit/
# PySwip 0.2.3 https://code.google.com/p/pyswip/
# PySpeech https://code.google.com/p/pyspeech/
# Myro 2.9.5 http://myro.roboteducation.org/download/
# Swi-Prolog 6.2.6 http://www.swi-prolog.org/Download.html
# IMPORTS
from sys import exit
from myro import *
from pyswip import Prolog
import speech as pyspeech
# GLOBAL SETTINGS
verNum = '0.6' # ScribPro Version Number
scribblerPort = 'COM5' # Scribbler Bluetooth Port
prologFile = 'nlp.pl' # Prolog file to consult
debug = 'on' # Debug mode, if enabled, shows detailed prolog parse information
inputMode = "text" # Input in text mode by default
outputMode = "text" # Output to text by default
# CLASSES
class PrologResult(object):
def __init__(self, prologcall, status, pythoncode):
self.prologcall = prologcall
self.status = status
self.pythoncode = pythoncode
# FUNCTIONS
def strClean(string):
""" strClean(str) -- converts "A string like This" into a list of lowercase atoms
i.e. [a,string,like,this] for use in Prolog. also converts commas to "and"s """
finalstring = "["
loweredstring = string.lower()
exclude = '!"#$%&'()*+-./:;<=>?@[\]^_`{|}~'
for c in exclude:
loweredstring = loweredstring.replace(c,"")
for ch in loweredstring:
if ch == ' ':
finalstring = finalstring + ","
elif ch == ',':
finalstring = finalstring + ",comma"
#elif ch == u'xb0' or ch == '°':
# finalstring = finalstring + ",degrees"
else:
finalstring = finalstring + ch
return finalstring + "]"
def parse(inStr):
"""parse(str) -- cleans the user input and then throws it at Prolog for analysis"""
codeOut = "null"
sentence = strClean(inStr)
prologCall = 'parseToCode(' + sentence + ', Status, CodeOut).'
for soln in prolog.query(prologCall, maxresult=1):
statusOut = ''.join(soln["Status"])
if statusOut == 'valid':
codeOut = buildMultiCode(soln["CodeOut"])
return PrologResult(prologCall,statusOut,codeOut)
def buildCode(prologterms):
"""converts a [instruction, param1, param2, ...] list into Python-exec-friendly "instruction(param1,param2,...)" functor format"""
parameters = [str(item) for item in prologterms]
if len(parameters) > 2:
codeOut = parameters[0] + '('
for x in range(len(parameters)-2):
codeOut = codeOut + parameters[x+1] + ', '
codeOut = codeOut + parameters[len(parameters)-1] + ')'
elif len(parameters) == 2:
codeOut = parameters[0] + '(' + parameters[1] + ')'
else:
codeOut = parameters[0] + '()'
return codeOut
def buildMultiCode(instructions):
"""converts a list of [instruction, param1, param2] codes into a list of Python-exec-friendly "instruction(param1,param2,...)" strings
i.e. buildMultiCode([('forward',['1.0','3.0']),('stop',[])]) -> ['forward(1.0, 3.0)', 'stop()']"""
multiCodeOut = []
for x in instructions:
line = buildCode(x)
multiCodeOut.append(line)
return multiCodeOut
def execute(code): # potential security concerns here!
"""executes multiple lines of python script in sequence"""
for line in code:
if type(line) is list:
execute(line)
else:
exec line
def toggle_inputMode(mode):
"""toggles between text and voice mode"""
if mode == "text":
print "Now activating voice input."
return "voice"
else:
print "Switching back to text-input mode."
return "text"
def toggle_outputMode(mode):
"""toggles between text and voice mode"""
if mode == "text":
print "Now activating voice output."
return "voice"
else:
print "Switching back to text-only output mode."
return "text"
def toggle_debugMode():
"""toggles between debug mode being on/off"""
if debug == "on":
print "Disabling debug mode."
return "off"
else:
print "Enabling debug mode."
return "on"
# addon robot commands
def takePhoto(mode):
"""Take and display a photo in the specified color mode. 2=color(fast), 1=color, 0=gray"""
if mode == 2:
pic = takePicture("jpeg-fast")
elif mode == 1:
pic = takePicture("color")
else:
pic = takePicture("gray")
show(pic,"Scribby Cam")
return
def move_until_wall():
"""Repeatedly moves forward in small increments until wall is sensed"""
while not wall():
forward(1.0, 0.7)
return
def moonwalk(time):
"""Moonwalk backwards for time"""
while timeRemaining(time):
forward(.25,.1)
backward(1,.3)
# MAIN
def main():
# global variable references
global debug
global prolog
global inputMode
global outputMode
# initialize the robot
print "***************** ScribPro v" + verNum + " *****************"
init(scribblerPort)
print "Scribbler II found on bluetooth (" + scribblerPort + ")."
# instantiation of Prolog interface
prolog = Prolog()
prolog.consult(prologFile)
print "Prolog has been initialized."
print "Loaded " + prologFile + " into SWI-Prolog!"
print "Starting in " + inputMode + " input mode. ['voice' to toggle]"
print "Starting in " + outputMode + " output mode. ['sound' to toggle]"
print "*************************************************"
# main routine
while(1):
# grab a line of input, either text or voice
if inputMode == "voice":
try:
inStr = pyspeech.input("nAwaiting voice command. (Ctrl-C to enter text)")
print "Voice command: " + inStr + "n"
except KeyboardInterrupt:
inStr = raw_input("Type in a command: ")
else:
inStr = raw_input("nEnter a command: ")
# parse the input and behave appropriately
if (inStr == 'quit' or inStr == 'exit'):
print "Exiting..."
exit(0)
elif inStr == 'reload':
newprolog = Prolog()
newprolog.consult(prologFile)
prolog = newprolog
print "Reloaded " + prologFile + " into Prolog!"
elif inStr == 'voice':
inputMode = toggle_inputMode(inputMode)
elif inStr == 'sound':
outputMode = toggle_outputMode(outputMode)
elif inStr == 'debug':
debug = toggle_debugMode()
else:
Result = parse(inStr)
if Result.status == 'valid':
if outputMode == "voice":
pyspeech.say("Okay")
print "Okay."
execute(Result.pythoncode)
else:
if outputMode == "voice":
errorMsg = "I don't understand " + inStr
pyspeech.say(errorMsg)
print "I don't understand "" + inStr + ""."
# Print detailed debug information
if debug == "on":
print "nProlog query executed:", Result.prologcall
print "Status:", Result.status
print "CodeOut:", Result.pythoncode
if __name__ == "__main__":
main()
scribpro.py
Week 9: Cleaning House (And Code)
This week, I began wrapping up my project. Only one week remains until this thing must be worthy of demonstration in front of a class full of peers, so adding new features has to be de-emphasized in favor of reinforcing stability & adding documentation to the project.
Major New Features:
- Added the ability to give a conditional statement. For now, this is limited to wall sensing. So now, “go forward until you hit a wall” is a legit command.
- Added the ability to repeat a command a specified number of times. i.e. “Turn left ninety degrees twice” or “Beep six times.”
Quality of Life Improvements:
- I reorganized quite a bit of the python code and added docstrings to all the functions.
- I added the ability to toggle voice/text input/output modes as well as debug mode. Previously, these were command-line options or hardcoded.
- Added punctuation filtering to user text input in order to avoid ugly crashes. Should be a lot more user-proof now.
- I added some more documentation to my prolog code.
- Stabilized the photo-taking code. It still crashes in IDLE, but works great via console python.
- MANY small grammar reinforcements & bug fixes.
The last major feature I hope to add is a “help” menu, which will give the user an overview of commands and some examples of usage.
In the meantime, I am beginning to brainstorm about my in-class presentation and am also working on outlining my final project paper.
Next week I will post all of the sources for the final program along with a project summary. In the meantime, here is a writeup of the full command grammar that the robot accepts:
Week 8: You Can Walk The Walk, Now Talk The Talk
I accomplished two tasks this week… adding more robustness to the movement grammar, and integrating speech recognition.
In particular, I added the abilities to parse commands such as:
- “go left/right” — turns left 90 degrees and then moves the given distance.
- “wait” — pauses the robot for x seconds.
- “stop” — stops all movement.
- “turn around” — turns 180 degrees, optionally “to the left”/”to the right”.
- “spin around” — turns 360 degrees, optionally “to the left”/”to the right”.
- “take a picture” — takes a photo using the fluke camera, which is transmitted to the computer via bluetooth and displayed. This functionality is still a little buggy, which I suspect is due to some instability with my version of the Python Imaging Library.
I also loosened up the grammar in some other places to make it more flexible. I wrote the following DCG predicate which has really come in handy:
optional(X) –> [X] ; [].
This is a case of something so obvious, that I don’t know why I didn’t think of it earlier. That simple little line of Prolog code enables the insertion of “optional” words or phrases in a sentence and has become invaluable. It is analogous to square-bracket notation in EBNF grammar definitions.
Integrating the Microsoft Speech Engine with the program proved to be quite a bit easier than I had imagined. I was able to do this via a library called PySpeech. After running through only one round of Microsoft’s voice training program, the engine is fairly accurate and fast enough to be useful. So now, the parser can be run in either text or speech mode– pretty cool! It is a blast to call out voice commands and see the scribbler react.
Next week I plan to wrap up the feature side of the project by adding a few more fun things (“beep” springs to mind) as well as conditional statements (“go forward until you hit a wall”). All that will then remain is to flush out my comments and documentation.
Week 7: Scribby’s First Steps
This week, I worked solely on the Prolog natural language parsing component of the project. This is basically a set of grammar rules along with very simple code-generation based on the results of the parse. I made quite a bit of progress this week… Basic movement commands are now parsed, along with quantities and conversions of a handful of different units. For example, you can now tell the Scribbler such things as:
* go forward for 3 seconds and then turn left
* drive forward three feet and then turn right 180 degrees
* move backwards 6.5 inches
And so on and so forth. Here is a sample run:
Enter a command: go forward two feet, turn right 90 degrees, drive backwards for three seconds
Prolog query executed: parseToCode([go,forward,two,feet,and,turn,right,90,degrees,and,drive,backwards,for,three,seconds], Status, CodeOut).
Status: valid
CodeOut: [‘forward(1.0, 4.1)’, ‘turnRight(1.0, 0.7875)’, ‘backward(1.0, 3)’]
A big component that I had to work on this week was english -> integer translation. It actually requires quite a bit of code to convert an English phrase like “nine hundred forty two point threee” to “942.3”!
My next task is to add some “fun” commands to the program; beeping, taking pictures, and maybe a wander/doodle capability. I would also like to add some flavorful error messages to return upon a failed parse.
Another potential goal is to add conditional statements to the parser, i.e. parsing of a statement like “go forward until you hit a wall and then turn left and take a picture.”
Here is the source code for the Prolog parser as it stands today…
Week 6: Finishing the Python/Prolog interface
This week I finished the python script that handles initialization, user input, and all messaging to & from Prolog. All that is left to do is the natural language parser component in Prolog.
I was learning Python as I wrote this, so a lot of this code could be certainly written in a more elegant way. But it gets the job done.
The biggest obstacle was a silly grammatical problem I encountered was interpreting the Prolog list of code to be executed into Python code. A typical prolog code result arrives in list format [forward, ‘1.0’, ‘3.0’]. I was trying to parse that into Python as forward(1.0, 3.0). Notice the lack of string quotes around Prolog’s forward; this is because in Prolog, its data type is “atom” rather than “string”. But this caused no end of trouble when pulling it into Python, as it was interpreting forward as a functor rather than as a string. No problem, right? It should be easy to just cast it as a string in Python. But despite all my efforts, it wouldn’t work… casting it was yielding a big mess, since it was interpreting it as a function pointer. I ended up having to resolve this issue on the Prolog side with a bit of a hack to get quotes around outgoing atoms.
Anyways, here is the Python code. The main routine parses user input, passes it to Prolog, and then checks for a solution and returns a PrologResult instance with fields prologcall, status, and pythoncode. Based on the result of status, it then executes pythoncode.
A typical command of “scribbler move forward” might yield the following PrologResult:
- prologCall = parseToCode([scribbler,move,forward],Status,CodeOut).
- status = valid
- pythoncode = [forward(3.0, 1.0)]
Week 5 – Adventures in Integration
I came down with a brutal cold this week and did not accomplish as much as I would have liked to. I did make some decent progress, however.
I spent an evening browsing the user documentation for the Myro library, which is a collection of Python functions to send basic commands to the Scribbler. I also worked my way through “Learning Computing With Robots” by Deepak Kumar, a freely-available textbook about using Myro as a platform to learn Python. In the process, I sharped my Python skills and learned my way around Myro.
My other, somewhat more significant advancement this week was successfully integrating Python, Myro and Prolog. I found a hobbyist-made library called PySWIP to facilitate messaging between the Python shell and SWIPl, the implementation of Prolog that I am using. Unfortunately getting this to work turned out to be a nightmare because of dependency and legacy issues… Myro runs best on Python 2.4, while PySWIP requires Python 2.7 and wouldn’t work with newer versions of SWIPL. I had similar issues regarding to 32-bit version versus 64-bit versions.
I could not find any online to solve my quandary– I think I may be the first person to ever try using Prolog to control a Scribbler! After brute forcing my way through several different combinations of versions, eventually I got Myro to run under Python 2.7 by updating all its dependencies manually, and found a version of SWIPL that was compatible with PySWIP.
So now I can import a Prolog file into Python by inputting:
> prolog.consult(‘sample.pl’)
And query it from Python by inputting:
> prolog.query(“male_ancestor(edward_vi, Ancestor)”)
Which returns a Python generator with all of the solutions that Prolog finds. Pretty cool!
Week 4: A sample vocab list…
Working on the parser tonight, I quickly realize that I needed to define a relatively small subset of English to handle. I brainstormed up the following list of Scribbler-relevant commands:
Verbs:
go move drive roll stop turn rotate accelerate decelerate “speed up” “slow down” “back up” beep sing dance wander
Nouns:
light lights forwards backwards left right middle center reverse second seconds millisecond milliseconds degrees percent obstacle object
Proper Nouns:
i me you scribbler
Numbers:
0-? zero one two …
Adjectives:
maximum minimum fast faster slow slower quickly slowly
Misc conjunctions:
then and or
Funny:
love hate don’t
—
Even with such a limited subset, this parser may be more work than I had imagined just due to the sheer number of combinations…
Week 4: My Scribbler 2 has arrived!
I arrived home yesterday to find, sitting on my doorstep, this bad-boy:
(He was inside a box, obviously, but it’s more fun to pretend that he just drove up to my door.)
I ran through the instruction manual and experimented using the different sensors and playing with the built-in modes, which included a cool line-following mode as well as an exploration & obstacle avoidance mode.
Unfortunately, I almost immediately realized that this puppy does not support Bluetooth connectivity out of the gate. Having it tethered to my PC via a Cable is going to be a nuisance, so I ordered a IPRE Fluke2 board. The Fluke2 connects to the Scribbler II via the serial port and allows Bluetooth connectivity, as well as providing a low-resolution camera that may be fun to play with. Once he has his Fluke2 board, my Scribbler will look like this:
While I wait for the Fluke2 to arrive, I have begun brainstorming about the types of tasks I want the user to be able to assign via natural language:
- Movement — Movement control, speed controls, composite movement instructions (i.e. “move forward for 4 seconds then turn left 90 degrees”)
- Sensor Queries — Is an object in front? On the left or right? Is there a detectable line beneath?
- LED Light Control — Control which of the 3 LED lights are turned on. Other possibilities including flashing or siren modes. Also control the Fluke2 “ultra bright” LED.
- Sound Control — Play a basic tune, or just some beeps & boops.
- Miscellaneous — preprogrammed “fun” routines. A draw mode, a dance mode, etc. Remaining battery power queries.
I’m not sure if it’s feasible, but I would also love to show the camera feed in the window next to the command input, to aid in controlling.
I also decided that I want my program to have the option to display the actual code that it is sending to the Scribbler II. This would give the program some utility as a Python education tool.
The bulk of my work this week will be on starting the Natural Language Processing component. Prolog, here I come…
Coding away in the ACC.
To-Do:
- Come up with a name for the program I’m writing, so I’m not just constantly saying “my program”.
- Come up with a name for my Scribbler!
Week 3: Project Proposal – Final Draft
I did some research as well as some soul-searching this week and discovered that my previous project idea, while interesting to me, just wouldn’t show off the strengths of Prolog as much I had hoped. Also, let’s be honest– simulations are great and all, but producing code that executes on a real, tangible robot is just WAY cooler!
So I have revised my project idea to focus on using Prolog as a natural language parser to control a robot. I will be using the Scribbler II, as it has pre-existing support for controlling it wirelessly using Python code– and Python can make calls to SWiPl, the Prolog interpreter I am most familiar with.
Following is the final revised draft of my project proposal:
Project Title: Controlling Robots Using Natural Language Processing
Areas of research
Natural Language Parsing, Prolog, Python, AI, robotics, as well as cross-paradigm language interaction.
Project description
I plan to use the Prolog programming language to build a natural language parsing engine to be used as a controller for a simple robot. Why is this of any value? Because Prolog’s resolution engine offers very finely-tuned depth-first-search and backtracking systems, and Prolog itself benefits from a rich history of natural language parsing application (see IBM’s Watson).
The end goal is to produce a python console program which prompts the user for instructions. The user can give natural language commands that python parses and evaluates using Prolog, which then determines an optimal sequence of instructions that it passes back to Python for execution.
I have created a blog to track my progress located at:
I hope to come away from this project having developed my knowledge of Prolog, as well as gaining an understanding for basic natural language processing methodology.
Prior knowledge
I previously studied Prolog in the Computability program here at Evergreen. The main projects I have written so far include a lexer and parser for a subset of C, an implementation of the Bellman-Ford shortest path algorithm, and some constraint-based logic puzzle solvers.
Resources needed to accomplish the project (textbooks, manuals)
I will be primarily using the “Scribbler” hobby robot, which has existing support for control using Python.
On the Prolog end, I will be using the SwiPL implementation which offers some good built-in API functionality for calling it from outside languages.
Other students you know of (if any) interested in related topics
My affinity group consists of Victoria, Jennifer, Jesse, Brian, David, Savannah, and Jason, who are all doing robotics-related projects. Richard will be the faculty advisor to this project. I know of no one else who is interested in Prolog!
Short Bibliography
I intend to use the two following textbooks to reinforce my Prolog knowledge:
“The Art of Prolog”, Sterling and Shapiro.
“Prolog Programming for Artificial Intelligence”, Ivan Bratko.
I still need to find some more good web resources on interacting with the Scribbler using Python.
Week 2: Revised Project Proposal
I have revised my project, having honed in the scope of the project after discussing it with fellow students and faculty. I have also done some research and decided on target robotics simulation suite.
Following is the revised project proposal:
Project title: Using Prolog As A Robotics Decision-Making Tool
Area of research
AI and robotic simulation, as well as cross-paradigm language interaction.
Short project description
I hope to integrate the Prolog language with a robotics simulation suite. The end goal of this is to allow for usage of logic-based programming in simple robotic decision making.
Why is this of any value? Because Prolog’s resolution engine offers very finely-tuned depth-first-search and backtracking systems, and Prolog itself benefits from a rich history of AI application.
If it works, I should be able to create a simple robotic simulation which is using a Prolog component to perform some simple decision-making tasks. I hope to extend this to touch on “Truth Maintenance Systems”, under recommendation from Richard.
I have created a blog to track my progress located at:
http://www.musicallyinept.com/blog/category/computing/cpat-project/
Prior knowledge
I previously studied Prolog in the Computability program here at Evergreen. The main projects I have written so far include a lexer and parser for a subset of C, an implementation of the Bellman-Ford shortest path algorithm, and some constraint-based logic puzzle solvers.
Resources needed to accomplish the project (textbooks, manuals)
I have found a simulation package called Webots which has a free 30-day trial and supports C, which I am much more confident that I can successfully integrate Prolog with.
On the Prolog end, I will be using the SwiPL implementation which offers some good built-in API functionality for calling it from outside languages.
Other students you know of (if any) interested in related topics
My affinity group consists of Victoria, Jennifer, Jesse, Brian, David, and Jason, who are all doing robotics-related projects. Richard will be the faculty advisor to this project. I know of no one else who is interested in Prolog!
Short Bibliography
I intend to use the two following textbooks to reinforce my Prolog knowledge:
I still need to find some good sources on robotics simulation and truth maintenance systems.