Last active
August 29, 2015 13:57
-
-
Save Seraf/9449266 to your computer and use it in GitHub Desktop.
lisa client
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from twisted.internet import glib2reactor # for non-GUI apps | |
glib2reactor.install() | |
import sys | |
import signal | |
import gobject | |
from dbus.mainloop.glib import DBusGMainLoop | |
DBusGMainLoop(set_as_default=True) | |
from twisted.internet import ssl, utils | |
from twisted.internet.protocol import ReconnectingClientFactory | |
from twisted.internet.defer import inlineCallbacks, DeferredQueue | |
from twisted.protocols.basic import LineReceiver | |
from twisted.application import internet, service | |
from twisted.python import log | |
import subprocess | |
import json, os | |
from OpenSSL import SSL | |
import platform | |
from twisted.application.internet import TimerService | |
import urllib2 | |
import pygst | |
pygst.require('0.10') | |
gobject.threads_init() | |
import gst | |
path = os.path.abspath(__file__) | |
dir_path = os.path.dirname(path) | |
soundfile = os.path.normpath(dir_path + '/tmp/output.wav') | |
configuration = json.load(open(os.path.normpath(dir_path + '/' + 'configuration/lisa.json'))) | |
sound_queue = DeferredQueue() | |
botname = "" | |
class Recorder: | |
def __init__(self, listener): | |
self.listener = listener | |
self.started = False | |
self.finished = False | |
# Take the tee'd audio input source and record to disk | |
self.pipeline = self.listener.get_pipeline() | |
print dir (self.pipeline) | |
self.recording = self.listener.get_flac_file_location() | |
vader = self.pipeline.get_by_name('vader') | |
vader.connect('vader_start', self.__start__) | |
vader.connect('vader_stop', self.__stop__) | |
bus = self.pipeline.get_bus() | |
bus.add_signal_watch() | |
bus.connect('message::application', self.__application_message__) | |
self.pipeline.set_state(gst.STATE_PLAYING) | |
print " * Listening closely..." | |
gobject.timeout_add_seconds(10, self.cancel) | |
def start(self): | |
self.started = True | |
print " * Recording..." | |
def stop(self): | |
print " * (silence)" | |
gobject.timeout_add_seconds(1, self.stop_now) | |
def stop_now(self): | |
print " # stop_now" | |
if self.finished == True: | |
self.listener.cancel_listening() | |
return | |
self.finished = True | |
print " * Stored recording to ", self.recording | |
self.pipeline.set_state(gst.STATE_NULL) | |
self.listener.answer(self.recording) | |
def cancel(self): | |
print " # cancel", self.finished, self.started | |
if self.finished == True: | |
print " # cancel - noop" | |
return | |
if self.started == False: | |
self.finished = True | |
print " * Not a word in the past 10 seconds, cancelling" | |
self.pipeline.set_state(gst.STATE_NULL) | |
self.listener.cancel_listening() | |
def __start__(self, vader, arg0): | |
print " # vader:start" | |
struct = gst.Structure('vader_start') | |
struct.set_value('arg0', arg0) | |
vader.post_message(gst.message_new_application(vader, struct)) | |
def __stop__(self, vader, arg0): | |
print " # vader:stop" | |
struct = gst.Structure('vader_stop') | |
struct.set_value('arg0', arg0) | |
vader.post_message(gst.message_new_application(vader, struct)) | |
def __application_message__(self, bus, msg): | |
msgtype = msg.structure.get_name() | |
if msgtype == 'vader_stop': | |
self.stop() | |
elif msgtype == 'vader_start': | |
self.start() | |
class Listener: | |
def __init__(self, LisaClient): | |
"""Initialize the speech components""" | |
self.lisaclient = LisaClient | |
self.failed = 0 | |
self.recording = 'tmp/google.flac' | |
self.pipeline = gst.Pipeline("mypipeline") | |
gst.debug("Adding autoaudiosrc") | |
self.source = gst.element_factory_make("autoaudiosrc", "autoaudiosrc") | |
self.pipeline.add(self.source) | |
# add a queue to allow pocketsphinx to recognize more data | |
gst.debug("Adding encoding queue") | |
self.qone = gst.element_factory_make("queue", "qone") | |
self.pipeline.add(self.qone) | |
# audio convert | |
gst.debug("Adding encoding audioconvert") | |
self.recfileconvert = gst.element_factory_make("audioconvert", "recfileconvert") | |
self.pipeline.add(self.recfileconvert) | |
# resample the wav | |
gst.debug("Adding encoding audioresample") | |
self.resampleOne = gst.element_factory_make("audioresample", "resampleOne") | |
self.pipeline.add(self.resampleOne) | |
# adding capsfilter | |
self.capsfilterOne = gst.element_factory_make("capsfilter", "capsfilterOne") | |
self.capsfilterOne.set_property('caps', gst.caps_from_string('audio/x-raw-int, rate=16000, width=16, depth=16, channels=1')) | |
self.pipeline.add(self.capsfilterOne) | |
# Add our tee | |
gst.debug("Adding tee") | |
self.rectee = gst.element_factory_make("tee", "rectee") | |
self.pipeline.add(self.rectee) | |
# Add another queue | |
gst.debug("Adding encoding queue") | |
self.qtwo = gst.element_factory_make("queue", "qtwo") | |
self.pipeline.add(self.qtwo) | |
# Add another audio resample | |
gst.debug("Adding encoding audioresample") | |
self.resampleTwo = gst.element_factory_make("audioresample", "resampleTwo") | |
self.pipeline.add(self.resampleTwo) | |
# adding capsfiltertwo | |
self.capsfilterTwo = gst.element_factory_make("capsfilter", "capsfilterTwo") | |
self.capsfilterTwo.set_property('caps', gst.caps_from_string('audio/x-raw-int, rate=8000')) | |
self.pipeline.add(self.capsfilterTwo) | |
# Add another vader | |
gst.debug("Adding vader element") | |
self.vader = gst.element_factory_make("vader","vader") | |
self.vader.set_property("auto-threshold",False) | |
self.pipeline.add(self.vader) | |
# add pocketsphinx | |
gst.debug("Adding pocketsphinx element") | |
self.pocketsphinx = gst.element_factory_make("pocketsphinx","listener") | |
#print "Pocketsphinx: " | |
#print dir( self.pocketsphinx ) | |
self.pocketsphinx.set_property("lm",'lisa.lm') | |
self.pocketsphinx.set_property("dict",'lisa.dic') | |
self.pipeline.add(self.pocketsphinx) | |
# Add Fakesink | |
gst.debug("Adding fakesink") | |
self.fakesink = gst.element_factory_make("fakesink", "fakesink") | |
self.fakesink.set_property("dump", True) | |
self.pipeline.add(self.fakesink) | |
# creating valve now | |
gst.debug("Adding Valve element") | |
self.recording_valve = gst.element_factory_make('valve') | |
self.recording_valve.set_property("drop",True) | |
self.pipeline.add(self.recording_valve) | |
# another qthree | |
gst.debug("Adding encoding queue") | |
self.qthree = gst.element_factory_make("queue", "qthree") | |
self.pipeline.add(self.qthree) | |
# adding wavenc element | |
gst.debug("Adding wavenc") | |
self.flacenc = gst.element_factory_make("flacenc", "flacenc") | |
self.pipeline.add(self.flacenc) | |
# adding filesink element | |
gst.debug("Adding filesink") | |
self.filesink = gst.element_factory_make("filesink", "filesink") | |
self.filesink.set_property("location", self.recording) | |
self.filesink.set_property("async", False) | |
self.pipeline.add(self.filesink) | |
# link everything needed for listener here | |
# initiate the microphone | |
self.source.link(self.qone) | |
self.qone.link(self.recfileconvert) | |
self.recfileconvert.link(self.resampleOne) | |
self.resampleOne.link(self.rectee) | |
# Take audio source and tee it into pocketsphinx | |
self.rectee.get_request_pad('src%d').link(self.capsfilterOne.get_pad('sink')) | |
self.capsfilterOne.link(self.qtwo) | |
self.qtwo.link(self.resampleTwo) | |
self.resampleTwo.link(self.capsfilterTwo) | |
self.capsfilterTwo.link(self.vader) | |
self.vader.link(self.pocketsphinx) | |
self.pocketsphinx.link(self.fakesink) | |
# Trying to use this here in listener. then only turn on the recorder when we need it on | |
self.rectee.get_request_pad('src%d').link(self.recording_valve.get_pad('sink')) | |
self.recording_valve.set_property('drop',True) | |
self.recording_valve.link(self.qthree) | |
self.qthree.link(self.flacenc) | |
self.flacenc.link(self.filesink) | |
listener = self.pipeline.get_by_name('listener') | |
listener.connect('result', self.__result__) | |
listener.set_property('configured', True) | |
bus = self.pipeline.get_bus() | |
bus.add_signal_watch() | |
bus.connect('message::application', self.__application_message__) | |
self.pipeline.set_state(gst.STATE_PLAYING) | |
def result(self, hyp, uttid): | |
global botname | |
if hyp.lower() == botname.lower(): | |
log.msg("======================") | |
log.msg("%s keyword detected" % botname) | |
self.failed = 0 | |
self.listen() | |
def listen(self): | |
self.pipeline.set_state(gst.STATE_PAUSED) | |
#play a sound to record the user sentence | |
log.msg("listening") | |
self.recording_valve.set_property('drop',False) | |
Recorder(self) | |
self.recording_valve.set_property('drop',True) | |
def cancel_listening(self): | |
self.recording_valve.set_property('drop',True) | |
self.pipeline.set_state(gst.STATE_PLAYING) | |
# question - sound recording | |
def answer(self, question): | |
#play a sound to notify sentence has been recorded | |
print " * Contacting Google" | |
self.pipeline.set_state(gst.STATE_PLAYING) | |
def get_pipeline(self): | |
return self.pipeline | |
def get_flac_file_location(self): | |
return self.recording | |
def __result__(self, listener, text, uttid): | |
struct = gst.Structure('result') | |
struct.set_value('hyp', text) | |
struct.set_value('uttid', uttid) | |
listener.post_message(gst.message_new_application(listener, struct)) | |
def __application_message__(self, bus, msg): | |
msgtype = msg.structure.get_name() | |
if msgtype == 'result': | |
self.result(msg.structure['hyp'], msg.structure['uttid']) | |
@inlineCallbacks | |
def SoundWorker(): | |
data = yield sound_queue.get() | |
command_create = ('-w', soundfile, | |
'-l', configuration['lang'], '"'+ data.encode('UTF-8') + '"') | |
create_sound = yield utils.getProcessOutputAndValue('/usr/bin/pico2wave', path='/usr/bin', args=command_create) | |
command_play = ( '-P', soundfile ) | |
play_sound = yield utils.getProcessOutputAndValue('/usr/bin/aplay', path='/usr/bin', args=command_play) | |
os.remove(soundfile) | |
class LisaClient(LineReceiver): | |
def __init__(self,factory): | |
self.factory = factory | |
self.bot_name = "lisa" | |
botname = "lisa" | |
def sendMessage(self, message, type='chat'): | |
if configuration['debug']['debug_output']: | |
log.msg('OUTPUT: "from": ' + unicode(platform.node()) + ',"type": ' + type + ', "body": ' + unicode(message) + | |
', "zone": ' + configuration['zone'] | |
) | |
self.sendLine(json.dumps( | |
{"from": unicode(platform.node()), "type": type, "body": unicode(message), "zone": configuration['zone']}) | |
) | |
def lineReceived(self, data): | |
datajson = json.loads(data) | |
if configuration['debug']['debug_input']: | |
log.msg("INPUT: " + unicode(datajson)) | |
if datajson['type'] == 'chat': | |
sound_queue.put(datajson['body']) | |
elif datajson['type'] == 'command': | |
if datajson['command'] == 'LOGIN': | |
print "I found login" | |
self.bot_name = unicode(datajson['bot_name']) | |
global botname | |
botname = unicode(datajson['bot_name']) | |
print "setting botname to %s" % self.bot_name | |
sound_queue.put(datajson['body']) | |
def connectionMade(self): | |
log.msg('Connected to Lisa.') | |
if configuration['enable_secure_mode']: | |
ctx = ClientTLSContext() | |
self.transport.startTLS(ctx, self.factory) | |
self.sendMessage(message='LOGIN', type='command') | |
#init gobject threads | |
gobject.threads_init() | |
#we want a main loop | |
main_loop = gobject.MainLoop() | |
#handle sigint | |
signal.signal(signal.SIGINT, signal.SIG_DFL) | |
Listener(self) | |
class LisaClientFactory(ReconnectingClientFactory): | |
def startedConnecting(self, connector): | |
log.msg('Started to connect.') | |
def buildProtocol(self, addr): | |
self.protocol = LisaClient(self) | |
log.msg('Resetting reconnection delay') | |
self.resetDelay() | |
return self.protocol | |
def clientConnectionLost(self, connector, reason): | |
log.err('Lost connection. Reason:', reason) | |
ReconnectingClientFactory.clientConnectionLost(self, connector, reason) | |
def clientConnectionFailed(self, connector, reason): | |
log.err('Connection failed. Reason:', reason) | |
ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) | |
class ClientTLSContext(ssl.ClientContextFactory): | |
isClient = 1 | |
def getContext(self): | |
return SSL.Context(SSL.TLSv1_METHOD) | |
class CtxFactory(ssl.ClientContextFactory): | |
def getContext(self): | |
self.method = SSL.SSLv23_METHOD | |
ctx = ssl.ClientContextFactory.getContext(self) | |
ctx.use_certificate_file(os.path.normpath(dir_path + '/' + 'configuration/ssl/client.crt')) | |
ctx.use_privatekey_file(os.path.normpath(dir_path + '/' + 'configuration/ssl//client.key')) | |
return ctx | |
# Creating MultiService | |
application = service.Application("LISA-Client") | |
multi = service.MultiService() | |
multi.setServiceParent(application) | |
sound_service = TimerService(0.01, SoundWorker) | |
sound_service.setServiceParent(multi) | |
LisaFactory = LisaClientFactory() | |
if configuration['enable_secure_mode']: | |
lisaclientService = internet.TCPClient(configuration['lisa_url'], configuration['lisa_engine_port_ssl'], LisaFactory, CtxFactory()) | |
else: | |
lisaclientService = internet.TCPClient(configuration['lisa_url'], configuration['lisa_engine_port'], LisaFactory) | |
lisaclientService.setServiceParent(multi) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment