Skip to content

Instantly share code, notes, and snippets.

@Seraf
Last active August 29, 2015 13:57
Show Gist options
  • Save Seraf/9449266 to your computer and use it in GitHub Desktop.
Save Seraf/9449266 to your computer and use it in GitHub Desktop.
lisa client
from twisted.internet import glib2reactor # for non-GUI apps
glib2reactor.install()
import sys
import signal
import gobject
from dbus.mainloop.glib import DBusGMainLoop
DBusGMainLoop(set_as_default=True)
from twisted.internet import ssl, utils
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.internet.defer import inlineCallbacks, DeferredQueue
from twisted.protocols.basic import LineReceiver
from twisted.application import internet, service
from twisted.python import log
import subprocess
import json, os
from OpenSSL import SSL
import platform
from twisted.application.internet import TimerService
import urllib2
import pygst
pygst.require('0.10')
gobject.threads_init()
import gst
path = os.path.abspath(__file__)
dir_path = os.path.dirname(path)
soundfile = os.path.normpath(dir_path + '/tmp/output.wav')
configuration = json.load(open(os.path.normpath(dir_path + '/' + 'configuration/lisa.json')))
sound_queue = DeferredQueue()
botname = ""
class Recorder:
def __init__(self, listener):
self.listener = listener
self.started = False
self.finished = False
# Take the tee'd audio input source and record to disk
self.pipeline = self.listener.get_pipeline()
print dir (self.pipeline)
self.recording = self.listener.get_flac_file_location()
vader = self.pipeline.get_by_name('vader')
vader.connect('vader_start', self.__start__)
vader.connect('vader_stop', self.__stop__)
bus = self.pipeline.get_bus()
bus.add_signal_watch()
bus.connect('message::application', self.__application_message__)
self.pipeline.set_state(gst.STATE_PLAYING)
print " * Listening closely..."
gobject.timeout_add_seconds(10, self.cancel)
def start(self):
self.started = True
print " * Recording..."
def stop(self):
print " * (silence)"
gobject.timeout_add_seconds(1, self.stop_now)
def stop_now(self):
print " # stop_now"
if self.finished == True:
self.listener.cancel_listening()
return
self.finished = True
print " * Stored recording to ", self.recording
self.pipeline.set_state(gst.STATE_NULL)
self.listener.answer(self.recording)
def cancel(self):
print " # cancel", self.finished, self.started
if self.finished == True:
print " # cancel - noop"
return
if self.started == False:
self.finished = True
print " * Not a word in the past 10 seconds, cancelling"
self.pipeline.set_state(gst.STATE_NULL)
self.listener.cancel_listening()
def __start__(self, vader, arg0):
print " # vader:start"
struct = gst.Structure('vader_start')
struct.set_value('arg0', arg0)
vader.post_message(gst.message_new_application(vader, struct))
def __stop__(self, vader, arg0):
print " # vader:stop"
struct = gst.Structure('vader_stop')
struct.set_value('arg0', arg0)
vader.post_message(gst.message_new_application(vader, struct))
def __application_message__(self, bus, msg):
msgtype = msg.structure.get_name()
if msgtype == 'vader_stop':
self.stop()
elif msgtype == 'vader_start':
self.start()
class Listener:
def __init__(self, LisaClient):
"""Initialize the speech components"""
self.lisaclient = LisaClient
self.failed = 0
self.recording = 'tmp/google.flac'
self.pipeline = gst.Pipeline("mypipeline")
gst.debug("Adding autoaudiosrc")
self.source = gst.element_factory_make("autoaudiosrc", "autoaudiosrc")
self.pipeline.add(self.source)
# add a queue to allow pocketsphinx to recognize more data
gst.debug("Adding encoding queue")
self.qone = gst.element_factory_make("queue", "qone")
self.pipeline.add(self.qone)
# audio convert
gst.debug("Adding encoding audioconvert")
self.recfileconvert = gst.element_factory_make("audioconvert", "recfileconvert")
self.pipeline.add(self.recfileconvert)
# resample the wav
gst.debug("Adding encoding audioresample")
self.resampleOne = gst.element_factory_make("audioresample", "resampleOne")
self.pipeline.add(self.resampleOne)
# adding capsfilter
self.capsfilterOne = gst.element_factory_make("capsfilter", "capsfilterOne")
self.capsfilterOne.set_property('caps', gst.caps_from_string('audio/x-raw-int, rate=16000, width=16, depth=16, channels=1'))
self.pipeline.add(self.capsfilterOne)
# Add our tee
gst.debug("Adding tee")
self.rectee = gst.element_factory_make("tee", "rectee")
self.pipeline.add(self.rectee)
# Add another queue
gst.debug("Adding encoding queue")
self.qtwo = gst.element_factory_make("queue", "qtwo")
self.pipeline.add(self.qtwo)
# Add another audio resample
gst.debug("Adding encoding audioresample")
self.resampleTwo = gst.element_factory_make("audioresample", "resampleTwo")
self.pipeline.add(self.resampleTwo)
# adding capsfiltertwo
self.capsfilterTwo = gst.element_factory_make("capsfilter", "capsfilterTwo")
self.capsfilterTwo.set_property('caps', gst.caps_from_string('audio/x-raw-int, rate=8000'))
self.pipeline.add(self.capsfilterTwo)
# Add another vader
gst.debug("Adding vader element")
self.vader = gst.element_factory_make("vader","vader")
self.vader.set_property("auto-threshold",False)
self.pipeline.add(self.vader)
# add pocketsphinx
gst.debug("Adding pocketsphinx element")
self.pocketsphinx = gst.element_factory_make("pocketsphinx","listener")
#print "Pocketsphinx: "
#print dir( self.pocketsphinx )
self.pocketsphinx.set_property("lm",'lisa.lm')
self.pocketsphinx.set_property("dict",'lisa.dic')
self.pipeline.add(self.pocketsphinx)
# Add Fakesink
gst.debug("Adding fakesink")
self.fakesink = gst.element_factory_make("fakesink", "fakesink")
self.fakesink.set_property("dump", True)
self.pipeline.add(self.fakesink)
# creating valve now
gst.debug("Adding Valve element")
self.recording_valve = gst.element_factory_make('valve')
self.recording_valve.set_property("drop",True)
self.pipeline.add(self.recording_valve)
# another qthree
gst.debug("Adding encoding queue")
self.qthree = gst.element_factory_make("queue", "qthree")
self.pipeline.add(self.qthree)
# adding wavenc element
gst.debug("Adding wavenc")
self.flacenc = gst.element_factory_make("flacenc", "flacenc")
self.pipeline.add(self.flacenc)
# adding filesink element
gst.debug("Adding filesink")
self.filesink = gst.element_factory_make("filesink", "filesink")
self.filesink.set_property("location", self.recording)
self.filesink.set_property("async", False)
self.pipeline.add(self.filesink)
# link everything needed for listener here
# initiate the microphone
self.source.link(self.qone)
self.qone.link(self.recfileconvert)
self.recfileconvert.link(self.resampleOne)
self.resampleOne.link(self.rectee)
# Take audio source and tee it into pocketsphinx
self.rectee.get_request_pad('src%d').link(self.capsfilterOne.get_pad('sink'))
self.capsfilterOne.link(self.qtwo)
self.qtwo.link(self.resampleTwo)
self.resampleTwo.link(self.capsfilterTwo)
self.capsfilterTwo.link(self.vader)
self.vader.link(self.pocketsphinx)
self.pocketsphinx.link(self.fakesink)
# Trying to use this here in listener. then only turn on the recorder when we need it on
self.rectee.get_request_pad('src%d').link(self.recording_valve.get_pad('sink'))
self.recording_valve.set_property('drop',True)
self.recording_valve.link(self.qthree)
self.qthree.link(self.flacenc)
self.flacenc.link(self.filesink)
listener = self.pipeline.get_by_name('listener')
listener.connect('result', self.__result__)
listener.set_property('configured', True)
bus = self.pipeline.get_bus()
bus.add_signal_watch()
bus.connect('message::application', self.__application_message__)
self.pipeline.set_state(gst.STATE_PLAYING)
def result(self, hyp, uttid):
global botname
if hyp.lower() == botname.lower():
log.msg("======================")
log.msg("%s keyword detected" % botname)
self.failed = 0
self.listen()
def listen(self):
self.pipeline.set_state(gst.STATE_PAUSED)
#play a sound to record the user sentence
log.msg("listening")
self.recording_valve.set_property('drop',False)
Recorder(self)
self.recording_valve.set_property('drop',True)
def cancel_listening(self):
self.recording_valve.set_property('drop',True)
self.pipeline.set_state(gst.STATE_PLAYING)
# question - sound recording
def answer(self, question):
#play a sound to notify sentence has been recorded
print " * Contacting Google"
self.pipeline.set_state(gst.STATE_PLAYING)
def get_pipeline(self):
return self.pipeline
def get_flac_file_location(self):
return self.recording
def __result__(self, listener, text, uttid):
struct = gst.Structure('result')
struct.set_value('hyp', text)
struct.set_value('uttid', uttid)
listener.post_message(gst.message_new_application(listener, struct))
def __application_message__(self, bus, msg):
msgtype = msg.structure.get_name()
if msgtype == 'result':
self.result(msg.structure['hyp'], msg.structure['uttid'])
@inlineCallbacks
def SoundWorker():
data = yield sound_queue.get()
command_create = ('-w', soundfile,
'-l', configuration['lang'], '"'+ data.encode('UTF-8') + '"')
create_sound = yield utils.getProcessOutputAndValue('/usr/bin/pico2wave', path='/usr/bin', args=command_create)
command_play = ( '-P', soundfile )
play_sound = yield utils.getProcessOutputAndValue('/usr/bin/aplay', path='/usr/bin', args=command_play)
os.remove(soundfile)
class LisaClient(LineReceiver):
def __init__(self,factory):
self.factory = factory
self.bot_name = "lisa"
botname = "lisa"
def sendMessage(self, message, type='chat'):
if configuration['debug']['debug_output']:
log.msg('OUTPUT: "from": ' + unicode(platform.node()) + ',"type": ' + type + ', "body": ' + unicode(message) +
', "zone": ' + configuration['zone']
)
self.sendLine(json.dumps(
{"from": unicode(platform.node()), "type": type, "body": unicode(message), "zone": configuration['zone']})
)
def lineReceived(self, data):
datajson = json.loads(data)
if configuration['debug']['debug_input']:
log.msg("INPUT: " + unicode(datajson))
if datajson['type'] == 'chat':
sound_queue.put(datajson['body'])
elif datajson['type'] == 'command':
if datajson['command'] == 'LOGIN':
print "I found login"
self.bot_name = unicode(datajson['bot_name'])
global botname
botname = unicode(datajson['bot_name'])
print "setting botname to %s" % self.bot_name
sound_queue.put(datajson['body'])
def connectionMade(self):
log.msg('Connected to Lisa.')
if configuration['enable_secure_mode']:
ctx = ClientTLSContext()
self.transport.startTLS(ctx, self.factory)
self.sendMessage(message='LOGIN', type='command')
#init gobject threads
gobject.threads_init()
#we want a main loop
main_loop = gobject.MainLoop()
#handle sigint
signal.signal(signal.SIGINT, signal.SIG_DFL)
Listener(self)
class LisaClientFactory(ReconnectingClientFactory):
def startedConnecting(self, connector):
log.msg('Started to connect.')
def buildProtocol(self, addr):
self.protocol = LisaClient(self)
log.msg('Resetting reconnection delay')
self.resetDelay()
return self.protocol
def clientConnectionLost(self, connector, reason):
log.err('Lost connection. Reason:', reason)
ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
log.err('Connection failed. Reason:', reason)
ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
class ClientTLSContext(ssl.ClientContextFactory):
isClient = 1
def getContext(self):
return SSL.Context(SSL.TLSv1_METHOD)
class CtxFactory(ssl.ClientContextFactory):
def getContext(self):
self.method = SSL.SSLv23_METHOD
ctx = ssl.ClientContextFactory.getContext(self)
ctx.use_certificate_file(os.path.normpath(dir_path + '/' + 'configuration/ssl/client.crt'))
ctx.use_privatekey_file(os.path.normpath(dir_path + '/' + 'configuration/ssl//client.key'))
return ctx
# Creating MultiService
application = service.Application("LISA-Client")
multi = service.MultiService()
multi.setServiceParent(application)
sound_service = TimerService(0.01, SoundWorker)
sound_service.setServiceParent(multi)
LisaFactory = LisaClientFactory()
if configuration['enable_secure_mode']:
lisaclientService = internet.TCPClient(configuration['lisa_url'], configuration['lisa_engine_port_ssl'], LisaFactory, CtxFactory())
else:
lisaclientService = internet.TCPClient(configuration['lisa_url'], configuration['lisa_engine_port'], LisaFactory)
lisaclientService.setServiceParent(multi)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment