Voximplant VoxEngine Dialogflow example voicebot showing Dialogflow use on a phone call with DTMF entry

voxengine-pizzabot-demo

// Voximplant Demo showing how to order a Pizza using Dialogflow on the phone
// Includes DTMF entry, custom payloads, no input handling, 

require(Modules.AI)

// Global Vars
const BOT_VOICE = Language.Premium.US_ENGLISH_MALE2
let dialogflow, call, hangup = false;

// Configurable via custom payload
let maxNoInputTime = 10000             // Number of seconds of not hearing anything to wait before prompting the user again
let interToneTime = 3000               // Default time to wait for the next DTMF digit after one is pressed
let maxToneDigits = 12                 // Default number of DTMF digits to look for after one is pressed
let dtmfMode = "event"                 // Send DTMF key presses as events or as "text"
let stopTones = []                     // DTMF tones used to indicate end of digit entry

// Fire a callback if the timer is exceeded
class Timer {
  constructor(period, name, callback) {
    this.expired = false
    this.noInputTimer = null
    this.period = period || 15 * 1000
    this.name = name || "timer" + Date.now()
    this.callback = callback
  }
  start() {
    if (!this.expired) {
      Logger.write("DEBUG: " + this.name + " already running; resetting.")
      this.expired = false
      clearTimeout(this.noInputTimer)
    }

    this.noInputTimer = setTimeout(() => {
      Logger.write("DEBUG: " + this.name + " exceeded")
      this.callback()
    }, this.period)
    Logger.write("DEBUG: " + this.name + " started")

  }
  stop() {
    this.expired = false
    clearTimeout(this.noInputTimer)
    Logger.write("DEBUG: " + this.name + " cleared")
  }
}

// Timers

// This did not work for some reason
//let noInputTimerCallback = () => dialogflow.sendQuery({ event: { name: "voximplant_NO_INPUT", language_code: "en" } })

function noInputTimerCallback(){
  Logger.write("DEBUG: noInputTimerCallback invoked")
  dialogflow.sendQuery({ event: { name: "voximplant_NO_INPUT", language_code: "en" } })
}
let noInputTimer = new Timer(maxNoInputTime, "noInputTimer", noInputTimerCallback)

// Timer for managing DTMF tone input
let toneTimer = new Timer(interToneTime, "toneTimer", toneTimerCallback)

// Inbound call processing
VoxEngine.addEventListener(AppEvents.CallAlerting, (e) => {
  call = e.call
  call.addEventListener(CallEvents.Connected, onCallConnected)
  call.addEventListener(CallEvents.Disconnected, VoxEngine.terminate)
  call.answer()

  // Handle DTMF
  call.handleTones(true)
  call.addEventListener(CallEvents.ToneReceived, onTone)
})

function onCallConnected(e) {

  // Create Dialogflow object
  dialogflow = AI.createDialogflow({
    lang: DialogflowLanguage.ENGLISH_US
  })
  dialogflow.addEventListener(AI.Events.DialogflowResponse, onDialogflowResponse)
  // Set a phone context with phone parameters

  // ToDo: this didnt' work???
  Logger.write("DEBUG: PhoneNumber.getInfo(call.callerid()).number: " + PhoneNumber.getInfo(call.callerid()).number)
  
  let friendlyCallId = PhoneNumber.getInfo(call.callerid()).number.replace("+1", "")
  Logger.write("DEBUG: friendlyCallId: " + friendlyCallId)

  voximplantContext = {
    name: "phone",
    lifespanCount: 99,
    parameters: {
      caller_id_e164: call.callerid(),
      caller_id: friendlyCallId,  // ToDo: see if we can use regex to remove country codes for the US
      called_number: call.number()
    }
  }

  dialogflow.setQueryParameters({ contexts: [voximplantContext] })

  // Sending TELEPHONY_WELCOME event to let the agent says a welcome message
  dialogflow.sendQuery({ event: { name: "TELEPHONY_WELCOME", language_code: "en" } })
  // Playback marker used for better user experience
  dialogflow.addMarker(-300)
  // Start sending media from Dialogflow to the call
  dialogflow.sendMediaTo(call)
  // Dialogflow TTS playback started
  dialogflow.addEventListener(AI.Events.DialogflowPlaybackStarted, (e) => {
    noInputTimer.stop()
  })
  // Playback marker reached - start sending audio from the call to Dialogflow
  dialogflow.addEventListener(AI.Events.DialogflowPlaybackMarkerReached, (e) => {
    call.sendMediaTo(dialogflow)
  })
  // Dialogflow TTS playback finished. Hangup the call if hangup flag was set to true
  dialogflow.addEventListener(AI.Events.DialogflowPlaybackFinished, (e) => {
    if (hangup) {
      call.hangup() // ToDo: see if this handles sequential fulfillment messages properly 
      Logger.write("DEBUG: hanging up")

    }
    noInputTimer.start()
  })

}

// Handle Dialogflow responses
function onDialogflowResponse(e) {
  // If DialogflowResponse with queryResult received - the call stops sending media to Dialogflow
  // in case of response with queryResult but without responseId we can continue sending media to dialogflow
  if (e.response.queryResult !== undefined && e.response.responseId === undefined) {
    if (!noInputTimer.expired && !toneTimer.expired)
      call.sendMediaTo(dialogflow)
  } else if (e.response.queryResult !== undefined && e.response.responseId !== undefined) {
    // Do whatever required with e.response.queryResult or e.response.webhookStatus

    // If we need to hangup because end of conversation has been reached
    if (e.response.queryResult.diagnosticInfo !== undefined &&
      e.response.queryResult.diagnosticInfo.end_conversation == true) {
      Logger.write("DEBUG: end_conversation marked. Hanging-up")

      hangup = true
    }

    // Telephony messages arrive in fulfillmentMessages array
    if (e.response.queryResult.fulfillmentMessages != undefined) {

      // Handle custom payloads
      let customPayload = e.response.queryResult.fulfillmentMessages.filter(msg => msg.payload !==undefined).reduce(((r, c) => Object.assign(r, c.payload)), {})
      Logger.write("DEBUG: custom payload: " + JSON.stringify(customPayload) )

      // ToDo: this was not working - need to skip if payload is empty
      if (Object.keys(customPayload).length !== 0)
        customPayloadHandler(customPayload)

      let telephonyMsgs = e.response.queryResult.fulfillmentMessages
        .filter(msg => msg.platform !== undefined && msg.platform === "TELEPHONY")
      if (telephonyMsgs.length > 0)
        handleTelephony(telephonyMsgs)
    }
  }
}

async function handleTelephony(msgs) {

  // stop the user from interrupting
  call.stopMediaTo(dialogflow)

  for (m of msgs) {
    noInputTimer.stop()
    let r = await processTelephonyMessage(m)
    Logger.write("DEBUG: " + r)
  }

  //if (hangup) call.hangup() //Handle hang-up request after Telephony interaction

  // Start two-way media again
  dialogflow.sendMediaTo(call)
  call.sendMediaTo(dialogflow)
  noInputTimer.start()
  Logger.write("DEBUG: done with handleTelephony")
}

// Process telephony messages from Dialogflow
function processTelephonyMessage(msg) {
  // cnt++;
  // Logger.write("DEBUG: processTelephonyMessage iteration #" + cnt)

  return new Promise(resolve => {

    // Play audio file located at msg.telephonyPlayAudio.audioUri
    if (msg.telephonyPlayAudio !== undefined) {
      // Wait for autio file playback to finish and then process the other Telephony messages 
      call.addEventListener(CallEvents.PlaybackFinished, e => {
        // Logger.write("DEBUG: call.startPlayback from Dialogflow audio file playback finished")
        resolve("call.startPlayback from Dialogflow audio file playback finished")
      })

      // audioUri contains Google Storage URI (gs://), we need to transform it to URL (https://)
      let url = msg.telephonyPlayAudio.audioUri.replace("gs://", "https://storage.googleapis.com/")
      call.startPlayback(url)
    }
    // Synthesize speech from msg.telephonySynthesizeSpeech.text
    else if (msg.telephonySynthesizeSpeech !== undefined) {
      // Wait for autio file playback to finish and then process the other Telephony messages 
      call.addEventListener(CallEvents.PlaybackFinished, e => {
        // Logger.write("DEBUG: call.startPlayback from Dialogflow syntheize speech playback finished")
        resolve("call.startPlayback from Dialogflow synthesize speech playback finished")
      })

      // See the list of available TTS languages at https://voximplant.com/docs/references/voxengine/language
      if (msg.telephonySynthesizeSpeech.ssml !== undefined) call.say(msg.telephonySynthesizeSpeech.ssml, BOT_VOICE)
      else call.say(msg.telephonySynthesizeSpeech.text, BOT_VOICE)
    }
    // Transfer call to msg.telephonyTransferCall.phoneNumber
    else if (msg.telephonyTransferCall !== undefined) {
      Logger.write("DEBUG: telephonyTransferCall not setup")
      // Not used in PizzaBot yet
      // dialogflow.stop()
      // let newcall = VoxEngine.callPSTN(msg.telephonyTransferCall.phoneNumber, VOICEBOT_PHONE_NUMBER)
      // VoxEngine.easyProcess(call, newcall)
      resolve("telephonyTransferCall not setup")

    }
    else {
      Logger.write("DEBUG: unhandled processTelephonyMessage") // on iteration #" + cnt)
      resolve("unhandled processTelephonyMessage : " + JSON.stringify(msg))
    }
  })

}

// Handle custom payload messages
function customPayloadHandler(payload) {
  Logger.write("DEBUG: custom payload was: " + JSON.stringify(payload))

  // ToDo: change to switch case?

  // Allow changing the no input timer
  if (payload.max_no_input_time !== undefined) {
    maxNoInputTime = payload.max_no_input_time
    Logger.write("DEBUG: changed maxNoInputTime to: " + payload.max_no_input_time)
  }
  // Change the playback marker
  if (payload.playbackMarker !== undefined) {
    dialogflow.addMarker(payload.playbackMarker)
    Logger.write("DEBUG: changed playbackMarker to: " + payload.playbackMarker)
  }
  // Set DTMF intertone time
  if (payload.interToneTime !== undefined) {
    interToneTime = payload.interToneTime
    toneTimer.period = payload.interToneTime
    Logger.write("DEBUG: changed interToneTime to: " + payload.interToneTime)
  }
  // Set DTMF max digits to look for before sending digits
  if (payload.maxToneDigits !== undefined) {
    maxToneDigits = payload.maxToneDigits
    Logger.write("DEBUG: changed maxToneDigits to: " + payload.maxToneDigits)
  }
  // Set the DTMF mode to send as event or text
  if (payload.dtmfMode !== undefined && (payload.dtmfMode == "event" || payload.dtmfMode == "text")) {
    dtmfMode = payload.dtmfMode
    Logger.write("DEBUG: changed dtmfMode to: " + payload.dtmfMode)
  }
  // Set DTMF stop tones
  if (payload.stopTones !== undefined && payload.stopTones.length > 0) {
    stopTones = payload.stopTones
    Logger.write("DEBUG: stopTones changed to: " + payload.stopTones.join(", "))
  }

  // ToDo: set a lifespan for payload message parameters
}

let tones = []
let toneCount = 0

function toneTimerCallback() {
  let toneString = tones.join('').toString()  // bug requires adding an extra .toString()
  
  if (toneString == '') {
    Logger.write("DEBUG: toneTimerCallback - invalid toneString: " + toneString)
    return
  }

  Logger.write("DEBUG: sending DTMF in " + dtmfMode + "mode : " + toneString)
  if (dtmfMode == "event")
    dialogflow.sendQuery({ event: { name: "DTMF", language_code: "en", parameters: { dtmf_digits: toneString } } })
  else if (dtmfMode == "text")
     dialogflow.sendQuery({ text: { text: toneString.toString(),  language_code: "en" }}) // bug requires adding an extra .toString()

  toneCount = 0
  tones = []
}

function onTone(e) {
  Logger.write("DEBUG: onTone called: " + e.tone) // on iteration #" + cnt)

  if (stopTones.includes(e.tone)){
    Logger.write("DEBUG: stopTone entered: " + e.tone);
    toneTimerCallback()
    return
  }

  noInputTimer.stop()
  toneCount++;
  tones.push(e.tone)

  if (toneCount >= maxToneDigits){
    Logger.write("DEBUG: maximum number of specified tones reached: " + toneCount) // on iteration #" + cnt)
    toneTimerCallback()
  }
  else
    toneTimer.start()

}

Thanks for your code, it was really helpful
I am able to get 1 set of multiple DTMF values and pass them through Actions through fulfilment in Dialogflow Es
How should we proceed if we want multiple set of dtmf values corresponding to different intents in Dialogflow Es?