Talking wrist watch for M5Stack

0_talking_watch.ino

// Talking wrist watch for M5Stack
// 2018/4/8 @ksasao
// 
// see:
// https://twitter.com/meganetaaan/status/982258576361078784
// http://blog-yama.a-quest.com/?eid=970191
// prerequisite: AquesTalk pico for ESP32
// https://www.a-quest.com/download.html#a-etc
#include <M5Stack.h>
#include "avator.h"
#include "M5StackUpdater.h"
#include "AquesTalkTTS.h"

const char* licencekey = "xxxx-xxxx-xxxx-xxxx";  // AquesTalk-ESP licencekey

const char* ssid       = "your ssid";   // WiFi SSID
const char* password   = "your password";   // WiFi PW
const char* ntpServer =  "ntp.jst.mfeed.ad.jp";
const long  gmtOffset_sec = 9 * 3600;
const int daylightOffset_sec = 0;

Avator *avator;
int count = 0;
int displayOffTimer = 0;
bool displayOn = true;

void setup()
{
  M5.begin();
  
  if(digitalRead(BUTTON_A_PIN) == 0) {
    Serial.println("Will Load menu binary");
    updateFromFS(SD);
    ESP.restart();
  }
  
  avator = new Avator();
  setDisplayOn(true);
  //avator->init();

  int iret;
  
   // Init Text-to-Speech (AquesTalk-ESP + I2S + Internal-DAC)
  iret = TTS.create(licencekey);
  if(iret){
       Serial.print("ERR: TTS_create():");
       Serial.println(iret);
  }
  // run clock task
  xTaskCreate(taskClock, "taskClock", 4096, NULL, 2, NULL);
}

void loop()
{
  int iret;
  struct tm timeinfo;
  char koe[100];

  if(M5.BtnA.wasPressed()){
    setDisplayOn(true);

    if(getLocalTime(&timeinfo)){
      // 年月日の読み上げ
      sprintf(koe,"<NUMK VAL=%d COUNTER=nenn>/<NUMK VAL=%d COUNTER=gatu>/<NUMK VAL=%d COUNTER=nichi>.",
        timeinfo.tm_year+1900,timeinfo.tm_mon+1, timeinfo.tm_mday);
      iret = TTS.play(koe, 100);
      if(iret){
        Serial.println("ERR:TTS.play()");
      }
    }
  }
  else if(M5.BtnB.wasPressed()){
    setDisplayOn(true);

    if(getLocalTime(&timeinfo)){
      // 時分秒の読み上げ
      sprintf(koe,"<NUMK VAL=%d COUNTER=ji>/<NUMK VAL=%d COUNTER=funn>/<NUMK VAL=%d COUNTER=byo->.",
        timeinfo.tm_hour,timeinfo.tm_min,timeinfo.tm_sec);
      iret = TTS.play(koe, 100);
      if(iret){
        Serial.println("ERR:TTS.play()");
      }
    }
  }
  else if(M5.BtnC.wasPressed()){
    TTS.stop();
    setDisplayOn(false);
  }
  if(displayOn){
    avatorUpdate();
  }
  M5.update();
}

void setDisplayOn(bool onState){
  if(onState){
    M5.Lcd.setBrightness(60);
    displayOn = true;
  }else{
    M5.Lcd.setBrightness(0);
    displayOn = false;
  }
}

void avatorUpdate(){
  count++;
  int percent = count * 10 % 100;
  avator->openMouth(TTS.getLevel());
  if (count % 100 == 97)
  {
    avator->openEye(false);
  }
  if (count % 100 == 0)
  {
    avator->openEye(true);
    count = random(50);
  }
  delay(50);
}

void syncDateTime(){
  //connect to WiFi
  Serial.printf("Connecting to %s ", ssid);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
      delay(500);
      Serial.print(".");
  }
  Serial.println(" CONNECTED");
  //init and get the time
  configTime(gmtOffset_sec, daylightOffset_sec, ntpServer);
  struct tm timeinfo;
  getLocalTime(&timeinfo);  // すぐにWiFiのdisconnectするとNG

  //disconnect WiFi as it's no longer needed
  WiFi.disconnect(true);
  WiFi.mode(WIFI_OFF);
}
void taskClock(void *arg)
{
  M5.Lcd.fillScreen(TFT_BLACK);

  //if(digitalRead(BUTTON_C_PIN) == 0) {
    syncDateTime();  
  //}
  for(;;){
    struct tm timeinfo;
    if(!getLocalTime(&timeinfo)){
      Serial.println("Failed to obtain time");
    }
    else {
      Serial.println(&timeinfo, "%A, %B %d %Y %H:%M:%S");
      //DrawTime(timeinfo.tm_hour, timeinfo.tm_min, timeinfo.tm_sec);
    }
    delay(1000);
  }
  
}

AquesTalkTTS.cpp

// Copyright (c) 2018 AQUEST
//	AquesTalk-ESP + I2S + internal-DAC

#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
#include <driver/i2s.h>
#include <aquestalk.h>
#include "AquesTalkTTS.h"

#define LEN_FRAME 32
#define	TASK_PRIORITY		10
#define	SAMPLING_FREQ   24000	// 8KHz x 3
#define	DMA_BUF_COUNT 3
#define	DMA_BUF_LEN	(LEN_FRAME*3)	// one buffer size(one channnel samples)
#define DMA_BUF_SIZE (DMA_BUF_COUNT*DMA_BUF_LEN)
#define I2S_FIFO_LEN  (64/2)
#define	TICKS_TO_WAIT	(2*LEN_FRAME/8/portTICK_PERIOD_MS)

static uint32_t *workbuf = 0;
static TaskHandle_t taskAquesTalk=0;
static SemaphoreHandle_t muxAquesTalk=0;
static int level = 0;


static void task_TTS_synthe(void *arg);
static void DAC_create();
static void DAC_release();
static void DAC_start();
static void DAC_stop();
static int DAC_write(int len, int16_t *wav);
static int DAC_write_val(uint16_t val);

AquesTalkTTS	TTS;		// the only instance of AquesTalkTTS class

int AquesTalkTTS::create(const char *licencekey)
{
	int iret;
	
	// Initialize	AquesTalk-ESP
	if(!workbuf){
	  workbuf = (uint32_t*)malloc(AQ_SIZE_WORKBUF*sizeof(uint32_t));
	  if(workbuf==0) return 401;  // no heap memory
	}
  iret = CAqTkPicoF_Init(workbuf, LEN_FRAME, licencekey);
  if(iret)		return iret;	// AquesTalk Init error
  
  if(!muxAquesTalk)  muxAquesTalk = xSemaphoreCreateMutex();

  return 0;
}

void AquesTalkTTS::release()
{
	stop();
	if(taskAquesTalk)	vTaskDelete(taskAquesTalk);
  if(muxAquesTalk)	vSemaphoreDelete(muxAquesTalk);
  if(workbuf)	free(workbuf);
  workbuf = 0; taskAquesTalk = 0; muxAquesTalk = 0;
}

int AquesTalkTTS::play(const char *koe, int speed)
{
	int iret;
	if(!muxAquesTalk) return 402;	// not TTS_create

  xSemaphoreTake(muxAquesTalk, (portTickType)portMAX_DELAY);
  iret = CAqTkPicoF_SetKoe((const uint8_t*)koe, speed, 256);
	xSemaphoreGive(muxAquesTalk);
  if(iret)   return iret;

  if(taskAquesTalk==0){
     xTaskCreate(task_TTS_synthe, "task_TTS_synthe", 4096, NULL, TASK_PRIORITY, &taskAquesTalk);
  }
  else {
		vTaskResume(taskAquesTalk);
	}
  return 0;
}

int AquesTalkTTS::getLevel()
{
	return level;
}

void AquesTalkTTS::stop()
{
	if(taskAquesTalk==0) return;	// not playing
	if(eTaskGetState(taskAquesTalk)==eSuspended) return;	// already suspended.
	
  xSemaphoreTake(muxAquesTalk, (portTickType)portMAX_DELAY);
  CAqTkPicoF_SetKoe((const uint8_t*)"#", 100, 256);	// generate error
	xSemaphoreGive(muxAquesTalk);
	// wait until the task suspend
	for(;;){
		if(eTaskGetState(taskAquesTalk)==eSuspended) break;
	}
}

void task_TTS_synthe(void *arg)
{
	for(;;){
	  DAC_create();
	  DAC_start();
		for(;;){
			int iret;
			uint16_t len;
			int16_t wav[LEN_FRAME];
			
	    xSemaphoreTake(muxAquesTalk, (portTickType)portMAX_DELAY);
			iret = CAqTkPicoF_SyntheFrame(wav, &len);
			xSemaphoreGive(muxAquesTalk);
			
			if(iret) break;	 // EOD or ERROR
			DAC_write((int)len, wav);
			
			int sum = wav[0];
			level = (int)(sum / 100);
			if(level > 100) level = 100;
		}
		DAC_stop();
		DAC_release();
		level = 0;
		vTaskSuspend(NULL);	// suspend this task
	}
}

////////////////////////////////
//i2s configuration 
static const int i2s_num = 0; // i2s port number
static i2s_config_t i2s_config = {
     .mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_TX | I2S_MODE_DAC_BUILT_IN),
     .sample_rate = SAMPLING_FREQ,
     .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT,
     .channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT,
     .communication_format = (i2s_comm_format_t)I2S_COMM_FORMAT_I2S_MSB,
     .intr_alloc_flags = 0,
     .dma_buf_count = DMA_BUF_COUNT,
     .dma_buf_len = DMA_BUF_LEN,
     .use_apll = 0
};

static void DAC_create()
{

  i2s_driver_install((i2s_port_t)i2s_num, &i2s_config, 0, NULL);
  i2s_set_pin((i2s_port_t)i2s_num, NULL);
  i2s_stop((i2s_port_t)i2s_num);  // Create時はstop状態
}

static void DAC_release()
{
  i2s_driver_uninstall((i2s_port_t)i2s_num); //stop & destroy i2s driver 
}

static void DAC_start()
{
	int k;
  i2s_start((i2s_port_t)i2s_num);
  
  for(k=0;k<DMA_BUF_LEN;k++){
		DAC_write_val(0);
	}
  for(k=0;k<=32768;k+=256) {
    DAC_write_val((uint16_t)k);
  }
	AqResample_Reset();
}

static void DAC_stop()
{
  int k;
  for(k=32768;k>=0;k-=256) {
    DAC_write_val((uint16_t)k);
  }
  for(k=0;k<DMA_BUF_SIZE+I2S_FIFO_LEN;k++){
		DAC_write_val(0);
	}
  i2s_stop((i2s_port_t)i2s_num);  
}

// upsampling & write to I2S
static int DAC_write(int len, int16_t *wav)
{
  int i;
  for(i=0;i<len;i++){
    // upsampling x3
    int16_t wav3[3];
    AqResample_Conv(wav[i], wav3);

    for(int k=0;k<3; k++){
			int iret = DAC_write_val(((uint16_t)wav3[k])^0x8000U);
      if(iret<0) return 404; // -1:parameter error
      if(iret==0) break;  //  0:TIMEOUT
    }
  }
  return i;
}

// write to I2S DMA buffer
static int DAC_write_val(uint16_t val)
{
	uint16_t sample[2];
  sample[0]=sample[1]=val; // mono -> stereo
  return i2s_push_sample((i2s_port_t)i2s_num, (const char *)sample, TICKS_TO_WAIT);
}

AquesTalkTTS.h

/**
 * \par Copyright (C), 2018, AQUEST
 * \class AquesTalkTTS
 * \brief   Text-to-Sppech library. uses AquesTalk-ESP, I2S, intenal-DAC.
 * @file    AquesTalkTTS.h
 * @author  AQUEST
 * @version V0.1.0
 * @date    2018/03/29
 * @brief   Header for AquesTalkTTS.cpp module
 *
 * \par Description
 * This file is a TTS class for ESP32.
 *
 * \par Method List:
 *    
 *
        TTS.create();
        TTS.release();
        TTS.play(const char *koe, int speed);
        TTS.stop();
 *
 * \par History:
 * <pre>
 * `<Author>`         `<Time>`        `<Version>`        `<Descr>`
 * Nobuhide Yamazaki    2018/03/29        0.0.1          Creation.
 * </pre>
 *
 */

#ifndef _AQUESTALK_TTS_H_
#define _AQUESTALK_TTS_H_

class AquesTalkTTS {
	public:
		int create(const char *licencekey);
		void release();

		int play(const char *koe, int speed);
		void stop();
	int getLevel();
};
extern AquesTalkTTS TTS;

#endif // !defined(_AQUESTALK_TTS_H_)

avator.cpp

// see https://twitter.com/meganetaaan/status/982258576361078784
// (2018/4/6 @meganetaaan)
#include <M5Stack.h>
#include "avator.h"

// Mouth
Mouth::Mouth(void)
{

}
Mouth::Mouth(int x, int y, int minWidth, int maxWidth, int minHeight, int maxHeight, uint32_t primaryColor, uint32_t secondaryColor)
{
  // TODO: validation
  this->x = x;
  this->y = y;
  this->minWidth = minWidth;
  this->maxWidth = maxWidth;
  this->minHeight = minHeight;
  this->maxHeight = maxHeight;
  this->primaryColor = primaryColor;
  this->secondaryColor = secondaryColor;
  this->lastX = 0;
  this->lastY = 0;
  this->lastW = 0;
  this->lastH = 0;
}
void Mouth::clear()
{
  M5.Lcd.fillRect(lastX, lastY, lastW, lastH, secondaryColor);
}
void Mouth::draw(int x, int y, int w, int h)
{
  if(lastX != x || lastY != y || lastW != w || lastH != h){
    clear();
  }
  M5.Lcd.fillRect(x, y, w, h, primaryColor);
  lastX = x;
  lastY = y;
  lastW = w;
  lastH = h;
}
void Mouth::open(int percent)
{
  int h = minHeight + (maxHeight - minHeight) * percent / 100;
  int w = minWidth + (maxWidth - minWidth) * (100 - percent) / 100;
  int x = this->x - w / 2;
  int y = this->y - h / 2;
  draw(x, y, w, h);
}

// Eye
Eye::Eye(void)
{
  
}
Eye::Eye(int x, int y, int r, uint32_t primaryColor, uint32_t secondaryColor)
{
  this->x = x;
  this->y = y;
  this->r = r;
  this->lastX = 0;
  this->lastY = 0;
  this->lastR = 0;
  this->primaryColor = primaryColor;
  this->secondaryColor = secondaryColor;
}
void Eye::clear()
{
  M5.Lcd.fillRect(lastX - lastR - 2, lastY - lastR - 2,
                  lastR * 2 + 4, lastR * 2 + 4, secondaryColor);
}
void Eye::drawCircle(int x, int y, int r)
{
  clear();
  M5.Lcd.fillCircle(x, y, r, primaryColor);
  lastX = x;
  lastY = y;
  lastR = r;
}
void Eye::drawRect(int x, int y, int w, int h)
{
  clear();
  M5.Lcd.fillRect(x, y, w, h, primaryColor);
  lastX = x + w / 2;
  lastY = y + h / 2;
  lastR = w; // TODO: ellipse
}
void Eye::open(boolean isOpen)
{
  if (isOpen)
  {
    // TODO: "wideness"
    drawCircle(x, y, r);
  }
  else
  {
    int x1 = x - r;
    int y1 = y - 2;
    int w = r * 2;
    int h = 4;
    drawRect(x1, y1, w, h);
  }
}

#define PRIMARY_COLOR WHITE
#define SECONDARY_COLOR BLACK

Avator::Avator()
{
  mouth = new Mouth(163, 145, 40, 100, 4, 60, PRIMARY_COLOR, SECONDARY_COLOR);
  eyeR = new Eye(90, 93, 8, PRIMARY_COLOR, SECONDARY_COLOR);
  eyeL = new Eye(230, 96, 8, PRIMARY_COLOR, SECONDARY_COLOR);
}
void Avator::openMouth(int percent)
{
  mouth->open(percent);
}
void Avator::openEye(boolean isOpen)
{
  eyeR->open(isOpen);
  eyeL->open(isOpen);
}
void Avator::smile()
{
  // TODO
}
void Avator::init()
{
  mouth->open(0);
  eyeR->open(true);
  eyeL->open(true);
  // TODO: start animation
}

avator.h

// see https://twitter.com/meganetaaan/status/982258576361078784
// (2018/4/6 @meganetaaan)
#pragma once
#include <M5Stack.h>

class Mouth
{
private:
  int x;
  int y;
  int minWidth;
  int maxWidth;
  int minHeight;
  int maxHeight;
  int lastX;
  int lastY;
  int lastW;
  int lastH;
  uint32_t primaryColor;
  uint32_t secondaryColor;
  void clear(void);
  void draw(int x1, int y1, int x2, int y2);
public:
  // constructor
  Mouth();
  Mouth(int x, int y,
  int minWidth, int maxWidth,
  int minHeight, int maxHeight,
  uint32_t primaryColor, uint32_t secondaryColor);

  void open(int percent);
};

class Eye
{
private:
  int x;
  int y;
  int r;
  int lastX;
  int lastY;
  int lastR;
  uint32_t primaryColor;
  uint32_t secondaryColor;
  void clear(void);
  void drawCircle(int x, int y, int r);
  void drawRect(int x, int y, int w, int h);
public:
  // constructor
  Eye();
  Eye(int x, int y, int r, uint32_t primaryColor, uint32_t secondaryColor);

  void open(boolean isOpen);
};

class Avator
{
private:
  Mouth* mouth;
  Eye* eyeR;
  Eye* eyeL;
public:
  // constructor
  Avator(void);

  void openMouth(int percent);
  void openEye(boolean isOpen);
  void smile();
  void init();
};