Skip to content

Instantly share code, notes, and snippets.

@kudoh
Created October 9, 2024 09:22
Show Gist options
  • Save kudoh/a76e83482f49ecc811e9a08be6118d9c to your computer and use it in GitHub Desktop.
Save kudoh/a76e83482f49ecc811e9a08be6118d9c to your computer and use it in GitHub Desktop.
OpenAI Realtime API with Function calling
import { customsearch } from '@googleapis/customsearch';
const API_KEY = process.env.CSE_API_KEY ?? '';
const ENGINE_ID = process.env.CSE_ENGINE_ID ?? '';
export async function webSearch({ query }: { query: string }) {
console.log('Web Search:', query);
const api = customsearch({
auth: API_KEY,
version: 'v1'
});
// https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list
const result = await api.cse.list({
q: query,
cx: ENGINE_ID
});
return (result.data.items ?? []).map(item => ({
title: item.title,
link: item.link,
snippet: item.snippet
}));
}
import WebSocket from 'ws';
import { spawn } from 'child_process';
import { webSearch } from './google-search.js';
const url = 'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01';
const ws = new WebSocket(url, {
headers: {
'Authorization': 'Bearer ' + process.env.OPENAI_API_KEY,
'OpenAI-Beta': 'realtime=v1'
}
});
const instructions = `あなたは物知りなAIアシスタントです。
ユーザーの質問には、webSearch関数をこっそり使って、驚きや笑いを交えた答えを返してください。
技術的な裏側は内緒にして、親しみやすくユニークな会話を楽しみましょう。
ユーザーを楽しませつつ、役立つ情報も忘れずに!`;
const recorder = spawn('sox', [
'--default-device',
'--no-show-progress',
'--rate', '24000',
'--channels', '1',
'--encoding', 'signed-integer',
'--bits', '16',
'--type', 'raw',
'-' // 標準出力
]);
const recorderStream = recorder.stdout;
const player = spawn('sox', [
'--type', 'raw',
'--rate', '24000',
'--encoding', 'signed-integer',
'--bits', '16',
'--channels', '1',
'-', // 標準入力
'--no-show-progress',
'--default-device',
]);
const audioStream = player.stdin;
ws.on('open', () => {
ws.send(JSON.stringify({
type: 'session.update',
session: {
voice: 'shimmer',
instructions: instructions,
input_audio_transcription: { model: 'whisper-1' },
turn_detection: { type: 'server_vad' }
}
}));
ws.send(JSON.stringify({
type: 'session.update',
session: {
tools: [{
type: 'function',
name: 'webSearch',
description: 'Performs an internet search using a search engine with the given query.',
parameters: {
type: 'object',
properties: {
query: {
type: 'string',
description: 'The search query'
}
},
required: ['query']
}
}],
tool_choice: 'auto'
}
}));
recorderStream.on('data', (chunk: Buffer) => {
ws.send(JSON.stringify({
type: 'input_audio_buffer.append',
audio: chunk.toString('base64')
}));
});
});
ws.on('message', (message) => {
const event = JSON.parse(message.toString());
if (!['response.audio_transcript.delta', 'response.audio.delta'].includes(event.type)) {
console.log(event.type);
}
// console.log('EVENT:', JSON.stringify(event, null, 2));
switch (event.type) {
case 'response.audio.delta':
audioStream.write(Buffer.from(event.delta, 'base64'));
break;
case 'response.output_item.done':
const { item } = event;
// 1. 関数実行依頼判定(function_call)
if (item.type === 'function_call') {
if (item.name === 'webSearch') {
// 2. 関数実行
webSearch(JSON.parse(item.arguments)).then(output => {
// 3. 実行結果連携
ws.send(JSON.stringify({
type: 'conversation.item.create',
item: {
type: 'function_call_output',
call_id: item.call_id,
output: JSON.stringify(output)
}
}));
// 4. レスポンス生成要求
ws.send(JSON.stringify({ type: 'response.create' }));
});
}
}
break;
case 'response.audio_transcript.done':
case 'conversation.item.input_audio_transcription.completed':
console.log(event.type, event.transcript);
break;
case 'error':
console.error('ERROR', event.error);
break;
}
});
ws.on('error', (error) => {
console.error('WebSocketエラー', { error });
});
ws.on('close', (event) => {
console.log('close connection');
audioStream.end();
recorderStream.pause();
recorder.kill('SIGHUP');
player.kill('SIGHUP');
process.exit();
});
// 標準入力を待機。`q`で終了
process.stdin.resume();
process.stdin.setEncoding('utf8');
process.stdin.on('data', (input: string) => {
if (input.trim().toLowerCase() === 'q') {
console.log('終了します...');
ws.close();
setTimeout(() => {
process.exit();
}, 1000);
}
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment