Created
October 9, 2024 09:22
-
-
Save kudoh/a76e83482f49ecc811e9a08be6118d9c to your computer and use it in GitHub Desktop.
OpenAI Realtime API with Function calling
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import { customsearch } from '@googleapis/customsearch'; | |
const API_KEY = process.env.CSE_API_KEY ?? ''; | |
const ENGINE_ID = process.env.CSE_ENGINE_ID ?? ''; | |
export async function webSearch({ query }: { query: string }) { | |
console.log('Web Search:', query); | |
const api = customsearch({ | |
auth: API_KEY, | |
version: 'v1' | |
}); | |
// https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list | |
const result = await api.cse.list({ | |
q: query, | |
cx: ENGINE_ID | |
}); | |
return (result.data.items ?? []).map(item => ({ | |
title: item.title, | |
link: item.link, | |
snippet: item.snippet | |
})); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import WebSocket from 'ws'; | |
import { spawn } from 'child_process'; | |
import { webSearch } from './google-search.js'; | |
const url = 'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01'; | |
const ws = new WebSocket(url, { | |
headers: { | |
'Authorization': 'Bearer ' + process.env.OPENAI_API_KEY, | |
'OpenAI-Beta': 'realtime=v1' | |
} | |
}); | |
const instructions = `あなたは物知りなAIアシスタントです。 | |
ユーザーの質問には、webSearch関数をこっそり使って、驚きや笑いを交えた答えを返してください。 | |
技術的な裏側は内緒にして、親しみやすくユニークな会話を楽しみましょう。 | |
ユーザーを楽しませつつ、役立つ情報も忘れずに!`; | |
const recorder = spawn('sox', [ | |
'--default-device', | |
'--no-show-progress', | |
'--rate', '24000', | |
'--channels', '1', | |
'--encoding', 'signed-integer', | |
'--bits', '16', | |
'--type', 'raw', | |
'-' // 標準出力 | |
]); | |
const recorderStream = recorder.stdout; | |
const player = spawn('sox', [ | |
'--type', 'raw', | |
'--rate', '24000', | |
'--encoding', 'signed-integer', | |
'--bits', '16', | |
'--channels', '1', | |
'-', // 標準入力 | |
'--no-show-progress', | |
'--default-device', | |
]); | |
const audioStream = player.stdin; | |
ws.on('open', () => { | |
ws.send(JSON.stringify({ | |
type: 'session.update', | |
session: { | |
voice: 'shimmer', | |
instructions: instructions, | |
input_audio_transcription: { model: 'whisper-1' }, | |
turn_detection: { type: 'server_vad' } | |
} | |
})); | |
ws.send(JSON.stringify({ | |
type: 'session.update', | |
session: { | |
tools: [{ | |
type: 'function', | |
name: 'webSearch', | |
description: 'Performs an internet search using a search engine with the given query.', | |
parameters: { | |
type: 'object', | |
properties: { | |
query: { | |
type: 'string', | |
description: 'The search query' | |
} | |
}, | |
required: ['query'] | |
} | |
}], | |
tool_choice: 'auto' | |
} | |
})); | |
recorderStream.on('data', (chunk: Buffer) => { | |
ws.send(JSON.stringify({ | |
type: 'input_audio_buffer.append', | |
audio: chunk.toString('base64') | |
})); | |
}); | |
}); | |
ws.on('message', (message) => { | |
const event = JSON.parse(message.toString()); | |
if (!['response.audio_transcript.delta', 'response.audio.delta'].includes(event.type)) { | |
console.log(event.type); | |
} | |
// console.log('EVENT:', JSON.stringify(event, null, 2)); | |
switch (event.type) { | |
case 'response.audio.delta': | |
audioStream.write(Buffer.from(event.delta, 'base64')); | |
break; | |
case 'response.output_item.done': | |
const { item } = event; | |
// 1. 関数実行依頼判定(function_call) | |
if (item.type === 'function_call') { | |
if (item.name === 'webSearch') { | |
// 2. 関数実行 | |
webSearch(JSON.parse(item.arguments)).then(output => { | |
// 3. 実行結果連携 | |
ws.send(JSON.stringify({ | |
type: 'conversation.item.create', | |
item: { | |
type: 'function_call_output', | |
call_id: item.call_id, | |
output: JSON.stringify(output) | |
} | |
})); | |
// 4. レスポンス生成要求 | |
ws.send(JSON.stringify({ type: 'response.create' })); | |
}); | |
} | |
} | |
break; | |
case 'response.audio_transcript.done': | |
case 'conversation.item.input_audio_transcription.completed': | |
console.log(event.type, event.transcript); | |
break; | |
case 'error': | |
console.error('ERROR', event.error); | |
break; | |
} | |
}); | |
ws.on('error', (error) => { | |
console.error('WebSocketエラー', { error }); | |
}); | |
ws.on('close', (event) => { | |
console.log('close connection'); | |
audioStream.end(); | |
recorderStream.pause(); | |
recorder.kill('SIGHUP'); | |
player.kill('SIGHUP'); | |
process.exit(); | |
}); | |
// 標準入力を待機。`q`で終了 | |
process.stdin.resume(); | |
process.stdin.setEncoding('utf8'); | |
process.stdin.on('data', (input: string) => { | |
if (input.trim().toLowerCase() === 'q') { | |
console.log('終了します...'); | |
ws.close(); | |
setTimeout(() => { | |
process.exit(); | |
}, 1000); | |
} | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
https://developer.mamezou-tech.com/blogs/2024/10/09/openai-realtime-api-function-calling/