Last active
April 25, 2024 21:38
-
-
Save rodydavis/c2c2c86adf1a1fe2cc45505ecdd58399 to your computer and use it in GitHub Desktop.
Ask the Menu
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Copyright 2024 the Dart project authors. All rights reserved. | |
// Use of this source code is governed by a BSD-style license | |
// that can be found in the LICENSE file. | |
import 'dart:async'; | |
import 'dart:typed_data'; | |
import 'package:flutter/material.dart'; | |
import 'package:flutter_markdown/flutter_markdown.dart'; | |
import 'package:google_generative_ai/google_generative_ai.dart'; | |
import 'package:url_launcher/link.dart'; | |
import 'package:web/web.dart' as html; | |
import 'package:http/http.dart' as http; | |
import 'dart:js_interop'; | |
final themeColor = ValueNotifier<Color>(Colors.orangeAccent); | |
void main() { | |
runApp(const GenerativeAISample()); | |
} | |
class GenerativeAISample extends StatelessWidget { | |
const GenerativeAISample({super.key}); | |
ThemeData theme(Brightness brightness) { | |
final colors = ColorScheme.fromSeed( | |
brightness: brightness, | |
seedColor: themeColor.value, | |
); | |
return ThemeData( | |
brightness: brightness, | |
colorScheme: colors, | |
scaffoldBackgroundColor: colors.surface, | |
); | |
} | |
@override | |
Widget build(BuildContext context) { | |
return AnimatedBuilder( | |
animation: themeColor, | |
builder: (context, child) { | |
return MaterialApp( | |
debugShowCheckedModeBanner: false, | |
title: 'Ask the Menu', | |
theme: theme(Brightness.light), | |
darkTheme: theme(Brightness.dark), | |
themeMode: ThemeMode.system, | |
home: const ChatScreen(title: 'Ask the Menu'), | |
); | |
}, | |
); | |
} | |
} | |
class ChatScreen extends StatefulWidget { | |
const ChatScreen({ | |
super.key, | |
required this.title, | |
}); | |
final String title; | |
@override | |
State<ChatScreen> createState() => _ChatScreenState(); | |
} | |
class _ChatScreenState extends State<ChatScreen> { | |
String? apiKey; | |
@override | |
Widget build(BuildContext context) { | |
return switch (apiKey) { | |
final providedKey? => AskTheMenu( | |
title: widget.title, | |
apiKey: providedKey, | |
), | |
_ => ApiKeyWidget( | |
title: widget.title, | |
onSubmitted: (key) { | |
setState(() => apiKey = key); | |
}, | |
), | |
}; | |
} | |
} | |
class ApiKeyWidget extends StatelessWidget { | |
ApiKeyWidget({ | |
super.key, | |
required this.onSubmitted, | |
required this.title, | |
}); | |
final String title; | |
final ValueChanged onSubmitted; | |
final _textController = TextEditingController(); | |
@override | |
Widget build(BuildContext context) { | |
return Scaffold( | |
appBar: AppBar( | |
title: Text(title), | |
), | |
body: Center( | |
child: Padding( | |
padding: const EdgeInsets.all(8.0), | |
child: Column( | |
mainAxisSize: MainAxisSize.min, | |
children: [ | |
const Text( | |
'To use the Gemini API, you\'ll need an API key. ' | |
'If you don\'t already have one, ' | |
'create a key in Google AI Studio.', | |
textAlign: TextAlign.center, | |
), | |
const SizedBox(height: 8), | |
Link( | |
uri: Uri.https('aistudio.google.com', '/app/apikey'), | |
target: LinkTarget.blank, | |
builder: (context, followLink) => TextButton( | |
onPressed: followLink, | |
child: const Text('Get an API Key'), | |
), | |
), | |
], | |
), | |
), | |
), | |
bottomNavigationBar: BottomAppBar( | |
padding: const EdgeInsets.all(8), | |
child: Row( | |
children: [ | |
Expanded( | |
child: TextField( | |
decoration: textFieldDecoration(context, 'Enter your API key'), | |
controller: _textController, | |
obscureText: true, | |
onSubmitted: (value) { | |
onSubmitted(value); | |
}, | |
), | |
), | |
const SizedBox(height: 8), | |
TextButton( | |
onPressed: () { | |
onSubmitted(_textController.value.text); | |
}, | |
child: const Text('Submit'), | |
), | |
], | |
), | |
), | |
); | |
} | |
} | |
class AskTheMenu extends StatefulWidget { | |
const AskTheMenu({ | |
super.key, | |
required this.apiKey, | |
required this.title, | |
}); | |
final String apiKey, title; | |
@override | |
State<AskTheMenu> createState() => _AskTheMenuState(); | |
} | |
class _AskTheMenuState extends State<AskTheMenu> { | |
final loading = ValueNotifier(false); | |
final menu = ValueNotifier(''); | |
final messages = ValueNotifier<List<(Sender, String)>>([]); | |
final controller = TextEditingController(); | |
late final ChatSession chat = chatModel.startChat(); | |
late final chatModel = GenerativeModel( | |
model: 'gemini-pro', | |
apiKey: widget.apiKey, | |
); | |
late final imageModel = GenerativeModel( | |
model: 'gemini-pro-vision', | |
apiKey: widget.apiKey, | |
); | |
Future<void> sendMessage() async { | |
final message = controller.text.trim(); | |
if (message.isEmpty) return; | |
controller.clear(); | |
addMessage(Sender.user, message); | |
loading.value = true; | |
try { | |
String prompt = chatSessionPrompt; | |
prompt = prompt.replaceAll('{{menu}}', menu.value); | |
prompt = prompt.replaceAll('{{questions}}', message); | |
final response = await chat.sendMessage(Content.text(prompt)); | |
if (response.text != null) { | |
addMessage(Sender.system, response.text!); | |
} else { | |
addMessage(Sender.system, 'No Response'); | |
} | |
} catch (e) { | |
addMessage(Sender.system, 'Error sending message: $e'); | |
} finally { | |
loading.value = false; | |
} | |
} | |
void addMessage(Sender sender, String value, {bool clear = false}) { | |
if (clear) messages.value = []; | |
messages.value = messages.value.toList()..add((sender, value)); | |
} | |
Future<String> extractMenu(String mimeType, Uint8List bytes) async { | |
final theme = Theme.of(context); | |
final response = await imageModel.generateContent([ | |
Content.multi([ | |
TextPart(extractDataFromMenuPrompt), | |
DataPart(mimeType, bytes), | |
]), | |
]).then((res) => res.text ?? ''); | |
try { | |
final color = await ColorScheme.fromImageProvider( | |
provider: MemoryImage(bytes), | |
brightness: theme.brightness, | |
); | |
themeColor.value = color.primary; | |
} catch (e) { | |
debugPrint('Error extracting image color: $e'); | |
} | |
return response; | |
} | |
Future<void> loadMenu() async { | |
final file = await pickFile(); | |
if (file == null) return; | |
addMessage(Sender.system, 'Uploading menu...', clear: true); | |
controller.clear(); | |
try { | |
menu.value = await extractMenu('image/png', file); | |
addMessage(Sender.system, menu.value, clear: true); | |
} catch (e) { | |
menu.value = ''; | |
addMessage(Sender.system, 'Error uploading menu: $e', clear: true); | |
} | |
} | |
@override | |
Widget build(BuildContext context) { | |
return AnimatedBuilder( | |
animation: messages, | |
builder: (context, child) { | |
final reversed = messages.value.reversed; | |
return Scaffold( | |
appBar: AppBar( | |
title: Text(widget.title), | |
actions: [ | |
IconButton( | |
onPressed: loadMenu, | |
icon: const Icon(Icons.image), | |
tooltip: 'Upload a menu', | |
) | |
], | |
), | |
body: messages.value.isEmpty | |
? AnimatedBuilder( | |
animation: menu, | |
builder: (context, _) { | |
if (menu.value.isEmpty) { | |
return Center( | |
child: Column( | |
mainAxisAlignment: MainAxisAlignment.center, | |
mainAxisSize: MainAxisSize.min, | |
children: [ | |
const Text('Upload a new menu'), | |
const SizedBox(height: 10), | |
ElevatedButton( | |
onPressed: loadMenu, | |
child: const Text('Select Image/PDF'), | |
), | |
], | |
), | |
); | |
} | |
return SingleChildScrollView( | |
padding: const EdgeInsets.all(8), | |
child: MarkdownBody( | |
data: menu.value, | |
), | |
); | |
}, | |
) | |
: ListView.builder( | |
padding: const EdgeInsets.all(8), | |
reverse: true, | |
itemCount: reversed.length, | |
itemBuilder: (context, index) { | |
final (sender, message) = reversed.elementAt(index); | |
return MessageWidget( | |
isFromUser: sender == Sender.user, | |
text: message, | |
); | |
}, | |
), | |
bottomNavigationBar: messages.value.isEmpty | |
? null | |
: BottomAppBar( | |
padding: const EdgeInsets.all(8), | |
child: Row( | |
children: [ | |
Expanded( | |
child: TextField( | |
controller: controller, | |
decoration: textFieldDecoration(context, | |
'Ask questions about price, food types, and suggestions on combos'), | |
onEditingComplete: sendMessage, | |
onSubmitted: (value) => sendMessage(), | |
), | |
), | |
const SizedBox(width: 8), | |
AnimatedBuilder( | |
animation: loading, | |
builder: (context, _) { | |
if (loading.value) { | |
return const CircularProgressIndicator(); | |
} | |
return IconButton( | |
onPressed: sendMessage, | |
icon: const Icon(Icons.send), | |
tooltip: 'Send a message', | |
); | |
}, | |
), | |
], | |
), | |
), | |
); | |
}, | |
); | |
} | |
} | |
const chatSessionPrompt = r''' | |
Given the following menu, respond with valid | |
markdown the answers to the users questions: | |
``` | |
{{menu}} | |
``` | |
Questions about the menu: | |
``` | |
{{questions}} | |
``` | |
'''; | |
const extractDataFromMenuPrompt = ''' | |
Convert the restaurant menu to markdown and return all the items on the menu | |
showing the title, description and price for each. | |
Should return pretty markdown that includes the section, item name, price and the full details of the item if available. | |
'''; | |
enum Sender { | |
user, | |
system, | |
} | |
class MessageWidget extends StatelessWidget { | |
const MessageWidget({ | |
super.key, | |
this.text, | |
this.image, | |
required this.isFromUser, | |
}); | |
final Image? image; | |
final String? text; | |
final bool isFromUser; | |
@override | |
Widget build(BuildContext context) { | |
return Row( | |
mainAxisAlignment: | |
isFromUser ? MainAxisAlignment.end : MainAxisAlignment.start, | |
children: [ | |
Flexible( | |
child: Container( | |
constraints: const BoxConstraints(maxWidth: 520), | |
decoration: BoxDecoration( | |
color: isFromUser | |
? Theme.of(context).colorScheme.primaryContainer | |
: Theme.of(context).colorScheme.surfaceVariant, | |
borderRadius: BorderRadius.circular(18), | |
), | |
padding: const EdgeInsets.symmetric( | |
vertical: 15, | |
horizontal: 20, | |
), | |
margin: const EdgeInsets.only(bottom: 8), | |
child: Column(children: [ | |
if (text case final text?) MarkdownBody(data: text), | |
if (image case final image?) image, | |
]), | |
), | |
), | |
], | |
); | |
} | |
} | |
InputDecoration textFieldDecoration(BuildContext context, String hintText) => | |
InputDecoration( | |
contentPadding: const EdgeInsets.all(15), | |
hintText: hintText, | |
border: OutlineInputBorder( | |
borderRadius: const BorderRadius.all( | |
Radius.circular(14), | |
), | |
borderSide: BorderSide( | |
color: Theme.of(context).colorScheme.secondary, | |
), | |
), | |
focusedBorder: OutlineInputBorder( | |
borderRadius: const BorderRadius.all( | |
Radius.circular(14), | |
), | |
borderSide: BorderSide( | |
color: Theme.of(context).colorScheme.secondary, | |
), | |
), | |
); | |
Future<Uint8List?> pickFile() async { | |
final el = html.document.createElement('input') as html.HTMLInputElement; | |
el.type = 'file'; | |
el.accept = 'image/*'; | |
el.click(); | |
final completer = Completer<Uri?>(); | |
el.onchange = (html.Event e) { | |
final files = el.files; | |
if (files != null && files.length != 0) { | |
final file = files.item(0); | |
if (file != null) { | |
final reader = html.FileReader(); | |
reader.onload = (html.Event _) { | |
final url = reader.result; | |
if (url != null) { | |
completer.complete(Uri.parse(url as String)); | |
} else { | |
completer.complete(null); | |
} | |
}.toJS; | |
reader.readAsDataURL(file); | |
} else { | |
completer.complete(null); | |
} | |
} else { | |
completer.complete(null); | |
} | |
}.toJS; | |
final result = await completer.future; | |
el.remove(); | |
if (result != null) { | |
final res = await http.get(result); | |
if (res.statusCode == 200) { | |
return res.bodyBytes; | |
} | |
} | |
return null; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment