Skip to content

Instantly share code, notes, and snippets.

@hanishi
Last active March 7, 2024 03:36
Show Gist options
  • Save hanishi/99e25b8aaf8a1f3d193a0b38b6eeef1c to your computer and use it in GitHub Desktop.
Save hanishi/99e25b8aaf8a1f3d193a0b38b6eeef1c to your computer and use it in GitHub Desktop.
AutocompleteServer for ChatGPT
type CacheEntry<T> = {
value: T,
timeout: ReturnType<typeof setTimeout>,
}
class ExpiringCache<T> {
private readonly cache: Record<string, CacheEntry<T>>;
private readonly expirationTime: number;
constructor(expirationTime: number = 5000) {
this.cache = {};
this.expirationTime = expirationTime;
}
set(key: string, value: T): void {
// If there's an existing timeout for this key, clear it
if (this.cache[key] && this.cache[key].timeout) {
clearTimeout(this.cache[key].timeout);
}
// Set the new value
this.cache[key] = {
value: value,
timeout: setTimeout(() => {
delete this.cache[key];
}, this.expirationTime)
};
}
get(key: string): T | null {
const entry = this.cache[key];
if (entry) {
console.log("HIT")
return entry.value;
}
return null;
}
}
class AutocompleteServer {
LATENCY: number;
timeout: null | ReturnType<typeof setTimeout> = null;
cache = new ExpiringCache<string>(15000)
constructor(latency: number = 3000) {
this.LATENCY = latency;
}
query = (searchText: string): SearchPromise => {
if (this.timeout !== null) {
clearTimeout(this.timeout);
this.timeout = null;
}
const dismiss = () => {
if (this.timeout !== null) {
console.log("DISMISSED")
clearTimeout(this.timeout);
this.timeout = null;
}
};
const promise: Promise<null | string> = new Promise((resolve, reject) => {
this.timeout = setTimeout(async () => {
const searchTextLength = searchText.length;
if (searchText === '' || searchTextLength < 4) {
return resolve(null);
}
const data
= this.cache.get(searchText) || await fetch(`/api/chat`, {method: 'POST', body: JSON.stringify({input: searchText})}).then(res => res.json())
.then(data => data.text).catch(e => reject(e));
this.cache.set(searchText, data);
return resolve(data);
}, this.LATENCY);
});
return {
dismiss,
promise,
};
};
}
// use this instead of the original useQuery
function useQuery(server: AutocompleteServer): (searchText: string) => SearchPromise {
return useCallback((searchText: string) => {
console.time('query');
const response = server.query(searchText);
console.timeEnd('query');
return response;
}, [server]);
}
@hanishi
Copy link
Author

hanishi commented Aug 24, 2023

AI_writer2.mov

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment