langchain-pdf-embeddings-supabase.js

import fs from 'fs';
import path from 'path';
import { PDFLoader } from 'langchain/document_loaders/fs/pdf';
import { OpenAI } from 'langchain/llms/openai';
import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
import { SupabaseVectorStore } from 'langchain/vectorstores/supabase';
import { RetrievalQAChain } from 'langchain/chains';
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
import { createClient } from '@supabase/supabase-js';
import * as dotenv from 'dotenv';

dotenv.config();

const supabaseClient = createClient(
  process.env.NEXT_PUBLIC_SUPABASE_URL!,
  process.env.SUPABASE_SERVICE_ROLE_KEY!,
  {
    auth: {
      persistSession: false,
      autoRefreshToken: false,
    },
  },
);

async function generateEmbeddings() {
  const pdfs = fs.readdirSync(path.join(__dirname, '../assets'));
  const loaders = pdfs.map(
    (pdf) => new PDFLoader(path.join(__dirname, `../assets/${pdf}`)),
  );

  const model = new OpenAI();
  const embeddings = new OpenAIEmbeddings();

  const vectorStore = new SupabaseVectorStore(embeddings, {
    client: supabaseClient,
    tableName: 'documents',
    queryName: 'match_documents',
  });
  const splitter = new RecursiveCharacterTextSplitter();

  await Promise.all(
    loaders.map(async (loader) => {
      const docs = await loader.load();
      const splitedDocs = await splitter.splitDocuments(docs);
      await vectorStore.addDocuments(splitedDocs);
    }),
  );

  const chain = RetrievalQAChain.fromLLM(model, vectorStore.asRetriever());

  const question = '';
  const res = await chain.call({
    query: question,
  });
  console.log({ res });
}

async function main() {
  await generateEmbeddings();
}

main().catch((err) => console.error(err));