components_knowledge_knowledge-new.tsx

"use client";

import { useState } from "react";
import { useRouter } from "next/navigation";
import Loading from "../../loading";

type Props = {
  knowledge_urls:
    | {
        url: string;
      }[]
    | null;
};

const KnowledgeNew: React.FC<Props> = ({ knowledge_urls }) => {
  const [urls, setUrls] = useState<string[]>([]);
  const [loading, setLoading] = useState(false);
  const [message, setMessage] = useState("");
  const router = useRouter();

  // 送信
  const onSubmit = async (e: React.FormEvent<HTMLFormElement>) => {
    e.preventDefault();
    setLoading(true);
    setMessage("");

    // 知識データベースに保存
    const body = JSON.stringify({ urls });
    const response = await fetch("/api/knowledge", {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
      },
      body,
    });

    // エラーチェック
    if (!response.ok) {
      alert("データベースが作成できませんでした");
      setLoading(false);
      return;
    }

    setMessage("データベースに追加されました");
    setUrls([]);
    router.refresh();
    setLoading(false);
  };

  return (
    <div>
      <div className="mb-3 text-center text-xl font-bold">知識データベース</div>
      <div className="mb-5 text-center">
        指定したURLの文章を知識データベースに追加します
      </div>
      <form onSubmit={onSubmit}>
        <div className="mb-5">
          <textarea
            className="focus:outline-none p-2 w-full text-sm bg-gray-50 rounded-lg border border-gray-300 focus:ring-yellow-500 focus:border-yellow-500 focus:ring-2"
            rows={10}
            placeholder="URLを入力(複数URLは改行)"
            value={urls.join("\n")}
            onChange={(e) => setUrls(e.target.value.split("\n"))}
            disabled={loading}
          />
        </div>

        <div className="text-center my-5 font-bold text-red-500">{message}</div>

        <div className="flex justify-center mb-5">
          {loading ? (
            <Loading />
          ) : (
            <button
              className="bg-yellow-400 hover:bg-yellow-500 focus:ring-4 focus:ring-yellow-300 focus:outline-none text-white font-medium rounded-lg text-sm px-5 py-3"
              type="submit"
              disabled={loading}
            >
              追加する
            </button>
          )}
        </div>
      </form>

      <div>
        <div className="text-center font-bold mb-5">格納URL</div>
        {knowledge_urls && knowledge_urls.length ? (
          <div className="border rounded">
            {knowledge_urls.map((data, index) => {
              return (
                <a
                  href={data.url}
                  key={index}
                  target="_blank"
                  rel="noopener noreferrer"
                >
                  <div
                    className={`${
                      knowledge_urls.length - 1 != index && "border-b"
                    } p-2 text-blue-500 underline hover:bg-gray-100`}
                  >
                    {data.url}
                  </div>
                </a>
              );
            })}
          </div>
        ) : (
          <div className="text-center">知識データベースがありません</div>
        )}
      </div>
    </div>
  );
};

export default KnowledgeNew;

pages_api_knowledge.ts

import { NextApiRequest, NextApiResponse } from "next";
import { Configuration, OpenAIApi } from "openai";
import { supabase } from "../../utils/supabase-js";
import { JSDOM } from "jsdom";
import { Readability } from "@mozilla/readability";

const configuration = new Configuration({
  apiKey: process.env.OPENAI_API_KEY,
});

const openai = new OpenAIApi(configuration);

// 文章の最大文字数
const maxSize = 500;

const handler = async (req: NextApiRequest, res: NextApiResponse) => {
  try {
    const { urls } = req.body;
    const documents = [];

    for (const url of urls) {
      // 記事の内容を取得
      const dom = await JSDOM.fromURL(url);
      const reader = new Readability(dom.window.document);
      const article = reader.parse();

      if (!article) {
        throw new Error(`内容の取得に失敗しました: ${url}`);
      }

      const articleText =
        new dom.window.DOMParser().parseFromString(article.content, "text/html")
          .body.textContent || "";

      let start = 0;
      while (start < articleText.length) {
        // 文章を分割
        const end = start + maxSize;
        const chunk = articleText.slice(start, end);
        documents.push({ url, body: chunk });
        start = end;
      }

      // 古いデータを削除
      await supabase.from("documents").delete().eq("url", url);
    }

    for (const { url, body } of documents) {
      // 改行を空白に変換
      const input = body.replace(/\n/g, " ");

      // Embeddings
      // 文章をベクトルに変換
      const res_embedding = await openai.createEmbedding({
        model: "text-embedding-ada-002",
        input,
      });

      // ベクトル取得
      const [{ embedding }] = res_embedding.data.data;

      // テーブル作成
      await supabase.from("documents").insert({
        content: input,
        embedding,
        url,
      });
    }

    return res.status(200).json({ success: true });
  } catch (error) {
    console.error(error);
    res.status(500).send("Something went wrong");
  }
};

export default handler;