Hastily-written Go program to poll the RSS feed at TrumpsTruth.org and submit results to a RAG knowledge base on Open-WebUI

tsingest.go

package main

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/mmcdole/gofeed"
)

const (
	feedUrl     = "https://trumpstruth.org/feed"
	filePath    = "items"
	invFilePath = ".lastId"

	apiEndpoint = "https://ai/api"
	apiToken    = "YOUR_OPEN-WEBUI_API_TOKEN_HERE"

	knowledgeBaseId = "YOUR_KNOWLEDGE_BASE_ID_HERE"
)

var (
	knowledgeBaseUrl = fmt.Sprintf("%s/v1/knowledge/%s/file/add", apiEndpoint, knowledgeBaseId)
)

type FileAddBody struct {
	FileId string `json:"file_id"`
}

func main() {
	fp := gofeed.NewParser()

	feed, _ := fp.ParseURL(feedUrl)

	highestId := uint32(0)
	newHighestId := uint32(0)

	hidFh, err := os.OpenFile(invFilePath, os.O_RDONLY, 0644)
	if err == nil {
		hidBytes := make([]byte, 8)
		hidFh.Read(hidBytes)
		hidFh.Close()
		highestId = binary.LittleEndian.Uint32(hidBytes)
	}

	fmt.Printf("Highest ID at Start: %v\n", highestId)

	for _, item := range feed.Items {
		if item.Description == "<p></p>" || item.Title == "" {
			continue
		}
		if strings.Contains(item.Title, "[No Title]") {
			continue
		}

		guid_parts := strings.Split(item.GUID, "/")
		tId := guid_parts[len(guid_parts)-1]
		tIdInt, _ := strconv.Atoi(tId)
		if uint32(tIdInt) <= highestId {
			continue
		}
		if uint32(tIdInt) > newHighestId {
			newHighestId = uint32(tIdInt)
		}

		fileName := fmt.Sprintf("%s/TruthSocial-Trump_%s.md", filePath, tId)

		fh, _ := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644)

		fh.WriteString(fmt.Sprintf("## %v\n\n", feed.Description))
		fh.WriteString(fmt.Sprintf("### %v\n\n", item.Published))
		fh.WriteString(fmt.Sprintf("%v\n\n", item.Description))

		fh.Close()

		sendKnowledge(fileName)
	}

	if newHighestId < highestId {
		newHighestId = highestId
	}
	fmt.Printf("Highest ID at End: %v\n", newHighestId)

	hidFh, err = os.OpenFile(invFilePath, os.O_CREATE|os.O_RDWR, 0644)
	if err == nil {
		hidBytes := make([]byte, 8)
		binary.LittleEndian.PutUint32(hidBytes, newHighestId)
		hidFh.Write(hidBytes)
		hidFh.Close()
	}
}

func sendKnowledge(fileName string) error {
	fh, err := os.Open(fileName)
	if err != nil {
		fmt.Printf("Error opening file %s: %v\n", fileName, err)
		return err
	}

	var body bytes.Buffer
	mpW := multipart.NewWriter(&body)
	fw, _ := mpW.CreateFormFile("file", fh.Name())
	io.Copy(fw, fh)
	mpW.Close()
	fh.Close()

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/v1/files/", apiEndpoint), &body)
	if err != nil {
		fmt.Printf("Error generating new request: %v\n", err)
		return err
	}
	req.Header.Add("Content-Type", mpW.FormDataContentType())
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiToken))

	hc := http.Client{}
	resp, err := hc.Do(req)
	if err != nil {
		fmt.Printf("Error at execute file upload: %v\n", err)
		return err
	}

	respBodyContents := make([]byte, 4096)
	rbLen, err := resp.Body.Read(respBodyContents)
	if err != nil {
		fmt.Printf("Error at unmarshal read response body: %v\n", err)
		return err
	}
	resp.Body.Close()

	respBody := map[string]any{}
	err = json.Unmarshal(respBodyContents[0:rbLen], &respBody)
	if err != nil {
		fmt.Printf("Error at unmarshal response body: %v\n", err)
		return err
	}

	knowledgeAddBody := FileAddBody{}
	knowledgeAddBody.FileId = respBody["id"].(string)
	knowledgeAddBytes, err := json.Marshal(knowledgeAddBody)
	if err != nil {
		fmt.Printf("Error at Marshal knowledgebase add body: %v\n", err)
		return err
	}

	req, err = http.NewRequest(http.MethodPost, knowledgeBaseUrl, bytes.NewReader(knowledgeAddBytes))
	if err != nil {
		fmt.Printf("Error at knowledgebase add request: %v\n", err)
		return err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiToken))

	resp, err = hc.Do(req)
	if err != nil {
		fmt.Printf("Error at knowledgebase add request do: %v\n", err)
		return err
	}

	if resp.StatusCode == 200 {
		fmt.Printf("Successfully added %s to knowledge base.\n", fileName)
	}

	return nil
}