README.md

EfficientTextHandler

Usage:

// The LLM Workflow contains the prompt and pass it to the LLM
const workflow = new ConceptExtractorChapterWorkflow()

const content = `${document.title}\n\n${document.content}`

const textHandler = new EfficientTextHandler(content)

const result = await workflow.execute({
  context: this.context,
  // textHandler.wrappedText contains the text wrapped in <div id="n">...</div>
  content: textHandler.wrappedText,
})

const chapters: string[] = []

// The LLM returns an array of chapter start and end using div IDs
for (const chapter of result.chapters) {
  chapters.push(
    textHandler.extractPart({ start: chapter.start, end: chapter.end })
  )
}

return chapters

EfficientTextHandler.ts

export class EfficientTextHandler {
  /**
   * Wrap each line in a <div> tag with an id attribute.
   * The LLM will better understand the structure of the text.
   */
  static wrap(text: string) {
    return text
      .split('\n')
      .map((p, index) => `<div id="div-id-${index + 1}">${p}</div>`)
      .join('\n')
  }

  /**
   * Remove the <div> tags from the wrapped text.
   */
  static unwrap(wrappedText: string) {
    return wrappedText
      .split('\n')
      .map((div) => div.replace(/<div id="div-id-\d+">(.+?)<\/div>/, '$1'))
      .join('\n')
  }

  static getLineNumber(divId: string) {
    const lineNumber = parseInt(divId.replace('div-id-', ''), 10)

    if (Number.isNaN(lineNumber)) {
      throw new Error(`Invalid div ID: ${divId}`)
    }

    return lineNumber
  }

  public text: string
  public wrappedText: string

  constructor(text: string) {
    this.text = text
    this.wrappedText = EfficientTextHandler.wrap(text)
  }

  /**
   * Extract a part of the text between two lines. (included)
   */
  extractPart({ start, end }: { start: number; end: number }): string {
    const divPattern = /<div id="div-id-(\d+)">(.*?)<\/div>/g
    let match
    const result: string[] = []
    let capturing = false
    let startFound = false
    let endFound = false

    while ((match = divPattern.exec(this.wrappedText)) !== null) {
      const divId = parseInt(match[1], 10)

      if (divId === start) {
        capturing = true
        startFound = true
      }

      if (capturing) {
        result.push(match[2])
      }

      if (divId === end) {
        endFound = true
        break
      }
    }

    if (!startFound) {
      throw new Error(`Start ID ${start} not found`)
    }

    if (!endFound) {
      throw new Error(`End ID ${end} not found`)
    }

    return result.join('\n')
  }
}