txtai-summary.py

import time

from txtai.pipeline import LLM, Summary, Textractor
from txtai.workflow import Task, Workflow

# Extract text from HTML, ignore boilerplate text
textractor = Textractor(lines=True, join=True, minlength=100)
text = textractor("https://github.com/neuml/txtai")

# Summarization with standard models
summary = Summary()

start = time.time()
print(summary(text))
print(f"ELAPSED = {time.time() - start:.2f}s")

# LLM summarization
llm = LLM("TheBloke/Mistral-7B-OpenOrca-AWQ", template="""
Create a summary in very simple terms for the following text.

text:
{text}

summary:
""")

start = time.time()
print(llm(text, maxlength=1024))
print(f"ELAPSED = {time.time() - start:.2f}s")

# Run as workflow
workflow = Workflow([Task(textractor), Task(summary)])
print(next(workflow(["https://github.com/neuml/txtai"])))

# Run as a FastAPI service
# config.yml:
#
# textractor:
#   lines: True
#   join: True
#   minlength: 100
#
# llm:
#   path: TheBloke/Mistral-7B-OpenOrca-AWQ
#   template: |
#     Create a summary in simple terms for the following text.
#
#     text:
#    {text}
# 
#     summary:
#
# summary:
#
# workflow:
#   llmsum:
#     tasks:
#       - textractor
#       - action: llm
#         args:
#           maxlength: 1024
#
#   summarize:
#     tasks:
#       - textractor
#       - summary

$ CONFIG=config.yml uvicorn "txtai.api:app"
$ curl -XPOST http://localhost:8000/workflow \
   -H 'Content-Type: application/json'
   -d '{"name": "llmsum", "elements": ["https://github.com/neuml/txtai"]}'

$ curl -XPOST http://localhost:8000/workflow \
   -H 'Content-Type: application/json'
   -d '{"name": "summarize", "elements": ["https://github.com/neuml/txtai"]}'

# Run as a Docker API service
$ docker build -t txtai-api --build-arg BASE_IMAGE=neuml/txtai-gpu api/.
$ docker run -p 8000:8000 txtai-api