Scrape web page that exceeds prompt context length

app.py

from dotenv import load_dotenv
from griptape.drivers import MarkdownifyWebScraperDriver
from griptape.structures import Agent
from griptape.loaders import WebLoader
from griptape.tools import TaskMemoryClient, WebScraper

load_dotenv()


url = "https://www.kantata.com/careers"

agent = Agent(
    tools=[
        WebScraper(
            web_loader=WebLoader(
                web_scraper_driver=MarkdownifyWebScraperDriver(timeout=1000),
            ),
            off_prompt=True,
        ),
        TaskMemoryClient(off_prompt=False),
    ],
)
agent.run(f"List all job openings at '{url}' in a flat numbered markdown list.")


# Ouput:
# ```md
# 1. Senior Functional Consultant in London / Remote, United Kingdom
# 2. Senior Functional Consultant in Remote, United States
# 3. Senior Business Intelligence Consultant in London / Remote, United Kingdom
# 4. Senior Project Manager in Remote, United States
# 5. Technical Account Manager in London / Remote, United Kingdom
# 6. Technical Account Manager in Remote, United States
# 7. Technical Account Manager in Remote, United States
# 8. Business Development Representative in London (Hybrid), United Kingdom
# 9. Business Development Representative in London (Hybrid), United Kingdom
# 10. Business Development Representative in Boston (Hybrid), United States
# 11. Business Development Representative in Irvine (Hybrid), CA, United States
# 12. Business Development Representative, German Speaking in London (Hybrid), United Kingdom
# 13. Senior Solutions Engineer in London / Remote, United Kingdom
# 14. Commercial Account Executive in London / Remote, United Kingdom
# 15. Customer Community Manager in Remote, United States
# 16. Senior Customer Success Manager in London / Remote, United Kingdom
# 17. Customer Success Manager in Remote, United States
# ```