embedding_pipelines.py

def urls(base_url: str, max_scrape_depth: int = 1, cutoff: int = None) -> pd.DataFrame:
    """Gives all recursive URLs from the given base URL."""

def embedder() -> URLEmbedder:
    """Sets up an embedder to embed URLs on a remote GPU box."""

def embeddings_df(urls: pd.DataFrame, embedder: URLEmbedder) -> pd.DataFrame:
    """Adds embeddings to the given URL DataFrame."""

def saved_embeddings(embeddings_df: pd.DataFrame) -> str:
    """Saves the embeddings to disk + returns URL"""