tomtom103

# /// script
# requires-python = ">=3.8"
# dependencies = [
#     "httpx>=0.27.0",
#     "pyarrow>=14.0.0",
#     "h2>=4.3.0"
# ]
# ///
import argparse
import asyncio

tomtom103

import logging
from contextlib import asynccontextmanager
from functools import lru_cache
from typing import AsyncIterator, Optional

from sqlalchemy.ext.asyncio import (
    AsyncEngine,
    AsyncSession,
    async_sessionmaker,
    create_async_engine,

tomtom103

from psycopg2.pool import ThreadedConnectionPool as _ThreadedConnectionPool
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Semaphore
import pandas as pd
import numpy as np
import pickle

NUM_WORKERS = 10

class ThreadedConnectionPool(_ThreadedConnectionPool):