airtable-iterator-error.py

def load(connection, airtable, table_name, field_info, destination):
    restarts = 0
    while True:
        try:
            chunks = airtable.iter_chunks(table_name)
            # Airtable will give us the records in chunks of 100, which is
            # inefficiently small for loading into Snowflake, so buffer them up into
            # chunks of a few thousand
            chunks = rechunked(chunks, 5000)
            return _load_chunks(connection, chunks, field_info, destination)
        except AirtableIteratorError:
            if restarts == 3:
                raise
            else:
                restarts += 1
                log.warn(
                    "airtable list records iterator not available, restarting",
                    restart=restarts,
                )