austospumanto

import multiprocessing
import pickle
import struct
from typing import Optional, Callable, List, Any

from tqdm import tqdm


import pickle
import struct

austospumanto

"""
System/Runtime Requirements:
    Python3.7
    Linux / Mac

Must pip install:
    numpy, pandas

Must build from source:
    shared_memory (https://github.com/SleepProgger/py_shared_memory)

austospumanto

"""
pip install \
    bigquery-schema-generator \
    google-api-python-client \
    pandas-gbq \
    pandas
"""

import logging
import os

austospumanto

"""
pip install \
    numpy \
    pandas \
    numba
"""

import time
from dataclasses import dataclass
from functools import wraps

austospumanto

"""
pip install \
    pandas \
    sqlalchemy \
    pyarrow
"""

import json
import os
import time

austospumanto

"""
from .diskcached import clear_all, diskcached


@diskcached()
def cached_fn(x):
    print(x)
    return x + 2

austospumanto

"""
Modified file (<VIRTUALENV>/lib/python3.7/site-packages/sqlalchemy/dialects/mssql/base.py)
for sqlalchemy==1.3.7 to get it to work with sqlacodegen==2.1.0 
for an Azure SQL Data Warehouse server using the following get_engine function:

```
def get_engine(username=None, password=None, hostname=None, database=None) -> Engine:
    from skykick_ds.runconfig import RunConfig

    username = username or RunConfig.sql_username

austospumanto

"""
################
# Installation #
################
memory_profiler:
    pip install memory-profiler

pympler:
    pip install pympler

austospumanto

"""
System/Runtime Requirements:
    >=Python3.7
    Linux / Mac
    >=2 CPU Cores

Must pip install to run `processit`:
    pickle5, tqdm

Must pip install to run the tests and some functions:

austospumanto

"""
System/Runtime Requirements:
    >=Python3.7
    Linux / Mac
    >=2 CPU Cores

Must pip install to use `pd_processit`:
    pickle5, tqdm, numpy, pandas

To use this file, have processit.py in same folder as pd_processit.py (this file).