chrishop

{
    "remote_url": "http://api.skymapper.nci.org.au/public/tap",
    "remote_table": "dr1.dr1p1_master",
    "remote_table_id": "object_id",
    "remote_table_id_min": 0,
    "remote_table_id_max": 10000000,
    "download_batch_size": 500000,
    "local_host": "localhost",
    "local_user": "root",
    "local_password": "12345",

chrishop

# install mysql
sudo apt update
sudo apt install mysql-server

# should now open a window to set your root password if not do this:
sudo mysql_secure_instalation utility
sudo ufw enable
sudo ufw allow mysql

chrishop

def clean_column_names(columns: str) -> str:
    return strip_lower(underscore(columns))

def strip_lower(a_str: str) -> str:
    return a_str.strip().lower()

def underscore(a_str: str) -> str:
    return a_str.replace(" ","_").replace(punctuation,"")

chrishop

def read_data(path, useful_columns: List[str]) -> pd.DataFrame:
  return tidy_column_names(keep_useful_columns(pd.read_excel(path, usecols=columns_to_keep)))