/default.nix Secret
Last active
February 6, 2021 15:22
-
-
Save djanatyn/4f177b9bb46b80389bb90f0a02eb9a83 to your computer and use it in GitHub Desktop.
Troubles with propagated build inputs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ pkgs ? | |
import (fetchTarball "http://nixos.org/channels/nixos-20.09/nixexprs.tar.xz") | |
{ } }: | |
pkgs.python3Packages.buildPythonApplication { | |
pname = "load"; | |
src = ./.; | |
version = "0.3.0"; | |
propagatedBuildInputs = with pkgs.python38Packages; [ | |
python-dotenv | |
joblib | |
multiprocess | |
]; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import multiprocessing | |
from optparse import OptionParser | |
from pathlib import Path | |
from dotenv import dotenv_values | |
from joblib import Parallel | |
num_cores = multiprocessing.cpu_count() | |
parser = OptionParser( | |
usage="%prog [options] file...\n\nLoad CSV files into a sql database", | |
version="%prog v0.3.0", | |
) | |
parser.add_option( | |
"-d", | |
"--drop-schema", | |
dest="drop_schema", | |
action="store_true", | |
help="Drop existing database schema before create", | |
) | |
parser.add_option( | |
"-f", | |
"--environment-file", | |
dest="environment_file", | |
help="Location of environment file containing credentials", | |
default=Path(".") / ".env", | |
) | |
parser.add_option( | |
"-s", | |
"--db-schema", | |
dest="db_schema", | |
type="string", | |
help="The database schema to load .csv files into", | |
) | |
parser.add_option( | |
"-t", | |
"--type", | |
dest="db_type", | |
type="string", | |
default="postgresql", | |
help="Type of sql database (postgresql or athena)", | |
) | |
(options, args) = parser.parse_args() | |
def print_arg(arg): | |
print(f"arg: {arg}") | |
def run(): | |
Parallel(n_jobs=num_cores)(delayed(print_arg)(i) for i in args) | |
if __name__ == "__main__": | |
run() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from setuptools import setup | |
setup( | |
name="load", | |
version="0.3.0", | |
py_modules=["load"], | |
entry_points={"console_scripts": ["load = load:run"]}, | |
) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hey @djanatyn did you have a chance to take a look at this build issue with python? Any tips would be greatly appreciated. Thanks.