Draft

.gitignore

/.cache/
/.netrc
/netrc
/result

default.nix

(import ./setup.nix {}).env

Makefile

# Private repositories require .netrc file with
#
# machine my.private.repository.tld
# login username
# password secret
#
# This Makefile tries to symlink it from home directory or
# create it from environment variables (preferred on CI).

INDEX_URL ?= https://pypi.org/simple
INDEX_HOSTNAME ?= my.private.repository.tld
PYPI_USERNAME ?= guest
PYPI_PASSWORD ?= guest

PYTHON ?= python27
NIX_OPTIONS ?= --argstr python $(PYTHON)
PIP2NIX_OPTIONS ?= --licenses --no-binary :all:
REF_NIXPKGS = branches nixos-20.03

all: requirements

# Symlink .cache to share pip cache between projects
.cache:
	@if [ -f ~/.cache ]; then ln -s ~/.cache.; else \
	  mkdir -p .cache; \
	fi

.netrc:
	@if [ -f ~/.netrc ]; then ln -s ~/.netrc .; else \
	  echo machine ${INDEX_HOSTNAME} > .netrc && \
	  echo login ${PYPI_USERNAME} >> .netrc && \
	  echo password ${PYPI_PASSWORD} >> .netrc; \
	fi

netrc: .netrc
	@ln -s .netrc netrc

.PHONY: requirements
requirements: .cache requirements-$(PYTHON).nix

# HOME and NIX_CONF_DIR is set to allow both pip and nix to read private
# repository credentials from netrc.
requirements-$(PYTHON).nix: requirements-$(PYTHON).txt
	SOURCE_DATE_EPOCH=315532800 HOME=$(PWD) NIX_CONF_DIR=$(PWD) \
	nix-shell setup.nix $(NIX_OPTIONS) -A pip2nix --run "HOME=$(PWD) NIX_CONF_DIR=$(PWD) pip2nix generate -r requirements-$(PYTHON).txt --index-url $(INDEX_URL) $(PIP2NIX_OPTIONS) --output=requirements-$(PYTHON).nix"

requirements-$(PYTHON).txt: requirements.txt
	HOME=$(PWD) NIX_CONF_DIR=$(PWD) \
	nix-shell setup.nix $(NIX_OPTIONS) -A pip2nix --run "HOME=$(PWD) NIX_CONF_DIR=$(PWD) pip2nix generate -r requirements.txt --index-url $(INDEX_URL) --output=requirements-$(PYTHON).nix"
	@grep "pname =\|version =" requirements-$(PYTHON).nix|awk "ORS=NR%2?FS:RS"|sed 's|.*"\(.*\)";.*version = "\(.*\)".*|\1==\2|' > requirements-$(PYTHON).txt

.PHONY: upgrade
upgrade:
	nix-shell --pure -p cacert curl gnumake jq nix --run "make setup.nix"

.PHONY: setup.nix
setup.nix:
	@set -e pipefail; \
	echo "Updating nixpkgs @ setup.nix using $(REF_NIXPKGS)"; \
	rev=$$(curl https://api.github.com/repos/NixOS/nixpkgs-channels/$(firstword $(REF_NIXPKGS)) \
		| jq -er '.[]|select(.name == "$(lastword $(REF_NIXPKGS))").commit.sha'); \
	echo "Latest commit sha: $$rev"; \
	sha=$$(nix-prefetch-url --unpack https://github.com/NixOS/nixpkgs-channels/archive/$$rev.tar.gz); \
	sed -i \
		-e "2s|.*|    # $(REF_NIXPKGS)|" \
		-e "3s|.*|    url = \"https://github.com/NixOS/nixpkgs-channels/archive/$$rev.tar.gz\";|" \
		-e "4s|.*|    sha256 = \"$$sha\";|" \
		setup.nix

requirements.txt

zc.buildout

setup.nix

{ pkgs ? import (fetchTarball {
    # branches nixos-20.03
    url = "https://github.com/NixOS/nixpkgs-channels/archive/91cdcf313578e9520bb45cb21e6a9b1773bd656c.tar.gz";
    sha256 = "133sxph6qzankv5v63v8vlvg9dkbrki3b8xgi5pa88c0njx5x8qp";
  }) {}
, python ? "python27"
, pythonPackages ? builtins.getAttr (python + "Packages") pkgs
, requirements ?  ./. + "/requirements-${python}.nix"
}:

with builtins;
with pkgs;
with pkgs.lib;

let

  # Requirements for generating requirements.nix
  requirementsBuildInputs = [ cacert nix nix-prefetch-git ];

  # Load generated requirements
  requirementsFunc = import requirements {
    inherit pkgs;
    inherit (builtins) fetchurl;
    inherit (pkgs) fetchgit fetchhg;
  };

  # List package names in requirements
  requirementsNames = attrNames (requirementsFunc {} {});

  # Return base name from python drv name or name when not python drv
  pythonNameOrName = drv:
    if hasAttr "overridePythonAttrs" drv then drv.pname else drv.name;

  # Merge named input list from nixpkgs drv with input list from requirements drv
  mergedInputs = old: new: inputsName: self: super:
    (attrByPath [ inputsName ] [] new) ++ map
    (x: attrByPath [ (pythonNameOrName x) ] x self)
    (filter (x: !isNull x) (attrByPath [ inputsName ] [] old));

  # Merge package drv from nixpkgs drv with requirements drv
  mergedPackage = old: new: self: super:
    if isString new.src
       && !isNull (match ".*\.whl" new.src)  # do not merge build inputs for wheels
       && new.pname != "wheel"               # ...
    then new.overridePythonAttrs(old: rec {
      propagatedBuildInputs =
        mergedInputs old new "propagatedBuildInputs" self super;
    })
    else old.overridePythonAttrs(old: rec {
      inherit (new) pname version src;
      name = "${pname}-${version}";
      checkInputs =
        mergedInputs old new "checkInputs" self super;
      buildInputs =
        mergedInputs old new "buildInputs" self super;
      nativeBuildInputs =
        mergedInputs old new "nativeBuildInputs" self super;
      propagatedBuildInputs =
        mergedInputs old new "propagatedBuildInputs" self super;
      doCheck = false;
    });

  # Build python with manual aliases for naming differences between world and nix
  buildPython = (pythonPackages.python.override {
    packageOverrides = self: super:
      listToAttrs (map (name: {
        name = name; value = getAttr (getAttr name aliases) super;
      }) (filter (x: hasAttr (getAttr x aliases) super) (attrNames aliases)));
  });

  # Build target python with all generated & customized requirements
  targetPython = (buildPython.override {
    packageOverrides = self: super:
      # 1) Merge packages already in pythonPackages
      let super_ = (requirementsFunc self buildPython.pkgs);  # from requirements
          results = (listToAttrs (map (name: let new = getAttr name super_; in {
        inherit name;
        value = mergedPackage (getAttr name buildPython.pkgs) new self super_;
      })
      (filter (name: hasAttr "overridePythonAttrs"
                     (if (tryEval (attrByPath [ name ] {} buildPython.pkgs)).success
                      then (attrByPath [ name ] {} buildPython.pkgs) else {}))
       requirementsNames)))
      // # 2) with packages only in requirements or disabled in nixpkgs
      (listToAttrs (map (name: { inherit name; value = (getAttr name super_); })
      (filter (name: (! ((hasAttr name buildPython.pkgs) &&
                         (tryEval (getAttr name buildPython.pkgs)).success)))
       requirementsNames)));
      in # 3) finally, apply overrides (with aliased drvs mapped back)
      (let final = (super // (results //
        (listToAttrs (map (name: {
          name = getAttr name aliases; value = getAttr name results;
        }) (filter (x: hasAttr x results) (attrNames aliases))))
      )); in (final // (overrides self final)));
    self = buildPython;
  });

  # Alias packages with different names in requirements and in nixpkgs
  aliases = {
  };

  # Final overrides to fix issues all the magic above cannot fix automatically
  overrides = self: super: {
    pip = pythonPackages."pip";  # always use nixpkgs version of pip
  };

in rec {

  # shell with 'pip2nix' for resolving requirements.txt into requirements-pythonXX.nix
  pip2nix = mkShell {
    buildInputs = requirementsBuildInputs ++ [
      (pythonPackages.python.withPackages(ps: with ps; [
        (getAttr
          ("python" + replaceStrings ["."] [""] pythonPackages.python.pythonVersion)
          ( import (fetchTarball {
              url = "https://github.com/datakurre/pip2nix/archive/9ad83dba2c07f8bbdbb88c9f85b46f5635393238.tar.gz";
              sha256 = "0d344mgwl0b9ykmgvfx5cw7jarl1ynfib45i7laps7iq3dxx6m4r";
          } + "/release.nix") { inherit pkgs; }).pip2nix)
      ]))
    ];
  };

  shell = mkShell {
    buildInputs = [
      (targetPython.withPackages(ps: map (name: getAttr name ps) requirementsNames))
    ];
    shellHook = ''
    '';
  };

  env = pkgs.buildEnv {
    name = "env";
    paths = [
      (targetPython.withPackages(ps: map (name: getAttr name ps) requirementsNames))
    ];
  };

}

shell.nix

(import ./setup.nix {}).shell