README.rst

Python development with Nix

DRAFT

Scope:

• Developing small or middle-size services in Python.
• Mixing PYPI and in-house dependencies

Restictions:

• No Plone yet, because of unsolved issues in generating Nix-expression, and issues with setuptools namespackages and wheels.

Issues working with Nixpkgs:

• Which nixpkgs to use
  • nixpkgs is growing fast and stable is probably missing packages
  • unstable updates may bring breakages
• How to mix stable with unstable with security patches
  • e.g. fork release, rebase to next release, cherry-pick patches (but may cause world rebuild)
• Adding new packages
  • add to fork, create pull request, get from upstream in the next release

Issues working with in-house packages

• Each package requires:
  • expression to build the package (incl. tests)
  • expression to build Docker image with the package
  • shell expression for running tests for the package
  • buildable interpreter expression with package requiremens for IDE
• The most "logical" place for the expressions are each package's repository, but
  • that's different to nixpkgs
  • makes composing expressions harder
• Apparently it's possible also to package's default.nix with traditional python sdist release and require it in Nix expression via import fetchTarball. Would you see any downsides with this approach?
• Other issues with in-house packages
  • double work in defining both python and nix packaging
    • maybe one JSON where all data flows to default.nix and setup.py
  • updating versions in both places
  • extra work in making python package releases to in-house package respository, even Nix could build from version control

Misc issues

• How to package apps with plugins?
  • Let A be the main app. A includes the CLI script.
  • Let B and C be add-ons for A. They require A.
  • Building either B or C includes A, but don't install CLI script.
  • Building A installs CLI script, but A cannot have B or C as its inputs.
  • So, how to package A with B and C without python.buildEnv?

Proposed structure

• Using Nixpkgs best practices
• Creating Python packages best practices
  • setup.py
  • tests
  • default.nix
  • packaging dist
  • Makefile
• Managing local Nix package tree
• Running local hydra

default.nix

{ pkgs ? import (builtins.fetchTarball  # revision for reproducible builds
  "https://github.com/nixos/nixpkgs-channels/archive/d9f5e94bae088234791ae28f0d813a9fad5b8163.tar.gz") {}
, pythonPackages ? pkgs.python35Packages
}:

let self = {
  version = builtins.replaceStrings ["\n"] [""] (builtins.readFile ./VERSION);

  tornado = with pythonPackages; tornado.override {
    name = "tornado-4.3b1";
    src = pkgs.fetchurl {
      url = "https://pypi.python.org/packages/source/t/tornado/tornado-4.3b1.tar.gz";
      sha256 = "c7ddda61d9469c5745f3ac00e480ede0703dd1a4ef540a3d9bd5e03e9796e430";
    };
    doCheck = false;  # Oh dear, it used to pass.
  };
};

in pythonPackages.buildPythonPackage rec {
  namePrefix = "";
  name = "example-${self.version}";
  src = builtins.filterSource
    (path: type: baseNameOf path != ".git"
              && baseNameOf path != "result")
    ./.;
  buildInputs = [
    pythonPackages.pytest
    pythonPackages.pytestrunner
  ];
  propagatedBuildInputs = with self; [
    tornado
  ];
}

MANIFEST.in

include VERSION *.py *.rst *.nix
exclude .* Makefile test_example.py

release.nix

{ image_name ? "example", image_tag ? "master", image_entrypoint ? "/bin/sh"
, supportedSystems ? [ "x86_64-linux" ]
, nixpkgs ? builtins.fetchTarball  # until > 15.09, unstable is required
  "https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz"
}:

let

  pkgs = import nixpkgs {};

  pkgFor = system: import ./default.nix {
    pkgs = import pkgs.path { inherit system; };
  };

in rec {

  build = pkgs.lib.genAttrs supportedSystems (system: pkgs.lib.hydraJob (
    pkgFor system
  ));

  python = pkgs.lib.genAttrs supportedSystems (system: pkgs.lib.hydraJob (
    let package = pkgFor system;
        syspkgs = import pkgs.path { inherit system; };
    in syspkgs.python35.buildEnv.override {
      extraLibs = package.nativeBuildInputs
                  ++ package.propagatedNativeBuildInputs;
      ignoreCollisions = true;
    }
  ));

  tarball = pkgs.lib.hydraJob((pkgFor "x86_64-linux")
                    .overrideDerivation(args: {
    phases = [ "unpackPhase" "buildPhase" ];
    buildPhase = ''
      ${python."x86_64-linux"}/bin/python3 setup.py sdist --formats=gztar
      mkdir -p $out/tarballs $out/nix-support
      mv dist/${args.name}.tar.gz $out/tarballs
      echo "file source-dist $out/tarballs/${args.name}.tar.gz" > \
           $out/nix-support/hydra-build-products
      echo ${args.name} > $out/nix-support/hydra-release-name
    '';
  }));

  image = pkgs.lib.hydraJob (
    let package = pkgFor "x86_64-linux";
        syspkgs = import pkgs.path { system = "x86_64-linux"; };
    in pkgs.dockerTools.buildImage {
      name = image_name;
      tag = image_tag;
      contents = [ syspkgs.busybox package ];
      runAsRoot = ''
        #!${pkgs.stdenv.shell}
        ${pkgs.dockerTools.shadowSetup}
        groupadd --system --gid 65534 nobody
        useradd --system --uid 65534 --gid 65534 -d / -s /sbin/nologin nobody
      '';
      config = {
        EntryPoint = [ "${image_entrypoint}" ];
        User = "nobody";
      };
    }
  );

  docker = pkgs.lib.hydraJob (
    pkgs.runCommand "${builtins.baseNameOf image_name}" {} ''
      mkdir -p $out/nix-support
      echo "file binary-dist ${image}" > $out/nix-support/hydra-build-products
      echo ${image_name}:${image_tag} > $out/nix-support/hydra-release-name
    ''
  );
}

service.py

# -*- coding: utf-8 -*-
import asyncio
import tornado.platform.asyncio
import tornado.web


class MainHandler(tornado.web.RequestHandler):
    async def get(self):
        self.write('Hello World!')


app = tornado.web.Application([
    (r'.*', MainHandler),
])


def main():
    tornado.platform.asyncio.AsyncIOMainLoop().install()
    app.listen(8080)
    asyncio.get_event_loop().run_forever()


if __name__ == "__main__":
    main()

setup.cfg

[aliases]
test = pytest

setup.py

from setuptools import setup

setup(
    name='example',
    version=open('VERSION').read().strip(),
    py_modules=[
        'service'
    ],
    install_requires=[
        'tornado'
    ],
    setup_requires=[
        'pytest-runner',
    ],
    tests_require=[
        'pytest'
    ],
    entry_points={
        'console_scripts': ['service = service:main']
    }
)

test_example.py

def test_dummy():
    assert True

VERSION

1.0.0