nix gluster python scripts packaging

almost-works-only-glusterfind-fails.nix

{stdenv, fetchurl, fuse, bison, flex_2_5_35, openssl, python2, ncurses, readline,
 autoconf, automake, libtool, pkgconfig, zlib, libaio, libxml2, acl, sqlite
 , liburcu, attr, makeWrapper, coreutils, gnused, gnugrep, which, python2Packages
}:
let
  s =
  rec {
    baseName="glusterfs";
    # NOTE: On each glusterfs release, it should be checked if gluster added
    #       new, or changed, Python scripts whose PYTHONPATH has to be set in
    #       `postFixup` below, and whose runtime deps need to go into
    #       `nativeBuildInputs`.
    #       The command
    #         find /nix/store/...-glusterfs-.../ -name '*.py' -executable
    #       can help with finding new Python scripts.
    version = "3.10.1";
    name="${baseName}-${version}";
    url="https://github.com/nh2/glusterfs/archive/v3.10.1-runner-log.tar.gz";
    sha256 = "0d0n981vhyrxxwbfkdkbhyswiipa8pipz8j80mbvwb8aiqsk6kqs";
  };
  buildInputs = [
    fuse bison flex_2_5_35 openssl ncurses readline
    autoconf automake libtool pkgconfig zlib libaio libxml2
    acl sqlite liburcu attr makeWrapper
    (python2.withPackages (pkgs: [
      pkgs.flask
      pkgs.prettytable
      pkgs.requests
      pkgs.xattr
    ]))
    python2
  ];
  # Some of the headers reference acl
  propagatedBuildInputs = [
    acl
  ];
in
stdenv.mkDerivation
rec {
  inherit (s) name version;
  inherit buildInputs propagatedBuildInputs;

  preConfigure = ''
    ./autogen.sh
    '';

  configureFlags = [
    ''--localstatedir=/var''
    ];

  makeFlags = "DESTDIR=$(out)";

  enableParallelBuilding = true;

  postInstall = ''
    cp -r $out/$out/* $out
    rm -r $out/nix
    wrapProgram $out/sbin/mount.glusterfs --set PATH "${stdenv.lib.makeBinPath [ coreutils gnused attr gnugrep which]}"
    '';

  postFixup = ''
    # Set Python environment for the Python based utilities.
    # (@nh2) I'm not sure if there's a better way to do this, automatically for all of them.
    # Also, this is brittle: If we forget a dependency or gluster adds a new one, things will break deep inside gluster.
    # We should better try to get an explicit list of Python dependencies from gluster and ensure all of them are in the PYTHONPATH of all these python scripts.

    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out)"
    # glusterfind is the last one that fails
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out)"

    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out)"

    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out)"

    wrapProgram $out/share/glusterfs/scripts/eventsdash.py --set PYTHONPATH "$(toPythonPath $out)"
    '';

  src = fetchurl {
    inherit (s) url sha256;
  };

  meta = {
    inherit (s) version;
    description = "Distributed storage system";
    maintainers = [
      stdenv.lib.maintainers.raskin
    ];
    platforms = with stdenv.lib.platforms;
      linux ++ freebsd;
  };
}

glusterfs.nix

{stdenv, fetchurl, fuse, bison, flex_2_5_35, openssl, python2, ncurses, readline,
 autoconf, automake, libtool, pkgconfig, zlib, libaio, libxml2, acl, sqlite
 , liburcu, attr, makeWrapper, coreutils, gnused, gnugrep, which, python2Packages
}:
let
  s =
  rec {
    baseName="glusterfs";
    version = "3.10.1";
    name="${baseName}-${version}";
    url="https://github.com/nh2/glusterfs/archive/v3.10.1-runner-log.tar.gz";
    sha256 = "0d0n981vhyrxxwbfkdkbhyswiipa8pipz8j80mbvwb8aiqsk6kqs";
  };
  buildInputs = [
    fuse bison flex_2_5_35 openssl python2 ncurses readline
    autoconf automake libtool pkgconfig zlib libaio libxml2
    acl sqlite liburcu attr makeWrapper
    (python2.withPackages (pkgs: [pkgs.flask])) # doesn't work
  ];
  # Some of the headers reference acl
  propagatedBuildInputs = [
    acl
  ];
in
stdenv.mkDerivation
rec {
  inherit (s) name version;
  inherit buildInputs propagatedBuildInputs;

  # nativeBuildInputs = [
  #   (python2.withPackages (pkgs: [pkgs.flask])) # works
  # ];

  preConfigure = ''
    ./autogen.sh
    '';

  configureFlags = [
    ''--localstatedir=/var''
    ];

  makeFlags = "DESTDIR=$(out)";

  enableParallelBuilding = true;

  postInstall = ''
    cp -r $out/$out/* $out
    rm -r $out/nix
    wrapProgram $out/sbin/mount.glusterfs --set PATH "${stdenv.lib.makeBinPath [ coreutils gnused attr gnugrep which]}"
    '';

  postFixup = ''
    # Set Python environment for the Python based utilities.
    # (@nh2) I'm not sure if there's a better way to do this, automatically for all of them.
    # Also, this is brittle: If we forget a dependency or gluster adds a new one, things will break deep inside gluster.
    # We should better try to get an explicit list of Python dependencies from gluster and ensure all of them are in the PYTHONPATH of all these python scripts.

    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.requests}):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out)"

    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.requests}):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.prettytable})"

    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.xattr}):$(toPythonPath ${python2Packages.cffi}):$(toPythonPath ${python2Packages.pycparser})"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.xattr}):$(toPythonPath ${python2Packages.cffi}):$(toPythonPath ${python2Packages.pycparser})"

    wrapProgram $out/share/glusterfs/scripts/eventsdash.py --set PYTHONPATH "$(toPythonPath $out)"
    '';

  src = fetchurl {
    inherit (s) url sha256;
  };

  meta = {
    inherit (s) version;
    description = "Distributed storage system";
    maintainers = [
      stdenv.lib.maintainers.raskin
    ];
    platforms = with stdenv.lib.platforms;
      linux ++ freebsd;
  };
}

test.sh

$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../bin/gluster-eventsapi
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../bin/gluster-mountbroker
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../bin/gluster-georep-sshkey
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../bin/glusterfind
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/peer_eventsapi.py
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/peer_georep-sshkey.py
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/peer_mountbroker.py
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/glusterfind/changelog.py
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/gfind_missing_files/gfid_to_path.py
$(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../share/glusterfs/scripts/eventsdash.py --help

working-with-tests.nix

{stdenv, fetchurl, fuse, bison, flex_2_5_35, openssl, python2, ncurses, readline,
 autoconf, automake, libtool, pkgconfig, zlib, libaio, libxml2, acl, sqlite
 , liburcu, attr, makeWrapper, coreutils, gnused, gnugrep, which, python2Packages
}:
let
  s =
  rec {
    baseName="glusterfs";
    # NOTE: On each glusterfs release, it should be checked if gluster added
    #       new, or changed, Python scripts whose PYTHONPATH has to be set in
    #       `postFixup` below, and whose runtime deps need to go into
    #       `nativeBuildInputs`.
    #       The command
    #         find /nix/store/...-glusterfs-.../ -name '*.py' -executable
    #       can help with finding new Python scripts.
    version = "3.10.1";
    name="${baseName}-${version}";
    url="https://github.com/nh2/glusterfs/archive/v3.10.1-runner-log.tar.gz";
    sha256 = "0d0n981vhyrxxwbfkdkbhyswiipa8pipz8j80mbvwb8aiqsk6kqs";
  };
  buildInputs = [
    fuse bison flex_2_5_35 openssl ncurses readline
    autoconf automake libtool pkgconfig zlib libaio libxml2
    acl sqlite liburcu attr makeWrapper
    (python2.withPackages (pkgs: [
      pkgs.flask
      pkgs.prettytable
      pkgs.requests
      pkgs.xattr
    ]))
    # NOTE: `python2` has to be *AFTER* the above `python2.withPackages`,
    #       to ensure that the packages are available but the `toPythonPath`
    #       shell function used in `postFixup` is also still available.
    python2
  ];
  # Some of the headers reference acl
  propagatedBuildInputs = [
    acl
  ];
in
stdenv.mkDerivation
rec {
  inherit (s) name version;
  inherit buildInputs propagatedBuildInputs;

  preConfigure = ''
    ./autogen.sh
    '';

  configureFlags = [
    ''--localstatedir=/var''
    ];

  makeFlags = "DESTDIR=$(out)";

  enableParallelBuilding = true;

  postInstall = ''
    cp -r $out/$out/* $out
    rm -r $out/nix
    wrapProgram $out/sbin/mount.glusterfs --set PATH "${stdenv.lib.makeBinPath [ coreutils gnused attr gnugrep which]}"
    '';

  postFixup = ''
    # Set Python environment for the Python based utilities.
    # It would be nice if there was a better way to do this, automatically for all of them.
    # Also, this is brittle: If we forget a dependency or gluster adds a new one, things will break deep inside gluster.
    # We should better try to get an explicit list of Python dependencies from gluster and ensure all of them are in the PYTHONPATH of all these python scripts.
    # But at the time of writing (gluster 3.10), gluster only provides this in form of a gluster.spec file for RPM creation,
    # and even that one is not complete (for example it doesn't mention the `flask` dependency).

    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out):$out/libexec/glusterfs"

    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out)"

    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out)"

    wrapProgram $out/share/glusterfs/scripts/eventsdash.py --set PYTHONPATH "$(toPythonPath $out)"

    # Tests that the above programs work without import errors.
    # For testing it manually in a shell you may want to substitute `$out` with `$(dirname $(readlink -f $(which gluster)))/../`.
    $out/bin/gluster-eventsapi --help
    $out/bin/gluster-mountbroker --help
    $out/bin/gluster-georep-sshkey --help
    $out/bin/glusterfind --help
    $out/libexec/glusterfs/peer_eventsapi.py --help
    $out/libexec/glusterfs/peer_georep-sshkey.py --help
    $out/libexec/glusterfs/peer_mountbroker.py --help
    $out/libexec/glusterfs/glusterfind/changelog.py --help
    # gfid_to_path.py doesn't accept --help, and it requires different arguments
    # (a dir as single argument) than the usage prints when stdin is not a TTY.
    # The `echo ""` is just so that stdin is not a TTY even if you try this line
    # on a real TTY for testing purposes.
    echo "" | (mkdir -p nix-test-dir-for-gfid_to_path && touch b && $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py nix-test-dir-for-gfid_to_path)
    $out/share/glusterfs/scripts/eventsdash.py --help
    '';

  src = fetchurl {
    inherit (s) url sha256;
  };

  meta = {
    inherit (s) version;
    description = "Distributed storage system";
    maintainers = [
      stdenv.lib.maintainers.raskin
    ];
    platforms = with stdenv.lib.platforms;
      linux ++ freebsd;
  };
}

working.nix

{stdenv, fetchurl, fuse, bison, flex_2_5_35, openssl, python2, ncurses, readline,
 autoconf, automake, libtool, pkgconfig, zlib, libaio, libxml2, acl, sqlite
 , liburcu, attr, makeWrapper, coreutils, gnused, gnugrep, which, python2Packages
}:
let
  s =
  rec {
    baseName="glusterfs";
    # NOTE: On each glusterfs release, it should be checked if gluster added
    #       new, or changed, Python scripts whose PYTHONPATH has to be set in
    #       `postFixup` below, and whose runtime deps need to go into
    #       `nativeBuildInputs`.
    #       The command
    #         find /nix/store/...-glusterfs-.../ -name '*.py' -executable
    #       can help with finding new Python scripts.
    version = "3.10.1";
    name="${baseName}-${version}";
    url="https://github.com/nh2/glusterfs/archive/v3.10.1-runner-log.tar.gz";
    sha256 = "0d0n981vhyrxxwbfkdkbhyswiipa8pipz8j80mbvwb8aiqsk6kqs";
  };
  buildInputs = [
    fuse bison flex_2_5_35 openssl ncurses readline
    autoconf automake libtool pkgconfig zlib libaio libxml2
    acl sqlite liburcu attr makeWrapper
    (python2.withPackages (pkgs: [
      pkgs.flask
      pkgs.prettytable
      pkgs.requests
      pkgs.xattr
    ]))
    # NOTE: `python2` has to be *AFTER* the above `python2.withPackages`,
    #       to ensure that the packages are available but the `toPythonPath`
    #       shell function used in `postFixup` is also still available.
    python2
  ];
  # Some of the headers reference acl
  propagatedBuildInputs = [
    acl
  ];
in
stdenv.mkDerivation
rec {
  inherit (s) name version;
  inherit buildInputs propagatedBuildInputs;

  preConfigure = ''
    ./autogen.sh
    '';

  configureFlags = [
    ''--localstatedir=/var''
    ];

  makeFlags = "DESTDIR=$(out)";

  enableParallelBuilding = true;

  postInstall = ''
    cp -r $out/$out/* $out
    rm -r $out/nix
    wrapProgram $out/sbin/mount.glusterfs --set PATH "${stdenv.lib.makeBinPath [ coreutils gnused attr gnugrep which]}"
    '';

  postFixup = ''
    # Set Python environment for the Python based utilities.
    # (@nh2) I'm not sure if there's a better way to do this, automatically for all of them.
    # Also, this is brittle: If we forget a dependency or gluster adds a new one, things will break deep inside gluster.
    # We should better try to get an explicit list of Python dependencies from gluster and ensure all of them are in the PYTHONPATH of all these python scripts.

    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out):$out/libexec/glusterfs"

    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out)"

    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out)"

    wrapProgram $out/share/glusterfs/scripts/eventsdash.py --set PYTHONPATH "$(toPythonPath $out)"
    '';

  src = fetchurl {
    inherit (s) url sha256;
  };

  meta = {
    inherit (s) version;
    description = "Distributed storage system";
    maintainers = [
      stdenv.lib.maintainers.raskin
    ];
    platforms = with stdenv.lib.platforms;
      linux ++ freebsd;
  };
}