Skip to content

Instantly share code, notes, and snippets.

@nh2

nh2/almost-works-only-glusterfind-fails.nix

Last active May 10, 2017 00:57
Download ZIP

Revisions

  1. nh2 revised this gist May 10, 2017. 1 changed file with 117 additions and 0 deletions.
    117 changes: 117 additions & 0 deletions working-with-tests.nix
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,117 @@
    {stdenv, fetchurl, fuse, bison, flex_2_5_35, openssl, python2, ncurses, readline,
    autoconf, automake, libtool, pkgconfig, zlib, libaio, libxml2, acl, sqlite
    , liburcu, attr, makeWrapper, coreutils, gnused, gnugrep, which, python2Packages
    }:
    let
    s =
    rec {
    baseName="glusterfs";
    # NOTE: On each glusterfs release, it should be checked if gluster added
    # new, or changed, Python scripts whose PYTHONPATH has to be set in
    # `postFixup` below, and whose runtime deps need to go into
    # `nativeBuildInputs`.
    # The command
    # find /nix/store/...-glusterfs-.../ -name '*.py' -executable
    # can help with finding new Python scripts.
    version = "3.10.1";
    name="${baseName}-${version}";
    url="https://github.com/nh2/glusterfs/archive/v3.10.1-runner-log.tar.gz";
    sha256 = "0d0n981vhyrxxwbfkdkbhyswiipa8pipz8j80mbvwb8aiqsk6kqs";
    };
    buildInputs = [
    fuse bison flex_2_5_35 openssl ncurses readline
    autoconf automake libtool pkgconfig zlib libaio libxml2
    acl sqlite liburcu attr makeWrapper
    (python2.withPackages (pkgs: [
    pkgs.flask
    pkgs.prettytable
    pkgs.requests
    pkgs.xattr
    ]))
    # NOTE: `python2` has to be *AFTER* the above `python2.withPackages`,
    # to ensure that the packages are available but the `toPythonPath`
    # shell function used in `postFixup` is also still available.
    python2
    ];
    # Some of the headers reference acl
    propagatedBuildInputs = [
    acl
    ];
    in
    stdenv.mkDerivation
    rec {
    inherit (s) name version;
    inherit buildInputs propagatedBuildInputs;

    preConfigure = ''
    ./autogen.sh
    '';

    configureFlags = [
    ''--localstatedir=/var''
    ];

    makeFlags = "DESTDIR=$(out)";

    enableParallelBuilding = true;

    postInstall = ''
    cp -r $out/$out/* $out
    rm -r $out/nix
    wrapProgram $out/sbin/mount.glusterfs --set PATH "${stdenv.lib.makeBinPath [ coreutils gnused attr gnugrep which]}"
    '';

    postFixup = ''
    # Set Python environment for the Python based utilities.
    # It would be nice if there was a better way to do this, automatically for all of them.
    # Also, this is brittle: If we forget a dependency or gluster adds a new one, things will break deep inside gluster.
    # We should better try to get an explicit list of Python dependencies from gluster and ensure all of them are in the PYTHONPATH of all these python scripts.
    # But at the time of writing (gluster 3.10), gluster only provides this in form of a gluster.spec file for RPM creation,
    # and even that one is not complete (for example it doesn't mention the `flask` dependency).
    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out):$out/libexec/glusterfs"
    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/share/glusterfs/scripts/eventsdash.py --set PYTHONPATH "$(toPythonPath $out)"
    # Tests that the above programs work without import errors.
    # For testing it manually in a shell you may want to substitute `$out` with `$(dirname $(readlink -f $(which gluster)))/../`.
    $out/bin/gluster-eventsapi --help
    $out/bin/gluster-mountbroker --help
    $out/bin/gluster-georep-sshkey --help
    $out/bin/glusterfind --help
    $out/libexec/glusterfs/peer_eventsapi.py --help
    $out/libexec/glusterfs/peer_georep-sshkey.py --help
    $out/libexec/glusterfs/peer_mountbroker.py --help
    $out/libexec/glusterfs/glusterfind/changelog.py --help
    # gfid_to_path.py doesn't accept --help, and it requires different arguments
    # (a dir as single argument) than the usage prints when stdin is not a TTY.
    # The `echo ""` is just so that stdin is not a TTY even if you try this line
    # on a real TTY for testing purposes.
    echo "" | (mkdir -p nix-test-dir-for-gfid_to_path && touch b && $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py nix-test-dir-for-gfid_to_path)
    $out/share/glusterfs/scripts/eventsdash.py --help
    '';

    src = fetchurl {
    inherit (s) url sha256;
    };

    meta = {
    inherit (s) version;
    description = "Distributed storage system";
    maintainers = [
    stdenv.lib.maintainers.raskin
    ];
    platforms = with stdenv.lib.platforms;
    linux ++ freebsd;
    };
    }
  2. nh2 revised this gist May 10, 2017. 1 changed file with 98 additions and 0 deletions.
    98 changes: 98 additions & 0 deletions working.nix
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,98 @@
    {stdenv, fetchurl, fuse, bison, flex_2_5_35, openssl, python2, ncurses, readline,
    autoconf, automake, libtool, pkgconfig, zlib, libaio, libxml2, acl, sqlite
    , liburcu, attr, makeWrapper, coreutils, gnused, gnugrep, which, python2Packages
    }:
    let
    s =
    rec {
    baseName="glusterfs";
    # NOTE: On each glusterfs release, it should be checked if gluster added
    # new, or changed, Python scripts whose PYTHONPATH has to be set in
    # `postFixup` below, and whose runtime deps need to go into
    # `nativeBuildInputs`.
    # The command
    # find /nix/store/...-glusterfs-.../ -name '*.py' -executable
    # can help with finding new Python scripts.
    version = "3.10.1";
    name="${baseName}-${version}";
    url="https://github.com/nh2/glusterfs/archive/v3.10.1-runner-log.tar.gz";
    sha256 = "0d0n981vhyrxxwbfkdkbhyswiipa8pipz8j80mbvwb8aiqsk6kqs";
    };
    buildInputs = [
    fuse bison flex_2_5_35 openssl ncurses readline
    autoconf automake libtool pkgconfig zlib libaio libxml2
    acl sqlite liburcu attr makeWrapper
    (python2.withPackages (pkgs: [
    pkgs.flask
    pkgs.prettytable
    pkgs.requests
    pkgs.xattr
    ]))
    # NOTE: `python2` has to be *AFTER* the above `python2.withPackages`,
    # to ensure that the packages are available but the `toPythonPath`
    # shell function used in `postFixup` is also still available.
    python2
    ];
    # Some of the headers reference acl
    propagatedBuildInputs = [
    acl
    ];
    in
    stdenv.mkDerivation
    rec {
    inherit (s) name version;
    inherit buildInputs propagatedBuildInputs;

    preConfigure = ''
    ./autogen.sh
    '';

    configureFlags = [
    ''--localstatedir=/var''
    ];

    makeFlags = "DESTDIR=$(out)";

    enableParallelBuilding = true;

    postInstall = ''
    cp -r $out/$out/* $out
    rm -r $out/nix
    wrapProgram $out/sbin/mount.glusterfs --set PATH "${stdenv.lib.makeBinPath [ coreutils gnused attr gnugrep which]}"
    '';

    postFixup = ''
    # Set Python environment for the Python based utilities.
    # (@nh2) I'm not sure if there's a better way to do this, automatically for all of them.
    # Also, this is brittle: If we forget a dependency or gluster adds a new one, things will break deep inside gluster.
    # We should better try to get an explicit list of Python dependencies from gluster and ensure all of them are in the PYTHONPATH of all these python scripts.
    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out):$out/libexec/glusterfs"
    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/share/glusterfs/scripts/eventsdash.py --set PYTHONPATH "$(toPythonPath $out)"
    '';

    src = fetchurl {
    inherit (s) url sha256;
    };

    meta = {
    inherit (s) version;
    description = "Distributed storage system";
    maintainers = [
    stdenv.lib.maintainers.raskin
    ];
    platforms = with stdenv.lib.platforms;
    linux ++ freebsd;
    };
    }
  3. nh2 revised this gist May 10, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion almost-works-only-glusterfind-fails.nix
    View file
    Original file line number Diff line number Diff line change
    @@ -67,13 +67,13 @@ rec {
    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out)"
    # glusterfind is the last one that fails
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out)"
    # glusterfind is the last one that fails
    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out)"
  4. nh2 revised this gist May 10, 2017. 1 changed file with 96 additions and 0 deletions.
    96 changes: 96 additions & 0 deletions almost-works-only-glusterfind-fails.nix
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,96 @@
    {stdenv, fetchurl, fuse, bison, flex_2_5_35, openssl, python2, ncurses, readline,
    autoconf, automake, libtool, pkgconfig, zlib, libaio, libxml2, acl, sqlite
    , liburcu, attr, makeWrapper, coreutils, gnused, gnugrep, which, python2Packages
    }:
    let
    s =
    rec {
    baseName="glusterfs";
    # NOTE: On each glusterfs release, it should be checked if gluster added
    # new, or changed, Python scripts whose PYTHONPATH has to be set in
    # `postFixup` below, and whose runtime deps need to go into
    # `nativeBuildInputs`.
    # The command
    # find /nix/store/...-glusterfs-.../ -name '*.py' -executable
    # can help with finding new Python scripts.
    version = "3.10.1";
    name="${baseName}-${version}";
    url="https://github.com/nh2/glusterfs/archive/v3.10.1-runner-log.tar.gz";
    sha256 = "0d0n981vhyrxxwbfkdkbhyswiipa8pipz8j80mbvwb8aiqsk6kqs";
    };
    buildInputs = [
    fuse bison flex_2_5_35 openssl ncurses readline
    autoconf automake libtool pkgconfig zlib libaio libxml2
    acl sqlite liburcu attr makeWrapper
    (python2.withPackages (pkgs: [
    pkgs.flask
    pkgs.prettytable
    pkgs.requests
    pkgs.xattr
    ]))
    python2
    ];
    # Some of the headers reference acl
    propagatedBuildInputs = [
    acl
    ];
    in
    stdenv.mkDerivation
    rec {
    inherit (s) name version;
    inherit buildInputs propagatedBuildInputs;

    preConfigure = ''
    ./autogen.sh
    '';

    configureFlags = [
    ''--localstatedir=/var''
    ];

    makeFlags = "DESTDIR=$(out)";

    enableParallelBuilding = true;

    postInstall = ''
    cp -r $out/$out/* $out
    rm -r $out/nix
    wrapProgram $out/sbin/mount.glusterfs --set PATH "${stdenv.lib.makeBinPath [ coreutils gnused attr gnugrep which]}"
    '';

    postFixup = ''
    # Set Python environment for the Python based utilities.
    # (@nh2) I'm not sure if there's a better way to do this, automatically for all of them.
    # Also, this is brittle: If we forget a dependency or gluster adds a new one, things will break deep inside gluster.
    # We should better try to get an explicit list of Python dependencies from gluster and ensure all of them are in the PYTHONPATH of all these python scripts.
    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out)"
    # glusterfind is the last one that fails
    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/share/glusterfs/scripts/eventsdash.py --set PYTHONPATH "$(toPythonPath $out)"
    '';

    src = fetchurl {
    inherit (s) url sha256;
    };

    meta = {
    inherit (s) version;
    description = "Distributed storage system";
    maintainers = [
    stdenv.lib.maintainers.raskin
    ];
    platforms = with stdenv.lib.platforms;
    linux ++ freebsd;
    };
    }
  5. nh2 revised this gist May 10, 2017. 1 changed file with 10 additions and 0 deletions.
    10 changes: 10 additions & 0 deletions test.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,10 @@
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../bin/gluster-eventsapi
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../bin/gluster-mountbroker
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../bin/gluster-georep-sshkey
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../bin/glusterfind
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/peer_eventsapi.py
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/peer_georep-sshkey.py
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/peer_mountbroker.py
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/glusterfind/changelog.py
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../libexec/glusterfs/gfind_missing_files/gfid_to_path.py
    $(dirname $(readlink -f /run/current-system/sw/bin/gluster))/../share/glusterfs/scripts/eventsdash.py --help
  6. nh2 created this gist May 9, 2017.
    86 changes: 86 additions & 0 deletions glusterfs.nix
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,86 @@
    {stdenv, fetchurl, fuse, bison, flex_2_5_35, openssl, python2, ncurses, readline,
    autoconf, automake, libtool, pkgconfig, zlib, libaio, libxml2, acl, sqlite
    , liburcu, attr, makeWrapper, coreutils, gnused, gnugrep, which, python2Packages
    }:
    let
    s =
    rec {
    baseName="glusterfs";
    version = "3.10.1";
    name="${baseName}-${version}";
    url="https://github.com/nh2/glusterfs/archive/v3.10.1-runner-log.tar.gz";
    sha256 = "0d0n981vhyrxxwbfkdkbhyswiipa8pipz8j80mbvwb8aiqsk6kqs";
    };
    buildInputs = [
    fuse bison flex_2_5_35 openssl python2 ncurses readline
    autoconf automake libtool pkgconfig zlib libaio libxml2
    acl sqlite liburcu attr makeWrapper
    (python2.withPackages (pkgs: [pkgs.flask])) # doesn't work
    ];
    # Some of the headers reference acl
    propagatedBuildInputs = [
    acl
    ];
    in
    stdenv.mkDerivation
    rec {
    inherit (s) name version;
    inherit buildInputs propagatedBuildInputs;

    # nativeBuildInputs = [
    # (python2.withPackages (pkgs: [pkgs.flask])) # works
    # ];

    preConfigure = ''
    ./autogen.sh
    '';

    configureFlags = [
    ''--localstatedir=/var''
    ];

    makeFlags = "DESTDIR=$(out)";

    enableParallelBuilding = true;

    postInstall = ''
    cp -r $out/$out/* $out
    rm -r $out/nix
    wrapProgram $out/sbin/mount.glusterfs --set PATH "${stdenv.lib.makeBinPath [ coreutils gnused attr gnugrep which]}"
    '';

    postFixup = ''
    # Set Python environment for the Python based utilities.
    # (@nh2) I'm not sure if there's a better way to do this, automatically for all of them.
    # Also, this is brittle: If we forget a dependency or gluster adds a new one, things will break deep inside gluster.
    # We should better try to get an explicit list of Python dependencies from gluster and ensure all of them are in the PYTHONPATH of all these python scripts.
    wrapProgram $out/bin/gluster-eventsapi --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.requests}):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/bin/gluster-mountbroker --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/bin/gluster-georep-sshkey --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/bin/glusterfind --set PYTHONPATH "$(toPythonPath $out)"
    wrapProgram $out/libexec/glusterfs/peer_eventsapi.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.requests}):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/libexec/glusterfs/peer_georep-sshkey.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/libexec/glusterfs/peer_mountbroker.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.prettytable})"
    wrapProgram $out/libexec/glusterfs/glusterfind/changelog.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.xattr}):$(toPythonPath ${python2Packages.cffi}):$(toPythonPath ${python2Packages.pycparser})"
    wrapProgram $out/libexec/glusterfs/gfind_missing_files/gfid_to_path.py --set PYTHONPATH "$(toPythonPath $out):$(toPythonPath ${python2Packages.xattr}):$(toPythonPath ${python2Packages.cffi}):$(toPythonPath ${python2Packages.pycparser})"
    wrapProgram $out/share/glusterfs/scripts/eventsdash.py --set PYTHONPATH "$(toPythonPath $out)"
    '';

    src = fetchurl {
    inherit (s) url sha256;
    };

    meta = {
    inherit (s) version;
    description = "Distributed storage system";
    maintainers = [
    stdenv.lib.maintainers.raskin
    ];
    platforms = with stdenv.lib.platforms;
    linux ++ freebsd;
    };
    }