NixOS BeeGFS 7.4.0

beegfs.nix

{ pkgs, kernel ? pkgs.linux, ... } :
with pkgs;
let
  version = "7.4.0";
in stdenvNoCC.mkDerivation {
  pname = "beegfs";
  inherit version;

  src = fetchurl {
    name = "beegfs-archive-${version}.tar.bz2";
    # url = "https://git.beegfs.com/pub/v7/repository/archive.tar.bz2?ref=${version}";
    url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.bz2";
    sha256 = "sha256-VwD3z3lZIs5aOIBbwUvEkOxkFggTCv8OWuJMCga2ooo=";
  };

  nativeBuildInputs = [ which unzip pkg-config cppunit perl makeWrapper ];

  buildInputs = [
    gcc12
    libuuid
    attr
    xfsprogs
    zlib
    openssl
    sqlite
    rdma-core
    openssh
    gfortran
    influxdb
    curl
    rdma-core
    pahole
  ];

  hardeningDisable = [ "format" ]; # required for building beeond

  postPatch = ''
    patchShebangs ./
    find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
    find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
    find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
  '';

  buildPhase = ''
    make BEEGFS_OPENTK_IBVERBS=1 \
    KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build \
    ''${enableParallelBuilding:+-j''${NIX_BUILD_CORES} \
    -l''${NIX_BUILD_CORES}}
  '';

  enableParallelBuilding = true;

  installPhase = ''
    binDir=$out/bin
    docDir=$out/share/doc/beegfs
    includeDir=$out/include/beegfs
    libDir=$out/lib
    libDirPkg=$out/lib/beegfs
    mkdir -p $binDir $libDir $libDirPkg $docDir $includeDir
    cp common/build/libbeegfs_ib.so $libDir
    cp ctl/build/beegfs-ctl $binDir
    cp fsck/build/beegfs-fsck $binDir
    cp utils/scripts/beegfs-check-servers $binDir
    cp utils/scripts/beegfs-df $binDir
    cp utils/scripts/beegfs-net $binDir
    cp helperd/build/beegfs-helperd $binDir
    cp helperd/build/dist/etc/beegfs-helperd.conf $docDir
    cp client_module/build/dist/sbin/beegfs-setup-client $binDir
    cp client_module/build/dist/etc/beegfs-client.conf $docDir
    cp meta/build/beegfs-meta $binDir
    cp meta/build/dist/sbin/beegfs-setup-meta $binDir
    cp meta/build/dist/etc/beegfs-meta.conf $docDir
    cp mgmtd/build/beegfs-mgmtd $binDir
    cp mgmtd/build/dist/sbin/beegfs-setup-mgmtd $binDir
    cp mgmtd/build/dist/etc/beegfs-mgmtd.conf $docDir
    cp storage/build/beegfs-storage $binDir
    cp storage/build/dist/sbin/beegfs-setup-storage $binDir
    cp storage/build/dist/etc/beegfs-storage.conf $docDir
    cp client_devel/build/dist/usr/share/doc/beegfs-client-devel/examples/* $docDir
    cp -r client_devel/include/* $includeDir
  '';

  # postFixup = ''
  #   for i in $(find $out/bin -type f -executable); do
  #     wrapProgram "$i" --prefix LD_LIBRARY_PATH : $out/lib
  #   done
  # '';

  doCheck = true;

  # checkPhase = ''
  #   LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/opentk_lib/build/ \
  #     common/build/test-runner --text
  # '';

  meta = with lib; {
    description = "High performance distributed filesystem with RDMA support";
    homepage = "https://www.beegfs.io";
    platforms = [ "i686-linux" "x86_64-linux" ];
    license = {
      fullName = "BeeGFS_EULA";
      url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt";
      free = true;
    };
    maintainers = with maintainers; [ "juselius" ];
  };
}

default.nix

# Heavily based on the great work by @markuskowa
{ config, lib, pkgs, ...} :
 
with lib;

let
  cfg = config.features.hpc.beegfs.beegfs;

  # kernel = pkgs.linuxPackages_5_4.kernel;
  kernel = config.boot.kernelPackages.kernel;

  beegfs = pkgs.callPackage ./beegfs.nix {
    inherit kernel;
  };
  beegfs-module = pkgs.callPackage ./kernel-module.nix {
    inherit kernel;
  };

  # functions for the generations of config files

  configMgmtd = name: cfg: pkgs.writeText "mgmt-${name}.conf" ''
    storeMgmtdDirectory = ${cfg.mgmtd.storeDir}
    storeAllowFirstRunInit = false
    connAuthFile = ${cfg.connAuthFile}
    connPortShift = ${toString cfg.connPortShift}
    ${cfg.mgmtd.extraConfig}
  '';

  configMeta = name: cfg: pkgs.writeText "meta-${name}.conf" ''
    storeMetaDirectory = ${cfg.meta.storeDir}
    sysMgmtdHost = ${cfg.mgmtdHost}
    connAuthFile = ${cfg.connAuthFile}
    connPortShift = ${toString cfg.connPortShift}
    storeAllowFirstRunInit = false
    ${cfg.meta.extraConfig}
  '';

  configStorage = name: cfg: pkgs.writeText "storage-${name}.conf" ''
    storeStorageDirectory = ${cfg.storage.storeDir}
    sysMgmtdHost = ${cfg.mgmtdHost}
    connAuthFile = ${cfg.connAuthFile}
    connPortShift = ${toString cfg.connPortShift}
    storeAllowFirstRunInit = false
    ${cfg.storage.extraConfig}
  '';

  configHelperd = name: cfg: pkgs.writeText "helperd-${name}.conf" ''
    connAuthFile = ${cfg.connAuthFile}
    ${cfg.helperd.extraConfig}
  '';

  configClientFilename = name : "/etc/beegfs/client-${name}.conf";

  configClient = name: cfg: ''
    sysMgmtdHost = ${cfg.mgmtdHost}
    connAuthFile = ${cfg.connAuthFile}
    connPortShift = ${toString cfg.connPortShift}
    ${cfg.client.extraConfig}
  '';

  serviceList = [
    { service = "meta"; cfgFile = configMeta; }
    { service = "mgmtd"; cfgFile = configMgmtd; }
    { service = "storage"; cfgFile = configStorage; }
  ];

  # functions to generate systemd.service entries

  systemdEntry = service: cfgFile: (mapAttrs' ( name: cfg:
    (nameValuePair "beegfs-${service}-${name}" (mkIf cfg.${service}.enable {
    wantedBy = [ "multi-user.target" ];
    requires = [ "network-online.target" ];
    after = [ "network-online.target" ];
    serviceConfig = rec {
      ExecStart = ''
        ${beegfs}/bin/beegfs-${service} \
          cfgFile=${cfgFile name cfg} \
          pidFile=${PIDFile}
      '';
      PIDFile = "/run/beegfs-${service}-${name}.pid";
      TimeoutStopSec = "300";
    };
  }))) cfg);

  systemdHelperd =  mapAttrs' ( name: cfg:
    (nameValuePair "beegfs-helperd-${name}" (mkIf cfg.client.enable {
    wantedBy = [ "multi-user.target" ];
    requires = [ "network-online.target" ];
    after = [ "network-online.target" ];
    environment = {
      LD_LIBRARY_PATH = "${beegfs}/lib";
    };
    serviceConfig = rec {
      ExecStart = ''
        ${beegfs}/bin/beegfs-helperd \
          cfgFile=${configHelperd name cfg} \
          pidFile=${PIDFile}
      '';
      PIDFile = "/run/beegfs-helperd-${name}.pid";
      TimeoutStopSec = "300";
    };
   }))) cfg;

  # wrappers to beegfs tools. Avoid typing path of config files
  utilWrappers = mapAttrsToList ( name: cfg:
    ( pkgs.runCommand "beegfs-utils-${name}" {
        nativeBuildInputs = [ pkgs.makeWrapper ];
        preferLocalBuild = true;
        } ''
        mkdir -p $out/bin
        makeWrapper ${beegfs}/bin/beegfs-check-servers \
                    $out/bin/beegfs-check-servers-${name} \
                    --add-flags "-c ${configClientFilename name}" \
                    --prefix PATH : ${lib.makeBinPath [ beegfs ]}
        makeWrapper ${beegfs}/bin/beegfs-ctl \
                    $out/bin/beegfs-ctl-${name} \
                    --add-flags "--cfgFile=${configClientFilename name}"
        makeWrapper ${beegfs}/bin/beegfs-ctl \
                    $out/bin/beegfs-df-${name} \
                    --add-flags "--cfgFile=${configClientFilename name}" \
                    --add-flags --listtargets  \
                    --add-flags --hidenodeid \
                    --add-flags --pools \
                    --add-flags --spaceinfo
        makeWrapper ${beegfs}/bin/beegfs-fsck \
                    $out/bin/beegfs-fsck-${name} \
                    --add-flags "--cfgFile=${configClientFilename name}"
      ''
     )) cfg;
   beegfsOptions = {
     options = {
       mgmtdHost = mkOption {
         type = types.str;
         default = null;
         example = "master";
         description = ''Hostname of managament host.'';
       };

       connAuthFile = mkOption {
         type = types.str;
         default = "";
         example = "/etc/my.key";
         description = "File containing shared secret authentication.";
       };

       connPortShift = mkOption {
         type = types.int;
         default = 0;
         example = 5;
         description = ''
            For each additional beegfs configuration shift all
            service TCP/UDP ports by at least 5.
         '';
       };

       client = {
         enable = mkEnableOption "BeeGFS client";

         mount = mkOption {
           type = types.bool;
           default = true;
           description = "Create fstab entry automatically";
         };

         mountPoint = mkOption {
           type = types.str;
           default = "/run/beegfs";
           description = ''
              Mount point under which the beegfs filesytem should be mounted.
              If mounted manually the mount option specifing the config file is needed:
              cfgFile=/etc/beegfs/beegfs-client-<name>.conf
           '';
         };

         extraConfig = mkOption {
           type = types.lines;
           default = "";
           description = ''
              Additional lines for beegfs-client.conf.
              See documentation for further details.
           '';
         };
       };

       helperd = {
         enable = mkOption {
           type = types.bool;
           default = true;
           description = ''
              Enable the BeeGFS helperd.
              The helpered is need for logging purposes on the client.
              Disabling <literal>helperd</literal> allows for runing the client
              with <literal>allowUnfree = false</literal>.
           '';
         };

         extraConfig = mkOption {
           type = types.lines;
           default = "";
           description = ''
              Additional lines for beegfs-helperd.conf. See documentation
              for further details.
           '';
         };
       };

       mgmtd = {
         enable = mkEnableOption "BeeGFS mgmtd daemon";

         storeDir = mkOption {
           type = types.path;
           default = null;
           example = "/data/beegfs-mgmtd";
           description = ''
              Data directory for mgmtd.
              Must not be shared with other beegfs daemons.
              This directory must exist and it must be initialized
              with beegfs-setup-mgmtd, e.g. "beegfs-setup-mgmtd -C -p &lt;storeDir&gt;"
           '';
         };

         extraConfig = mkOption {
           type = types.lines;
           default = "";
           description = ''
              Additional lines for beegfs-mgmtd.conf. See documentation
              for further details.
           '';
         };
       };

       meta = {
         enable = mkEnableOption "BeeGFS meta data daemon";

         storeDir = mkOption {
           type = types.path;
           default = null;
           example = "/data/beegfs-meta";
           description = ''
              Data directory for meta data service.
              Must not be shared with other beegfs daemons.
              The underlying filesystem must be mounted with xattr turned on.
              This directory must exist and it must be initialized
              with beegfs-setup-meta, e.g.
              "beegfs-setup-meta -C -s &lt;serviceID&gt; -p &lt;storeDir&gt;"
           '';
         };

         extraConfig = mkOption {
           type = types.str;
           default = "";
           description = ''
              Additional lines for beegfs-meta.conf. See documentation
              for further details.
           '';
         };
       };

       storage = {
         enable = mkEnableOption "BeeGFS storage daemon";

         storeDir = mkOption {
           type = types.path;
           default = null;
           example = "/data/beegfs-storage";
           description = ''
              Data directories for storage service.
              Must not be shared with other beegfs daemons.
              The underlying filesystem must be mounted with xattr turned on.
              This directory must exist and it must be initialized
              with beegfs-setup-storage, e.g.
              "beegfs-setup-storage -C -s &lt;serviceID&gt; -i &lt;storageTargetID&gt; -p &lt;storeDir&gt;"
           '';
         };

         extraConfig = mkOption {
           type = types.str;
           default = "";
           description = ''
              Addional lines for beegfs-storage.conf. See documentation
              for further details.
           '';
         };
       };
     };
   };
in
{
  ###### interface

  options.features.hpc.beegfs = {
    enable = mkEnableOption "BeeGFS";

    beegfs = mkOption {
      type = with types; attrsOf (submodule ({ ... } : beegfsOptions ));
      default = {};
      description = ''
          BeeGFS configurations. Every mount point requires a separate configuration.
      '';
    };
  };

  ###### implementation

  config = mkIf config.features.hpc.beegfs.enable {

    environment.systemPackages = utilWrappers;

    # Put the client.conf files in /etc since they are needed
    # by the commandline tools
    environment.etc = mapAttrs' ( name: cfg:
      (nameValuePair "beegfs/client-${name}.conf" (mkIf (cfg.client.enable)
    {
      enable = true;
      text = configClient name cfg;
    }))) cfg;

    # Kernel module, we need it only once per host.
    boot = mkIf (
      foldr (a: b: a || b) false
        (map (x: x.client.enable) (collect (x: x ? client) cfg)))
    {
      kernelModules = [ "beegfs" ];
      extraModulePackages = [ beegfs-module ];
    };

    # generate fstab entries
    fileSystems = mapAttrs' (name: cfg:
      (nameValuePair cfg.client.mountPoint (optionalAttrs cfg.client.mount (mkIf cfg.client.enable {
      device = "beegfs_nodev";
      fsType = "beegfs";
      mountPoint = cfg.client.mountPoint;
      options = [ "cfgFile=${configClientFilename name}" "_netdev" ];
    })))) cfg;

    # generate systemd services
    systemd.services = systemdHelperd //
      foldr (a: b: a // b) {}
        (map (x: systemdEntry x.service x.cfgFile) serviceList);
  };
}

kernel-module.nix

{ pkgs, kernel ? pkgs.linux, ... } :
with pkgs;
let
  version = "7.4.0";
  beegfs = pkgs.callPackage ./beegfs.nix { inherit kernel; };
in stdenvNoCC.mkDerivation {
  name = "beegfs-module-${version}-${kernel.version}";

  src = fetchurl {
    name = "beegfs-archive-${version}.tar.bz2";
    # url = "https://git.beegfs.com/pub/v7/repository/archive.tar.bz2?ref=${version}";
    url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.bz2";
    sha256 = "sha256-VwD3z3lZIs5aOIBbwUvEkOxkFggTCv8OWuJMCga2ooo=";
  };

  hardeningDisable = [ "fortify" "pic" "stackprotector" ];

  nativeBuildInputs = [ gcc12 which kmod pahole ];

  buildInputs = kernel.moduleBuildDependencies;

  makeFlags = [ "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/" ];

  postPatch = ''
    patchShebangs ./
    find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
    find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
    find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
    find -type f -name "configure" -exec sed -i "s:/bin/:/usr/bin/env :" \{} \;
    find -type f -name "configure" -exec sed -i "s:/usr/bin/:/usr/bin/env :" \{} \;
    sed -i 's,libbeegfs_ib.so,${beegfs}/lib/&,' common/source/common/net/sock/RDMASocket.cpp
  '';

  preBuild = "cd client_module/build";

  installPhase = ''
    instdir=$out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs
    mkdir -p $instdir
    cp beegfs.ko $instdir
  '';

  meta = with lib; {
    description = "High performance distributed filesystem with RDMA support";
    homepage = "https://www.beegfs.io";
    platforms = [ "i686-linux" "x86_64-linux" ];
    license = licenses.gpl2;
    maintainers = with maintainers; [ "juselius" ];
    # broken = stdenv.lib.versionAtLeast kernel.version "4.18";
  };
}