get-fixed-output-pathv2.pl

#! /usr/bin/env nix-shell
#! nix-shell -i perl -p perl nixUnstable.perl-bindings

# pipe using stdin, to this script, where each line has the format
# <algo> <hash> <name>
# sha256 0q6gwd9pqh41f6j5p9jl92aji4p8kx9inffpha93d7jc3ndsfq6q rematch2_2.1.2.tar.gz

use strict;
use Nix::Store;

foreach my $line ( <STDIN> ) {
    chomp( $line );
    my @spl = split(' ', $line);
    
    print makeFixedOutputPath(0, $spl[0], $spl[1], $spl[2]) . "\n";
}

nixos-release-paths.sh

#!/usr/bin/env nix-shell
#!nix-shell -i bash -p nix-eval-jobs git
WORKERS=1

BINARY_CACHE="file:///home/eyjhb/nix-cache-test/cache-dir"
STORE_CACHE="https://cache.nixos.org"

# current directory of this script
SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"

# Git
GIT_REVISION="5550a85a087c04ddcace7f892b0bdc9d8bb080c8"

TMP_PATH="/tmp/nix-cache-tmp-dir"
TMP_NIXPKGS="$TMP_PATH/nixpkgs"
TMP_STORE_PATHS="$TMP_PATH/store-paths"

# move into our tmp directory
mkdir $TMP_PATH
pushd $TMP_PATH

# clone nixpkgs into TMP_PATH
git clone https://github.com/NixOS/nixpkgs.git $TMP_NIXPKGS
pushd $TMP_NIXPKGS
git pull
git checkout $GIT_REVISION
GIT_SHORT_REV="$(git rev-parse --short $GIT_REVISION)"
GIT_REV_COUNT="$(git rev-list --count $GIT_REVISION)"
popd

TMP_NIXPKGS=$(nix store add-path $TMP_NIXPKGS)

# TMP_NIXPKGS="/nix/store/i333qx94x4mcqqm84miqrj02kg3hlpsz-nixpkgs"

# eval `nixos/release-combined.nix` using nix-eval-jobs
mkdir $TMP_STORE_PATHS
pushd $TMP_STORE_PATHS

# tmp files for store paths
TMP_CACHE_RELEASE_FILE="$TMP_STORE_PATHS/eval-cache.json"
TMP_CACHE_TARBALLS_FILE="$TMP_STORE_PATHS/tarballs-cache.json"
TMP_RELEASE_STORE_PATHS_DRV_FILE="$TMP_STORE_PATHS/store-paths-drv"
TMP_RELEASE_STORE_PATHS_FILE="$TMP_STORE_PATHS/store-paths"
TMP_TARBALLS_STORE_PATHS_FILE="$TMP_STORE_PATHS/tarballs-store-paths"

echo "Using file $TMP_CACHE_RELEASE_FILE outputs of evaluating $TMP_NIXPKGS/nixos/release-combined.nix"
nix-eval-jobs \
    --workers $WORKERS \
    --keep-going \
    --quiet \
    --force-recurse \
    "$TMP_NIXPKGS/nixos/release-combined.nix" \
    --arg nixpkgs '{ outPath = builtins.storePath '$TMP_NIXPKGS'; rev = "'$GIT_REVISION'"; revCount = '$GIT_REV_COUNT'; shortRev = "'$GIT_SHORT_REV'"; }' \
    --arg stableBranch true \
    --arg supportedPlatforms '[ "x86_64-linux"]' \
    --option extra-binary-caches 'https://hydra.nixos.org/' \
    --option system x86_64-linux \
    -I nixpkgs=$TMP_NIXPKGS \
    > $TMP_CACHE_RELEASE_FILE
echo "Resulting file $TMP_CACHE_RELEASE_FILE"

# get store paths for both drv and outputs to separate files
cat $TMP_CACHE_RELEASE_FILE | jq -r '.drvPath' | sed '/^null$/d' | sort | uniq > $TMP_RELEASE_STORE_PATHS_DRV_FILE
cat $TMP_CACHE_RELEASE_FILE | jq -r '.outputs.out' | sed '/^null$/d' | sort | uniq > $TMP_RELEASE_STORE_PATHS_FILE

# get tarballs from tarballs.json
nix-instantiate --eval --json --strict "$TMP_NIXPKGS/maintainers/scripts/find-tarballs.nix" --arg expr "import \"$TMP_NIXPKGS/maintainers/scripts/all-tarballs.nix\"" > $TMP_CACHE_TARBALLS_FILE
cat $TMP_CACHE_TARBALLS_FILE | jq -r '.[] | (if (.type | length) > 0 then .type else (.hash | split("(-|:)";""))[0] end) + " " + .hash + " " + .name' | sort | uniq | "$SCRIPTPATH/get-fixed-output-pathv2.pl" > $TMP_TARBALLS_STORE_PATHS_FILE

# nix copy them
# cat store-paths | xargs -I{} bash -c 'nix copy --from https://cache.nixos.org --to "$BINARY_CACHE" --derivation "{}^*" | true'
echo "Copying stage 1"
cat $TMP_RELEASE_STORE_PATHS_FILE | xargs -t -I{} nix copy --from https://cache.nixos.org --to "$BINARY_CACHE" --derivation '{}^*'
echo "Copying stage 2"
cat $TMP_RELEASE_STORE_PATHS_FILE | xargs -t -I{} nix copy --from https://cache.nixos.org --to "$BINARY_CACHE" '{}'
echo "Copying stage 3"
cat $TMP_RELEASE_STORE_PATHS_DRV_FILE | xargs -t -I{} nix copy --to "$BINARY_CACHE" '{}'
# cat $TMP_RELEASE_STORE_PATHS_DRV_FILE | xargs -I{} nix copy --from https://cache.nixos.org --to "$BINARY_CACHE" --derivation '{}^*'
echo "Copying stage 4"
cat $TMP_TARBALLS_STORE_PATHS_FILE | xargs -t -I{} nix copy --to "$BINARY_CACHE" '{}'