magic.sh

#!/bin/bash

# URL prefix
url_prefix='http://testing.mainnet.beacon-api.nimbus.team/eth/v2/beacon/blocks'

# Range of numbers to loop over
start=6209538
end=9453725

# Number of workers
num_workers=32

# Function to fetch JSON files
fetch_json() {
    local i=$1
    local url="${url_prefix}/${i}"
    local output_file="blocks/${i}.json"
    if [[ -f $output_file ]]; then
        echo "$output_file : already exists"
    else
        curl -s -o "$output_file" "$url"
        echo "$output_file : fetched from $url"
    fi
}

# Export the function so it can be used by parallel
export -f fetch_json
export url_prefix

# Loop over the range of numbers
for i in $(seq $start $end); do
    # Start a background job for each worker
    ((worker_num=i%num_workers))
    fetch_json $i &
    
    # Wait for all background jobs to finish every num_workers iterations
    if ((worker_num == num_workers-1)); then
        wait
    fi
done

# Wait for any remaining background jobs to finish
wait

echo "All files fetched."