Retrieval Test

retrieval

#!/usr/bin/env bash

set -eo pipefail

DIR="$(dirname "${BASH_SOURCE[0]}")"

source "$DIR/lib/helpers.bash"

export GO_FILECOIN_LOG_LEVEL=3
export FILECOIN_PROOFS_FAST_DELAY_SECONDS=1

if [ -z "$1" ]; then
    FIXTURES_PATH="./fixtures/test"
    COMMIT_SECTOR_AND_POST_TIMEOUT=120
else
    FIXTURES_PATH="./fixtures/live"
    COMMIT_SECTOR_AND_POST_TIMEOUT=3600
fi

go run ./build/*.go build

if [ -z "$2" ]; then
    AUTO_SEAL_INTERVAL_SECONDS="0"
else
    AUTO_SEAL_INTERVAL_SECONDS="$2"
fi

# forward-declare stuff that we need to clean up
STORAGE_MN_PID=""
BOOTSTRAP_MN_PID=""
CL_PID=""
STORAGE_MN_REPO_DIR=""
BOOTSTRAP_MN_REPO_DIR=""
CL_REPO_DIR=""
PIECE_1_PATH=$(mktemp)
PIECE_2_PATH=$(mktemp)
UNSEAL_PATH=$(mktemp)
BLOCK_TIME="5s"
HODL="HODL HODL HODL HODL HODL HODL HODL HODL HODL HODL HODL HODL HODL HODL HODL"

if [ "${FIXTURES_PATH}" = "./fixtures/test" ] ; then
    dd if=/dev/urandom of="${PIECE_2_PATH}" bs=1 count=700
    dd if=/dev/urandom of="${PIECE_1_PATH}" bs=1 count=500
else
    # Maximum number of user piece-bytes in a Live-configuration sector is
    # equal to 266338304. Our first piece will not fill up the whole sector.
    dd if=/dev/urandom of="${PIECE_1_PATH}" bs=$((1024*1024)) count=180
    # Adding this second piece will cause the sector sealing process to run.
    dd if=/dev/urandom of="${PIECE_2_PATH}" bs=$((1024*1024)) count=180
fi

trap finish EXIT

STORAGE_MN_REPO_DIR=$(mktemp -d)
STORAGE_MN_CMDAPI_PORT=$(free_port)
STORAGE_MN_SWARM_PORT=$(free_port)

BOOTSTRAP_MN_REPO_DIR=$(mktemp -d)
BOOTSTRAP_MN_CMDAPI_PORT=$(free_port)
BOOTSTRAP_MN_SWARM_PORT=$(free_port)

CL_REPO_DIR=$(mktemp -d)
CL_CMDAPI_PORT=$(free_port)
CL_SWARM_PORT=$(free_port)

echo ""
echo "generating private keys..."
BOOTSTRAP_MN_MINER_FIL_ADDR=$(jq -r '.Miners[] | select(.Owner == 0).Address' < "${FIXTURES_PATH}/gen.json")

echo ""
echo "initializing daemons..."
init_local_daemon "${BOOTSTRAP_MN_REPO_DIR}" "${BOOTSTRAP_MN_CMDAPI_PORT}" "${FIXTURES_PATH}/genesis.car"
init_local_daemon "${STORAGE_MN_REPO_DIR}" "${STORAGE_MN_CMDAPI_PORT}" "${FIXTURES_PATH}/genesis.car"
init_local_daemon "${CL_REPO_DIR}" "${CL_CMDAPI_PORT}" "${FIXTURES_PATH}/genesis.car"

echo ""
echo "start daemons..."
start_daemon "${STORAGE_MN_REPO_DIR}" "${STORAGE_MN_CMDAPI_PORT}" "${STORAGE_MN_SWARM_PORT}"
STORAGE_MN_PID=$!
start_daemon "${BOOTSTRAP_MN_REPO_DIR}" "${BOOTSTRAP_MN_CMDAPI_PORT}" "${BOOTSTRAP_MN_SWARM_PORT}"
BOOTSTRAP_MN_PID=$!
start_daemon "${CL_REPO_DIR}" "${CL_CMDAPI_PORT}" "${CL_SWARM_PORT}"
CL_PID=$!

sleep 2

echo ""
echo "client imports pieces..."
PIECE_1_CID=$(./go-filecoin client import --repodir="${CL_REPO_DIR}" < "${PIECE_1_PATH}")
PIECE_2_CID=$(./go-filecoin client import --repodir="${CL_REPO_DIR}" < "${PIECE_2_PATH}")

echo ""
echo "importing private keys..."
BOOTSTRAP_MN_MINER_OWNER_FIL_ADDR=$(import_private_key 0 "${BOOTSTRAP_MN_REPO_DIR}")
CL_FIL_ADDRESS=$(import_private_key 1 "${CL_REPO_DIR}")
STORAGE_MN_MINER_OWNER_FIL_ADDR=$(import_private_key 2 "${STORAGE_MN_REPO_DIR}")

echo ""
echo "ensure that miner address is set so that the bootstrap miner-node can mine..."
set_mining_address_in_config "${BOOTSTRAP_MN_MINER_FIL_ADDR}" "${BOOTSTRAP_MN_REPO_DIR}"

echo ""
echo "node default address should match what's associated with imported SK..."
set_wallet_default_address_in_config "${CL_FIL_ADDRESS}" "${CL_REPO_DIR}"
set_wallet_default_address_in_config "${BOOTSTRAP_MN_MINER_OWNER_FIL_ADDR}" "${BOOTSTRAP_MN_REPO_DIR}"
set_wallet_default_address_in_config "${STORAGE_MN_MINER_OWNER_FIL_ADDR}" "${STORAGE_MN_REPO_DIR}"

echo ""
echo "get storage mining node's libp2p identity..."
STORAGE_MN_PEER_ID=$(get_peer_id "${STORAGE_MN_REPO_DIR}")

echo ""
echo "connecting daemons..."
swarm_connect "$(get_first_address "${CL_REPO_DIR}")" "${BOOTSTRAP_MN_REPO_DIR}"
swarm_connect "$(get_first_address "${BOOTSTRAP_MN_REPO_DIR}")" "${STORAGE_MN_REPO_DIR}"

echo ""
echo ""
echo ""
echo "********************** BEGIN STORAGE PROTOCOL"
echo ""
echo ""
echo ""

echo ""
echo "bootstrap miner node starts mining (so that messages can be processed)..."
./go-filecoin mining start \
  --repodir="$BOOTSTRAP_MN_REPO_DIR" \

echo ""
echo "bootstrap miner shares some funds with the storage miner..."
SEND_FIL_MSG_CID=$(send_fil "$BOOTSTRAP_MN_MINER_OWNER_FIL_ADDR" 100 "${STORAGE_MN_MINER_OWNER_FIL_ADDR}" "${BOOTSTRAP_MN_REPO_DIR}")

echo ""
echo "block until FIL-transferring messages are in blockchain..."
message_wait "${SEND_FIL_MSG_CID}" "${BOOTSTRAP_MN_REPO_DIR}"
message_wait "${SEND_FIL_MSG_CID}" "${CL_REPO_DIR}"
message_wait "${SEND_FIL_MSG_CID}" "${STORAGE_MN_REPO_DIR}"

echo ""
echo "create a new miner actor (for storage miner)..."
STORAGE_MN_MINER_FIL_ADDR=$(create_miner "${STORAGE_MN_REPO_DIR}")

echo ""
echo "storage miner node starts mining (so that it processes storage proposals)..."
./go-filecoin mining start \
  --repodir="$STORAGE_MN_REPO_DIR" \

echo ""
echo "update miner's libp2p identity to match its node's..."
STORAGE_MN_MINER_UPDATE_PID_MSG_CID=$(miner_update_pid "${STORAGE_MN_MINER_FIL_ADDR}" "${STORAGE_MN_PEER_ID}" "${STORAGE_MN_REPO_DIR}")

echo ""
echo "storage miner adds its ask to the market..."
STORAGE_MN_MINER_SET_PRICE_MSG_CID=$(set_price 10 10000 "${STORAGE_MN_REPO_DIR}")

echo ""
echo "block until miner peer id-update and set price-messages appear in chains..."
message_wait "${STORAGE_MN_MINER_UPDATE_PID_MSG_CID}" "${BOOTSTRAP_MN_REPO_DIR}"
message_wait "${STORAGE_MN_MINER_UPDATE_PID_MSG_CID}" "${CL_REPO_DIR}"
message_wait "${STORAGE_MN_MINER_UPDATE_PID_MSG_CID}" "${STORAGE_MN_REPO_DIR}"
message_wait "${STORAGE_MN_MINER_SET_PRICE_MSG_CID}" "${BOOTSTRAP_MN_REPO_DIR}"
message_wait "${STORAGE_MN_MINER_SET_PRICE_MSG_CID}" "${CL_REPO_DIR}"
message_wait "${STORAGE_MN_MINER_SET_PRICE_MSG_CID}" "${STORAGE_MN_REPO_DIR}"

echo ""
echo "client proposes a storage deal, which transfers file 1..."
./go-filecoin client propose-storage-deal "${STORAGE_MN_MINER_FIL_ADDR}" "${PIECE_1_CID}" 0 5 \
  --repodir="$CL_REPO_DIR" \

echo ""
echo "client proposes a storage deal, which transfers piece 2..."
./go-filecoin client propose-storage-deal "${STORAGE_MN_MINER_FIL_ADDR}" "${PIECE_2_CID}" 0 5 \
  --repodir="$CL_REPO_DIR" \

echo ""
echo "wait for commitSector sent by miner owner to be included in a block viewable by all nodes..."
wait_for_message_in_chain_by_method_and_sender commitSector "${STORAGE_MN_MINER_OWNER_FIL_ADDR}" "${CL_REPO_DIR}" "${COMMIT_SECTOR_AND_POST_TIMEOUT}"
wait_for_message_in_chain_by_method_and_sender commitSector "${STORAGE_MN_MINER_OWNER_FIL_ADDR}" "${BOOTSTRAP_MN_REPO_DIR}" "${COMMIT_SECTOR_AND_POST_TIMEOUT}"
wait_for_message_in_chain_by_method_and_sender commitSector "${STORAGE_MN_MINER_OWNER_FIL_ADDR}" "${STORAGE_MN_REPO_DIR}" "${COMMIT_SECTOR_AND_POST_TIMEOUT}"

echo ""
echo "wait for submitPoSt, too..."
wait_for_message_in_chain_by_method_and_sender submitPoSt "${STORAGE_MN_MINER_OWNER_FIL_ADDR}" "${CL_REPO_DIR}" "${COMMIT_SECTOR_AND_POST_TIMEOUT}"
wait_for_message_in_chain_by_method_and_sender submitPoSt "${STORAGE_MN_MINER_OWNER_FIL_ADDR}" "${BOOTSTRAP_MN_REPO_DIR}" "${COMMIT_SECTOR_AND_POST_TIMEOUT}"
wait_for_message_in_chain_by_method_and_sender submitPoSt "${STORAGE_MN_MINER_OWNER_FIL_ADDR}" "${STORAGE_MN_REPO_DIR}" "${COMMIT_SECTOR_AND_POST_TIMEOUT}"

echo ""
echo ""
echo ""
echo "********************** BEGIN RETRIEVAL PROTOCOL"
echo ""
echo ""
echo ""

./go-filecoin retrieval-client retrieve-piece  "${STORAGE_MN_MINER_FIL_ADDR}" "${PIECE_1_CID}" \
  --repodir="${CL_REPO_DIR}" > "${UNSEAL_PATH}"

GOT=$(shasum < "${UNSEAL_PATH}")
EXPECTED=$(shasum < "${PIECE_1_PATH}")

if [ "${GOT}" = "${EXPECTED}" ]; then
    echo "Round trip passed!"
    exit 0
else
    echo "Round trip Failed!, expected file"
    echo "${UNSEAL_PATH}"
    echo "to have same contents as file"
    echo "${PIECE_1_PATH}"
    exit 1
fi