Repro script for neondatabase/neon#365

test_compute_restart.sh

#!/usr/bin/env sh

export PATH=$(pwd)/target/debug/:$(pwd)/tmp_install/bin:$PATH

# Remove all traces of the previous run
killall pageserver
killall wal_acceptor
killall postgres

rm -rf zenith_tmp

# Create directories structure
mkdir zenith_tmp
mkdir zenith_tmp/page
mkdir zenith_tmp/safe1
mkdir zenith_tmp/safe2
mkdir zenith_tmp/safe3

# Init pageserver
pageserver --init -D $(pwd)/zenith_tmp/page --postgres-distrib $(pwd)/tmp_install

# Run all pageservers and safekeepers
RUST_LOG=debug RUST_BACKTRACE=full pageserver -l 0.0.0.0:6400 -D $(pwd)/zenith_tmp/page >> zenith_tmp/zenith.log 2>&1 &
RUST_LOG=debug RUST_BACKTRACE=full wal_acceptor --no-sync --recall '1 second' -l 127.0.0.1:6501 -p 127.0.0.1:6400 -D $(pwd)/zenith_tmp/safe1 >> zenith_tmp/zenith.log 2>&1 &
RUST_LOG=debug RUST_BACKTRACE=full wal_acceptor --no-sync --recall '1 second' -l 127.0.0.1:6502 -p 127.0.0.1:6400 -D $(pwd)/zenith_tmp/safe2 >> zenith_tmp/zenith.log 2>&1 &
RUST_LOG=debug RUST_BACKTRACE=full wal_acceptor --no-sync --recall '1 second' -l 127.0.0.1:6503 -p 127.0.0.1:6400 -D $(pwd)/zenith_tmp/safe3 >> zenith_tmp/zenith.log 2>&1 &

# Wait until everything is ready
sleep 1

# Create tenant
TENANT_ID=c5a71cf0722a92664b3b829bed1ee5a1
psql -p6400 -hlocalhost -Xqtc "tenant_create ${TENANT_ID}"

# Create branch
psql -p6400 -hlocalhost -Xqtc "branch_create ${TENANT_ID} br main"

# Get default timeline from the pageserver stdout
# MAIN_TIMELINE=$(cat zenith_tmp/zenith.log | grep 'created initial timeline' | awk -F ' ' '{print $4}')

# Get new branch timelineid
TIMELINE=$(cat zenith_tmp/page/tenants/${TENANT_ID}/refs/branches/br)

# Remove compute datadir, initialize it using basebackup and start
init_compute () {
    rm -rf zenith_tmp/compute
    mkdir zenith_tmp/compute
    sh -c "cd zenith_tmp/compute && psql -p6400 -hlocalhost -Xqtc 'basebackup ${TENANT_ID} ${TIMELINE}' | tar -x"
    chmod -R 750 zenith_tmp/compute

    echo "
listen_addresses='0.0.0.0'
password_encryption=md5
max_connections=100
max_wal_senders=10
max_replication_slots=10
hot_standby=on
wal_log_hints=on
wal_level=replica
wal_sender_timeout=0
wal_keep_size=256MB
shared_buffers=256MB
fsync=off
port=54325
shared_preload_libraries='zenith'
synchronous_standby_names='walproposer'
wal_acceptors='127.0.0.1:6501,127.0.0.1:6502,127.0.0.1:6503'
zenith.zenith_tenant='${TENANT_ID}'
zenith.zenith_timeline='${TIMELINE}'
zenith.page_server_connstring='host=127.0.0.1 port=6400'" >> zenith_tmp/compute/postgresql.conf

    pg_ctl -D zenith_tmp/compute -l zenith_tmp/pg.log start
}

init_compute
# Make some payload and check the result
psql -p54325 -hlocalhost -Uzenith_admin -c 'create role zenith_user with login' postgres
psql -p54325 -hlocalhost -Uzenith_admin -c 'create database db1 owner zenith_user' postgres
pgbench -p54325 -hlocalhost -Uzenith_user -i -s1 -n db1
psql -p54325 -hlocalhost -Uzenith_user -c 'select count(*) from pgbench_accounts' db1

# Try 'immediate' first
pg_ctl -D zenith_tmp/compute -m immediate stop

init_compute
psql -p54325 -hlocalhost -Uzenith_user -c 'select count(*) from pgbench_accounts' db1

# Try 'fast'
pg_ctl -D zenith_tmp/compute -m fast stop

init_compute
# This hangs forever
psql -p54325 -hlocalhost -Uzenith_user -c 'select count(*) from pgbench_accounts' db1