Skip to content

Instantly share code, notes, and snippets.

@mohamedawnallah
Last active November 8, 2025 16:17
Show Gist options
  • Select an option

  • Save mohamedawnallah/7f77cfe3e34a729c549e4d7c9627ce87 to your computer and use it in GitHub Desktop.

Select an option

Save mohamedawnallah/7f77cfe3e34a729c549e4d7c9627ce87 to your computer and use it in GitHub Desktop.
Reproducing LND blocks rescan issue when wallet created during headers sync
#!/bin/bash
# LND Benchmark Script with Linux Namespaces and Temporary Storage
# Runs LND in isolated network, PID, and mount namespaces with ephemeral tmpfs storage
# Configuration
VERSION="${1:-$(git describe --tags --always --dirty 2>/dev/null || echo 'unknown')}"
ALIAS="${2:-default}"
LND_BIN="$(pwd)/lnd-debug-${VERSION}"
LNCLI_BIN="$(pwd)/lncli-debug-${VERSION}"
POLL_INTERVAL=5
RESULTS_FILE="benchmark_${VERSION}_${ALIAS}.json"
# Resource limits
CPU_LIMIT="${CPU_LIMIT:-4}" # Number of CPU cores (e.g., 4 = 400% CPU)
MEMORY_LIMIT="${MEMORY_LIMIT:-8G}" # Memory limit (e.g., 8G, 4096M)
# Check if running as root (required for namespaces)
if [ "$EUID" -ne 0 ]; then
echo "This script must be run as root for namespace isolation"
echo "Usage: sudo $0 [version] [alias]"
echo " version: LND version (default: v0.20.0.latest.experimental)"
echo " alias: Instance alias for identification (default: default)"
exit 1
fi
# Create a temporary directory for this run (will be bind-mounted in namespace)
TEMP_DIR=$(mktemp -d -t "lnd-bench-${VERSION}-${ALIAS}-XXXXXX")
LND_DIR="${TEMP_DIR}/lnd"
mkdir -p "${LND_DIR}"
# Namespace mount point with alias
NS_MOUNT_POINT="/tmp/lnd_data_ns_${ALIAS}"
# Set permissions so original user can access (get the actual user if running via sudo)
ACTUAL_USER="${SUDO_USER:-$USER}"
if [ ! -z "$ACTUAL_USER" ] && [ "$ACTUAL_USER" != "root" ]; then
chown -R "$ACTUAL_USER:$ACTUAL_USER" "$TEMP_DIR"
fi
# Cleanup function
cleanup() {
if [ ! -z "$LND_PID" ] && kill -0 $LND_PID 2>/dev/null; then
echo "Cleaning up LND process..."
kill -TERM $LND_PID 2>/dev/null || true
wait $LND_PID 2>/dev/null || true
fi
# Cleanup cgroup if it exists
if [ -d "${CGROUP_PATH}" ]; then
rmdir "${CGROUP_PATH}" 2>/dev/null || true
fi
# Cleanup namespace mount point
if mountpoint -q ${NS_MOUNT_POINT} 2>/dev/null; then
echo "Unmounting ${NS_MOUNT_POINT}..."
umount ${NS_MOUNT_POINT} 2>/dev/null || true
fi
if [ -d "${NS_MOUNT_POINT}" ]; then
echo "Removing ${NS_MOUNT_POINT}..."
rm -rf ${NS_MOUNT_POINT}
fi
# Remove temporary directory
if [ -d "${TEMP_DIR}" ]; then
echo "Cleaning up temporary directory: ${TEMP_DIR}"
# Fix permissions before removal so user can inspect if needed
if [ ! -z "$ACTUAL_USER" ] && [ "$ACTUAL_USER" != "root" ]; then
chown -R "$ACTUAL_USER:$ACTUAL_USER" "$TEMP_DIR" 2>/dev/null || true
fi
rm -rf "${TEMP_DIR}"
fi
}
trap cleanup EXIT INT TERM
# Clean up any leftover state from previous runs
echo "Checking for leftover state from previous runs..."
if mountpoint -q ${NS_MOUNT_POINT} 2>/dev/null; then
echo "Unmounting stale ${NS_MOUNT_POINT}..."
umount ${NS_MOUNT_POINT} 2>/dev/null || true
fi
if [ -d "${NS_MOUNT_POINT}" ]; then
echo "Removing stale ${NS_MOUNT_POINT}..."
rm -rf ${NS_MOUNT_POINT}
fi
echo "=========================================="
echo "Benchmarking LND ${VERSION} [${ALIAS}]"
echo "Using temporary storage: ${TEMP_DIR}"
echo "Namespace mount: ${NS_MOUNT_POINT}"
echo "Resource Limits:"
echo " - CPU: ${CPU_LIMIT} cores"
echo " - Memory: ${MEMORY_LIMIT}"
echo "=========================================="
# Create cgroup for resource isolation and monitoring
CGROUP_PATH="/sys/fs/cgroup/lnd_bench_${VERSION}_${ALIAS}"
mkdir -p "$CGROUP_PATH"
# Set CPU limit (number of cores * 100000)
# Example: 4 cores = 400000 (400% CPU)
CPU_QUOTA=$((${CPU_LIMIT%.*} * 100000))
echo $CPU_QUOTA > "${CGROUP_PATH}/cpu.max" 2>/dev/null || echo "Warning: Could not set CPU limit (cgroup v1?)"
# Set memory limit
echo "$MEMORY_LIMIT" > "${CGROUP_PATH}/memory.max" 2>/dev/null || echo "Warning: Could not set memory limit (cgroup v1?)"
# Enable memory high watermark (soft limit at 90% of max)
MEMORY_HIGH=$(echo "$MEMORY_LIMIT" | sed 's/G/*1024*1024*1024*0.9/' | sed 's/M/*1024*1024*0.9/' | bc | cut -d. -f1)
echo "$MEMORY_HIGH" > "${CGROUP_PATH}/memory.high" 2>/dev/null || true
# Generate random ports to allow multiple instances
RPC_PORT=$((10000 + RANDOM % 50000))
REST_PORT=$((10000 + RANDOM % 50000))
# LND startup arguments
LND_ARGS=(
--lnddir="${LND_DIR}"
--alias="${ALIAS}"
--debuglevel="debug"
--noseedbackup
--nolisten
--rpclisten="127.0.0.1:${RPC_PORT}"
--restlisten="127.0.0.1:${REST_PORT}"
--bitcoin.active
--bitcoin.mainnet
--bitcoin.node=neutrino
--feeurl="https://nodes.lightning.computer/fees/v1/btc-fee-estimates.json"
--routing.assumechanvalid
--tlsdisableautofill
--neutrino.addpeer=asia.blixtwallet.com
--neutrino.addpeer=europe.blixtwallet.com
--neutrino.addpeer=btcd.lnolymp.us
--neutrino.addpeer=btcd-mainnet.lightning.computer
)
# Start time
START_TIME=$(date +%s.%N)
# Build complete command string with all arguments for the namespace
LND_ARGS_STR=""
for arg in "${LND_ARGS[@]}"; do
# Skip the lnddir argument since we'll override it
if [[ ! "$arg" =~ ^--lnddir= ]]; then
LND_ARGS_STR="${LND_ARGS_STR} ${arg}"
fi
done
echo "Starting LND in isolated namespace with tmpfs..."
# Run LND in isolated namespaces using unshare
# - PID namespace: isolated process tree
# - Mount namespace: isolated mount points with tmpfs for data
# - UTS namespace: isolated hostname
# - IPC namespace: isolated IPC
# Network namespace is shared so neutrino can connect to peers
unshare --fork --pid --mount --uts --ipc \
--mount-proc \
bash -c "
# Move process to cgroup
echo \$\$ > ${CGROUP_PATH}/cgroup.procs
# Create tmpfs mount for LND data (optional: limit size with -o size=2G)
mkdir -p ${NS_MOUNT_POINT}
mount -t tmpfs -o size=4G tmpfs ${NS_MOUNT_POINT}
# Copy the empty LND directory structure to tmpfs
cp -a ${LND_DIR} ${NS_MOUNT_POINT}/lnd
# Execute LND with data on tmpfs
exec ${LND_BIN} --lnddir=${NS_MOUNT_POINT}/lnd ${LND_ARGS_STR} > ${TEMP_DIR}/lnd.log 2>&1
" &
LND_PID=$!
# Give LND a moment to start
sleep 3
echo "Started LND [${ALIAS}] (PID: ${LND_PID})"
echo "RPC Port: ${RPC_PORT}, REST Port: ${REST_PORT}"
echo "Monitoring sync status..."
echo ""
echo "To manually check status, run:"
echo " sudo nsenter --target ${LND_PID} --pid --mount --uts --ipc \\"
echo " ${LNCLI_BIN} --lnddir=${NS_MOUNT_POINT}/lnd --rpcserver=127.0.0.1:${RPC_PORT} getinfo"
echo ""
# Arrays for metrics
declare -a MEMORY_SAMPLES=()
declare -a CPU_SAMPLES=()
MAX_MEMORY=0
PREV_CPU_TIME=0
PREV_TIMESTAMP=0
# Function to get cgroup memory stats
get_cgroup_memory() {
if [ -f "${CGROUP_PATH}/memory.current" ]; then
cat "${CGROUP_PATH}/memory.current"
else
# Fallback to process RSS
cat /proc/${LND_PID}/status | awk '/VmRSS/ {print $2*1024}'
fi
}
# Function to get cgroup CPU usage
get_cgroup_cpu() {
if [ -f "${CGROUP_PATH}/cpu.stat" ]; then
awk '/usage_usec/ {print $2}' "${CGROUP_PATH}/cpu.stat"
else
echo "0"
fi
}
# Function to get getinfo output
get_lnd_info() {
nsenter --target ${LND_PID} --pid --mount --uts --ipc \
${LNCLI_BIN} --lnddir=${NS_MOUNT_POINT}/lnd --rpcserver=127.0.0.1:${RPC_PORT} getinfo 2>/dev/null
}
# Function to check sync to chain status
check_sync_to_chain_status() {
local info_output="$1"
if [ -z "$info_output" ]; then
SYNCED_TO_CHAIN=false
return 0
fi
# Get block height for monitoring output
local block_height=$(echo "$info_output" | grep -o '"block_height"[[:space:]]*:[[:space:]]*[0-9]*' | grep -o '[0-9]*')
CURRENT_BLOCK_HEIGHT=$block_height
# Check if synced_to_chain is true
local synced=$(echo "$info_output" | grep -o '"synced_to_chain"[[:space:]]*:[[:space:]]*true')
if [ ! -z "$synced" ]; then
SYNCED_TO_CHAIN=true
else
SYNCED_TO_CHAIN=false
fi
return 0
}
# Function to check sync to graph status
check_sync_to_graph_status() {
local info_output="$1"
if [ -z "$info_output" ]; then
SYNCED_TO_GRAPH=false
return 0
fi
# Get synced_to_graph status
local synced_graph=$(echo "$info_output" | grep -o '"synced_to_graph"[[:space:]]*:[[:space:]]*true')
if [ ! -z "$synced_graph" ]; then
SYNCED_TO_GRAPH=true
else
SYNCED_TO_GRAPH=false
fi
return 0
}
# Monitoring loop
SAMPLE_COUNT=0
while true; do
# Check if LND is still running
if ! kill -0 ${LND_PID} 2>/dev/null; then
echo "LND process died unexpectedly"
cat ${TEMP_DIR}/lnd.log
exit 1
fi
# Get current metrics
CURRENT_MEM=$(get_cgroup_memory)
CURRENT_CPU=$(get_cgroup_cpu)
CURRENT_TIME=$(date +%s.%N)
# Calculate memory stats
MEMORY_SAMPLES+=($CURRENT_MEM)
if [ $CURRENT_MEM -gt $MAX_MEMORY ]; then
MAX_MEMORY=$CURRENT_MEM
fi
# Calculate CPU percentage
if [ $SAMPLE_COUNT -gt 0 ]; then
CPU_DELTA=$((CURRENT_CPU - PREV_CPU_TIME))
TIME_DELTA=$(echo "$CURRENT_TIME - $PREV_TIMESTAMP" | bc)
CPU_PERCENT=$(echo "scale=2; ($CPU_DELTA / ($TIME_DELTA * 1000000)) * 100" | bc)
CPU_SAMPLES+=($CPU_PERCENT)
fi
PREV_CPU_TIME=$CURRENT_CPU
PREV_TIMESTAMP=$CURRENT_TIME
# Get LND info
INFO_OUTPUT=$(get_lnd_info)
# Check RPC status
if [ -z "$INFO_OUTPUT" ]; then
RPC_STATUS="down"
else
RPC_STATUS="up"
fi
# Check sync statuses
check_sync_to_chain_status "$INFO_OUTPUT"
check_sync_to_graph_status "$INFO_OUTPUT"
# Display status
MEM_MB=$(echo "scale=2; $CURRENT_MEM/1048576" | bc)
echo "[Sample $SAMPLE_COUNT] RPC: ${RPC_STATUS} | Block: ${CURRENT_BLOCK_HEIGHT:-unknown} | synced_to_chain: ${SYNCED_TO_CHAIN} | synced_to_graph: ${SYNCED_TO_GRAPH:-false} | Memory: ${MEM_MB} MB | CPU: ${CPU_PERCENT:-0}%"
SAMPLE_COUNT=$((SAMPLE_COUNT + 1))
sleep ${POLL_INTERVAL}
done
# Calculate execution time
EXECUTION_TIME=$(echo "$END_TIME - $START_TIME" | bc)
# Stopping LND.
echo "Stopping LND..."
# First, try to find and kill the actual lnd process inside the namespace
if [ ! -z "${LND_PID}" ] && kill -0 ${LND_PID} 2>/dev/null; then
# Kill all processes in the namespace (this kills lnd inside)
pkill -9 -P ${LND_PID} 2>/dev/null || true
# Then kill the unshare process itself
kill -KILL ${LND_PID} 2>/dev/null || true
# Wait for cleanup
wait ${LND_PID} 2>/dev/null || true
sleep 2
fi
# Double-check: kill any remaining lnd-debug processes for this version
pkill -9 -f "lnd-debug-${VERSION}" 2>/dev/null || true
# Ensure the tmpfs is unmounted
umount ${NS_MOUNT_POINT} 2>/dev/null || true
# Calculate statistics
TOTAL_MEM=0
for mem in "${MEMORY_SAMPLES[@]}"; do
TOTAL_MEM=$(echo "$TOTAL_MEM + $mem" | bc)
done
AVG_MEMORY=$(echo "$TOTAL_MEM / ${#MEMORY_SAMPLES[@]}" | bc)
TOTAL_CPU=0
for cpu in "${CPU_SAMPLES[@]}"; do
TOTAL_CPU=$(echo "$TOTAL_CPU + $cpu" | bc)
done
if [ ${#CPU_SAMPLES[@]} -gt 0 ]; then
AVG_CPU=$(echo "scale=2; $TOTAL_CPU / ${#CPU_SAMPLES[@]}" | bc)
else
AVG_CPU=0
fi
# Get final cgroup stats if available
if [ -f "${CGROUP_PATH}/memory.peak" ]; then
PEAK_MEMORY=$(cat "${CGROUP_PATH}/memory.peak")
else
PEAK_MEMORY=$MAX_MEMORY
fi
# Convert to MB
PEAK_MEMORY_MB=$(echo "scale=2; $PEAK_MEMORY/1048576" | bc)
AVG_MEMORY_MB=$(echo "scale=2; $AVG_MEMORY/1048576" | bc)
# Print results
echo ""
echo "========================================"
echo "BENCHMARK RESULTS - ${VERSION} [${ALIAS}]"
echo "========================================"
echo "Execution Time: ${EXECUTION_TIME} seconds"
echo "Peak Memory Usage: ${PEAK_MEMORY_MB} MB"
echo "Average Memory Usage: ${AVG_MEMORY_MB} MB"
echo "Average CPU Usage: ${AVG_CPU}%"
echo "Samples Collected: ${#MEMORY_SAMPLES[@]}"
echo ""
# Save results as JSON
cat > "$RESULTS_FILE" << EOF
{
"version": "${VERSION}",
"alias": "${ALIAS}",
"execution_time_seconds": ${EXECUTION_TIME},
"peak_memory_bytes": ${PEAK_MEMORY},
"peak_memory_mb": ${PEAK_MEMORY_MB},
"average_memory_bytes": ${AVG_MEMORY},
"average_memory_mb": ${AVG_MEMORY_MB},
"average_cpu_percent": ${AVG_CPU},
"samples_collected": ${#MEMORY_SAMPLES[@]},
"poll_interval_seconds": ${POLL_INTERVAL},
"storage_type": "tmpfs"
}
EOF
echo "Results saved to: ${RESULTS_FILE}"
echo "Temporary directory will be cleaned up automatically"
# Cleanup cgroup
rmdir "$CGROUP_PATH" 2>/dev/null || true
{
"version": "v0.18.3-beta",
"execution_time_seconds": 70.434696628,
"peak_memory_bytes": 458911744,
"peak_memory_mb": 437.65,
"average_memory_bytes": 243024164,
"average_memory_mb": 231.76,
"average_cpu_percent": 157.76,
"samples_collected": 14,
"poll_interval_seconds": 5,
"storage_type": "tmpfs"
}
{
"version": "v0.18.4-beta",
"execution_time_seconds": 70.358434022,
"peak_memory_bytes": 476459008,
"peak_memory_mb": 454.38,
"average_memory_bytes": 245991131,
"average_memory_mb": 234.59,
"average_cpu_percent": 157.61,
"samples_collected": 14,
"poll_interval_seconds": 5,
"storage_type": "tmpfs"
}
{
"version": "v0.18.5-beta",
"execution_time_seconds": 65.119188935,
"peak_memory_bytes": 498163712,
"peak_memory_mb": 475.08,
"average_memory_bytes": 241432418,
"average_memory_mb": 230.24,
"average_cpu_percent": 162.00,
"samples_collected": 13,
"poll_interval_seconds": 5,
"storage_type": "tmpfs"
}
{
"version": "v0.20.0-beta.rc1-after",
"execution_time_seconds": 70.416534861,
"peak_memory_bytes": 513572864,
"peak_memory_mb": 489.78,
"average_memory_bytes": 323389440,
"average_memory_mb": 308.40,
"average_cpu_percent": 208.15,
"samples_collected": 14,
"poll_interval_seconds": 5,
"storage_type": "tmpfs"
}
dev@dev:~/lnd$ sudo ./compare_lnd_versions.sh
==========================================
LND Version Comparison Benchmark
==========================================
Comparing 4 versions:
- v0.18.3-beta
- v0.18.4-beta
- v0.18.5-beta
- v0.20.0-beta.rc1-after
Each version will run in isolated namespaces
for fair comparison.
>>> Running v0.18.3-beta (1/4)...
==========================================
Benchmarking LND v0.18.3-beta
Using temporary storage: /tmp/lnd-bench-v0.18.3-beta-tQ9SWS
Resource Limits:
- CPU: 4 cores
- Memory: 8G
==========================================
Starting LND in isolated namespace with tmpfs...
Started LND (PID: 851327)
Monitoring sync status...
To manually check status, run:
sudo nsenter --target 851327 --pid --mount --uts --ipc \
/home/dev/lnd/lncli-debug-v0.18.3-beta --lnddir=/tmp/lnd_data_ns/lnd getinfo
[Sample 0] Block: unknown | synced: false | Memory: 348.76 MB | CPU: 0%
[Sample 1] Block: unknown | synced: false | Memory: 437.65 MB | CPU: 223.00%
[Sample 2] Block: 150000 | synced: false | Memory: 119.67 MB | CPU: 190.00%
[Sample 3] Block: 243000 | synced: false | Memory: 131.54 MB | CPU: 158.00%
[Sample 4] Block: 337000 | synced: false | Memory: 150.61 MB | CPU: 171.00%
[Sample 5] Block: 412000 | synced: false | Memory: 160.77 MB | CPU: 151.00%
[Sample 6] Block: 464000 | synced: false | Memory: 174.42 MB | CPU: 143.00%
[Sample 7] Block: 536000 | synced: false | Memory: 193.35 MB | CPU: 157.00%
[Sample 8] Block: 635000 | synced: false | Memory: 207.39 MB | CPU: 167.00%
[Sample 9] Block: 700000 | synced: false | Memory: 217.50 MB | CPU: 153.00%
[Sample 10] Block: 764000 | synced: false | Memory: 242.05 MB | CPU: 154.00%
[Sample 11] Block: 822000 | synced: false | Memory: 253.53 MB | CPU: 152.00%
[Sample 12] Block: 914000 | synced: false | Memory: 278.78 MB | CPU: 144.00%
synced_to_chain
✓ synced_to_chain: true (block height: 918649)
Stopping LND...
========================================
BENCHMARK RESULTS - v0.18.3-beta
========================================
Execution Time: 70.434696628 seconds
Peak Memory Usage: 437.65 MB
Average Memory Usage: 231.76 MB
Average CPU Usage: 157.76%
Samples Collected: 14
Results saved to: benchmark_v0.18.3-beta.json
Temporary directory will be cleaned up automatically
Cleaning up temporary directory: /tmp/lnd-bench-v0.18.3-beta-tQ9SWS
Waiting 10 seconds before next benchmark...
>>> Running v0.18.4-beta (2/4)...
==========================================
Benchmarking LND v0.18.4-beta
Using temporary storage: /tmp/lnd-bench-v0.18.4-beta-2Z0bCV
Resource Limits:
- CPU: 4 cores
- Memory: 8G
==========================================
Starting LND in isolated namespace with tmpfs...
Started LND (PID: 852086)
Monitoring sync status...
To manually check status, run:
sudo nsenter --target 852086 --pid --mount --uts --ipc \
/home/dev/lnd/lncli-debug-v0.18.4-beta --lnddir=/tmp/lnd_data_ns/lnd getinfo
[Sample 0] Block: unknown | synced: false | Memory: 363.35 MB | CPU: 0%
[Sample 1] Block: unknown | synced: false | Memory: 454.38 MB | CPU: 240.00%
[Sample 2] Block: 197000 | synced: false | Memory: 118.01 MB | CPU: 183.00%
[Sample 3] Block: 275000 | synced: false | Memory: 132.96 MB | CPU: 158.00%
[Sample 4] Block: 351000 | synced: false | Memory: 150.52 MB | CPU: 153.00%
[Sample 5] Block: 432000 | synced: false | Memory: 163.67 MB | CPU: 158.00%
[Sample 6] Block: 508000 | synced: false | Memory: 178.56 MB | CPU: 164.00%
[Sample 7] Block: 583000 | synced: false | Memory: 190.03 MB | CPU: 156.00%
[Sample 8] Block: 635000 | synced: false | Memory: 209.26 MB | CPU: 148.00%
[Sample 9] Block: 705000 | synced: false | Memory: 226.96 MB | CPU: 151.00%
[Sample 10] Block: 791000 | synced: false | Memory: 248.11 MB | CPU: 161.00%
[Sample 11] Block: 816000 | synced: false | Memory: 255.58 MB | CPU: 132.00%
[Sample 12] Block: 918000 | synced: false | Memory: 251.60 MB | CPU: 132.00%
synced_to_chain
✓ synced_to_chain: true (block height: 918649)
Stopping LND...
========================================
BENCHMARK RESULTS - v0.18.4-beta
========================================
Execution Time: 70.358434022 seconds
Peak Memory Usage: 454.38 MB
Average Memory Usage: 234.59 MB
Average CPU Usage: 157.61%
Samples Collected: 14
Results saved to: benchmark_v0.18.4-beta.json
Temporary directory will be cleaned up automatically
Cleaning up temporary directory: /tmp/lnd-bench-v0.18.4-beta-2Z0bCV
Waiting 10 seconds before next benchmark...
>>> Running v0.18.5-beta (3/4)...
==========================================
Benchmarking LND v0.18.5-beta
Using temporary storage: /tmp/lnd-bench-v0.18.5-beta-XHFt2i
Resource Limits:
- CPU: 4 cores
- Memory: 8G
==========================================
Starting LND in isolated namespace with tmpfs...
Started LND (PID: 852853)
Monitoring sync status...
To manually check status, run:
sudo nsenter --target 852853 --pid --mount --uts --ipc \
/home/dev/lnd/lncli-debug-v0.18.5-beta --lnddir=/tmp/lnd_data_ns/lnd getinfo
[Sample 0] Block: unknown | synced: false | Memory: 352.79 MB | CPU: 0%
[Sample 1] Block: unknown | synced: false | Memory: 475.08 MB | CPU: 244.00%
[Sample 2] Block: 207000 | synced: false | Memory: 116.33 MB | CPU: 180.00%
[Sample 3] Block: 256000 | synced: false | Memory: 138.87 MB | CPU: 144.00%
[Sample 4] Block: 347000 | synced: false | Memory: 156.42 MB | CPU: 163.00%
[Sample 5] Block: 434000 | synced: false | Memory: 166.09 MB | CPU: 163.00%
[Sample 6] Block: 514000 | synced: false | Memory: 179.01 MB | CPU: 161.00%
[Sample 7] Block: 571000 | synced: false | Memory: 190.89 MB | CPU: 150.00%
[Sample 8] Block: 635000 | synced: false | Memory: 208.35 MB | CPU: 143.00%
[Sample 9] Block: 721000 | synced: false | Memory: 221.60 MB | CPU: 163.00%
[Sample 10] Block: 795000 | synced: false | Memory: 237.88 MB | CPU: 162.00%
[Sample 11] Block: 854000 | synced: false | Memory: 262.40 MB | CPU: 156.00%
synced_to_chain
✓ synced_to_chain: true (block height: 918649)
Stopping LND...
========================================
BENCHMARK RESULTS - v0.18.5-beta
========================================
Execution Time: 65.119188935 seconds
Peak Memory Usage: 475.08 MB
Average Memory Usage: 230.24 MB
Average CPU Usage: 162.00%
Samples Collected: 13
Results saved to: benchmark_v0.18.5-beta.json
Temporary directory will be cleaned up automatically
Cleaning up temporary directory: /tmp/lnd-bench-v0.18.5-beta-XHFt2i
Waiting 10 seconds before next benchmark...
>>> Running v0.20.0-beta.rc1-after (4/4)...
==========================================
Benchmarking LND v0.20.0-beta.rc1-after
Using temporary storage: /tmp/lnd-bench-v0.20.0-beta.rc1-after-LeHuC4
Resource Limits:
- CPU: 4 cores
- Memory: 8G
==========================================
Starting LND in isolated namespace with tmpfs...
Started LND (PID: 854003)
Monitoring sync status...
To manually check status, run:
sudo nsenter --target 854003 --pid --mount --uts --ipc \
/home/dev/lnd/lncli-debug-v0.20.0-beta.rc1-after --lnddir=/tmp/lnd_data_ns/lnd getinfo
[Sample 0] Block: unknown | synced: false | Memory: 356.35 MB | CPU: 0%
[Sample 1] Block: unknown | synced: false | Memory: 489.78 MB | CPU: 249.00%
[Sample 2] Block: 207000 | synced: false | Memory: 144.97 MB | CPU: 222.00%
[Sample 3] Block: 278000 | synced: false | Memory: 172.72 MB | CPU: 203.00%
[Sample 4] Block: 355000 | synced: false | Memory: 207.43 MB | CPU: 221.00%
[Sample 5] Block: 436000 | synced: false | Memory: 242.13 MB | CPU: 224.00%
[Sample 6] Block: 510000 | synced: false | Memory: 269.44 MB | CPU: 216.00%
[Sample 7] Block: 577000 | synced: false | Memory: 303.05 MB | CPU: 222.00%
[Sample 8] Block: 657000 | synced: false | Memory: 324.72 MB | CPU: 228.00%
[Sample 9] Block: 721000 | synced: false | Memory: 354.95 MB | CPU: 214.00%
[Sample 10] Block: 771000 | synced: false | Memory: 378.89 MB | CPU: 208.00%
[Sample 11] Block: 795000 | synced: false | Memory: 405.87 MB | CPU: 183.00%
[Sample 12] Block: 872000 | synced: false | Memory: 392.42 MB | CPU: 227.00%
synced_to_chain
✓ synced_to_chain: true (block height: 918649)
Stopping LND...
========================================
BENCHMARK RESULTS - v0.20.0-beta.rc1-after
========================================
Execution Time: 70.416534861 seconds
Peak Memory Usage: 489.78 MB
Average Memory Usage: 308.40 MB
Average CPU Usage: 208.15%
Samples Collected: 14
Results saved to: benchmark_v0.20.0-beta.rc1-after.json
Temporary directory will be cleaned up automatically
Cleaning up temporary directory: /tmp/lnd-bench-v0.20.0-beta.rc1-after-LeHuC4
==========================================
COMPARISON RESULTS
==========================================
Metric | v0.18.3-beta | v0.18.4-beta | v0.18.5-beta | v0.20.0-beta.rc1-after
------------------------- | ----------------- | ----------------- | ----------------- | -----------------
Execution Time (s) | 70.434696628 | 70.358434022 | 65.119188935 | 70.416534861
Peak Memory (MB) | 437.65 | 454.38 | 475.08 | 489.78
Average Memory (MB) | 231.76 | 234.59 | 230.24 | 308.4
Average CPU (%%) | 157.76 | 157.61 | 162 | 208.15
Summary:
--------
✓ Fastest: v0.18.5-beta (65.119188935s)
✓ Lowest Peak Memory: v0.18.3-beta (437.65 MB)
✓ Lowest Average CPU: v0.18.4-beta (157.61%)
Detailed logs available in:
lnd_v0.18.3-beta.log
lnd_v0.18.4-beta.log
lnd_v0.18.5-beta.log
lnd_v0.20.0-beta.rc1-after.log
dev@dev:~/lnd$ cat compare_lnd_versions.sh
#!/bin/bash
# Compare multiple LND versions in isolated namespaces
# Usage: sudo ./compare_lnd_versions.sh
set -e
# Define versions to compare
VERSIONS=(
"v0.18.3-beta"
"v0.18.4-beta"
"v0.18.5-beta"
#"v0.19.0-beta"
#"v0.19.1-beta"
#"v0.19.2-beta"
#"v0.19.3-beta"
#"v0.20.0-beta.rc1"
"v0.20.0-beta.rc1-after"
)
# Check if running as root
if [ "$EUID" -ne 0 ]; then
echo "This script must be run as root for namespace isolation"
echo "Usage: sudo $0"
exit 1
fi
echo "=========================================="
echo "LND Version Comparison Benchmark"
echo "=========================================="
echo "Comparing ${#VERSIONS[@]} versions:"
for version in "${VERSIONS[@]}"; do
echo " - $version"
done
echo ""
echo "Each version will run in isolated namespaces"
echo "for fair comparison."
echo ""
# Run benchmarks for each version
for i in "${!VERSIONS[@]}"; do
version="${VERSIONS[$i]}"
echo ">>> Running ${version} ($((i+1))/${#VERSIONS[@]})..."
./benchmark_lnd_namespace.sh "$version"
# Wait between benchmarks (except after the last one)
if [ $i -lt $((${#VERSIONS[@]} - 1)) ]; then
echo ""
echo "Waiting 10 seconds before next benchmark..."
sleep 10
echo ""
fi
done
echo ""
echo "=========================================="
echo "COMPARISON RESULTS"
echo "=========================================="
echo ""
# Verify all benchmark files exist
ALL_FILES_EXIST=true
for version in "${VERSIONS[@]}"; do
if [ ! -f "benchmark_${version}.json" ]; then
echo "Error: benchmark_${version}.json not found"
ALL_FILES_EXIST=false
fi
done
if [ "$ALL_FILES_EXIST" = false ]; then
exit 1
fi
# Parse and compare results
if command -v jq &> /dev/null; then
# Build header
printf "%-25s" "Metric"
for version in "${VERSIONS[@]}"; do
printf " | %-17s" "$version"
done
echo ""
printf "%-25s" "-------------------------"
for version in "${VERSIONS[@]}"; do
printf " | %-17s" "-----------------"
done
echo ""
# Execution time
printf "%-25s" "Execution Time (s)"
TIMES=()
for version in "${VERSIONS[@]}"; do
TIME=$(jq -r '.execution_time_seconds' "benchmark_${version}.json")
TIMES+=("$TIME")
printf " | %-17s" "$TIME"
done
echo ""
# Peak memory
printf "%-25s" "Peak Memory (MB)"
MEMS=()
for version in "${VERSIONS[@]}"; do
MEM=$(jq -r '.peak_memory_mb' "benchmark_${version}.json")
MEMS+=("$MEM")
printf " | %-17s" "$MEM"
done
echo ""
# Average memory
printf "%-25s" "Average Memory (MB)"
AVGS=()
for version in "${VERSIONS[@]}"; do
AVG=$(jq -r '.average_memory_mb' "benchmark_${version}.json")
AVGS+=("$AVG")
printf " | %-17s" "$AVG"
done
echo ""
# CPU usage
printf "%-25s" "Average CPU (%%)"
CPUS=()
for version in "${VERSIONS[@]}"; do
CPU=$(jq -r '.average_cpu_percent' "benchmark_${version}.json")
CPUS+=("$CPU")
printf " | %-17s" "$CPU"
done
echo ""
echo ""
echo "Summary:"
echo "--------"
# Find fastest version
FASTEST_IDX=0
FASTEST_TIME="${TIMES[0]}"
for i in "${!TIMES[@]}"; do
if (( $(echo "${TIMES[$i]} < $FASTEST_TIME" | bc -l) )); then
FASTEST_TIME="${TIMES[$i]}"
FASTEST_IDX=$i
fi
done
echo "✓ Fastest: ${VERSIONS[$FASTEST_IDX]} (${FASTEST_TIME}s)"
# Find most memory efficient
LOWEST_IDX=0
LOWEST_MEM="${MEMS[0]}"
for i in "${!MEMS[@]}"; do
if (( $(echo "${MEMS[$i]} < $LOWEST_MEM" | bc -l) )); then
LOWEST_MEM="${MEMS[$i]}"
LOWEST_IDX=$i
fi
done
echo "✓ Lowest Peak Memory: ${VERSIONS[$LOWEST_IDX]} (${LOWEST_MEM} MB)"
# Find lowest CPU usage
LOWEST_CPU_IDX=0
LOWEST_CPU="${CPUS[0]}"
for i in "${!CPUS[@]}"; do
if (( $(echo "${CPUS[$i]} < $LOWEST_CPU" | bc -l) )); then
LOWEST_CPU="${CPUS[$i]}"
LOWEST_CPU_IDX=$i
fi
done
echo "✓ Lowest Average CPU: ${VERSIONS[$LOWEST_CPU_IDX]} (${LOWEST_CPU}%)"
else
echo "Install 'jq' for formatted comparison, or view raw JSON files:"
for version in "${VERSIONS[@]}"; do
echo " benchmark_${version}.json"
done
echo ""
echo "Raw comparison:"
for version in "${VERSIONS[@]}"; do
echo ""
echo "${version}:"
cat "benchmark_${version}.json"
done
fi
echo ""
echo "Detailed logs available in:"
for version in "${VERSIONS[@]}"; do
echo " lnd_${version}.log"
done
#!/usr/bin/env python3
"""
LND Benchmark Visualization
Generates comparison charts for multiple LND versions
Usage: python visualize_benchmarks.py benchmark_*.json
"""
import json
import sys
import glob
from pathlib import Path
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from typing import List, Dict
def load_benchmarks(file_pattern: str = "benchmark_*.json") -> List[Dict]:
"""Load all benchmark JSON files matching the pattern."""
benchmarks = []
if len(sys.argv) > 1:
# Use command line arguments if provided
files = sys.argv[1:]
else:
# Use glob pattern
files = glob.glob(file_pattern)
if not files:
print(f"No benchmark files found matching: {file_pattern}")
print("Usage: python visualize_benchmarks.py benchmark_*.json")
print(" or: python visualize_benchmarks.py file1.json file2.json ...")
sys.exit(1)
for filepath in sorted(files):
try:
with open(filepath, 'r') as f:
data = json.load(f)
benchmarks.append(data)
print(f"Loaded: {filepath}")
except Exception as e:
print(f"Error loading {filepath}: {e}")
# Sort by version for consistent ordering
benchmarks.sort(key=lambda x: x['version'])
return benchmarks
def extract_version_short(version: str) -> str:
"""Extract short version name for x-axis labels."""
return version
# Convert "v0.18.3-beta" to "v0.18.3"
#return version.split('-')[0] if '-' in version else version
def create_single_chart(versions, values, ylabel, title, output_file, colors):
"""Create a single bar chart and save it."""
fig, ax = plt.subplots(figsize=(10, 6))
# Each version gets its own color
bars = ax.bar(range(len(versions)), values, color=colors, edgecolor='white', linewidth=2, alpha=0.9)
ax.set_ylabel(ylabel, fontweight='bold', fontsize=13)
ax.set_title(title, fontweight='bold', fontsize=15, pad=15)
ax.set_xticks(range(len(versions)))
ax.set_xticklabels([''] * len(versions)) # Remove x-axis labels
ax.grid(axis='y', alpha=0.3, linestyle='--', linewidth=1)
ax.set_axisbelow(True)
# Add value labels on bars
for bar in bars:
height = bar.get_height()
ax.text(bar.get_x() + bar.get_width()/2., height,
f'{height:.1f}',
ha='center', va='bottom', fontsize=11, fontweight='bold')
# Add some padding to y-axis
ymin, ymax = ax.get_ylim()
ax.set_ylim(0, ymax * 1.15)
# Create legend at the top right (always horizontal)
legend_patches = [mpatches.Patch(color=colors[i], label=versions[i]) for i in range(len(versions))]
ax.legend(handles=legend_patches, loc='upper right', ncol=len(versions),
frameon=True, fontsize=11, edgecolor='gray', fancybox=True, shadow=True)
plt.tight_layout()
plt.savefig(output_file, dpi=300, bbox_inches='tight', facecolor='white')
print(f"✓ Saved: {output_file}")
plt.close()
def create_comparison_charts(benchmarks: List[Dict]):
"""Create three separate comparison charts: Peak Memory, CPU Usage, Execution Time."""
if not benchmarks:
print("No benchmarks to visualize")
return
# Extract data
versions = [extract_version_short(b['version']) for b in benchmarks]
peak_memory = [b['peak_memory_mb'] for b in benchmarks]
avg_cpu = [b['average_cpu_percent'] for b in benchmarks]
exec_time = [b['execution_time_seconds'] for b in benchmarks]
# Color scheme - different color for each version
color_palette = ['#3b82f6', '#10b981', '#f59e0b', '#ef4444', '#8b5cf6', '#ec4899', '#06b6d4', '#84cc16']
colors = [color_palette[i % len(color_palette)] for i in range(len(versions))]
print("\nGenerating charts...")
# Chart 1: Peak Memory Usage
create_single_chart(
versions, peak_memory,
ylabel='Peak Memory (MB)',
title='LND Peak Memory Usage (Until Chain Mainnet Synced)',
output_file='lnd_peak_memory.png',
colors=colors
)
# Chart 2: Average CPU Usage
create_single_chart(
versions, avg_cpu,
ylabel='Average CPU (%)',
title='LND Average CPU Usage (Until Chain Mainnet Synced)',
output_file='lnd_cpu_usage.png',
colors=colors
)
# Chart 3: Execution Time
create_single_chart(
versions, exec_time,
ylabel='Execution Time (seconds)',
title='LND Execution Time (Until Chain Mainnet Synced)',
output_file='lnd_execution_time.png',
colors=colors
)
def print_comparison_table(benchmarks: List[Dict]):
"""Print a text-based comparison table."""
print("\n" + "="*100)
print("BENCHMARK COMPARISON TABLE")
print("="*100)
# Header
print(f"{'Version':<20} {'Exec Time (s)':<15} {'Peak Mem (MB)':<15} {'Avg CPU (%)':<15} {'Samples':<10}")
print("-"*100)
# Rows
for b in benchmarks:
print(f"{b['version']:<20} "
f"{b['execution_time_seconds']:<15.2f} "
f"{b['peak_memory_mb']:<15.2f} "
f"{b['average_cpu_percent']:<15.2f} "
f"{b['samples_collected']:<10}")
print("-"*100)
# Calculate improvements vs first version (baseline)
if len(benchmarks) > 1:
baseline = benchmarks[0]
print(f"\nComparison vs {baseline['version']} (baseline):")
print("-"*100)
for b in benchmarks[1:]:
time_diff = ((b['execution_time_seconds'] - baseline['execution_time_seconds']) /
baseline['execution_time_seconds'] * 100)
mem_diff = ((b['peak_memory_mb'] - baseline['peak_memory_mb']) /
baseline['peak_memory_mb'] * 100)
cpu_diff = ((b['average_cpu_percent'] - baseline['average_cpu_percent']) /
baseline['average_cpu_percent'] * 100)
print(f"{b['version']:<20} "
f"Time: {time_diff:+.1f}% "
f"Memory: {mem_diff:+.1f}% "
f"CPU: {cpu_diff:+.1f}%")
print("="*100 + "\n")
def main():
"""Main function to load data and create visualizations."""
# Load benchmark data
benchmarks = load_benchmarks()
if not benchmarks:
return
print(f"\nLoaded {len(benchmarks)} benchmark(s)")
# Print comparison table
print_comparison_table(benchmarks)
# Create charts
create_comparison_charts(benchmarks)
if __name__ == "__main__":
main()
(.venv) dev@dev:~/results$ python visualize_benchmarks.py
Loaded: benchmark_v0.18.3-beta.json
Loaded: benchmark_v0.18.4-beta.json
Loaded: benchmark_v0.18.5-beta.json
Loaded: benchmark_v0.20.0-beta.rc1-after.json
Loaded 4 benchmark(s)
====================================================================================================
BENCHMARK COMPARISON TABLE
====================================================================================================
Version Exec Time (s) Peak Mem (MB) Avg CPU (%) Samples
----------------------------------------------------------------------------------------------------
v0.18.3-beta 70.43 437.65 157.76 14
v0.18.4-beta 70.36 454.38 157.61 14
v0.18.5-beta 65.12 475.08 162.00 13
v0.20.0-beta.rc1-after 70.42 489.78 208.15 14
----------------------------------------------------------------------------------------------------
Comparison vs v0.18.3-beta (baseline):
----------------------------------------------------------------------------------------------------
v0.18.4-beta Time: -0.1% Memory: +3.8% CPU: -0.1%
v0.18.5-beta Time: -7.5% Memory: +8.6% CPU: +2.7%
v0.20.0-beta.rc1-after Time: -0.0% Memory: +11.9% CPU: +31.9%
====================================================================================================
Generating charts...
✓ Saved: lnd_peak_memory.png
✓ Saved: lnd_cpu_usage.png
✓ Saved: lnd_execution_time.png
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment