Setup Rclone Storage Nodes

Setup Rclone Storage Nodes.md

1. On each storage node run the setup-store.sh file.

bash setup-store.sh <global_username> <global_password>

OR

sudo -v ; curl -sL https://gist.githubusercontent.com/Wamy-Dev/4f10e107d812f13813a20fc7de2ce24a/raw/e059eaa697eb2a38378e8acb79ffd68ce45b7d75/setup-store.sh | sudo bash -s -- <global_username> <global_password>

• Runs the webdav instance as a service that persists on reboot
• Secured using the global_username and global_password, make sure these are the same on every system

2. On each gateway node run the setup-gateway.sh file.

bash setup-gateway.sh <global_username> <global_password>

OR

sudo -v ; curl -sL https://gist.githubusercontent.com/Wamy-Dev/4f10e107d812f13813a20fc7de2ce24a/raw/e059eaa697eb2a38378e8acb79ffd68ce45b7d75/setup-gateway.sh | sudo bash -s -- <global_username> <global_password>

• Make sure you edit the webdav_ips.txt file in the home directory with the ips of each webdav instance
• The gateway runs a webdav instance pointing to the union of all the store nodes (protected by the global_username and global_password)

setup-gateway.sh

#!/bin/bash
# Check for required arguments
if [ "$#" -ne 2 ]; then
    echo "Usage: $0 <webdav_username> <webdav_password>"
    exit 1
fi

# Assign arguments to variables
webdav_username="$1"
webdav_password="$2"

shift 2

# Variables
user_home=$(getent passwd ${SUDO_USER:-$USER} | cut -d: -f6)
log_dir="$user_home/rclone_setup_logs"
log_file="$log_dir/rclone_setup.log"
ip_file="$user_home/webdav_ips.txt"

# Ensure directories and files exist
initialize() {
    mkdir -p "$log_dir"
    touch "$log_file"
    if [ ! -f "$ip_file" ]; then
        echo "Creating empty WebDAV IPs file."
        touch "$ip_file"
    fi
}

# Read WebDAV server IPs from file
read_webdav_ips() {
    if [ -s "$ip_file" ]; then
        readarray -t webdav_servers < "$ip_file"
    else
        echo "No WebDAV server IPs found in $ip_file. Exiting."
        exit 1
    fi
}

# Add new IPs to the file (avoid duplicates)
add_ips_to_file() {
    for ip in "$@"; do
        if ! grep -q "^$ip$" "$ip_file"; then
            echo "$ip" >> "$ip_file"
        fi
    done
}

# Install or update rclone
install_update_rclone() {
    echo "Installing or updating rclone..."
    curl https://rclone.org/install.sh | sudo bash
}

# Configure a single WebDAV remote
configure_webdav() {
    local name=$1
    local ip=$2
    echo "Configuring WebDAV remote: $name with IP: $ip"
    rclone config create "$name" webdav url "http://$ip" vendor "other" user "$webdav_username" pass "$webdav_password" > /dev/null 2>&1
}

# Configure the union remote
configure_union() {
    if [ "${#webdav_servers[@]}" -lt 1 ]; then
        echo "Union remote requires at least two upstreams to be configured. Exiting."
        exit 1
    fi
    local remotes=""
    for i in "${!webdav_servers[@]}"; do
        remotes+="webdav$i:/ "
    done
    echo "Configuring union remote with remotes: $remotes"
    rclone config create union_remote union upstreams "$remotes" > /dev/null 2>&1
}

# Update WebDAV IPs, reconfigure remotes
update_and_configure() {
    echo "Updating WebDAV IPs and reconfiguring remotes..."
    rclone config delete union_remote
    for i in "${!webdav_servers[@]}"; do
        configure_webdav "webdav$i" "${webdav_servers[$i]}"
    done
    configure_union
}

# Stop existing rclone services
stop_services() {
    echo "Stopping any existing rclone serve services..."
    sudo systemctl stop rclone-serve-webdav.service || true
    sudo systemctl stop rclone-mount.service || true
}

# Create and start systemd services for rclone serve
create_and_start_services() {
    echo "Creating and starting systemd services for rclone serve..."

    # WebDAV Serve Service
    sudo bash -c "cat <<EOF > /etc/systemd/system/rclone-serve-webdav.service
[Unit]
Description=Rclone Serve WebDAV
After=network.target

[Service]
Type=simple
ExecStart=/usr/bin/rclone serve webdav union_remote: --addr 0.0.0.0:8080 --user \"$webdav_username\" --pass \"$webdav_password\"
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF"

# WebDAV Serve Service
    sudo bash -c "cat <<EOF > /etc/systemd/system/rclone-mount.service
[Unit]
Description=Rclone Mount
After=network.target

[Service]
Type=simple
ExecStart=/usr/bin/rclone mount union_remote: /mnt/union --allow-other --allow-non-empty --dir-cache-time 30s --poll-interval 5s --vfs-read-chunk-size 32M --vfs-read-chunk-size-limit 2G --buffer-size 512M --read-only --no-modtime --vfs-cache-mode=full
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF"

    # Reload systemd, enable and start services
    sudo systemctl daemon-reload
    sudo systemctl enable rclone-serve-webdav.service
    sudo systemctl enable rclone-mount.service
    sudo systemctl start rclone-serve-webdav.service
    sudo systemctl start rclone-mount.service
}

# Main execution logic
initialize
sudo mkdir -p /mnt/union
stop_services
add_ips_to_file "$@"
read_webdav_ips

install_update_rclone

# If there are WebDAV servers to configure, proceed
if [ ${#webdav_servers[@]} -ne 0 ]; then
    update_and_configure
    create_and_start_services
else
    echo "No WebDAV servers to configure. Exiting."
    exit 1
fi

echo "Rclone setup and serving configuration complete."

setup-store.sh

#!/bin/bash


# Check for required arguments
if [ "$#" -ne 2 ]; then
    echo "Usage: $0 <webdav_username> <webdav_password>"
    exit 1
fi

# Assign arguments to variables
webdav_username="$1"
webdav_password="$2"
# Define a log file location in the home directory
LOG_FILE="~/update_rclone_service.log"

# Ensure the log file path is expanded correctly
LOG_FILE=$(eval echo $LOG_FILE)

# Function to log a message
log_message() {
    echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

# Function to log an error but not exit
log_error_continue() {
    echo "$(date +'%Y-%m-%d %H:%M:%S') - ERROR: $1" | tee -a "$LOG_FILE"
}

# Log start
log_message "Starting rclone update and service restart"

# Stop the rclone WebDAV service
log_message "Stopping rclone WebDAV service..."
sudo systemctl stop rclone-webdav.service || log_error_continue "Failed to stop rclone-webdav.service"

# Update rclone and handle possible outcomes
update_output=$(curl https://rclone.org/install.sh | sudo bash 2>&1)
update_success=$?
echo "$update_output" | tee -a "$LOG_FILE"

if [[ $update_success -ne 0 ]]; then
    if echo "$update_output" | grep -q "is already installed"; then
        log_message "The latest version of rclone is already installed. Continuing..."
    else
        log_error_continue "rclone update failed but continuing with the script."
    fi
else
    log_message "rclone updated successfully."
fi

# Ensure the /mnt/storage directory exists
if [ ! -d "/mnt/storage" ]; then
    log_message "Creating /mnt/storage directory..."
    sudo mkdir -p /mnt/storage || log_error_continue "Failed to create /mnt/storage"
fi

# Check if the rclone config exists, if not create it
if [ ! -f "/root/.config/rclone/rclone.conf" ]; then
    log_message "Creating rclone config..."
    echo -e "[Local]\ntype = alias\nremote = /mnt/storage" | sudo tee /root/.config/rclone/rclone.conf > /dev/null || log_error_continue "Failed to create rclone config"
fi

# Ensure the rclone WebDAV service is correctly set up
log_message "Setting up rclone WebDAV service..."
sudo tee /etc/systemd/system/rclone-webdav.service > /dev/null <<EOL || log_error_continue "Failed to set up rclone-webdav.service"
[Unit]
Description=Rclone WebDAV Service
After=network.target

[Service]
Type=simple
ExecStart=/usr/bin/rclone serve webdav Local: -vv --addr 0.0.0.0:8080 --user $webdav_username --pass $webdav_password --poll-interval 0m5s --dir-cache-time 0m30s
Restart=on-abort

[Install]
WantedBy=multi-user.target
EOL

# Reload systemd to apply any changes
log_message "Reloading systemd daemon..."
sudo systemctl daemon-reload || log_error_continue "Failed to reload systemd"

# Ensure the service is enabled (to start at boot)
log_message "Enabling rclone WebDAV service..."
sudo systemctl enable rclone-webdav.service || log_error_continue "Failed to enable rclone-webdav.service"

# Start the service again
log_message "Starting rclone WebDAV service..."
sudo systemctl start rclone-webdav.service || log_error_continue "Failed to start rclone-webdav.service"

log_message "Rclone WebDAV service has been updated, restarted, and is now running."