Wrapper for rsync, optimized for faster network transfers. It simplifies rsync's configuration to enhance usability & performance

frsync.sh

#!/bin/bash
# shellcheck disable=SC2034

# Author: Nikita Neverov (BMTLab)
# Version: 2.1.7

readonly ERR_FRSYNC_INVALID_OPTIONS=101

#######################################
# frsync: Optimizes the file synchronization process from local to remote using rsync over SSH or vice versa.
# The function is designed for advanced users needing fine-grained control over file transfer settings,
# aiming to achieve the highest possible performance. It includes options for enabling SSH compression,
# utilizing custom SSH private keys, managing SSH connection multiplexing, and ensuring file consistency
# with the --delete option. It also provides mechanisms to bypass SSH security checks in trusted networks
# for faster setup. This function embodies a balance between speed, security, and flexibility, offering
# users a powerful tool for their file synchronization needs.
#
# Globals:
#   ERR_FRSYNC_INVALID_OPTIONS - Error code for invalid options.
#
# Arguments:
#   -h - Displays help message and exits.
#   -c - Enables compression during the transfer to improve speed over slow connections.
#   -i <private_key> - Specifies a path to the SSH private key for secure authentication.
#   -m - Disables SSH connection multiplexing (ControlMaster and ControlPath), useful in environments where SSH multiplexing might not be supported.
#   -d - Enables the --delete option in rsync to ensure the receiving side mirrors the source by deleting extraneous files.
#   -u - Enables unsafe SSH options for faster communication in trusted networks, such as disabling strict host key checking.
#   <local_path1> [<local_path2> ...] <remote_user@remote_ip:remote_path> - Specifies local and remote paths for rsync.
#
# Outputs:
#   Writes messages to stdout and stderr, providing detailed feedback on the transfer process.
#   Exits with various status codes based on success or type of error encountered during execution.
#
# Returns:
#   0 - Successful execution.
#   ERR_FRSYNC_INVALID_OPTIONS(101) - Returned in case of invalid command-line options.
#   Other non-zero status - Returned in case of errors during rsync execution.
#######################################
function frsync() {
  local help_message

  # Multi-line usage message
  read -r -d '' help_message << 'EOF'
Usage: frsync [-h] [-c] [-i private_key] [-m] [-d] [-u] <local_path1> [<local_path2> ...] <remote_user@remote_ip:remote_path>

Options:
  -h: Display this help message and exit.
  -c: Enable compression during transfer.
  -i: Specify a path to the SSH private key for authentication.
  -m: Disable SSH connection multiplexing (ControlMaster and ControlPath).
  -d: Enable the --delete option in rsync to delete extraneous files from the receiving side.
  -u: Enable unsafe SSH options for faster communication, but only in trusted networks.

This function preconfigures rsync for faster network transfers by optimizing various settings.
EOF

  # Prints help message
  function __frsync_usage() {
    printf '%b\n' "$help_message"
    unset -f __frsync_usage
  }

  # Prints error message
  function __frsync_error() {
    local -r message="$1"
    local -ir code="${2:-$ERR_FRSYNC_INVALID_OPTIONS}"

    printf 'Error: %s.\n\n' "$message" >&2
    __frsync_usage
    unset -f __frsync_error

    return $code
  }

  local private_key_option=''
  local is_private_key_option_set=false
  local enable_compression=false
  local disable_multiplexing=false
  local enable_rsync_delete=false
  local enable_unsafe_options=false

  ## Process command-line options
  # This ensures that getopts always starts processing arguments from the first argument each time
  # the frsync function is called, and should fix the problem of missing options on repeated calls.
  OPTIND=1

  local opt
  while getopts ':hci:mdu' opt; do
    case "$opt" in
      h)
        __frsync_usage
        return 0
        ;;
      c)
        enable_compression=true
        ;;
      i)
        if [[ -z $OPTARG ]]; then
          __frsync_error 'Option -i requires an argument' || return $?
        fi

        private_key_option="$OPTARG"
        is_private_key_option_set=true
        ;;
      m)
        disable_multiplexing=true
        ;;
      d)
        enable_rsync_delete=true
        ;;
      u)
        enable_unsafe_options=true
        ;;
      \?)
        __frsync_error "Invalid option: -${OPTARG}" || return $?
        ;;
      :)
        __frsync_error "Option -${OPTARG} requires an argument" || return $?
        ;;
    esac
  done

  # We remove the processed options, leaving only the path arguments
  shift $((OPTIND - 1))

  # Verify that at least two arguments remain (local and remote paths)
  if [ "$#" -lt 2 ]; then
    __frsync_error 'Local and remote host must be specified' || return $?
  fi

  # Calculates local and remote hosts
  local -r remote_path="${!#}" # Get the last argument as the remote path
  local -ar local_path_arr=("${@:1:$#-1}") # Define as an array of all arguments except the last one

  # Tuning RSYNC options
  local -a rsync_options_arr=('-aHAXxv' '--numeric-ids' '--progress' '--partial' '--inplace')
  [[ $enable_rsync_delete == true ]] && rsync_options_arr+=('--delete')
  [[ $enable_compression == true ]] && rsync_options_arr+=('-z')

  # Tuning SSH options
  local -a ssh_options_arr=('-T' '-c' 'chacha20-poly1305@openssh.com') # Fastest encryption algorthym
  [[ $is_private_key_option_set == true ]] && ssh_options_arr+=('-i' "$private_key_option")
  [[ $disable_multiplexing == false ]] && ssh_options_arr+=('-o' 'ControlMaster=auto' '-o' 'ControlPath=/tmp/frsync-%r@%h:%p')
  [[ $enable_unsafe_options == true ]] && ssh_options_arr+=('-o' 'StrictHostKeyChecking=no' '-o' 'UserKnownHostsFile=/dev/null')
  ssh_options_arr+=('-o' 'ForwardX11=no')
  # ssh_options_arr+=('-v') # DEBUG

  local -r ssh_command="ssh $(printf '%q ' "${ssh_options_arr[@]}")"

  # Execute rsync with the predefined options
  rsync "${rsync_options_arr[@]}" -e "$ssh_command" "${local_path_arr[@]}" "$remote_path"

  unset -f __frsync_usage __frsync_error
}

readonly -f frsync

frsync: Advanced File Synchronization Utility

The frsync script is a robust and versatile tool designed to optimize file synchronization processes using rsync over SSH, catering to advanced users who require detailed control over their file transfer operations.

Features

• Compression: Reduces data size during transfer, enhancing speed on slower connections.
• SSH Private Key Authentication: Utilizes custom SSH keys for secure connections.
• SSH Multiplexing Control: Option to enable or disable SSH connection multiplexing.
• File Consistency: Includes an option to mirror the source directory on the target by deleting extraneous files.
• Performance Optimization: Optimizes various settings to achieve high-performance file transfers.

Prerequisites

• bash environment compatible with your operating system.
• rsync and ssh installed on your system.

Installation

1. Download this .frsync.sh script.
2. Integrate it into your current shell, for example in .bashrc, like this:

# ~/.bashrc or .profile

if [ -f "${HOME}/.frsync.sh" ]; then
    source "${HOME}/.frsync.sh"
fi

Usage

Execute the script from your command line with the desired options:

frsync [OPTIONS]... <local_path>... <remote_user@remote_ip:remote_path>

Command-Line Options:

-h: Displays the help message and exits.
-c: Enables compression during the transfer.
-i <private_key>: Specifies the SSH private key for authentication.
-m: Disables SSH connection multiplexing.
-d: Activates the --delete option in rsync.
-u: Enables faster, less secure SSH options for trusted networks.

Examples:

Basic Usage:

Transfer files with default options:

frsync -i ~/.ssh/id_rsa my_local_folder/ user@remote_host:/remote_folder

Advanced Usage:

Transfer files with compression and custom SSH key, deleting extraneous files on the remote side:

frsync -cud -i ~/.ssh/custom_rsa my_project/ user@192.168.0.1:/projects/my_project

Handling Errors

frsync provides clear error messages for common issues such as invalid options or missing arguments. It exits with specific codes that indicate different types of errors, making it suitable for integration into scripts where error handling is crucial.

101 - ERR_FRSYNC_INVALID_OPTIONS
Trigger: This error code is returned when invalid command-line options are provided to the script. This could be due to a typo, using an unsupported flag, or forgetting to provide a required argument for an option.

Modifying the Script

You can modify frsync to add more features or adjust existing ones to better suit your needs. Always ensure that any modifications are tested in a safe environment before use in production 😇

#!/usr/bin/env bats

load 'test_helper/bats-support/load'
load 'test_helper/bats-assert/load'

setup() {
  source ./.frsync.sh
}

@test 'Display help message with -h option' {
  ## Act
  run frsync -h

  ## Assert
  assert_success
  assert_output --partial 'Usage: frsync'
}

@test 'Error with invalid option' {
  ## Act
  run frsync -z

  ## Assert
  assert_failure
  assert_output --partial 'Error: Invalid option'
}

@test 'Error when mandatory arguments are not provided' {
  ## Act
  run frsync

  ## Assert
  assert_failure
  assert_output --partial 'Local and remote host must be specified'
}

@test 'Enable compression with -c option' {
  ## Arrange
  # Mocking `rsync` command to prevent actual execution
  # shellcheck disable=SC2317
  function rsync() {
    echo "rsync called with args: $*"
  }
  export -f rsync

  ## Act
  run frsync -c 'source' 'target'

  ## Assert
  assert_success
  assert_output --partial '-z'
}

@test 'Specify private key with -i option' {
  ## Arrange
  # Mocking `rsync` command to prevent actual execution
  # shellcheck disable=SC2317
  function rsync() {
    echo "rsync called with args: $*"
  }
  export -f rsync

  ## Act
  run frsync -i '/path/to/key' 'source' 'target'

  ## Assert
  assert_success
  assert_output --partial '-i /path/to/key'
}

teardown() {
  unset -f rsync
}