Remote sync files with various method for various protocol

-remote_sync_files.md

Remote sync/copy files with various method for various protocol

copy_with_tool_rclone_protocol_http_1fichier.sh

#!/bin/bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# 1fichier shared folder : https://1fichier.com/dir/folder_id
# NOTE : this method do not sync but use copy method of rclone. So files not existing in source will not be deleted from the destination folder

# Do this at least once to add a rclone source named "foo" of type "fichier" for the given shared_folder
# (cat <<'EOL'
# [foo]
# type = fichier
# shared_folder = folder_id
# api_key = xxxxxxxxxxxxxx
# EOL
# )>> "${HOME}/.config/rclone/rclone.conf"

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log


process_rclone_copy() {
   # generate a list of files to process
   rclone lsl --human-readable "${URI}" >"${WORKSPACE}/list-${NICK_NAME}.log"
    
  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      rclone copy --dry-run -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
    ;;
    
    *)
      # launch copy
      # NOTE : this is a copy, not a sync. Files are not deleted from destination if not exist in source
      rclone copy -vv --human-readable --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
    ;;
	esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage


NICK_NAME="foo"
URI="foo:"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content"

process_rclone_copy

sync_with_rool_rclone_protocol_http_internetarchive.sh

#!/bin/bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# Sync file from 2 internet archive url into one folder
# Sync files : https://archive.org/details/foo1
#		           https://archive.org/details/foo2

# Do this at least once to add a rclone source named "internetarchive" of type "internetarchive"
# (cat <<'EOL'
# [internetarchive]
# type = internetarchive
# EOL
# )>> "${HOME}/.config/rclone/rclone.conf"

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log



process_rclone_sync() {
  # generate a list of files to process
  rclone lsl "${URI}" --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" >"${WORKSPACE}/list-${NICK_NAME}.log"

  case $MODE in
    test)
      # test run in dry-run mode
      echo "** Test mode activated, only a dry-run will be made **"
      rclone sync --dry-run -vv --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;

    *)
      # launch sync
      rclone sync -vv --human-readable --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" --log-file="${WORKSPACE}/sync-${NICK_NAME}.log" "${URI}" "${WORKSPACE}/content/${NICK_NAME}"
    ;;
  esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi



# PART1
NICK_NAME="foo-part1"
# NOTE foo1 is the subpath of the internet archive url
URI="internetarchive:foo1"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content/${NICK_NAME}"

process_rclone_sync


# PART2
NICK_NAME="foo-part2"
# NOTE foo2 is the subpath of the internetarchive url
URI="internetarchive:foo2"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content/${NICK_NAME}"

process_rclone_sync

sync_with_tool_rysnc_protocol_rsync_and_exclusion.sh

#!/bin/bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# Myrient No-Intro
# Sync files : rsync://rsync.myrient.erista.me/files/No-Intro (see https://myrient.erista.me/files/No-Intro/)

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log


process_rsync_archive_with_exclude() {
	# generate a list of files to process
	rsync --list-only -vv --archive --stats --human-readable --recursive --exclude-from="${EXCLUDE}" --log-file="${WORKSPACE}/list-${NICK_NAME}.log" "${URI}"

	case $MODE in
		test)
			# test run in dry-run mode
			echo "** Test mode activated, only a dry-run will be made **"
			rsync --dry-run -vv --delete-before --delete-excluded --stats --human-readable --progress --recursive --sparse --exclude-from="${EXCLUDE}" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"	
		;;

		*)
			# launch sync
			rsync -vv --archive --delete-before --delete-excluded --stats --human-readable --progress --recursive --sparse --exclude-from="${EXCLUDE}" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
		;;
	esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage


NICK_NAME="myrient-nointro"
URI="rsync://rsync.myrient.erista.me/files/No-Intro"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content"

# exclude list
EXCLUDE="${WORKSPACE}/exclude-${NICK_NAME}.txt"
(cat <<'EOL'
Audio CD/
CD-ROM/
EOL
)> "${EXCLUDE}"





process_rsync_archive_with_exclude

sync_with_tool_rysnc_protocol_rsync_and_inclusion.sh

#!/bin/bash
# https://gist.github.com/StudioEtrange/564f09bf3ec563dbb13e4c3cb8f8b801
# Sync files : "rsync://rsync.myrient.erista.me/files/Redump (see https://myrient.erista.me/files/Redump/)

CURRENT_FILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORKSPACE="${CURRENT_FILE_DIR}"
rm -f "${WORKSPACE}"/*.log


process_rsync_archive_with_include() {
	# generate a list of files to process
	# NOTE rsync option '--list-only' seems to not work with '--files-from' so use a dry-run instead
	rsync --dry-run -vv --archive --stats --human-readable --recursive --files-from="${INCLUDE}" "${URI}" "${TARGET}" >"${WORKSPACE}/list-${NICK_NAME}.log"

	case $MODE in
	  test)
	    # test run in dry-run mode
	    echo "** Test mode activated, only a dry-run will be made **"
	    rsync --dry-run -vv --archive --delete-before --delete-excluded --stats --human-readable --progress --recursive --sparse --files-from="${INCLUDE}" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
	  ;;

	  *)
	    # launch sync
	    rsync -vv --archive --delete-before --delete-excluded --stats --human-readable --progress --recursive --sparse --files-from="${INCLUDE}" --log-file="${WORKSPACE}/log-${NICK_NAME}.log" "${URI}" "${TARGET}"
	  ;;
	esac
}
usage() {
	echo "Usage : $0 [test]"
}
if [ "$1" = "test" ]; then
	echo "** Test mode activated, only a dry-run will be made **"
	MODE="test"
fi
usage


NICK_NAME="myrient-redump"
URI="rsync://rsync.myrient.erista.me/files/Redump"
# processed files will be in $TARGET folder
TARGET="${WORKSPACE}/content"

# include list
INCLUDE="${WORKSPACE}/include-${NICK_NAME}.txt"
(cat <<'EOL'
/Palm/
EOL
)> "${INCLUDE}"

process_rsync_archive_with_include