Python Rclone Script with Telegram and InfluxDB Reporting

rclone_backup.py

# Python Rclone Script with Telegram and InfluxDB Reporting
# - This Script backups your data to any rclone location (see rclone.org)
# - You can execute scripts, commands and database dumps before running a single rclone command
# - It sends automatic error reports and summaries
# - It saves all statistics and operations to an Influx database
# - The data can be used for visualizations in e.g. Grafana
# - You can automate it by using cron

# Created by: Luca Koroll - https://github.com/lucanello
# Last update: 2021-07-04

import json
import os
import requests
import datetime
import math
import subprocess

from influxdb import InfluxDBClient

# Set to True if you want to make a test run
dryrun = False

# Specify logfile directory (where temporary logs are saved)
logfiledir = '/root/'
# Specify host tag for Influx data, used for filtering if you backup several hosts
host = 'vhost'

# Specify extra options, if you want to filter for example (see the calls below, where they are used)
opts = ['--local-no-check-updated', '--delete-excluded', '--exclude-from=excluded.txt']

# Specify your Influx and Telegram credentials
influxclient = InfluxDBClient('host', 8086, 'db_user', 'db_pass', 'db_name')
telegram_token = "123456789:AAABBBCCCDDDEEEFFFGGGHHHIII"
telegram_chat_id = "-123456789"

result_msg = []


def rclone(source, target, configfile=None, transfers=48, dryrun=False, extra_opts=None, mode='sync', db_export=None,
           db_export_location=None, pre_cmd=None):
    # Construct logfilepath
    logfilepath = os.path.join(logfiledir + source.replace('/', '_') + '-rclone.log')
    # Construct rclone sync command
    basecmd = ('rclone ' + str(mode) + ' --transfers=' + str(transfers) + ' --log-file=' + str(logfilepath) +
               ' --use-json-log ' + '--log-level=INFO --stats=90m --fast-list').split()
    # Add dry-run for testing purposes
    basecmd.append('--dry-run') if dryrun else ''
    # Add explicit config if given
    basecmd.append('--config=' + str(configfile)) if configfile else ''
    # Add extra options
    if extra_opts:
        basecmd.extend(extra_opts)
    # Add source and target
    basecmd.append(str(source))
    basecmd.append(str(target))
    try:
        # Execute pre command
        if pre_cmd:
            result = subprocess.run(pre_cmd)
            if result.returncode != 0:
                raise Exception("Error in pre command: " + str(result))
        # Execute database export
        if db_export:
            with open(db_export_location, "w") as outfile:
                result = subprocess.run(db_export, stdout=outfile)
            if result.returncode != 0:
                raise Exception("Error in database export: " + str(result))
        # Execute command
        result = subprocess.run(basecmd)
        if result.returncode != 0:
            with open(logfilepath, 'r') as file:
                data = file.read()
            raise Exception(str(data) + " when executing: " + str(result))
        # Parse logfile
        checks, transfers, bytes = parse_log(logfilepath, source, target)
        # Remove log
        os.remove(logfilepath)
        # Append to result message
        statistic = "\n  ⤷ Checks: " + str(checks) + ", Transfers: " + str(transfers) + ", Bytes: " + \
                    str(convert_size(bytes))
        icon = "✅ " if transfers > 0 else "🆗 "
        result_msg.append(icon + source + " -> " + target + statistic)
    except BaseException as e:
        notify_telegram(("<b>🚨 ERROR in Backup occurred 🚨</b>\n<i>Source</i>: %s\n<i>Target</i>: %s\n<i>Error</i>: %s"
                         % (source, target, str(e))))
        result_msg.append("❌ " + source + " -> " + target)



def parse_log(logfile, source, target):
    with open(logfile) as f:
        data = [json.loads(line.rstrip()) for line in f if line[0] == "{"]
    stats = []
    operations = []
    for obj in data:
        if 'accounting/stats' in obj['source']:
            stats.append(obj)
        elif 'operations/operations' in obj['source']:
            operations.append(obj)
    for obj in stats:
        obj['stats']['speed'] = float(obj['stats']['speed'])
        json_body = [
            {
                "measurement": "stats",
                "tags": {
                    "host": host,
                    "level": obj['level'],
                    "log_entry_source": obj['source'],
                    "source": source,
                    "target": target
                },
                "time": obj['time'],
                "fields": obj['stats']
            }
        ]
        if not dryrun:
            influxclient.write_points(json_body)
    for obj in operations:
        json_body = [
            {
                "measurement": "operations",
                "tags": {
                    "host": host,
                    "level": obj['level'],
                    "log_entry_source": obj['source'],
                    "objType": obj['objectType'],
                    "msg": obj['msg'],
                    "source": source,
                    "target": target
                },
                "time": obj['time'],
                "fields": {
                    "obj": obj['object']
                }
            }
        ]
        if not dryrun:
            influxclient.write_points(json_body)
    return stats[0]['stats']['totalChecks'], stats[0]['stats']['totalTransfers'], stats[0]['stats']['totalBytes']


def notify_telegram(text):
    params = {"parse_mode": "HTML", "chat_id": telegram_chat_id, "text": text}
    url = f"https://api.telegram.org/bot{telegram_token}/sendMessage"
    requests.post(url, params=params)


def convert_size(size_bytes):
   if size_bytes == 0:
       return "0B"
   size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
   i = int(math.floor(math.log(size_bytes, 1024)))
   p = math.pow(1024, i)
   s = round(size_bytes / p, 2)
   return "%s %s" % (s, size_name[i])


if __name__ == '__main__':
    start = datetime.datetime.now()

    # Using extra options (specified in the beginning)
    rclone('/source/folder', 'target:folder/', dryrun=dryrun,
                extra_opts=opts)
    
    # Add another extra option 
    rclone('/source/folder', 'target:folder/', dryrun=dryrun,
                extra_opts=opts + ['--min-age=30m'])

    # Backup a database which is dumped in before
    # with rclone(..., db_expot=command, db_export_location=location) you can execute a dump to the given location
    db_export_cmd = '/usr/bin/mysqldump --no-tablespaces --skip-dump-date --single-transaction -h localhost -u db_user -pdb_pass db_name'
    rclone('/source/folder', 'target:folder/',
           dryrun=dryrun, extra_opts=opts, mode='copy',
           db_export=db_export_cmd.split(), db_export_location='/dump/target/db_export.sql')

    # Run a general command in before
    # I am using this for gitea backups for example
    pre_cmd = 'bash /sample/script.sh'
    rclone('/source/folder', 'target:folder/',
           dryrun=dryrun, pre_cmd=pre_cmd.split())

    end = datetime.datetime.now()
    duration = end - start

    # Send Telegram notification
    notify_telegram(("<b>Backup Statistics:</b>\n\n%s\n\n<i>Start Time</i>: %s\n<i>End Time</i>: %s\n<i>Duration</i>: %s minutes" %
                     ("\n\n".join(result_msg), start.strftime("%Y-%m-%d %H:%M:%S"), end.strftime("%Y-%m-%d %H:%M:%S"),
                      divmod(duration.total_seconds(), 60)[0])))

rclone_grafana_dashboard.json

{
  "__inputs": [
    {
      "name": "DS_BACKUPS@INFLUXDB",
      "label": "backups@InfluxDB",
      "description": "",
      "type": "datasource",
      "pluginId": "influxdb",
      "pluginName": "InfluxDB"
    }
  ],
  "__requires": [
    {
      "type": "panel",
      "id": "gauge",
      "name": "Gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "7.5.7"
    },
    {
      "type": "datasource",
      "id": "influxdb",
      "name": "InfluxDB",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "piechart",
      "name": "Pie chart v2",
      "version": ""
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "panels": [
    {
      "datasource": "${DS_BACKUPS@INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "hideFrom": {
              "graph": false,
              "legend": false,
              "tooltip": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "decbytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 14,
        "x": 0,
        "y": 0
      },
      "id": 10,
      "interval": "1d",
      "options": {
        "graph": {},
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltipOptions": {
          "mode": "single"
        }
      },
      "pluginVersion": "7.5.7",
      "targets": [
        {
          "alias": "$tag_source",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "source"
              ],
              "type": "tag"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "totalBytes"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "transfers",
              "operator": ">",
              "value": "0"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Transferred data per source",
      "type": "timeseries"
    },
    {
      "datasource": "${DS_BACKUPS@INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 5,
        "x": 14,
        "y": 0
      },
      "id": 8,
      "interval": "1d",
      "options": {
        "displayLabels": [],
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "values": []
        },
        "pieType": "pie",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {}
      },
      "pluginVersion": "7.5.7",
      "targets": [
        {
          "alias": "$tag_source",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "source"
              ],
              "type": "tag"
            }
          ],
          "limit": "",
          "measurement": "stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT last(\"transfers\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval), \"source\" DESC LIMIT 5",
          "rawQuery": false,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "transfers"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "last"
              }
            ]
          ],
          "tags": [
            {
              "key": "transfers",
              "operator": ">",
              "value": "0"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Transfers by source",
      "type": "piechart"
    },
    {
      "datasource": "${DS_BACKUPS@INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "max": 5000000,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "dark-red",
                "value": null
              },
              {
                "color": "dark-yellow",
                "value": 50
              },
              {
                "color": "dark-green",
                "value": 75
              }
            ]
          },
          "unit": "Bps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 5,
        "x": 19,
        "y": 0
      },
      "id": 12,
      "interval": "1d",
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "text": {}
      },
      "pluginVersion": "7.5.7",
      "targets": [
        {
          "alias": "",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            }
          ],
          "limit": "",
          "measurement": "stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT last(\"transfers\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval), \"source\" DESC LIMIT 5",
          "rawQuery": false,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "speed"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "transfers",
              "operator": ">",
              "value": "0"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Transfer speed",
      "type": "gauge"
    },
    {
      "datasource": "${DS_BACKUPS@INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "hideFrom": {
              "graph": false,
              "legend": false,
              "tooltip": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 14,
        "x": 0,
        "y": 8
      },
      "id": 2,
      "interval": "1d",
      "options": {
        "graph": {},
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltipOptions": {
          "mode": "single"
        }
      },
      "pluginVersion": "7.5.7",
      "targets": [
        {
          "alias": "$tag_msg",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "msg"
              ],
              "type": "tag"
            },
            {
              "params": [
                "0"
              ],
              "type": "fill"
            }
          ],
          "measurement": "operations",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "obj"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Operation types",
      "type": "timeseries"
    },
    {
      "datasource": "${DS_BACKUPS@INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 5,
        "x": 14,
        "y": 8
      },
      "id": 11,
      "interval": "1d",
      "options": {
        "displayLabels": [],
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "values": []
        },
        "pieType": "pie",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {}
      },
      "pluginVersion": "7.5.7",
      "targets": [
        {
          "alias": "$tag_msg",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "msg"
              ],
              "type": "tag"
            },
            {
              "params": [
                "0"
              ],
              "type": "fill"
            }
          ],
          "measurement": "operations",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "obj"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Operation types",
      "type": "piechart"
    },
    {
      "datasource": "${DS_BACKUPS@INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "dark-red",
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "dark-green",
                "value": null
              },
              {
                "color": "dark-red",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 5,
        "x": 19,
        "y": 8
      },
      "id": 13,
      "interval": "1d",
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "7.5.7",
      "targets": [
        {
          "alias": "",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            }
          ],
          "limit": "",
          "measurement": "stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT last(\"transfers\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval), \"source\" DESC LIMIT 5",
          "rawQuery": false,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "errors"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "transfers",
              "operator": ">",
              "value": "0"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Errors",
      "type": "stat"
    },
    {
      "datasource": "${DS_BACKUPS@INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "hideFrom": {
              "graph": false,
              "legend": false,
              "tooltip": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "decbytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Checks"
            },
            "properties": [
              {
                "id": "unit",
                "value": "none"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Transfers"
            },
            "properties": [
              {
                "id": "unit",
                "value": "none"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 10,
        "w": 24,
        "x": 0,
        "y": 16
      },
      "id": 3,
      "interval": "1d",
      "options": {
        "graph": {},
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltipOptions": {
          "mode": "single"
        }
      },
      "pluginVersion": "7.5.7",
      "targets": [
        {
          "alias": "Bytes transferred",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "0"
              ],
              "type": "fill"
            }
          ],
          "measurement": "stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "totalBytes"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "sum"
              }
            ]
          ],
          "tags": []
        },
        {
          "alias": "Checks",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "0"
              ],
              "type": "fill"
            }
          ],
          "hide": false,
          "measurement": "stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT sum(\"totalChecks\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval) fill(0)",
          "rawQuery": false,
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "totalChecks"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "sum"
              }
            ]
          ],
          "tags": []
        },
        {
          "alias": "Transfers",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "0"
              ],
              "type": "fill"
            }
          ],
          "hide": false,
          "measurement": "stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT sum(\"totalChecks\") FROM \"stats\" WHERE $timeFilter GROUP BY time($__interval) fill(0)",
          "rawQuery": false,
          "refId": "C",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "totalTransfers"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "sum"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Operations",
      "type": "timeseries"
    },
    {
      "datasource": "${DS_BACKUPS@INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": null,
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 155
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "msg"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 92
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "source"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 462
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "msg"
            },
            "properties": [
              {
                "id": "mappings",
                "value": [
                  {
                    "from": "",
                    "id": 1,
                    "text": "Updated",
                    "to": "",
                    "type": 1,
                    "value": "Updated modification time in destination"
                  },
                  {
                    "from": "",
                    "id": 2,
                    "text": "Updated",
                    "to": "",
                    "type": 1,
                    "value": "Copied (replaced existing)"
                  },
                  {
                    "from": "",
                    "id": 3,
                    "text": "Added",
                    "to": "",
                    "type": 1,
                    "value": "Copied (New)"
                  }
                ]
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 12,
        "w": 24,
        "x": 0,
        "y": 26
      },
      "id": 5,
      "interval": "1d",
      "options": {
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "Time"
          }
        ]
      },
      "pluginVersion": "7.5.7",
      "targets": [
        {
          "groupBy": [],
          "measurement": "operations",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT \"obj\" FROM \"operations\" WHERE (\"msg\" = 'Deleted') AND $timeFilter",
          "rawQuery": false,
          "refId": "A",
          "resultFormat": "table",
          "select": [
            [
              {
                "params": [
                  "msg"
                ],
                "type": "field"
              }
            ],
            [
              {
                "params": [
                  "source"
                ],
                "type": "field"
              }
            ],
            [
              {
                "params": [
                  "obj"
                ],
                "type": "field"
              }
            ]
          ],
          "tags": []
        }
      ],
      "title": "File update log",
      "type": "table"
    }
  ],
  "refresh": false,
  "schemaVersion": 27,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-7d",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Backup Dashboard (Rclone)",
  "uid": "6xQTJPk7k",
  "version": 25
}

Sample Telegram Notification

Sample Grafana Dashboard

You can find the Grafana dashboard here: https://grafana.com/grafana/dashboards/14686

could you provide some instruction to deploy it ?

could you provide some instruction to deploy it ?

Things you're going to need:

• Rclone set up with backup targets
• InfluxDB for metrics storage
• Grafana for visualization
• (optionally) a Telegram account and the chat ID, you might have to look up how to get it

Instructions:

1. Copy the script to your machine.
2. Install the specified python modules using pip install <module>, you can see them imported at the beginning of the script.
3. You might change the options (line 22 to 35) to match your preferences.
4. Change lines 162 to 181 to match your sources and targets you want to backup. You can use the sample commands. Just play around.
5. Execute the script using python3 rclone_script.py. Check the log to see if it's all running fine.
6. Check if the data is added to the Influx database.
7. Log in to Grafana, add your InfluxDB as data source, create a new dashboard by importing the rclone_grafana_dashboard.json file and adapt it to the specified data source.
8. You might setup a cronjob to execute the script regularly.

Please check the comments inside the script.