archiveteam-stats.py

# SPDX-FileCopyrightText: 2023 Nicolás Alvarez <nicolas.alvarez@gmail.com>
#
# SPDX-License-Identifier: MIT

import requests
import time
import sys
import datetime

sess = requests.session()

def seconds_to_human(s):
    text = f"{s:.0f} seconds"
    if s > 60:
        s /= 60
        text = f"{s:.0f} minutes"
    else: return text
    if s > 90:
        s /= 60
        text = f"{s:.0f} hours"
    else: return text
    if s > 40:
        s /= 24
        text = f"{s:.0f} days"
    else: return text
    if s > 120:
        s /= 30
        text = f"{s:.0f} months"
    return text

KB = 1024
MB = KB*1024
GB = MB*1024
TB = GB*1024

bytes_last=[]
items_last=[]
todo_last=[]

QUERIES_PER_MINUTE=4
minutes_window=int(sys.argv[2])
project=sys.argv[1]
next_ts=(time.time() // (60/QUERIES_PER_MINUTE) + 1) * (60/QUERIES_PER_MINUTE)

item_size = 275000

while True:
    while time.time() < next_ts:
        time.sleep(0.1)
    next_ts = (time.time() // (60/QUERIES_PER_MINUTE) + 1) * (60/QUERIES_PER_MINUTE)
    resp = None
    while not resp:
        try:
            resp = sess.get(f"https://legacy-api.arpa.li/{project}/stats.json")
        except requests.exceptions.ConnectionError:
            print("request failed, retrying", file=sys.stderr)
            continue # retry!
    s = resp.json()

    bytes_done = s["domain_bytes"]["data"]
    items_done = s["total_items_done"]
    items_todo = s["total_items_todo"]
    items_out = s["total_items_out"]
    #if project == 'imgur':
    #    items_todo -= 100_536131 + 100000
        #items_todo = items_out - 1000000
    if project == 'blogger':
        items_todo += items_out
    bytes_remaining = item_size * items_todo
    fraction_done = items_done / (items_done + items_todo)

    if len(bytes_last) > 0:
        # calculate rate based on the last N minutes, or the closest we can get
        WINDOW=minutes_window*QUERIES_PER_MINUTE
        if len(bytes_last) >= WINDOW:
            bytes_per_minute = (bytes_done-bytes_last[-WINDOW])/WINDOW*QUERIES_PER_MINUTE
            items_per_minute = (items_done-items_last[-WINDOW])/WINDOW*QUERIES_PER_MINUTE
            todo_per_minute  =-(items_todo- todo_last[-WINDOW])/WINDOW*QUERIES_PER_MINUTE
        else:
            bytes_per_minute = (bytes_done-bytes_last[0])/len(bytes_last)*QUERIES_PER_MINUTE
            items_per_minute = (items_done-items_last[0])/len(items_last)*QUERIES_PER_MINUTE
            todo_per_minute  =-(items_todo- todo_last[0])/len( todo_last)*QUERIES_PER_MINUTE

        bytes_remaining /= max(todo_per_minute/(items_per_minute or 0.1), 1)

        print("%.1f TiB done, %.1f TiB left, %.1f TiB total" % (
            bytes_done / TB,
            bytes_remaining / TB,
            (bytes_done+bytes_remaining) / TB
            ),
            end=""
        )

        if items_per_minute == 0:
            print(", stalled", end="")
        else:
            print(", %.2f GiB/minute, %.2f MiB/s, %.0f items/m, %.0f items/s, q %.0f items/m" % (
                (bytes_per_minute)/GB,
                (bytes_per_minute)/MB/60,
                items_per_minute,
                items_per_minute/60,
                todo_per_minute
            ), end="")

            if todo_per_minute < items_per_minute:
                print(", disc %.2fx" % ((items_per_minute - todo_per_minute) / items_per_minute), end="")
            else:
                print(", succ %.1f%%" % (items_per_minute / todo_per_minute * 100), end="")

            if todo_per_minute > 0:
                print(", ETA %s (%s)" % (
                    seconds_to_human(items_todo/todo_per_minute*60),
                    (datetime.datetime.utcnow() + datetime.timedelta(seconds=items_todo/todo_per_minute*60)).strftime('%b %d %H:%M')
                ), end="")
            else:
                print(", ETA* %s (%s)" % (
                    seconds_to_human(items_todo/items_per_minute*60),
                    (datetime.datetime.utcnow() + datetime.timedelta(seconds=items_todo/items_per_minute*60)).strftime('%b %d %H:%M')
                ), end="")

            if items_per_minute > 0:
                print(", avg item %.2fKB" % ((bytes_per_minute/items_per_minute)/KB), end="")
                #print(", avg item total %.2fKB" % (bytes_done/items_done/KB), end="")
                item_size = bytes_per_minute/items_per_minute

    bytes_last.append(bytes_done)
    items_last.append(items_done)
    todo_last.append(items_todo)
    print()
    sys.stdout.flush()