Python Freenet fetchpull

README.md

Insert-and-fetch statistics for Freenet

Disclaimer: this is by all means unfinished work.

This script should be run at most once a day (preferably exactly once a day), and the uid must be unique. It requires a modified pyFreenet with this patch applied.

To run, first edit fetchpull.py to set a unique uid, then:

python2 fetchpull.py

By default, it inserts 100 SSK keys for each of the 8 deltas on each run, where the deltas are given as powers of two (e.g. 1, 2, 4, 8, 16, 32, 64, 128 days), amounting to 800 keys in total. The total insert duration is by default limited to 7200 seconds = 2 hours. All keys that were successfully inserted during that period of time are saved in a pickle for future reference. After inserting, previously inserted keys are fetched for the different deltas (i.e., keys inserted n days ago that were intended to be fetched today, that is, with delta of n days), and the number of successful fetches vs. total keys attempted is saved in a pickle.

It creates several pickle files in the working directory:

• the keys-<uid>-<day>.pkl pickle holds the inserted keys on a given day
• the results-<day>.pkl pickle holds the fetch results for a given day

Here <uid> is the self-assigned unique ID, <day> the number of days passed since epoch (January 1st 1970).

In addition, it outputs the results in human-readable form to the standard output. You are advised to output this to some sort of log file for future reference.

fetchpull.py

#!/usr/bin/python2

import datetime
import math
import pickle
import random
import time

import fcp

# Set this to a unique ID string yourself!
uid = None

kskbase = 'KSK@fetchpull'

today = (datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).days

deltas = 8

def picklefile(day):
    return "keys-" + uid + "-" + str(day) + ".pkl"

def keystofetch(delta):
    try:
        with open(picklefile(today - pow(2, delta)), "rb") as f:
            return pickle.load(f)[delta]
    except:
        return []

def keystoinsert(delta, n):
    return ['-'.join([kskbase, uid, str(today), str(delta), str(i)]) for i in range(n)]

def insertkeys(n=100, timeout=7200):
    node = fcp.FCPNode()
    starttime = time.time()
    keys = []
    for delta in range(deltas):
        for key in keystoinsert(delta, n):
            keys.append((key, delta))
    random.shuffle(keys)
    tickets = []
    for key, delta in keys:
        tickets.append((node.put(uri=key, data='', nocompress=True, async=True), delta))
    success = { delta: [] for delta in range(deltas) }
    for ticket, delta in tickets:
        while not ticket.isComplete() and time.time() < starttime + timeout:
            time.sleep(1)
        try:
            resultkey = ticket.getResult()
            if resultkey is not None:
                success[delta].append(resultkey)
        except:
            pass
    with open(picklefile(today), 'wb') as f:
        pickle.dump(success, f)
    totalsuccess = 0
    for l in success.values():
        totalsuccess = totalsuccess + len(l)
    print "{pct}% of inserts succeeded".format(pct=round(float(totalsuccess) / float(len(keys)) * 100.0, 2))
    node.shutdown()

def fetchkeys(timeout=3600):
    node = fcp.FCPNode()
    starttime = time.time()
    keys = []
    totals = { delta: 0 for delta in range(deltas) }
    for delta in range(deltas):
        for key in keystofetch(delta):
            totals[delta] = totals[delta] + 1
            keys.append((key, delta))
    random.shuffle(keys)
    tickets = []
    for key, delta in keys:
        tickets.append((node.get(uri=key, nodata=True, ignoreds=True, maxretries=0, async=True), delta))
    success = { delta: 0 for delta in range(deltas) }
    for ticket, delta in tickets:
        while not ticket.isComplete() and time.time() < starttime + timeout:
            time.sleep(1)
        try:
            if ticket.getResult() is not None:
                success[delta] = success[delta] + 1
        except:
            pass
    print "Fetch results:"
    for delta in range(deltas):
        print str(pow(2, delta)) + ": " + str(success[delta]) + " / " + str(totals[delta])
    with open("results-" + str(today) + ".pkl", 'wb') as f:
        pickle.dump({"success": success, "totals": totals}, f)
    node.shutdown()

print "=== " + str(today) + " ==="
t1 = time.time()
insertkeys()
t2 = time.time()
print "Inserting took " + str(round(t2 - t1)) + " seconds"
time.sleep(60)
t3 = time.time()
fetchkeys()
t4 = time.time()
print "Fetching took " + str(round(t4 - t3)) + " seconds"