Last active
July 20, 2018 20:31
-
-
Save jkcgs/1456c1461bccfe27e679968c81b65019 to your computer and use it in GitHub Desktop.
Mini API to generate basic mirror.ufro.cl mirror status, using Flask
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
import os | |
import subprocess | |
from datetime import datetime | |
from util import reverse_readline, sizeof_fmt, isint | |
from flask import Flask, jsonify | |
app = Flask(__name__) | |
mirror_path = '/srv/mirror' | |
default_logdir = '/var/log/mirror' | |
special_logdirs = { | |
'archlinux': '@/arch.log', | |
'debian': '/opt/ftpsync/log/rsync-ftpsync.log', | |
'debian-cd': '@/debiancd.log', | |
'linuxmint': '@/mint.log', | |
'linuxmint-cd': '@/mintcd.log', | |
'ubuntu-releases': '@/ubunturelease.log' | |
} | |
@app.route('/status') | |
def index(): | |
# Measure generation time | |
start_time = datetime.now() | |
# Retrieve all mirror directories | |
mirror_files = os.listdir(mirror_path) | |
dirs = [d for d in mirror_files if (os.path.isdir(os.path.join(mirror_path, d)) and not d.startswith('.'))] | |
resp = {d: {} for d in dirs} | |
for process in dirs: | |
# Disk usage | |
usage_path = os.path.join(mirror_path, process + '.usage.txt') | |
resp[process]['usage'] = None | |
resp[process]['usageHuman'] = None | |
if os.path.exists(usage_path): | |
with open(usage_path, 'r') as usage_file: | |
# Usage file contains output from du -h <mirror_path> | |
# Format: <bytes>\t<path> | |
cont = usage_file.read() | |
val = cont.split('\t')[0] | |
if isint(val): | |
val = int(val) | |
resp[process]['usage'] = val | |
resp[process]['usageHuman'] = sizeof_fmt(val) | |
resp[process]['lastUpdate'] = str(datetime.fromtimestamp(os.path.getmtime(usage_path))) | |
# Process status | |
logpath = '' | |
# Resolve log path | |
if process in special_logdirs: | |
logpath = special_logdirs[process].replace('@', default_logdir) | |
else: | |
logpath = default_logdir + '/' + process + '.log' | |
# Try <path>.0, default value will be "idk" | |
resp[process]['status'] = 'idk' | |
if not os.path.exists(logpath): | |
if os.path.exists(logpath + '.0'): | |
logpath += '.0' | |
else: | |
continue | |
lines = reverse_readline(logpath) | |
line = next(lines) | |
if 'Waiting 4h' in line or 'total size is' in line: | |
resp[process]['status'] = 'ready' | |
continue | |
# Seek for progress | |
resp[process]['status'] = line | |
i = 50 | |
# Check last 50 lines | |
while i > 0: | |
if 'ir-chk' in line: | |
cont_start = line.find('ir-chk') + 7 | |
cont_line = line[cont_start:-1].split('/') | |
if len(cont_line) != 2 or not isint(cont_line[0]) or not isint(cont_line[1]): | |
resp[process]['status'] = 'idk' | |
else: | |
total = int(cont_line[1]) | |
remaining = int(cont_line[0]) | |
resp[process]['status'] = (total - remaining) / total | |
resp[process]['total'] = total | |
resp[process]['remaining'] = remaining | |
break | |
i -= 1 | |
line = next(lines) | |
# Retrieve disk usage info | |
df = subprocess.run(['df', '-B1', mirror_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
df = [x for x in df.stdout.decode('utf-8').split('\n')[1].replace(' ', ' ').split(' ') if x != ''] | |
# Measure execution time | |
finish_time = datetime.now() | |
delta = finish_time - start_time | |
return jsonify({ | |
'mirrors': resp, | |
'disk': { | |
'total': int(df[1]), | |
'used': int(df[2]), | |
'free': int(df[3]), | |
'totalHuman': sizeof_fmt(int(df[1])), | |
'usedHuman': sizeof_fmt(int(df[2])), | |
'freeHuman': sizeof_fmt(int(df[3])) | |
}, | |
'_ms': delta.total_seconds() * 1000 | |
}) | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
def reverse_readline(filename, buf_size=8192): | |
# https://stackoverflow.com/a/23646049 | |
"""a generator that returns the lines of a file in reverse order""" | |
with open(filename) as fh: | |
segment = None | |
offset = 0 | |
fh.seek(0, os.SEEK_END) | |
file_size = remaining_size = fh.tell() | |
while remaining_size > 0: | |
offset = min(file_size, offset + buf_size) | |
fh.seek(file_size - offset) | |
buffer = fh.read(min(remaining_size, buf_size)) | |
remaining_size -= buf_size | |
lines = buffer.split('\n') | |
# the first line of the buffer is probably not a complete line so | |
# we'll save it and append it to the last line of the next buffer | |
# we read | |
if segment is not None: | |
# if the previous chunk starts right from the beginning of line | |
# do not concact the segment to the last line of new chunk | |
# instead, yield the segment first | |
if buffer[-1] is not '\n': | |
lines[-1] += segment | |
else: | |
yield segment | |
segment = lines[0] | |
for index in range(len(lines) - 1, 0, -1): | |
if len(lines[index]): | |
yield lines[index] | |
# Don't yield None if the file was empty | |
if segment is not None: | |
yield segment | |
def sizeof_fmt(num, suffix='B'): | |
# https://stackoverflow.com/a/1094933 | |
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: | |
if abs(num) < 1024.0: | |
return "%3.2f%s%s" % (num, unit, suffix) | |
num /= 1024.0 | |
return "%.2f%s%s" % (num, 'Yi', suffix) | |
def isint(val): | |
try: | |
int(val) | |
return True | |
except ValueError: | |
return False |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment