Skip to content

Instantly share code, notes, and snippets.

@jkcgs
Last active July 20, 2018 20:31
Show Gist options
  • Save jkcgs/1456c1461bccfe27e679968c81b65019 to your computer and use it in GitHub Desktop.
Save jkcgs/1456c1461bccfe27e679968c81b65019 to your computer and use it in GitHub Desktop.
Mini API to generate basic mirror.ufro.cl mirror status, using Flask
#!/usr/bin/env python3
import os
import subprocess
from datetime import datetime
from util import reverse_readline, sizeof_fmt, isint
from flask import Flask, jsonify
app = Flask(__name__)
mirror_path = '/srv/mirror'
default_logdir = '/var/log/mirror'
special_logdirs = {
'archlinux': '@/arch.log',
'debian': '/opt/ftpsync/log/rsync-ftpsync.log',
'debian-cd': '@/debiancd.log',
'linuxmint': '@/mint.log',
'linuxmint-cd': '@/mintcd.log',
'ubuntu-releases': '@/ubunturelease.log'
}
@app.route('/status')
def index():
# Measure generation time
start_time = datetime.now()
# Retrieve all mirror directories
mirror_files = os.listdir(mirror_path)
dirs = [d for d in mirror_files if (os.path.isdir(os.path.join(mirror_path, d)) and not d.startswith('.'))]
resp = {d: {} for d in dirs}
for process in dirs:
# Disk usage
usage_path = os.path.join(mirror_path, process + '.usage.txt')
resp[process]['usage'] = None
resp[process]['usageHuman'] = None
if os.path.exists(usage_path):
with open(usage_path, 'r') as usage_file:
# Usage file contains output from du -h <mirror_path>
# Format: <bytes>\t<path>
cont = usage_file.read()
val = cont.split('\t')[0]
if isint(val):
val = int(val)
resp[process]['usage'] = val
resp[process]['usageHuman'] = sizeof_fmt(val)
resp[process]['lastUpdate'] = str(datetime.fromtimestamp(os.path.getmtime(usage_path)))
# Process status
logpath = ''
# Resolve log path
if process in special_logdirs:
logpath = special_logdirs[process].replace('@', default_logdir)
else:
logpath = default_logdir + '/' + process + '.log'
# Try <path>.0, default value will be "idk"
resp[process]['status'] = 'idk'
if not os.path.exists(logpath):
if os.path.exists(logpath + '.0'):
logpath += '.0'
else:
continue
lines = reverse_readline(logpath)
line = next(lines)
if 'Waiting 4h' in line or 'total size is' in line:
resp[process]['status'] = 'ready'
continue
# Seek for progress
resp[process]['status'] = line
i = 50
# Check last 50 lines
while i > 0:
if 'ir-chk' in line:
cont_start = line.find('ir-chk') + 7
cont_line = line[cont_start:-1].split('/')
if len(cont_line) != 2 or not isint(cont_line[0]) or not isint(cont_line[1]):
resp[process]['status'] = 'idk'
else:
total = int(cont_line[1])
remaining = int(cont_line[0])
resp[process]['status'] = (total - remaining) / total
resp[process]['total'] = total
resp[process]['remaining'] = remaining
break
i -= 1
line = next(lines)
# Retrieve disk usage info
df = subprocess.run(['df', '-B1', mirror_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
df = [x for x in df.stdout.decode('utf-8').split('\n')[1].replace(' ', ' ').split(' ') if x != '']
# Measure execution time
finish_time = datetime.now()
delta = finish_time - start_time
return jsonify({
'mirrors': resp,
'disk': {
'total': int(df[1]),
'used': int(df[2]),
'free': int(df[3]),
'totalHuman': sizeof_fmt(int(df[1])),
'usedHuman': sizeof_fmt(int(df[2])),
'freeHuman': sizeof_fmt(int(df[3]))
},
'_ms': delta.total_seconds() * 1000
})
import os
def reverse_readline(filename, buf_size=8192):
# https://stackoverflow.com/a/23646049
"""a generator that returns the lines of a file in reverse order"""
with open(filename) as fh:
segment = None
offset = 0
fh.seek(0, os.SEEK_END)
file_size = remaining_size = fh.tell()
while remaining_size > 0:
offset = min(file_size, offset + buf_size)
fh.seek(file_size - offset)
buffer = fh.read(min(remaining_size, buf_size))
remaining_size -= buf_size
lines = buffer.split('\n')
# the first line of the buffer is probably not a complete line so
# we'll save it and append it to the last line of the next buffer
# we read
if segment is not None:
# if the previous chunk starts right from the beginning of line
# do not concact the segment to the last line of new chunk
# instead, yield the segment first
if buffer[-1] is not '\n':
lines[-1] += segment
else:
yield segment
segment = lines[0]
for index in range(len(lines) - 1, 0, -1):
if len(lines[index]):
yield lines[index]
# Don't yield None if the file was empty
if segment is not None:
yield segment
def sizeof_fmt(num, suffix='B'):
# https://stackoverflow.com/a/1094933
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
if abs(num) < 1024.0:
return "%3.2f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.2f%s%s" % (num, 'Yi', suffix)
def isint(val):
try:
int(val)
return True
except ValueError:
return False
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment