sutaburosu/sexta.py

## sexta.py
#!/usr/bin/env python3
'''
## Installation

# Windows

Install [Python 3](https://www.python.org/downloads/), then continue with the
[All operating systems](#allOS) section.

# Gentoo Linux

`emerge -av flask colorama pyelftools crossdev`

Then follow the
[Gentoo wiki crossdev instructions](https://wiki.gentoo.org/wiki/Arduino#Recommended:_Install_the_toolchain_using_crossdev)
to create an AVR profile. I built mine with these options:

`time USE="-gold lto pie" nice -n19 crossdev --stage4 --stable --binutils '~2.34' --gcc '~10.2.0' --target avr`

This builds compilers for *every* AVR architecture. It took about 6 hours in a single core 2GB VM on a 2.1 GHz Xeon Skylake.

# other Linux

Python 3 and pip are probably installed already, so continue with the next section.

# <a name='allOS'></a>All operating systems

Download [arduino-cli](https://arduino.github.io/arduino-cli/latest/installation/#latest-packages)
Extract it into the same folder as this script, then:

`./arduino-cli update`
`./arduino-cli core install arduino:avr arduino:megaavr`
`./arduino-cli lib install "FastLED" "Servo" "FastLED NeoMatrix" "Adafruit NeoMatrix" "Adafruit SSD1306" "DHT Sensor Library" "Adafruit GFX Library" "Framebuffer GFX"`
`pip3 install flask colorama pyelftools`

# Start

To run it locally:

`python3 sexta.py`

Or you can run it in a WSGI server like Apache + mod_wsgi or Nginx + uWSGI.
'''

import flask
import os
import platform
import re
import subprocess
import tempfile
import traceback
from flask import request, jsonify
from pprint import pprint
from sys import stderr

try:
    from elftools.elf.elffile import ELFFile
    from elftools.elf.sections import SymbolTableSection
    from elftools.dwarf.lineprogram import LineProgram
    from elftools.dwarf.constants import (DW_LNS_copy, DW_LNS_set_file, DW_LNE_define_file)
    # from elftools.dwarf.descriptions import (describe_DWARF_expr, set_global_machine_arch)
    # from elftools.dwarf.locationlists import (LocationEntry, LocationExpr, LocationParser)
except ModuleNotFoundError:
    pprint("'elftools' not found. Line numbers and "
            "symbol addresses WILL NOT be generated.", file=stderr)
    ELFFile = None

SCRIPTNAME = os.path.splitext(os.path.basename(__file__))[0]
SCRIPTFOLDER = os.path.dirname(os.path.realpath(__file__))
ARDUINO_CLI_PATH = os.path.join(SCRIPTFOLDER, 'arduino-cli')
if platform.system() == 'Windows':
    ARDUINO_CLI_PATH += '.exe'

app = flask.Flask(SCRIPTNAME)
app.config["DEBUG"] = True  # TODO

def get_include_paths():
    # get gcc include paths
    subp = subprocess.Popen(
        ['avr-gcc', '-E', '-Wp,-v', '-'],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    stdout, stderr = subp.communicate(b'\r\n')
    includes = []
    in_inc = False
    for line in stderr.decode('utf-8').splitlines():
        if not in_inc:
            m = re.match(r'^\s*#include (\"|\')\.\.\.(\"|\') search starts here:\s*$', line)
            if m is not None:
                in_inc = True
            continue
        # print(line)
        if re.match(r'^End of search list\.$', line):
            in_inc = False
            continue
        m = re.match(r'^\s+(.*)$', line)
        if m is not None:
            includes.append(m[1])
    return includes

def get_version(cmd):
    if cmd in ('avr-gcc', 'avr-ld'):
        subp = subprocess.run([cmd, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if subp.returncode != 0:
            pprint(subp, file=stderr)
            return cmd + ' ???'
        return subp.stdout.decode('utf-8').splitlines()[0]
    elif cmd == 'avr-libc':
        gcc_inc_paths = get_include_paths()
        # find a file called */avr/version.h
        for incpath in gcc_inc_paths:
            v_fn = os.path.join(incpath, 'avr', 'version.h')
            if os.path.exists(v_fn):
                with open(v_fn) as version_h:
                    for line in version_h:
                        vm = re.match(r'^\s*#define\s+__AVR_LIBC_VERSION_STRING__\s+\"(.+)\"\s+$', line)
                        if vm is not None:
                            return 'avr-libc ' + vm[1]
        return 'avr-libc ???'

def get_elf_info(genelf, sketchDir, sketchFiles):
    with open (genelf, 'rb') as f:
        elffile = ELFFile(f)
        if not elffile.has_dwarf_info():
            return {}, {}
        symtab = [s for s in elffile.iter_sections() if isinstance(s, SymbolTableSection)]
        symbols = {}
        for section in symtab:
            for sym in section.iter_symbols():
                if sym.name == '':
                    continue
                if sym.name not in ('__brkval', '_end', '__stack', 'leds'):
                    continue
                type = sym['st_info']['type'][4:]   # trim the STT_ prefix
                addr = sym['st_value'] & ~(1 << 23) # remove 0x800000 Flash offset if present
                symbols[sym.name] = { 'addr': '%x' % addr, 'size': '%x' % sym['st_size'], 'type': type}

        dwarfinfo = elffile.get_dwarf_info()
        lineInfo = {}
        for cu in dwarfinfo.iter_CUs():
            lineprogram = dwarfinfo.line_program_for_CU(cu)
            for fileentry in lineprogram['file_entry']:
                filename = fileentry.name.decode('utf-8')
                folder = lineprogram['include_directory'][fileentry.dir_index - 1].decode('utf-8')
                thisfn = filename

                for entry in lineprogram.get_entries():
                    state = entry.state
                    if entry.state is None:
                        if entry.command in (DW_LNS_set_file, DW_LNE_define_file, DW_LNS_copy):
                            fentry = lineprogram['file_entry'][entry.args[0] - 1]
                            thisfn = fentry.name.decode('utf-8')
                            # thisfn = filename + '(' + fentry.name.decode('utf-8') + ')'
                            # print("fentry.name", filename)
                        continue
                    elif entry.state.end_sequence:
                        thisfn = filename
                        continue
                    # filter out files that weren't supplied in the request
                    if not any([thisfn == file['name'] for file in sketchFiles]):
                        continue
                    if thisfn not in lineInfo:
                        lineInfo[thisfn] = {}
                    lineInfo[thisfn]['%x' % entry.state.address] = entry.state.line
        return lineInfo, symbols

def make_build(board, sketchDir, sketchFiles, sketchName):
    main = sketchName + '.ino'

    # merge all source files
    merged = ''
    for file in sketchFiles:
        if file['name'] == main:
            header_included = re.match(
                r'^\s*#\s*include\s+<\s*Arduino\.h\s*>',
                file['content'])
            if header_included is not None:
                merged += '#include <Arduino.h>\n\n'
            merged += file['content'] + '\n\n// ' + '-' * 72 + '\n\n'

    for file in sketchFiles:
        if file['name'] != main:
            merged += file['content'] + '\n\n// ' + '-' * 72 + '\n\n'

    with open(os.path.join(sketchDir, main), 'w') as outf:
        outf.write(merged)

    dummy = [{'name': main, 'content': merged}]
    env = {'ARDUINO_SKETCHBOOK_DIR': sketchDir,
            'ARDUINO_DOWNLOADS_DIR': os.path.join(SCRIPTFOLDER, '.arduino15b', 'staging'),
            'ARDUINO_DATA_DIR': os.path.join(SCRIPTFOLDER, '.arduino15b')}
    return arduino_build(board, sketchDir, dummy, sketchName, env)

def arduino_build(board, sketchDir, sketchFiles, sketchName, envVars={}):
    buildDir = os.path.join(sketchDir, 'build')
    outputDir = os.path.join(sketchDir, 'output')
    # artefactDir = os.path.join(outputDir, board.replace(':', '.'))
    artefactDir = outputDir
    resp = {}

    # write out files
    for file in sketchFiles:
        with open(os.path.join(sketchDir, file['name']), 'w') as outf:
            outf.write(file['content'])

    env = dict(os.environ)
    env.update(envVars)
    cmd = [ARDUINO_CLI_PATH, 'compile', '-b', board, '--warnings', 'all',
        '--build-path', buildDir, '--output-dir', outputDir, sketchDir]
    comp = subprocess.run(cmd, cwd=sketchDir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    resp['stdout'], resp['stderr'] = comp.stdout.decode('utf-8'), comp.stderr.decode('utf-8')
    if comp.returncode != 0:
        return resp

    outprefix = os.path.join(artefactDir, sketchName + '.ino')
    if ELFFile is None:
        resp['lineInfo'] = { sketchName + '.ino': {} }
        resp['symbols'] = {}
    else:
        resp['lineInfo'], resp['symbols'] = get_elf_info(outprefix + '.elf', sketchDir, sketchFiles)

    with open(outprefix + '.hex', 'rb') as hex_file:
        resp['hex'] = hex_file.read().decode('utf-8')
    with open(outprefix + '.eep', 'rb') as eep_file:
        resp['eep'] = eep_file.read().decode('utf-8')
    return resp

try:
    versions = {
        'gcc': get_version('avr-gcc'),
        'binutils': get_version('avr-ld'),
        'libc': get_version('avr-libc'),
    }
except FileNotFoundError:
    versions = {
        'gcc': '???',
        'binutils': '???',
        'libc': '???',
    }

@app.route('/', methods=['GET'])
def home():
    return "<h1>%s build server</h1>" \
        "<p>You're very <!--expletive deleted--> inquisitive, aren't you. " \
        "Perchance this pertains to your purpose:</p>" \
        "<list><li>%s</li><li>%s</li><li>%s</li></list>" \
        % (SCRIPTNAME, versions['gcc'], versions['binutils'], versions['libc'])

@app.route('/build', methods=['POST'])
def build():
    req = request.get_json()

    sketchFiles = []
    if 'files' in req:
        sketchFiles = req['files']

    if 'sketch' in req:
        sketchFiles.append({'name': 'sketch.ino', 'content': req['sketch']})
        sketchName = 'sketch'
    else:
        if 'sketchName' in req:
            sketchName = req['sketchName']
        else:
            return {'oof': 'No sketch'}, 406, {'ContentType': 'application/json'}

    board = 'uno'
    if 'board' in req:
        board = req['board']
        if board not in ('uno', 'mega', 'nano'):
            return {'oof': 'unsupported MCU'}, 501, {'ContentType': 'application/json'}
    board = 'arduino:avr:' + board

    # try to reject dangerous filenames
    for file in sketchFiles:
        if any([x in file['name'] for x in
            ("~", "|", "\\", "/", "..", "%", "$", "{", "[", "`", "\"", "'", "?", ">", "<", "&")
        ]):
            return {'oof': 'you devil!'}, 406, {'ContentType': 'application/json'}

    # create unique temp folder
    tmpdir = tempfile.TemporaryDirectory(prefix=SCRIPTNAME + '-')
    sketchDir = os.path.join(tmpdir.name, sketchName)
    os.mkdir(sketchDir)

    # compile
    try:
        if 'compiler' in req and req['compiler'] == 'make':
            resp = make_build(board, sketchDir, sketchFiles, sketchName)
        else:
            resp = arduino_build(board, sketchDir, sketchFiles, sketchName)
    except Exception:
        app.logger.error(traceback.format_exc())
        return {'oof': 'idk'}, 500
    return resp, 200, {'ContentType': 'application/json'}

if __name__ == '__main__':
    app.run(host='localhost', port='16666')
	#!/usr/bin/env python3
	'''
	## Installation

	# Windows

	Install [Python 3](https://www.python.org/downloads/), then continue with the
	[All operating systems](#allOS) section.

	# Gentoo Linux

	`emerge -av flask colorama pyelftools crossdev`

	Then follow the
	[Gentoo wiki crossdev instructions](https://wiki.gentoo.org/wiki/Arduino#Recommended:_Install_the_toolchain_using_crossdev)
	to create an AVR profile. I built mine with these options:

	`time USE="-gold lto pie" nice -n19 crossdev --stage4 --stable --binutils '~2.34' --gcc '~10.2.0' --target avr`

	This builds compilers for every AVR architecture. It took about 6 hours in a single core 2GB VM on a 2.1 GHz Xeon Skylake.

	# other Linux

	Python 3 and pip are probably installed already, so continue with the next section.

	# <a name='allOS'></a>All operating systems

	Download [arduino-cli](https://arduino.github.io/arduino-cli/latest/installation/#latest-packages)
	Extract it into the same folder as this script, then:

	`./arduino-cli update`
	`./arduino-cli core install arduino:avr arduino:megaavr`
	`./arduino-cli lib install "FastLED" "Servo" "FastLED NeoMatrix" "Adafruit NeoMatrix" "Adafruit SSD1306" "DHT Sensor Library" "Adafruit GFX Library" "Framebuffer GFX"`
	`pip3 install flask colorama pyelftools`

	# Start

	To run it locally:

	`python3 sexta.py`

	Or you can run it in a WSGI server like Apache + mod_wsgi or Nginx + uWSGI.
	'''

	import flask
	import os
	import platform
	import re
	import subprocess
	import tempfile
	import traceback
	from flask import request, jsonify
	from pprint import pprint
	from sys import stderr

	try:
	from elftools.elf.elffile import ELFFile
	from elftools.elf.sections import SymbolTableSection
	from elftools.dwarf.lineprogram import LineProgram
	from elftools.dwarf.constants import (DW_LNS_copy, DW_LNS_set_file, DW_LNE_define_file)
	# from elftools.dwarf.descriptions import (describe_DWARF_expr, set_global_machine_arch)
	# from elftools.dwarf.locationlists import (LocationEntry, LocationExpr, LocationParser)
	except ModuleNotFoundError:
	pprint("'elftools' not found. Line numbers and "
	"symbol addresses WILL NOT be generated.", file=stderr)
	ELFFile = None

	SCRIPTNAME = os.path.splitext(os.path.basename(__file__))[0]
	SCRIPTFOLDER = os.path.dirname(os.path.realpath(__file__))
	ARDUINO_CLI_PATH = os.path.join(SCRIPTFOLDER, 'arduino-cli')
	if platform.system() == 'Windows':
	ARDUINO_CLI_PATH += '.exe'

	app = flask.Flask(SCRIPTNAME)
	app.config["DEBUG"] = True # TODO

	def get_include_paths():
	# get gcc include paths
	subp = subprocess.Popen(
	['avr-gcc', '-E', '-Wp,-v', '-'],
	stdin=subprocess.PIPE,
	stdout=subprocess.PIPE,
	stderr=subprocess.PIPE)
	stdout, stderr = subp.communicate(b'\r\n')
	includes = []
	in_inc = False
	for line in stderr.decode('utf-8').splitlines():
	if not in_inc:
	m = re.match(r'^\s#include (\"\|\')\.\.\.(\"\|\') search starts here:\s$', line)
	if m is not None:
	in_inc = True
	continue
	# print(line)
	if re.match(r'^End of search list\.$', line):
	in_inc = False
	continue
	m = re.match(r'^\s+(.*)$', line)
	if m is not None:
	includes.append(m[1])
	return includes

	def get_version(cmd):
	if cmd in ('avr-gcc', 'avr-ld'):
	subp = subprocess.run([cmd, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	if subp.returncode != 0:
	pprint(subp, file=stderr)
	return cmd + ' ???'
	return subp.stdout.decode('utf-8').splitlines()[0]
	elif cmd == 'avr-libc':
	gcc_inc_paths = get_include_paths()
	# find a file called */avr/version.h
	for incpath in gcc_inc_paths:
	v_fn = os.path.join(incpath, 'avr', 'version.h')
	if os.path.exists(v_fn):
	with open(v_fn) as version_h:
	for line in version_h:
	vm = re.match(r'^\s*#define\s+__AVR_LIBC_VERSION_STRING__\s+\"(.+)\"\s+$', line)
	if vm is not None:
	return 'avr-libc ' + vm[1]
	return 'avr-libc ???'

	def get_elf_info(genelf, sketchDir, sketchFiles):
	with open (genelf, 'rb') as f:
	elffile = ELFFile(f)
	if not elffile.has_dwarf_info():
	return {}, {}
	symtab = [s for s in elffile.iter_sections() if isinstance(s, SymbolTableSection)]
	symbols = {}
	for section in symtab:
	for sym in section.iter_symbols():
	if sym.name == '':
	continue
	if sym.name not in ('__brkval', '_end', '__stack', 'leds'):
	continue
	type = sym['st_info']['type'][4:] # trim the STT_ prefix
	addr = sym['st_value'] & ~(1 << 23) # remove 0x800000 Flash offset if present
	symbols[sym.name] = { 'addr': '%x' % addr, 'size': '%x' % sym['st_size'], 'type': type}

	dwarfinfo = elffile.get_dwarf_info()
	lineInfo = {}
	for cu in dwarfinfo.iter_CUs():
	lineprogram = dwarfinfo.line_program_for_CU(cu)
	for fileentry in lineprogram['file_entry']:
	filename = fileentry.name.decode('utf-8')
	folder = lineprogram['include_directory'][fileentry.dir_index - 1].decode('utf-8')
	thisfn = filename

	for entry in lineprogram.get_entries():
	state = entry.state
	if entry.state is None:
	if entry.command in (DW_LNS_set_file, DW_LNE_define_file, DW_LNS_copy):
	fentry = lineprogram['file_entry'][entry.args[0] - 1]
	thisfn = fentry.name.decode('utf-8')
	# thisfn = filename + '(' + fentry.name.decode('utf-8') + ')'
	# print("fentry.name", filename)
	continue
	elif entry.state.end_sequence:
	thisfn = filename
	continue
	# filter out files that weren't supplied in the request
	if not any([thisfn == file['name'] for file in sketchFiles]):
	continue
	if thisfn not in lineInfo:
	lineInfo[thisfn] = {}
	lineInfo[thisfn]['%x' % entry.state.address] = entry.state.line
	return lineInfo, symbols

	def make_build(board, sketchDir, sketchFiles, sketchName):
	main = sketchName + '.ino'

	# merge all source files
	merged = ''
	for file in sketchFiles:
	if file['name'] == main:
	header_included = re.match(
	r'^\s#\sinclude\s+<\sArduino\.h\s>',
	file['content'])
	if header_included is not None:
	merged += '#include <Arduino.h>\n\n'
	merged += file['content'] + '\n\n// ' + '-' * 72 + '\n\n'

	for file in sketchFiles:
	if file['name'] != main:
	merged += file['content'] + '\n\n// ' + '-' * 72 + '\n\n'

	with open(os.path.join(sketchDir, main), 'w') as outf:
	outf.write(merged)

	dummy = [{'name': main, 'content': merged}]
	env = {'ARDUINO_SKETCHBOOK_DIR': sketchDir,
	'ARDUINO_DOWNLOADS_DIR': os.path.join(SCRIPTFOLDER, '.arduino15b', 'staging'),
	'ARDUINO_DATA_DIR': os.path.join(SCRIPTFOLDER, '.arduino15b')}
	return arduino_build(board, sketchDir, dummy, sketchName, env)

	def arduino_build(board, sketchDir, sketchFiles, sketchName, envVars={}):
	buildDir = os.path.join(sketchDir, 'build')
	outputDir = os.path.join(sketchDir, 'output')
	# artefactDir = os.path.join(outputDir, board.replace(':', '.'))
	artefactDir = outputDir
	resp = {}

	# write out files
	for file in sketchFiles:
	with open(os.path.join(sketchDir, file['name']), 'w') as outf:
	outf.write(file['content'])

	env = dict(os.environ)
	env.update(envVars)
	cmd = [ARDUINO_CLI_PATH, 'compile', '-b', board, '--warnings', 'all',
	'--build-path', buildDir, '--output-dir', outputDir, sketchDir]
	comp = subprocess.run(cmd, cwd=sketchDir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	resp['stdout'], resp['stderr'] = comp.stdout.decode('utf-8'), comp.stderr.decode('utf-8')
	if comp.returncode != 0:
	return resp

	outprefix = os.path.join(artefactDir, sketchName + '.ino')
	if ELFFile is None:
	resp['lineInfo'] = { sketchName + '.ino': {} }
	resp['symbols'] = {}
	else:
	resp['lineInfo'], resp['symbols'] = get_elf_info(outprefix + '.elf', sketchDir, sketchFiles)

	with open(outprefix + '.hex', 'rb') as hex_file:
	resp['hex'] = hex_file.read().decode('utf-8')
	with open(outprefix + '.eep', 'rb') as eep_file:
	resp['eep'] = eep_file.read().decode('utf-8')
	return resp

	try:
	versions = {
	'gcc': get_version('avr-gcc'),
	'binutils': get_version('avr-ld'),
	'libc': get_version('avr-libc'),
	}
	except FileNotFoundError:
	versions = {
	'gcc': '???',
	'binutils': '???',
	'libc': '???',
	}

	@app.route('/', methods=['GET'])
	def home():
	return "<h1>%s build server</h1>" \
	"<p>You're very <!--expletive deleted--> inquisitive, aren't you. " \
	"Perchance this pertains to your purpose:</p>" \
	"<list><li>%s</li><li>%s</li><li>%s</li></list>" \
	% (SCRIPTNAME, versions['gcc'], versions['binutils'], versions['libc'])

	@app.route('/build', methods=['POST'])
	def build():
	req = request.get_json()

	sketchFiles = []
	if 'files' in req:
	sketchFiles = req['files']

	if 'sketch' in req:
	sketchFiles.append({'name': 'sketch.ino', 'content': req['sketch']})
	sketchName = 'sketch'
	else:
	if 'sketchName' in req:
	sketchName = req['sketchName']
	else:
	return {'oof': 'No sketch'}, 406, {'ContentType': 'application/json'}

	board = 'uno'
	if 'board' in req:
	board = req['board']
	if board not in ('uno', 'mega', 'nano'):
	return {'oof': 'unsupported MCU'}, 501, {'ContentType': 'application/json'}
	board = 'arduino:avr:' + board

	# try to reject dangerous filenames
	for file in sketchFiles:
	if any([x in file['name'] for x in
	("~", "\|", "\\", "/", "..", "%", "$", "{", "[", "`", "\"", "'", "?", ">", "<", "&")
	]):
	return {'oof': 'you devil!'}, 406, {'ContentType': 'application/json'}

	# create unique temp folder
	tmpdir = tempfile.TemporaryDirectory(prefix=SCRIPTNAME + '-')
	sketchDir = os.path.join(tmpdir.name, sketchName)
	os.mkdir(sketchDir)

	# compile
	try:
	if 'compiler' in req and req['compiler'] == 'make':
	resp = make_build(board, sketchDir, sketchFiles, sketchName)
	else:
	resp = arduino_build(board, sketchDir, sketchFiles, sketchName)
	except Exception:
	app.logger.error(traceback.format_exc())
	return {'oof': 'idk'}, 500
	return resp, 200, {'ContentType': 'application/json'}

	if __name__ == '__main__':
	app.run(host='localhost', port='16666')