-
-
Save ijgnd/17f3fdd2bf1e8457692f37b7ff01339a to your computer and use it in GitHub Desktop.
my build script
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
# for windows cmd: #! python3 and make sure it has a ".py" ending | |
""" | |
copy add-on from development folder to Anki and/or make .ankiaddon file that you can | |
share or upload to Ankiweb | |
For most of my Anki add-ons it's enough to copy the subfolder "src" from my repo into | |
your addon21 folder. | |
If the add-on also includes dialogs designed with QtDesigner it's more complicated: QtDesigner | |
creates ".ui" files. These are in the subfolder "designer" in my repo. My code assumes that the | |
compiled versions of these are in the add-on folder in a subfolder named "forms". So when you're | |
in the designer subfolder in my repo you need to run something like | |
for f in *.ui; do pyuic5 -o "addonprofile/addons21/myaddon/forms/${i%.*}.py" "${i}"; done | |
If the repo folder contains a folder named "ressources" you need to run pyrcc5 to build the | |
qt ressources files. Make sure that the "ressources" folder contains a ".qrc" file. This is used | |
for only very few of my addons. The qrc file will be copied into the body of the built add-on. | |
E.g. if you have the file "rc_icons.qrc" you can later use in your __init__.py | |
"from . import rc_icons" | |
An .ankiaddon file is a zipfile that contains a file named "manifest.json" that has the entries | |
"package", "name", "mod" (= epoch time). | |
To make copying easier I use a short python build script. I use only Anki 2.1 so this is 2.1 only. | |
This add-on reads a global config from Path.home()/.aam_anki_addon_make_config.json | |
If the folder of the add-on that is built contains a file .aam.json | |
this overrides global defaults. | |
If the user passes commandline parameters these override what's set in the files. | |
The add-on folder that is processed is determined by the argument --path/-p. | |
If this argument is missing the current folder is processed in Windows - in Linux I prefer | |
to get an error message. | |
I use this build file because when I made the first version glutanimate's aab wasn't | |
released and I know that this script works for me on Windows and Linux. | |
I mainly use a small subset of this build script: Copying to the dev folder set in the | |
global json file and building an ankiaddon file in the default location. The rest of | |
this add-on is hardly used and might not or no longer work. The first version of this file | |
is quite old and has seen many ad-hoc additions ... | |
Use this at your own risk. | |
This build script assumes that pyuic5 and pyrcc5 are on your path. | |
This build script assumes the following folder layout in the add-on folder: | |
minimal layout: | |
└── src | |
└── __init__.py | |
usual layout: | |
├── screenshots (optional; unprocessed. I sometimes put links to these into ankiweb.html) | |
├── designer (optional; processed by pyuic5) | |
│ └── dialog.ui | |
├── rc (optional; processed with pyrcc5) | |
│ ├── image.png | |
│ └── my.qrc (you must create this file yourself, eg with QtDesigner) | |
├── src | |
│ ├── libs (optional) | |
│ ├── icons (optional; unprocessed ones I just link to these from my code) | |
│ ├── __init__.py | |
│ ├── LICENSE | |
│ └── ... | |
├── .gitignore (opional) | |
├── readme.MD (opional) | |
├── checksums.csv (opional) | |
├── ankiweb.html (opional) | |
└── ... | |
If the repo folder contains a file named "checksums.csv" this script reads it and compares it with | |
the files in the subfolder src to avoid accidental changes in bundled dependencies. The format in | |
"checksums.csv" is "md5sum\tRelativePathToFileFromSrc". | |
.aam_anki_addon_make_config.json layout | |
{ | |
"profile_dev": "/home/user/.var/app/net.ankiweb.Anki/data/Anki2/addons21/", | |
"profile_main": "/home/user/path/to/Anki_profile_main/addons21/", | |
"ankiaddon_dest": "", | |
"ignore_files_pattern": [ | |
"*.pyc", | |
"*.vscode", | |
"*___bak_*", | |
"*___note*", | |
"*___nobundle*" | |
] | |
} | |
""" | |
import csv | |
import datetime | |
import hashlib | |
import json | |
import os | |
import pathlib | |
from pprint import pprint as pp | |
import random | |
import shutil | |
import subprocess | |
import sys | |
import tempfile | |
import threading | |
import time | |
import uuid | |
import zipfile | |
from argparse import ArgumentParser | |
from PyQt5.uic import compileUiDir as compileUiDir_5 | |
from PyQt6.uic import compileUiDir as compileUiDir_6 | |
def md5sum(path, blocksize=2**20): | |
# https://stackoverflow.com/questions/1131220/get-md5-hash-of-big-files-in-python | |
m = hashlib.md5() | |
with open(path, "rb") as f: | |
while True: | |
buf = f.read(blocksize) | |
if not buf: | |
break | |
m.update(buf) | |
return m.hexdigest() | |
def list_all_in_dir(folder, kind, prefix, myfilterfunc=lambda *_: True): | |
""" | |
return all from FOLDER of KIND ("files" or "dirs"). | |
to return absolute path set PREFIX to 'absolute', else to "relative" | |
MYFILTERFUNC only include file if this function returns true | |
""" | |
folder = os.path.abspath(folder) | |
li = [] | |
for root, dirs, files in os.walk(folder): | |
for f in eval(kind): | |
if myfilterfunc(f): | |
if prefix == 'absolute': | |
o=os.path.join(root,f) | |
li.append(o) | |
elif prefix == 'relative': | |
relDir = os.path.relpath(root, folder) | |
if not relDir == '.': | |
o = os.path.join(relDir, f) | |
else: | |
o = f | |
li.append(o) | |
elif prefix == 'rel_prefixed_with_base': | |
pass | |
return li | |
def exactly_one_true_in_list(li): | |
one_found = False | |
for e in li: | |
if e: | |
if one_found: | |
return False | |
else: | |
one_found = True | |
return one_found | |
def config_from_file(fi): | |
if not os.path.isfile(fi): | |
return | |
with open(fi, 'r') as f: | |
try: | |
config = json.loads(f.read()) | |
except: | |
print("error while reading config file '{}'. Aborting ...".format(fi)) | |
sys.exit() | |
else: | |
return config | |
def val_from_confs(val, local, glob, ignorefalse, checkpath): | |
# "ankiaddon_dest", locConfFi, gloConfFi | |
fromlocal = True | |
locconf = config_from_file(local) | |
out = False | |
if locconf: # False is the config file doesn't exist | |
out = locconf.get(val, False) | |
if not out: | |
fromlocal = False | |
globconf = config_from_file(glob) | |
if globconf: | |
out = globconf.get(val, False) | |
if not ignorefalse: | |
if not out: | |
print("no value found for {} in config files".format(val)) | |
sys.exit() | |
else: | |
if checkpath: | |
if not os.path.isdir(out): | |
source = local if fromlocal else glob | |
print(f"path '{out}' doesn't exist which is set as '{val}' in the " | |
f"file '{source}'. Aborting ...") | |
sys.exit() | |
return out | |
def parse_arguments(): | |
prs = ArgumentParser() | |
prs.add_argument("-p", "--path", dest="path", | |
help="path to the location where the development code/source repo is") | |
prs.add_argument("-m", "--meta", "--nometa", action="store_false", | |
help="don't copy meta.json to add-on folder (default TRUE)") | |
prs.add_argument("--action", "-a", dest="action", | |
help=('actions: "a" create ankiaddon file in --path, ' | |
'"d" copy/build to dev profile, ' | |
'"m" copy/build to main/regular profile.')) | |
prs.add_argument("--profile", dest="profile", | |
help=('override dest profile, may not be combined with "--action" or "--aadest"')) | |
prs.add_argument("--times", dest="times", | |
help=('override ctime, atime for files in ankiaddon, e.g. "2020,11,05,12,24,37"')) | |
prs.add_argument("--nosrc", dest="nosrc", action='store_true', | |
help=('copy from folder and not from src subfolder (for third party add-ons)')) | |
prs.add_argument("--aadest", dest="aadest", | |
help=('override --ankiaddon destination, may not be combined with ' | |
'"--action" or "--profile"')) | |
prs.add_argument("--no_qt6-just_forms", action="store_true", | |
help="do not build qt6 forms, use forms folder (and not forms5/forms6) (default FALSE)") | |
return prs.parse_args() | |
def check_for_illegal_combination_in_args(args): | |
if not any([args.action, args.profile, args.aadest]): | |
print('exactly one parameter of "--action", "--profile", "--aadest" is needed. Aborting ...') | |
sys.exit() | |
if not exactly_one_true_in_list([args.action, args.profile, args.aadest]): | |
print('only one parameter of "--action", "--profile", "--aadest" may be used. Aborting ...') | |
sys.exit() | |
if args.action not in ["a", "ankiaddon", "d", "dev", "m", "main", "regular"]: | |
print('unknown value for "--action". May only be "a" (ankiaddon), "d" (devprofile), ' | |
'"m" (mainprofile)') | |
sys.exit() | |
def adjust_args(args): | |
syn_addons = ["a", "ankiaddon",] | |
syn_dev = ["d", "dev",] | |
sync_main = ["m", "main", "regular"] | |
if args.action in syn_addons: | |
args.action = "a" | |
elif args.action in syn_dev: | |
args.action = "d" | |
elif args.action in sync_main: | |
args.action = "m" | |
if args.path and args.path != "." and not args.path.endswith(os.path.sep): | |
args.path = args.path + os.path.sep | |
def get_source_path(args): | |
if args.path: | |
path = args.path | |
if not os.path.isdir(args.path): | |
print("value for '--path' doesn't exist. Aborting ...") | |
sys.exit() | |
if not os.path.isabs(args.path): | |
print("value for '--path' must be an absolute paths. Aborting ...") | |
sys.exit() | |
else: | |
path = os.getcwd() | |
if not os.path.isdir(os.path.join(path, "src")): | |
print("The script is executed without a '--path' argument and uses your current " | |
"working dir\n '{}'\nwhich does not include a 'src' folder.\nAre you sure you are in " | |
"the right directory?\nAborting ...".format(path)) | |
sys.exit() | |
return path | |
def set_config_file_paths(): | |
global gloConfFi | |
global locConfFi | |
gloConfFi = os.path.join(pathlib.Path().home(), ".aam_anki_addon_make_config.json") | |
if not os.path.isfile(gloConfFi): | |
print(f"Info: config file '{gloConfFi}' doesn't exist.") | |
locConfFi = os.path.join(sourcepath, ".aam.json") | |
if not os.path.isfile(locConfFi): | |
print(f"Info: config file '{locConfFi}' doesn't exist.") | |
def get_dests_and_skip(args): | |
addondest = False | |
regdest = False | |
if args.profile: | |
regdest = args.profile | |
elif args.aadest: | |
addondest = args.aadest | |
else: | |
# TODO tidy up | |
if args.action == "a": | |
# can only be a folder since args.aadest is False BUT leave empty so that defaults to | |
# addonfolder | |
addondest = val_from_confs("ankiaddon_dest", locConfFi, gloConfFi, True, False) | |
if not addondest: | |
addondest = sourcepath | |
else: | |
if args.action == "d": | |
regdest = val_from_confs("profile_dev", locConfFi, gloConfFi, False, True) | |
elif args.action == "m": | |
regdest = val_from_confs("profile_main", locConfFi, gloConfFi, False, True) | |
skip = val_from_confs("ignore_files_pattern", locConfFi, gloConfFi, True, False) | |
return addondest, regdest, skip | |
def setup_adjust_commonly_used_global_variables(args): | |
global tmpdir | |
global manifest_in_temp | |
global meta_in_temp | |
global addonSrcFolderName | |
global SrcInSrcFolder | |
global UiInSrcFolder | |
global ResInSrcFolder | |
global addondest | |
tmpdir = tempfile.mkdtemp() | |
manifest_in_temp = os.path.join(tmpdir, 'manifest.json') | |
meta_in_temp = os.path.join(tmpdir, 'meta.json') | |
addonSrcFolderName = os.path.basename(os.path.dirname(sourcepath)) # used for filename | |
if args.nosrc: | |
SrcInSrcFolder = sourcepath | |
else: | |
if os.path.isdir(os.path.join(sourcepath, "src")): | |
SrcInSrcFolder = os.path.join(sourcepath, "src") | |
elif os.path.isdir(os.path.join(sourcepath, "addon")): | |
SrcInSrcFolder = os.path.join(sourcepath, "addon") | |
else: | |
print("no src or addon folder. Aborting ...") | |
sys.exit() | |
UiInSrcFolder = os.path.join(sourcepath, "designer") | |
ResInSrcFolder = os.path.join(sourcepath, "ressources") | |
if addondest: | |
if os.path.isdir(addondest): | |
if not args.times: | |
version = time.strftime('%Y-%m-%d_%H-%M') | |
else: | |
print(args.times) | |
try: | |
y,mo,day,h,mi,sec= [int(x) for x in args.times.split(",")] | |
except: | |
print("error while parsing args.times. Only use integers and split them with a comma like '2020,1,1,12,30'. Aborting...") | |
sys.exit() | |
new = datetime.datetime(y,mo,day,h,mi,sec) | |
version = new.strftime('%Y-%m-%d_%H-%M') | |
addondest = os.path.join(sourcepath, f"{addonSrcFolderName}__{version}.ankiaddon") | |
def copy_from_temp_to_profile(): | |
# read name of install folder from manifest.json if exists | |
if os.path.isfile(manifest_in_temp): | |
with open(manifest_in_temp, "r") as f: | |
d = json.load(f) | |
if d.get("package", False): | |
dest = os.path.join(regulardest, str(d["package"])) | |
else: | |
dest = os.path.join(regulardest, addonSrcFolderName) | |
else: | |
dest = os.path.join(regulardest, addonSrcFolderName) | |
if os.path.exists(dest): | |
shutil.rmtree(dest) | |
shutil.copytree(tmpdir, dest) | |
def if_manifestjson_insert_current_time_in_tmp(args): | |
if os.path.isfile(manifest_in_temp): | |
with open(manifest_in_temp, "r") as f: | |
d = json.load(f) | |
if args.times: | |
try: | |
y,mo,day,h,mi,sec= [int(x) for x in args.times.split(",")] | |
except: | |
print("error while parsing args.times. Only use integers and split them with a comma like '2020,1,1,12,30'. Aborting...") | |
sys.exit() | |
new = int(datetime.datetime(y,mo,day,h,mi,sec).timestamp()) | |
d['mod'] = int(new) | |
else: | |
d['mod'] = int(time.time()) | |
manifest = json.dumps(d) | |
with open(manifest_in_temp, 'w') as f: | |
f.write(manifest) | |
def metajson_with_filename_and_mod_from_manifestjson_in_tmp(): | |
if os.path.isfile(manifest_in_temp): | |
with open(manifest_in_temp, "r") as f: | |
j = json.load(f) | |
if os.path.isfile(meta_in_temp): | |
with open(meta_in_temp, "r") as f: | |
md = json.load(f) | |
else: | |
md = {} | |
if "name" not in md and "name" in j: | |
md['name'] = j['name'] | |
# why did I originally use if mod not in - this way Anki often wants to download an "update" from ankiweb | |
# if "mod" not in md and "mod" in j: | |
if "mod" in j: | |
md['mod'] = j['mod'] | |
with open(meta_in_temp, "w") as f: | |
f.write(json.dumps(md)) | |
def copy_to_temp_and_build_ui_and_ressources(args): | |
# shutil.copytreee doesn't work: "The destination directory, named by dst, must not already exist;", | |
# https://docs.python.org/3.7/library/shutil.html#shutil.copytree | |
# distutils.dir_util.copy_tree doesn't help since it doesn't have an ignore pattern, see | |
# https://docs.python.org/3.7/distutils/apiref.html#distutils.dir_util.copy_tree | |
if os.path.isdir(tmpdir): | |
os.rmdir(tmpdir) | |
if not os.path.isdir(SrcInSrcFolder): | |
print('src folder in --path is not a directory. Aborting ...') | |
sys.exit() | |
shutil.copytree(SrcInSrcFolder, tmpdir, ignore=shutil.ignore_patterns(*ignorewhencopying)) #'*.json', | |
#compile ui if necessary | |
if os.path.isdir(UiInSrcFolder): | |
def build_copy_ui(dest_folder, binary): | |
formsfolder = os.path.join(tmpdir, dest_folder) | |
pathlib.Path(formsfolder).mkdir(parents=True, exist_ok=True) | |
# pyuic5 writes the source path into the py file. I just want the relative locations | |
# so that the diff between versions is unchanged even if after reorganizing my dev folder | |
os.chdir(UiInSrcFolder) | |
relativesource = os.path.relpath(os.getcwd()) | |
binary(relativesource, recurse=False) | |
for f in list_all_in_dir(UiInSrcFolder, "files", "absolute", lambda f:f.endswith('.py')): | |
shutil.move(f, formsfolder) | |
if args.no_qt6_just_forms: | |
build_copy_ui("forms", compileUiDir_5) | |
else: | |
build_copy_ui("forms5", compileUiDir_5) | |
build_copy_ui("forms6", compileUiDir_6) | |
if os.path.isdir(ResInSrcFolder): | |
#res_in_temp = os.path.join(tmpdir, "ressources") | |
#pathlib.Path(res_in_temp).mkdir(parents=True, exist_ok=True) | |
for f in list_all_in_dir(ResInSrcFolder, "files", "relative", lambda f:f.endswith('.qrc')): | |
# aab also uses call_shell | |
# pyrcc5 resource_file.qrc -o icons.py | |
cmd = """cd "{ResInSrcFolder}"; pyrcc5 {input} -o {dest}""".format( | |
ResInSrcFolder=ResInSrcFolder, | |
input=f, | |
dest=os.path.join(tmpdir, os.path.splitext(f)[0] + ".py") | |
) | |
subprocess.run(cmd, shell=True, check=True) | |
if os.path.isfile(meta_in_temp): | |
if args.meta: | |
print('Info: meta.json copied from source') | |
else: | |
os.unlink(meta_in_temp) | |
print('Info: meta.json not copied because of -m/--meta argument') | |
def verify_files(relevant_files): | |
mismatched = [] | |
matched = [] | |
unchecked = [] | |
valfile = os.path.join(sourcepath, "checksums.csv") | |
if not os.path.exists(valfile): | |
print('\nno checksums file found\n') | |
else: | |
checksumsdict = {} | |
with open(valfile, "r") as f: | |
csv_reader = csv.reader(f, delimiter='\t') | |
for row in csv_reader: | |
try: | |
checksumsdict[row[1]] = row[0] | |
except: | |
pass | |
if checksumsdict: | |
for k in relevant_files: | |
if k in checksumsdict: | |
if md5sum(k) == checksumsdict[k]: | |
matched.append(k) | |
else: | |
mismatched.append(k) | |
else: | |
unchecked.append(k) | |
print('matched files: %i ' % len(matched)) | |
print('unchecked files: %i ' % len(unchecked)) | |
for f in unchecked: | |
print(' %s ' %f) | |
print('\n\n\n') | |
print('MISMATCHED files: %i ' % len(mismatched)) | |
for f in mismatched: | |
print(' %s ' %f) | |
def change_times(files, folder, args): | |
if not args.times: | |
# adjust to now | |
now = datetime.datetime.now() | |
new = int(now.timestamp()) | |
else: | |
try: | |
y,mo,day,h,mi,sec= [int(x) for x in args.times.split(",")] | |
except: | |
print("error while parsing args.times. Only use integers and split them with a comma like '2020,1,1,12,30,37'. Aborting...") | |
sys.exit() | |
new = int(datetime.datetime(y,mo,day,h,mi,sec).timestamp()) | |
for f in files: | |
os.utime(f, (new, new)) | |
# adjust folders | |
for d in list_all_in_dir(tmpdir, "dirs", "relative"): | |
os.utime(d, (new, new)) | |
def zip_tmpfolder(files): | |
if "meta.json" in files: | |
files.remove("meta.json") | |
with zipfile.ZipFile(addondest, 'w', zipfile.ZIP_DEFLATED) as z: | |
for f in files: | |
z.write(f) | |
#empty directories | |
for f in list_all_in_dir(tmpdir, "dirs", "relative"): | |
z.write(f) | |
def tmpfolder_cleanup(): | |
#shutil.rmtree(tmpdir) | |
pass | |
def create_ankiaddon(args): | |
copy_to_temp_and_build_ui_and_ressources(args) | |
if_manifestjson_insert_current_time_in_tmp(args) | |
# Todo: Adjust relevant_files | |
os.chdir(tmpdir) | |
relevant_files = list_all_in_dir(tmpdir, "files", "relative") | |
verify_files(relevant_files) | |
change_times(relevant_files, tmpdir, args) | |
zip_tmpfolder(relevant_files) | |
tmpfolder_cleanup() | |
def write_to_profile(args): | |
copy_to_temp_and_build_ui_and_ressources(args) | |
if_manifestjson_insert_current_time_in_tmp(args) | |
metajson_with_filename_and_mod_from_manifestjson_in_tmp() | |
copy_from_temp_to_profile() | |
tmpfolder_cleanup() | |
args = parse_arguments() | |
if args.times: | |
sec = random.randint(0,59) | |
if args.times.count(",") == 4: | |
args.times = args.times + f",{sec}" | |
check_for_illegal_combination_in_args(args) | |
adjust_args(args) | |
sourcepath = get_source_path(args) | |
set_config_file_paths() | |
addondest, regulardest, ignorewhencopying = get_dests_and_skip(args) | |
setup_adjust_commonly_used_global_variables(args) | |
if addondest: | |
create_ankiaddon(args) | |
else: | |
write_to_profile(args) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment