Skip to content

Instantly share code, notes, and snippets.

@ijgnd

ijgnd/aam.py Secret

Last active October 30, 2022 12:15
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ijgnd/17f3fdd2bf1e8457692f37b7ff01339a to your computer and use it in GitHub Desktop.
Save ijgnd/17f3fdd2bf1e8457692f37b7ff01339a to your computer and use it in GitHub Desktop.
my build script
#!/usr/bin/env python3
# for windows cmd: #! python3 and make sure it has a ".py" ending
"""
copy add-on from development folder to Anki and/or make .ankiaddon file that you can
share or upload to Ankiweb
For most of my Anki add-ons it's enough to copy the subfolder "src" from my repo into
your addon21 folder.
If the add-on also includes dialogs designed with QtDesigner it's more complicated: QtDesigner
creates ".ui" files. These are in the subfolder "designer" in my repo. My code assumes that the
compiled versions of these are in the add-on folder in a subfolder named "forms". So when you're
in the designer subfolder in my repo you need to run something like
for f in *.ui; do pyuic5 -o "addonprofile/addons21/myaddon/forms/${i%.*}.py" "${i}"; done
If the repo folder contains a folder named "ressources" you need to run pyrcc5 to build the
qt ressources files. Make sure that the "ressources" folder contains a ".qrc" file. This is used
for only very few of my addons. The qrc file will be copied into the body of the built add-on.
E.g. if you have the file "rc_icons.qrc" you can later use in your __init__.py
"from . import rc_icons"
An .ankiaddon file is a zipfile that contains a file named "manifest.json" that has the entries
"package", "name", "mod" (= epoch time).
To make copying easier I use a short python build script. I use only Anki 2.1 so this is 2.1 only.
This add-on reads a global config from Path.home()/.aam_anki_addon_make_config.json
If the folder of the add-on that is built contains a file .aam.json
this overrides global defaults.
If the user passes commandline parameters these override what's set in the files.
The add-on folder that is processed is determined by the argument --path/-p.
If this argument is missing the current folder is processed in Windows - in Linux I prefer
to get an error message.
I use this build file because when I made the first version glutanimate's aab wasn't
released and I know that this script works for me on Windows and Linux.
I mainly use a small subset of this build script: Copying to the dev folder set in the
global json file and building an ankiaddon file in the default location. The rest of
this add-on is hardly used and might not or no longer work. The first version of this file
is quite old and has seen many ad-hoc additions ...
Use this at your own risk.
This build script assumes that pyuic5 and pyrcc5 are on your path.
This build script assumes the following folder layout in the add-on folder:
minimal layout:
└── src
└── __init__.py
usual layout:
├── screenshots (optional; unprocessed. I sometimes put links to these into ankiweb.html)
├── designer (optional; processed by pyuic5)
│ └── dialog.ui
├── rc (optional; processed with pyrcc5)
│ ├── image.png
│ └── my.qrc (you must create this file yourself, eg with QtDesigner)
├── src
│ ├── libs (optional)
│ ├── icons (optional; unprocessed ones I just link to these from my code)
│ ├── __init__.py
│ ├── LICENSE
│ └── ...
├── .gitignore (opional)
├── readme.MD (opional)
├── checksums.csv (opional)
├── ankiweb.html (opional)
└── ...
If the repo folder contains a file named "checksums.csv" this script reads it and compares it with
the files in the subfolder src to avoid accidental changes in bundled dependencies. The format in
"checksums.csv" is "md5sum\tRelativePathToFileFromSrc".
.aam_anki_addon_make_config.json layout
{
"profile_dev": "/home/user/.var/app/net.ankiweb.Anki/data/Anki2/addons21/",
"profile_main": "/home/user/path/to/Anki_profile_main/addons21/",
"ankiaddon_dest": "",
"ignore_files_pattern": [
"*.pyc",
"*.vscode",
"*___bak_*",
"*___note*",
"*___nobundle*"
]
}
"""
import csv
import datetime
import hashlib
import json
import os
import pathlib
from pprint import pprint as pp
import random
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import uuid
import zipfile
from argparse import ArgumentParser
from PyQt5.uic import compileUiDir as compileUiDir_5
from PyQt6.uic import compileUiDir as compileUiDir_6
def md5sum(path, blocksize=2**20):
# https://stackoverflow.com/questions/1131220/get-md5-hash-of-big-files-in-python
m = hashlib.md5()
with open(path, "rb") as f:
while True:
buf = f.read(blocksize)
if not buf:
break
m.update(buf)
return m.hexdigest()
def list_all_in_dir(folder, kind, prefix, myfilterfunc=lambda *_: True):
"""
return all from FOLDER of KIND ("files" or "dirs").
to return absolute path set PREFIX to 'absolute', else to "relative"
MYFILTERFUNC only include file if this function returns true
"""
folder = os.path.abspath(folder)
li = []
for root, dirs, files in os.walk(folder):
for f in eval(kind):
if myfilterfunc(f):
if prefix == 'absolute':
o=os.path.join(root,f)
li.append(o)
elif prefix == 'relative':
relDir = os.path.relpath(root, folder)
if not relDir == '.':
o = os.path.join(relDir, f)
else:
o = f
li.append(o)
elif prefix == 'rel_prefixed_with_base':
pass
return li
def exactly_one_true_in_list(li):
one_found = False
for e in li:
if e:
if one_found:
return False
else:
one_found = True
return one_found
def config_from_file(fi):
if not os.path.isfile(fi):
return
with open(fi, 'r') as f:
try:
config = json.loads(f.read())
except:
print("error while reading config file '{}'. Aborting ...".format(fi))
sys.exit()
else:
return config
def val_from_confs(val, local, glob, ignorefalse, checkpath):
# "ankiaddon_dest", locConfFi, gloConfFi
fromlocal = True
locconf = config_from_file(local)
out = False
if locconf: # False is the config file doesn't exist
out = locconf.get(val, False)
if not out:
fromlocal = False
globconf = config_from_file(glob)
if globconf:
out = globconf.get(val, False)
if not ignorefalse:
if not out:
print("no value found for {} in config files".format(val))
sys.exit()
else:
if checkpath:
if not os.path.isdir(out):
source = local if fromlocal else glob
print(f"path '{out}' doesn't exist which is set as '{val}' in the "
f"file '{source}'. Aborting ...")
sys.exit()
return out
def parse_arguments():
prs = ArgumentParser()
prs.add_argument("-p", "--path", dest="path",
help="path to the location where the development code/source repo is")
prs.add_argument("-m", "--meta", "--nometa", action="store_false",
help="don't copy meta.json to add-on folder (default TRUE)")
prs.add_argument("--action", "-a", dest="action",
help=('actions: "a" create ankiaddon file in --path, '
'"d" copy/build to dev profile, '
'"m" copy/build to main/regular profile.'))
prs.add_argument("--profile", dest="profile",
help=('override dest profile, may not be combined with "--action" or "--aadest"'))
prs.add_argument("--times", dest="times",
help=('override ctime, atime for files in ankiaddon, e.g. "2020,11,05,12,24,37"'))
prs.add_argument("--nosrc", dest="nosrc", action='store_true',
help=('copy from folder and not from src subfolder (for third party add-ons)'))
prs.add_argument("--aadest", dest="aadest",
help=('override --ankiaddon destination, may not be combined with '
'"--action" or "--profile"'))
prs.add_argument("--no_qt6-just_forms", action="store_true",
help="do not build qt6 forms, use forms folder (and not forms5/forms6) (default FALSE)")
return prs.parse_args()
def check_for_illegal_combination_in_args(args):
if not any([args.action, args.profile, args.aadest]):
print('exactly one parameter of "--action", "--profile", "--aadest" is needed. Aborting ...')
sys.exit()
if not exactly_one_true_in_list([args.action, args.profile, args.aadest]):
print('only one parameter of "--action", "--profile", "--aadest" may be used. Aborting ...')
sys.exit()
if args.action not in ["a", "ankiaddon", "d", "dev", "m", "main", "regular"]:
print('unknown value for "--action". May only be "a" (ankiaddon), "d" (devprofile), '
'"m" (mainprofile)')
sys.exit()
def adjust_args(args):
syn_addons = ["a", "ankiaddon",]
syn_dev = ["d", "dev",]
sync_main = ["m", "main", "regular"]
if args.action in syn_addons:
args.action = "a"
elif args.action in syn_dev:
args.action = "d"
elif args.action in sync_main:
args.action = "m"
if args.path and args.path != "." and not args.path.endswith(os.path.sep):
args.path = args.path + os.path.sep
def get_source_path(args):
if args.path:
path = args.path
if not os.path.isdir(args.path):
print("value for '--path' doesn't exist. Aborting ...")
sys.exit()
if not os.path.isabs(args.path):
print("value for '--path' must be an absolute paths. Aborting ...")
sys.exit()
else:
path = os.getcwd()
if not os.path.isdir(os.path.join(path, "src")):
print("The script is executed without a '--path' argument and uses your current "
"working dir\n '{}'\nwhich does not include a 'src' folder.\nAre you sure you are in "
"the right directory?\nAborting ...".format(path))
sys.exit()
return path
def set_config_file_paths():
global gloConfFi
global locConfFi
gloConfFi = os.path.join(pathlib.Path().home(), ".aam_anki_addon_make_config.json")
if not os.path.isfile(gloConfFi):
print(f"Info: config file '{gloConfFi}' doesn't exist.")
locConfFi = os.path.join(sourcepath, ".aam.json")
if not os.path.isfile(locConfFi):
print(f"Info: config file '{locConfFi}' doesn't exist.")
def get_dests_and_skip(args):
addondest = False
regdest = False
if args.profile:
regdest = args.profile
elif args.aadest:
addondest = args.aadest
else:
# TODO tidy up
if args.action == "a":
# can only be a folder since args.aadest is False BUT leave empty so that defaults to
# addonfolder
addondest = val_from_confs("ankiaddon_dest", locConfFi, gloConfFi, True, False)
if not addondest:
addondest = sourcepath
else:
if args.action == "d":
regdest = val_from_confs("profile_dev", locConfFi, gloConfFi, False, True)
elif args.action == "m":
regdest = val_from_confs("profile_main", locConfFi, gloConfFi, False, True)
skip = val_from_confs("ignore_files_pattern", locConfFi, gloConfFi, True, False)
return addondest, regdest, skip
def setup_adjust_commonly_used_global_variables(args):
global tmpdir
global manifest_in_temp
global meta_in_temp
global addonSrcFolderName
global SrcInSrcFolder
global UiInSrcFolder
global ResInSrcFolder
global addondest
tmpdir = tempfile.mkdtemp()
manifest_in_temp = os.path.join(tmpdir, 'manifest.json')
meta_in_temp = os.path.join(tmpdir, 'meta.json')
addonSrcFolderName = os.path.basename(os.path.dirname(sourcepath)) # used for filename
if args.nosrc:
SrcInSrcFolder = sourcepath
else:
if os.path.isdir(os.path.join(sourcepath, "src")):
SrcInSrcFolder = os.path.join(sourcepath, "src")
elif os.path.isdir(os.path.join(sourcepath, "addon")):
SrcInSrcFolder = os.path.join(sourcepath, "addon")
else:
print("no src or addon folder. Aborting ...")
sys.exit()
UiInSrcFolder = os.path.join(sourcepath, "designer")
ResInSrcFolder = os.path.join(sourcepath, "ressources")
if addondest:
if os.path.isdir(addondest):
if not args.times:
version = time.strftime('%Y-%m-%d_%H-%M')
else:
print(args.times)
try:
y,mo,day,h,mi,sec= [int(x) for x in args.times.split(",")]
except:
print("error while parsing args.times. Only use integers and split them with a comma like '2020,1,1,12,30'. Aborting...")
sys.exit()
new = datetime.datetime(y,mo,day,h,mi,sec)
version = new.strftime('%Y-%m-%d_%H-%M')
addondest = os.path.join(sourcepath, f"{addonSrcFolderName}__{version}.ankiaddon")
def copy_from_temp_to_profile():
# read name of install folder from manifest.json if exists
if os.path.isfile(manifest_in_temp):
with open(manifest_in_temp, "r") as f:
d = json.load(f)
if d.get("package", False):
dest = os.path.join(regulardest, str(d["package"]))
else:
dest = os.path.join(regulardest, addonSrcFolderName)
else:
dest = os.path.join(regulardest, addonSrcFolderName)
if os.path.exists(dest):
shutil.rmtree(dest)
shutil.copytree(tmpdir, dest)
def if_manifestjson_insert_current_time_in_tmp(args):
if os.path.isfile(manifest_in_temp):
with open(manifest_in_temp, "r") as f:
d = json.load(f)
if args.times:
try:
y,mo,day,h,mi,sec= [int(x) for x in args.times.split(",")]
except:
print("error while parsing args.times. Only use integers and split them with a comma like '2020,1,1,12,30'. Aborting...")
sys.exit()
new = int(datetime.datetime(y,mo,day,h,mi,sec).timestamp())
d['mod'] = int(new)
else:
d['mod'] = int(time.time())
manifest = json.dumps(d)
with open(manifest_in_temp, 'w') as f:
f.write(manifest)
def metajson_with_filename_and_mod_from_manifestjson_in_tmp():
if os.path.isfile(manifest_in_temp):
with open(manifest_in_temp, "r") as f:
j = json.load(f)
if os.path.isfile(meta_in_temp):
with open(meta_in_temp, "r") as f:
md = json.load(f)
else:
md = {}
if "name" not in md and "name" in j:
md['name'] = j['name']
# why did I originally use if mod not in - this way Anki often wants to download an "update" from ankiweb
# if "mod" not in md and "mod" in j:
if "mod" in j:
md['mod'] = j['mod']
with open(meta_in_temp, "w") as f:
f.write(json.dumps(md))
def copy_to_temp_and_build_ui_and_ressources(args):
# shutil.copytreee doesn't work: "The destination directory, named by dst, must not already exist;",
# https://docs.python.org/3.7/library/shutil.html#shutil.copytree
# distutils.dir_util.copy_tree doesn't help since it doesn't have an ignore pattern, see
# https://docs.python.org/3.7/distutils/apiref.html#distutils.dir_util.copy_tree
if os.path.isdir(tmpdir):
os.rmdir(tmpdir)
if not os.path.isdir(SrcInSrcFolder):
print('src folder in --path is not a directory. Aborting ...')
sys.exit()
shutil.copytree(SrcInSrcFolder, tmpdir, ignore=shutil.ignore_patterns(*ignorewhencopying)) #'*.json',
#compile ui if necessary
if os.path.isdir(UiInSrcFolder):
def build_copy_ui(dest_folder, binary):
formsfolder = os.path.join(tmpdir, dest_folder)
pathlib.Path(formsfolder).mkdir(parents=True, exist_ok=True)
# pyuic5 writes the source path into the py file. I just want the relative locations
# so that the diff between versions is unchanged even if after reorganizing my dev folder
os.chdir(UiInSrcFolder)
relativesource = os.path.relpath(os.getcwd())
binary(relativesource, recurse=False)
for f in list_all_in_dir(UiInSrcFolder, "files", "absolute", lambda f:f.endswith('.py')):
shutil.move(f, formsfolder)
if args.no_qt6_just_forms:
build_copy_ui("forms", compileUiDir_5)
else:
build_copy_ui("forms5", compileUiDir_5)
build_copy_ui("forms6", compileUiDir_6)
if os.path.isdir(ResInSrcFolder):
#res_in_temp = os.path.join(tmpdir, "ressources")
#pathlib.Path(res_in_temp).mkdir(parents=True, exist_ok=True)
for f in list_all_in_dir(ResInSrcFolder, "files", "relative", lambda f:f.endswith('.qrc')):
# aab also uses call_shell
# pyrcc5 resource_file.qrc -o icons.py
cmd = """cd "{ResInSrcFolder}"; pyrcc5 {input} -o {dest}""".format(
ResInSrcFolder=ResInSrcFolder,
input=f,
dest=os.path.join(tmpdir, os.path.splitext(f)[0] + ".py")
)
subprocess.run(cmd, shell=True, check=True)
if os.path.isfile(meta_in_temp):
if args.meta:
print('Info: meta.json copied from source')
else:
os.unlink(meta_in_temp)
print('Info: meta.json not copied because of -m/--meta argument')
def verify_files(relevant_files):
mismatched = []
matched = []
unchecked = []
valfile = os.path.join(sourcepath, "checksums.csv")
if not os.path.exists(valfile):
print('\nno checksums file found\n')
else:
checksumsdict = {}
with open(valfile, "r") as f:
csv_reader = csv.reader(f, delimiter='\t')
for row in csv_reader:
try:
checksumsdict[row[1]] = row[0]
except:
pass
if checksumsdict:
for k in relevant_files:
if k in checksumsdict:
if md5sum(k) == checksumsdict[k]:
matched.append(k)
else:
mismatched.append(k)
else:
unchecked.append(k)
print('matched files: %i ' % len(matched))
print('unchecked files: %i ' % len(unchecked))
for f in unchecked:
print(' %s ' %f)
print('\n\n\n')
print('MISMATCHED files: %i ' % len(mismatched))
for f in mismatched:
print(' %s ' %f)
def change_times(files, folder, args):
if not args.times:
# adjust to now
now = datetime.datetime.now()
new = int(now.timestamp())
else:
try:
y,mo,day,h,mi,sec= [int(x) for x in args.times.split(",")]
except:
print("error while parsing args.times. Only use integers and split them with a comma like '2020,1,1,12,30,37'. Aborting...")
sys.exit()
new = int(datetime.datetime(y,mo,day,h,mi,sec).timestamp())
for f in files:
os.utime(f, (new, new))
# adjust folders
for d in list_all_in_dir(tmpdir, "dirs", "relative"):
os.utime(d, (new, new))
def zip_tmpfolder(files):
if "meta.json" in files:
files.remove("meta.json")
with zipfile.ZipFile(addondest, 'w', zipfile.ZIP_DEFLATED) as z:
for f in files:
z.write(f)
#empty directories
for f in list_all_in_dir(tmpdir, "dirs", "relative"):
z.write(f)
def tmpfolder_cleanup():
#shutil.rmtree(tmpdir)
pass
def create_ankiaddon(args):
copy_to_temp_and_build_ui_and_ressources(args)
if_manifestjson_insert_current_time_in_tmp(args)
# Todo: Adjust relevant_files
os.chdir(tmpdir)
relevant_files = list_all_in_dir(tmpdir, "files", "relative")
verify_files(relevant_files)
change_times(relevant_files, tmpdir, args)
zip_tmpfolder(relevant_files)
tmpfolder_cleanup()
def write_to_profile(args):
copy_to_temp_and_build_ui_and_ressources(args)
if_manifestjson_insert_current_time_in_tmp(args)
metajson_with_filename_and_mod_from_manifestjson_in_tmp()
copy_from_temp_to_profile()
tmpfolder_cleanup()
args = parse_arguments()
if args.times:
sec = random.randint(0,59)
if args.times.count(",") == 4:
args.times = args.times + f",{sec}"
check_for_illegal_combination_in_args(args)
adjust_args(args)
sourcepath = get_source_path(args)
set_config_file_paths()
addondest, regulardest, ignorewhencopying = get_dests_and_skip(args)
setup_adjust_commonly_used_global_variables(args)
if addondest:
create_ankiaddon(args)
else:
write_to_profile(args)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment