Skip to content

Instantly share code, notes, and snippets.

@VOID001 VOID001/check.py
Last active Dec 28, 2018

Embed
What would you like to do?
Arch Linux CN static dependency check
import os
import sys
import importlib.util
import logging
import shlex
import subprocess
from typing import List
import yaml
import pycman.config
import pyalpm
class DependencyChecker:
check_pass = {}
REPO_ROOT = ""
build_prefix_pkgs = {}
def run(self, pkg_item: any):
print("run pkg_item = {0}".format(pkg_item))
pkg_name = ""
pkg_base = ""
err_dep = []
is_split = False
if isinstance(pkg_item, tuple):
pkg_base, pkg_name = pkg_item
is_split = True
else:
pkg_name = pkg_base = pkg_item
# Retrive the meta info from PKGBUILD, lilac.py, lilac.yaml
# Note that lilac.yaml will be overridden by lilac.py
package_info = self._get_pkgbuild_info(
pkg_base, pkg_name, split=is_split)
depends = package_info.get("depends")
cn_deps, build_prefix = self._parse_lilac(pkg_base)
if cn_deps is None:
print("package {} doesn't contain a lilac.py, skip checking".format(pkg_item))
return []
# Search all packages, Init pyalpm handle from pacman.conf
conf_path = os.path.join(
"/usr/share/devtools/", "pacman-" + build_prefix.rstrip("-x86_64") + ".conf")
handle = pycman.config.init_with_config(conf_path)
syncdbs = handle.get_syncdbs() # type: pyalpm.DB
allpkgs = []
if self.build_prefix_pkgs.get(build_prefix) is not None:
allpkgs = self.build_prefix_pkgs.get(build_prefix)
else:
for db in syncdbs:
allpkgs.extend(db.search(""))
self.build_prefix_pkgs[build_prefix] = allpkgs
print("len allpkgs = {}".format(len(allpkgs)))
for dep in depends:
if self.check_pass.get(dep):
continue
if self._dep_satisfied_in_syncdb(dep, allpkgs):
continue
if self._dep_satisfied_in_cn_repo(dep, cn_deps):
continue
err_dep.append(dep)
if len(err_dep) == 0:
self.check_pass[pkg_item] = True
return err_dep
def _get_pkgbuild_info(self, pkg_base, pkg_name, split=False) -> dict:
info = {}
pkgbuild_dir = os.path.join(self.REPO_ROOT, pkg_base, "PKGBUILD")
deps = []
provides = []
pkgname = []
info = {}
# These are args we need to parse
PARSE_ARGS = ["depends", "makedepends",
"pkgname", "provides", "depends_x86_64"]
for arg in PARSE_ARGS:
info[arg] = []
if not split:
cmd = shlex.split(
"/bin/bash -c '{}/printpkgenv.sh {} 2>/dev/null'".format(os.getcwd(), pkgbuild_dir))
else:
cmd = shlex.split(
"/bin/bash -c '{}/printpkgenv.sh {} {} 2>/dev/null'".format(os.getcwd(), pkgbuild_dir, pkg_name))
# print("command = {}".format(cmd))
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
encoding="UTF-8", stderr=None)
if proc.returncode != 0 and proc.returncode is not None:
raise Exception("PKGBUILD Parse error {}".format(proc.returncode))
for line in proc.stdout:
(k, _, v) = str(line).partition("=")
v = v.lstrip("(").rstrip(")\n") # type: str
items = v.split(" ")
if k in PARSE_ARGS:
info[k] = []
for item in items:
if item != "":
info[k].append(item)
proc.communicate()
print("DEBUG info = {}".format(info))
info["depends"].extend(info.get("makedepends"))
info["depends"].extend(info.get("depends_x86_64"))
return info
def _parse_lilac(self, pkg_base) -> (list, str):
""" Read yaml first, and then read the py file, note that py will override the yaml """
cn_deps = []
build_prefix = ""
lilac_py_path = os.path.join(self.REPO_ROOT, pkg_base, "lilac.py")
lilac_yaml_path = os.path.join(self.REPO_ROOT, pkg_base, "lilac.yaml")
try:
_file = open(lilac_py_path, "r")
_file.close()
except FileNotFoundError:
return (None, None)
yaml_cn_deps, yaml_build_prefix = self._parse_lilac_yaml(
lilac_yaml_path)
py_cn_deps, py_build_prefix = self._parse_lilac_py(lilac_py_path)
cn_deps = py_cn_deps if py_cn_deps is not None else yaml_cn_deps
build_prefix = py_build_prefix if py_build_prefix is not None else yaml_build_prefix
if cn_deps is None:
cn_deps = []
if build_prefix is None or build_prefix == "":
build_prefix = "extra-x86_64"
print("DEBUG: cn_dep = {}, build_prefix = {}".format(cn_deps, build_prefix))
return (cn_deps, build_prefix)
def _parse_lilac_yaml(self, path):
cn_deps = build_prefix = None
build_prefix = ""
try:
_file = open(path, "r")
yaml_obj = yaml.load(_file)
# print("DEBUG package = {} yaml_obj = {}".format(path, yaml_obj))
if "depends" in yaml_obj:
cn_deps = []
depends = yaml_obj.get("depends")
for depend in depends:
if isinstance(depend, dict): # 'is' is different from 'isinstance'
for k, v in depend.items():
# cn_deps would contain either string or tuple
cn_deps.append((k, v))
else:
cn_deps.append(depend)
_file.close()
if "build_prefix" in yaml_obj:
build_prefix = yaml_obj.get("build_prefix")
except FileNotFoundError:
pass
return cn_deps, build_prefix
def _parse_lilac_py(self, path):
cn_deps = build_prefix = None
spec = importlib.util.spec_from_file_location(
"dumb" + ".pkg", path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
try:
assert mod.depends
cn_deps = mod.depends
except Exception as e:
pass
try:
assert mod.build_prefix
build_prefix = mod.build_prefix
except Exception as e:
pass
return cn_deps, build_prefix
def _dep_satisfied_in_syncdb(self, dep, packages: List[pyalpm.Package]):
pkgobj = pyalpm.find_satisfier(packages, dep)
if pkgobj is None:
return False
return True
def _dep_satisfied_in_cn_repo(self, dep, cn_deps):
# If the package is explictly list in cn_dep, we return true
ok = False
dep_need_check = None
dep_name, _ = self._parse_dep_str(dep)
if dep_name in cn_deps:
dep_need_check = dep_name
ok = True
# TODO: deal with CN repo version string
# We need to figure out provided packages
if not ok:
for cn_item in cn_deps:
if not isinstance(cn_item, tuple):
provides = self._get_pkgbuild_info(
cn_item, cn_item, split=False).get("provides")
if dep_name in provides:
ok = True
dep_need_check = dep_name
break
else:
# It's a tuple, which means split package
base, name = cn_item
info = self._get_pkgbuild_info(base, name, split=True)
# The split package might provide dep
provides = info.get("provides")
# The split package might be dep
pkgnames = info.get("pkgname")
if dep_name in provides or dep_name in pkgnames:
ok = True
dep_need_check = (base, name)
break
if not ok:
return False
err_deps = self.run(dep_need_check)
if len(err_deps) > 0:
return False
self.check_pass[dep_need_check] = True
return True
def _parse_dep_str(self, depstr)-> (str, str):
name_index = -1
ver_index = -1
for k, v in enumerate(depstr):
v: str
if name_index == -1 and (v.isalpha() or v.isnumeric() or v in "@._+-"):
continue
if name_index != -1 and v in "><=":
continue
else:
if name_index == -1:
name_index = k
elif ver_index == -1:
ver_index = k
break
if name_index == -1:
return depstr, ""
pkgname = depstr[0:name_index]
pkgver = depstr[ver_index:]
return pkgname, pkgver
ck = DependencyChecker()
ck.REPO_ROOT = "/home/void001/packager/repo/"
err_deps = ck.run("mingw-w64-gcc")
if len(err_deps) > 0:
print("check error: {}".format(err_deps))
else:
print("check ok")
#!/bin/bash
grep_function() {
{ declare -f "$1" || declare -f package; } 2>/dev/null | grep -E "$2"
}
extract_function_variable() {
# $1: function name
# $2: variable name
# $3: multivalued
# $4: name of output var
local funcname=$1 attr=$2 isarray=$3 outputvar=$4 attr_regex= decl= r=1
if (( isarray )); then
printf -v attr_regex '^[[:space:]]* %s\+?=\(' "$2"
else
printf -v attr_regex '^[[:space:]]* %s\+?=[^(]' "$2"
fi
# this function requires extglob - save current status to restore later
local shellopts=$(shopt -p extglob)
shopt -s extglob
#declare -f $funcname
while read -r; do
# strip leading whitespace and any usage of declare
decl=${REPLY##*([[:space:]])}
# printf "${decl/#$attr/$outputvar}"
eval "${decl/#$attr/$outputvar}"
# entering this loop at all means we found a match, so notify the caller.
r=0
done < <(grep_function "$funcname" "$attr_regex")
eval "$shellopts"
return $r
}
set +e
env -i
source "$1" || true
echo "pkgname=(" "${pkgname[@]}" ")"
echo "makedepends=(" "${makedepends[@]}" ")"
overrides=(provides depends)
# echo "Len =" "${#pkgname[@]}"
if [[ -n "${2// }" ]]; then
echo "# We are going to process split"
extract_function_variable "package_$2" "depends" 1 depends
extract_function_variable "package_$2" "provides" 1 provides
fi
echo "provides=(" "${provides[@]}" ")"
echo "depends=(" "${depends[@]}" ")"
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.