parallelman script

parallelman.py

#!/usr/bin/env python3

from subprocess import check_output
import os.path
import subprocess
import time

cachedir = "/var/cache/pacman/pkg" # NO "/" AT THE END!

print(check_output(["pacman", "-Sqy"]).decode("utf-8")) # update package data base

print("Building trees\n...", end="")

packagesToInstall = check_output(["pacman", "-Qqu"]).decode("utf-8").split()

trees = []
for thisPkg in packagesToInstall:
    trees.append(set(check_output(["pacman", "-Qqs", thisPkg]).decode("utf-8").split()))

# Where the magic happens: merge all trees that have at least one common package
# -> leaves us with indipendend trees
class EscapeFor(Exception): pass
oldLen = -1
while (oldLen != len(trees)) and (len(trees)>=2): # merge as long as >=2 trees and as long as they get less
    oldLen= len(trees)
    try:
        for j in range(0, len(trees)-2):
            for i in range(j+1, len(trees)-1):
                if trees[j].intersection(trees[i]): # if trees have >=1 common package
                    trees[j] = trees[j] | trees[i] # merge one into the other
                    del trees[i] # and remove one
                    raise EscapeFor() # exit whole for-hell since number of trees changed -> mess
    except EscapeFor:
        pass
        
print("list of indipendend trees("+str(len(trees))+"): ")
for tree in trees:
    print(tree)

packagesReady = []
InstallationProcess = None

for tree in trees:
    print("\n********* Download of tree: ", tree)
    for package in tree:
        packageURLs = check_output(["pacman", "-Sp", package]).decode("utf-8").strip().split()
        for packageURL in packageURLs:
            packageFileName = packageURL.split("/")[-1]
            if os.path.exists(cachedir+"/"+packageFileName):
                print(packageFileName + " already in cache, skipping download")
            else:
                print(packageFileName + " downloading:")
                print(check_output(["curl", "-#", "-o", cachedir+"/"+packageFileName, packageURL]).decode("utf-8").strip(), end="")
            
        packagesReady.append(package)
            
    
    print("******** Installation of packages: ",packagesReady)
    if os.path.exists("/var/lib/pacman/db.lck"): # check if pacman is running
        print("pacman is busy, trying again after next tree download finished")
    else:
        if InstallationProcess:
            print("\n///// PACMAN installation status of last tree //////")
            out, err = InstallationProcess.communicate()
            print(out.decode("utf-8"))
            print(err.decode("utf-8"))
            print("///// PACMAN installation status END //////\n")
        print("pacman is free, starting background installation of these packages: ", packagesReady)
        InstallationProcess = subprocess.Popen(["pacman", "--noconfirm", "--needed", "-S"]+packagesReady, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        packagesReady = []
        
if InstallationProcess: # output of last pacman-session doesn't get shown by for-loop -> do it now
    InstallationProcess.wait()
    print("\n///// PACMAN installation status of last tree //////")
    out, err = InstallationProcess.communicate()
    print(out.decode("utf-8"))
    print("///// PACMAN installation status END //////\n")
    
        
if len(packagesReady)>0: # in last round of the for-loop pacman might have been busy -> install remaining packages now
    print("******** Installation of packages: ",packagesReady)
    while os.path.exists("/var/lib/pacman/db.lck"):
        time.sleep(0.1)
    print(check_output(["pacman", "--noconfirm", "--needed", "-S"]+packagesReady).decode("utf-8"))