Creates and maintains a list of all manually-installed packages in a Ubuntu distro.

export-custom-packages

#!/usr/bin/env bash

# Script name
PROGNAME=$(basename "$0")

if [[ "$1" == '--help' || "$1" == '-h' ]]; then
    fmt -s <<- HELP_TEXT
	
	Writes and updates a list of all manually-installed packages in a Ubuntu distro.
	
	Keeping a permanent, separate list is necessary because the Apt install log is rotated and archived every few months (depending on the system configuration), and the archived logs are purged eventually.
	
	$PROGNAME evaluates the current as well as the archived logs, ie everything which is still there, and updates or creates the permanent list.
	
	If the list is empty, the logs don't contain manually-installed packages (and never have, at least when $PROGNAME was executed).
	
	Usage: 
	$PROGNAME [options]
	
	Options:
	
	-d dirname   The path to the output directory. Trailing slash is 
	             optional. Defaults to the current "." directory.
	-f filename  The name of the output file. Defaults to 'packages.txt'.
	-c           Create the output directory and its parents if necessary.
	-q           Quiet, don't print the path to the output file and a
	             success message.
	-l           List the packages (to stdout). The output file is still
	             required. The option prints the content of the updated 
	             output file. When combined with -q, just the package 
	             names are printed, one package per line, without
	             additional text.
	-h, --help   Show help.
	
	Limitations:
	
	- The script does not detect manual installs with Aptitude. 
	
	  The format of the Aptitude log entries make it almost impossible to
	  distinguish between user-installed, top-level packages and their
	  countless dependencies, which would clutter the output hopelessly.
	
	- Under rare cicumstances, a package which has been added manually,
	  and is removed later on, can still remain on the list.
	
	  That happens if the package has been added to the list long ago, and
	  the install action is no longer present in the Apt log (because the 
	  log has been rotated), and neither in the archived logs (because the 
	  archive has eventually been deleted). If the package is removed, it 
	  still remains on the list.
	
	  (This issue could be fixed, but it doesn't seem worth the effort.)

	HELP_TEXT
	exit 0
fi

# Apt log location
APT_LOG="/var/log/apt/history.log"

[ ! -f "$APT_LOG" ] && { echo "$PROGNAME: Cannot find the Apt log. Skript aborted. Expected location: $APT_LOG" >&2; exit 1; }

# Option default values
quiet=false
create_output_dir=false
output_filename="packages.txt"
output_dir="$(realpath .)"
print_list_to_stdout=false

while getopts ":cd:f:lq" option; do
    case $option in
        c)
            create_output_dir=true
            ;;
        d)
            output_dir="$OPTARG"
            ;;
        f)
            output_filename="$OPTARG"
            ;;
        l)
            print_list_to_stdout=true
            ;;
        q)
            quiet=true
            ;;
        \?)
            echo "$PROGNAME: Option '-$OPTARG' is invalid. Skript aborted." >&2
            exit 1
            ;;
        :)
            echo "$PROGNAME: The argument for option '-$OPTARG' is missing. Skript aborted." >&2
            exit 1
            ;;
    esac
done

# Locations
#
# (Remove trailing slash from output dir if present.)
output_dir="$(sed -r 's|([^/])/+$|\1|' <<<"$output_dir")"
output_filepath="$output_dir/$output_filename"

if [ ! -d "$output_dir" ]; then
    [ $create_output_dir == false ] && { echo "$PROGNAME: Cannot find the output directory. Use the -c option to create it." >&2; echo "Expected location: $output_dir" >&2; exit 1; }
    mkdir -p "$output_dir" || { echo "$PROGNAME: Failed to create the output directory at: $output_dir" >&2; exit 1; }
fi

touch "$output_filepath" || { echo "$PROGNAME: Failed to create or access the output file '$output_filename' at: $output_filepath" >&2; exit 1; }

if [ $quiet == false ]; then
    echo "The list of packages is kept at"
    echo
    echo "    $output_filepath"
    echo
fi

# Find packages in the apt log which are manually installed. Extract them, one
# package per line, and append the new ones to the existing list.
#
# - safegrep 
#   `grep` utility function, for better readability, which suppresses an error
#   if `grep` doesn't find a match. Used as a drop-in replacement for `grep`. 
#   See https://stackoverflow.com/a/49627999/508355
# - `ls -tr $APT_LOG*`
#   Gets the paths of the (current) log file and archived older ones 
#   (history.log.1.gz, ...), sorted by modification date, oldest first
# - zgrep, grep (safegrep): 
#   extracts apt/apt-get install/remove/purge command lines from logs
# - sed: 
#   + extracts command name and package names from command lines
#   + removes redundant white space
#   + ensures one package per line (from multiple installs), with the command
#     preceding it
#   + normalizes the 'purge' and 'remove' commands as 'remove'
# - nl, sort, uniq, sort:
#   + removes lines with duplicate packages, keeping the last occurrance. The
#     command in that line tells whether the final action was 'install' or 
#     'remove'
#   + restores the original sort order
# - grep (safegrep):
#   keeps only lines with install commands
# - cut:
#   extracts the package names, discards line numbers and commands
# - comm:
#   discards packages which are already recorded in the package list
# 
# The result is appended to the existing package list.

safegrep() { grep "$@" || test $? = 1; }

set -o pipefail    # See https://stackoverflow.com/a/19804002/508355

zgrep -Pi '^CommandLine: +apt.* (install|remove|purge) +[a-z]+' `ls -tr $APT_LOG*` | safegrep -iv 'autoinstall=yes' | \
    sed -r -e 's/^.* (install|remove|purge) +([a-z].*)$/\1 \2/I' -e 's/ +/ /g' -e 's/ $//' -e '/^install / s/([^ ]+) /\1\ninstall /2gI' -e '/^purge |^remove / s/([^ ]+) /\1\nremove /2gI' | \
    nl -s ' ' -n 'rz' | sort -k 3 -k 1rn | uniq -f 2 | sort -k 1n | \
    safegrep -Pi '^\d+\s+install' | \
    cut -d " " -f 3 | \
    comm -13 --nocheck-order "$output_filepath" - >> "$output_filepath"

[ $? -ne 0 ] && {  echo "$PROGNAME: Error while processing the package install log. Skript aborted." >&2; exit 1; } 
set +o pipefail

if [ $print_list_to_stdout == true ]; then
    [ $quiet == false ] && { echo "The following packages have been installed manually:"; echo; }
    cat < "$output_filepath"
    [ $quiet == false ] && echo
fi

if [ $quiet == false ]; then
    if [ -n "$WSL_DISTRO_NAME" ]; then
        name="WSL distro '$WSL_DISTRO_NAME' @ $(hostname)"
    else
        name="host '$(hostname)'"
    fi
    echo "The list of manually-installed packages for $name has been updated successfully."
fi

is there a license for this? might add timestamp info and would like to publish

@cphouser You are right, I should have clarified this, so I'll make it MIT-licensed by way of this comment ;) In other words, fork and edit as much as you like.