Organize http://archzfs.com/archive_archzfs/ as Arch Linux Archives

symlinker.sh

#!/bin/bash
# Written by github.com/eoli3n

# Script stops if any error
set -e

############
### Test ###
# Here i tried to keep you current uploaddir and just generates a new one with symlinks
# To test without symlink actual datas, just use "-s"
# It will dump actual repo by touching empty files.
# $ mkdir test && cd test && wget $raw.gist.url/symlink_gen.sh -s && ./symlink_gen.sh
### Prod ###
# Your webserver needs to follow symlinks : for nginx https://unix.stackexchange.com/a/157098
# Set absolute paths for vars $UPLOADDIR, $WORKDIR and $LOGFILE
# Use cron to automates with the process. Use '-f' but runs once manually to create toptree.
############

### Vars

SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
UPLOADDIR="./archive_archzfs"
WORKDIR="./archives"

# Write logfile in current script dir
LOGFILE="$SCRIPTDIR/archive.log"

# Source url for simulate
url="http://archzfs.com"

# Set lists separator for loops
IFS=$'\n'

### Functions

usage() {
    echo "usage: $0 [-h] [-f] [-s] [-d]"
    echo "    -h     print that usage"
    echo "    -f     force without asking anything"
    echo "    -s     simulate with dump of site"
    echo "    -d     dryrun"
    exit 1
}

create_local_dump() {

    # Query repos lines
    lines=$(curl -s "$url"/archive_archzfs/ | grep href | grep 'pkg')
    
    echo ">>> Starting extraction !"
    for line in $lines
    do
        # Create upload dir
        mkdir -p "$UPLOADDIR"
        # Extract filename
        file=$(echo "$line" | sed -E 's/^.*href="(.*)">.*$/\1/' | cut -d'"' -f1 )
        # Extract date
        date=$(echo "$line" | sed -E 's/^.*([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}).*$/\1/' )
        # Create file
        touch --date="$date" "$UPLOADDIR/$file"
        echo "Extracted $UPLOADDIR/$file"
    done
    echo ">>> Extraction done !"
}

run() {

    # Browse files to add
    for filename in $(cat "$TMPFILE")
    do
    
        # Set filepath
        filepath="$UPLOADDIR/$filename"

        # Extract datas from files
        date=$(stat -c '%w' "$filepath")
        year=$(echo "$date" | awk -F'-' '{print $1}')
        month=$(echo "$date" | awk -F'-' '{print $2}')
        day=$(echo "$date" | awk -F'-' '{print $3}' | cut -d" " -f1)
        firstletter=$(echo "$filename" | head -c 1)
        name=$(echo "$filename" | sed -E 's/^(.*)-[0-9].*-[0-9].*$/\1/')

        # Create repos and packages subdirs
        repos_bydate="$repos/$year/$month/$day"
        packages_byname="$packages/$firstletter/$name"

        # Create relative symlinks
        if test -z "$dryrun"
        then

            # Create directories by name and by date
            mkdir -p "$repos_bydate" "$packages_byname"

            # Create symlinks
            ln -rs "$filepath" "$repos_bydate/$filename"
            echo "$(date -u) : symlink $filepath to $repos_bydate" >> "$LOGFILE"
            printf '.'
            ln -rs "$filepath" "$packages_byname/$filename"
            echo "$(date -u) : symlink $filepath to $packages_byname" >> "$LOGFILE"
            printf '.'
            ln -rs "$filepath" "$all/$filename"
            echo "$(date -u) : symlink $filepath to $all" >> "$LOGFILE"
            printf '.'
        else
            echo "symlink $filepath to $repos_bydate"
            echo "symlink $filepath to $packages_byname"
            echo "symlink $filepath to $all"
        fi
    done

    # Clean tmp file
    if test -f "$TMPFILE"
    then
      rm "$TMPFILE"
    fi
}

### Options

while getopts "hsfd" o; do
    case "${o}" in
        # simulate
        s)
            create_local_dump
            ;;
        # force without asking
        f)
            force=1
            ;;
        # dryrun
        d)
            dryrun=1
            ;;
        # help
        h)
            usage 
            ;;
        *)
            usage
            ;;
    esac
done
shift $((OPTIND-1))

### Main

# Test if UPLOADDIR is present
if ! test -d "$UPLOADDIR"
then
    echo -e "Error: No upload dir found\nexited"
    exit 1
fi 

# Set toptree vars
# https://wiki.archlinux.org/index.php/Arch_Linux_Archive#/repos
repos="$WORKDIR/repos"

# https://wiki.archlinux.org/index.php/Arch_Linux_Archive#/packages
packages="$WORKDIR/packages"

# .all contains all files flat and will be used for some tests
all="$packages/.all"

# Test if WORKDIR top trees exists
if ! test -d "$repos" || ! test -d "$packages" || ! test -d "$all"
then 

    # If not force
    if test -z $force
        then
        # Ask for toptree creation if no present
        echo ">>> Toptree not found"
        read -p "Do you want to create it (y/N)? " -n 1 -r
        echo -e "\n"
        if [[ $REPLY =~ ^[Yy]$ ]]
        then
            # Create toptree
            mkdir -p $repos $packages $all
            echo ">>> Created toptree"
        else
            echo "exited"
            exit
        fi
    else
        # Create toptree
        mkdir -p $repos $packages $all
        echo ">>> Created toptree"
    fi
fi

# Diff $UPLOADDIR files and $all to know which packages has not been already symlinked
TMPFILE="/tmp/symlink_gen.tmp"
comm -23 <(find "$UPLOADDIR" -type f -printf '%f\n' | sort) <(find "$all" -type l -printf '%f\n' | sort) > "$TMPFILE"
NUMBERFILES=$(cat "$TMPFILE" | wc -l)

# Test if something to do
if test $NUMBERFILES -eq 0
then
    echo "Nothing to do"
    exit 0
fi

# If not force
if test -z $force
then
    cat "$TMPFILE"
    echo -e ">>> $NUMBERFILES files to symlink"
    if ! test -z $dryrun
    then
      echo "(dryrun mode)"
    fi
    read -p "Do you want to proceed (y/N)? " -n 1 -r
    echo -e "\n"

    if [[ $REPLY =~ ^[Yy]$ ]]
    then
        run
    else
        echo "exited"
        exit
    fi
else
    run
fi