An exercise in torturing ADIF log files

filter_confirmed.sh

#!/bin/bash

### This produces a list of all QSOs that are confirmed by *either* LoTW or eQSL
### from a given log file -- in my case, a cqrlog backup file, which keeps
### these flags separately -- to be used with diploma applications that
### accept either of the two and verify them by their own mysterious means.

# Requires:
# * adifmt - https://github.com/flwyd/adif-multitool
# * ripgrep - https://github.com/BurntSushi/ripgrep

# Where to find your files.
SOURCE=$HOME/Syncthing/Radio/cqrlog/R2AZE_backup.adi
RESULT=$HOME/Documents/GridTracker/confirmed.adif

# LOTW is easy.

adifmt find -if 'lotw_qsl_rcvd=Y' $SOURCE >/tmp/lotw_qsl.adi

# I need a way to filter out non-AG eQSL correspondents, which is not information
# cqrlog seems to keep.
# We have to download and parse the official list and then filter out based on that.

adifmt find -if 'eqsl_qsl_rcvd=Y' -if-not 'lotw_qsl_rcvd=Y' $SOURCE |
	adifmt fix >/tmp/eqsl_qsl.adi

wget -q -O /tmp/aglist.csv \
	https://www.eqsl.cc/qslcard/DownloadedFiles/AGMemberListDated.txt

# Transmute the CSV file into a list of grep strings that indicate
# that a given ADIF record is valid.
awk -F',' '{printf("<CALL:%d>%s <\n",length($1),$1);}' /tmp/aglist.csv |
	tail -n +2 >/tmp/aglist.txt

# Something about the resulting unicode is broken because someone uses unusual
# characters in their callsign on eQSL. WHATEVER!
recode -f utf8..iso8859-1 /tmp/aglist.txt

# And that kills grep on out of memory in fucking 8 gigabytes of it!
# grep -F -f /tmp/aglist.txt /tmp/eqsl_qsl.adi
# Fortunately ripgrep can also do this and isn't so stupid.
rg -FN -f /tmp/aglist.txt /tmp/eqsl_qsl.adi >/tmp/eqsl_ag_qsl.adi

# Now merge them.
adifmt fix /tmp/lotw_qsl.adi /tmp/eqsl_ag_qsl.adi |
	adifmt select -fields CALL,QSO_DATE,TIME_ON,TIME_OFF,STATION_CALLSIGN,$(
	)MODE,FREQ,BAND,RST_SENT,RST_RCVD,COUNTRY,QTH,NAME,GRIDSQUARE,$(
	)MY_GRIDSQUARE,TX_PWR,DXCC,LOTW_QSL_RCVD,LOTW_QSLRDATE,$(
	)EQSL_QSL_RCVD,EQSL_QSLRDATE |
	adifmt sort -fields QSO_DATE \
		>$RESULT

rm /tmp/lotw_qsl.adi /tmp/eqsl_qsl.adi /tmp/eqsl_ag_qsl.adi \
	/tmp/aglist.txt /tmp/aglist.csv