filter_log2.sh

#!/bin/bash

# loop, read last line for the input values
while read line; do
  # read values from the line delimitted by `|`
  consumer_key=$(echo $line | cut -d '|' -f1)
  username=$(echo $line | cut -d '|' -f2)
  appname=$(echo $line | cut -d '|' -f3)

  # get all the access tokens for the consumer key
  grep "$consumer_key" db_output_sorted.txt  | cut -d '|' -f4 > tmp_access_tokens.txt

  # collect all the counts for access keys found and sum them together using awk
  while read line; do
    grep $line access-log-with-thousands-of-lines.log | wc -l
  done < tmp_access_tokens.txt | awk '{ SUM += $1 } END { print SUM }' > tmp_total_count.txt

  total_count=$(cat tmp_total_count.txt)

  # remove temp files
  rm -rf tmp_access_tokens.txt tmp_total_count.txt

  # output!
  echo -e "$consumer_key\t$username\t$appname\t$total_count"
done < <(cat db_output_sorted.txt | cut -d '|' -f1,2,3 | sort | uniq)
# input the list of unique consumer keys, their usernames, and app names