filter_log.sh

#!/bin/bash

prev_consumer_key=""
prev_access_token=""
prev_username=""
prev_appname=""

total_count=0

# start looping
while read line; do
  # cut up the line by separator `|`
  consumer_key=$(echo $line | cut -d '|' -f1)
  username=$(echo $line | cut -d '|' -f2)
  appname=$(echo $line | cut -d '|' -f3)
  access_token=$(echo $line | cut -d '|' -f4)

  # search for access token in the log file and get the line count for the results
  grep_count=$(grep $access_token access-log-with-thousands-of-lines.log | wc -l)

  # first loop
  if [[ $prev_consumer_key == "" ]]; then
    prev_consumer_key="$consumer_key"
    prev_appname="$appname"
    prev_username="$username"
    prev_access_token="$access_token"
  fi

  if [[ $prev_consumer_key == $consumer_key ]]; then
    # Add up to the request count found for earlier access token
    total_count=$(( total_count + grep_count ))
  else [[ $prev_consumer_key != $consumer_key ]];
    # Display the total for the previous consumer key
    echo -e "$prev_consumer_key\t$prev_username\t$prev_appname\t$total_count"

    # Start counting for the next one
    total_count="$grep_count"
    prev_consumer_key="$consumer_key"
    prev_appname="$appname"
    prev_username="$username"
    prev_access_token="$access_token"
  fi
done < db_output_sorted.txt

# show the final line too
echo -e "$prev_consumer_key\t$prev_username\t$prev_appname\t$total_count"