twitter_link_collector.sh

#!/bin/sh
home_dir="./fall22-hw2-antonrasmussen/"
search_term_array=("virginia" "beach" "coronavirus" "dall-e" "openai")
cd $home_dir
for search_term in "${search_term_array[@]}"
do
    /opt/homebrew/bin/python3 $home_dir/collect-tweets.py $search_term
    /opt/homebrew/bin/python3 $home_dir/process-tweets.py < tweets.jsonl > tweets-info.txt
    
    link_pref="https:/"
    # get only https links | remove twitter links | remove .ly links | remove youtube links | remove instagram | get both www and non www
    declare -a link_sufx=$(cat tweets-info.txt | grep https | grep -v twitter | grep -v ".ly" | grep -v youtu | grep -v instagram | cut -f3-  -d "/" )
    for link in $link_sufx
    do
        # Stich the links back together
        link="$link_pref/$link"
        echo $link >> link_list_unsorted.txt
    done
done

# unsorted list will grow but link_list will always be new unique and sorted
cat link_list_unsorted.txt | sort | uniq > link_list.txt