To extract URLs of technologies from httpx & webanalyze results combined

techdetect.sh

#!/bin/bash

if [ -z "$1" ]; then
    echo "Usage: techdetect <technology>"
    exit 1
fi

urls=()

# Check existence of httpxout*.json files and process each one
for file in httpxout*.json; do
    [ -e "$file" ] && urls+=($(cat "$file" | jq -r 'select(.tech // [] | length > 0) | [.url, .tech[]] | @csv' | grep -i "$1" | cut -d , -f 1 | tr -d '"'))
done

# Check existence of webtech.json file
[ -e webtech.json ] && urls+=($(cat webtech.json | jq -r '. | [.hostname, .matches[].app_name] | @csv' | grep -i "$1" | cut -d , -f 1 | tr -d '"'))

result=$(printf "%s\n" "${urls[@]}")

for url in $result; do
    echo "$url" | grep -oE "^https?://[^/]*(:[0-9]+)?"
done