search for django static files from a list of urls and ensure it's json response.

sta.go

package main

import (
	"bufio"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"sync"
	"time"
)

type Result struct {
	URL      string
	Found    bool
	HasError bool
}

func main() {
	filePath := "urls.txt"
	outputFilePath := "results.txt"

	urls, err := readURLsFromFile(filePath)
	if err != nil {
		fmt.Printf("Error reading URLs from file: %s\n", err)
		return
	}

	var wg sync.WaitGroup
	results := make(chan Result)

	// Spawn goroutines for checking URLs
	for _, url := range urls {
		wg.Add(1)
		go checkURL(url, results, &wg)
	}

	// Start a separate goroutine to collect results and write to file
	go writeResultsToFile(results, outputFilePath, &wg)

	wg.Wait()
	close(results)

	fmt.Println("All checks completed!")
}

func readURLsFromFile(filePath string) ([]string, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var urls []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		url := scanner.Text()
		urls = append(urls, url)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return urls, nil
}

func checkURL(url string, results chan<- Result, wg *sync.WaitGroup) {
	defer wg.Done()

	tr := &http.Transport{
		// Ignore SSL certificate errors
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{
		Timeout:   10 * time.Second,
		Transport: tr,
	}

	response, err := client.Get(url + "/static/staticfiles.json")
	if err != nil {
		results <- Result{URL: url, Found: false, HasError: true}
		return
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		results <- Result{URL: url, Found: false, HasError: false}
		return
	}

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		results <- Result{URL: url, Found: false, HasError: true}
		return
	}

	var jsonBody map[string]interface{}
	err = json.Unmarshal(body, &jsonBody)
	if err != nil {
		results <- Result{URL: url, Found: false, HasError: true}
		return
	}

	results <- Result{URL: url, Found: true, HasError: false}
}

func writeResultsToFile(results <-chan Result, outputFilePath string, wg *sync.WaitGroup) {
	file, err := os.Create(outputFilePath)
	if err != nil {
		fmt.Printf("Error creating output file: %s\n", err)
		wg.Done()
		return
	}
	defer file.Close()

	for result := range results {
		if result.Found {
			fmt.Printf("Whoo! Found '%s'\n", result.URL)
			file.WriteString(fmt.Sprintf("%s\n", result.URL))
		} else {
			fmt.Printf("Boo! '%s' not found\n", result.URL)
		}
	}

	wg.Done()
}