fetch_news.go

package main

import (
"fmt"
"io/ioutil"
"net/http"
"encoding/xml"
"strings"
"os"
"encoding/json"
)

type IndexXML struct {
	Locations []string `xml:"sitemap>loc"`
}

type SubXML struct {
	Titles []string `xml:"url>news>title"`
	Loc []string `xml:"url>loc"`
	Keywords []string `xml:"url>news>keywords"`
}

type NewsMapStruct struct {
	Keyword string
	Location string
}

func makeCall(url string) []byte{
	temp := strings.TrimSpace(url)
	resp, err := http.Get(temp)
	if err != nil {
		fmt.Println("Fetch Error", err)
		return nil
	}
	bytes, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	fmt.Println("Call Done", temp)
	return bytes
}

func main() {
	var indexSiteMap IndexXML
	var subXMLMap SubXML
	newsMap := make(map[string]NewsMapStruct)
	byteData := makeCall("https://www.washingtonpost.com/news-sitemaps/index.xml")
	if byteData != nil {
		xml.Unmarshal(byteData, &indexSiteMap)

		for _, loc := range(indexSiteMap.Locations) {
			byteData = makeCall(loc)
			if byteData != nil {
				xml.Unmarshal(byteData, &subXMLMap)

				for idx := range subXMLMap.Keywords {
					newsMap[subXMLMap.Titles[idx]] = NewsMapStruct{subXMLMap.Keywords[idx], subXMLMap.Loc[idx]}
				}
				jsonString, _ := json.Marshal(newsMap)
				f, _ := os.Create(fmt.Sprintf("/tmp/go-test/%s.txt", loc))
				f.Write(jsonString)
				f.Close()
			} else {
				fmt.Println("Error in fetching Sub XML", loc)
			}
		}
	}
}