Skip to content

Instantly share code, notes, and snippets.

@Sahil624
Created November 10, 2019 16:38
Show Gist options
  • Save Sahil624/388fe6a31d6177aa9bdca1ef162767ef to your computer and use it in GitHub Desktop.
Save Sahil624/388fe6a31d6177aa9bdca1ef162767ef to your computer and use it in GitHub Desktop.
package main
import (
"fmt"
"io/ioutil"
"net/http"
"encoding/xml"
"strings"
"os"
"encoding/json"
)
type IndexXML struct {
Locations []string `xml:"sitemap>loc"`
}
type SubXML struct {
Titles []string `xml:"url>news>title"`
Loc []string `xml:"url>loc"`
Keywords []string `xml:"url>news>keywords"`
}
type NewsMapStruct struct {
Keyword string
Location string
}
func makeCall(url string) []byte{
temp := strings.TrimSpace(url)
resp, err := http.Get(temp)
if err != nil {
fmt.Println("Fetch Error", err)
return nil
}
bytes, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
fmt.Println("Call Done", temp)
return bytes
}
func main() {
var indexSiteMap IndexXML
var subXMLMap SubXML
newsMap := make(map[string]NewsMapStruct)
byteData := makeCall("https://www.washingtonpost.com/news-sitemaps/index.xml")
if byteData != nil {
xml.Unmarshal(byteData, &indexSiteMap)
for _, loc := range(indexSiteMap.Locations) {
byteData = makeCall(loc)
if byteData != nil {
xml.Unmarshal(byteData, &subXMLMap)
for idx := range subXMLMap.Keywords {
newsMap[subXMLMap.Titles[idx]] = NewsMapStruct{subXMLMap.Keywords[idx], subXMLMap.Loc[idx]}
}
jsonString, _ := json.Marshal(newsMap)
f, _ := os.Create(fmt.Sprintf("/tmp/go-test/%s.txt", loc))
f.Write(jsonString)
f.Close()
} else {
fmt.Println("Error in fetching Sub XML", loc)
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment