tpsrep0rts/README.md

## README.md

      
    Raw
  

              README.md
            
          
    Improvements to source made available on pythonprogramming.net tutorial series.
Original source can be found here:
https://pythonprogramming.net/go/concurrency-web-app-go-language-programming-tutorial/

  
## aggregatorfinish.html
<head>
    <script type="text/javascript" charset="utf8" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
    <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/1.10.16/css/jquery.dataTables.css">
    <script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.16/js/jquery.dataTables.js"></script>
</head>

<h1>{{.Title}}</h1>

<table id="example" class="display" cellspacing="0" width="100%">
    <col width="35%">
    <col width="65%">
    <thead>
        <tr>
            <th>Title</th>
            <th>Keywords</th>
        </tr>
    </thead>
    <tbody>

        {{ range $key, $value := .News }}
         <tr>
            <td><a href="{{ $value.Location }}" target='_blank'>{{ $key }}</td>
            <td>{{ $value.Keyword }}</td>
        </tr>
        {{ end }}
    </tbody>
</table>


<script>$(document).ready(function() {
    $('#example').DataTable();
} );</script>

## handler.go
package main

import (
    "fmt"
    "net/http"
    "html/template"
    "newsfeed"
)

func newsAggHandler(w http.ResponseWriter, r *http.Request) {
    p := newsfeed.NewsAggPage{Title: "Amazing News Aggregator", News: newsfeed.GetNews()}
    t, _ := template.ParseFiles("aggregatorfinish.html")
    t.Execute(w, p)
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "<h1>Whoa, Go is neat!</h1>")
}

func main() {
    go newsfeed.RefreshNews()
    http.HandleFunc("/", indexHandler)
    http.HandleFunc("/agg/", newsAggHandler)
    http.ListenAndServe(":8000", nil)
}

## newsfeed.go
package newsfeed

import (
    "net/http"
    "encoding/xml"
    "io/ioutil"
    "sync"
    "time"
)

var wg sync.WaitGroup

type NewsMap struct {
    Keyword string
    Location string
}

type NewsAggPage struct {
    Title string
    News map[string]NewsMap
}

type Sitemapindex struct {
    Locations []string `xml:"sitemap>loc"`
}

type News struct {
    Titles []string `xml:"url>news>title"`
    Keywords []string `xml:"url>news>keywords"`
    Locations []string `xml:"url>loc"`
}

var news_map = make(map[string]NewsMap)

func newsRoutine(c chan News, Location string){
    defer wg.Done()
    var n News
    resp, _ := http.Get(Location)
    bytes, _ := ioutil.ReadAll(resp.Body)
    xml.Unmarshal(bytes, &n)
    resp.Body.Close()
    c <- n
}

func fetchNews() map[string]NewsMap {
	var s Sitemapindex
	resp, _ := http.Get("https://www.washingtonpost.com/news-sitemap-index.xml")
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &s)
	resp.Body.Close()
	queue := make(chan News, 30)

	for _, Location := range s.Locations {
	    wg.Add(1)
	    go newsRoutine(queue, Location)
	}
	wg.Wait()
	close(queue)

	for elem := range queue {
	    for idx, _ := range elem.Keywords {
	        news_map[elem.Titles[idx]] = NewsMap{elem.Keywords[idx], elem.Locations[idx]}
	    }
	}
	return news_map
}

func RefreshNews() {
	for {
		news_map = fetchNews()
		time.Sleep(60 * time.Second)
	}
}

func GetNews() map[string]NewsMap {
	return news_map
}
	<head>
	<script type="text/javascript" charset="utf8" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
	<link rel="stylesheet" type="text/css" href="//cdn.datatables.net/1.10.16/css/jquery.dataTables.css">
	<script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.16/js/jquery.dataTables.js"></script>
	</head>

	<h1>{{.Title}}</h1>

	<table id="example" class="display" cellspacing="0" width="100%">
	<col width="35%">
	<col width="65%">
	<thead>
	<tr>
	<th>Title</th>
	<th>Keywords</th>
	</tr>
	</thead>
	<tbody>

	{{ range $key, $value := .News }}
	<tr>
	<td><a href="{{ $value.Location }}" target='_blank'>{{ $key }}</td>
	<td>{{ $value.Keyword }}</td>
	</tr>
	{{ end }}
	</tbody>
	</table>



	<script>$(document).ready(function() {
	$('#example').DataTable();
	} );</script>
	package main

	import (
	"fmt"
	"net/http"
	"html/template"
	"newsfeed"
	)

	func newsAggHandler(w http.ResponseWriter, r *http.Request) {
	p := newsfeed.NewsAggPage{Title: "Amazing News Aggregator", News: newsfeed.GetNews()}
	t, _ := template.ParseFiles("aggregatorfinish.html")
	t.Execute(w, p)
	}

	func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "<h1>Whoa, Go is neat!</h1>")
	}

	func main() {
	go newsfeed.RefreshNews()
	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/agg/", newsAggHandler)
	http.ListenAndServe(":8000", nil)
	}
	package newsfeed

	import (
	"net/http"
	"encoding/xml"
	"io/ioutil"
	"sync"
	"time"
	)

	var wg sync.WaitGroup

	type NewsMap struct {
	Keyword string
	Location string
	}

	type NewsAggPage struct {
	Title string
	News map[string]NewsMap
	}

	type Sitemapindex struct {
	Locations []string `xml:"sitemap>loc"`
	}

	type News struct {
	Titles []string `xml:"url>news>title"`
	Keywords []string `xml:"url>news>keywords"`
	Locations []string `xml:"url>loc"`
	}

	var news_map = make(map[string]NewsMap)

	func newsRoutine(c chan News, Location string){
	defer wg.Done()
	var n News
	resp, _ := http.Get(Location)
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &n)
	resp.Body.Close()
	c <- n
	}

	func fetchNews() map[string]NewsMap {
	var s Sitemapindex
	resp, _ := http.Get("https://www.washingtonpost.com/news-sitemap-index.xml")
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &s)
	resp.Body.Close()
	queue := make(chan News, 30)

	for _, Location := range s.Locations {
	wg.Add(1)
	go newsRoutine(queue, Location)
	}
	wg.Wait()
	close(queue)

	for elem := range queue {
	for idx, _ := range elem.Keywords {
	news_map[elem.Titles[idx]] = NewsMap{elem.Keywords[idx], elem.Locations[idx]}
	}
	}
	return news_map
	}

	func RefreshNews() {
	for {
	news_map = fetchNews()
	time.Sleep(60 * time.Second)
	}
	}

	func GetNews() map[string]NewsMap {
	return news_map
	}