miner.go

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/sha1"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/mmcdole/gofeed"
)

func hashLink(link string) string {
	h := sha1.New()
	h.Write([]byte(link))
	return fmt.Sprintf("%x", h.Sum(nil))
}

var feedURL string
var targetDir string
var verbose bool

func parseFlags() {
	flag.StringVar(&feedURL, "feed", "", "URL to RSS or Atom feed")
	flag.StringVar(&targetDir, "out", ".", "Target directory")
	flag.BoolVar(&verbose, "verbose", false, "Enable debug output")
	flag.Parse()

	if feedURL == "" {
		log.Fatalln("Flag --feed is required")
	}
}

func main() {
	// Disable date/time printing in logs
	log.SetFlags(0)

	parseFlags()

	// Download feed
	fp := gofeed.Parser{}
	feed, err := fp.ParseURL(feedURL)
	if err != nil {
		log.Fatalln(err)
	}

	// Create tar.gz archive file
	timestamp := time.Now().Format("20060102-150405")
	file, err := os.Create(path.Join(targetDir, timestamp+".tar.gz"))
	defer file.Close()
	if err != nil {
		log.Fatalln(err)
	}
	gz, err := gzip.NewWriterLevel(file, gzip.BestCompression)
	defer gz.Close()
	if err != nil {
		log.Fatalln(err)
	}
	targz := tar.NewWriter(gz)
	defer targz.Close()

	// Save every referenced link in the archive
	for _, item := range feed.Items {
		if verbose {
			log.Println(item.Link)
		}

		// Request news page
		resp, err := http.Get(item.Link)
		if err != nil {
			log.Fatalln(err)
		}
		defer resp.Body.Close()

		// Copy news page to temporary buffer
		buf := new(bytes.Buffer)
		if _, err := io.Copy(buf, resp.Body); err != nil {
			log.Fatalln(err)
		}

		// Write temporary buffer into archive
		hdr := &tar.Header{
			Name: hashLink(item.Link),
			Mode: 0664,
			Size: int64(buf.Len()),
		}
		if err := targz.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := io.Copy(targz, buf); err != nil {
			log.Fatalln(err)
		}
	}
}