export chrome history to markdown file.

weekly.go

package main

import (
	"bytes"
	"database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"time"

	// import sqlite3 package
	_ "github.com/mattn/go-sqlite3"
)

var (
	historyDefaultLoc = `/Library/Application Support/Google/Chrome/Default/History`
	historyLoc        string
	filterRuleFile    string
	outputDir         string
)

func main() {
	userhome, err := os.UserHomeDir()
	if err != nil {
		log.Fatalln("userhome", err)
	}

	flag.StringVar(&historyLoc, "f", userhome+historyDefaultLoc, "your chrome history file location; visit https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md")
	flag.StringVar(&filterRuleFile, "filter", "filter.txt", "path/to/your/filter/file, one host per line")
	flag.StringVar(&outputDir, "output", "", "output/path/to/your/markdownfile")
	flag.Parse()
	if _, err = os.Stat(historyLoc); err != nil {
		log.Println("Cannot find your Chrome Histroy File!")
		log.Println("Please visit this page to get your chrome history file: https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md")
		log.Println("Then manual special your chrome history file location using this way:")
		log.Println("./weekly -f /path/to/your/chrome/history/file")
		log.Fatalln("Try again with pleasure!")
	}
	var out bytes.Buffer
	cmd := exec.Command("cp", historyLoc, ".")
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		log.Fatal("cp err: ", err, " , ", out.String())
	}
	Write2MD(filterRuleFile, outputDir)
	log.Print("write finished!")
	if err := os.Remove("./History"); err != nil {
		log.Fatalln("remove history file err", err)
	}
}

// -------------- chrome history ------------- //

const (
	selectURLs = "SELECT id, url, title, datetime(last_visit_time/1000000-11644473600, 'unixepoch') as last_visit_time FROM urls WHERE last_visit_time > ? ORDER BY last_visit_time DESC"
)

var (
	chromeHistoryDBPath = "./History"

	windowsFiletime = int64(((1970-1601)*365 + 89) * 24 * 60 * 60 * 1000 * 1000)
)

// HistoryChrome read chrome history.
func HistoryChrome() map[string][]*History {
	dsn := fmt.Sprintf("file:%s?mode=ro&cache=private", chromeHistoryDBPath)
	db, err := sql.Open("sqlite3", dsn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	rows, err := db.Query(selectURLs, unixtime2filetime(time.Now().Add(-7*24*time.Hour)))
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	historys := []*History{}
	for rows.Next() {
		h := &History{}
		if err := rows.Scan(&h.ID, &h.URL, &h.Title, &h.LastVisitTime); err != nil {
			continue
		}
		historys = append(historys, h)
	}
	groupedHs := map[string][]*History{}
	for _, v := range historys {
		us, err := url.Parse(v.URL)
		if err != nil {
			continue
		}
		if _, ok := groupedHs[us.Host]; !ok {
			groupedHs[us.Host] = []*History{}
		}
		groupedHs[us.Host] = append(groupedHs[us.Host], v)
	}
	return groupedHs
}

// History info of history
type History struct {
	ID            int64
	URL           string
	Title         string
	LastVisitTime string
}

func unixtime2filetime(t time.Time) int64 {
	return t.UnixNano()/1000 + windowsFiletime
}

func filetime2unixtime(f int64) int64 {
	return (f - windowsFiletime) / 1000 / 1000
}

// --------- writer.go --------- //

var (
	filters = map[string]bool{}
)

// Write2MD Write2MD
func Write2MD(filterFile, outputDir string) {
	if filterFile != "" {
		fs, err := ioutil.ReadFile(filterFile)
		if err != nil {
			log.Fatalln("open filter file err", err)
		}
		ss := strings.Split(string(fs), "\n")
		for _, v := range ss {
			if _, ok := filters[v]; !ok {
				filters[v] = true
			}
		}
	}
	hs := HistoryChrome()
	newFilename := fmt.Sprintf("weekly-%s.md", time.Now().Format("2006-01-02"))
	if outputDir != "" {
		newFilename = outputDir + "/" + newFilename
	}
	if _, err := os.Stat(newFilename); err == nil {
		os.Remove(newFilename)
	}
	f, err := os.OpenFile(newFilename, os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		log.Fatalln("create file", err)
	}
	f.WriteString("---\n")
	title := fmt.Sprintf("一周阅读(%s~%s)", time.Now().Add(-7*24*time.Hour).Format("2006-01-02"), time.Now().Format("2006-01-02"))
	f.WriteString(fmt.Sprintf(`title : "%s"`, title))
	f.WriteString("\n")
	f.WriteString(`tags: "一周阅读"`)
	f.WriteString("\n")
	f.WriteString("---\n")
	f.WriteString("\n")
	for host, ss := range hs {
		if filter(host) {
			continue
		}
		f.WriteString(fmt.Sprintf("## %s\n", host))
		f.WriteString("\n")
		um := map[string]bool{}
		for i, s := range ss {
			if filterURL(s.URL) {
				continue
			}
			if _, ok := um[s.URL]; !ok {
				f.WriteString(fmt.Sprintf("%d. 【%s】[%s](%s) %s\n", i, s.Title, s.URL, s.URL, s.LastVisitTime))
			}
		}
		f.WriteString("\n")
	}
}

func filter(s string) bool {
	if _, ok := filters[s]; ok {
		return true
	}
	if strings.HasPrefix("local", s) || strings.HasPrefix("127.0", s) || strings.HasPrefix("0.0", s) {
		return true
	}
	switch s {
	case "map.baidu.com":
	case "accounts.google.com":
		return true
	}
	return false
}

var (
	urlsmap = map[string]bool{}
)

func filterURL(s string) bool {
	url, err := url.Parse(s)
	if err != nil {
		return false
	}
	u := fmt.Sprintf("%s/%s", url.Host, url.Path)
	if _, ok := urlsmap[u]; ok {
		return true
	}
	urlsmap[u] = true
	if strings.HasPrefix(s, "chrome-extension:") {
		return true
	}
	return false
}