Skip to content

Instantly share code, notes, and snippets.

@peterbe
Created May 15, 2018 19:40
Show Gist options
  • Save peterbe/0ad409604e3970f32e39e793effba3cf to your computer and use it in GitHub Desktop.
Save peterbe/0ad409604e3970f32e39e793effba3cf to your computer and use it in GitHub Desktop.
package main
import (
"compress/gzip"
"encoding/csv"
"fmt"
"log"
"os"
"path/filepath"
"strconv"
"time"
)
func count(fn string, index int) (int, int) {
fmt.Printf("%d %v\n", index, fn)
f, err := os.Open(fn)
if err != nil {
log.Fatal(err)
}
defer f.Close()
gr, err := gzip.NewReader(f)
if err != nil {
log.Fatal(err)
}
defer gr.Close()
cr := csv.NewReader(gr)
rec, err := cr.ReadAll()
if err != nil {
log.Fatal(err)
}
var count = 0
var total = 0
layout := "2006-01-02T15:04:05.000Z"
minimum, err := time.Parse(layout, "2017-11-02T00:00:00.000Z")
if err != nil {
log.Fatal(err)
}
for _, v := range rec {
last_modified := v[3]
t, err := time.Parse(layout, last_modified)
if err != nil {
log.Fatal(err)
}
if t.After(minimum) {
count += 1
}
total += 1
}
return total, count
}
func FloatToString(input_num float64) string {
// to convert a float number to a string
return strconv.FormatFloat(input_num, 'f', 2, 64)
}
func main() {
var pattern = "*.csv.gz"
files, err := filepath.Glob(pattern)
if err != nil {
panic(err)
}
total := int(0)
recent := int(0)
for i, fn := range files {
if len(fn) == 39 {
// fmt.Println(fn)
c, t := count(fn, i)
total += t
recent += c
}
}
fmt.Println(total)
fmt.Println(recent)
ratio := float64(recent) / float64(total)
fmt.Println(FloatToString(100.0 * ratio))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment