Skip to content

Instantly share code, notes, and snippets.

@fasmide
Last active May 15, 2016 21:42
Show Gist options
  • Save fasmide/8f4cb5620c58f7b1a2ce4e30120513ad to your computer and use it in GitHub Desktop.
Save fasmide/8f4cb5620c58f7b1a2ce4e30120513ad to your computer and use it in GitHub Desktop.
package main
import (
"fmt"
"log"
"regexp"
"strconv"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/influxdata/influxdb/client/v2"
)
/*
* Today we will be scraping the webpage
* https://www.google.dk/search?q=weather+aalborg&oq=weather+aalborg&sourceid=chrome&ie=UTF-8
* and logging into influxdb
*/
type Weather struct {
Temp float64
Humidity uint64
}
var re = regexp.MustCompile("[^0-9]")
func Save(w *Weather) {
// Make client
c, err := client.NewHTTPClient(client.HTTPConfig{
Addr: "http://localhost:8086",
Username: "",
Password: "",
})
if err != nil {
log.Fatalln("Influx error: ", err)
}
// Create a new point batch
bp, err := client.NewBatchPoints(client.BatchPointsConfig{
Database: "house",
Precision: "s",
})
if err != nil {
log.Fatalln("Influx error: ", err)
}
// Create a point and add to batch
tags := map[string]string{"location": "outside"}
fields := map[string]interface{}{
"temperature": w.Temp,
"humidity": w.Humidity,
}
pt, err := client.NewPoint("house", tags, fields, time.Now())
if err != nil {
log.Fatalln("Error: ", err)
}
bp.AddPoint(pt)
// Write the batch
c.Write(bp)
log.Printf("Data saved to influx")
}
func Scrape() Weather {
doc, err := goquery.NewDocument("https://www.google.dk/search?q=weather+aalborg&oq=weather+aalborg&sourceid=chrome&ie=UTF-8")
if err != nil {
log.Fatal(err)
}
tempText := doc.Find("#ires > ol > div:nth-child(1) > div > table:nth-child(2) > tbody > tr:nth-child(1) > td:nth-child(2) > span").Text()
humidityText := doc.Find("#ires > ol > div:nth-child(1) > div > table:nth-child(2) > tbody > tr:nth-child(5) > td:nth-child(1)").Text()
temp, err := strconv.ParseFloat(re.ReplaceAllString(tempText, ""), 64)
if err != nil {
log.Printf("Cannot parse %s: %s", tempText, err.Error())
}
humidity, err := strconv.ParseUint(re.ReplaceAllString(humidityText, ""), 10, 64)
if err != nil {
log.Printf("Cannot parse %s: %s", humidityText, err.Error())
}
return Weather{Temp: temp, Humidity: humidity}
}
func main() {
fmt.Println("Hej mide")
w := Scrape()
fmt.Printf("Weather: %+v\n", w)
Save(&w)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment