scrape-vex-step.go

package main

import (
  "fmt"
  "net/http"
  "regexp"
  "io/ioutil"
  "time"
)

func main() {
  resp, err := http.Get("http://www.vexforum.com/wiki/index.php/VEXCAD")
  if err != nil {
    fmt.Println("Error!")
    fmt.Println(err)
    return
  }
  defer resp.Body.Close()
  contents, err := ioutil.ReadAll(resp.Body)
  if err != nil {
    fmt.Println("Error!")
    fmt.Println(err)
  }
  re, _ := regexp.Compile(`<a href="(.*?)" class="external text" title="(.*?)" rel="nofollow">Download</a>`)
  res := re.FindAllStringSubmatch(string(contents), -1)
  for _, link := range res {
    re, _ = regexp.Compile(`http://content.vexrobotics.com/cad/STEP/((.*?).zip)`)
    res = re.FindAllStringSubmatch(link[1], -1)
    if len(res) > 0 {
      resp, _ := http.Get(string(res[0][0]))
      if body, err := ioutil.ReadAll(resp.Body); err == nil {
        fmt.Println("Downloading ", res[0][1])
        if err := ioutil.WriteFile(res[0][1], body, 0644); err != nil {
          fmt.Println("Failed to write ", res[0][1])
        }
      }
    }
    time.Sleep(time.Second * 30)
  }
}