Skip to content

Instantly share code, notes, and snippets.

@eminaksehirli
Last active August 29, 2015 14:08
Show Gist options
  • Save eminaksehirli/5107600f36536cbab0ef to your computer and use it in GitHub Desktop.
Save eminaksehirli/5107600f36536cbab0ef to your computer and use it in GitHub Desktop.
package main
import (
"io/ioutil"
"net/http"
"os"
fp "path/filepath"
re "regexp"
"sync"
)
func main() {
p, e := re.Compile("(http://www.standaard.be/cnt/dmf.*?)\"")
check(e)
r, e := http.Get("http://www.standaard.be/nieuws/chronologisch")
check(e)
b, e := ioutil.ReadAll(r.Body)
check(e)
ls := p.FindAll(b, 10000)
wg := new(sync.WaitGroup)
wg.Add(len(ls))
for _, l := range ls {
s := string(l)
go func() {
c, _ := http.Get(s)
b, e = ioutil.ReadAll(c.Body)
check(e)
ioutil.WriteFile(fp.Join("/tmp/a/", fp.Base(s)), b, os.ModePerm)
wg.Done()
}()
}
wg.Wait()
}
func check(e error) {
if e != nil {
panic(e)
}
}
package main
import (
"io/ioutil"
"net/http"
"os"
fp "path/filepath"
re "regexp"
"sync"
)
func main() {
p, e := re.Compile("(http://www.standaard.be/cnt/dmf.*?)\"")
check(e)
r, e := http.Get("http://www.standaard.be/nieuws/chronologisch")
check(e)
b, e := ioutil.ReadAll(r.Body)
check(e)
ls := p.FindAll(b, 10000)
wg := new(sync.WaitGroup)
wg.Add(8)
ps := len(ls) / 8
for i := 0; i < 7; i++ {
go dl(ls[:ps], wg)
ls = ls[ps:]
}
dl(ls, wg)
wg.Wait()
}
func dl(ls [][]byte, wg *sync.WaitGroup) {
for _, l := range ls {
s := string(l)
c, _ := http.Get(s)
b, e := ioutil.ReadAll(c.Body)
check(e)
ioutil.WriteFile(fp.Join("/tmp/a/", fp.Base(s)), b, os.ModePerm)
}
wg.Done()
}
func check(e error) {
if e != nil {
panic(e)
}
}
package main
import (
"io/ioutil"
"net/http"
"os"
fp "path/filepath"
re "regexp"
)
func main() {
p, e := re.Compile("(http://www.standaard.be/cnt/dmf.*?)\"")
check(e)
r, e := http.Get("http://www.standaard.be/nieuws/chronologisch")
check(e)
b, e := ioutil.ReadAll(r.Body)
check(e)
ls := p.FindAll(b, 10000)
for _, l := range ls {
s := string(l)
c, _ := http.Get(s)
b, e = ioutil.ReadAll(c.Body)
check(e)
ioutil.WriteFile(fp.Join("/tmp/a/", fp.Base(s)), b, os.ModePerm)
}
}
func check(e error) {
if e != nil {
panic(e)
}
}
For 1 Processor:
dl_1:
real 0m1.081s
user 0m0.112s
sys 0m0.084s
dl_2:
real 0m1.016s
user 0m0.092s
sys 0m0.056s
dl_3:
real 0m4.330s
user 0m0.104s
sys 0m0.088s
For 8 processors:
dl_1:
real 0m1.263s
user 0m0.092s
sys 0m0.164s
dl_2:
real 0m1.176s
user 0m0.124s
sys 0m0.108s
dl_3:
real 0m4.323s
user 0m0.100s
sys 0m0.144s
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment