scraperapikerins

package main
  
import (
   "encoding/json"
   "fmt"
   "net"
   "net/http"
   "os"
   "time"
  

scraperapikerins

c.WithTransport(&http.Transport{
    DialContext: (&net.Dialer{
        Timeout:   60 * time.Second,
        KeepAlive: 30 * time.Second,
        DualStack: true,
    }).DialContext,
    MaxIdleConns:          100,
    IdleConnTimeout:       90 * time.Second,
    TLSHandshakeTimeout:   10 * time.Second,
    ExpectContinueTimeout: 1 * time.Second,

scraperapikerins

c.OnHTML("a.paging-controls__next.js-page-control", func(p *colly.HTMLElement) {
    nextPage := p.Request.AbsoluteURL(p.Attr("data-href"))
    c.Visit(nextPage)
})

scraperapikerins

import (
   "encoding/json"
   "fmt"
   "os"
  
   "github.com/gocolly/colly"
)

scraperapikerins

   //We pass the data to Marshal
   content, err := json.Marshal(allProducts)
   //We need to handle someone the potential error
   if err != nil {
       fmt.Println(err.Error())
   }
   //We write a new file passing the file name, data, and permission
   os.WriteFile("jack-shoes.json", content, 0644)
}

scraperapikerins

c.OnHTML("div.product-tile__content-wrapper", func(h *colly.HTMLElement) {
    products := products{
        Name:  h.ChildText("a.product-tile__name__link.js-product-tile-link"),
        Price: h.ChildText("em.value__price"),
        URL:   h.ChildAttr("a.product-tile__name__link.js-product-tile-link", "href"),
    }
    
    fmt.Println(products)
})

scraperapikerins

type products struct {
   Name, Price, URL string
}

scraperapikerins

type products struct {
   Name  string `json:"name"`
   Price string `json:"price"`
   URL   string `json:"url"`
}

scraperapikerins

type products struct {
   Name  string
   Price string
   URL   string
}

scraperapikerins

c.OnHTML("div.product-tile__content-wrapper", func(h *colly.HTMLElement) {
    name := h.ChildText("a.product-tile__name__link.js-product-tile-link")
    price := h.ChildText("em.value__price")
    url := h.ChildAttr("a.product-tile__name__link.js-product-tile-link", "href")
    fmt.Println(name, price, url)
})