parse_amazon_xml_th.go

package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"strconv"
	"strings"
	"time"
)

type ImageInfo struct {
	url    string
	width  int
	height int
}

type BookInfo struct {
	asin            string
	title           string
	binding         string
	author          string
	publisher       string
	publicationDate string
	images          map[string]ImageInfo
}

func parseXmls(result chan []BookInfo, xmls []string) {
	bookInfos := []BookInfo{}
	for _, xml := range xmls {
		dom, _ := goquery.NewDocumentFromReader(strings.NewReader(xml))
		dom.Find("Item").Each(func(_ int, item *goquery.Selection) {
			bookInfo := BookInfo{}
			bookInfo.asin = item.Find("ASIN").Text()
			attributes := item.Find("ItemAttributes").First()
			if attributes.Length() > 0 {
				bookInfo.title = attributes.Find("Title").Text()
				bookInfo.binding = attributes.Find("Binding").Text()
				bookInfo.author = attributes.Find("Author").Text()
				bookInfo.publisher = attributes.Find("Publisher").Text()
				bookInfo.publicationDate = attributes.Find("PublicationDate").Text()
			}
			imageLabels := []string{
				"SmallImage",
				"MediumImage",
				"LargeImage",
			}
			images := map[string]ImageInfo{}
			for _, imageLabel := range imageLabels {
				xml := item.Find(imageLabel).First()
				url := xml.Find("URL").Text()
				width, _ := strconv.Atoi(xml.Find("Height").Text())
				height, _ := strconv.Atoi(xml.Find("Width").Text())
				image := ImageInfo{url, width, height}
				images[imageLabel] = image
			}
			bookInfo.images = images
			bookInfos = append(bookInfos, bookInfo)
		})
	}
	result <- bookInfos
}

func getXmls() []string {
	xmls := []string{}
	for i := 0; i <= 1440; i += 10 {
		path := fmt.Sprintf("xmls/%d.xml", i)
		xml, _ := ioutil.ReadFile(path)
		xmls = append(xmls, string(xml))
	}
	return xmls
}

func divideXmls(xmls []string, num int) [][]string {
	xmlsNum := len(xmls)
	size := xmlsNum / num
	result := [][]string{}
	for i := 0; i < num; i++ {
		start := size * i
		end := size * (i + 1)
		if i == (num - 1) {
			end = xmlsNum
		}
		result = append(result, xmls[start:end])
	}
	return result
}

func main() {
	allXmls := getXmls()
	divXmls := divideXmls(allXmls, 4)
	start := time.Now()

	result := make(chan []BookInfo)
	for _, xmls := range divXmls {
		go parseXmls(result, xmls)
	}
	bookInfos := []BookInfo{}
	for _, _ = range divXmls {
		bookInfos = append(bookInfos, <-result...)
	}

	end := time.Now()
	fmt.Printf("xml数: %d\n", len(allXmls))
	fmt.Printf("book数: %d\n", len(bookInfos))
	fmt.Printf("parse時間: %f秒\n", (end.Sub(start)).Seconds())
}