Download Guides Pdf from https://developer.apple.com/library/ios/navigation/#section=Resource%20Types&topic=Guides

download_guide_pdf.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os/exec"
	"strconv"
	"strings"

	"github.com/jmoiron/jsonq"
)

func main() {
	resp, err := http.Get("https://developer.apple.com/library/ios/navigation/library.json")
	PanicIfError(err)
	defer resp.Body.Close()

	data := map[string]interface{}{}
	dec := json.NewDecoder(resp.Body)
	dec.Decode(&data)
	jq := jsonq.NewQuery(data)

	// get guides' key
	contents, err := jq.ArrayOfObjects("topics", "0", "contents")
	PanicIfError(err)
	var guideKey int
	for _, v := range contents {
		name := v["name"]
		if name == "Guides" {
			keyStr := v["key"].(string)
			guideKey, err = strconv.Atoi(keyStr)
			PanicIfError(err)
			break
		}
	}

	// assmenble url
	basePath := "https://developer.apple.com/library/ios"
	documents, err := jq.ArrayOfArrays("documents")
	var guideDocPaths []string
	for _, v := range documents {
		key := v[2].(float64)
		if int(key) == guideKey {
			relativePath := v[9].(string)

			if relativePath[0:3] != "../" {
				fmt.Println("maybe some error at:", relativePath)
				continue
			}

			relativePath = relativePath[2:len(relativePath)]
			path := basePath + relativePath
			guideDocPaths = append(guideDocPaths, path)
			// fmt.Println(path)
		}
	}
	// fmt.Println("first guide path:", guideDocPaths[0])
	fmt.Println("library json loaded")

	downloadPdf(guideDocPaths)
}

func downloadPdf(guideDocPaths []string) {
	// download pdf
	// https://developer.apple.com/library/ios/documentation/Performance/Conceptual/ManagingMemory/book.json
	// https://developer.apple.com/library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/book.json

	downloadedFileNames := make(chan string, len(guideDocPaths))
	workers := make(chan bool, 10)

	for _, v := range guideDocPaths {
		workers <- true
		go func(guideDocPath string) {
			fmt.Println("begin parse:", guideDocPath)
			u, err := url.Parse(guideDocPath)
			PanicIfError(err)

			oldPath := u.Path
			u.Fragment = ""
			// get book.json
			// /library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/Introduction/Introduction.html
			// =>>>
			// /library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/book.json
			oldComps := strings.Split(oldPath, "/")
			lastNeededCompIndex := len(oldComps) - 1

		TryGetBookJSON:
			newComps := oldComps[0:lastNeededCompIndex]
			u.Path = strings.Join(newComps, "/") + "/book.json"
			bookFullPath := u.String()

			// get Pdf file name and download link
			resp, err := http.Get(bookFullPath)
			PanicIfError(err)
			defer resp.Body.Close()
			if resp.StatusCode != 200 {
				lastNeededCompIndex--
				if lastNeededCompIndex >= 0 {
					goto TryGetBookJSON
				} else {
					fmt.Println("ERROR:", err, "oldPath:", oldPath)
					releaseWorkerAndFileNames(downloadedFileNames, workers)
					return
				}
			}

			data := map[string]interface{}{}
			dec := json.NewDecoder(resp.Body)
			dec.Decode(&data)
			jq := jsonq.NewQuery(data)

			// fmt.Println("begin find pdf link for:", bookFullPath)
			pdfName, err := jq.String("PDF", "href")
			if err != nil {
				fmt.Println("find pdf href err:", err, " bookFullPath:", bookFullPath)
				releaseWorkerAndFileNames(downloadedFileNames, workers)
				return
			}
			// fmt.Println("pdfName:::", pdfName)
			pdfPath := strings.Join(newComps, "/") + "/" + pdfName
			u.Path = pdfPath
			pdfFullPath := u.String()
			// fmt.Println("pdfFullPath:::", pdfFullPath)

			// download it
			fmt.Println("begin download:", pdfFullPath)
			cmd := exec.Command("curl", pdfFullPath, "-o", pdfName)
			// var out bytes.Buffer
			// cmd.Stdout = &out
			// cmd.Stderr = &out
			err = cmd.Run()
			if err != nil {
				fmt.Println("download pdf err:", err, " pdfFullPath:", pdfFullPath)
				releaseWorkerAndFileNames(downloadedFileNames, workers)
				return
			}

			downloadedFileNames <- pdfName
			<-workers
			return
		}(v)
	}

	for i := 0; i < len(guideDocPaths); i++ {
		<-downloadedFileNames
	}
	fmt.Println("----------DONE----------")
}

func releaseWorkerAndFileNames(downloadedFileNames chan string, workers chan bool) {
	<-workers
	downloadedFileNames <- "DOWNLOAD FILE ERROR!!"
}

func PanicIfError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

download_prerelease_sample_code.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os/exec"
	"strconv"
	"strings"

	"github.com/jmoiron/jsonq"
)

func main() {
	resp, err := http.Get("https://developer.apple.com/library/prerelease/ios/navigation/library.json")
	PanicIfError(err)
	defer resp.Body.Close()

	data := map[string]interface{}{}
	dec := json.NewDecoder(resp.Body)
	dec.Decode(&data)
	jq := jsonq.NewQuery(data)

	// get guides' key
	contents, err := jq.ArrayOfObjects("topics", "0", "contents")
	PanicIfError(err)
	var guideKey int
	for _, v := range contents {
		name := v["name"]
		if name == "Sample Code" {
			keyStr := v["key"].(string)
			guideKey, err = strconv.Atoi(keyStr)
			PanicIfError(err)
			break
		}
	}

	// fmt.Println("key = ", guideKey)
	// assmenble url
	basePath := "https://developer.apple.com/library/prerelease/ios"
	documents, err := jq.ArrayOfArrays("documents")
	var guideDocPaths []string
	for _, v := range documents {
		key := v[2].(float64)
		if int(key) == guideKey {
			relativePath := v[9].(string)

			if relativePath[0:3] != "../" {
				fmt.Println("maybe some error at:", relativePath)
				continue
			}

			relativePath = relativePath[2:len(relativePath)]
			path := basePath + relativePath
			guideDocPaths = append(guideDocPaths, path)
			// fmt.Println(path)
		}
	}
	// fmt.Println("first guide path:", guideDocPaths[0])
	fmt.Println("library json loaded")

	downloadPdf(guideDocPaths)
}

func downloadPdf(guideDocPaths []string) {
	// download pdf
	// https://developer.apple.com/library/ios/documentation/Performance/Conceptual/ManagingMemory/book.json
	// https://developer.apple.com/library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/book.json

	downloadedFileNames := make(chan string, len(guideDocPaths))
	workers := make(chan bool, 10)

	for _, v := range guideDocPaths {
		workers <- true
		go func(guideDocPath string) {
			// fmt.Println("begin parse:", guideDocPath)
			u, err := url.Parse(guideDocPath)
			PanicIfError(err)

			oldPath := u.Path
			u.Fragment = ""
			// get book.json
			// /library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/Introduction/Introduction.html
			// =>>>
			// /library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/book.json
			oldComps := strings.Split(oldPath, "/")
			lastNeededCompIndex := len(oldComps) - 1

		TryGetBookJSON:
			newComps := oldComps[0:lastNeededCompIndex]
			u.Path = strings.Join(newComps, "/") + "/book.json"
			bookFullPath := u.String()

			// get Pdf file name and download link
			fmt.Println("before get: ", bookFullPath)
			resp, err := http.Get(bookFullPath)
			PanicIfError(err)
			defer resp.Body.Close()
			if resp.StatusCode != 200 {
				lastNeededCompIndex--
				if lastNeededCompIndex >= 0 {
					goto TryGetBookJSON
				} else {
					fmt.Println("ERROR:", err, "oldPath:", oldPath)
					releaseWorkerAndFileNames(downloadedFileNames, workers)
					return
				}
			}

			data := map[string]interface{}{}
			dec := json.NewDecoder(resp.Body)
			dec.Decode(&data)
			jq := jsonq.NewQuery(data)

			// fmt.Println("begin find pdf link for:", bookFullPath)
			pdfName, err := jq.String("sampleCode")
			if err != nil {
				fmt.Println("find pdf href err:", err, " bookFullPath:", bookFullPath)
				releaseWorkerAndFileNames(downloadedFileNames, workers)
				return
			}
			// fmt.Println("pdfName:::", pdfName)
			pdfPath := strings.Join(newComps, "/") + "/" + pdfName
			u.Path = pdfPath
			pdfFullPath := u.String()
			// fmt.Println("pdfFullPath:::", pdfFullPath)

			// download it
			fmt.Println("begin download:", pdfFullPath)
			cmd := exec.Command("curl", pdfFullPath, "-o", pdfName)
			// var out bytes.Buffer
			// cmd.Stdout = &out
			// cmd.Stderr = &out
			err = cmd.Run()
			if err != nil {
				fmt.Println("download pdf err:", err, " pdfFullPath:", pdfFullPath)
				releaseWorkerAndFileNames(downloadedFileNames, workers)
				return
			}

			downloadedFileNames <- pdfName
			<-workers
			return
		}(v)
	}

	for i := 0; i < len(guideDocPaths); i++ {
		<-downloadedFileNames
	}
	fmt.Println("----------DONE----------")
}

func releaseWorkerAndFileNames(downloadedFileNames chan string, workers chan bool) {
	<-workers
	downloadedFileNames <- "DOWNLOAD FILE ERROR!!"
}

func PanicIfError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}