Golang NGM patches crawler

main.go

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
)

var (
	hostname           string
	patchDir           string
	startPatch         int
	maxErrorCount         int
	patchVersionFormat string
)

func GetFiles(version int) ([]string, error) {
	var contents string = ""

	localDir := patchDir + fmt.Sprintf(patchVersionFormat, version) + "/"
	remoteDir := hostname + localDir

	if binaryContents, err := ioutil.ReadFile(localDir + "files.nfo2"); err != nil && len(contents) > 0 {
		contents = string(binaryContents)
	} else {

		res, err := http.Get(remoteDir + "files.nfo2")
		if err != nil {
			return nil, err
		}

		if res.StatusCode != 200 {
			return nil, errors.New("File not found: " + remoteDir + "files.nfo2")
		}

		os.MkdirAll(localDir, 0755)

		binaryContents, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		contents = string(binaryContents)

		ioutil.WriteFile(localDir+"files.nfo2", binaryContents, 0755)

	}

	lines := strings.Split(contents, "\r\n")

	startLine := lines[0]

	if !strings.HasPrefix(startLine, "NFO") {
		return nil, errors.New("This is not a valid NFO2 file")
	}

	nfoVersion, err := strconv.Atoi(startLine[3:6])
	if err != nil {
		return nil, err
	}
	if nfoVersion != 200 && nfoVersion != 400 {
		return nil, errors.New("Unsupported NFO version: " + strconv.Itoa(nfoVersion))
	}

	l := make([]string, 0, len(lines))
	l = append(l, remoteDir+"files.nfo2") // Add main files.nfo2 thing
	lines = lines[1 : len(lines)-1]

	for _, line := range lines {
		elements := strings.Split(line, ",")
		elements = elements[:len(elements)-1]

		for indx, element := range elements {
			end := len(element)
			elements[indx] = element[1 : end-1]
		}

		filename := elements[0]

		// Replace backslashes with forward slashes
		filename = strings.Replace(filename, "\\", "/", -1)

		if nfoVersion == 200 {
			// GZ compressed files
			filename += ".gz"
		} else if nfoVersion == 400 {
			// GZ compressed files with special hash

			magicNumber, _ := strconv.Atoi(elements[1])
			if magicNumber < 0 {
				magicNumber = int((uint(0xFFFFFFFF) + uint(magicNumber)) + 1)
			}

			filename += fmt.Sprintf(".%08x.nxgz", magicNumber)
		}

		downloadURL := remoteDir + filename

		l = append(l, downloadURL)
	}

	return l, nil

}

func main() {
	fmt.Println("NGM patch downloader")

	flag.StringVar(&hostname, "hostname", "http://maplestory2.dn.nexoncdn.co.kr/", "Hostname of the download location")
	flag.StringVar(&patchDir, "patchdir", "real/Patch/", "Directory of the patchfiles")
	flag.IntVar(&startPatch, "startpatch", 1, "Patch to start scanning from")
	flag.StringVar(&patchVersionFormat, "patch-version-format", "%d", "Sprintf format of the patch version.")
	flag.IntVar(&maxErrorCount, "max-errors", 5, "Max amount of 404's until stopping")
	flag.Parse()

	downloads := make([]string, 0)

	os.MkdirAll(patchDir, 0755)

	failCounter := 0
	foundDownload := false
	for i := startPatch; ; i++ {
		if l, err := GetFiles(i); err != nil {
			if foundDownload && failCounter >= maxErrorCount {
				fmt.Println("End!")
				break
			} else {
				fmt.Println("Not found: ", err)
				failCounter++
			}
		} else {
			fmt.Println("Added files for version ", i)
			foundDownload = true
			downloads = append(downloads, l...)
			failCounter = 0
		}
	}

	ioutil.WriteFile("todownload.txt", []byte(strings.Join(downloads, "\n")), 0755)
}