Golang MapleStory patches crawler

main.go

package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/dutchcoders/goftp"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

var (
	hostname   string
	patchDir   string
	newFormat  bool
	ftp        *goftp.FTP   = nil
	httpClient *http.Client = &http.Client{
		CheckRedirect: redirectPolicyFunc,
	}
)

func redirectPolicyFunc(req *http.Request, via []*http.Request) error {
	return errors.New("We no need no redirects.")
}

func GetFile(fileurl string) ([]byte, error) {
	urlInfo, _ := url.Parse(fileurl)
	if urlInfo.Scheme == "ftp" {
		var err error
		if ftp == nil {
			hostAndPort := urlInfo.Host
			if strings.Index(hostAndPort, ":") == -1 {
				hostAndPort += ":21"
			}

			ftp, err = goftp.ConnectDbg(hostAndPort)
			if err != nil {
				return nil, err
			}

			if err := ftp.Login("anonymous", ""); err != nil {
				return nil, err
			}
		}

		var alldata []byte
		_, err = ftp.Retr(urlInfo.Path, func(r io.Reader) error {
			alldata, _ = ioutil.ReadAll(r)
			return err
		})

		if err != nil {
			return nil, err
		}

		return alldata, nil
	} else if urlInfo.Scheme == "http" || urlInfo.Scheme == "https" {

		res, err := httpClient.Get(fileurl)
		if err != nil {
			return nil, err
		}

		if res.StatusCode != 200 {
			return nil, errors.New("File not found: " + fileurl)
		}

		binaryContents, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		return binaryContents, nil
	} else {
		return nil, errors.New("Unsupported protocol")
	}
}

func GetFiles(version int) ([]string, error) {

	localDir := patchDir
	if !newFormat {
		localDir += "patchdir/"
	}
	localDir += fmt.Sprintf("%05d", version) + "/"
	remoteDir := hostname + localDir

	binaryContents, err := GetFile(remoteDir + "Version.info")
	if err != nil {
		return nil, err
	}

	contents := string(binaryContents)

	lines := strings.Split(contents, "\r\n")

	patches := make([]string, 0)
	for _, line := range lines {
		if !strings.HasPrefix(line, "0x") && line != "" {
			// This must be version number
			oldVersion, _ := strconv.Atoi(line)

			if oldVersion != version {
				patchFilename := remoteDir + fmt.Sprintf("%05dto%05d.patch", oldVersion, version)

				patches = append(patches, patchFilename)
			}
		}
	}

	patches = append(patches, remoteDir+"Version.info", remoteDir+"ExePatch.dat", remoteDir+"NewPatcher.dat", remoteDir+"Crc.info")

	if !newFormat {
		patches = append(patches, hostname+patchDir+"notice/"+fmt.Sprintf("%05d.txt", version))
	}

	return patches, nil
}

func main() {
	fmt.Println("MapleStory patches downloader")

	var startVersion int
	flag.IntVar(&startVersion, "startversion", 2, "Start version")
	flag.StringVar(&hostname, "hostname", "", "Host of the download server")
	flag.StringVar(&patchDir, "patchdir", "patchdir/", "Path of 'patchdir'")
	flag.BoolVar(&newFormat, "new-format", false, "New format type of patching (no patchdir)")

	flag.Parse()

	downloads := make([]string, 0)
	errorCounter := 0
	foundFiles := false

	for i := startVersion; ; i++ {
		if files, err := GetFiles(i); err != nil {
			fmt.Println("Error while fetching files:", err)
			if foundFiles {
				if errorCounter > 2 {
					fmt.Println("Stopping!")
					break
				} else {
					errorCounter++
				}
			}
		} else {
			fmt.Println("Got files for version", i)
			downloads = append(downloads, files...)
			errorCounter = 0
			foundFiles = true
		}
	}

	ioutil.WriteFile("todownload.txt", []byte(strings.Join(downloads, "\n")), 0755)
}