main.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path"

	"github.com/hibiken/asynq"
)

type Links []Link

func UnmarshalLinks(data []byte) (Links, error) {
	var r Links
	err := json.Unmarshal(data, &r)
	return r, err
}

func (r *Links) Marshal() ([]byte, error) {
	return json.Marshal(r)
}

type Link struct {
	ID     ID       `json:"_id"`
	Links  []string `json:"links"`
	Title  string   `json:"title"`
	Number float64  `json:"number"`
	Manga  ID       `json:"manga"`
}

type ID struct {
	OID string `json:"$oid"`
}

const redisAddr = "127.0.0.1:6379"

const (
	EmailDelivery = "email:deliver"
)

func main() {

	redis := asynq.RedisClientOpt{Addr: redisAddr}
	client := asynq.NewClient(redis)
	defer client.Close()

	// Open our jsonFile
	jsonFile, err := os.Open("links.json")
	// if we os.Open returns an error then handle it
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("Successfully Opened users.json")
	// defer the closing of our jsonFile so that we can parse it later on
	defer jsonFile.Close()

	// read our opened jsonFile as a byte array.
	byteValue, _ := ioutil.ReadAll(jsonFile)

	links, err := UnmarshalLinks(byteValue)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println(len(links))

	for i, item := range links {
		if _, err := os.Stat(item.Manga.OID); os.IsNotExist(err) {
			os.MkdirAll(item.Manga.OID, 0755)
		}

		for _, fileUrl := range item.Links {
			myUrl, err := url.Parse(fileUrl)
			if err != nil {
				fmt.Println(err)
			}
			folder := item.Manga.OID + "/" + fmt.Sprintf("%f", item.Number)
			if _, err := os.Stat(folder); os.IsNotExist(err) {
				os.MkdirAll(folder, 0755)
			}

			payload := map[string]interface{}{"filepath": folder + "/" + path.Base(myUrl.Path), "imageURL": fileUrl}
			client.Enqueue(asynq.NewTask(EmailDelivery, payload)) //NewTask(EmailDelivery, payload)
			// go DownloadFile(folder+"/"+path.Base(myUrl.Path), fileUrl)
		}
		if i == 50 {
			os.Exit(0)
		}
	}
}

// func DownloadFile(filepath string, url string) {

// 	// Get the data
// 	resp, err := http.Get(url)
// 	if err != nil {
// 		fmt.Println(err)
// 		return
// 	}
// 	defer resp.Body.Close()

// 	// Create the file
// 	out, err := os.Create(filepath)
// 	if err != nil {
// 		fmt.Println(err)
// 		return
// 	}
// 	defer out.Close()

// 	// Write the body to file
// 	_, err = io.Copy(out, resp.Body)
// 	if err != nil {
// 		fmt.Println(err)
// 		return
// 	}
// }

server.go

package main

import (
	"context"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/hibiken/asynq"
)

const redisAddr = "127.0.0.1:6379"

const (
	EmailDelivery = "email:deliver"
)

func main() {
	r := asynq.RedisClientOpt{Addr: redisAddr}

	srv := asynq.NewServer(r, asynq.Config{
		Concurrency: 10,
	})

	// mux maps a type to a handler
	mux := asynq.NewServeMux()
	mux.HandleFunc(EmailDelivery, HandleEmailDeliveryTask)

	if err := srv.Run(mux); err != nil {
		log.Fatalf("could not run server: %v", err)
	}
}

func HandleEmailDeliveryTask(ctx context.Context, t *asynq.Task) error {
	filepath, err := t.Payload.GetString("filepath")
	if err != nil {
		return err
	}
	imageURL, err := t.Payload.GetString("imageURL")
	if err != nil {
		return err
	}

	// Get the data
	resp, err := http.Get(imageURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Create the file
	out, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer out.Close()

	// Write the body to file
	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return err
	}
	return nil
}