Migration tool from IPFS pinning provider to Storj IPFS.

main.go

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"

	"storj.io/ipfs-user-mapping-proxy/proxy"
)

const (
	inputFile            = "pinned_hashes.json"
	downloadBaseURL      = "https://cloudflare-ipfs.com/ipfs"
	dagImportEndpointURL = "https://www.storj-ipfs.com/api/v0/dag/import"
	pinUser              = ""
	pinPassword          = ""
)

func main() {
	cids, err := readInputFile(inputFile)
	if err != nil {
		log.Fatal(err)
	}

	for _, cid := range cids {
		err := migrate(cid)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func readInputFile(file string) ([]string, error) {
	var cids []string

	log.Println("Reading", file)

	data, err := os.ReadFile(file)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(data, &cids)
	if err != nil {
		return nil, err
	}

	log.Printf("Read %d cids", len(cids))

	return cids, err
}

func migrate(cid string) error {
	log.Println("Migrating", cid)

	car, err := downloadCAR(cid)
	if err != nil {
		return err
	}

	err = importCAR(cid, car)
	if err != nil {
		return err
	}

	return nil
}

func downloadCAR(cid string) ([]byte, error) {
	url, err := url.JoinPath(downloadBaseURL, cid)
	if err != nil {
		return nil, err
	}
	url += "?format=car"

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("unexpected response status code: expected %d, got %d", http.StatusOK, resp.StatusCode)
	}

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	log.Printf("Downloaded %s (%d bytes)", url, len(data))

	return data, nil
}

func importCAR(cid string, data []byte) error {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	err := func() error {
		defer writer.Close()

		fw, err := writer.CreateFormFile("file", "car")
		if err != nil {
			return err
		}

		_, err = fw.Write(data)
		if err != nil {
			return err
		}

		return nil
	}()
	if err != nil {
		return err
	}

	url := dagImportEndpointURL + "?stats"

	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body.Bytes()))
	if err != nil {
		return err
	}

	req.SetBasicAuth(pinUser, pinPassword)
	req.Header.Set("Content-Type", writer.FormDataContentType())

	log.Printf("Uploading to %s for user %s", dagImportEndpointURL, pinUser)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)

	var rootMsg proxy.DAGImportResponseMessage
	err = decoder.Decode(&rootMsg)
	if err != nil {
		return err
	}

	var statsMsg proxy.DAGImportResponseMessage
	err = decoder.Decode(&statsMsg)
	if err != nil {
		return err
	}

	if cid == rootMsg.Root.Cid["/"] {
		log.Printf("CID matches the hash of the uploaded content (%d blocks, %d bytes).",
			statsMsg.Stats.BlockCount, statsMsg.Stats.BlockBytesCount)
	} else {
		return fmt.Errorf("CID does not match the hash of the uploaded content (%s != %s)",
			cid, rootMsg.Root.Cid["/"])
	}

	return nil
}