use erasure encoding to survive data loss

erasure_encoding.go

package main

import (
	"fmt"
	"os"

	"github.com/klauspost/reedsolomon"
)

// You can use erasure encoding to;
// 1. survive data loss
// 2. reduce tail latencies
// 3. etc
// https://twitter.com/copyconstruct/status/1520085040260034560
// https://brooker.co.za/blog/2023/01/06/erasure.html

// Also LOOK into CONTENT-DEFINED CHUNKING
// 1. https://blog.gopheracademy.com/advent-2018/split-data-with-cdc/
// 2. https://restic.net/blog/2015-09-12/restic-foundation1-cdc/

const originalText = `Reed-Solomon codes are a group of error-correcting codes that were introduced by 
Irving S. Reed and Gustave Solomon in 1960.
They have many applications, the most prominent of which include consumer technologies such as 
MiniDiscs, CDs, DVDs, Blu-ray discs, QR codes, 
data transmission technologies such as DSL and WiMAX, broadcast systems such as satellite 
communications, DVB and ATSC, and storage systems such as RAID 6.`

func main() {
	data := []byte(originalText)
	lenData := len(data)

	// Create an encoder with N data and X parity slices.
	totalShards := 20
	parityShards := 3
	dataShards := totalShards - parityShards
	enc, _ := reedsolomon.New(dataShards, parityShards)

	// Split the data into shards
	shards, _ := enc.Split(data)
	data = nil

	// Encode the parity set
	_ = enc.Encode(shards)

	// Verify the parity set
	ok, _ := enc.Verify(shards)
	if !ok {
		panic("verification failed")
	}

	//  Delete X shards(ie, simulate partial data loss)
	for i := 0; i < parityShards; i++ {
		shards[i] = nil
	}

	// Reconstruct the shards
	_ = enc.Reconstruct(shards)

	// Verify the data set
	ok, _ = enc.Verify(shards)
	if !ok {
		panic("verification failed")
	}

	// join the data.
	err := enc.Join(os.Stdout, shards, lenData)
	if err != nil {
		panic(fmt.Sprint("writing data to Stdout failed: ", err))
	}
}