Gerador de orders em formato de string desenvolvido em GO | Lindando com sincronismo em GOLang

order_generation.md

Gerador com estrutura de objetos e channels

https://gist.github.com/gbzarelli/a745ede4804137b8a9525623120d6b5c#file-order_generation_challange_goroutines_and_channels-go

Gerando: 2MM a 30s com 10k * 100 * 10

Gerador sem estrutura, direto no sync.Map sem channels e com waitGroup

https://gist.github.com/gbzarelli/a745ede4804137b8a9525623120d6b5c#file-order_generation_challange_goroutines_and_wait_group-go

Gerando: 2MM a 25s com 10k * 100 * 10

Gerador simples só no for

https://gist.github.com/gbzarelli/a745ede4804137b8a9525623120d6b5c#file-order_generation_challange_simple-go

Bem lento

---

Notes

Testes passando a gravar no file direto dentro da goroutina deixava consideravelmente mais lento, devido ao lock do file que isso gerava, por isso, armazenar em um sync.Map e fazer o laço com writeString deixou-o mais rápido

• https://pkg.go.dev/sync

order_generation_challange_goroutines_and_channels.go

package main

import (
	"fmt"
	"github.com/jaswdr/faker"
	"log"
	"math/rand"
	"os"
	"sync/atomic"
	"time"
)

type User struct {
	id     int
	name   string
	orders []Order
}

type Order struct {
	id       int32
	products []Product
	date     string
}

type Product struct {
	id    int
	value float32
}

type UserCreated struct {
	index int
	user  User
	data  []string
}
type OrderCreated struct {
	index int
	order Order
	data  []string
}

const (
	qtdUsersToGenerate      = 10_000
	maxRandomOrdersByUser   = 100
	maxRandomProductByOrder = 10
	//--
	maxRandomProductsId = 10
)

var (
	timeMinToRandomDateOrder, _ = time.Parse("20060102", "20100101")
	file                        = openNewFile()
)

func main() {
	startDate := time.Now()
	data, _ := generateData()
	fmt.Printf("Duration to generate: %d\n", time.Now().UnixMilli()-startDate.UnixMilli())
	startDate2 := time.Now()
	shuffleAndWrite(data, file)
	fmt.Printf("Duration write: %d\n", time.Now().UnixMilli()-startDate2.UnixMilli())
	fmt.Printf("Duration total: %d\n", time.Now().UnixMilli()-startDate.UnixMilli())
}

func generateData() ([]string, []User) {
	users := make([]User, qtdUsersToGenerate)
	userCh := make(chan UserCreated)

	var usersString []string
	var sequentialOrderId int32 = 100

	// start goroutines to create users with orders
	for userIndex := 0; userIndex < qtdUsersToGenerate; userIndex++ {
		go createUserWithOrders(&sequentialOrderId, userIndex, userIndex+1, userCh)
	}

	// wait all users be created
	for userIndex := 0; userIndex < qtdUsersToGenerate; userIndex++ {
		newUser := <-userCh
		users[newUser.index] = newUser.user
		usersString = append(usersString, newUser.data...)
	}
	return usersString, users
}

func createUserWithOrders(sequentialOrderId *int32, userIndex int, userId int, ch chan UserCreated) {
	var ordersData []string
	qtdOrdersByUser := randomInt(maxRandomOrdersByUser)
	orderCh := make(chan OrderCreated)

	user := User{
		id:     userId,
		name:   newFaker().Person().Name(),
		orders: make([]Order, qtdOrdersByUser),
	}

	// start goroutines to create orders
	for orderIndex := 0; orderIndex < qtdOrdersByUser; orderIndex++ {
		orderId := atomic.AddInt32(sequentialOrderId, 1)
		go createOrderByUser(orderIndex, orderId, user, orderCh)
	}

	// listen all orders created
	for orderIndex := 0; orderIndex < qtdOrdersByUser; orderIndex++ {
		newOrder := <-orderCh
		user.orders[newOrder.index] = newOrder.order
		ordersData = append(ordersData, newOrder.data...)
	}
	ch <- UserCreated{index: userIndex, user: user, data: ordersData}
}

func createOrderByUser(orderIndex int, orderId int32, user User, ch chan OrderCreated) {
	var ordersString []string
	qtdProductsByOrder := randomInt(maxRandomProductByOrder)
	order := Order{
		id:       orderId,
		products: make([]Product, qtdProductsByOrder),
		date:     newFaker().Time().TimeBetween(timeMinToRandomDateOrder, time.Now()).Format("20060102"),
	}
	for productIndex := 0; productIndex < qtdProductsByOrder; productIndex++ {
		product, orderString := createProductToOrder(order, user)
		order.products[productIndex] = product
		ordersString = append(ordersString, orderString)
	}
	ch <- OrderCreated{index: orderIndex, order: order, data: ordersString}
}

func createProductToOrder(order Order, user User) (Product, string) {
	product := Product{
		id:    randomInt(maxRandomProductsId),
		value: newFaker().Float32(2, 10, 2000),
	}
	orderString := fmt.Sprintf("%010d%45s%010d%010d%12.2f%s\n",
		user.id, user.name, order.id, product.id, product.value, order.date,
	)
	return product, orderString
}

func newFaker() faker.Faker {
	return faker.NewWithSeed(rand.NewSource(time.Now().UnixNano()))
}

func randomInt(max int) int {
	value := rand.New(rand.NewSource(time.Now().UnixNano())).Intn(max)
	if value <= 0 {
		value = 1
	}
	return value
}

func openNewFile() *os.File {
	file, err := os.OpenFile(
		fmt.Sprintf("./orders_%s.txt", time.Now().Format("20060102_150405")),
		os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func shuffleAndWrite(data []string, file *os.File) {
	rand.New(rand.NewSource(time.Now().Unix())).Shuffle(len(data), func(i, j int) { data[i], data[j] = data[j], data[i] })
	for _, dataValue := range data {
		_, _ = file.WriteString(dataValue)
	}
}

order_generation_challange_goroutines_and_wait_group.go

package main

import (
	"fmt"
	"github.com/jaswdr/faker"
	"log"
	"math/rand"
	"os"
	"sync"
	"sync/atomic"
	"time"
)

type User struct {
	id   int
	name string
}

type Order struct {
	id   int32
	date string
}

const (
	qtdUsersToGenerate      = 10_000
	maxRandomOrdersByUser   = 100
	maxRandomProductByOrder = 10
	//--
	maxRandomProductsId = 10
)

var (
	timeMinToRandomDateOrder, _       = time.Parse("20060102", "20100101")
	file                              = openNewFile()
	counter                     int32 = 0
	data                              = new(sync.Map)
)

func main() {
	startDate := time.Now()
	generateDataAndWrite()
	data.Range(func(key, value interface{}) bool {
		_, _ = file.WriteString(value.(string))
		return true
	})
	fmt.Printf("Duration total: %d - Counter: %d \n",
		time.Now().UnixMilli()-startDate.UnixMilli(),
		counter)
}

func generateDataAndWrite() {
	var wg = new(sync.WaitGroup)
	var sequentialOrderId int32 = 100
	for userIndex := 0; userIndex < qtdUsersToGenerate; userIndex++ {
		wg.Add(1)
		userIndex := userIndex
		go func() {
			defer wg.Done()
			createUserWithOrders(&sequentialOrderId, userIndex+1)
		}()
	}
	wg.Wait()
}

func createUserWithOrders(sequentialOrderId *int32, userId int) {
	qtdOrdersByUser := randomInt(maxRandomOrdersByUser)
	var wg = new(sync.WaitGroup)
	user := User{
		id:   userId,
		name: newFaker().Person().Name(),
	}

	for orderIndex := 0; orderIndex < qtdOrdersByUser; orderIndex++ {
		orderId := atomic.AddInt32(sequentialOrderId, 1)
		wg.Add(1)
		go func() {
			defer wg.Done()
			createOrderByUser(orderId, user)
		}()
	}
	wg.Wait()
}

func createOrderByUser(orderId int32, user User) {
	qtdProductsByOrder := randomInt(maxRandomProductByOrder)
	order := Order{
		id:   orderId,
		date: newFaker().Time().TimeBetween(timeMinToRandomDateOrder, time.Now()).Format("20060102"),
	}

	for productIndex := 0; productIndex < qtdProductsByOrder; productIndex++ {
		createProductToOrder(order, user)
	}
}

func createProductToOrder(order Order, user User) {
	value := fmt.Sprintf("%010d%45s%010d%010d%12.2f%s\n",
		user.id,
		user.name,
		order.id,
		randomInt(maxRandomProductsId),
		newFaker().Float32(2, 10, 2000),
		order.date,
	)
	id := atomic.AddInt32(&counter, 1)
	data.Store(id, value)
}

func newFaker() faker.Faker {
	return faker.NewWithSeed(rand.NewSource(time.Now().UnixNano()))
}
func randomInt(max int) int {
	value := rand.New(rand.NewSource(time.Now().UnixNano())).Intn(max)
	if value <= 0 {
		value = 1
	}
	return value
}

func openNewFile() *os.File {
	file, err := os.OpenFile(
		fmt.Sprintf("./orders_%s.txt", time.Now().Format("20060102_150405")),
		os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

order_generation_challange_simple.go

package main

import (
	"fmt"
	"github.com/jaswdr/faker"
	"log"
	"math/rand"
	"os"
	"time"
)

type User struct {
	id     int
	name   string
	orders []Order
}

type Order struct {
	id       int
	products []Product
	date     string
}

type Product struct {
	id    int
	value float32
}

const (
	qtdUsersToGenerate      = 10_000
	maxRandomOrdersByUser   = 100
	maxRandomProductByOrder = 10
	//--
	maxRandomProductsId = 10
)

var (
	generator                   = faker.New()
	timeMinToRandomDateOrder, _ = time.Parse("20060102", "20100101")
	file                        = openNewFile()
	random                      = rand.New(rand.NewSource(time.Now().Unix()))
)

func main() {
	data, _ := generateData()
	shuffleAndWrite(data, file)
}

func generateData() ([]string, []User) {
	users := make([]User, qtdUsersToGenerate)
	var data []string
	sequentialOrderId := 100
	for userIndex := 0; userIndex < qtdUsersToGenerate; userIndex++ {
		qtdOrdersByUser := random.Intn(maxRandomOrdersByUser)
		if qtdOrdersByUser <= 0 {
			qtdOrdersByUser = 1
		}
		orders := make([]Order, qtdOrdersByUser)
		user := User{
			id:     userIndex + 1,
			name:   generator.Person().Name(),
			orders: orders,
		}

		for orderIndex := 0; orderIndex < qtdOrdersByUser; orderIndex++ {
			qtdProductsByOrder := random.Intn(maxRandomProductByOrder)
			if qtdProductsByOrder <= 0 {
				qtdProductsByOrder = 1
			}
			products := make([]Product, qtdProductsByOrder)
			order := Order{
				id:       sequentialOrderId,
				products: products,
				date:     generator.Time().TimeBetween(timeMinToRandomDateOrder, time.Now()).Format("20060102"),
			}
			for productIndex := 0; productIndex < qtdProductsByOrder; productIndex++ {
				product := Product{
					id:    random.Intn(maxRandomProductsId),
					value: generator.Float32(2, 10, 2000),
				}
				data = append(data, fmt.Sprintf("%010d%45s%010d%010d%12.2f%s\n",
					user.id, user.name, order.id, product.id, product.value, order.date,
				))
				products[productIndex] = product
			}
			orders[orderIndex] = order
			sequentialOrderId++
		}
		users[userIndex] = user
	}
	return data, users
}

func openNewFile() *os.File {
	file, err := os.OpenFile(
		fmt.Sprintf("./orders_%s.txt", time.Now().Format("20060102_150405")),
		os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func shuffleAndWrite(data []string, file *os.File) {
	random.Shuffle(len(data), func(i, j int) { data[i], data[j] = data[j], data[i] })
	for _, dataValue := range data {
		_, _ = file.WriteString(dataValue)
	}
}

sample_data_generated.txt

0000000090                         Esteban Bogisich DVM00000002970000000002     1267.2020190826
0000000081                                Ike Bahringer00000002780000000006      248.2020120831
0000000015                              Rashawn Carroll00000001350000000001      807.1020140919
0000000095                                  Paris O"Kon00000003060000000004       34.1020160817
0000000082                           Mr. Garrison Grady00000002810000000001     1879.2020170306
0000000011                                 Max Reynolds00000001270000000000     1108.1020151018
0000000075                             Mr. Darion Fahey00000002650000000008      164.2020100227
0000000008                               Ms. Zena Crist00000001180000000006      760.2020180904
0000000086                                  Maeve Mertz00000002920000000002      874.2020130520
0000000011                                 Max Reynolds00000001240000000008      347.1020161001
0000000091                         Mortimer Heidenreich00000003000000000001     1590.1020200505
0000000031                            Winnifred Carroll00000001690000000009     1516.2020170510
0000000095                                  Paris O"Kon00000003070000000000     1388.1020200930
0000000042                         Christian Wintheiser00000001880000000005     1543.1020180506
........