Fake Data Generator - Generate some fake data to postgesql

fakegen.go

package main

import (
    "database/sql"
    "fmt"
    "log"
    "sync"
    "time"

    "github.com/bwmarrin/snowflake"
    "github.com/icrowley/fake"
    _ "github.com/lib/pq"
)

var IDGenerator *snowflake.Node

// Data doc
type Data struct {
    ID          string
    Name        string
    Address     string
    S2ID        int64
    PlaceID     string
    GateID      string
    CountryCode string
    CreatedAt   string
    UpdatedAt   string
}

type messageCH chan Package

type Package struct {
    Seq  int
    Data Data
}

func main() {
    seed()
}

func seed() {
    maxNumTask := 1000000
    numWorker := 20000

    host := "127.0.0.1"
    port := "5432"
    user := "postgres"
    password := ""
    dbName := "customer_locations_development"
    dsn := fmt.Sprintf("host=%s port=%s user=%s dbname=%s sslmode=disable", host, port, user, dbName)
    if password != "" {
        dsn = fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, password, dbName)
    }
    db, err := sql.Open("postgres", dsn)
    if err != nil {
        log.Panic("error while connecting to database", err)
    }

    err = db.Ping()
    if err != nil {
        log.Panic("can't connect to database", err)
    }
    db.SetMaxOpenConns(64)
    db.SetMaxIdleConns(64)
    db.SetConnMaxLifetime(time.Minute)

    var wg sync.WaitGroup
    ch := make(messageCH)
    println("preparing worker...")
    for i := 1; i <= numWorker; i++ {
        go func(workerID int) {
            println(fmt.Sprintf("started worker %d", workerID))
            for task := range ch {
                wg.Add(1)
                data := task.Data
                println(fmt.Sprintf("processing %d", task.Seq))
                sql := fmt.Sprintf("INSERT into pois (id, name, address, s2id, place_id, country_code, created_at, updated_at) VALUES ('%s', '%s', '%s', '%d', '%s', '%s', '%s', '%s');", data.ID, data.Name, data.Address, data.S2ID, data.PlaceID, data.CountryCode, data.CreatedAt, data.UpdatedAt)
                _, err = db.Exec(sql)
                if err != nil {
                    log.Fatalf("error while inserting, got:%+v", err)
                }
                wg.Done()
            }
        }(i)
    }

    IDGenerator, err := snowflake.NewNode(1)
    if err != nil {
        log.Fatal(err)
        return
    }

    for i := 1; i <= maxNumTask; i++ {
        id := IDGenerator.Generate().Int64()
        placeID := fmt.Sprintf("%s-%d", "GOOG", id)
        timestamp := time.Now().Format(time.RFC3339)
        data := Data{
            ID:          fmt.Sprint(id),
            Name:        fake.Street(),
            Address:     fake.StreetAddress(),
            S2ID:        id,
            PlaceID:     placeID,
            CountryCode: "ID",
            CreatedAt:   timestamp,
            UpdatedAt:   timestamp,
        }
        task := Package{
            Seq:  i,
            Data: data,
        }
        ch <- task
    }

    wg.Wait()
    println("all done...")
}