readInf

go.mod

module readInf

go 1.21

require (
	github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 // indirect
	golang.org/x/text v0.13.0 // indirect
)

main.go

package main

import (
	"crypto/rand"
	"fmt"
	"github.com/mfonda/simhash"
	"sync/atomic"
	"time"
)

type Node struct {
	Val  []byte
	Next *Node
}

func randBlocks(num int, bufSize int) *Node {
	var head *Node = nil
	cur := head

	for i := 0; i < num; i++ {
		b := make([]byte, bufSize)
		n, err := rand.Read(b)
		if err != nil {
			panic(err)
		}

		if n == 0 {
			break
		}

		newNode := &Node{
			Val:  b,
			Next: nil,
		}

		if cur == nil {
			cur = newNode
			head = cur
		} else {
			cur.Next = newNode
			cur = cur.Next
		}
	}

	cur.Next = head
	return head
}

func main() {
	blocks := randBlocks(8*10, 128*1024) // total: 10MB

	var c int64 = 0
	for i := 0; i < 10; i++ {
		go func() {
			cur := blocks
			for {
				featureSet := simhash.NewWordFeatureSet(cur.Val)
				simhash.Simhash(featureSet)
				atomic.AddInt64(&c, 1)
				cur = cur.Next
			}
		}()
	}

	var p int64 = 0
	t := time.Now()
	for {
		time.Sleep(2 * time.Second)
		fmt.Printf("%.2f op/s\n", float64(c-p)/time.Since(t).Seconds())
		p = c
		t = time.Now()
	}
}