filter_func.go

func hasPrefixCaseInsensitive(s, prefix string) bool {
    if len(prefix) > len(s) {
        return false
    }
    return strings.EqualFold(s[:len(prefix)], prefix)
}

!hasPrefixCaseInsensitive(single, lSearchTerm)

func IMatch(s, match string) bool {
	return strings.EqualFold(s, match)
}

filters.go

	func filteredOptimized(items Items, operations GroupedOperations, query Query) Items {
		registerFuncs := RegisterFuncMap //operations.Funcs
		excludes := query.Excludes
		filters := query.Filters
		anys := query.Anys
	
		anyFuncs := make([]func(*Item, string) bool, 0)
		anyArgs := make([][]string, 0)
		for funcName, args := range anys {
			anyFunc := registerFuncs[funcName]
			if anyFunc != nil {
				anyFuncs = append(anyFuncs, anyFunc)
				anyArgs = append(anyArgs, args)
			}
		}
	
		allFuncs := make([]func(*Item, string) bool, 0)
		allArgs := make([][]string, 0)
		for funcName, args := range filters {
			allFunc := registerFuncs[funcName]
			if allFunc != nil {
				allFuncs = append(allFuncs, allFunc)
				allArgs = append(allArgs, args)
			}
		}
	
		excludeFuncs := make([]func(*Item, string) bool, 0)
		excludeArgs := make([][]string, 0)
		for funcName, args := range excludes {
			excludeFunc := registerFuncs[funcName]
			if excludeFunc != nil {
				excludeFuncs = append(excludeFuncs, excludeFunc)
				excludeArgs = append(excludeArgs, args)
			}
		}
	
		filteredItems := make(Items, 0)
		for _, item := range items {
			if !anyOptimized(item, anyFuncs, anyArgs) {
				continue
			}
			if !allOptimized(item, allFuncs, allArgs) {
				continue
			}
			if !excludeOptimized(item, excludeFuncs, excludeArgs) {
				continue
			}
			filteredItems = append(filteredItems, item)
		}
		return filteredItems
	}
	
	func allOptimized(item *Item, funcs []func(*Item, string) bool, args [][]string) bool {
		for i, filterFunc := range funcs {
			for _, arg := range args[i] {
				if !filterFunc(item, arg) {
					return false
				}
			}
		}
		return true
	}
	
	func anyOptimized(item *Item, funcs []func(*Item, string) bool, args [][]string) bool {
		if len(funcs) == 0 {
			return true
		}
		for i, filterFunc := range funcs {
			for _, arg := range args[i] {
				if filterFunc(item, arg) {
					return true
				}
			}
		}
		return false
	}
	
	func excludeOptimized(item *Item, funcs []func(*Item, string) bool, args [][]string) bool {
		for i, excludeFunc := range funcs {
			for _, arg := range args[i] {
				if excludeFunc(item, arg) {
					return false
				}
			}
		}
		return true
	}

set_intersect_index.go

package main

import (
	"fmt"
)

// Represents individual sets.
type DataSet struct {
	id   int
	data map[string]bool
}

// Index structure to allow for fast lookups.
type Index struct {
	wordToSets map[string][]int // Change to store IDs directly
}

func NewIndex() *Index {
	return &Index{
		wordToSets: make(map[string][]int),
	}
}

func (idx *Index) AddSet(dataSet *DataSet) {
	for word := range dataSet.data {
		if _, found := idx.wordToSets[word]; !found {
			idx.wordToSets[word] = []int{}
		}
		idx.wordToSets[word] = append(idx.wordToSets[word], dataSet.id)
	}
}

func (idx *Index) IntersectingSets(query []string) []int {
	seen := make(map[int]int)

	for _, word := range query {
		sets, found := idx.wordToSets[word]
		if found {
			for _, setID := range sets {
				seen[setID]++
			}
		}
	}

	var result []int
	for setID, count := range seen {
		if count == len(query) {
			result = append(result, setID)
		}
	}

	return result
}

func main() {
	// Sample sets.
	set1 := &DataSet{id: 1, data: map[string]bool{"foo": true, "bar": true, "baz": true}}
	set2 := &DataSet{id: 2, data: map[string]bool{"foo": true, "apple": true}}
	set3 := &DataSet{id: 3, data: map[string]bool{"banana": true, "cherry": true, "apple": true}}

	index := NewIndex()

	index.AddSet(set1)
	index.AddSet(set2)
	index.AddSet(set3)

	query := []string{"foo", "apple"}
	matchingSetIDs := index.IntersectingSets(query)

	fmt.Println("Matching set IDs:", matchingSetIDs)
}



package main

import (
	"fmt"
	"math/rand"
	"testing"
	"time"
)

func generateMockData(numEntries int) []*DataSet {
	rand.Seed(time.Now().UnixNano())
	sets := make([]*DataSet, numEntries)

	for i := 0; i < numEntries; i++ {
		setData := make(map[string]bool)
		for j := 0; j < 10; j++ { // Assuming each set has 10 random words.
			setData[fmt.Sprintf("word%d", rand.Intn(numEntries))] = true
		}
		sets[i] = &DataSet{id: i, data: setData}
	}
	return sets
}

func BenchmarkIntersectingSets10k(b *testing.B) {
	sets := generateMockData(10000)
	index := NewIndex()

	for _, set := range sets {
		index.AddSet(set)
	}

	query := []string{"word500", "word1000"} // Some random words to query

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		index.IntersectingSets(query)
	}
}

func BenchmarkIntersectingSets100k(b *testing.B) {
	sets := generateMockData(100000)
	index := NewIndex()

	for _, set := range sets {
		index.AddSet(set)
	}

	query := []string{"word5000", "word10000"} // Some random words to query

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		index.IntersectingSets(query)
	}
}

func BenchmarkIntersectingSets1M(b *testing.B) {
	sets := generateMockData(1000000)
	index := NewIndex()

	for _, set := range sets {
		index.AddSet(set)
	}

	query := []string{"word50000", "word100000"} // Some random words to query

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		index.IntersectingSets(query)
	}
}

sorting.go

package main

import (
	"fmt"
	"sort"
)

func indexOf(slice []string, item string) int {
	for i, v := range slice {
		if v == item {
			return i
		}
	}
	return -1
}

func GenericSortBy(items Items, sortingL []string) Items {
	sort.SliceStable(items, func(i, j int) bool {
		columns := items[i].Columns()
		rowI := items[i].Row()
		rowJ := items[j].Row()

		for _, field := range sortingL {
			isDescending := false
			if field[0] == '-' {
				isDescending = true
				field = field[1:]
			}
			idx := indexOf(columns, field)
			if idx == -1 {
				continue // if field not found, skip
			}
			if rowI[idx] < rowJ[idx] {
				return !isDescending
			} else if rowI[idx] > rowJ[idx] {
				return isDescending
			}
		}
		return false
	})
	return items
}

func singleSortBy(items Items, sorting string) Items {
	columns := items[0].Columns()
	isDescending := false

	if sorting[0] == '-' {
		isDescending = true
		sorting = sorting[1:]
	}

	idx := indexOf(columns, sorting)
	if idx == -1 {
		return items // if field not found, return items as is
	}

	sort.SliceStable(items, func(i, j int) bool {
		rowI := items[i].Row()
		rowJ := items[j].Row()

		if rowI[idx] < rowJ[idx] {
			return !isDescending
		} else if rowI[idx] > rowJ[idx] {
			return isDescending
		}
		return false
	})

	return items
}

func sortBy(items Items, sortingL []string) Items {
	if len(sortingL) == 1 {
		return singleSortBy(items, sortingL[0])
	}
	return GenericSortBy(items, sortingL)
}

// Assuming you have the map of sortFuncs somewhere else.
func keysFromMap(m map[string]func(int, int) bool) []string {
	keys := []string{}
	for key := range m {
		keys = append(keys, key)
	}
	return keys
}

type Item struct {
	Column_a string
	Column_b string
	Column_c string
}

func (i Item) Columns() []string {
	return []string{"Column_a", "Column_b", "Column_c"}
}

func (i Item) Row() []string {
	return []string{i.Column_a, i.Column_b, i.Column_c}
}

type Items []Item


func main() {
	items := Items{
		{Column_a: "apple", Column_b: "banana", Column_c: "cherry"},
		{Column_a: "orange", Column_b: "grape", Column_c: "apple"},
		{Column_a: "banana", Column_b: "cherry", Column_c: "banana"},
	}

	// Sort by Column_a in ascending order
	sortedItems := sortBy(items, []string{"Column_a"})
	for _, item := range sortedItems {
		fmt.Println(item.Column_a, item.Column_b, item.Column_c)
	}

	fmt.Println("----------")
	sortedItems = sortBy(items, []string{"-Column_a"})
	for _, item := range sortedItems {
		fmt.Println(item.Column_a, item.Column_b, item.Column_c)
	}
    fmt.Println("reversed")

	// Sort by Column_b in descending order
	sortedItems = sortBy(items, []string{"-Column_b"})
	for _, item := range sortedItems {
		fmt.Println(item.Column_a, item.Column_b, item.Column_c)
	}

	fmt.Println("----------")

	// Multi-level sort: Sort by Column_a in ascending order, then Column_b in descending order
	sortedItems = sortBy(items, []string{"Column_a", "-Column_b"})
	for _, item := range sortedItems {
		fmt.Println(item.Column_a, item.Column_b, item.Column_c)
	}
}

stream_csv.go

func streamSliceToCSV(w http.ResponseWriter, items []Item) {
	w.Header().Set("Content-Type", "text/csv")
	w.Header().Set("Content-Disposition", "attachment; filename=data.csv")

	const bufferSize = 2 * 1024 * 1024 // 2MB
	bw := bufio.NewWriterSize(w, bufferSize)
	writer := csv.NewWriter(bw)
	defer writer.Flush()

	if len(items) > 0 {
		writer.Write(items[0].Columns()) // Write the header
	}

	// Iterate through items and write to CSV
	for _, item := range items {
		writer.Write(item.Rows())
	}

	if err := bw.Flush(); err != nil {
		http.Error(w, err.Error(), 500)
	}
}


func streamSliceToCSV(w http.ResponseWriter, items []Item) {
	w.Header().Set("Content-Type", "text/csv")
	w.Header().Set("Content-Disposition", "attachment; filename=data.csv")

	const bufferSize = 2 * 1024 * 1024 // 2MB
	bw := bufio.NewWriterSize(w, bufferSize)
	writer := csv.NewWriter(bw)
	defer writer.Flush()

	structType := reflect.TypeOf(items[0])
	numFields := structType.NumField()
	headers := make([]string, numFields)

	// Extract CSV tags from the struct fields for header
	for i := 0; i < numFields; i++ {
		field := structType.Field(i)
		headers[i] = field.Tag.Get("csv")
	}
	writer.Write(headers)

	row := make([]string, numFields)

	// Iterate through items and write to CSV
	for _, item := range items {
		itemVal := reflect.ValueOf(item)
		for j := 0; j < numFields; j++ {
			field := itemVal.Field(j)
			row[j] = fmt.Sprintf("%v", field.Interface())
		}
		writer.Write(row)
	}

	if err := bw.Flush(); err != nil {
		http.Error(w, err.Error(), 500)
	}
}

streaming_JSON.go

func streamJSON_MEM(w http.ResponseWriter, items []Item) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)

	const bufferSize = 2 * 1024 * 1024 // 2MB

	bw := bufio.NewWriterSize(w, bufferSize)

	if _, err := bw.Write([]byte("[")); err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	isFirst := true
	for _, item := range items {
		if isFirst {
			isFirst = false
		} else {
			if _, err := bw.Write([]byte(",")); err != nil {
				http.Error(w, err.Error(), 500)
				return
			}
		}

		if err := json.NewEncoder(bw).Encode(item); err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
	}

	if _, err := bw.Write([]byte("]")); err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	if err := bw.Flush(); err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}