ashvardanian

import argparse
import time
from datasets import load_dataset
from datasets import disable_caching

# Set up argument parser
parser = argparse.ArgumentParser(description="Benchmark HuggingFace datasets library for a large textual file.")
parser.add_argument("file_path", type=str, help="Path to the textual file to be parsed and chunked.")
parser.add_argument("--sample_by", type=str, help="How to split - by line or by paragraph.", default="line")

ashvardanian

~/StringZilla$ ./build_release/stringzilla_bench_search leipzig1M.txt 
StringZilla. Starting search benchmarks.
Parsed the file with 8388608 words of 5 mean length!
Benchmarking for whitespaces:
- std::string_view.find_first_of          0.1882 GB/s     356648820.2 ns          0 errors in         32 iterations  
- sz_find_from_set_serial          0.2286 GB/s     293561982.7 ns          0 errors in         36 iterations  
- sz_find_from_set_neon          0.2289 GB/s     293157591.5 ns          0 errors in         36 iterations  
- strcspn                       0.2956 GB/s     226998443.9 ns          0 errors in         48 iterations  
- std::string_view.find_last_of          0.2117 GB/s     317041158.2 ns          0 errors in         32 iterations  
- sz_find_last_from_set_serial          0.2333 GB/s     287684198.9 ns          0 errors in         36 iterations  

ashvardanian

#include <string>
#include <iostream>
#include <cstdlib>

template <typename T = char>
struct failing_alloc : public std::allocator<T> {
    using value_type = T;
    failing_alloc() = default;
    template <class U> failing_alloc(failing_alloc<U> const&) {}
    template <typename U> struct rebind { typedef failing_alloc<U> other; };

ashvardanian

ashvardanian

#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
#include <Windows.h>
#endif

namespace ashvardanian {
  

#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)

ashvardanian

def read_matrix(filename: str, start_row: int = 0, count_rows: Optional[int] = None):
    """
    Read *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
    Args:
        :param filename (str): path to the matrix file
        :param start_row (int): start reading vectors from this index
        :param count_rows (int): number of vectors to read. If None, read all vectors
    Returns:
        Parsed matrix (numpy.ndarray)
    """