mooreniemi

use std::collections::HashMap;
use std::time::Instant;

/**
The IA-Select algorithm translated into Rust from
[Diversifying Search Results](https://www.microsoft.com/en-us/research/wp-content/uploads/2009/02/diversifying-wsdm09.pdf)
which is a "(1 − 1/`e`)-approximation algorithm for `Diversify(k)`." `Diversify(k)` is an NP-hard function that maximizes
the probability that you find a subset of documents from multiple categories of documents that "satisfies the average user."

IA stands for "intent aware" but intents are then mapped to categories.

mooreniemi

import ray
import time
import logging
from threading import Thread

log = logging.getLogger(__name__)


@ray.remote
class Worker:

mooreniemi

extern crate blas_src;

use hnsw::{Hnsw, Params, Searcher};
use ndarray_npy::{ViewNpyExt, WriteNpyExt};

use memmap2::{Mmap, MmapMut};
use rand_pcg::Pcg64;
use space::Metric;
use std::{fs::OpenOptions, io::BufWriter, time::Instant};

mooreniemi

extern crate blas_src;

use hnsw::{Hnsw, Params, Searcher};
use ndarray_npy::{ViewNpyExt, WriteNpyExt};

use memmap2::{Mmap, MmapMut};
use rand_pcg::Pcg64;
use space::Metric;
use std::{fs::OpenOptions, io::BufWriter, time::Instant};

mooreniemi

[package]
name = "mem"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
rayon = "1.5"
indicatif = {version = "0.16", features = ["rayon"]}

mooreniemi

[package]
name = "mem"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
rayon = "1.5"
indicatif = {version = "0.16", features = ["rayon"]}

mooreniemi

# https://github.com/matsui528/faiss_tips
import faiss
import numpy as np

D = 256
N = 10000
X = np.random.random((N, D)).astype(np.float32)

# Setup
index = faiss.index_factory(D, "IVF25,Flat", faiss.METRIC_INNER_PRODUCT)

mooreniemi

use std::{
    fs::File,
    io::{BufRead, BufReader},
    path::PathBuf,
    time::Instant,
};

use faiss::{error::Error, index_factory, read_index, write_index, Index, MetricType};
use itertools_num::linspace;
use structopt::StructOpt;

mooreniemi

use std::{path::PathBuf, time::Instant};

use faiss::{error::Error, index_factory, read_index, write_index, Index, MetricType};
use itertools_num::linspace;
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
#[structopt(name = "dann", about = "Demo ANN")]
struct DannOpt {
    /// FAISS Index factory string

mooreniemi

use std::time::Instant;

use faiss::{error::Error, index_factory, Index, MetricType};
use itertools_num::linspace;

fn main() -> Result<(), Error> {
    let d: u32 = 64;
    // let my_data: Vec<f32> = (1u16..(d * 10) as u16).map(f32::from).collect();
    // above way of generating f32 ranges is limited in d size wrt default traits
    // https://users.rust-lang.org/t/collect-f32-range-into-a-vector/15936/3