get_config_test

check_status.py

import random
import time

import jubatus
from jubatus.common import Datum

def generate_data(num = 10, mu = 1.0, sigma = 1.0):
    res = []
    for i in range(num):
        x = random.gauss(mu, sigma)
        y = random.gauss(mu, sigma)
        z = random.gauss(mu, sigma)
        res.append((x, y, z))
    return res

def add_data():
    data = generate_data()
    client = jubatus.Anomaly("127.0.0.1", 9199, "hoge")
    for d in data:
        dt = Datum({"x": d[0],
                    "y": d[1],
                    "z": d[2]})
        print(client.add(dt))

def get_status():
    client = jubatus.Anomaly("127.0.0.1", 9199, "hoge")
    return client.get_status()

def print_status():
    r = get_status()
    for k in r:
        print(k)
        print("\t{}:{}".format("num_id", r[k]["num_id"]))
        print("\t{}:{}".format("num_unlearner_ids", r[k]["num_unlearner_ids"]))

def do_mix():
    client = jubatus.Anomaly("127.0.0.1", 19199, "hoge")
    client.do_mix()

def clear():
    client = jubatus.Anomaly("127.0.0.1", 9199, "hoge")
    client.clear()

def main():
    clear()
    print("1. add 10 data")
    add_data()
    print("2. status before mix")
    print_status()
    print("3. do mix")
    do_mix()
    time.sleep(10)
    print("4. status after mix")
    print_status()
    print("5. add more 10 data")
    add_data()
    print("6. status before mix")
    print_status()
    print("7. do mix")
    do_mix()
    time.sleep(10)
    print("8. status after mix")
    print_status()

if __name__ == "__main__":
    main()

light_lof_ignore_unlearner.json

{
  "converter" : {
    "string_filter_types" : {},
    "string_filter_rules" : [],
    "num_filter_types" : {},
    "num_filter_rules" : [],
    "string_types" : {},
    "string_rules" : [
      { "key" : "*", "type" : "str", "sample_weight" : "bin", "global_weight" : "bin" }
    ],
    "num_types" : {},
    "num_rules" : [
      { "key" : "*", "type" : "num" }
    ]
  },
  "parameter" : {
    "nearest_neighbor_num" : 10,
    "reverse_nearest_neighbor_num" : 30,
    "method" : "lsh",
    "parameter" : {
        "hash_num": 512
    },
      "ignore_kth_same_point": true,
      "unlearner": "random",
      "unlearner_parameter": {
          "max_size": 15
      }

  },
  "method" : "light_lof"
}

lof_config.json

{
  "converter" : {
    "string_filter_types" : {},
    "string_filter_rules" : [],
    "num_filter_types" : {},
    "num_filter_rules" : [],
    "string_types" : {},
    "string_rules" : [
      { "key" : "*", "type" : "str", "sample_weight" : "bin", "global_weight" : "bin" }
    ],
    "num_types" : {},
    "num_rules" : [
      { "key" : "*", "type" : "num" }
    ]
  },
  "parameter" : {
    "nearest_neighbor_num" : 10,
    "reverse_nearest_neighbor_num" : 30,
    "method" : "inverted_index",
    "parameter" : {
    },
      "ignore_kth_same_point": true,
      "unlearner": "lru",
      "unlearner_parameter": {
          "max_size": 15
      }
  },
  "method" : "lof"
}

result.txt

$ python script/check_num.py
1. add 10 data
id_with_score{id: 241, score: inf}
id_with_score{id: 242, score: 1.0}
id_with_score{id: 243, score: 0.9759825468063354}
id_with_score{id: 244, score: inf}
id_with_score{id: 245, score: 1.0}
id_with_score{id: 246, score: 0.9696969389915466}
id_with_score{id: 247, score: 0.960365891456604}
id_with_score{id: 248, score: 1.085548758506775}
id_with_score{id: 249, score: 1.0459892749786377}
id_with_score{id: 250, score: 0.9652501344680786}
2. status before mix
10.0.2.15_19199
	num_id:5
	num_unlearner_ids:5
10.0.2.15_19200
	num_id:7
	num_unlearner_ids:7
3. do mix
4. status after mix
10.0.2.15_19199
	num_id:10
	num_unlearner_ids:10
10.0.2.15_19200
	num_id:10
	num_unlearner_ids:10
5. add more 10 data
id_with_score{id: 251, score: 1.0014467239379883}
id_with_score{id: 252, score: 1.0222411155700684}
id_with_score{id: 253, score: 1.0287809371948242}
id_with_score{id: 254, score: 0.9923465847969055}
id_with_score{id: 255, score: 1.0239719152450562}
id_with_score{id: 256, score: 1.0072060823440552}
id_with_score{id: 257, score: 1.1723639965057373}
id_with_score{id: 258, score: 1.052030086517334}
id_with_score{id: 259, score: 1.0083149671554565}
id_with_score{id: 260, score: 1.0868561267852783}
6. status before mix
10.0.2.15_19199
	num_id:15
	num_unlearner_ids:15
10.0.2.15_19200
	num_id:14
	num_unlearner_ids:14
7. do mix
8. status after mix
10.0.2.15_19199
	num_id:13
	num_unlearner_ids:15
10.0.2.15_19200
	num_id:10
	num_unlearner_ids:15

run_test.sh

jubaconfig -c write -f conf/light_lof_ignore_unlearner.json -n hoge -t anomaly -z 127.0.0.1:2181
jubaanomaly_proxy -z 127.0.0.1:2181 -p 9199 &
jubaanomaly -p 19199 -z 127.0.0.1:2181 -n hoge &
jubaanomaly -p 19200 -z 127.0.0.1:2181 -n hoge &
python check_status.py