1_teach.sh

$ ./prodigy dataset test
  ✨  Successfully added 'test' to database SQLite.

$ cat data/test.jsonl

$ ./prodigy textcat.teach test initial_model_topics data/test.jsonl --label economie
  ✨  Starting the web server at http://localhost:8080 ...
  Open the app in your browser and start annotating!

Saved 2 annotations to database SQLite
Dataset: test
Session ID: 2018-08-11_13-28-10

$ ./prodigy textcat.teach test initial_model_topics data/test.jsonl --label wonen

Using 1 labels: wonen

  ✨  Starting the web server at http://localhost:8080 ...
  Open the app in your browser and start annotating!

^C
Saved 2 annotations to database SQLite
Dataset: test
Session ID: 2018-08-11_13-30-20

$ ./prodigy db-out test

{"text":"This is text B","meta":{"id":"2","score":0.0643216446},"_input_hash":-1530009927,"_task_hash":-340967702,"label":"economie","score":0.0643216446,"priority":0.0643216446,"spans":[],"answer":"accept"}
{"text":"This is text A","meta":{"id":"1","score":0.02744749},"_input_hash":499043849,"_task_hash":2003036685,"label":"economie","score":0.02744749,"priority":0.02744749,"spans":[],"answer":"reject"}
{"text":"This is text B","meta":{"id":"2","score":0.0643216446},"_input_hash":-1530009927,"_task_hash":-340967702,"label":"wonen","score":0.0643216446,"priority":0.0643216446,"spans":[],"answer":"accept"}
{"text":"This is text A","meta":{"id":"1","score":0.02744749},"_input_hash":499043849,"_task_hash":2003036685,"label":"wonen","score":0.02744749,"priority":0.02744749,"spans":[],"answer":"reject"}

2_eval.sh

$ ./prodigy textcat.eval test-eval initial_model_topics data/test.jsonl --label economie
Using 1 labels: economie

✨  ERROR: Invalid task format for view ID 'classification'
'label' is a required property 

{'text': 'This is text A', 'meta': {'id': '1'}, '_input_hash': 499043849, '_task_hash': 2003036685}
$ ./prodigy textcat.eval test-eval initial_model_topics data/test_labels.jsonl --label economie
Using 1 labels: economie

  ✨  Starting the web server at http://localhost:8080 ...
  Open the app in your browser and start annotating!

^C
Saved 2 annotations to database SQLite
Dataset: test-eval
Session ID: 2018-08-11_13-33-41

Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/prodigy/__main__.py", line 261, in <module>
    server(controller, controller.config)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/prodigy/app.py", line 52, in server
    controller.save()
  File "cython_src/prodigy/core.pyx", line 136, in prodigy.core.Controller.save
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/prodigy/recipes/textcat.py", line 219, in on_exit
    data = dict(model.evaluate(examples))
  File "cython_src/prodigy/models/textcat.pyx", line 207, in prodigy.models.textcat.TextClassifier.evaluate
KeyError: 'wonen'

$ ./prodigy db-out test-eval
{"text":"This is text A","meta":{"id":"1"},"label":"economie","_input_hash":499043849,"_task_hash":-1902286938,"answer":"reject"}
{"text":"This is text B","meta":{"id":"2"},"label":"wonen","_input_hash":-1530009927,"_task_hash":-628091562,"answer":"accept"}


$ ./prodigy drop test-eval
  ✨  Removed 'test-eval' from database SQLite.

$ ./prodigy dataset test-eval
  ✨  Successfully added 'test-eval' to database SQLite.

$ ./prodigy textcat.eval test-eval nl_core_news_sm data/test_labels2.jsonl
  ✨  Starting the web server at http://localhost:8080 ...
  Open the app in your browser and start annotating!

^C
Saved 2 annotations to database SQLite
Dataset: test-eval
Session ID: 2018-08-11_13-52-08

Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/prodigy/__main__.py", line 261, in <module>
    server(controller, controller.config)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/prodigy/app.py", line 52, in server
    controller.save()
  File "cython_src/prodigy/core.pyx", line 136, in prodigy.core.Controller.save
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/prodigy/recipes/textcat.py", line 219, in on_exit
    data = dict(model.evaluate(examples))
  File "cython_src/prodigy/models/textcat.pyx", line 204, in prodigy.models.textcat.TextClassifier.evaluate
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/spacy/language.py", line 554, in pipe
    for doc, context in izip(docs, contexts):
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/spacy/language.py", line 578, in pipe
    for doc in docs:
  File "pipeline.pyx", line 860, in pipe
  File "pipeline.pyx", line 865, in spacy.pipeline.TextCategorizer.predict
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/thinc/neural/_classes/model.py", line 161, in __call__
    return self.predict(x)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/thinc/api.py", line 55, in predict
    X = layer(X)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/thinc/neural/_classes/model.py", line 161, in __call__
    return self.predict(x)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/thinc/neural/_classes/model.py", line 125, in predict
    y, _ = self.begin_update(X)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/thinc/api.py", line 176, in begin_update
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/thinc/api.py", line 176, in <listcomp>
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/thinc/api.py", line 258, in wrap
    output = func(*args, **kwargs)
  File "/home/wva/prodigy_env/lib/python3.6/site-packages/thinc/api.py", line 61, in begin_update
    X, inc_layer_grad = layer.begin_update(X, drop=drop)
  File "linear.pyx", line 43, in thinc.linear.linear.LinearModel.begin_update
ValueError: Buffer and memoryview are not contiguous in the same dimension.

$ ./prodigy db-out test-eval
{"text":"This is text A","meta":{"id":"1"},"label":"economie","_input_hash":499043849,"_task_hash":-1902286938,"answer":"reject"}
{"text":"This is text B","meta":{"id":"2"},"label":"economie","_input_hash":-1530009927,"_task_hash":-252583655,"answer":"accept"}

test.jsonl

{"text": "This is text A", "meta": {"id": "1"}}
{"text": "This is text B", "meta": {"id": "2"}}

test_labels.jsonl

{"text": "This is text A", "meta": {"id": "1"}, "label": "economie"}
{"text": "This is text B", "meta": {"id": "2"}, "label": "wonen"}

test_labels2.jsonl

{"text": "This is text A", "meta": {"id": "1"}, "label": "economie"}
{"text": "This is text B", "meta": {"id": "2"}, "label": "economie"}

hello