I hereby claim:
- I am petekalo on github.
- I am netfl0 (https://keybase.io/netfl0) on keybase.
- I have a public key ASDvy2NRqUBBMsDYH6F2pmCuGAVUdeh6x-9PDFIGoeixJwo
To claim this, I am signing this object:
I hereby claim:
To claim this, I am signing this object:
"""Extract several BOW models from a corpus of text files. | |
The models are stored in Matrix Market format which can be read | |
by gensim. The texts are read from .txt files in the directory | |
specified as TOPDIR. The output is written to the current directory.""" | |
# NB: All strings are utf8 (not unicode). | |
import os | |
import glob | |
import nltk | |
import gensim |
class IterableQuerySet(object): | |
"""Allows iteration over a QuerySet breaking it off into smaller chunks.""" | |
"""Take from http://justcramer.com/2009/02/09/large-sql-result-sets-in-django/""" | |
def __init__(self, queryset, batch=10000): | |
self.batch = batch | |
self.queryset = queryset | |
def __iter__(self): | |
at = 0 | |
---------------------------------------------- | |
#TOR CONFIGURATION: | |
# transparent tor: | |
#TransPort 9040 | |
#TransListenAddress 10.0.0.1 | |
#DNSPort 53 | |
#DNSListenAddress 10.0.0.1 | |
----------------------------------------------- |