LMDB exploration

lmdb.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"LMDB Exploration.ipynb","version":"0.3.2","views":{},"default_view":{},"provenance":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"cells":[{"metadata":{"id":"P9wQ-WalTdSQ","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["import lmdb"],"execution_count":0,"outputs":[]},{"metadata":{"id":"GICZ6SB0TdSY","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["## Let's create two databases one for tags and other for data"],"execution_count":0,"outputs":[]},{"metadata":{"id":"52mOb88cTdSd","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["env = lmdb.Environment(path='data/explore', max_dbs=2)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"6SS_L7oeTdSh","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Insert data to tag DB"],"execution_count":0,"outputs":[]},{"metadata":{"id":"0TRmo1arTdSm","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["tag_db = env.open_db(b'tag_db')\n","data_db = env.open_db(b'data_db')"],"execution_count":0,"outputs":[]},{"metadata":{"id":"JaKuRx__TdSr","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Can I open third db?"],"execution_count":0,"outputs":[]},{"metadata":{"id":"nnuq87yHTdSx","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"3430a517-70ad-493b-9762-0d6a72f97953"},"cell_type":"code","source":["env.open_db(b'trail')"],"execution_count":0,"outputs":[{"output_type":"error","ename":"DbsFullError","evalue":"mdb_dbi_open: MDB_DBS_FULL: Environment maxdbs limit reached","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mDbsFullError\u001b[0m                              Traceback (most recent call last)","\u001b[0;32m<ipython-input-7-75ea9bdd6f83>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_db\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mb'trail'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mDbsFullError\u001b[0m: mdb_dbi_open: MDB_DBS_FULL: Environment maxdbs limit reached"]}]},{"metadata":{"id":"LnEKlnoFTdS7","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Add some 100 tags"],"execution_count":0,"outputs":[]},{"metadata":{"id":"mDC3UkmyTdTB","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["import faker"],"execution_count":0,"outputs":[]},{"metadata":{"id":"iu3MZm0DTdTF","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"aaad37a0-f533-4b1a-d9a8-6c5e8caf9c5d"},"cell_type":"code","source":["fake = faker.Faker()\n","import time\n","start = time.time()\n","tags = []\n","with env.begin(write=True) as txn:\n","    # Add to tag DB\n","    cursor = txn.cursor(tag_db)\n","    for i in range(100):\n","        # Only byte like object accepted\n","        key = fake.name().encode('UTF-8')\n","        cursor.put(key=key, value=str(i).encode('UTF-8'))\n","        tags.append(key)\n","print(f'Inserting 100 items took {time.time() - start} seconds')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Inserting 100 items took 0.2672431468963623 seconds\n"],"name":"stdout"}]},{"metadata":{"id":"2FaXVtZlTdTN","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Can I add int as key?"],"execution_count":0,"outputs":[]},{"metadata":{"id":"ACimyALzTdTR","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"cf530079-1f45-4cb7-f52b-7c67e571a61c"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    cursor = txn.cursor(tag_db)\n","    key = fake.name().encode('UTF-8')\n","    cursor.put(key=key, value=1)"],"execution_count":0,"outputs":[{"output_type":"error","ename":"TypeError","evalue":"a bytes-like object is required, not 'int'","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-12-364753f09464>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mcursor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtxn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtag_db\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfake\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'UTF-8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mcursor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mTypeError\u001b[0m: a bytes-like object is required, not 'int'"]}]},{"metadata":{"id":"Cw9GgSBQTdTY","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Read values back from the tag_db\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"CQPSK154TdTf","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"5fd34614-f21a-4902-f933-799c4a2d4650"},"cell_type":"code","source":["with env.begin(write=False) as txn:\n","    cursor = txn.cursor(tag_db)\n","    print(cursor.get(tags[10]))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["b'10'\n"],"name":"stdout"}]},{"metadata":{"id":"I_AzHuNYTdTm","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Can we store dict as a value?"],"execution_count":0,"outputs":[]},{"metadata":{"id":"Pp_ZJQA6TdTr","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"979cf5f0-9bf1-4c65-b499-cfa5621dc5b5"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    cursor = txn.cursor(tag_db)\n","    key = fake.name().encode('UTF-8')\n","    cursor.put(key=key, value={'1': b'1'})"],"execution_count":0,"outputs":[{"output_type":"error","ename":"TypeError","evalue":"a bytes-like object is required, not 'dict'","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-16-04775a8f091a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mcursor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtxn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtag_db\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfake\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'UTF-8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mcursor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'1'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34mb'1'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mTypeError\u001b[0m: a bytes-like object is required, not 'dict'"]}]},{"metadata":{"id":"VUgoDCrvTdTw","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Convert dict to JSON?"],"execution_count":0,"outputs":[]},{"metadata":{"id":"Ko1L9hr9TdT0","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["import json"],"execution_count":0,"outputs":[]},{"metadata":{"id":"Cw_ZugsHTdT4","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    cursor = txn.cursor(tag_db)\n","    key = fake.name().encode('UTF-8')\n","    value = json.dumps({'1': '1'})\n","    cursor.put(key=key, value=value.encode('UTF-8'))"],"execution_count":0,"outputs":[]},{"metadata":{"id":"R0GjR_9CTdT9","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"11dcec53-d073-44c0-db0e-c5d3e1e10a78"},"cell_type":"code","source":["print(key)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["b'Melanie Davis'\n"],"name":"stdout"}]},{"metadata":{"id":"mFYLRPnhTdUE","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["## Get back the value"],"execution_count":0,"outputs":[]},{"metadata":{"id":"EUU7DiW5TdUL","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"7e0bb846-214b-405c-d7b5-7d1b6ef3ef53"},"cell_type":"code","source":["with env.begin(write=False) as txn:\n","    cursor = txn.cursor(tag_db)\n","    print(cursor.get(key))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["b'{\"1\": \"1\"}'\n"],"name":"stdout"}]},{"metadata":{"id":"u1obxvubTdUR","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Awesome! Is it possible to store bytes of length 100000 characters?"],"execution_count":0,"outputs":[]},{"metadata":{"id":"6OOepfBrTdUV","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    cursor = txn.cursor(tag_db)\n","    key = fake.name().encode('UTF-8')\n","    value = b'u' * 1000_000\n","    cursor.put(key=key, value=value)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"KaIAcU78TdUa","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"f5014ea9-008a-42c4-cbce-ee81baf0ce47"},"cell_type":"code","source":["with env.begin(write=False) as txn:\n","    cursor = txn.cursor(tag_db)\n","    print(len(cursor.get(key)))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["1000000\n"],"name":"stdout"}]},{"metadata":{"id":"1qYHTNK8TdUg","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### how many entries are in tags_db?"],"execution_count":0,"outputs":[]},{"metadata":{"id":"wQ2w0itZTdUj","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"50ef7836-c106-4742-bb74-d254ae3048f2"},"cell_type":"code","source":["env.stat()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'branch_pages': 0,\n"," 'depth': 1,\n"," 'entries': 2,\n"," 'leaf_pages': 1,\n"," 'overflow_pages': 0,\n"," 'psize': 4096}"]},"metadata":{"tags":[]},"execution_count":27}]},{"metadata":{"id":"UOHunVe2TdUo","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Let's adda some sample data to data_db"],"execution_count":0,"outputs":[]},{"metadata":{"id":"pNLNgRvmTdUr","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"d4605c90-0f5a-4e69-9092-f3d1bee8c447"},"cell_type":"code","source":["fake = faker.Faker()\n","import time\n","start = time.time()\n","data = []\n","with env.begin(write=True) as txn:\n","    # Add to tag DB\n","    cursor = txn.cursor(data_db)\n","    for i in range(100):\n","        # Only byte like object accepted\n","        key = fake.name().encode('UTF-8')\n","        cursor.put(key=key, value=str(i).encode('UTF-8'))\n","        data.append(key)\n","print(f'Inserting 100 items took {time.time() - start} seconds')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Inserting 100 items took 0.04271888732910156 seconds\n"],"name":"stdout"}]},{"metadata":{"id":"pze6pjJZTdUv","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"1277a557-bed4-4f1e-c6ea-da46815e70fe"},"cell_type":"code","source":["### How many records?\n","env.stat()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'branch_pages': 0,\n"," 'depth': 1,\n"," 'entries': 2,\n"," 'leaf_pages': 1,\n"," 'overflow_pages': 0,\n"," 'psize': 4096}"]},"metadata":{"tags":[]},"execution_count":30}]},{"metadata":{"id":"i4080hMeTdU1","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"96866e8d-b715-46ed-fd05-32b689316fc5"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    cursor = txn.cursor(data_db)\n","    print(sum(1 for _ in cursor))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["100\n"],"name":"stdout"}]},{"metadata":{"id":"_iLTZ_7mTdU8","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"5157512d-4ac4-46a8-d17b-cc75d60bff37"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    cursor = txn.cursor(tag_db)\n","    print(sum(1 for _ in cursor))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["102\n"],"name":"stdout"}]},{"metadata":{"id":"BU5nFsGZTdVB","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"ad8a2161-014f-457a-8b94-1f5f47b733ef"},"cell_type":"code","source":["!ls -lah data/explore/"],"execution_count":0,"outputs":[{"output_type":"stream","text":["total 1.1M\n","drwxr-xr-x 2 krace krace   36 Mar  2 02:04 .\n","drwxr-xr-x 4 krace krace   68 Mar  2 02:04 ..\n","-rw-r--r-- 1 krace krace 1.0M Mar  2 02:06 data.mdb\n","-rw-r--r-- 1 krace krace 8.0K Mar  2 02:06 lock.mdb\n"],"name":"stdout"}]},{"metadata":{"id":"wwvj5RbyTdVG","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"4ad408d9-8384-43dc-b272-9c98072cf881"},"cell_type":"code","source":["### How many records?\n","env.stat()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'branch_pages': 0,\n"," 'depth': 1,\n"," 'entries': 2,\n"," 'leaf_pages': 1,\n"," 'overflow_pages': 0,\n"," 'psize': 4096}"]},"metadata":{"tags":[]},"execution_count":34}]},{"metadata":{"id":"0R11qbXRTdVK","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"1c5f6068-64ee-46e3-b0ca-6cb133d66af9"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    print(txn.stat(data_db))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["{'psize': 4096, 'depth': 1, 'branch_pages': 0, 'leaf_pages': 1, 'overflow_pages': 0, 'entries': 100}\n"],"name":"stdout"}]},{"metadata":{"id":"nFzvSzxnTdVP","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"8a7dc80d-e2db-413f-e485-3c6952feb0b3"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    print(txn.stat(tag_db))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["{'psize': 4096, 'depth': 1, 'branch_pages': 0, 'leaf_pages': 1, 'overflow_pages': 245, 'entries': 102}\n"],"name":"stdout"}]},{"metadata":{"id":"dUNZIMyBTdVV","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### how to get 50th entry? "],"execution_count":0,"outputs":[]},{"metadata":{"id":"SIGTnl8jTdVY","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"d6a9d134-c402-4a2d-b7e2-f0aef3c5aeda"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    cursor = txn.cursor(data_db)\n","    cursor.set_range(b'50')\n","    print(cursor.item())"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(b'Alexander Christensen', b'81')\n"],"name":"stdout"}]},{"metadata":{"id":"K_1VTauBTdVc","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["## doesn't work"],"execution_count":0,"outputs":[]},{"metadata":{"id":"rBTnm4znTdVf","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"0096d722-a021-4b6c-c620-13ec081ec587"},"cell_type":"code","source":["### How long does it take to read 100 keys 128 times\n","start = time.time()\n","with env.begin(write=False) as txn:\n","    cursor = txn.cursor(data_db)\n","    for _ in range(128):\n","        for key in data:\n","            cursor.get(key)\n","print(f'To read 12800 keys it took {time.time() - start} seconds')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["To read 12800 keys it took 0.027528047561645508 seconds\n"],"name":"stdout"}]},{"metadata":{"id":"dsZcJidATdVk","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Of course there are only 100 records in the tree."],"execution_count":0,"outputs":[]},{"metadata":{"id":"cxKj3wNITdVn","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["# Let me add a 100_000+ records"],"execution_count":0,"outputs":[]},{"metadata":{"id":"IF0lsi-oTdVr","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"b31b6e2c-6bff-4b20-9c8d-d73ef82fe2be"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    #cursor = txn.cursor(data_db)\n","    print(txn.stat(data_db))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["{'psize': 4096, 'depth': 1, 'branch_pages': 0, 'leaf_pages': 1, 'overflow_pages': 0, 'entries': 100}\n"],"name":"stdout"}]},{"metadata":{"id":"AN-LrkYdTdVv","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"dc996836-af59-43e9-b5d4-a6112d0c27ca"},"cell_type":"code","source":[""],"execution_count":0,"outputs":[{"output_type":"stream","text":["To read 12800 keys it took 0.01929330825805664 seconds\n"],"name":"stdout"}]},{"metadata":{"id":"_eYWEg8NTdVz","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["# Add 1000K  record\n","with env.begin(write=True) as txn:\n","    # Add to tag DB\n","    cursor = txn.cursor(data_db)\n","    for i in range(100_000):\n","        # Only byte like object accepted\n","        key = fake.name() + str(i)\n","        key = key.encode('UTF-8')\n","        cursor.put(key=key, value=str(i).encode('UTF-8'))"],"execution_count":0,"outputs":[]},{"metadata":{"id":"zEU1XtyITdV2","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"d22fb6e6-5ee4-4298-a5ff-10f278e21468"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    #cursor = txn.cursor(data_db)\n","    print(txn.stat(data_db))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["{'psize': 4096, 'depth': 3, 'branch_pages': 14, 'leaf_pages': 1165, 'overflow_pages': 0, 'entries': 100100}\n"],"name":"stdout"}]},{"metadata":{"id":"yoKkg7mATdV7","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":["### Add more record causes MAP_SIZE Error"],"execution_count":0,"outputs":[]},{"metadata":{"id":"IseUGXhaTdWC","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"501414d3-2e5c-4f9c-da60-df2c6bd16a44"},"cell_type":"code","source":["### How long does it take to read 100 keys 128 times\n","start = time.time()\n","with env.begin(write=False) as txn:\n","    cursor = txn.cursor(data_db)\n","    for _ in range(128):\n","        for key in data:\n","            cursor.get(key)\n","print(f'To read 12800 keys it took {time.time() - start} seconds')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["To read 12800 keys it took 0.029829978942871094 seconds\n"],"name":"stdout"}]},{"metadata":{"id":"oWaDQ7FCTdWH","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"0c64e484-e44b-4a42-dd98-ce4c3c8209d1"},"cell_type":"code","source":["env.stat()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'branch_pages': 0,\n"," 'depth': 1,\n"," 'entries': 2,\n"," 'leaf_pages': 1,\n"," 'overflow_pages': 0,\n"," 'psize': 4096}"]},"metadata":{"tags":[]},"execution_count":51}]},{"metadata":{"id":"qjuhiqq8TdWL","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"26627007-84ba-4adf-bc91-00a29bb406a5"},"cell_type":"code","source":["with env.begin(write=True) as txn:\n","    #cursor = txn.cursor(data_db)\n","    print(txn.stat(data_db))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["{'psize': 4096, 'depth': 3, 'branch_pages': 14, 'leaf_pages': 1165, 'overflow_pages': 0, 'entries': 100100}\n"],"name":"stdout"}]},{"metadata":{"id":"lB523PzZTdWP","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0},"output_extras":[{}]},"outputId":"b4df4d3b-741f-4add-ae32-cdcdefa84e7d"},"cell_type":"code","source":["env.info()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'last_pgno': 1433,\n"," 'last_txnid': 7,\n"," 'map_addr': 0,\n"," 'map_size': 10485760,\n"," 'max_readers': 126,\n"," 'num_readers': 1}"]},"metadata":{"tags":[]},"execution_count":53}]},{"metadata":{"id":"IwYvGtVKTdWX","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"y7m9fV7wTdWa","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"0O5sQjh_TdWe","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"z_7u8EemTdWi","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"O30G6QvkTdWo","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"_uqpATIQTdWv","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"251BovsiTdWz","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"mQvGJmKaTdW8","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"VSGaUPS3TdXH","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"OPJ8HRusTdXP","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"6YVrEjmtTdXU","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"wPnG_aQxTdXW","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"Ze5R4OKHTdXZ","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"N2M2w9RcTdXf","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"j5GdiAxcTdXi","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"5izZJbDzTdXm","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"cl3O-GIMTdXo","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"nxYzFBqkTdXr","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"zkeXoaz6TdXu","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"jR7gwXXxTdX0","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"wwKbtXABTdX6","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"MhZuGbx9TdX-","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"UDaqlHogTdYG","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"7AV73PXITdYL","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"SJX8piOuTdYO","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"oqqnTGovTdYR","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"58X1TOOQTdYX","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"W-xihXjZTdYa","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"CZIVTKxbTdYk","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"fgaqGKUYTdYn","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"tDMT-Lp3TdYq","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"w7J0lI4sTdYu","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"Cz44eJyJTdYx","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"_BMd1O66TdY0","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"SB-Fk7LMTdY5","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"4Sn0uhZLTdY7","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"LUtMVeDTTdY-","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"jWqlmcTCTdZA","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"e0Yn9JQTTdZC","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"XIwtFJZmTdZE","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"_DZEzgbHTdZG","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"53AHRZEuTdZI","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"d0g_2VNyTdZK","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"oF461OWiTdZM","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"M-WAkqgATdZO","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"nkUnPwdGTdZQ","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"4aIcZGX-TdZR","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"6yQ5nBpXTdZT","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"slvo4Dg2TdZV","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"tU6CVG19TdZY","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"26EsDcDwTdZb","colab_type":"code","colab":{"autoexec":{"startup":false,"wait_interval":0}}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]}]}