Skip to content

Instantly share code, notes, and snippets.

@louis030195

louis030195/quotesnearestneighbour.ipynb

Last active November 11, 2020 15:40
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save louis030195/3fe7b242a033b6016f2243c40e5b36c6 to your computer and use it in GitHub Desktop.
Save louis030195/3fe7b242a033b6016f2243c40e5b36c6 to your computer and use it in GitHub Desktop.
Download ZIP
Nearest neighbor search on Readwise quotes using FAISS and HuggingFace tools
Display the source blob
Display the rendered blob
Raw
quotesnearestneighbour.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "QuotesNearestNeighbour.ipynb",
"provenance": [],
"collapsed_sections": [],
"authorship_tag": "ABX9TyOZb11Zh/cucgTMZ2aSFyr5",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU",
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"7f1abff69fda4443b2f3bccf341e4a00": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"state": {
"_view_name": "HBoxView",
"_dom_classes": [],
"_model_name": "HBoxModel",
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.5.0",
"box_style": "",
"layout": "IPY_MODEL_2c41dc9bc36d43e9b83576290ac19a85",
"_model_module": "@jupyter-widgets/controls",
"children": [
"IPY_MODEL_c91e2dee23d9476cbe6b39534952237b",
"IPY_MODEL_415dc56492f54bb2b21c714a633156ca"
]
}
},
"2c41dc9bc36d43e9b83576290ac19a85": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
},
"c91e2dee23d9476cbe6b39534952237b": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"state": {
"_view_name": "ProgressView",
"style": "IPY_MODEL_9bbcd00a9d504026852c75ea4463ab71",
"_dom_classes": [],
"description": "100%",
"_model_name": "FloatProgressModel",
"bar_style": "success",
"max": 401,
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"value": 401,
"_view_count": null,
"_view_module_version": "1.5.0",
"orientation": "horizontal",
"min": 0,
"description_tooltip": null,
"_model_module": "@jupyter-widgets/controls",
"layout": "IPY_MODEL_0d925904c5a64e21a881498ad619ba2c"
}
},
"415dc56492f54bb2b21c714a633156ca": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"state": {
"_view_name": "HTMLView",
"style": "IPY_MODEL_fef88aaa2acc48e7a7e942aabde7a965",
"_dom_classes": [],
"description": "",
"_model_name": "HTMLModel",
"placeholder": "​",
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"value": " 401/401 [00:00<00:00, 3261.36ex/s]",
"_view_count": null,
"_view_module_version": "1.5.0",
"description_tooltip": null,
"_model_module": "@jupyter-widgets/controls",
"layout": "IPY_MODEL_e057c61daf024dbcbb24edea822f7c7c"
}
},
"9bbcd00a9d504026852c75ea4463ab71": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"state": {
"_view_name": "StyleView",
"_model_name": "ProgressStyleModel",
"description_width": "initial",
"_view_module": "@jupyter-widgets/base",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.2.0",
"bar_color": null,
"_model_module": "@jupyter-widgets/controls"
}
},
"0d925904c5a64e21a881498ad619ba2c": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
},
"fef88aaa2acc48e7a7e942aabde7a965": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"state": {
"_view_name": "StyleView",
"_model_name": "DescriptionStyleModel",
"description_width": "",
"_view_module": "@jupyter-widgets/base",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.2.0",
"_model_module": "@jupyter-widgets/controls"
}
},
"e057c61daf024dbcbb24edea822f7c7c": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
},
"11c280f594cf49769af0f91e9a52021e": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"state": {
"_view_name": "HBoxView",
"_dom_classes": [],
"_model_name": "HBoxModel",
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.5.0",
"box_style": "",
"layout": "IPY_MODEL_db6638fe62e348b591b46254f41758b8",
"_model_module": "@jupyter-widgets/controls",
"children": [
"IPY_MODEL_9fc1aea915704a2a9aa54aebe9c33e16",
"IPY_MODEL_ed75da21096e4bbf844427d3f950599d"
]
}
},
"db6638fe62e348b591b46254f41758b8": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
},
"9fc1aea915704a2a9aa54aebe9c33e16": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"state": {
"_view_name": "ProgressView",
"style": "IPY_MODEL_9eef96beec9941f996e9288cd5100202",
"_dom_classes": [],
"description": "100%",
"_model_name": "FloatProgressModel",
"bar_style": "success",
"max": 401,
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"value": 401,
"_view_count": null,
"_view_module_version": "1.5.0",
"orientation": "horizontal",
"min": 0,
"description_tooltip": null,
"_model_module": "@jupyter-widgets/controls",
"layout": "IPY_MODEL_286d73526d684f08a9d4e5f86f8c8a10"
}
},
"ed75da21096e4bbf844427d3f950599d": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"state": {
"_view_name": "HTMLView",
"style": "IPY_MODEL_6c5778cfcafb46349e942223a9e07be1",
"_dom_classes": [],
"description": "",
"_model_name": "HTMLModel",
"placeholder": "​",
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"value": " 401/401 [01:13<00:00, 5.43ex/s]",
"_view_count": null,
"_view_module_version": "1.5.0",
"description_tooltip": null,
"_model_module": "@jupyter-widgets/controls",
"layout": "IPY_MODEL_f3b13d7f087c4775b1416dffa297af3d"
}
},
"9eef96beec9941f996e9288cd5100202": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"state": {
"_view_name": "StyleView",
"_model_name": "ProgressStyleModel",
"description_width": "initial",
"_view_module": "@jupyter-widgets/base",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.2.0",
"bar_color": null,
"_model_module": "@jupyter-widgets/controls"
}
},
"286d73526d684f08a9d4e5f86f8c8a10": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
},
"6c5778cfcafb46349e942223a9e07be1": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"state": {
"_view_name": "StyleView",
"_model_name": "DescriptionStyleModel",
"description_width": "",
"_view_module": "@jupyter-widgets/base",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.2.0",
"_model_module": "@jupyter-widgets/controls"
}
},
"f3b13d7f087c4775b1416dffa297af3d": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
},
"005e3e0dd60842bf904eaf3f9bb4aae9": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"state": {
"_view_name": "HBoxView",
"_dom_classes": [],
"_model_name": "HBoxModel",
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.5.0",
"box_style": "",
"layout": "IPY_MODEL_ed61d0b41b6f412e8603fa5769daf714",
"_model_module": "@jupyter-widgets/controls",
"children": [
"IPY_MODEL_46f5f68619d14560865dcb08e4d2e052",
"IPY_MODEL_49b404d8096c411387789540377f92aa"
]
}
},
"ed61d0b41b6f412e8603fa5769daf714": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
},
"46f5f68619d14560865dcb08e4d2e052": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"state": {
"_view_name": "ProgressView",
"style": "IPY_MODEL_7bcbc73eddb044f6bae964390b786488",
"_dom_classes": [],
"description": "100%",
"_model_name": "FloatProgressModel",
"bar_style": "success",
"max": 1,
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"value": 1,
"_view_count": null,
"_view_module_version": "1.5.0",
"orientation": "horizontal",
"min": 0,
"description_tooltip": null,
"_model_module": "@jupyter-widgets/controls",
"layout": "IPY_MODEL_2db64961119549d3ba6a9d0808e19b38"
}
},
"49b404d8096c411387789540377f92aa": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"state": {
"_view_name": "HTMLView",
"style": "IPY_MODEL_4aa17fabf3e04c58a5d618710a205866",
"_dom_classes": [],
"description": "",
"_model_name": "HTMLModel",
"placeholder": "​",
"_view_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"value": " 1/1 [00:04<00:00, 4.74s/it]",
"_view_count": null,
"_view_module_version": "1.5.0",
"description_tooltip": null,
"_model_module": "@jupyter-widgets/controls",
"layout": "IPY_MODEL_c353c18d317448e782b923bbdf95f731"
}
},
"7bcbc73eddb044f6bae964390b786488": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"state": {
"_view_name": "StyleView",
"_model_name": "ProgressStyleModel",
"description_width": "initial",
"_view_module": "@jupyter-widgets/base",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.2.0",
"bar_color": null,
"_model_module": "@jupyter-widgets/controls"
}
},
"2db64961119549d3ba6a9d0808e19b38": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
},
"4aa17fabf3e04c58a5d618710a205866": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"state": {
"_view_name": "StyleView",
"_model_name": "DescriptionStyleModel",
"description_width": "",
"_view_module": "@jupyter-widgets/base",
"_model_module_version": "1.5.0",
"_view_count": null,
"_view_module_version": "1.2.0",
"_model_module": "@jupyter-widgets/controls"
}
},
"c353c18d317448e782b923bbdf95f731": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"state": {
"_view_name": "LayoutView",
"grid_template_rows": null,
"right": null,
"justify_content": null,
"_view_module": "@jupyter-widgets/base",
"overflow": null,
"_model_module_version": "1.2.0",
"_view_count": null,
"flex_flow": null,
"width": null,
"min_width": null,
"border": null,
"align_items": null,
"bottom": null,
"_model_module": "@jupyter-widgets/base",
"top": null,
"grid_column": null,
"overflow_y": null,
"overflow_x": null,
"grid_auto_flow": null,
"grid_area": null,
"grid_template_columns": null,
"flex": null,
"_model_name": "LayoutModel",
"justify_items": null,
"grid_row": null,
"max_height": null,
"align_content": null,
"visibility": null,
"align_self": null,
"height": null,
"min_height": null,
"padding": null,
"grid_auto_rows": null,
"grid_gap": null,
"max_width": null,
"order": null,
"_view_module_version": "1.2.0",
"grid_template_areas": null,
"object_position": null,
"object_fit": null,
"grid_auto_columns": null,
"margin": null,
"display": null,
"left": null
}
}
}
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/louis030195/3fe7b242a033b6016f2243c40e5b36c6/quotesnearestneighbour.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"metadata": {
"id": "-fBX-0o3dCLD"
},
"source": [
"!pip install -q transformers datasets faiss-gpu # Or faiss-cpu"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "vnkMtwf_aOGy",
"outputId": "94a245e7-4438-4434-d55d-e8b3498ed72f",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"source": [
"import os\n",
"import pandas as pd\n",
"import requests\n",
"import datetime\n",
"os.environ[\"READWISE\"] = \"GRAB IT HERE https://readwise.io/access_token\"\n",
"\n",
"TOKEN = os.environ[\"READWISE\"]\n",
"response = requests.get(\n",
" url=\"https://readwise.io/api/v2/auth/\",\n",
" headers={\"Authorization\": f\"Token {TOKEN}\"},\n",
")\n",
"if response.status_code != 204: raise Exception(f\"Invalid Readwise token: {TOKEN}\")\n",
" \n",
"def books_as_dataframe(delay_in_days):\n",
" delay = datetime.datetime.now() - datetime.timedelta(days=delay_in_days)\n",
"\n",
" query = {\n",
" \"category\": \"books\",\n",
" \"updated__gt\": delay.strftime(\"%Y-%m-%dT%H:%M:%SZ\"),\n",
" }\n",
"\n",
" response = requests.get(\n",
" url=\"https://readwise.io/api/v2/books/\",\n",
" headers={\"Authorization\": f\"Token {TOKEN}\"},\n",
" params=query\n",
" )\n",
"\n",
" books = response.json()\n",
" return pd.DataFrame(books[\"results\"])\n",
"\n",
"def book_highlights_as_dataframe(book_id):\n",
" query = {\n",
" \"book_id\": book_id\n",
" }\n",
"\n",
" response = requests.get(\n",
" url=\"https://readwise.io/api/v2/highlights/\",\n",
" headers={\"Authorization\": f\"Token {TOKEN}\"},\n",
" params=query\n",
" )\n",
"\n",
" highlights = response.json()\n",
" return pd.DataFrame(highlights[\"results\"])\n",
"\n",
"books = books_as_dataframe(400)\n",
"quotes = []\n",
"for _, book in books.iterrows():\n",
" try:\n",
" q = [\n",
" quote for quote in book_highlights_as_dataframe(book.id).text.values \\\n",
" if quote and \\\n",
" isinstance(quote, str) and \\\n",
" len(quote) > 10 and \\\n",
" len(quote) < 512 # Some filters, like limiting quote sizing is necessary\n",
" ]\n",
" if not q: raise Exception() # Another filter to crappy Readwise data :D\n",
" quotes.append(q)\n",
" except:\n",
" print(f\"Ignoring {book.title}, data broken\")\n",
" # Cleanup\n",
" books = books.drop(books[ books['id'] == book.id ].index, axis=0)\n",
" \n",
"books[\"quote\"] = quotes\n",
"books = books.explode(\"quote\")"
],
"execution_count": 86,
"outputs": [
{
"output_type": "stream",
"text": [
"Ignoring Quick Passages, data broken\n",
"Ignoring Plato, data broken\n",
"Ignoring Turnbull, J. - The Art of Monitoring-James Turnbull, data broken\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "zwhfTGb3xvs9",
"outputId": "05de592b-4122-4fd4-8841-f8359c8464b1",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 844
}
},
"source": [
"books"
],
"execution_count": 48,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>title</th>\n",
" <th>author</th>\n",
" <th>category</th>\n",
" <th>num_highlights</th>\n",
" <th>last_highlight_at</th>\n",
" <th>updated</th>\n",
" <th>cover_image_url</th>\n",
" <th>highlights_url</th>\n",
" <th>quote</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4625724</td>\n",
" <td>Douglas R. Hofstadter - Gödel, Escher, Bach_ a...</td>\n",
" <td>1994</td>\n",
" <td>books</td>\n",
" <td>6</td>\n",
" <td>2020-09-14T13:09:37Z</td>\n",
" <td>2020-09-14T18:05:29.023564Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/4625724</td>\n",
" <td>ACHILLES: It goes like this: Two monks were ar...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4625724</td>\n",
" <td>Douglas R. Hofstadter - Gödel, Escher, Bach_ a...</td>\n",
" <td>1994</td>\n",
" <td>books</td>\n",
" <td>6</td>\n",
" <td>2020-09-14T13:09:37Z</td>\n",
" <td>2020-09-14T18:05:29.023564Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/4625724</td>\n",
" <td>TORTOISE: Zen Koan? Zen Master? What do you me...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4625724</td>\n",
" <td>Douglas R. Hofstadter - Gödel, Escher, Bach_ a...</td>\n",
" <td>1994</td>\n",
" <td>books</td>\n",
" <td>6</td>\n",
" <td>2020-09-14T13:09:37Z</td>\n",
" <td>2020-09-14T18:05:29.023564Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/4625724</td>\n",
" <td>ACHILLES: It goes like this: Two monks were ar...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4625724</td>\n",
" <td>Douglas R. Hofstadter - Gödel, Escher, Bach_ a...</td>\n",
" <td>1994</td>\n",
" <td>books</td>\n",
" <td>6</td>\n",
" <td>2020-09-14T13:09:37Z</td>\n",
" <td>2020-09-14T18:05:29.023564Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/4625724</td>\n",
" <td>Principia Mathematica (P.M.), a giant opus by ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4625724</td>\n",
" <td>Douglas R. Hofstadter - Gödel, Escher, Bach_ a...</td>\n",
" <td>1994</td>\n",
" <td>books</td>\n",
" <td>6</td>\n",
" <td>2020-09-14T13:09:37Z</td>\n",
" <td>2020-09-14T18:05:29.023564Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/4625724</td>\n",
" <td>fixed system of numbertheoretical reasoning to...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33</th>\n",
" <td>2530129</td>\n",
" <td>Steven Strogatz - Infinite Powers_ How Calculu...</td>\n",
" <td>2019, Houghton Mifflin Harcourt</td>\n",
" <td>books</td>\n",
" <td>3</td>\n",
" <td>2019-12-23T23:27:32Z</td>\n",
" <td>2020-05-09T18:44:27.011510Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/2530129</td>\n",
" <td>Thus, calculus proceeds in two phases: cutting...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33</th>\n",
" <td>2530129</td>\n",
" <td>Steven Strogatz - Infinite Powers_ How Calculu...</td>\n",
" <td>2019, Houghton Mifflin Harcourt</td>\n",
" <td>books</td>\n",
" <td>3</td>\n",
" <td>2019-12-23T23:27:32Z</td>\n",
" <td>2020-05-09T18:44:27.011510Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/2530129</td>\n",
" <td>In every case the strategy remains the same: s...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>34</th>\n",
" <td>2530128</td>\n",
" <td>Aldous Huxley</td>\n",
" <td>Brave New World 2006</td>\n",
" <td>books</td>\n",
" <td>3</td>\n",
" <td>2019-12-20T03:45:01Z</td>\n",
" <td>2020-05-09T18:44:26.983031Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/2530128</td>\n",
" <td>\"Alpha children wear grey They work much harde...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>34</th>\n",
" <td>2530128</td>\n",
" <td>Aldous Huxley</td>\n",
" <td>Brave New World 2006</td>\n",
" <td>books</td>\n",
" <td>3</td>\n",
" <td>2019-12-20T03:45:01Z</td>\n",
" <td>2020-05-09T18:44:26.983031Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/2530128</td>\n",
" <td>All the air of the fourteenth floor was sibila...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>34</th>\n",
" <td>2530128</td>\n",
" <td>Aldous Huxley</td>\n",
" <td>Brave New World 2006</td>\n",
" <td>books</td>\n",
" <td>3</td>\n",
" <td>2019-12-20T03:45:01Z</td>\n",
" <td>2020-05-09T18:44:26.983031Z</td>\n",
" <td>https://readwise-assets.s3.amazonaws.com/stati...</td>\n",
" <td>https://readwise.io/bookreview/2530128</td>\n",
" <td>And that,\" put in the Director sententiously, ...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>489 rows × 10 columns</p>\n",
"</div>"
],
"text/plain": [
" id ... quote\n",
"0 4625724 ... ACHILLES: It goes like this: Two monks were ar...\n",
"0 4625724 ... TORTOISE: Zen Koan? Zen Master? What do you me...\n",
"0 4625724 ... ACHILLES: It goes like this: Two monks were ar...\n",
"0 4625724 ... Principia Mathematica (P.M.), a giant opus by ...\n",
"0 4625724 ... fixed system of numbertheoretical reasoning to...\n",
".. ... ... ...\n",
"33 2530129 ... Thus, calculus proceeds in two phases: cutting...\n",
"33 2530129 ... In every case the strategy remains the same: s...\n",
"34 2530128 ... \"Alpha children wear grey They work much harde...\n",
"34 2530128 ... All the air of the fourteenth floor was sibila...\n",
"34 2530128 ... And that,\" put in the Director sententiously, ...\n",
"\n",
"[489 rows x 10 columns]"
]
},
"metadata": {
"tags": []
},
"execution_count": 48
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "Mw0as_WceHNZ",
"outputId": "0015ffed-1fcf-41e9-b911-435f6f02f9c3",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"source": [
"from datasets import Dataset\n",
"import pandas as pd\n",
"dataset = Dataset.from_pandas(books)\n",
"dataset"
],
"execution_count": 87,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Dataset(features: {'id': Value(dtype='int64', id=None), 'title': Value(dtype='string', id=None), 'author': Value(dtype='string', id=None), 'category': Value(dtype='string', id=None), 'num_highlights': Value(dtype='int64', id=None), 'last_highlight_at': Value(dtype='string', id=None), 'updated': Value(dtype='string', id=None), 'cover_image_url': Value(dtype='string', id=None), 'highlights_url': Value(dtype='string', id=None), 'quote': Value(dtype='string', id=None), '__index_level_0__': Value(dtype='int64', id=None)}, num_rows: 401)"
]
},
"metadata": {
"tags": []
},
"execution_count": 87
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "wTaS7b8ahQAG",
"outputId": "7d7e9074-d48f-413e-f7dc-ba3504ed88f7",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000,
"referenced_widgets": [
"7f1abff69fda4443b2f3bccf341e4a00",
"2c41dc9bc36d43e9b83576290ac19a85",
"c91e2dee23d9476cbe6b39534952237b",
"415dc56492f54bb2b21c714a633156ca",
"9bbcd00a9d504026852c75ea4463ab71",
"0d925904c5a64e21a881498ad619ba2c",
"fef88aaa2acc48e7a7e942aabde7a965",
"e057c61daf024dbcbb24edea822f7c7c"
]
}
},
"source": [
"dataset.map(lambda quote: print(quote[\"quote\"])) # Still low quality data"
],
"execution_count": 102,
"outputs": [
{
"output_type": "stream",
"text": [
"ACHILLES: It goes like this: Two monks were arguing about a flag. One said, “The flag is moving. ” The other said, “The wind is moving. ” The sixth patriarch, Zeno, happened to be passing by. He told them, “Not the wind, not the flag, mind is moving. ”\n"
],
"name": "stdout"
},
{
"output_type": "display_data",
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7f1abff69fda4443b2f3bccf341e4a00",
"version_minor": 0,
"version_major": 2
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=401.0), HTML(value='')))"
]
},
"metadata": {
"tags": []
}
},
{
"output_type": "stream",
"text": [
"ACHILLES: It goes like this: Two monks were arguing about a flag. One said, “The flag is moving. ” The other said, “The wind is moving. ” The sixth patriarch, Zeno, happened to be passing by. He told them, “Not the wind, not the flag, mind is moving. ”\n",
"TORTOISE: Zen Koan? Zen Master? What do you mean? ACHILLES: It goes like this: Two monks were arguing about a flag. One said, “The flag is\n",
"ACHILLES: It goes like this: Two monks were arguing about a flag. One said, “The flag is moving. ” The other said, “The wind is moving. ” The sixth patriarch, Zeno, happened to be passing by. He told them, “Not the wind, not the flag, mind is moving\n",
"Principia Mathematica (P.M.), a giant opus by Bertrand Russell and Alfred North Whitehead\n",
"fixed system of numbertheoretical reasoning to which the word \"proof\" refers is that of\n",
"system of numbertheoretical reasoning to which the word \"proof\" refers is that of Principia Mathematica (P.M.), a giant opus by Bertrand Russell and Alfred North\n",
"Women are likely to be different: Having sex with a stranger not only encumbered a Pleistocene woman with a possible pregnancy before she had won the man 's commitment to help rear the child, but it also exposed her to probable revenge from her husband\n",
"men are more likely to be tempted by an opportunity for casual sex than women\n",
"It would be easy to engineer a society with no sex difference in attitude between men and women: Inject all pregnant women with the right dose of hormones, and the result would be men and women with normal bodies but identical feminine brains: War, rape, boxing, car racing, pornography, and hamburgers and beer would soon be distant memories: A feminist paradise would have arrived\n",
"It would be easy to engineer a society with no sex difference in attitude between men and women: Inject all pregnant women with the right dose of hormones, and the result would be men and women with normal bodies but identical feminine brains: War, rape, boxing, car racing, pornography, and hamburgers and beer would\n",
"Jonathan Kingdon in his recent book Self-Made Man and His Undoing\n",
"This phenomenon—that people specialize in what they are good at and so create conditions that suit their genes—is known as the Baldwin effect\n",
"Men look for sources that are mobile, distant, and unpredictable (usually meat), while women, burdened with children, look for sources that are static, close, and predictable (usually plants)\n",
"Women's bodies evolved to suit the demands of bearing and rearing children and of gathering plant food. Men's bodies evolved to suit the demands of rising in a male hierarchy, fighting over women, and providing meat to a family\n",
"Women are and always have been far less interested in polygamy than men\n",
"Marriage is a child-rearing institution\n",
"If war is something we inherited directly from the hostility between groups of male apes over female apes, with territory as merely a means to the end—sex—then it follows that tribal people must be going to war over women rather than territory. For a long time anthropologists insisted that war was fought over scarce material resources, in particular protein, which was often in short supply\n",
"Monogamy, enforced by law, religion, or sanction, does seem to reduce murderous competition between men\n",
"The gradual synonymy of sex and sin in Christendom is surely based more on the fact that sex often leads to trouble rather than that there is anything inherently sinful about sex\n",
"By I.6 million years ago, when Homo erectus was living in Africa, he was without question the most carnivorous monkey or ape the world had ever known\n",
"Which woman would not rather be John Kennedy 's third wife than Bozo the Clown's first?\" said one (female) evolutionist\n",
"Gender, then, was invented as a means of resolving the conflict between the cytoplasmic genes of the two parents. Rather than let such conflict destroy the offspring, a sensible agreement was reached: All the cytoplasmic genes would come from the mother, none from the father. Since this made the father 's gametes smaller, they could specialize in being more numerous and mobile the better to find eggs. Gender is a bureaucratic solution to an antisocial habit\n",
"Each gene is descended from a gene that unwittingly jostled to get into the next generation by whatever means was in its power. Cooperation between them is marked, but so is competition. And it is that competition that led to the invention of gender\n",
"Finding the right balance between cooperation and competition has been the goal and bane of Western politics for centuries\n",
"if the tangled bank was right and competition between snails was the cause of sex, he would find more males in lakes than in streams because lakes are stable, crowded habitats; if the Red Queen was right, he would find more males where there were more parasites.\n",
"the tangled bank was right and competition between snails was the cause of sex, he would find more males in lakes than in streams because lakes are stable, crowded habitats; if the Red Queen was right, he would find more males where there were more parasites\n",
"By turning our fields over to monocultures of increasingly inbred strains of wheat and maize, we are inviting the very epidemics of disease that can only be fought by the pesticides we are forced to use in ever larger quantities\n",
"turning our fields over to monocultures of increasingly inbred strains of wheat and maize, we are inviting the very epidemics of disease that can only be fought by the pesticides we are forced to use in ever larger quantities\n",
"Because they are so short-lived compared with their hosts, parasites can be quicker to evolve and adapt. In about ten years, the genes of the AIDS virus change as much as human genes change in 10 million years\n",
"Bell concluded from his exhaustive survey of sex and asex in the animal kingdom that the tangled bank was the most promising of the ecological theories for sex.' Z\n",
"Bell concluded from his exhaustive survey of sex and asex in the animal kingdom that the tangled bank was the most promising of the ecological theories for\n",
"Asexual species tend to be small and live at high latitudes and high altitudes, in fresh water or disturbed ground\n",
"bdelloids gave up sex between 40 million and 80 million years ago.\n",
"Bell reckons that being sexual was a prerequisite for being big (and therefore few), or, conversely, sex is unnecessary if you stay small\n",
"Vicar of Bray theory Muller\n",
", with Hermann Muller, one of the fathers of the Vicar of\n",
"The leader of the molecular biologists is Harris Bernstein of the University of Arizona. His argument is that sex was invented to repair genes\n",
"Every creature on earth is in a Red Queen chess tournament with its parasites (or hosts), its predators (or prey), and, above all, with its mate\n",
"When the land was unoccupied by animals, the first amphibian to emerge from the sea could get away with being slow, lumbering, and fishlike, for it had no enemies and no competitors\n",
"Evolutionary history is no different. Progress and success are always relative: When the land was unoccupied by animals, the first amphibian to emerge from the sea could get away with being slow\n",
"When the land was unoccupied by animals, the first amphibian to emerge from the sea could get away with being slow, lumbering, and fishlike, for it had no enemies and no competitors. But if a fish were to take to the\n",
"When the land was unoccupied by animals, the first amphibian to emerge from the sea could get away with being slow, lumbering, and fishlike, for it had no enemies and no competitors. But if a fish were to take to the\n",
"Therefore, if you spot somebody with good genes, it is your inherited habit to seek to buy some of those genes; or, put more prosaically, people are attracted to people of high reproductive and genetic potential—the healthy, the fit, and the powerful\n",
"For example, males usually compete for access to females, rather than vice versa. There are good evolutionary reasons for this, and there are clear evolutionary consequences, too; for instance, men are more aggressive than women.\n",
"example, males usually compete for access to females, rather than vice versa. There are good evolutionary reasons for this, and there are clear evolutionary consequences, too; for instance, men are more aggressive than women.\n",
"Human nature was as carefully designed by natural selection for the use of a social, bipedal, originally African ape as human stomachs were designed for the use of an omnivorous African ape with a taste for meat\n",
"Why sex? Surely there are features of human nature other than this one overexposed and troublesome procreative pastime: True enough, but reproduction is the sole goal for which human beings are designed;\n",
"Making things is a joy—immensely satisfying. J. R. R. Tolkien suggests that God gave us the gift of subcreation, as a gift, just for our joy.2 After all, “The cattle on a thousand hills are mine. … If I were hungry, I would not tell you.”3 Designing per se is fun.\n",
"Making things is a joy—immensely satisfying. J. R. R. Tolkien suggests that God gave us the gift of subcreation, as a gift, just for our joy.2 After all, “The cattle on a thousand hills are mine. …\n",
"Life's fnest days, for u poor hu big \\ Fly fst\n",
"heaviest particles and any cloudiness settling to the bottom. It is just the same with human life. The best comes frst\n",
"Things tend, i fact, to go wrong ; part of the blame lies on the teachers of philosophy, who today teach us how to ague instead of how to live, part on their students, who come to the teachers i the frst place with a view to developig not their character but their itellect. The result ha been the trasformation of philosophy, the study of widom, into philology, the study ofword\n",
"To lose someone you love i something you'll regad as the hardest of all blows to bea, while all the time this will be a sly a cryig because the leaves fll fom the beautiful tree that add to the charm of your home\n",
"for people cee to possess everthing a soon a tey want everything for themselves.\n",
"Wat's the ue ofovercoming opponet a opponent in the wetl or boxng rings iyou c b overcome by you temper\n",
"What realy rs ou characers is the fc that none of u looks back over his life. We t about what we ae going to do, ad ony rrely of that, ad f to t about what we have done, yet ay pla for te fte ae dependent on the pat\n",
"Refl to b iueced by one's body asure one's feedom\n",
"I am too great, wa hom to too gret a destiny to be my body's slave\n",
"am too great, wa hom to too gret a destiny to be my body's slave\n",
"In the pleue we fd i the memory of departed fiends there i a resemblace to the way in which certain bitter fruit are agreeble or te very acidity of a exceedingly old wine h it atraction\n",
"Wen one h lost a fiend one's eyes shoud be neiter dry nor streaming. Tes, yes, there should be, but not lametaton\n",
"Wen one h lost a fiend one's eyes shoud be neiter dry nor streaming\n",
"For that is what philosophy ha promised me - that she wl me me God's equ. That's the invittion ad that's wht I've come for ; b a good a you word\n",
"How much longer are you going to sere under others' orders? Assume authority yourself and utter something that may be handed dow to posterity. Produce sometg fom you own resources\n",
"A consciousness of wrongdoing . is te fst step to salvation\n",
"A persn who h lened how to die h uneed how to be a slave.\n",
"If you shape your lie accordng to nte, you wl never be poor; i according to people's opion, you wil never be rch.'\n",
"you shape your lie accordng to nte, you wl never be poor; i according to people's opion, you wil never be rch.'\n",
"If you shape your lie accordng to nte, you wl never be poor\n",
"Without it no one c led a lfe fee of fe or worry. Every hou of the day countles situation arise tt cl for advce, ad for that advice we have to look to philosophy\n",
"I T i cle to you, I know, Lucilius, tat no one c led a happy lfe, or eve one that is beable, without te pursuit of wsdom, and that the perfection of wisdom i what make te happy life, although eve the bengs ofwisdom make life beable\n",
"Woever h said 'I have lived• receive a wdal ee day he get up i the mor.\n",
"Sel-ontented a he is, ten, he doe need fend - ad wants as many of them a possible - but not to eable him to lead a happy life; t he will have even without fends. The spreme ideal does not cl for ay exter aids. It is home grow, wholly self-developed. Once it start looking outide itselfor any pat ofitselit i on the way to beig domated by fortne.\n",
"I'm still ting over the pages of Epicurus, and the followng sying, one I read today, comes fom him: 'To win tre freedom you must be a slave to phlosophy.' A person who surenders ad subjects himsel to her doesn't have his application deferred from day to day ; he's emancipated on te spot, the very service of phosophy being fe dom.\n",
"Retire ito yourself as much a you c. Associate wth people who are likely to improve you. Welcome those whom you are capable of improving. The process is a mutual one : men learn as they teach.\n",
"Trstig everone is a much a fault a trsting no one (though I should cl te frst the worthier ad te second the safer behaviour\n",
"Nothing, to my way of tg, is a better proof of a well ordered mind than a man's ability to stop just where he i ad pass some time in his own compay\n",
"This, the summum bonum or 'spreme ideal', i uly sumzed in acient phlosophy as a combination of four qualities: wisdom (or moral iight), couage, selfcontrol adjutice (or upright dealing\n",
"If the meme is a scientific idea, its spread will depend on how acceptable it is to the population of individual scientists; a rough measure of its survival value could be obtained by counting the number of times it is referred to in successiveyears in scientific journals\n",
"An animal's behaviour tends to maximize the survival of thegenes 'for'that behaviour, whether or not those genes happen to be in the body of the particular animalperforming it\n",
"animal's behaviour tends to maximize the survival of thegenes 'for'that behaviour, whether or not those genes happen to be in the body of the particular animalperforming\n",
"We biologists have assimilated the idea of genetic evolution so deeply that we tend to forget that it is only one of many possible kinds of evolution\n",
"An arbitrary label like a green beard is just one wayin which a gene might 'recognize' copies of itself in other individuals\n",
"If all the five strategies I have mentioned are turned loose upon one another in a computer simulation, only one of them, retaliator, emerges as evolutionarily stable.* Prober—retaliator is nearly stable. Dove is not stable,\n",
"all the five strategies I have mentioned are turned loose upon one another in a computer simulation, only one of them, retaliator, emerges as evolutionarily stable.* Prober—retaliator is nearly stable. Dove is not stable, because a population of doves would be invaded by hawks and bullies. Hawk is not stable, because\n",
"Genes are the primarypolicy-makers; brains are the executives. But as brains became more highly developed, they took over more and more of the actual policy decisions, using tricks like learning and simulation in doing so. The logical conclusion to this trend, not yet reached in any species, would be for the genes to give the survivalmachine a single overall policyinstruction: do whatever you think best to keep us alive.\n",
"The evolution of the capacity to simulate seems to have culminated in subjectiveconsciousness. Why\n",
"One of the most interesting methods of predicting the future is simulation\n",
"Afor Andromeda by Fred Hoyle and John Elliot is\n",
"Computers do not yet play chess as well as human grand masters\n",
"Each one of us knows,from the evidence of our own introspection, that, at least in one modern survival machine, this purposiveness has evolved the property we call 'consciousness'. I am not philosopher enough to discuss what this means, but fortunately it does not matter for our present purposes because it is easy to talk about machines that behave as ^/motivated by a purpose, and to leave open the question whether they actually are conscious\n",
"The workings of the sensory systems are particularly baffling, because they can achieve far more sophisticated feats of pattern-recognition than the best and most expensive man-made machines; if this were not so, all typists would be redundant, superseded by speech-recognizing machines, or machines for reading handwriting. Human typistswill be needed for many decades yet.\n",
"The cornerstone of the argument, as given earlier, was the assumption that genes are potentially immortal\n",
"It is differences that matter in the competitive struggle to survive\n",
"Darwin's theory of evolution by natural selection is satisfying because it shows us away in which simplicity could change into complexity, how unordered atoms could group themselves into ever more complex patterns until they ended up manufacturing people\n",
"Darwin's theory of evolution by natural selection is satisfying because it shows us away in which simplicity could change into complexity, how unordered atoms could group themselves into ever\n",
"Lions and antelopes are both members of the class Mammalia, as are we. Should we then not expect lions to refrain from killing antelopes, 'for the good of the mammals\n",
"The stinging behaviour of worker bees is a very effective defence against honey robbers. But the bees who do the stinging are kamikaze fighters. In the act of stinging, vital internal organs are usually torn out of the body, and the bee dies soon afterwards. Her suicide mission mayhave saved the colony's vital food stocks, but she herself is not around to reap the benefits. Byour definition this is an altruistic behavioural act\n",
"The stinging behaviour of worker bees is a very effective defence against honey robbers. But the bees who do the stinging are kamikaze fighters. In the act of stinging, vital internal organs are usually torn out of the body, and the bee dies soon afterwards. Her suicide mission mayhave saved the colony's vital food stocks, but she herself is not around to reap the benefits. Byour definition this is an altruistic behavioural\n",
"A system cannot be successful if it is too strongly influenced by a single person. Once the initial design is complete and fairly robust, the real test begins as people with many different viewpoints undertake their own experiments\n",
"system cannot be successful if it is too strongly influenced by a single person. Once the initial design is complete and fairly robust, the real test begins as people with many different viewpoints undertake their own experiments.\n",
"If we want to make distributed systems work, we must accept the possibility of partial failure and build fault-tolerance mechanisms into the software\n",
"This is a deliberate choice in the design of computers: if an internal fault occurs, we prefer a computer to crash completely rather than returning a wrong result, because wrong results are difficult and confusing to deal with\n",
"Good abstractions can help reduce complexity and make the system easier to modify and adapt for new use cases. Good operability means having good visibility into the system’s health, and having effective ways of managing\n",
"scaling up (vertical scaling, moving to a more powerful machine) and scaling out (horizontal scaling, distributing the load across multiple smaller machines\n",
"even more often than that. People often talk of a dichotomy between scaling up (vertical scaling, moving to a more powerful machine) and scaling out\n",
"If you want to add response time percentiles to the monitoring dashboards for your services, you need to efficiently calculate them on an ongoing basis. For example, you may want to keep a rolling window of response times of requests in the last 10 minutes. Every minute, you calculate the median and various percentiles over the values in that window and plot those metrics on a graph.\n",
"Store data so that they, or another application, can find it again later (databases) Remember the result of an expensive operation, to speed up reads (caches) Allow users to search data by keyword or filter it in various ways (search indexes) Send a message to another process, to be handled asynchronously (stream processing) Periodically crunch a large amount of accumulated data (batch processing)\n",
"Many of the technologies described in this book fall within the realm of the Big Data buzzword. However, the term “Big Data” is so overused and underdefined that it is not useful in a serious engineering discussion. This book uses less ambiguous terms, such as single-node versus distributed systems, or online/interactive versus offline/batch processing systems\n",
"CPU clock speeds are barely increasing, but multi-core processors are standard, and networks are getting faster. This means parallelism is only going to increase\n",
"Businesses need to be agile, test hypotheses cheaply, and respond quickly to new market insights by keeping development cycles short and data models flexible\n",
"“He who climbeth on the highest mountains, laugheth at all tragic plays and tragic realities.”—ZARATHUSTRA,\n",
"Where is innocence? Where there is will to procreation. And he who seeketh to create beyond himself, hath for me the purest will.\n",
"Whatever cannot obey itself, is commanded. Such is the nature of living things.\n",
"It is more, verily, when out of one’s own burning cometh one’s own teaching!\n",
"The man of knowledge must be able not only to love his enemies, but also to hate his friends.\n",
"When, however, ye have an enemy, then return him not good for evil: for that would abash him. But prove that he hath done something good to you.\n",
"Two different things wanteth the true man: danger and diversion. Therefore wanteth he woman, as the most dangerous plaything.\n",
"Everything in woman is a riddle, and everything in woman hath one solution—it is called pregnancy.\n",
"“He who seeketh may easily get lost himself. All isolation is wrong”: so say the herd. And long didst thou belong to the herd.\n",
"There are the spiritually consumptive ones: hardly are they born when they begin to die, and long for doctrines of lassitude and renunciation.\n",
"What is heavy? so asketh the load-bearing spirit; then kneeleth it down like the\n",
"how the spirit becometh a camel, the camel a lion, and the lion at last a child.\n",
"one must still have chaos in one, to give birth to a dancing star.\n",
"Man is a rope stretched between the animal and the Superman—a rope over an abyss.\n",
"Altered is Zarathustra; a child hath Zarathustra become; an awakened one is Zarathustra: what wilt thou do in the land of the sleepers?\n",
"“The Greeks are interesting and extremely important because they reared such a vast number of great individuals.\n",
"“How can one praise and glorify a nation as a whole?—Even among the Greeks, it was the INDIVIDUALS that counted.”\n",
"From this Anaximander concluded that human beings arose from other animals with more self-reliant newborns: He proposed the spontaneous origin of life in mud, the first animals being fish covered with spines. Some descendants of these fishes eventually abandoned the water and moved to dry land, where they evolved into other animals by the transmutation of one form into another.\n",
"As a little psychological nudge to help you build your daily review habit, we've built a streak counter. You maintain your streak by reviewing your daily highlights at least once per day (within the 24 hour window of your email send time). Check out the leaderboard here: [Top Readwise Streaks](https://readwise.io/highscores)\n",
"Support for highlights from PDFs is one of our most requested features. Unfortunately, PDFs are a super fickle file format and there's no great existing solution that we know of for highlighting and exporting from PDF. That said, we intend to solve the PDF highlighting problem soon once we develop the Readwise native mobile app.\n",
"Highlights from PDFs\n",
"Struggling to come up with a gift idea for the voracious reader in your life? Give him or her the gift of Readwise by clicking [Gift Readwise](https://readwise.io/gift) in the footer.\n",
"Gift Readwise\n",
"Kindle Highlight Limits\n",
"We're often asked: What happens if I delete a highlight in Readwise? Does it disappear in Kindle? Conversely, we're also asked: What happens if I delete a highlight in Kindle? Does it disappear in Readwise? In both cases, the answer is no. If you delete a highlight in Kindle, it will be preserved in Readwise. If you delete a highlight in Readwise, it will be preserved in Kindle.\n",
"Deleting Highlights in Kindle\n",
"Many modern note-taking apps such as [Notion](https://notion.so) (our favorite!) accept [Markdown](https://en.wikipedia.org/wiki/Markdown) as their preferred import format of choice. Accordingly, Readwise enables you to export to Markdown all your notes and highlights on a book-by-basis for use elsewhere. Simply head to the [Books](https://readwise.io/library) or [Articles](https://readwise.io/articles) menu from the [Readwise Dashboard](https://readwise.io/dashboard) and click the down arrow.\n",
"Export to Markdown\n",
"Deleting Books\n",
"If you're an Evernote lover, you can use Readwise to automatically (and continuously) export all your highlights from all sources to Evernote. Simply select the Evernote Export option from the [Readwise Dashboard](https://readwise.io/dashboard) and follow the instructions from there. Note: Exporting to Evernote is a premium feature.\n",
"Export to Evernote\n",
"Concatenation Tags (Combining Highlights)\n",
"Heading Tags (Creating a Table of Contents)\n",
"Inline Tagging\n",
"Terms (Mastery)\n",
"Question & Answer Tag\n",
"Question & Answer (Mastery)\n",
"Cloze Deletion (Mastery)\n",
"Active Recall (Mastery)\n",
"Spaced Repetition Feedback Buttons\n",
"Spaced Repetition Basics\n",
"Why Retention?\n",
"Do you sometimes come across a profound highlight in your Daily Readwise that makes you wish that book were resurfaced more? Good news! You can instruct Readwise to \"Show this book more often\" by clicking the down arrow in the upper right corner of the web app (you can also use the keyboard shortcut: +). Going forward, the probability this book is resurfaced will be increased. You can tell Readwise to \"Show this book less often\" (keyboard shortcut: -) or even \"Never show this book again\" too.\n",
"In-Line Book Tuning\n",
"Book Tuning\n",
"Readwise Resurfacing Algorithm\n",
"Scroll Mode versus Review Mode\n",
"Advanced Usage\n",
"Bonus Highlight\n",
"Readwise is built for power readers so the web app has been designed for power users. This means, of course, that we have all kinds of keyboard shortcuts. There's not enough room to list them all here, but almost all shortcuts are discoverable if you hover over an action in the web app.\n",
"Using Keyboard Shortcuts in the Web App\n",
"Processing Highlights\n",
"Many times a highlight resurfaced in Readwise will not be enough. You'll want to return to the highlight in the context of the book or article. You can automatically launch the Kindle app (assuming it's installed) and open the book to the appropriate location by clicking the down arrow in the upper right of each highlight and selecting \"Open this book in Kindle\". Note: Due technical limitations, this only works on desktop (not mobile) and only from the web app (not email).\n",
"Opening in Kindle App\n",
"Want to quickly copy and paste a highlight for use elsewhere? Perhaps in an article you're writing, a Slack discussion, or a Twitter debate? Simply click the down arrow in the upper right of each highlight and select \"Copy highlight text\". Pro tip: You can also use the keyboard shortcuts cc and cx to copy a highlight to the clipboard.\n",
"Copying Highlight to Clipboard\n",
"Sharing to Twitter or Facebook\n",
"Basic Features\n",
"Disabling Books\n",
"Highlight Recency\n",
"If you've added [Paper or Audiobooks](https://readwise.io/quick) to your Readwise shelf, you'll typically receive one highlight from these books per Daily Readwise. If this is not ideal for you, you can easily increase or decrease the number of Paper and Audiobook highlights by selecting the [Preferences](https://readwise.io/preferences) menu from the [Readwise Dashboard](https://readwise.io/dashboard) and toggling the Physical/Audiobooks option.\n",
"Change the Number of Paper or Audiobook Highlights\n",
"By default, you're set to receive five (5) of your highlights per Daily Readwise. For many users, this is the optimal setting. If it's not ideal for you, you can easily increase or decrease the number of highlights by selecting the [Preferences](https://readwise.io/preferences) menu from the [Readwise Dashboard](https://readwise.io/dashboard) and toggling the Highlights Per Day option.\n",
"Change the Number of Daily Readwise Highlights\n",
"By default, you're set to receive your Daily Readwise email in the morning at 8:00 AM Eastern Time. For many users, this might be perfect. If you're in a different time zone, or if Readwise fits better into a different part of your daily routine, you can easily change this send time by selecting the [Preferences](https://readwise.io/preferences) menu from the [Readwise Dashboard](https://readwise.io/dashboard) and toggling the Email Send Time option.\n",
"Customize Email Send Time\n",
"All of us go through periods of extreme busyness from time to time. During those times, it can be a little overwhelming to watch emails pile up in your inbox. To account for this, Readwise will automatically downgrade your frequency if you're unable to read the emails. First from every day to every other day, then from every other day to every week. It works the same in reverse. Once you start reading your emails again, your frequency will be automatically upgraded.\n",
"Automatic Email Frequency Adjustment\n",
"By default, you'll receive a Daily Readwise email once per day. You can change this preference to be every other day, weekly, or never by selecting [Preferences](https://readwise.io/preferences) menu from the [Readwise Dashboard](https://readwise.io/dashboard) and toggling the Email Frequency option.\n",
"Change Email Frequency\n",
"Customizing Readwise\n",
"If you have a bunch of highlights from a random source, such as an obscure reading or note-taking app, and that source lets you export your annotations in CSV format, you can easily import those into your Readwise account in one fell swoop. You can do this by selecting [Bulk Import](https://readwise.io/import_bulk) from the [Add Highlights](https://readwise.io/sync) menu and following the instructions from there to properly format your file.\n",
"Readwise isn't limited to just books. You can also use it with highlights from articles. There are myriad apps out there, but one of the most popular is Pocket. To connect your Pocket account select the [Add Highlights](https://readwise.io/sync) menu from the [Readwise Dashboard](https://readwise.io/dashboard) and click on the [Pocket](https://readwise.io/pocket) link.\n",
"My Clippings.txt\n",
"Paper and Audiobooks\n",
"Even though Amazon Kindle commands a staggering share of the ebook market (some reports suggest north of 90%!), Readwise also supports highlights from Apple iBooks. You can synchronize your iBooks highlights by selecting the [Add Highlights](https://readwise.io/sync) menu from the [Readwise Dashboard](https://readwise.io/dashboard) and clicking on the [iBooks](https://readwise.io/ibooks) link. You'll need to follow the instructions from there.\n",
"And this is the whole basis of meditation: control thought. But he doesn’t ask who the controller is. The controller is still thought\n",
"the observer is the observed\n",
"There is knowledge of the present only when there is a complete understanding of what the structure and the nature of the past is—and ending it.\n",
"So the question arises: Is the mind mechanical? That is, in your mind, are your thoughts, your feelings, your reactions, your responsibilities, your relationships, your ways, your opinions, and so on, merely mechanical; that is, responding according to conditioning, according to environmental influence? If that is the totality of the mind, then we live in a tremendous, inescapable prison\n",
"Then, through negation of what is false—not through resistance or reaction to the false—through choiceless rejection of what is false, you have a different kind of energy\n",
"It is only when we understand the self— not according to Shankara, Buddha, or Christ, but as it actually is in each one of us in relation to people, to ideas, and to things—that there is the cessation of pain\n",
"is only when we understand the self— not according to Shankara, Buddha, or Christ, but as it actually is in each one of us in relation to people, to ideas, and to things—that there is the cessation of pain.\n",
"Let us discuss this. Can we as individuals put an end in ourselves to the causes of war? One of the causes is obviously belief, the division of ourselves as Hindus, Buddhists, Christians, communists, or capitalists. Can we put all that aside?\n",
"Let us discuss this. Can we as individuals put an end in ourselves to the causes of war? One of the causes is obviously belief, the division of ourselves as Hindus,\n",
"I see how foolish we are, searching the universe for that bliss which is to be found only in our own hearts when the mind is purged of its activities. You are perfectly right. I begin from where I started. I begin with what I am\n",
"see how foolish we are, searching the universe for that bliss which is to be found only in our own hearts when the mind is purged of its activities. You are perfectly right. I begin from where I started. I begin with what\n",
". At last he wearily returned to his home, and in his own house was the jewel! I see how foolish we are, searching the universe for that bliss which is to be found only in our own hearts when the mind is purged of its activities. You are perfectly right. I begin from where I started. I begin with what I am.\n",
"see how foolish we are, searching the universe for that bliss which is to be found only in our own hearts when the mind is purged of its activities. You are perfectly right. I begin from where I started. I begin with what I am\n",
"“To what extent can a person control his own actions?” A person does not control his own actions if he has not understood environment. Then he is only acting under the compulsion, the influence, of environment; such an action is not action at all, but is merely reaction or self-protectiveness\n",
"When you yourself become both the teacher and the disciple—disciple being a man who is learning, learning, learning, not accumulating knowledge—then you are an extraordinary human being\n",
"Is experience different from the experiencer? Give your brain to this, find out!\n",
"holding, becomes our knowledge. Experience is always limited, naturally. Is experience different from the experiencer? Give your brain to this, find out!\n",
"Is experience different from the experiencer? Give your brain to this, find out! If there is no experiencer\n",
"If you take a journey into yourself, empty all the content that you have collected and go very, very deeply, then there is that vast space, that so-called emptiness, that is full of energy. And in that state alone there is that which is most sacred, most holy.\n",
"Because if you carry all the burdens of yesterday, your brain becomes mechanical, dull. If you leave all the psychological memories, hurts, pains, behind, every day, then it means dying and living are together. In that there is no fear\n",
"What is living, what do you call living? Going to the office from nine o’clock in the morning to six o’clock in the evening every day of your life for the next sixty years, being bossed, being bullied, and you bullying somebody else? Or you are a businessman always wanting more and more money, more power, better position, and then go home and quarrel with your wife, sleep with her, and beat her up verbally, or actually\n",
"You don’t have to make the mind quiet. If you end all conflict, the mind naturally becomes quiet. And when the mind is absolutely silent, without any movement of thought, then perhaps you will see something, perhaps there is something sacred beyond all words. And this man has sought everlastingly, something that is beyond measure, beyond thought, which is incorruptible, unnameable, eternal. That can only take place when the mind is absolutely free and completely silent\n",
"You don’t have to make the mind quiet. If you end all conflict, the mind naturally becomes quiet. And when the mind is absolutely silent, without any movement of thought, then perhaps you will see something, perhaps there is something sacred beyond all words. And this man has sought everlastingly, something that is beyond measure, beyond thought, which is incorruptible, unnameable, eternal\n",
"You don’t have to make the mind quiet. If you end all conflict, the mind naturally becomes quiet. And when the mind is absolutely silent, without any movement of thought, then perhaps you will see something, perhaps there is something sacred beyond all words. And this man has sought everlastingly, something that is beyond measure, beyond thought, which is incorruptible, unnameable, eternal\n",
"The nature of dying can be found in the living. That is, death is the ending, the ending of my possessions, my wife, my children, my house, my bank account\n",
"Meditation then is the emptying of the content of consciousness—which means the fears, the anxieties, the conflicts in relationship—the ending of sorrow and, therefore, compassion. The ending of the content of consciousness is complete silence. Then that silence is full of energy\n",
"Meditation then is the emptying of the content of consciousness—which means the fears, the anxieties, the conflicts in relationship—the ending of sorrow and, therefore, compassion. The ending of the content of consciousness is complete silence. Then that silence is full of energy. It is not vacant silence. It is not a silence that wants something more. So meditation is not the repetition of mantras\n",
"Meditation then is the emptying of the content of consciousness—which means the fears, the anxieties, the conflicts in relationship—the ending of sorrow and, therefore, compassion. The ending of the content of consciousness is complete silence. Then that silence is full\n",
"You cannot hear what somebody else is saying if you are talking to yourself all the time\n",
"the logic of it, that you can see something clearly only when your mind is silent. You cannot hear what somebody else is saying if you are talking to yourself all the time\n",
"Simplicity is hacking away the unessential. - It is not daily increase but daily decrease - hack away the unessential! The closer to the source, the less wastage there is\n",
"Maturity vs. maturing. -There is no such word as \"maturity.\" Rather: maturing. Because when there is a maturity, there is a conclusion and a cessation. That's \"the end.\" That's when the coffin is closed\n",
"Zen reveals that there is no problem-and no solution. - Zen reveals that there is no where for man to go out of this world; no tavern in which he can overcome anxiety; no jail in which he can expiate guilt. So, instead of telling us what the problem is, Zen insists that the whole trouble is just our failure to realise that there is no problem. And, of course, this means that there is no solution, either\n",
"but if you look at the people, they all borrow concepts from other sources. Buber from Judaism, Tillich from Protestantism, Sartre from Socialism, Heidegger from language, Binswanger from psychoanalysis\n",
"Only eat what your body requires, and don't [become] carried away with foods that don't benefit you\n",
"To understand your fear is the beginning of really seeing\n",
"- In order to achieve happiness or the right conduct of life, one must acquire knowledge - so he can think, reason, and create\n",
"Acceptance of death. - The round of summer and winter becomes a blessing the moment we give up the fantasy of eternal spring\n",
"A teacher is never a giver of truth; he is a guide, a pointer to the truth that each student must find for himself. A good teacher is merely a catalyst.\n",
"Tissues that are especially prone to cancer tend to be tissues that do a lot of cell division throughout life either for repair or for other reasons: skin, testis, breast, colon, stomach, white blood cells\n",
"Natural selection has designed all parts of our bodies to last just long enough to see our children into independence, no more\n",
"Genes are just chunks of software that can run on any system: they use the same code and do the same\n",
"mutations, bred them by the thousand and slowly sorted out all the ones\n",
"The psychological precedes the physical. The mind drives the body, which drives the genome\n",
"In a massive, long-term study of 17,000 civil servants, an almost unbelievable conclusion emerged: the status of a person's job was more able to predict their likelihood of a heart attack than obesity, smoking or high blood pressure\n",
"the genome is a book that wrote itself\n",
"blend is because it involves more than one particle. In the early nineteenth\n",
"Heredity is a modifiable stored program; metabolism is a universal machine. The recipe that links them is a code, an abstract message that can be embodied in a chemical, physical or even immaterial form\n",
"Heredity is a modifiable stored program; metabolism is a universal machine\n",
"They do not defy the second law of thermodynamics, which says that in a closed system everything tends from order towards disorder\n",
"Many mutations are neither harmful nor beneficial, for instance if they change one codon to another that has the same amino acid 'meaning': there are sixty-four different codons and only twenty amino acids, so many D N A 'words' share the same meaning. Human beings\n",
"When genes are replicated, mistakes are sometimes made. A letter (base) is occasionally missed out or the wrong letter inserted. Whole sentences or paragraphs are sometimes duplicated, omitted or\n",
"Rent Seeking: trying to use protective regulations or “rights” to derive income without adding anything to economic activity, without increasing the wealth of others. As Fat Tony would define it, it is like being forced to pay protection money to the Mafia without getting the economic benefits of protection.\n",
"skin in the game falls at the convergence point of three main ethical systems: Kantian imperatives, consequentialism, and classical virtue.\n",
"When the beard (or hair) is black, heed the reasoning, but ignore the conclusion. When the beard is gray, consider both reasoning and conclusion. When the beard is white, skip the reasoning, but mind the conclusion.\n",
"We close this chapter with a few summarizing lines. One may be risk loving yet completely averse to ruin. The central asymmetry of life is: In a strategy that entails ruin, benefits never offset risks of ruin. Further: Ruin and other changes in condition are different animals. Every single risk you take adds up to reduce your life expectancy. Finally: Rationality is avoidance of systemic ruin.\n",
"“The difference between successful people and really successful people is that really successful people say no to almost everything\n",
"Rationality is risk management, period.\n",
"When you consider beliefs in evolutionary terms, do not look at how they compete with each other, but consider the survival of the populations that have them.\n",
"Herb Simon, who pioneered artificial intelligence; the psychologist Gerd Gigerenzer; and the mathematician, logician, and decision theorist Ken Binmore, who spent his life formulating the logical foundations of rationality.\n",
"This chapter will ease us to the next section: a) rationality resides in what you do, not in what you think or in what you “believe” (skin in the game), and b) rationality is about survival.\n",
"Finally, when young people who “want to help mankind” come to me asking, “What should I do? I want to reduce poverty, save the world,” and similar noble aspirations at the macro-level, my suggestion is: 1) Never engage in virtue signaling; 2) Never engage in rent-seeking; 3) You must start a business. Put yourself on the line, start a business. Yes, take risk, and if you get rich (which is optional), spend your money generously on others.\n",
"If your private actions do not generalize, then you cannot have general ideas.\n",
"If your private life conflicts with your intellectual opinion, it cancels your intellectual ideas, not your private life.\n",
"Journalism isn’t Lindy compatible. Information transmits organically by word of mouth, which circulates in a two-way manner.\n",
"The best enemy is the one you own by putting skin in his game and letting him know the exact rules that come with it. You keep him alive, with the knowledge that he owes his life to your benevolence. The notion that an enemy you own is better than a dead one was perfected by the order of the Assassins, so we will do some digging into the work of that secret society.\n",
"To put it another way: if wealth is giving you fewer options instead of more (and more varied) options, you’re doing it wrong.\n",
"The idea of the Lindy effect is itself Lindy-proof. The pre-Socratic thinker Periander of Corinth wrote, more than twenty-five hundred years ago: Use laws that are old but food that is fresh. Likewise, Alfonso X of Spain, nicknamed El Sabio, “the wise,” had as a maxim: Burn old logs. Drink old wine. Read old books. Keep old friends.\n",
"Lindy is a deli in New York, now a tourist trap, that proudly claims to be famous for its cheesecake, but in fact has been known for fifty or so years by physicists and mathematicians thanks to the heuristic that developed there. Actors who hung out there gossiping about other actors discovered that Broadway shows that lasted for, say, one hundred days, had a future life expectancy of a hundred more. For those that lasted two hundred days, two hundred more. The heuristic became known as the Lindy effect.\n",
"Before we end, take some Fat Tony wisdom: always do more than you talk. And precede talk with action. For it will always remain that action without talk supersedes talk without action.\n",
"scars, and character flaws increase the distance between a human and a ghost.fn2 Scars signal skin in the game.\n",
"The reason a dream is not reality is that when you suddenly wake up from falling from a Chinese skyscraper, life continues, and there is no absorbing barrier, the mathematical name for that irreversible state that we will discuss at length in Chapter 19, along with ergodicity, the most powerful concept I know.\n",
"If you do not undertake a risk of real harm, reparable or even potentially irreparable, from an adventure, it is not an adventure.\n",
"slavery in modern life: why employees exist because they have much more to lose than contractors.\n",
"the best pieces of advice I have ever received was the recommendation by a very successful (and happy) older entrepreneur, Yossi Vardi, to have no assistant.\n",
"How is it that we have more slaves today than we did during Roman times?\n",
"I submit that the world would be much happier, if men were as fully able to keep silence as they are to speak.\n",
"Proof.—If this be denied, conceive, if possible, that God does not exist: then his essence does not involve existence. But this (Prop. vii.) is absurd. Therefore God necessarily exists.\n",
"A man who can understand Buddha and has an intuition of the heaven and hell of humanity ought not to live in a world ruled by \"common sense\" and democracy and bourgeois standards. It is only from cowardice that he lives in it\n",
"His tendency is to explain Mozart's perfected being, just as a schoolmaster would, as a supreme and special gift rather than as the outcome of his immense powers of surrender and suffering, of his indifference to the ideals of the bourgeois, and of his patience under that last extremity of loneliness which rarefies the atmosphere of the bourgeois world to an ice-cold ether, around those who suffer to become men, that loneliness of the Garden of Gethsemane.\n",
"However, the inequality here was seen as the greatest in human history: inequality before death.\n",
"“I believe, not because I have any proof, but because it’s relatively safe: If there really is a God, then it’s right to believe in him. If there isn’t, then we don’t have anything to lose.”\n",
"“True. One of the biggest problems in artificial intelligence is identifying facial and eye expressions. Some experts even say that computers may never be able to read the eyes.”\n",
"“I get what you mean. Her smile is something that the sophons and the Trisolarans will never understand.” “That’s right. Human expressions, and people’s eyes in particular, are subtle and complex. A gaze or a smile can transmit so much information! And only humans can understand that information. Only humans have that sensitivity.”\n",
"Diferential privacy addresses the paradox of learning nothing about an individual while learning useful information about a population\n",
"curious neat language. Be neither a great talker, nor a great undertaker. Moreover, let\n",
"memory of him that begot me I have learned both shamefastness\n",
"There is a better chance of seeing a camel pass through the eye of a needle than of seeing a really great man ‘discovered’ through an election. 79\n",
"If by ‘free will’ you mean the freedom to do what you desire – then yes, humans have free will. But if by ‘free will’ you mean the freedom to choose what to desire – then no, humans have no free will.\n",
"But both the ‘self’ and freedom are mythological chimeras borrowed from the fairy tales of ancient times.\n",
"Have you seen those zombies who roam the streets with their faces glued to their smartphones?\n",
"In the past, it was a relatively safe bet to follow the adults, because they knew the world quite well, and the world changed slowly. But the twenty-first century is going to be different. Due to the growing pace of change you can never be certain whether what the adults are telling you is timeless wisdom or outdated bias.\n",
"When a thousand people believe some made-up story for one month – that’s fake news. When a billion people believe it for a thousand years – that’s a religion,\n",
"25,000 people globally (mostly in Iraq, Afghanistan, Pakistan, Nigeria and Syria).1 In contrast, each year traffic accidents kill about 80,000 Europeans, 40,000 Americans, 270,000 Chinese, and 1.25 million people altogether.2 Diabetes and high sugar levels kill up to 3.5 million people annually, while air pollution kills about 7 million\n",
"Karl Marx, Milton Friedman, Friedrich Hayek\n",
"digital dictatorships to the creation of a global useless class.\n",
"There are no essential links between consciousness and either organic biochemistry or high intelligence. Hence computers might develop consciousness – but not necessarily. They could become super-intelligent while still having zero consciousness.\n",
"Consciousness is somehow linked to organic biochemistry in such a way that it will never be possible to create consciousness in non-organic systems. Consciousness is not linked to organic biochemistry, but it is linked to intelligence in such a way that computers could develop consciousness, and computers will have to develop consciousness if they are to pass a certain threshold of intelligence.\n",
"Intelligence is the ability to solve problems. Consciousness is the ability to feel things such as pain, joy, love and anger.\n",
"the entire universe as a flow of data, see organisms as little more than biochemical algorithms, and believe that humanity’s cosmic vocation is to create an all-encompassing data-processing system\n",
"democracy were a matter of rational decision-making, there would be absolutely no reason to give all people equal voting rights – or perhaps any voting rights. There is ample evidence that some people are far more knowledgeable and rational than others, certainly when it comes to specific economic and political questions.\n",
"Referendums and elections are always about human feelings, not about human rationality.\n",
"free education, free healthcare, free transport and so forth. This is in fact the utopian vision of communism. Though the communist plan to start a working-class revolution might well become outdated, maybe we should still aim to realise the communist goal by other means?\n",
"My thanks for the loftiest spiritual elevation must go to the works of Goethe (his Faust is perhaps the greatest ofall the products ofart and from it I took the mottoes of my first books); likewise to the works of Shakespeare, and so\n",
"The two greatest philosophical points of view (both discovered by Germans). (a) That of becoming and that of evolution. (b) That based upon the values of existence (but the wretched form of German pessimism must first be overcome!)—\n",
"He is always in his own company, whether his intercourse be with books, with men, or with Nature.\n",
"He grows stronger under the misfortunes which threaten to annihilate him.\n",
"We must not let ourselves be seduced by blue eyes and heaving breasts: greatness of soul has absolutely nothing romantic about it. And unfortunately nothing whatever amiable either.\n",
"He knows that he cannot reveal himself to anybody: he thinks it bad taste to become familiar; and as a rule he is not familiar when people think he is. When he is not talking to his soul, he wears a mask. He would rather lie than tell the truth, because lying requires more spirit and will.\n",
"a man should look upon himself with an adventurous spirit; he should experiment with himself and run risks with himself—\n",
"We should substitute morality by the will to our own ends, and consequently to the means to them.\n",
"Closely related thereto is the \"genius.\" The \"great adventurers and criminals\" and all great men, the most healthy in particular, have always been sick at certain periods of their lives—\n",
"There is no relationship between work done and money received; the individual should, according to his kind, be so placed as to perform the highest that is compatible with his powers.\n",
"(in this connection let us recall Savonarola's judgment of Florence, Plato's indictment of Athens under Pericles, Luther's condemnation of Rome, Rousseau's anathemas against the society of Voltaire, and Germany's hostility to Goethe).\n",
"all great men have been criminals\n",
"Man has one terrible and fundamental wish; he desires power, and this impulse, which is called freedom, must be the longest restrained. Hence ethics has instinctively aimed at such an education as shall restrain the desire for power; thus our morality slanders the would-be tyrant, and glorifies charity, patriotism, and the ambition of the herd.\n",
"The fundamental phenomena: innumerable individuals are sacrificed for the sake of a few, in order to make the few possible.—\n",
"Here is the root of perspectivity, and it explains why a living organism is \"egoistic\" to the core.\n",
"Perspectivity is only a complex form, of specificness. My idea is that every specific body strives to become master of all space, and to extend its power (its will to power), and to thrust back everything that resists it. But inasmuch as it is continually meeting the same endeavours on the part of other bodies, it concludes by coming to terms with those (by \"combining\" with those) which are sufficiently related to it—and thus they conspire together for power. And the process continues.\n",
"First principles, bravery, patience, no \"stepping-back,\" not too much ardour to get to the fore.\n",
"All the idealism of mankind, hitherto, is on the point of turning into Nihilism—may be shown to be a belief in absolute worthlessness, i.e. purposelessness.\n",
"The uselessness of the mechanical theory—it gives the impression that there can be no purpose.\n",
"The uselessness of old ideals for the interpretation of all that takes place, once their bestial origin and utility have been recognised, they are, moreover, all hostile to life.\n",
"Becoming is inventing, willing, self-denying, self-overcoming; no subject but an action, it places things, it is creative, no \"causes and effects.\"\n",
"That everything recurs, is the very nearest approach of a world of Becoming to a world of Being, the height of contemplation.\n",
"To stamp Becoming with the character of Being—this is the highest Will to Power.\n",
"it is merely a vital condition under which, alone, a living organism can preserve itself and prosper: a great solid belt of ignorance must stand about you.\n",
"Wisdom is an attempt to overcome the perspective valuations (i.e. the \"will to power\"): it is a principle which is both unfriendly to Life, and also decadent; a symptom in the case of the Indians, etc.; weakness of the power of appropriation.\n",
"The ascertaining of \"truth\" and \"untruth,\" the ascertaining of facts in general, is fundamentally different from the creative placing, forming, moulding, subduing, and willing which lies at the root of philosophy.\n",
"philosophy, religion, and morality are symptoms of decadence.\n",
"The logical denial of the world and Nihilism is a consequence of the fact that we must oppose nonentity with Being, and that Becoming is denied.\n",
"Brave and creative men never make pleasure and pain ultimate questions—they\n",
"if A exists, then its opposite B must also exist).\n",
"That is to say; there is nothing without other things. That is to say; there is no \"thing-in-itself.\"\n",
"Knowing means: \"to place one's self in relation with something,\"\n",
"as soon as a superior power prevails over an inferior power, and the latter proceeds to work as a function of the former, an order of rank is established,\n",
"To combat determinism and teleology.—From the fact that something happens regularly, and that its occurrence may be reckoned upon, it does not follow that it happens necessarily. If a quantity of force determines and conducts itself in a certain way in every particular case, it does not prove that it has \"no free will.\"\n",
"The so-called instinct of causality is nothing more than the fear of the unfamiliar, and the attempt at finding something in it which is already known.—It is not a search for causes, but for the familiar.\n",
"There is no such thing as a cause or an effect.\n",
"We distinguish ourselves, the agents, from the action, and everywhere we make use of this scheme—we try to discover an agent behind every phenomenon.\n",
"\"Une croyance presque instinctive chez moi c'est que tout homme puissant ment quand il parle et à plus forte raison quand il écrit.\"\n",
"There are many kinds of eyes. Even the Sphinx has eyes—therefore there must be many kinds of \"truths,\" and consequently there can be no truth.\n",
"The teaching of Being, of things, and of all those constant entities, is a hundred times more easy than the teaching of Becoming and of evolution.\n",
"All human knowledge is either experience or mathematics.\n",
"The whole apparatus of knowledge is an abstracting and simplifying apparatus—\n",
"The metaphysico-logical postulates, the belief in substance, accident, attribute, etc. etc., draws its convincing character from our habit of regarding all our actions as the result of our will: so that the ego, as substance, does not vanish in the multiplicity of changes.—But there is no such thing as will\n",
"There are no such things as \"mind,\" reason, thought, consciousness, soul, will, or truth: they all belong to fiction,\n",
"The most valuable knowledge is always discovered last: but the most valuable knowledge consists of methods.\n",
"The great Methodologists: Aristotle, Bacon, Descartes, Auguste Comte.\n",
"to hear the inferior and the poor in spirit having their say is a terrible ear-splitting torment for him who knows and trembles at the thought,\n",
"The more lofty philosophical man who is surrounded by loneliness, not because he wishes to be alone, but because he is what he is, and cannot find his equal...\n",
"\"Prava corrigere, et recta corroborare, et sancta sublimare.\"\n",
"a capacity for shutting out the world; but above all, a much lower degree of sensitiveness than the average man has, who is too easily infected with the views of others.\n",
"For a personality to be possible, timely isolation and the necessity for an existence of offence and defence, are prerequisites;\n",
"Art is the only superior counter-agent to all will to the denial of life; it is par excellence the anti-Christian, the anti-Buddhistic, the anti-Nihilistic force.\n",
"it is far better to devote yourself to a few authors than to get lost among many\n",
"Still, you must especially avoid those who are gloomy and aways lamentg, and who grasp at every pretext for complaint. Though a man's loyalty and kdness may not be in doubt, a companion who is agtated and groaning about everg is an enemy to peace of mind\n",
"But life is very short and anxious for those who forget the past, neglect the present, and fear the fture\n",
"Of all people only those are at leisure who make time for phiosophy, only those are really alive.\n",
"Of all people only those are at leisure who make time for phiosophy, only those are really alive. For they not ony keep a good watch over their own\n",
"entangled in these. / Of all people only those are at leisure who make time for phiosophy, only those are really alive\n",
"Of all people only those are at leisure who\n",
"entangled in these. / Of all people only those are at leisure who make time for phiosophy, only those are really\n",
"The following example creates an alert when the error ratio over 10 minutes exceeds 1% and the total number of errors exceeds 1: rules <<< {var=dc:http_errors:ratio_rate10m,job=webserver} > 0.01 and by job, error {var=dc:http_errors:rate10m,job=webserver} > 1 for 2m => ErrorRatioTooHigh details \"webserver error ratio at [[trigger_value]]\" labels {severity=page}; >>> Our\n",
"and the total number of errors exceeds 1: rules <<< {var=dc:http_errors:ratio_rate10m,job=webserver} > 0.01 and by job, error {var=dc:http_errors:rate10m\n",
"rule evaluation cycles to ensure no missed collections cause a false alert. The following example creates an alert when the error ratio over 10 minutes exceeds 1% and the total number of errors exceeds 1: rules <<< {var=dc:http_errors:ratio_rate10m,job=webserver} > 0.01 and by job, error {var=dc:http_errors:rate10m,job=webserver} > 1 for 2m => ErrorRatioTooHigh details \"webserver error ratio at [[trigger_value]]\" labels {severity\n",
"following example creates an alert when the error ratio over 10 minutes exceeds 1% and the total number of errors exceeds 1: rules <<< {var=dc:http_errors:ratio_rate10m,job=webserver} > 0.01 and by job, error {var=dc:http_errors:rate10m,job=webserver} > 1 for 2m => ErrorRatioTooHigh details \"webserver error ratio at [[trigger_value]]\" labels {severity\n",
"Record the current CPU utilization each second. 2. Using buckets of 5% granularity, increment the appropriate CPU utilization bucket each second. 3. Aggregate those values every minute. This strategy allows you to observe brief CPU hotspots without incurring very high cost due to collection and retention\n",
"Worrying About Your Tail (or, Instrumentation and Performance\n",
"key principle of any effective software engineering, not only reliability-oriented engineering, simplicity is a quality that, once lost, can be extraordinarily difficult to recapture. Nevertheless, as the old adage goes, a complex system that works necessar‐ ily evolved from a simple system that works. Chapter 9, Simplicity, goes into this topic in detail.\n",
"key principle of any effective software engineering, not only reliability-oriented engineering, simplicity is a quality that, once lost, can be extraordinarily difficult to recapture. Nevertheless, as the old adage goes, a complex system that works necessar‐ ily evolved from\n",
"• Set up alerting for acute problems. • Compare behavior: did a software update make the server faster? • Examine how resource consumption behavior evolves over time, which is essen‐ tial for capacity planning\n",
"up alerting for acute problems. • Compare behavior: did a software update make the server faster? • Examine how resource consumption behavior evolves over time, which is essen‐ tial for capacity planning\n",
"What level of availability will the users be happy with, given how they use the product? • What alternatives are available to users who are dissatisfied with the product’s availability? • What happens to users’ usage of the product at different availability levels?\n",
"SREs should receive a maxi‐ mum of two events per 8–12-hour on-call shift. This target volume gives the on-call engineer enough time to handle the event accurately and quickly, clean up and restore normal service, and then conduct a postmortem\n",
"By far, UNIX system internals and networking (Layer 1 to Layer 3) expertise are the two most common types of alternate technical skills we seek\n",
"It is an iron rule documented in virtually all foraging people that ‘men hunt, women and children gather\n",
"The more people are drawn into the global division of labour, the more people can specialise and exchange, the wealthier we will all be\n",
"The more people are drawn into the global division of labour, the more people can specialise and exchange, the wealthier we will all\n",
"The more people are drawn into the global division of labour, the more people can specialise and exchange, the wealthier we will all be. Moreover, along the way there is no reason we cannot solve the\n",
"Friedrich Hayek\n",
"There are only two types of people in the world. People who want to live comfortably—they are seeking death, they want a comfortable grave. And people who want to live—they choose to live dangerously, because life thrives only when there is risk.\n",
"Living never flowers in security; it flowers only in insecurity\n",
"To live dangerously means to live\n",
"All meditative techniques are a help to destroy the false\n",
"Habits die hard. So many years of living in a false personality imposed by people whom you loved, whom you respected … and they were not intentionally doing anything bad to you\n",
"Habits die hard. So many years of living in a false personality imposed by people whom you loved, whom you respected … and they were not intentionally doing anything bad to\n",
"Everybody has to pass through a dark night before he reaches the sunrise\n",
"Life is not a problem. To look at it as a problem is to take a wrong step. It is a mystery to be lived, loved, experienced\n",
"Meditation takes you into the unknown, the uncharted. Meditation takes you slowly, slowly into a kind of dissolution where the observer and the observed become one\n",
"TRUTH IS AN EXPERIENCE, NOT A BELIEF. Truth never comes by studying about it; truth has to be encountered, truth has to be faced. The person who studies about love is like the person who studies about the Himalayas by looking at the map of the mountains. The map is not the mountain! And if you start believing in the map, you will go on missing the mountain. If you become too much obsessed with the map, the mountain may be there just in front of you, but still you will not be able to see it.\n",
"And remember, I am not guaranteeing you that it will always lead you to the right. Many times it will take you to the wrong, because to come to the right door one has to knock first on many wrong doors\n",
"It is your fear that makes you a slave—it is your fear\n",
"There are two possibilities. Either you close your eyes and become dogmatic, become a Christian or a Hindu or a Mohammedan … then you become like an ostrich. It doesn’t change life; it simply closes your eyes. It simply makes you stupid, it simply makes you unintelligent. In your unintelligence you feel secure—all idiots feel secure. In fact, only idiots feel secure. A really alive man will always feel insecure. What security can there be?\n",
"The language lawyer can 6nd a neat and efficient way to use the language to do difficult, obscure, or tricky things. Often he will need to do small studies (two or three days) on good technique. One language lawyer can service two or three surgeons\n",
"The tester. The surgeon will need a bank of suitable test cases for testing pieces of his work as he writes it, and then for testing the whole thing. The tester is therefore both an adversary who devises system test cases from the functional specs, and an assistant who devises test data for the day-by-day debugging. He would also plan testing sequences and set up the scaffolding required for component tests\n",
"The program clerk. He is responsible for maintaining all the technical records of the team in a programming-product library. The clerk is trained as a secretary and has responsibility for both machine-readable and human-readable files\n",
"Two secretaries. The administrator and the editor will each need a secretary; the administrator's secretary will handle project correspondence and non-product files\n",
"The editor. The surgeon is responsible for generating the documentation—for maximum clarity he must write it. This is true of both external and internal descriptions. The editor, however, takes the draft or dictated manuscript produced by the surgeon and criticizes it, reworks it, provides it with references and bibliography, nurses it through several versions, and oversees the mechanics of production\n",
"The surgeon. Mills calls him a chief programmer. He personally defines the functional and performance specifications, designs the program, codes it, tests it, and writes its documentation\n",
"—the essential difference between the two religions of décadence: Buddhism promises nothing, but actually fulfils; Christianity promises everything, but fulfils nothing.—\n",
"to be or not to be,\n",
"In Buddha’s teaching egoism is a duty.\n",
"He understands good, the state of goodness, as something which promotes health.\n",
"Buddhism is the only genuinely positive religion to be encountered in history,\n",
"The old word “will” now connotes only a sort of result, an individual reaction, that follows inevitably upon a series of partly discordant and partly harmonious stimuli—the will no longer “acts,” or “moves.”...\n",
"What is good?—Whatever augments the feeling of power, the will to power, power itself, in man. What is evil?—Whatever springs from weakness. What is happiness?—The feeling that power increases—that resistance is overcome.\n",
", ils semblent avoir d'autant moins à craindre de se mesconter. Toutefois encore cecy, par ce que c'est un corps vain, et sans prise, eschappe volontiers à la memoire, si elle n'est bien asseuree. Dequoy j'ay souvent\n",
"And as we gaze at circles, they gaze back at us, literally\n",
"Thus, calculus proceeds in two phases: cutting and rebuilding\n",
"In every case the strategy remains the same: split a\n",
"\"Alpha children wear grey They work much harder than we do, because they're so frightfully clever. I'm really awfuly glad I'm a Beta, because I don't work so hard. And then we are much better than the Gammas and Deltas. Gammas are stupid. They all wear green, and Delta children wear khaki. Oh no, I don't want to play with Delta children. And Epsilons are still worse. They're too stupid to be able\n",
"All the air of the fourteenth floor was sibilant with the categorical imperative\n",
"And that,\" put in the Director sententiously, \"that is the secret of happiness and virtue-liking what you've got to do. All conditioning aims at that: making people like their unescapable social destiny.\n",
"\n"
],
"name": "stdout"
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Dataset(features: {'id': Value(dtype='int64', id=None), 'title': Value(dtype='string', id=None), 'author': Value(dtype='string', id=None), 'category': Value(dtype='string', id=None), 'num_highlights': Value(dtype='int64', id=None), 'last_highlight_at': Value(dtype='string', id=None), 'updated': Value(dtype='string', id=None), 'cover_image_url': Value(dtype='string', id=None), 'highlights_url': Value(dtype='string', id=None), 'quote': Value(dtype='string', id=None), '__index_level_0__': Value(dtype='int64', id=None)}, num_rows: 401)"
]
},
"metadata": {
"tags": []
},
"execution_count": 102
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "Bq0v45axc-hj",
"outputId": "c596045d-23fe-4b68-88b6-31f902d1754a",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 152,
"referenced_widgets": [
"11c280f594cf49769af0f91e9a52021e",
"db6638fe62e348b591b46254f41758b8",
"9fc1aea915704a2a9aa54aebe9c33e16",
"ed75da21096e4bbf844427d3f950599d",
"9eef96beec9941f996e9288cd5100202",
"286d73526d684f08a9d4e5f86f8c8a10",
"6c5778cfcafb46349e942223a9e07be1",
"f3b13d7f087c4775b1416dffa297af3d",
"005e3e0dd60842bf904eaf3f9bb4aae9",
"ed61d0b41b6f412e8603fa5769daf714",
"46f5f68619d14560865dcb08e4d2e052",
"49b404d8096c411387789540377f92aa",
"7bcbc73eddb044f6bae964390b786488",
"2db64961119549d3ba6a9d0808e19b38",
"4aa17fabf3e04c58a5d618710a205866",
"c353c18d317448e782b923bbdf95f731"
]
}
},
"source": [
"from transformers import DPRContextEncoder, DPRContextEncoderTokenizer\n",
"import torch\n",
"torch.set_grad_enabled(False)\n",
"ctx_encoder = DPRContextEncoder.from_pretrained(\"facebook/dpr-ctx_encoder-single-nq-base\")\n",
"ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained(\"facebook/dpr-ctx_encoder-single-nq-base\")\n",
"dataset_with_embeddings = dataset.map(lambda quote: {'embeddings': ctx_encoder(**ctx_tokenizer(quote[\"quote\"], return_tensors=\"pt\"))[0][0].numpy()})\n",
"dataset_with_embeddings.add_faiss_index(column='embeddings')"
],
"execution_count": 89,
"outputs": [
{
"output_type": "display_data",
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "11c280f594cf49769af0f91e9a52021e",
"version_minor": 0,
"version_major": 2
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=401.0), HTML(value='')))"
]
},
"metadata": {
"tags": []
}
},
{
"output_type": "stream",
"text": [
"\n"
],
"name": "stdout"
},
{
"output_type": "display_data",
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "005e3e0dd60842bf904eaf3f9bb4aae9",
"version_minor": 0,
"version_major": 2
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
]
},
"metadata": {
"tags": []
}
},
{
"output_type": "stream",
"text": [
"\n"
],
"name": "stdout"
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Dataset(features: {'__index_level_0__': Value(dtype='int64', id=None), 'author': Value(dtype='string', id=None), 'category': Value(dtype='string', id=None), 'cover_image_url': Value(dtype='string', id=None), 'embeddings': Sequence(feature=Value(dtype='float64', id=None), length=-1, id=None), 'highlights_url': Value(dtype='string', id=None), 'id': Value(dtype='int64', id=None), 'last_highlight_at': Value(dtype='string', id=None), 'num_highlights': Value(dtype='int64', id=None), 'quote': Value(dtype='string', id=None), 'title': Value(dtype='string', id=None), 'updated': Value(dtype='string', id=None)}, num_rows: 401)"
]
},
"metadata": {
"tags": []
},
"execution_count": 89
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "7cd-3cliksXd",
"outputId": "82efc91e-d626-4aa2-d934-10fb3f5e0144",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"source": [
"def nearest_quote(quote):\n",
" quote_embedding = ctx_encoder(**ctx_tokenizer(quote, return_tensors=\"pt\"))[0][0].numpy()\n",
" scores, retrieved_examples = dataset_with_embeddings.get_nearest_examples('embeddings', quote_embedding, k=10)\n",
" return retrieved_examples['quote']\n",
"some_quotes = [ # Pick some quotes from https://readwise.io/@louis\n",
" \"\"\"Human nature is a product of culture, but culture is also a product of \\\n",
" human nature, and both are the products of evolution.\"\"\",\n",
" \"\"\"We join spokes together in a wheel, but it is the center hole that makes \\\n",
" the wagon move. We shape clay into a pot, but it is the emptiness inside \\\n",
" that holds whatever we want. We hammer wood for a house, but it is the \\\n",
" inner space that makes it livable. We work with being, but non-being \\\n",
" is what we use.\"\"\",\n",
" \"\"\"If you wish to have leisure for your mind, either be a poor man, \\\n",
" or resemble a poor man.\"\"\",\n",
" \"Those who know don’t talk. Those who talk don’t know.\"\n",
"]\n",
"\n",
"for quote in some_quotes:\n",
" print(\"quote:\", quote)\n",
" print(\"nearest_quotes:\")\n",
" [print(\" - \", n_q) for n_q in nearest_quote(quote)]\n",
" print(\"~~~~~~~~~~~~~\")\n"
],
"execution_count": 101,
"outputs": [
{
"output_type": "stream",
"text": [
"quote: Human nature is a product of culture, but culture is also a product of human nature, and both are the products of evolution.\n",
"nearest_quotes:\n",
" - Darwin's theory of evolution by natural selection is satisfying because it shows us away in which simplicity could change into complexity, how unordered atoms could group themselves into ever more complex patterns until they ended up manufacturing people\n",
" - We biologists have assimilated the idea of genetic evolution so deeply that we tend to forget that it is only one of many possible kinds of evolution\n",
" - Human nature was as carefully designed by natural selection for the use of a social, bipedal, originally African ape as human stomachs were designed for the use of an omnivorous African ape with a taste for meat\n",
" - Darwin's theory of evolution by natural selection is satisfying because it shows us away in which simplicity could change into complexity, how unordered atoms could group themselves into ever\n",
" - the entire universe as a flow of data, see organisms as little more than biochemical algorithms, and believe that humanity’s cosmic vocation is to create an all-encompassing data-processing system\n",
" - Heredity is a modifiable stored program; metabolism is a universal machine\n",
" - All human knowledge is either experience or mathematics.\n",
" - animal's behaviour tends to maximize the survival of thegenes 'for'that behaviour, whether or not those genes happen to be in the body of the particular animalperforming\n",
" - There are no such things as \"mind,\" reason, thought, consciousness, soul, will, or truth: they all belong to fiction,\n",
" - An animal's behaviour tends to maximize the survival of thegenes 'for'that behaviour, whether or not those genes happen to be in the body of the particular animalperforming it\n",
"~~~~~~~~~~~~~\n",
"quote: We join spokes together in a wheel, but it is the center hole that makes the wagon move. We shape clay into a pot, but it is the emptiness inside that holds whatever we want. We hammer wood for a house, but it is the inner space that makes it livable. We work with being, but non-being is what we use.\n",
"nearest_quotes:\n",
" - Making things is a joy—immensely satisfying. J. R. R. Tolkien suggests that God gave us the gift of subcreation, as a gift, just for our joy.2 After all, “The cattle on a thousand hills are mine. …\n",
" - Making things is a joy—immensely satisfying. J. R. R. Tolkien suggests that God gave us the gift of subcreation, as a gift, just for our joy.2 After all, “The cattle on a thousand hills are mine. … If I were hungry, I would not tell you.”3 Designing per se is fun.\n",
" - Because if you carry all the burdens of yesterday, your brain becomes mechanical, dull. If you leave all the psychological memories, hurts, pains, behind, every day, then it means dying and living are together. In that there is no fear\n",
" - That is to say; there is nothing without other things. That is to say; there is no \"thing-in-itself.\"\n",
" - The more people are drawn into the global division of labour, the more people can specialise and exchange, the wealthier we will all be. Moreover, along the way there is no reason we cannot solve the\n",
" - There is no relationship between work done and money received; the individual should, according to his kind, be so placed as to perform the highest that is compatible with his powers.\n",
" - You don’t have to make the mind quiet. If you end all conflict, the mind naturally becomes quiet. And when the mind is absolutely silent, without any movement of thought, then perhaps you will see something, perhaps there is something sacred beyond all words. And this man has sought everlastingly, something that is beyond measure, beyond thought, which is incorruptible, unnameable, eternal\n",
" - You don’t have to make the mind quiet. If you end all conflict, the mind naturally becomes quiet. And when the mind is absolutely silent, without any movement of thought, then perhaps you will see something, perhaps there is something sacred beyond all words. And this man has sought everlastingly, something that is beyond measure, beyond thought, which is incorruptible, unnameable, eternal\n",
" - Women's bodies evolved to suit the demands of bearing and rearing children and of gathering plant food. Men's bodies evolved to suit the demands of rising in a male hierarchy, fighting over women, and providing meat to a family\n",
" - You don’t have to make the mind quiet. If you end all conflict, the mind naturally becomes quiet. And when the mind is absolutely silent, without any movement of thought, then perhaps you will see something, perhaps there is something sacred beyond all words. And this man has sought everlastingly, something that is beyond measure, beyond thought, which is incorruptible, unnameable, eternal. That can only take place when the mind is absolutely free and completely silent\n",
"~~~~~~~~~~~~~\n",
"quote: If you wish to have leisure for your mind, either be a poor man, or resemble a poor man.\n",
"nearest_quotes:\n",
" - Of all people only those are at leisure who make time for phiosophy, only those are really alive.\n",
" - If your private life conflicts with your intellectual opinion, it cancels your intellectual ideas, not your private life.\n",
" - If your private actions do not generalize, then you cannot have general ideas.\n",
" - Of all people only those are at leisure who make time for phiosophy, only those are really alive. For they not ony keep a good watch over their own\n",
" - Of all people only those are at leisure who\n",
" - Only eat what your body requires, and don't [become] carried away with foods that don't benefit you\n",
" - How much longer are you going to sere under others' orders? Assume authority yourself and utter something that may be handed dow to posterity. Produce sometg fom you own resources\n",
" - If you do not undertake a risk of real harm, reparable or even potentially irreparable, from an adventure, it is not an adventure.\n",
" - To put it another way: if wealth is giving you fewer options instead of more (and more varied) options, you’re doing it wrong.\n",
" - Still, you must especially avoid those who are gloomy and aways lamentg, and who grasp at every pretext for complaint. Though a man's loyalty and kdness may not be in doubt, a companion who is agtated and groaning about everg is an enemy to peace of mind\n",
"~~~~~~~~~~~~~\n",
"quote: Those who know don’t talk. Those who talk don’t know.\n",
"nearest_quotes:\n",
" - If your private actions do not generalize, then you cannot have general ideas.\n",
" - There is knowledge of the present only when there is a complete understanding of what the structure and the nature of the past is—and ending it.\n",
" - to hear the inferior and the poor in spirit having their say is a terrible ear-splitting torment for him who knows and trembles at the thought,\n",
" - If your private life conflicts with your intellectual opinion, it cancels your intellectual ideas, not your private life.\n",
" - There is no such thing as a cause or an effect.\n",
" - He knows that he cannot reveal himself to anybody: he thinks it bad taste to become familiar; and as a rule he is not familiar when people think he is. When he is not talking to his soul, he wears a mask. He would rather lie than tell the truth, because lying requires more spirit and will.\n",
" - The most valuable knowledge is always discovered last: but the most valuable knowledge consists of methods.\n",
" - Because if you carry all the burdens of yesterday, your brain becomes mechanical, dull. If you leave all the psychological memories, hurts, pains, behind, every day, then it means dying and living are together. In that there is no fear\n",
" - You don’t have to make the mind quiet. If you end all conflict, the mind naturally becomes quiet. And when the mind is absolutely silent, without any movement of thought, then perhaps you will see something, perhaps there is something sacred beyond all words. And this man has sought everlastingly, something that is beyond measure, beyond thought, which is incorruptible, unnameable, eternal\n",
" - You don’t have to make the mind quiet. If you end all conflict, the mind naturally becomes quiet. And when the mind is absolutely silent, without any movement of thought, then perhaps you will see something, perhaps there is something sacred beyond all words. And this man has sought everlastingly, something that is beyond measure, beyond thought, which is incorruptible, unnameable, eternal\n",
"~~~~~~~~~~~~~\n"
],
"name": "stdout"
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment