Skip to content

Instantly share code, notes, and snippets.

@jcrist

jcrist/pypi.ipynb

Created August 2, 2023 20:34
Show Gist options
  • Save jcrist/38352309924b1816f2cb196f43536a1b to your computer and use it in GitHub Desktop.
Save jcrist/38352309924b1816f2cb196f43536a1b to your computer and use it in GitHub Desktop.
Download ZIP
Analyzing PyPI data with Ibis
Display the source blob
Display the rendered blob
Raw
pypi.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "a768a549",
"metadata": {},
"outputs": [],
"source": [
"import ibis\n",
"from ibis import _\n",
"\n",
"ibis.options.interactive = True"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "cd2cbbb4",
"metadata": {},
"outputs": [],
"source": [
"con = ibis.connect(\"pypi.db\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "19e1a36e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['deps',\n",
" 'maintainers',\n",
" 'package_urls',\n",
" 'packages',\n",
" 'scorecard_checks',\n",
" 'wheels']"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"con.list_tables()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "b063ad0d",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> package_name </span>┃<span style=\"font-weight: bold\"> version </span>┃<span style=\"font-weight: bold\"> dep_name </span>┃<span style=\"font-weight: bold\"> dep_specifier </span>┃<span style=\"font-weight: bold\"> extra </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │\n",
"├──────────────────────────────────────┼─────────┼─────────────────┼──────────────────────┼────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-orchestrator </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">aioredis </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">aioredis </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-orchestrator </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">etcd3 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">etcd3 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-orchestrator </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">jinja2 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">jinja2 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-orchestrator </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">netaddr </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">netaddr </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-orchestrator </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">python-dateutil</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">python-dateutil </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-core-client </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">redis </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">&gt;=2.10.5 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">01d61084-d29e-11e9-96d1-7c5cf84ffe8e</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">attrs </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">==19.1.0 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">01d61084-d29e-11e9-96d1-7c5cf84ffe8e</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">click </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">==7.0 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0x-contract-addresses </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">mypy-extensions</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">mypy-extensions </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0x-contract-addresses </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">bandit </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">bandit; extra=='dev'</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">dev </span> │\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │\n",
"└──────────────────────────────────────┴─────────┴─────────────────┴──────────────────────┴────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mpackage_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mversion\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdep_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdep_specifier\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mextra\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │\n",
"├──────────────────────────────────────┼─────────┼─────────────────┼──────────────────────┼────────┤\n",
"│ \u001b[32m0-orchestrator \u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32maioredis \u001b[0m │ \u001b[32maioredis \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m0-orchestrator \u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32metcd3 \u001b[0m │ \u001b[32metcd3 \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m0-orchestrator \u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32mjinja2 \u001b[0m │ \u001b[32mjinja2 \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m0-orchestrator \u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32mnetaddr \u001b[0m │ \u001b[32mnetaddr \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m0-orchestrator \u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32mpython-dateutil\u001b[0m │ \u001b[32mpython-dateutil \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m0-core-client \u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32mredis \u001b[0m │ \u001b[32m>=2.10.5 \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m01d61084-d29e-11e9-96d1-7c5cf84ffe8e\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32mattrs \u001b[0m │ \u001b[32m==19.1.0 \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m01d61084-d29e-11e9-96d1-7c5cf84ffe8e\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32mclick \u001b[0m │ \u001b[32m==7.0 \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m0x-contract-addresses \u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32mmypy-extensions\u001b[0m │ \u001b[32mmypy-extensions \u001b[0m │ \u001b[2mNULL\u001b[0m │\n",
"│ \u001b[32m0x-contract-addresses \u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[32mbandit \u001b[0m │ \u001b[32mbandit; extra=='dev'\u001b[0m │ \u001b[32mdev \u001b[0m │\n",
"│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n",
"└──────────────────────────────────────┴─────────┴─────────────────┴──────────────────────┴────────┘"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"deps = con.tables.deps\n",
"\n",
"deps"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "f3fa22cd",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> name </span>┃<span style=\"font-weight: bold\"> package_name </span>┃\n",
"┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │\n",
"├───────────────┼───────────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">hallazzang </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0 </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">zaibon </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0-orchestrator </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">Wenxuan </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">01changer </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">zaibon </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0-core-client </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">qu6zhi </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0-618 </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">piyush01 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">01-distributions </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">travistorline</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">024travis-test024</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">gabrielcdiniz</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">021 </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">kin </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">000 </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">sma.linux </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">00smalinux </span> │\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │\n",
"└───────────────┴───────────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mname\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mpackage_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │\n",
"├───────────────┼───────────────────┤\n",
"│ \u001b[32mhallazzang \u001b[0m │ \u001b[32m0 \u001b[0m │\n",
"│ \u001b[32mzaibon \u001b[0m │ \u001b[32m0-orchestrator \u001b[0m │\n",
"│ \u001b[32mWenxuan \u001b[0m │ \u001b[32m01changer \u001b[0m │\n",
"│ \u001b[32mzaibon \u001b[0m │ \u001b[32m0-core-client \u001b[0m │\n",
"│ \u001b[32mqu6zhi \u001b[0m │ \u001b[32m0-618 \u001b[0m │\n",
"│ \u001b[32mpiyush01 \u001b[0m │ \u001b[32m01-distributions \u001b[0m │\n",
"│ \u001b[32mtravistorline\u001b[0m │ \u001b[32m024travis-test024\u001b[0m │\n",
"│ \u001b[32mgabrielcdiniz\u001b[0m │ \u001b[32m021 \u001b[0m │\n",
"│ \u001b[32mkin \u001b[0m │ \u001b[32m000 \u001b[0m │\n",
"│ \u001b[32msma.linux \u001b[0m │ \u001b[32m00smalinux \u001b[0m │\n",
"│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n",
"└───────────────┴───────────────────┘"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"maintainers = con.tables.maintainers\n",
"\n",
"maintainers"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "6d1c2451",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> name </span>┃<span style=\"font-weight: bold\"> version </span>┃<span style=\"font-weight: bold\"> requires_python </span>┃<span style=\"font-weight: bold\"> yanked </span>┃<span style=\"font-weight: bold\"> has_binary_wheel </span>┃<span style=\"font-weight: bold\"> has_vulnerabilities </span>┃<span style=\"font-weight: bold\"> first_uploaded_at </span>┃<span style=\"font-weight: bold\"> last_uploaded_at </span>┃<span style=\"font-weight: bold\"> recorded_at </span>┃<span style=\"font-weight: bold\"> downloads </span>┃<span style=\"font-weight: bold\"> scorecard_overall </span>┃<span style=\"font-weight: bold\"> in_google_assured_oss </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">boolean</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">boolean</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">boolean</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">timestamp</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">timestamp</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">timestamp</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int32</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">float64</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">boolean</span> │\n",
"├───────────────────┼─────────┼─────────────────┼─────────┼──────────────────┼─────────────────────┼─────────────────────┼─────────────────────┼─────────────────────┼───────────┼───────────────────┼───────────────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0.0.0 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">~</span> │ False │ False │ False │ <span style=\"color: #800080; text-decoration-color: #800080\">2017-08-06 03:43:27</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2017-08-06 03:43:27</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:08</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-orchestrator </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">1.1.0a8</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">~</span> │ False │ False │ False │ <span style=\"color: #800080; text-decoration-color: #800080\">2017-10-11 13:21:48</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2017-10-11 13:21:50</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:08</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">01changer </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">1.0.0 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">~</span> │ False │ False │ False │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:09</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-core-client </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">1.1.0a8</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">~</span> │ False │ False │ False │ <span style=\"color: #800080; text-decoration-color: #800080\">2017-10-11 13:19:47</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2017-10-11 13:19:49</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:09</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">0-618 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0.1.0 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">&gt;=3.6.0 </span> │ False │ False │ False │ <span style=\"color: #800080; text-decoration-color: #800080\">2022-08-24 11:39:03</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2022-08-24 11:39:03</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:09</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">01-distributions </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0.1 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">~</span> │ False │ False │ False │ <span style=\"color: #800080; text-decoration-color: #800080\">2020-07-24 16:54:06</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2020-07-24 16:54:06</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:09</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">024travis-test024</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0.1.0 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">&gt;=3.9,&lt;4.0 </span> │ False │ False │ False │ <span style=\"color: #800080; text-decoration-color: #800080\">2021-03-06 17:32:48</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2021-03-06 17:32:49</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:09</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">021 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">1.0 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">~</span> │ False │ False │ False │ <span style=\"color: #800080; text-decoration-color: #800080\">2018-03-06 15:16:09</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2018-03-06 15:16:09</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:09</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">000 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">0.0.0 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">&gt;=3.6.0 </span> │ False │ False │ False │ <span style=\"color: #800080; text-decoration-color: #800080\">2022-05-29 11:20:53</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2022-05-29 11:20:53</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:09</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">00smalinux </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">1.0 </span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">~</span> │ False │ False │ False │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">NULL</span> │ <span style=\"color: #800080; text-decoration-color: #800080\">2023-07-12 14:21:09</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">nan</span> │ False │\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │\n",
"└───────────────────┴─────────┴─────────────────┴─────────┴──────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴───────────┴───────────────────┴───────────────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mname\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mversion\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mrequires_python\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1myanked\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mhas_binary_wheel\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mhas_vulnerabilities\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mfirst_uploaded_at\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mlast_uploaded_at\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mrecorded_at\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdownloads\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mscorecard_overall\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1min_google_assured_oss\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mboolean\u001b[0m │ \u001b[2mboolean\u001b[0m │ \u001b[2mboolean\u001b[0m │ \u001b[2mtimestamp\u001b[0m │ \u001b[2mtimestamp\u001b[0m │ \u001b[2mtimestamp\u001b[0m │ \u001b[2mint32\u001b[0m │ \u001b[2mfloat64\u001b[0m │ \u001b[2mboolean\u001b[0m │\n",
"├───────────────────┼─────────┼─────────────────┼─────────┼──────────────────┼─────────────────────┼─────────────────────┼─────────────────────┼─────────────────────┼───────────┼───────────────────┼───────────────────────┤\n",
"│ \u001b[32m0 \u001b[0m │ \u001b[32m0.0.0 \u001b[0m │ \u001b[2m~\u001b[0m │ False │ False │ False │ \u001b[35m2017-08-06 03:43:27\u001b[0m │ \u001b[35m2017-08-06 03:43:27\u001b[0m │ \u001b[35m2023-07-12 14:21:08\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m0-orchestrator \u001b[0m │ \u001b[32m1.1.0a8\u001b[0m │ \u001b[2m~\u001b[0m │ False │ False │ False │ \u001b[35m2017-10-11 13:21:48\u001b[0m │ \u001b[35m2017-10-11 13:21:50\u001b[0m │ \u001b[35m2023-07-12 14:21:08\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m01changer \u001b[0m │ \u001b[32m1.0.0 \u001b[0m │ \u001b[2m~\u001b[0m │ False │ False │ False │ \u001b[2mNULL\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[35m2023-07-12 14:21:09\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m0-core-client \u001b[0m │ \u001b[32m1.1.0a8\u001b[0m │ \u001b[2m~\u001b[0m │ False │ False │ False │ \u001b[35m2017-10-11 13:19:47\u001b[0m │ \u001b[35m2017-10-11 13:19:49\u001b[0m │ \u001b[35m2023-07-12 14:21:09\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m0-618 \u001b[0m │ \u001b[32m0.1.0 \u001b[0m │ \u001b[32m>=3.6.0 \u001b[0m │ False │ False │ False │ \u001b[35m2022-08-24 11:39:03\u001b[0m │ \u001b[35m2022-08-24 11:39:03\u001b[0m │ \u001b[35m2023-07-12 14:21:09\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m01-distributions \u001b[0m │ \u001b[32m0.1 \u001b[0m │ \u001b[2m~\u001b[0m │ False │ False │ False │ \u001b[35m2020-07-24 16:54:06\u001b[0m │ \u001b[35m2020-07-24 16:54:06\u001b[0m │ \u001b[35m2023-07-12 14:21:09\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m024travis-test024\u001b[0m │ \u001b[32m0.1.0 \u001b[0m │ \u001b[32m>=3.9,<4.0 \u001b[0m │ False │ False │ False │ \u001b[35m2021-03-06 17:32:48\u001b[0m │ \u001b[35m2021-03-06 17:32:49\u001b[0m │ \u001b[35m2023-07-12 14:21:09\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m021 \u001b[0m │ \u001b[32m1.0 \u001b[0m │ \u001b[2m~\u001b[0m │ False │ False │ False │ \u001b[35m2018-03-06 15:16:09\u001b[0m │ \u001b[35m2018-03-06 15:16:09\u001b[0m │ \u001b[35m2023-07-12 14:21:09\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m000 \u001b[0m │ \u001b[32m0.0.0 \u001b[0m │ \u001b[32m>=3.6.0 \u001b[0m │ False │ False │ False │ \u001b[35m2022-05-29 11:20:53\u001b[0m │ \u001b[35m2022-05-29 11:20:53\u001b[0m │ \u001b[35m2023-07-12 14:21:09\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[32m00smalinux \u001b[0m │ \u001b[32m1.0 \u001b[0m │ \u001b[2m~\u001b[0m │ False │ False │ False │ \u001b[2mNULL\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[35m2023-07-12 14:21:09\u001b[0m │ \u001b[1;36m0\u001b[0m │ \u001b[1;36mnan\u001b[0m │ False │\n",
"│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n",
"└───────────────────┴─────────┴─────────────────┴─────────┴──────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴───────────┴───────────────────┴───────────────────────┘"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"packages = con.tables.packages\n",
"\n",
"packages"
]
},
{
"cell_type": "markdown",
"id": "941cb55b",
"metadata": {},
"source": [
"### Total number of packages"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "9867857f",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
],
"text/plain": []
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"\u001b[1;36m452640\u001b[0m"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"packages.count()"
]
},
{
"cell_type": "markdown",
"id": "51dec6e1",
"metadata": {},
"source": [
"### Package counts by release day (sunday == 0)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "6915cf63",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━┳━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> day </span>┃<span style=\"font-weight: bold\"> CountStar() </span>┃\n",
"┡━━━━━━━╇━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int16</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├───────┼─────────────┤\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">68754</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">73183</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">72538</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">70664</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">66964</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">44071</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">47891</span> │\n",
"└───────┴─────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━┳━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mday\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mCountStar()\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━╇━━━━━━━━━━━━━┩\n",
"│ \u001b[2mint16\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├───────┼─────────────┤\n",
"│ \u001b[1;36m0\u001b[0m │ \u001b[1;36m68754\u001b[0m │\n",
"│ \u001b[1;36m1\u001b[0m │ \u001b[1;36m73183\u001b[0m │\n",
"│ \u001b[1;36m2\u001b[0m │ \u001b[1;36m72538\u001b[0m │\n",
"│ \u001b[1;36m3\u001b[0m │ \u001b[1;36m70664\u001b[0m │\n",
"│ \u001b[1;36m4\u001b[0m │ \u001b[1;36m66964\u001b[0m │\n",
"│ \u001b[1;36m5\u001b[0m │ \u001b[1;36m44071\u001b[0m │\n",
"│ \u001b[1;36m6\u001b[0m │ \u001b[1;36m47891\u001b[0m │\n",
"└───────┴─────────────┘"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"release_days = (\n",
" packages\n",
" .dropna(\"last_uploaded_at\")\n",
" .group_by(_.last_uploaded_at.day_of_week.index().name(\"day\"))\n",
" .count()\n",
")\n",
"\n",
"release_days"
]
},
{
"cell_type": "markdown",
"id": "18f28863",
"metadata": {},
"source": [
"### What maintainers have the most downloads?"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "11134b38",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> name </span>┃<span style=\"font-weight: bold\"> downloads </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├───────────────────┼────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">aws </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1681170346</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">google_opensource</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1167293745</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">gcloudpypi </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">874495196</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">microsoft </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">777314998</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">garnaat </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">700657025</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">jaraco </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">654999607</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">azure-sdk </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">627478370</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">Lukasa </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">576334805</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">graffatcolmingov </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">532278545</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dstufft </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">520993792</span> │\n",
"└───────────────────┴────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mname\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdownloads\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├───────────────────┼────────────┤\n",
"│ \u001b[32maws \u001b[0m │ \u001b[1;36m1681170346\u001b[0m │\n",
"│ \u001b[32mgoogle_opensource\u001b[0m │ \u001b[1;36m1167293745\u001b[0m │\n",
"│ \u001b[32mgcloudpypi \u001b[0m │ \u001b[1;36m874495196\u001b[0m │\n",
"│ \u001b[32mmicrosoft \u001b[0m │ \u001b[1;36m777314998\u001b[0m │\n",
"│ \u001b[32mgarnaat \u001b[0m │ \u001b[1;36m700657025\u001b[0m │\n",
"│ \u001b[32mjaraco \u001b[0m │ \u001b[1;36m654999607\u001b[0m │\n",
"│ \u001b[32mazure-sdk \u001b[0m │ \u001b[1;36m627478370\u001b[0m │\n",
"│ \u001b[32mLukasa \u001b[0m │ \u001b[1;36m576334805\u001b[0m │\n",
"│ \u001b[32mgraffatcolmingov \u001b[0m │ \u001b[1;36m532278545\u001b[0m │\n",
"│ \u001b[32mdstufft \u001b[0m │ \u001b[1;36m520993792\u001b[0m │\n",
"└───────────────────┴────────────┘"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"top_maintainers_by_downloads = (\n",
" maintainers\n",
" .join(packages, [(\"package_name\", \"name\")])\n",
" .group_by(\"name\")\n",
" .aggregate(downloads=_.downloads.sum())\n",
" .select(\"name\", \"downloads\")\n",
" .order_by(ibis.desc(\"downloads\"))\n",
" .limit(10)\n",
")\n",
"\n",
"top_maintainers_by_downloads"
]
},
{
"cell_type": "markdown",
"id": "7659d798",
"metadata": {},
"source": [
"### What packages depend on ibis-framework"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "7ad82673",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> package_name </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │\n",
"├───────────────────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">alertadengue </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask-clickhouse </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dbplot </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">epigraphhub </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">fugue </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">fugue-warehouses </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">genomic-features </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">google-pso-data-validator</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">holoviews </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">ibis-bigquery </span> │\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │\n",
"└───────────────────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mpackage_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │\n",
"├───────────────────────────┤\n",
"│ \u001b[32malertadengue \u001b[0m │\n",
"│ \u001b[32mdask-clickhouse \u001b[0m │\n",
"│ \u001b[32mdbplot \u001b[0m │\n",
"│ \u001b[32mepigraphhub \u001b[0m │\n",
"│ \u001b[32mfugue \u001b[0m │\n",
"│ \u001b[32mfugue-warehouses \u001b[0m │\n",
"│ \u001b[32mgenomic-features \u001b[0m │\n",
"│ \u001b[32mgoogle-pso-data-validator\u001b[0m │\n",
"│ \u001b[32mholoviews \u001b[0m │\n",
"│ \u001b[32mibis-bigquery \u001b[0m │\n",
"│ \u001b[2m…\u001b[0m │\n",
"└───────────────────────────┘"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ibis_dependents = (\n",
" deps\n",
" .filter(_.dep_name == \"ibis-framework\")\n",
" .select(\"package_name\")\n",
" .distinct()\n",
")\n",
"\n",
"ibis_dependents"
]
},
{
"cell_type": "markdown",
"id": "e82f5d53",
"metadata": {},
"source": [
"### What packages depend on things I maintain?"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "ce1c5c4a",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> package </span>┃<span style=\"font-weight: bold\"> dependent </span>┃\n",
"┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │\n",
"├─────────┼─────────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">a2 </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">abstar </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">acalib </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">msgspec</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">acb </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">hdfs3 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">accuinsight </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">aces-apps </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">acquire-imaging</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">acryo </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">acs-axiom </span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">dask </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">actk </span> │\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │\n",
"└─────────┴─────────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mpackage\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdependent\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │\n",
"├─────────┼─────────────────┤\n",
"│ \u001b[32mdask \u001b[0m │ \u001b[32ma2 \u001b[0m │\n",
"│ \u001b[32mdask \u001b[0m │ \u001b[32mabstar \u001b[0m │\n",
"│ \u001b[32mdask \u001b[0m │ \u001b[32macalib \u001b[0m │\n",
"│ \u001b[32mmsgspec\u001b[0m │ \u001b[32macb \u001b[0m │\n",
"│ \u001b[32mhdfs3 \u001b[0m │ \u001b[32maccuinsight \u001b[0m │\n",
"│ \u001b[32mdask \u001b[0m │ \u001b[32maces-apps \u001b[0m │\n",
"│ \u001b[32mdask \u001b[0m │ \u001b[32macquire-imaging\u001b[0m │\n",
"│ \u001b[32mdask \u001b[0m │ \u001b[32macryo \u001b[0m │\n",
"│ \u001b[32mdask \u001b[0m │ \u001b[32macs-axiom \u001b[0m │\n",
"│ \u001b[32mdask \u001b[0m │ \u001b[32mactk \u001b[0m │\n",
"│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n",
"└─────────┴─────────────────┘"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"my_dependents = (\n",
" deps\n",
" .join(\n",
" maintainers.filter(_.name == \"jcrist\"),\n",
" [(\"dep_name\", \"package_name\")]\n",
" )\n",
" .select(package=\"dep_name\", dependent=\"package_name\")\n",
" .distinct()\n",
")\n",
"\n",
"my_dependents"
]
},
{
"cell_type": "markdown",
"id": "95fad7e3",
"metadata": {},
"source": [
"### What packages are commonly used by `test*` extras?"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "f0e72ca4",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━┳━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> dep_name </span>┃<span style=\"font-weight: bold\"> count </span>┃\n",
"┡━━━━━━━━━━━━━━━━╇━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├────────────────┼───────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14430</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-cov </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7933</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">coverage </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4161</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">flake8 </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2388</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">tox </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1555</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">mypy </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1346</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">black </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1276</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-mock </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1156</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">mock </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1139</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-asyncio</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">997</span> │\n",
"└────────────────┴───────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━┳━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mdep_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcount\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━╇━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├────────────────┼───────┤\n",
"│ \u001b[32mpytest \u001b[0m │ \u001b[1;36m14430\u001b[0m │\n",
"│ \u001b[32mpytest-cov \u001b[0m │ \u001b[1;36m7933\u001b[0m │\n",
"│ \u001b[32mcoverage \u001b[0m │ \u001b[1;36m4161\u001b[0m │\n",
"│ \u001b[32mflake8 \u001b[0m │ \u001b[1;36m2388\u001b[0m │\n",
"│ \u001b[32mtox \u001b[0m │ \u001b[1;36m1555\u001b[0m │\n",
"│ \u001b[32mmypy \u001b[0m │ \u001b[1;36m1346\u001b[0m │\n",
"│ \u001b[32mblack \u001b[0m │ \u001b[1;36m1276\u001b[0m │\n",
"│ \u001b[32mpytest-mock \u001b[0m │ \u001b[1;36m1156\u001b[0m │\n",
"│ \u001b[32mmock \u001b[0m │ \u001b[1;36m1139\u001b[0m │\n",
"│ \u001b[32mpytest-asyncio\u001b[0m │ \u001b[1;36m997\u001b[0m │\n",
"└────────────────┴───────┘"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"top_test_deps = (\n",
" deps\n",
" .filter(_.extra.startswith(\"test\"))\n",
" .group_by(\"dep_name\")\n",
" .agg(count=_.count())\n",
" .order_by(ibis.desc(\"count\"))\n",
" .limit(10)\n",
")\n",
"\n",
"top_test_deps"
]
},
{
"cell_type": "markdown",
"id": "c299bf0d",
"metadata": {},
"source": [
"### What are the top pytest extensions?"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "c518d66e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━┳━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> dep_name </span>┃<span style=\"font-weight: bold\"> count </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━╇━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├──────────────────┼───────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-cov </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11408</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-mock </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1816</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-asyncio </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1605</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-xdist </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1449</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-timeout </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">533</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-django </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">502</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-runner </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">490</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-flake8 </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">475</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-sugar </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">364</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-benchmark</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">331</span> │\n",
"└──────────────────┴───────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━┳━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mdep_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcount\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━╇━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├──────────────────┼───────┤\n",
"│ \u001b[32mpytest-cov \u001b[0m │ \u001b[1;36m11408\u001b[0m │\n",
"│ \u001b[32mpytest-mock \u001b[0m │ \u001b[1;36m1816\u001b[0m │\n",
"│ \u001b[32mpytest-asyncio \u001b[0m │ \u001b[1;36m1605\u001b[0m │\n",
"│ \u001b[32mpytest-xdist \u001b[0m │ \u001b[1;36m1449\u001b[0m │\n",
"│ \u001b[32mpytest-timeout \u001b[0m │ \u001b[1;36m533\u001b[0m │\n",
"│ \u001b[32mpytest-django \u001b[0m │ \u001b[1;36m502\u001b[0m │\n",
"│ \u001b[32mpytest-runner \u001b[0m │ \u001b[1;36m490\u001b[0m │\n",
"│ \u001b[32mpytest-flake8 \u001b[0m │ \u001b[1;36m475\u001b[0m │\n",
"│ \u001b[32mpytest-sugar \u001b[0m │ \u001b[1;36m364\u001b[0m │\n",
"│ \u001b[32mpytest-benchmark\u001b[0m │ \u001b[1;36m331\u001b[0m │\n",
"└──────────────────┴───────┘"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"top_pytest_extensions = (\n",
" deps\n",
" .filter(_.dep_name.startswith(\"pytest-\"))\n",
" .select(\"package_name\", \"dep_name\")\n",
" .distinct()\n",
" .group_by(\"dep_name\")\n",
" .agg(count=_.count())\n",
" .order_by(ibis.desc(\"count\"))\n",
" .limit(10)\n",
")\n",
"\n",
"top_pytest_extensions"
]
},
{
"cell_type": "markdown",
"id": "83257a65",
"metadata": {},
"source": [
"### What packages are the most depended on"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "0644ad79",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━┳━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> dep_name </span>┃<span style=\"font-weight: bold\"> dep_count </span>┃\n",
"┡━━━━━━━━━━━━╇━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├────────────┼───────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">numpy </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">41656</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">requests </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">39331</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pandas </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">27275</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">25227</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">matplotlib</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">18046</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">scipy </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16837</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">click </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">15524</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pyyaml </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14392</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">odoo </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14003</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-cov</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11408</span> │\n",
"└────────────┴───────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━┳━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mdep_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdep_count\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━╇━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├────────────┼───────────┤\n",
"│ \u001b[32mnumpy \u001b[0m │ \u001b[1;36m41656\u001b[0m │\n",
"│ \u001b[32mrequests \u001b[0m │ \u001b[1;36m39331\u001b[0m │\n",
"│ \u001b[32mpandas \u001b[0m │ \u001b[1;36m27275\u001b[0m │\n",
"│ \u001b[32mpytest \u001b[0m │ \u001b[1;36m25227\u001b[0m │\n",
"│ \u001b[32mmatplotlib\u001b[0m │ \u001b[1;36m18046\u001b[0m │\n",
"│ \u001b[32mscipy \u001b[0m │ \u001b[1;36m16837\u001b[0m │\n",
"│ \u001b[32mclick \u001b[0m │ \u001b[1;36m15524\u001b[0m │\n",
"│ \u001b[32mpyyaml \u001b[0m │ \u001b[1;36m14392\u001b[0m │\n",
"│ \u001b[32modoo \u001b[0m │ \u001b[1;36m14003\u001b[0m │\n",
"│ \u001b[32mpytest-cov\u001b[0m │ \u001b[1;36m11408\u001b[0m │\n",
"└────────────┴───────────┘"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"most_dependents = (\n",
" deps\n",
" .select(\"package_name\", \"dep_name\")\n",
" .distinct()\n",
" .group_by(\"dep_name\")\n",
" .agg(dep_count=_.count())\n",
" .order_by(ibis.desc(\"dep_count\"))\n",
" .limit(10)\n",
")\n",
"\n",
"most_dependents"
]
},
{
"cell_type": "markdown",
"id": "11a36252",
"metadata": {},
"source": [
"### Histogram of maintainer count"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "a491e55a",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━┳━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> maintainers </span>┃<span style=\"font-weight: bold\"> count </span>┃\n",
"┡━━━━━━━━━━━━━╇━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├─────────────┼────────┤\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">409097</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">27310</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9118</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3472</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1548</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">721</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">378</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">192</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">106</span> │\n",
"│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">75</span> │\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │\n",
"└─────────────┴────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━┳━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mmaintainers\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcount\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━╇━━━━━━━━┩\n",
"│ \u001b[2mint64\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├─────────────┼────────┤\n",
"│ \u001b[1;36m1\u001b[0m │ \u001b[1;36m409097\u001b[0m │\n",
"│ \u001b[1;36m2\u001b[0m │ \u001b[1;36m27310\u001b[0m │\n",
"│ \u001b[1;36m3\u001b[0m │ \u001b[1;36m9118\u001b[0m │\n",
"│ \u001b[1;36m4\u001b[0m │ \u001b[1;36m3472\u001b[0m │\n",
"│ \u001b[1;36m5\u001b[0m │ \u001b[1;36m1548\u001b[0m │\n",
"│ \u001b[1;36m6\u001b[0m │ \u001b[1;36m721\u001b[0m │\n",
"│ \u001b[1;36m7\u001b[0m │ \u001b[1;36m378\u001b[0m │\n",
"│ \u001b[1;36m8\u001b[0m │ \u001b[1;36m192\u001b[0m │\n",
"│ \u001b[1;36m9\u001b[0m │ \u001b[1;36m106\u001b[0m │\n",
"│ \u001b[1;36m10\u001b[0m │ \u001b[1;36m75\u001b[0m │\n",
"│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n",
"└─────────────┴────────┘"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"maintainer_counts = (\n",
" maintainers\n",
" .group_by(\"package_name\")\n",
" .agg(maintainers=_.count())\n",
" .group_by(\"maintainers\")\n",
" .agg(count=_.count())\n",
")\n",
"\n",
"maintainer_counts"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "59ce0bf7",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"<style>\n",
" #altair-viz-7186c6b6a2e0423e9eb1189c1e94baf3.vega-embed {\n",
" width: 100%;\n",
" display: flex;\n",
" }\n",
"\n",
" #altair-viz-7186c6b6a2e0423e9eb1189c1e94baf3.vega-embed details,\n",
" #altair-viz-7186c6b6a2e0423e9eb1189c1e94baf3.vega-embed details summary {\n",
" position: relative;\n",
" }\n",
"</style>\n",
"<div id=\"altair-viz-7186c6b6a2e0423e9eb1189c1e94baf3\"></div>\n",
"<script type=\"text/javascript\">\n",
" var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
" (function(spec, embedOpt){\n",
" let outputDiv = document.currentScript.previousElementSibling;\n",
" if (outputDiv.id !== \"altair-viz-7186c6b6a2e0423e9eb1189c1e94baf3\") {\n",
" outputDiv = document.getElementById(\"altair-viz-7186c6b6a2e0423e9eb1189c1e94baf3\");\n",
" }\n",
" const paths = {\n",
" \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
" \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
" \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.8.0?noext\",\n",
" \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
" };\n",
"\n",
" function maybeLoadScript(lib, version) {\n",
" var key = `${lib.replace(\"-\", \"\")}_version`;\n",
" return (VEGA_DEBUG[key] == version) ?\n",
" Promise.resolve(paths[lib]) :\n",
" new Promise(function(resolve, reject) {\n",
" var s = document.createElement('script');\n",
" document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
" s.async = true;\n",
" s.onload = () => {\n",
" VEGA_DEBUG[key] = version;\n",
" return resolve(paths[lib]);\n",
" };\n",
" s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
" s.src = paths[lib];\n",
" });\n",
" }\n",
"\n",
" function showError(err) {\n",
" outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
" throw err;\n",
" }\n",
"\n",
" function displayChart(vegaEmbed) {\n",
" vegaEmbed(outputDiv, spec, embedOpt)\n",
" .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
" }\n",
"\n",
" if(typeof define === \"function\" && define.amd) {\n",
" requirejs.config({paths});\n",
" require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
" } else {\n",
" maybeLoadScript(\"vega\", \"5\")\n",
" .then(() => maybeLoadScript(\"vega-lite\", \"5.8.0\"))\n",
" .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
" .catch(showError)\n",
" .then(() => displayChart(vegaEmbed));\n",
" }\n",
" })({\"config\": {\"view\": {\"continuousWidth\": 300, \"continuousHeight\": 300}}, \"data\": {\"name\": \"data-9a27ef86aff8a18091b2e7afa60cb68a\"}, \"mark\": {\"type\": \"bar\"}, \"encoding\": {\"x\": {\"field\": \"maintainers\", \"type\": \"quantitative\"}, \"y\": {\"field\": \"count\", \"scale\": {\"type\": \"log\"}, \"type\": \"quantitative\"}}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.8.0.json\", \"datasets\": {\"data-9a27ef86aff8a18091b2e7afa60cb68a\": [{\"maintainers\": 1, \"count\": 409097}, {\"maintainers\": 2, \"count\": 27310}, {\"maintainers\": 3, \"count\": 9118}, {\"maintainers\": 4, \"count\": 3472}, {\"maintainers\": 5, \"count\": 1548}, {\"maintainers\": 6, \"count\": 721}, {\"maintainers\": 7, \"count\": 378}, {\"maintainers\": 8, \"count\": 192}, {\"maintainers\": 9, \"count\": 106}, {\"maintainers\": 10, \"count\": 75}, {\"maintainers\": 11, \"count\": 32}, {\"maintainers\": 12, \"count\": 229}, {\"maintainers\": 13, \"count\": 18}, {\"maintainers\": 14, \"count\": 12}, {\"maintainers\": 15, \"count\": 17}, {\"maintainers\": 16, \"count\": 9}, {\"maintainers\": 17, \"count\": 8}, {\"maintainers\": 18, \"count\": 23}, {\"maintainers\": 19, \"count\": 13}, {\"maintainers\": 20, \"count\": 10}, {\"maintainers\": 21, \"count\": 4}, {\"maintainers\": 22, \"count\": 2}, {\"maintainers\": 23, \"count\": 2}, {\"maintainers\": 24, \"count\": 39}, {\"maintainers\": 25, \"count\": 32}, {\"maintainers\": 26, \"count\": 29}, {\"maintainers\": 27, \"count\": 18}, {\"maintainers\": 28, \"count\": 21}, {\"maintainers\": 29, \"count\": 11}, {\"maintainers\": 30, \"count\": 10}, {\"maintainers\": 31, \"count\": 5}, {\"maintainers\": 32, \"count\": 4}, {\"maintainers\": 33, \"count\": 3}, {\"maintainers\": 34, \"count\": 2}]}}, {\"mode\": \"vega-lite\"});\n",
"</script>"
],
"text/plain": [
"alt.Chart(...)"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import altair as alt\n",
"chart = (\n",
" alt.Chart(maintainer_counts.to_pandas())\n",
" .mark_bar()\n",
" .encode(x=\"maintainers\", y=alt.Y(\"count\", scale=alt.Scale(type=\"log\")))\n",
")\n",
"\n",
"chart"
]
},
{
"cell_type": "markdown",
"id": "02fc5d80",
"metadata": {},
"source": [
"Followup question - what's up with the spike at 12?"
]
},
{
"cell_type": "markdown",
"id": "4c026015",
"metadata": {},
"source": [
"### What are the most common package prefixes?"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "b4778dcd",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> prefix </span>┃<span style=\"font-weight: bold\"> Count(prefix) </span>┃\n",
"┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├──────────────┼───────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">ftw </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">121</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">bob </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">48</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">collective </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">simplelayout</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">plonetheme </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> │\n",
"└──────────────┴───────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mprefix\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mCount(prefix)\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├──────────────┼───────────────┤\n",
"│ \u001b[32mftw \u001b[0m │ \u001b[1;36m121\u001b[0m │\n",
"│ \u001b[32mbob \u001b[0m │ \u001b[1;36m48\u001b[0m │\n",
"│ \u001b[32mcollective \u001b[0m │ \u001b[1;36m20\u001b[0m │\n",
"│ \u001b[32msimplelayout\u001b[0m │ \u001b[1;36m7\u001b[0m │\n",
"│ \u001b[32mplonetheme \u001b[0m │ \u001b[1;36m3\u001b[0m │\n",
"└──────────────┴───────────────┘"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"common_prefixes = (\n",
" maintainers\n",
" .group_by(\"package_name\")\n",
" .agg(maintainers=_.count())\n",
" .filter(_.maintainers == 12)\n",
" .package_name\n",
" .re_extract(r\"^(\\w*)-?\", 1).name(\"prefix\")\n",
" .topk(5)\n",
")\n",
"\n",
"common_prefixes"
]
},
{
"cell_type": "markdown",
"id": "d558ddb0",
"metadata": {},
"source": [
"### What packages have few downloads but lots of maintainers?\n",
"\n",
"Looking for \"clubs\", as described by Nadia Eghbal's \"Working in Public\"."
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "f43ee07e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
],
"text/plain": []
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"\u001b[1;36m82427\u001b[0m"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"packages.filter(_.downloads > 0).downloads.min()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "74cd818b",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> package_name </span>┃<span style=\"font-weight: bold\"> maintainers </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├────────────────────────┼─────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">zdaemon </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">27</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">apache-flink </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">25</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">py-pure-client </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">22</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">ray-cpp </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">18</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">hurry-query </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">oci-cli </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pyre-check </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">apache-flink-libraries</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">15</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">insights-core </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">tensorflow-federated </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span> │\n",
"└────────────────────────┴─────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mpackage_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmaintainers\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├────────────────────────┼─────────────┤\n",
"│ \u001b[32mzdaemon \u001b[0m │ \u001b[1;36m27\u001b[0m │\n",
"│ \u001b[32mapache-flink \u001b[0m │ \u001b[1;36m25\u001b[0m │\n",
"│ \u001b[32mpy-pure-client \u001b[0m │ \u001b[1;36m22\u001b[0m │\n",
"│ \u001b[32mray-cpp \u001b[0m │ \u001b[1;36m18\u001b[0m │\n",
"│ \u001b[32mhurry-query \u001b[0m │ \u001b[1;36m16\u001b[0m │\n",
"│ \u001b[32moci-cli \u001b[0m │ \u001b[1;36m16\u001b[0m │\n",
"│ \u001b[32mpyre-check \u001b[0m │ \u001b[1;36m16\u001b[0m │\n",
"│ \u001b[32mapache-flink-libraries\u001b[0m │ \u001b[1;36m15\u001b[0m │\n",
"│ \u001b[32minsights-core \u001b[0m │ \u001b[1;36m14\u001b[0m │\n",
"│ \u001b[32mtensorflow-federated \u001b[0m │ \u001b[1;36m14\u001b[0m │\n",
"└────────────────────────┴─────────────┘"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import datetime\n",
"\n",
"# These prefixes are all \"zope\" related, and are so prolific that they mask anything interesting.\n",
"# Zope used to be really popular, but downloads have waned. We'll ignore them for now since they\n",
"# mask anything else interesting.\n",
"ignore_prefixes = [\"zope\", \"zc\", \"z3c\", \"collective\", \"plone\", \"products\"]\n",
"\n",
"clubs = (\n",
" maintainers\n",
" .group_by(\"package_name\")\n",
" .agg(maintainers=_.count())\n",
" .join(\n",
" packages.filter(\n",
" [\n",
" _.downloads == 0,\n",
" _.last_uploaded_at > (datetime.datetime.now() - datetime.timedelta(days=365)),\n",
" *(~_.name.startswith(p) for p in ignore_prefixes)\n",
" ]\n",
" ),\n",
" [(\"package_name\", \"name\")]\n",
" )\n",
" .order_by([ibis.desc(\"maintainers\")])\n",
" .select(\"package_name\", \"maintainers\")\n",
" .limit(10)\n",
")\n",
"\n",
"clubs"
]
},
{
"cell_type": "markdown",
"id": "1c06e557",
"metadata": {},
"source": [
"### Find the top 20 most depended on packages that have only one maintainer"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "46fc05d3",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> package_name </span>┃<span style=\"font-weight: bold\"> name </span>┃<span style=\"font-weight: bold\"> dep_count </span>┃\n",
"┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├────────────────┼───────────────┼───────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytest-cov </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">ionel </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11408</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">six </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">gutworth </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8044</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pydantic </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">samuelcolvin </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6480</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">coverage </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">nedbatchelder</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6420</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">beautifulsoup4</span> │ <span style=\"color: #008000; text-decoration-color: #008000\">leonard </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4815</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">boto3 </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">aws </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4765</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">sqlalchemy </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">CaselIT </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4710</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pre-commit </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">asottile </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4135</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">rich </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">willmcgugan </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4094</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">pytz </span> │ <span style=\"color: #008000; text-decoration-color: #008000\">stub </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3872</span> │\n",
"└────────────────┴───────────────┴───────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mpackage_name\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mname\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mdep_count\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├────────────────┼───────────────┼───────────┤\n",
"│ \u001b[32mpytest-cov \u001b[0m │ \u001b[32mionel \u001b[0m │ \u001b[1;36m11408\u001b[0m │\n",
"│ \u001b[32msix \u001b[0m │ \u001b[32mgutworth \u001b[0m │ \u001b[1;36m8044\u001b[0m │\n",
"│ \u001b[32mpydantic \u001b[0m │ \u001b[32msamuelcolvin \u001b[0m │ \u001b[1;36m6480\u001b[0m │\n",
"│ \u001b[32mcoverage \u001b[0m │ \u001b[32mnedbatchelder\u001b[0m │ \u001b[1;36m6420\u001b[0m │\n",
"│ \u001b[32mbeautifulsoup4\u001b[0m │ \u001b[32mleonard \u001b[0m │ \u001b[1;36m4815\u001b[0m │\n",
"│ \u001b[32mboto3 \u001b[0m │ \u001b[32maws \u001b[0m │ \u001b[1;36m4765\u001b[0m │\n",
"│ \u001b[32msqlalchemy \u001b[0m │ \u001b[32mCaselIT \u001b[0m │ \u001b[1;36m4710\u001b[0m │\n",
"│ \u001b[32mpre-commit \u001b[0m │ \u001b[32masottile \u001b[0m │ \u001b[1;36m4135\u001b[0m │\n",
"│ \u001b[32mrich \u001b[0m │ \u001b[32mwillmcgugan \u001b[0m │ \u001b[1;36m4094\u001b[0m │\n",
"│ \u001b[32mpytz \u001b[0m │ \u001b[32mstub \u001b[0m │ \u001b[1;36m3872\u001b[0m │\n",
"└────────────────┴───────────────┴───────────┘"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"bus_factor_1 = (\n",
" maintainers\n",
" .group_by(\"package_name\")\n",
" .agg(maintainer_count=_.count())\n",
" .filter(_.maintainer_count == 1)\n",
" .join(maintainers, \"package_name\")\n",
" .join(\n",
" (\n",
" deps\n",
" .select(\"package_name\", \"dep_name\")\n",
" .distinct()\n",
" .group_by(\"dep_name\")\n",
" .agg(dep_count=_.count())\n",
" ),\n",
" [(\"package_name\", \"dep_name\")],\n",
" )\n",
" .select(\"package_name\", \"name\", \"dep_count\")\n",
" .order_by(ibis.desc(\"dep_count\"))\n",
" .limit(10)\n",
")\n",
"\n",
"bus_factor_1"
]
},
{
"cell_type": "markdown",
"id": "2612bd02",
"metadata": {},
"source": [
"### pypi users who have the most distinct collaborators"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "fce08fc0",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> name </span>┃<span style=\"font-weight: bold\"> n_collaborators </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├───────────────────┼─────────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">openstackci </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">386</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">davisagli </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">190</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">collective </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">186</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">hannosch </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">185</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">google_opensource</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">163</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">microsoft </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">161</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">gotcha </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">159</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">timo </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">152</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">plone </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">146</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">optilude </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">142</span> │\n",
"└───────────────────┴─────────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mname\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mn_collaborators\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├───────────────────┼─────────────────┤\n",
"│ \u001b[32mopenstackci \u001b[0m │ \u001b[1;36m386\u001b[0m │\n",
"│ \u001b[32mdavisagli \u001b[0m │ \u001b[1;36m190\u001b[0m │\n",
"│ \u001b[32mcollective \u001b[0m │ \u001b[1;36m186\u001b[0m │\n",
"│ \u001b[32mhannosch \u001b[0m │ \u001b[1;36m185\u001b[0m │\n",
"│ \u001b[32mgoogle_opensource\u001b[0m │ \u001b[1;36m163\u001b[0m │\n",
"│ \u001b[32mmicrosoft \u001b[0m │ \u001b[1;36m161\u001b[0m │\n",
"│ \u001b[32mgotcha \u001b[0m │ \u001b[1;36m159\u001b[0m │\n",
"│ \u001b[32mtimo \u001b[0m │ \u001b[1;36m152\u001b[0m │\n",
"│ \u001b[32mplone \u001b[0m │ \u001b[1;36m146\u001b[0m │\n",
"│ \u001b[32moptilude \u001b[0m │ \u001b[1;36m142\u001b[0m │\n",
"└───────────────────┴─────────────────┘"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"most_collaborators = (\n",
" maintainers\n",
" .join(maintainers, \"package_name\")\n",
" .select(\"name\", \"name_right\")\n",
" .filter(_.name != _.name_right)\n",
" .distinct()\n",
" .group_by(\"name\")\n",
" .agg(n_collaborators=_.count())\n",
" .order_by(ibis.desc(\"n_collaborators\"))\n",
" .limit(10)\n",
")\n",
"\n",
"most_collaborators"
]
},
{
"cell_type": "markdown",
"id": "ecb90e44",
"metadata": {},
"source": [
"### Finding the most popular transitive dependencies\n",
"\n",
"We can do this by using `.sql` to wrap a recursive CTE, then query it like a normal Ibis table. The recursive CTE will produce a table with a row for each package `package` and its recursive "
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "902ace7e",
"metadata": {},
"outputs": [],
"source": [
"transitive_deps = con.sql(\n",
" \"\"\"\n",
" WITH RECURSIVE\n",
" direct_deps(package, dependency) AS (\n",
" SELECT\n",
" package_name,\n",
" dep_name\n",
" FROM deps\n",
" WHERE\n",
" extra IS NULL\n",
" ),\n",
" transitive_deps(package, intermediate, dependency) AS (\n",
" SELECT\n",
" package,\n",
" package,\n",
" dependency\n",
" FROM direct_deps\n",
" UNION\n",
" SELECT\n",
" transitive_deps.package,\n",
" direct_deps.package,\n",
" direct_deps.dependency\n",
" FROM direct_deps\n",
" JOIN transitive_deps\n",
" ON direct_deps.package = transitive_deps.dependency\n",
" )\n",
" SELECT package, dependency FROM transitive_deps\n",
" \"\"\",\n",
" schema={\"package\": \"string\", \"dependency\": \"string\"}\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "0ee8a52c",
"metadata": {},
"outputs": [],
"source": [
"top_20_transitive_deps = (\n",
" transitive_deps\n",
" .group_by(\"dependency\")\n",
" .agg(n_dependents=_.package.nunique())\n",
" .order_by(ibis.desc(\"n_dependents\"))\n",
" .limit(20)\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "e533ea9f",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> dependency </span>┃<span style=\"font-weight: bold\"> n_dependents </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">string</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │\n",
"├────────────────────┼──────────────┤\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">typing-extensions </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">98812</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">idna </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">79193</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">zipp </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">75289</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">certifi </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">75101</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">urllib3 </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">74833</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">importlib-metadata</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">74184</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">charset-normalizer</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">73812</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">six </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">73315</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">requests </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">69332</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000\">numpy </span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">62662</span> │\n",
"│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">…</span> │\n",
"└────────────────────┴──────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mdependency\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mn_dependents\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩\n",
"│ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │\n",
"├────────────────────┼──────────────┤\n",
"│ \u001b[32mtyping-extensions \u001b[0m │ \u001b[1;36m98812\u001b[0m │\n",
"│ \u001b[32midna \u001b[0m │ \u001b[1;36m79193\u001b[0m │\n",
"│ \u001b[32mzipp \u001b[0m │ \u001b[1;36m75289\u001b[0m │\n",
"│ \u001b[32mcertifi \u001b[0m │ \u001b[1;36m75101\u001b[0m │\n",
"│ \u001b[32murllib3 \u001b[0m │ \u001b[1;36m74833\u001b[0m │\n",
"│ \u001b[32mimportlib-metadata\u001b[0m │ \u001b[1;36m74184\u001b[0m │\n",
"│ \u001b[32mcharset-normalizer\u001b[0m │ \u001b[1;36m73812\u001b[0m │\n",
"│ \u001b[32msix \u001b[0m │ \u001b[1;36m73315\u001b[0m │\n",
"│ \u001b[32mrequests \u001b[0m │ \u001b[1;36m69332\u001b[0m │\n",
"│ \u001b[32mnumpy \u001b[0m │ \u001b[1;36m62662\u001b[0m │\n",
"│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n",
"└────────────────────┴──────────────┘"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"top_20_transitive_deps"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment