Skip to content

Instantly share code, notes, and snippets.

@dutc
Created April 7, 2021 20:03
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dutc/a6d10400b5427eefad5c75c4cdfa74f7 to your computer and use it in GitHub Desktop.
Save dutc/a6d10400b5427eefad5c75c4cdfa74f7 to your computer and use it in GitHub Desktop.
“Python Expert” Newsletter (Apr 4, 2021): Slides for “What Does it All Really Mean?” (https://www.youtube.com/watch?v=hYi9S0GwVB8)
! pulse
What Does It All
<i>Really</i> <b>Mean</b> <i>?</i>
! fade
What Does It All
<i>Really</i> <b>Mean</b> <i>?</i>
\
☆ PyCon India ☆ Sat Oct 3, 2020 ☆
! fade
James Powell
@dontusethiscode
@dont
use
this
code
context
do
details
matter?
details
don't
matter
details
do
matter
details →
meaning
example…
\
details
small
details
\
xs = [1, 2, 3]
\
xs = {1, 2, 3}
\
xs = 1, 2, 3
from numpy import array
xs = array([1, 2, 3])
from pandas import Series
xs = Series([1, 2, 3])
from pandas import array
xs = array([1, 2, 3])
from pandas import DataFrame
xs = DataFrame([1, 2, 3])
who cares?
\
xs = [1, 2, 3]
assert sum(xs) == 6
\
xs = {1, 2, 3}
assert sum(xs) == 6
\
xs = 1, 2, 3
assert sum(xs) == 6
from numpy import array
xs = array([1, 2, 3])
assert sum(xs) == 6
from pandas import Series
xs = Series([1, 2, 3])
assert sum(xs) == 6
from pandas import array
xs = array([1, 2, 3])
assert sum(xs) == 6
from pandas import DataFrame
xs = DataFrame([1, 2, 3])
assert sum(xs) == 6
from pandas import DataFrame
xs = DataFrame([1, 2, 3])
assert sum(xs[0]) == 6
who
cares?
! mu
what does it
all <b>mean</b>?
thinking
exercise
“thinking
exercise”
“thought
experiment”
compare
xs = [1, 2, 3]
xs = {1, 2, 3}
\
\
xs = []
xs = {}
\
\
xs = [] # list
xs = {}
\
\
xs = [] # list
xs = {} # set
\
\
xs = [] # list
xs = {} # dict
\
\
xs = [] # list
xs = {*''} # set
\
\
xs = [] # list: Sequence, Collection
xs = {*''} # set: Set
\
\
xs = [] # list: Sequence, collection
xs = {*''} # set: Set
\
\
xs = [] # list: Sequence, collection
from collections.abc import Sequence
assert isinstance(xs, Sequence)
\
xs = [] # list: Sequence, collection
xs = {*''} # set: Set
\
\
from collections.abc import Set
xs = {*''} # set: Set
assert isinstance(xs, Set)
\
from collections.abc import Sequence
xs = [1, 2, 3]
assert isinstance(xs, Sequence)
assert xs[0] == 1
from collections.abc import Sequence
xs = [1, 2, 3]
assert isinstance(xs, Sequence)
assert xs[0:1] == [1]
from collections.abc import Sequence
xs = {1: 1, 2: 2, 3: 3}
assert not isinstance(xs, Sequence)
\
from collections.abc import Sequence
xs = {1: 1, 2: 2, 3: 3}
assert not isinstance(xs, Sequence)
assert xs[1] == 1
\
xs = {slice(0, 1): 1}
\
assert xs[0:1] == 1
from collections.abc import Sequence
Sequence.register(dict)
xs = {1: 1, 2: 2, 3: 3}
assert isinstance(xs, Sequence)
from collections.abc import MutableSequence, Sequence
MutableSequence.register(dict)
xs = {1: 1, 2: 2, 3: 3}
assert isinstance(xs, MutableSequence)
assert isinstance(xs, Sequence)
from collections.abc import Set
xs = {1, 2, 3}
assert isinstance(xs, Set)
from collections.abc import Set
Set.register(type(...))
x = ...
assert isinstance(x, Set)
from collections.abc import Set
xs = {1, 2, 3}
assert isinstance(xs, Set)
assert xs & xs is not None
assert xs | xs is not None
assert xs ^ xs is not None
assert xs - xs is not None
from collections.abc import Set
x = 10
assert not isinstance(x, Set)
assert x & x is not None
assert x | x is not None
assert x ^ x is not None
assert x - x is not None
xs = {1, 2, 3}
ys = {3, 4, 5}
assert xs & ys == {3}
assert xs | ys == {1, 2, 3, 4, 5}
assert xs ^ ys == {1, 2, 4, 5}
assert xs - ys == {1, 2}
from collections import Counter
xs = {1, 2, 3}
assert all(count == 1 for count in Counter(xs).values())
meaningful?
Set?
Set!
Sequence?
Sequence!
meaningful?
meaningful!
\
xs = [1, 2, 3]
xs = 1, 2, 3
\
\
xs = [1, 2, 3] # list: Sequence, Collection
xs = 1, 2, 3 # tuple: Sequence, Collection
\
from collections.abc import Sequence, Collection
xs = [1, 2, 3] # list: Sequence, Collection
assert isinstance(xs, Sequence)
assert isinstance(xs, Collection)
from collections.abc import Sequence, Collection
xs = 1, 2, 3 # tuple: Sequence, Collection
assert isinstance(xs, Sequence)
assert isinstance(xs, Collection)
\
xs = [1, 2, 3] # list: Sequence, Collection
assert xs[0] == 1 and xs[0:1] == [1]
\
\
xs = 1, 2, 3 # list: Sequence, Collection
assert xs[0] == 1 and xs[0:1] == (1,)
\
from collections.abc import MutableSequence
xs = [1, 2, 3] # list: MutableSequence
assert isinstance(xs, MutableSequence)
\
from collections.abc import MutableSequence
xs = 1, 2, 3 # tuple: Sequence
assert not isinstance(xs, MutableSequence)
\
\
xs = [1, 2, 3]
xs[0] = 10
assert xs[0] == 10
\
\
\
xs = 1, 2, 3
try:
xs[0] = 10
except TypeError:
assert xs[0] == 1
\
xs = [1, 2, 3]
xs.append(4)
assert xs[-1] == 4
\
\
append = lambda *_: ...
xs = [1, 2, 3]
append(xs, 4)
assert xs[-1] == 4
\
\
append = lambda *_: ...
xs = [1, 2, 3]
xs = append(xs, 4)
assert xs[-1] == 4
\
\
append = lambda xs, x: [*xs, x]
xs = [1, 2, 3]
xs = append(xs, 4)
assert xs[-1] == 4
\
\
\
xs = 1, 2, 3
\
\
\
\
\
xs = 1, 2, 3
# int PyTuple_SetItem(PyObject*,
# Py_ssize_t,
# PyObject*);
\
def f():
x = ...
\
\
def f():
x = ...
from dis import dis
dis(f)
1 0 LOAD_CONST 1 (Ellipsis)
2 STORE_FAST 0 (x)
4 LOAD_CONST 0 (None)
6 RETURN_VALUE
\
2 STORE_FAST 0 (x)
\
\
case TARGET(STORE_FAST): {
PREDICTED(STORE_FAST);
PyObject *value = POP();
SETLOCAL(oparg, value);
FAST_DISPATCH();
}
case TARGET(STORE_FAST): {
\
\
SETLOCAL(oparg, value);
\
}
#define SETLOCAL(i, value) \
do { PyObject *tmp = GETLOCAL(i); \
GETLOCAL(i) = value; \
Py_XDECREF(tmp); } while (0)
#define SETLOCAL(i, value) \
\
GETLOCAL(i) = value; \
\
#define GETLOCAL(i) \
(fastlocals[i])
1 0 LOAD_CONST 1 (Ellipsis)
2 STORE_FAST 0 (x)
4 LOAD_CONST 0 (None)
6 RETURN_VALUE
\
2 STORE_FAST 0 (x)
\
\
from types import CodeType
from numpy import array
\
\
\
\
\
\
\
\
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
\
\
\
\
\
\
\
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
\
\
\
\
\
\
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
xs = array([], dtype='uint64')
\
\
\
\
\
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
xs = array([], dtype='uint64')
loc = xs.__array_interface__['data'][0]
\
\
\
\
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
xs = array([], dtype='uint64')
loc = xs.__array_interface__['data'][0]
idx = id(t) - loc
\
\
\
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
xs = array([], dtype='uint64')
loc = xs.__array_interface__['data'][0]
idx = id(t) - loc
xs = as_strided(xs, strides=(1,), shape=(idx + 1,))
\
\
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
xs = array([], dtype='uint64')
loc = xs.__array_interface__['data'][0]
idx = id(t) - loc
xs = as_strided(xs, strides=(1,), shape=(idx + 1,))
ys = as_strided(xs[idx:], strides=(8,), shape=(4,))
\
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
xs = array([], dtype='uint64')
loc = xs.__array_interface__['data'][0]
idx = id(t) - loc
xs = as_strided(xs, strides=(1,), shape=(idx + 1,))
ys = as_strided(xs[idx:], strides=(8,), shape=(4,))
zs = as_strided(ys[3:], strides=(8,), shape=(i + 1,))
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
xs = array([], dtype='uint64')
loc = xs.__array_interface__['data'][0]
idx = id(t) - loc
xs = as_strided(xs, strides=(1,), shape=(idx + 1,))
ys = as_strided(xs[idx:], strides=(8,), shape=(4,))
zs = as_strided(ys[3:], strides=(8,), shape=(i + 1,))
ys[2] += max(0, i - (end := len(t)) + 1)
zs[min(i, end):] = id(v)
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
...
t = 0, 1, 2, None, 4
assert t[3] == None
\
\
from numpy import array
from numpy.lib.stride_tricks import as_strided
def setitem(t, i, v):
...
t = 0, 1, 2, None, 4
assert t[3] == None
setitem(t, 3, 3)
assert t[3] == 3
Sequence
vs
MutableSequence
xs = [1, 2, 3] # list: Sequence, Collection
xs = 1, 2, 3 # tuple: Sequence, Collection
xs = [1, 2, 3] # list: Sequence, collection
xs = 1, 2, 3 # tuple: Sequence, …
xs = [1, 2, 3] # list: Sequence, collection
xs = 1, 2, 3 # tuple: Sequence, record
xs = [1, 2, 3]
xs.append(4)
xs.pop()
xs = [1, 2, 3]
for x in xs:
pass
xs = [1, 2, 3]
for x in xs:
f(x)
xs = 1, 2, 3
a, b, c = xs
ordered
machine-ordered
human-ordered
human-ordered
- semantically similar
- semantically dissimilar
list: homogeneous in type
tuple: heterogeneous in type
xs = [1, 2, 3] # list: collection
xs = 1, 2, 3 # tuple: record
mutable
vs
immutable
hashability ⇋
immutability
\
\
hashability (suggests)
immutability
* assuming ‘random’
(i.e., non-intermediated) access
\
hostnames = ['abc.company.com', 'xyc.company.com']
hostnames = {'abc.company.com', 'xyc.company.com'}
hostnames = 'abc.company.com', 'xyc.company.com'
\
from asyncssh import connect
hostnames = ['abc.company.com', 'xyc.company.com']
for h in hostnames:
async with connect(h) as conn:
...
from asyncssh import connect
hostnames = {'abc.company.com', 'xyc.company.com'}
for h in hostnames:
async with connect(h) as conn:
...
\
hostnames = 'abc.company.com', 'xyc.company.com'
prod, dev = hostnames
async with connect(prod) as conn:
...
\
hostnames = 'abc.company.com', 'xyc.company.com'
primary, backup = hostnames
async with connect(primary) as conn:
...
details →
clarify
a choice
details →
convey
intention
machines
don't care
about
meaning
humans
care
about
meaning
interpretation
“thinking
exercise”
collect the facts
collect the details
find the differences
find the similarities
review conventions
review expectations
apply some judgment
derive some interpretation
! mu
what is this thing
<i>really</i>?
\
! mu
what does this
<i>really</i>
<b>mean</b>?
thus…
from numpy import array
xs = array([1, 2, 3])
xs = [1, 2, 3]
✓ Sequences
\
✓ Sequences
✓ MutableSequences
- Dynamic Size: `list`
- Fixed Size: `numpy.ndarray`
- Fixed Shape: `list`
- Dynamic Shape: `numpy.ndarray`
- Linear: `list`
- N-Dimensional: `numpy.ndarray`
xs = [1, 2, 3]
xs = [1, 2, 3]
for x in xs:
f(x)
assert (2 ** 20_000).bit_length() == 20_001
- boxed
\
\
- boxed
- non-contiguous
\
- boxed
- non-contiguous
- unconstrained behaviour
from numpy import array
xs = array([1, 2, 3], dtype='int64')
- unboxed
- contiguous
- constrained behaviour
from numpy import array
xs = array([1, 2, 3], dtype='int64')
for x in xs:
f(x)
from numpy import array
xs = array([1, 2, 3], dtype='int64')
xs.f()
→ “Restricted
Computation
Domain”
`list` → “collection”
`numpy.ndarray` → “vector”
from pandas import array as pd_array
from numpy import array as np_array
xs = pd_array([1, 2, 3])
xs = np_array([1, 2, 3])
from numpy import array
xs = array([1., 2., 3.])
from numpy import array, nan
xs = array([1., 2., nan])
\
modality
encoded
modality
IEEE-754
·111 1111 1··· ···· ···· ···· ···· ····
encode a
modality
from numpy import array, nan
xs = array([1, 2, nan])
assert xs.dtype == 'float64'
\
from pandas import array, NA
xs = array([1, 2, NA])
\
\
from pandas import array, NA
xs = array([1, 2, NA])
assert xs._mask[-1] == True
from numpy import ndarray
from pandas import array, NA
xs = array([1, 2, NA])
assert xs._data[0] == 1
from numpy import ndarray
from pandas import array, NA
xs = array([1, 2, NA])
assert type(xs._data) is ndarray
from numpy import array, ndarray
from pandas import Series
xs = array([1, 2, 3])
xs = Series([1, 2, 3])
\
\
from numpy import array, ndarray
from pandas import Series
xs = array([1, 2, 3])
xs = Series([1, 2, 3])
assert type(xs.__array__()) is ndarray
\
from numpy import array, ndarray
from pandas import Series
xs = array([1, 2, 3])
ys = Series(xs)
assert xs is ys.__array__()
\
from numpy import array, ndarray
from pandas import Series
xs = array([1, 2, 3])
ys = Series(xs)
xs[0] = 10
assert xs[0] == ys[0] == 10
from pandas import Series
xs = Series([1, 2, 3], index=[10, 20, 30])
\
from pandas import Series
xs = Series([1, 2, 3], index=[*'abc'])
\
from pandas import Series
xs = Series([1, 2, 3], index=[2, 1, 0])
\
modality
from pandas import Series
xs = Series([1, 2, 3], index=[2, 1, 0])
assert xs.iloc[0] == 1
from pandas import Series
xs = Series([1, 2, 3], index=[2, 1, 0])
assert xs.loc[0] == 3
from pandas import Index
- hierarchy
\
- hierarchy
- monotonicity
from pandas import DataFrame
xs = DataFrame([1, 2, 3])
\
\
from pandas import DataFrame
xs = DataFrame(zip([1, 2, 3], [4, 5, 6]))
\
\
from pandas import DataFrame
xs = DataFrame(zip([1, 2, 3], [4, 5, 6]))
assert (xs.index == [0, 1, 2]).all()
assert (xs.columns == [0, 1]).all()
from pandas import DataFrame
xs = DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]},
index=[*'xyz'])
\
from pandas import DataFrame
xs = DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]},
index=[*'xyz'])
xs.stack()
from pandas import DataFrame
xs = DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]},
index=[*'xyz'])
xs.stack().droplevel(0)
“thinking
exercise”
do
details
matter?
details
do
matter!
details
meaning
Why should
I care?
How does
this improve
my life?
! fade
! mu
What Does It All
<i>Really</i> <b>Mean</b> <i>?</i>
☆ PyCon India ☆ Sat Oct 3, 2020 ☆
☆ James Powell // @dontusethiscode ☆
@dutc
Copy link
Author

dutc commented Apr 7, 2021

For the full write-up and discussion, sign up for the “Python Expert” newsletter!

bit.ly/expert-python

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment