How Python runs all 3 written

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23import ast, dis

source = "total = a + 1"

# 1. Source -> tree. This is the stage that raises SyntaxError.
tree = ast.parse(source)
print("the tree:")
print(ast.dump(tree, indent=2))

# 2. Tree -> code object.
code = compile(tree, "<string>", "exec")
print()
print("names it refers to:", code.co_names)
print("constants it holds:", code.co_consts)

# 3. Code object -> instructions.
print()
dis.dis(code)

kinds = {type(node).__name__ for node in ast.walk(tree)}
test("the tree contains an assignment and an addition", {"Assign", "BinOp"} <= kinds, True)
test("both names ended up in co_names", set(code.co_names), {"a", "total"})
test("and the literal 1 in co_consts", 1 in code.co_consts, True)

the tree:
Module(
  body=[
    Assign(
      targets=[
        Name(id='total', ctx=Store())],
      value=BinOp(
        left=Name(id='a', ctx=Load()),
        op=Add(),
        right=Constant(value=1)))])

names it refers to: ('a', 'total')
constants it holds: (1, None)

  0           RESUME                   0

  1           LOAD_NAME                0 (a)
              LOAD_SMALL_INT           1
              BINARY_OP                0 (+)
              STORE_NAME               1 (total)
              LOAD_CONST               1 (None)
              RETURN_VALUE
ok   the tree contains an assignment and an addition
ok   both names ended up in co_names
ok   and the literal 1 in co_consts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18import dis, sys

def rate(amount, percent):
    return amount * percent / 100

print("this is CPython", ".".join(map(str, sys.version_info[:3])), "- the names below are its names")
print()
dis.dis(rate)

print()
for ins in dis.get_instructions(rate):
    print(f"  {ins.opname:<34} argval={ins.argval!r}")

names = [i.opname for i in dis.get_instructions(rate)]
test("it starts with RESUME and ends with RETURN_VALUE", (names[0], names[-1]), ("RESUME", "RETURN_VALUE"))
test("the two arguments are loaded", any("LOAD_FAST" in n for n in names), True)
test("100 is a small integer, loaded directly", "LOAD_SMALL_INT" in names, True)
test("and there are two arithmetic operations", names.count("BINARY_OP"), 2)

this is CPython 3.14.7 - the names below are its names

  3           RESUME                   0

  4           LOAD_FAST_BORROW_LOAD_FAST_BORROW 1 (amount, percent)
              BINARY_OP                5 (*)
              LOAD_SMALL_INT         100
              BINARY_OP               11 (/)
              RETURN_VALUE

  RESUME                             argval=0
  LOAD_FAST_BORROW_LOAD_FAST_BORROW  argval=('amount', 'percent')
  BINARY_OP                          argval=5
  LOAD_SMALL_INT                     argval=100
  BINARY_OP                          argval=11
  RETURN_VALUE                       argval=None
ok   it starts with RESUME and ends with RETURN_VALUE
ok   the two arguments are loaded
ok   100 is a small integer, loaded directly
ok   and there are two arithmetic operations

1
2
3
4
5
6
7
8
9
10
11
12import dis

code = compile("x = 2 ** 10", "<string>", "exec")
print("source: x = 2 ** 10")
dis.dis(code)
print("co_consts:", code.co_consts)
print()
print("1024 is in there as a constant. No exponentiation happens when this runs.")

names = [i.opname for i in dis.get_instructions(code)]
test("the result was folded in", 1024 in code.co_consts, True)
test("and there is no BINARY_OP left to do", "BINARY_OP" in names, False)

source: x = 2 ** 10
  0           RESUME                   0

  1           LOAD_CONST               2 (1024)
              STORE_NAME               0 (x)
              LOAD_CONST               1 (None)
              RETURN_VALUE
co_consts: (2, None, 1024)

1024 is in there as a constant. No exponentiation happens when this runs.
ok   the result was folded in
ok   and there is no BINARY_OP left to do

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21import dis

code = compile("a = 1000\nb = 1000\nsame = a is b", "<string>", "exec")
print("co_consts:", code.co_consts)
print("1000 appears once, not twice.")
print()

# Which is why this is True, and why it is not something to rely on.
namespace = {}
exec(code, namespace)
print("a is b ->", namespace["same"])

# Build one at runtime instead and the sharing disappears.
thousand = 1000
runtime = int("1000")
print("int(\"1000\") is the literal 1000 ->", runtime is thousand)

test("the constant was stored once", code.co_consts.count(1000), 1)
test("so the two literals are one object", namespace["same"], True)
test("but a value computed at runtime is a different object", runtime is thousand, False)
test("while still being equal", runtime == thousand, True)

co_consts: (1000, None)
1000 appears once, not twice.

a is b -> True
int("1000") is the literal 1000 -> False
ok   the constant was stored once
ok   so the two literals are one object
ok   but a value computed at runtime is a different object
ok   while still being equal

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22def greet(name, greeting="hi", *rest, loud=False, **extra):
    local = 1
    return greeting

c = greet.__code__

print("co_name:           ", c.co_name)
print("co_argcount:       ", c.co_argcount, "  (positional, not counting *rest)")
print("co_kwonlyargcount: ", c.co_kwonlyargcount)
print("co_varnames:       ", c.co_varnames)
print("co_consts:         ", c.co_consts)
print("co_stacksize:      ", c.co_stacksize, "  (deepest the value stack gets)")
print("co_code:           ", f"{len(c.co_code)} raw bytes of {type(c.co_code).__name__}")
print("takes *args?       ", bool(c.co_flags & 0x04))
print("takes **kwargs?    ", bool(c.co_flags & 0x08))

test("two positional parameters", c.co_argcount, 2)
test("one keyword-only parameter", c.co_kwonlyargcount, 1)
test("locals and parameters share one tuple", "local" in c.co_varnames, True)
test("in declaration order, with *args and **kwargs after the keyword-only ones",
     c.co_varnames, ("name", "greeting", "loud", "rest", "extra", "local"))
test("the instructions themselves are bytes", isinstance(c.co_code, bytes), True)

co_name:            greet
co_argcount:        2   (positional, not counting *rest)
co_kwonlyargcount:  1
co_varnames:        ('name', 'greeting', 'loud', 'rest', 'extra', 'local')
co_consts:          (1,)
co_stacksize:       1   (deepest the value stack gets)
co_code:            10 raw bytes of bytes
takes *args?        True
takes **kwargs?     True
ok   two positional parameters
ok   one keyword-only parameter
ok   locals and parameters share one tuple
ok   in declaration order, with *args and **kwargs after the keyword-only ones
ok   the instructions themselves are bytes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22import dis

def guarded(x):
    try:
        return 10 / x
    except ZeroDivisionError:
        return None

names = [i.opname for i in dis.get_instructions(guarded)]
print("instructions on the happy path, in order:")
print(" ", names[:6])
print()
print("nothing there sets up the try. The handler is here instead:")
print(" ", guarded.__code__.co_exceptiontable.hex())
print()
print("guarded(5) ->", guarded(5))
print("guarded(0) ->", guarded(0))

test("there is no SETUP_FINALLY or SETUP_EXCEPT instruction",
     any(n.startswith("SETUP_") for n in names), False)
test("the exception table is not empty", len(guarded.__code__.co_exceptiontable) > 0, True)
test("and the handler still works", guarded(0), None)

instructions on the happy path, in order:
  ['RESUME', 'NOP', 'LOAD_SMALL_INT', 'LOAD_FAST_BORROW', 'BINARY_OP', 'RETURN_VALUE']

nothing there sets up the try. The handler is here instead:
  82080b008b0b1a0399011a03

guarded(5) -> 2.0
guarded(0) -> None
ok   there is no SETUP_FINALLY or SETUP_EXCEPT instruction
ok   the exception table is not empty
ok   and the handler still works

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22import dis

def add(a, b):
    return a + b

cold = [i.opname for i in dis.get_instructions(add, adaptive=True)]
print("straight after definition:")
print(" ", cold)

for _ in range(100):
    add(1, 2)                  # always ints

warm = [i.opname for i in dis.get_instructions(add, adaptive=True)]
print()
print("after 100 calls with integers:")
print(" ", warm)
print()
dis.dis(add, adaptive=True)

test("it specialised", cold != warm, True)
test("the generic add was replaced by an integer add", any("BINARY_OP_ADD_INT" in n for n in warm), True)
test("and it still computes the same thing", add(2, 3), 5)

straight after definition:
  ['RESUME', 'LOAD_FAST_BORROW_LOAD_FAST_BORROW', 'BINARY_OP', 'RETURN_VALUE']

after 100 calls with integers:
  ['RESUME_CHECK', 'LOAD_FAST_BORROW_LOAD_FAST_BORROW', 'BINARY_OP_ADD_INT', 'RETURN_VALUE']

  3           RESUME_CHECK             0

  4           LOAD_FAST_BORROW_LOAD_FAST_BORROW 1 (a, b)
              BINARY_OP_ADD_INT        0 (+)
              RETURN_VALUE
ok   it specialised
ok   the generic add was replaced by an integer add
ok   and it still computes the same thing

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19import dis

def make(xs):
    return [x * 2 for x in xs]

nested = [c for c in make.__code__.co_consts if hasattr(c, "co_name")]
print("nested code objects inside make:", nested)
print()
names = [i.opname for i in dis.get_instructions(make)]
print("the comprehension's own loop, inlined:")
print(" ", [n for n in names if n in {"GET_ITER", "FOR_ITER", "LIST_APPEND", "JUMP_BACKWARD"}])
print()
print("and no CALL, because nothing is called:")
print(" ", "CALL" in names)

test("there is no hidden function to call", nested, [])
test("the loop is right there in make", "FOR_ITER" in names, True)
test("nothing is invoked", any(n.startswith("CALL") for n in names), False)
test("the comprehension variable does not escape", "x" not in globals(), True)

nested code objects inside make: []

the comprehension's own loop, inlined:
  ['GET_ITER', 'FOR_ITER', 'LIST_APPEND', 'JUMP_BACKWARD']

and no CALL, because nothing is called:
  False
ok   there is no hidden function to call
ok   the loop is right there in make
ok   nothing is invoked
ok   the comprehension variable does not escape

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15import importlib.util, py_compile, sys
from pathlib import Path

Path("mod.py").write_text("VALUE = 1 + 2\n", encoding="utf-8")
py_compile.compile("mod.py", cfile="mod.pyc", doraise=True)
raw = Path("mod.pyc").read_bytes()

print("the name Python would choose:", Path(importlib.util.cache_from_source("mod.py")).name)
print("this interpreter's cache tag: ", sys.implementation.cache_tag)
print("first four bytes:             ", raw[:4].hex(), "(the magic number)")
print("bytes in total:               ", len(raw))

test("the file starts with this build's magic number", raw[:4], importlib.util.MAGIC_NUMBER)
test("and its name carries the version", sys.implementation.cache_tag in
     Path(importlib.util.cache_from_source("mod.py")).name, True)

the name Python would choose: mod.cpython-314.pyc
this interpreter's cache tag:  cpython-314
first four bytes:              2b0e0d0a (the magic number)
bytes in total:                123
ok   the file starts with this build's magic number
ok   and its name carries the version

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25import dis, timeit

def with_append(n):
    out = []
    for i in range(n):
        out.append(i * 2)
    return out

def with_comprehension(n):
    return [i * 2 for i in range(n)]

append_count = len(list(dis.get_instructions(with_append)))
comp_count = len(list(dis.get_instructions(with_comprehension)))

append_time = timeit.timeit(lambda: with_append(2000), number=200)
comp_time = timeit.timeit(lambda: with_comprehension(2000), number=200)

print(f"append loop:   {append_count} instructions, {append_time * 1000:.1f} ms")
print(f"comprehension: {comp_count} instructions, {comp_time * 1000:.1f} ms")
print()
print("more instructions, less time. Counting them predicts nothing.")

test("the comprehension has more instructions", comp_count > append_count, True)
test("and is still faster", comp_time < append_time, True)
test("both produce the same list", with_append(5), with_comprehension(5))

append loop:   21 instructions, 22.5 ms
comprehension: 25 instructions, 19.0 ms

more instructions, less time. Counting them predicts nothing.
ok   the comprehension has more instructions
ok   and is still faster
ok   both produce the same list

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26import sys

original = [1, 2, 3]
alias = original                 # a second label, not a second list

shared = original is alias
print("same object?", shared)
print("same id?    ", id(original) == id(alias))
print("references: ", sys.getrefcount(original), "(one of those is getrefcount's own argument)")

alias.append(4)
print()
print("appended through alias, and original says:", original)

# Rebinding a name moves the label. It does not touch the object.
alias = [9]
print()
print("after alias = [9]:")
print("  original:", original)
print("  alias:   ", alias)
print("  still one object?", original is alias)

test("assignment created a second name for one object", shared, True)
test("mutation through either name is visible through both", original, [1, 2, 3, 4])
test("rebinding only moves the label", alias, [9])
test("after which they are different objects", original is alias, False)

same object? True
same id?     True
references:  3 (one of those is getrefcount's own argument)

appended through alias, and original says: [1, 2, 3, 4]

after alias = [9]:
  original: [1, 2, 3, 4]
  alias:    [9]
  still one object? False
ok   assignment created a second name for one object
ok   mutation through either name is visible through both
ok   rebinding only moves the label
ok   after which they are different objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18import sys

data = [1, 2, 3]
one_name = sys.getrefcount(data)
print("refs:", one_name)

keep = data
two_names = sys.getrefcount(data)
print("after keep = data:", two_names)

del keep
after_del = sys.getrefcount(data)
print("after del keep:   ", after_del)
print("the list is still here:", data)

test("a second name raised the count by one", two_names - one_name, 1)
test("and del put it back", after_del, one_name)
test("without destroying the object", data, [1, 2, 3])

refs: 2
after keep = data: 3
after del keep:    2
the list is still here: [1, 2, 3]
ok   a second name raised the count by one
ok   and del put it back
ok   without destroying the object

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19import copy

order = {"id": 1, "items": ["book"], "address": {"city": "Lagos"}}

shallow = copy.copy(order)
deep = copy.deepcopy(order)

shallow["items"].append("pen")          # reaches into the original
shallow["id"] = 99                      # does not: rebinding a key

print("original:", order)
print("shallow: ", shallow)
print("deep:    ", deep)

test("the shallow copy shares the inner list", order["items"], ["book", "pen"])
test("the deep copy did not", deep["items"], ["book"])
test("replacing a top-level key affects only the copy", order["id"], 1)
test("shared inner objects are the same object", order["address"] is shallow["address"], True)
test("but not in the deep copy", order["address"] is deep["address"], False)

original: {'id': 1, 'items': ['book', 'pen'], 'address': {'city': 'Lagos'}}
shallow:  {'id': 99, 'items': ['book', 'pen'], 'address': {'city': 'Lagos'}}
deep:     {'id': 1, 'items': ['book'], 'address': {'city': 'Lagos'}}
ok   the shallow copy shares the inner list
ok   the deep copy did not
ok   replacing a top-level key affects only the copy
ok   shared inner objects are the same object
ok   but not in the deep copy

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17import copy

grid = [[0, 0], [0, 0]]

sliced = grid[:]
constructed = list(grid)
deep = copy.deepcopy(grid)

sliced[0][0] = 1                        # mutates a row shared with grid

print("grid after mutating sliced[0][0]:", grid)
print("constructed sees it too:         ", constructed)
print("deep does not:                   ", deep)

test("a slice copies the outer list only", grid, [[1, 0], [0, 0]])
test("list() is the same kind of copy", constructed[0] is grid[0], True)
test("deepcopy copied the rows as well", deep[0] is grid[0], False)

grid after mutating sliced[0][0]: [[1, 0], [0, 0]]
constructed sees it too:          [[1, 0], [0, 0]]
deep does not:                    [[0, 0], [0, 0]]
ok   a slice copies the outer list only
ok   list() is the same kind of copy
ok   deepcopy copied the rows as well

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26import sys

literal_a = (1, 2, 3)
literal_b = (1, 2, 3)
built = tuple([1, 2, 3])

print("two identical tuple literals:", literal_a is literal_b)
print("one built at runtime:        ", literal_a is built)
print("equal either way:            ", literal_a == built)
print("interchangeable as dict keys:", len({literal_a: "x", built: "y"}))

words = "hello"
same_literal = "hello"
joined = "".join(["h", "e", "l", "l", "o"])

print()
print("two string literals:        ", words is same_literal)
print("a string built at runtime:  ", words is joined)
print("the same string, interned:  ", sys.intern(joined) is words)

test("equal values may or may not be one object", literal_a is built, False)
test("two string literals are shared", words is same_literal, True)
test("a runtime-built string is not", words is joined, False)
test("equality is the thing that is reliable", literal_a == built, True)
test("hashing follows equality, not identity", hash(literal_a) == hash(built), True)
test("so they are the same dict key", len({literal_a: "x", built: "y"}), 1)

two identical tuple literals: True
one built at runtime:         False
equal either way:             True
interchangeable as dict keys: 1

two string literals:         True
a string built at runtime:   False
the same string, interned:   True
ok   equal values may or may not be one object
ok   two string literals are shared
ok   a runtime-built string is not
ok   equality is the thing that is reliable
ok   hashing follows equality, not identity
ok   so they are the same dict key

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22import sys, tracemalloc

# Every element built at runtime, so every element is a distinct object.
rows = [f"row-{i}-" + "x" * 1000 for i in range(1000)]

print("getsizeof(the list):", sys.getsizeof(rows), "bytes")
print("getsizeof(one row): ", sys.getsizeof(rows[0]), "bytes")
print()

tracemalloc.start()
base = tracemalloc.get_traced_memory()[0]
again = [f"row-{i}-" + "y" * 1000 for i in range(1000)]
real = tracemalloc.get_traced_memory()[0] - base
tracemalloc.stop()

print("what tracemalloc says the same data costs:", round(real / 1024), "KiB")
print("what getsizeof would have told you:       ", round(sys.getsizeof(again) / 1024), "KiB")
print("out by a factor of", round(real / sys.getsizeof(again)))

test("the list itself is small", sys.getsizeof(rows) < 20_000, True)
test("while the data it refers to is not", real > 1_000_000, True)
test("getsizeof does not follow references", sys.getsizeof(again) < real / 10, True)

getsizeof(the list): 8856 bytes
getsizeof(one row):  1047 bytes

what tracemalloc says the same data costs: 1033 KiB
what getsizeof would have told you:        9 KiB
out by a factor of 119
ok   the list itself is small
ok   while the data it refers to is not
ok   getsizeof does not follow references

1
2
3
4
5
6
7
8
9
10
11folded = ["b" * 1000 for _ in range(3)]
runtime = [str(i) + "b" * 999 for i in range(3)]

print("elements of the folded list, all the same object?", folded[0] is folded[1] is folded[2])
print("elements built at runtime?                       ", runtime[0] is runtime[1])
print("both lists hold three strings of length", len(folded[0]), "and", len(runtime[0]))

test('"b" * 1000 was computed at compile time, so one object is reused',
     folded[0] is folded[2], True)
test("which means that list costs one string, not three", len(set(map(id, folded))), 1)
test("building the element at runtime gives three", len(set(map(id, runtime))), 3)

elements of the folded list, all the same object? True
elements built at runtime?                        False
both lists hold three strings of length 1000 and 1000
ok   "b" * 1000 was computed at compile time, so one object is reused
ok   which means that list costs one string, not three
ok   building the element at runtime gives three

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32import tracemalloc

class Plain:
    def __init__(self, x, y):
        self.x, self.y = x, y

class Slotted:
    __slots__ = ("x", "y")
    def __init__(self, x, y):
        self.x, self.y = x, y

def cost(cls, n=20_000):
    tracemalloc.start()
    base = tracemalloc.get_traced_memory()[0]
    objects = [cls(i, i) for i in range(n)]
    used = tracemalloc.get_traced_memory()[0] - base
    tracemalloc.stop()
    del objects
    return used

plain, slotted = cost(Plain), cost(Slotted)

print("20,000 plain instances:  ", round(plain / 1024), "KiB")
print("20,000 slotted instances:", round(slotted / 1024), "KiB")
print("saved:", round(100 * (1 - slotted / plain)), "%")
print()
print("a plain instance has a __dict__:  ", hasattr(Plain(1, 2), "__dict__"))
print("a slotted one does not:           ", hasattr(Slotted(1, 2), "__dict__"))

test("slots really used less", slotted < plain, True)
test("the saving is substantial", slotted < plain * 0.85, True)
test("and the dict is gone", hasattr(Slotted(1, 2), "__dict__"), False)

20,000 plain instances:   2507 KiB
20,000 slotted instances: 1720 KiB
saved: 31 %

a plain instance has a __dict__:   True
a slotted one does not:            False
ok   slots really used less
ok   the saving is substantial
ok   and the dict is gone

1
2
3
4
5
6
7class Point:
    __slots__ = ("x", "y")
    def __init__(self, x, y):
        self.x, self.y = x, y

p = Point(1, 2)
p.z = 3

Traceback (most recent call last):
  File "example.py", line 7, in <module>
    p.z = 3
    ^^^
AttributeError: 'Point' object has no attribute 'z' and no __dict__ for setting new attributes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30import gc

freed = []

class Node:
    def __init__(self, name):
        self.name = name
        self.peer = None
    def __del__(self):
        freed.append(self.name)

# No cycle: the last reference goes, and it is freed on the spot.
solo = Node("solo")
del solo
print("after del solo, freed:", freed)

# A cycle: each refers to the other, so neither count reaches zero.
a, b = Node("a"), Node("b")
a.peer, b.peer = b, a
del a, b
print("after del a, b:       ", freed, "-- still alive, despite no way to reach them")

collected = gc.collect()
print("after gc.collect():   ", sorted(freed))
print()
print("gc thresholds on this build:", gc.get_threshold())

test("refcounting freed the un-cycled object immediately", "solo" in freed, True)
test("the cycle needed the collector", collected >= 2, True)
test("which then freed both", sorted(freed), ["a", "b", "solo"])

after del solo, freed: ['solo']
after del a, b:        ['solo'] -- still alive, despite no way to reach them
after gc.collect():    ['a', 'b', 'solo']

gc thresholds on this build: (2000, 10, 10)
ok   refcounting freed the un-cycled object immediately
ok   the cycle needed the collector
ok   which then freed both

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30import gc, weakref

class Session:
    def __init__(self, user):
        self.user = user

strong = {}
weak = weakref.WeakValueDictionary()

for name in ("ada", "alan", "grace"):
    session = Session(name)
    strong[name] = session
    weak[name] = session

print("both hold:", len(strong), len(weak))

strong.clear()            # drop the strong dict's references
gc.collect()
after_clear = len(weak)
print("after clearing the strong dict:", after_clear)
print("  one survivor: the loop variable 'session' still names the last one")

del session               # the last strong reference in the program
gc.collect()
print("after del session:             ", len(weak))
print("the weak dict emptied itself, with nothing told to do it")

test("the strong cache is empty because we emptied it", len(strong), 0)
test("a single surviving reference keeps one entry alive", after_clear, 1)
test("and when that goes, so does the entry", len(weak), 0)

both hold: 3 3
after clearing the strong dict: 1
  one survivor: the loop variable 'session' still names the last one
after del session:              0
the weak dict emptied itself, with nothing told to do it
ok   the strong cache is empty because we emptied it
ok   a single surviving reference keeps one entry alive
ok   and when that goes, so does the entry

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26numbers = [1, 2, 3, 4, 5, 6]
visited = []

for n in numbers:
    visited.append(n)
    if n % 2 == 0:
        numbers.remove(n)      # the loop's internal index does not know

print("the loop visited:", visited)
print("the list is now: ", numbers)
print("3 and 5 were never visited, and nothing complained.")

mapping = {"a": 1, "b": 2}
try:
    for key in mapping:
        mapping[key + "!"] = 0
    complaint = None
except RuntimeError as exc:
    complaint = str(exc)

print()
print("a dict does complain:", complaint)

test("the list loop skipped items", visited, [1, 2, 4, 6])
test("which is not the whole list", set(visited) != {1, 2, 3, 4, 5, 6}, True)
test("a dict refuses instead", complaint, "dictionary changed size during iteration")

the loop visited: [1, 2, 4, 6]
the list is now:  [1, 3, 5]
3 and 5 were never visited, and nothing complained.

a dict does complain: dictionary changed size during iteration
ok   the list loop skipped items
ok   which is not the whole list
ok   a dict refuses instead

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18numbers = [1, 2, 3, 4, 5, 6]

# Build a new list rather than editing the one you are reading.
odds = [n for n in numbers if n % 2]

# Or, if it must be in place, iterate over a copy.
in_place = [1, 2, 3, 4, 5, 6]
for n in list(in_place):
    if n % 2 == 0:
        in_place.remove(n)

print("comprehension:", odds)
print("copy then edit:", in_place)
print("original untouched:", numbers)

test("the comprehension is correct and leaves the source alone", odds, [1, 3, 5])
test("and iterating a copy is correct too", in_place, [1, 3, 5])
test("the source list is unchanged", numbers, [1, 2, 3, 4, 5, 6])

comprehension: [1, 3, 5]
copy then edit: [1, 3, 5]
original untouched: [1, 2, 3, 4, 5, 6]
ok   the comprehension is correct and leaves the source alone
ok   and iterating a copy is correct too
ok   the source list is unchanged

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15import time

start_wall, start_cpu = time.perf_counter(), time.process_time()
time.sleep(0.1)                       # waiting, not working
total = sum(i * i for i in range(200_000))    # working
wall = time.perf_counter() - start_wall
cpu = time.process_time() - start_cpu

print(f"perf_counter (wall clock): {wall:.3f}s")
print(f"process_time (cpu only):   {cpu:.3f}s")
print(f"time spent waiting:        {wall - cpu:.3f}s")

test("the wall clock saw the sleep", wall > 0.1, True)
test("the cpu clock did not", cpu < wall, True)
test("the gap is roughly the sleep", 0.08 < wall - cpu < 0.2, True)

perf_counter (wall clock): 0.123s
process_time (cpu only):   0.019s
time spent waiting:        0.105s
ok   the wall clock saw the sleep
ok   the cpu clock did not
ok   the gap is roughly the sleep

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15import timeit

setup = "import random; random.seed(0); data = [random.random() for _ in range(1000)]"
runs = timeit.repeat("sorted(data)", setup=setup, number=100, repeat=5)

print("five batches of 100 sorts each:")
for r in runs:
    print(f"   {r * 1000:6.2f} ms")
print()
print(f"min: {min(runs) * 1000:.2f} ms   <- quote this")
print(f"max: {max(runs) * 1000:.2f} ms   <- {max(runs) / min(runs):.2f}x the min, and that is noise")

test("five measurements were taken", len(runs), 5)
test("they are not all identical", len(set(runs)) > 1, True)
test("and the spread is noise, not signal", max(runs) / min(runs) < 3, True)

five batches of 100 sorts each:
     9.06 ms
     8.49 ms
     8.82 ms
     8.35 ms
     8.42 ms

min: 8.35 ms   <- quote this
max: 9.06 ms   <- 1.09x the min, and that is noise
ok   five measurements were taken
ok   they are not all identical
ok   and the spread is noise, not signal

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29import cProfile, io, pstats

def parse(line):
    return line.strip().split(",")

def total(rows):
    return sum(int(row[1]) for row in rows)

def report(lines):
    rows = [parse(line) for line in lines]
    return total(rows)

lines = [f"row{i},{i}" for i in range(20_000)]

profiler = cProfile.Profile()
profiler.enable()
answer = report(lines)
profiler.disable()

buffer = io.StringIO()
stats = pstats.Stats(profiler, stream=buffer).sort_stats("tottime")
stats.print_stats(5)
print("\n".join(buffer.getvalue().splitlines()[2:]))

by_name = {key[2]: value for key, value in stats.stats.items()}

test("the report is correct", answer, sum(range(20_000)))
test("parse was called once per line", by_name["parse"][0], 20_000)
test("and report only once", by_name["report"][0], 1)

   Ordered by: internal time
   List reduced from 8 to 5 due to restriction <5>

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
    20000    0.013    0.000    0.023    0.000 example.py:3(parse)
    20000    0.008    0.000    0.008    0.000 {method 'split' of 'str' objects}
    20001    0.005    0.000    0.005    0.000 example.py:7(<genexpr>)
        1    0.004    0.004    0.036    0.036 example.py:9(report)
        1    0.003    0.003    0.008    0.008 {built-in method builtins.sum}


ok   the report is correct
ok   parse was called once per line
ok   and report only once

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26import time

known = [f"user{i}" for i in range(5000)]
wanted = [f"user{i}" for i in range(0, 5000, 5)]

def check_slow(names, wanted):
    return [w for w in wanted if w in names]          # scans the list every time

def check_fast(names, wanted):
    lookup = set(names)                               # hashed once
    return [w for w in wanted if w in lookup]

start = time.perf_counter(); slow_result = check_slow(known, wanted)
slow = time.perf_counter() - start

start = time.perf_counter(); fast_result = check_fast(known, wanted)
fast = time.perf_counter() - start

print(f"membership in a list: {slow * 1000:7.1f} ms")
print(f"membership in a set:  {fast * 1000:7.1f} ms")
print(f"speedup: {slow / fast:.0f}x")
print()
print("one line changed. Nothing was rewritten in C, and nothing was made harder to read.")

test("both give the same answer", fast_result, slow_result)
test("and the set version is far faster", slow / fast > 5, True)

membership in a list:    46.3 ms
membership in a set:      0.7 ms
speedup: 62x

one line changed. Nothing was rewritten in C, and nothing was made harder to read.
ok   both give the same answer
ok   and the set version is far faster

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24import timeit

setup = "data = list(range(1000))"

styles = {
    "list comprehension": "[x * 2 for x in data]",
    "map with a lambda":  "list(map(lambda x: x * 2, data))",
    "explicit append":    "out = []\nfor x in data:\n    out.append(x * 2)",
}

results = {}
for label, code in styles.items():
    results[label] = min(timeit.repeat(code, setup=setup, number=200, repeat=3))

slowest = max(results.values())
fastest = min(results.values())
for label, seconds in sorted(results.items(), key=lambda kv: kv[1]):
    print(f"  {label:<20} {seconds * 1000:6.1f} ms")
print()
print(f"best to worst: {slowest / fastest:.1f}x")
print("against the 50x above. This is not where your time is.")

test("all three styles were measured", len(results), 3)
test("and the spread between them is small", slowest / fastest < 5, True)

  list comprehension      6.9 ms
  explicit append         7.0 ms
  map with a lambda      12.7 ms

best to worst: 1.9x
against the 50x above. This is not where your time is.
ok   all three styles were measured
ok   and the spread between them is small

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49import time

def plain(n):
    out = ""
    for _ in range(n):
        out += "abcdefghij"
    return len(out)

def holding_a_reference(n):
    out = ""
    history = []
    for _ in range(n):
        history.append(out)          # a second name for the string being built
        out += "abcdefghij"
    return len(out)

def joined(n):
    parts = ["abcdefghij"] * n           # the parts, as you would really have them
    return len("".join(parts))

def elapsed(fn, n, repeat=3):
    # Best of three, for the reason the panel above gave.
    def once():
        start = time.perf_counter()
        fn(n)
        return time.perf_counter() - start
    return min(once() for _ in range(repeat))

print(f"{'n':>7}  {'plain +=':>11}  {'+= with a kept ref':>20}  {'join':>10}")
for n in (5_000, 20_000):
    print(f"{n:>7}  {elapsed(plain, n) * 1000:>9.1f}ms  "
          f"{elapsed(holding_a_reference, n) * 1000:>18.1f}ms  "
          f"{elapsed(joined, n) * 1000:>8.2f}ms")

# Growth rate: quadruple the work and see what happens to the time. Different
# ranges, because the quadratic one cannot be run at the sizes the others can.
def growth(fn, small, large):
    return elapsed(fn, large) / elapsed(fn, small)

plain_growth = growth(plain, 20_000, 80_000)
held_growth = growth(holding_a_reference, 2_500, 10_000)

print()
print(f"plain +=,   n 20,000 -> 80,000 (4x the work): {plain_growth:5.1f}x the time  -> linear")
print(f"kept a ref, n  2,500 -> 10,000 (4x the work): {held_growth:5.1f}x the time  -> quadratic")

test("all three build the same string length", (plain(10), joined(10)), (100, 100))
test("plain += grew about as fast as the input", plain_growth < 8, True)
test("holding a reference made it grow with the square", held_growth > 8, True)

      n     plain +=    += with a kept ref        join
   5000        0.5ms                56.2ms      0.05ms
  20000        2.0ms              1072.4ms      0.24ms

plain +=,   n 20,000 -> 80,000 (4x the work):   4.0x the time  -> linear
kept a ref, n  2,500 -> 10,000 (4x the work):  42.9x the time  -> quadratic
ok   all three build the same string length
ok   plain += grew about as fast as the input
ok   holding a reference made it grow with the square

1
2
3
4
5
6
7
8
9
10
11
12
13
14import timeit

from_list = timeit.timeit("xs.pop(0)", setup="xs = list(range(20_000))", number=20_000)
from_deque = timeit.timeit("xs.popleft()",
                           setup="from collections import deque; xs = deque(range(20_000))",
                           number=20_000)

print(f"list.pop(0)      {from_list * 1000:7.1f} ms")
print(f"deque.popleft()  {from_deque * 1000:7.1f} ms")
print(f"ratio            {from_list / from_deque:7.0f}x")
print()
print("pop(0) moves every remaining element up one. deque does not have to.")

test("the deque is much faster at the front", from_deque < from_list / 5, True)

list.pop(0)         25.1 ms
deque.popleft()      0.7 ms
ratio                 34x

pop(0) moves every remaining element up one. deque does not have to.
ok   the deque is much faster at the front

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28import tracemalloc

def build_list(n):
    return [i * i for i in range(n)]

def build_generator(n):
    return (i * i for i in range(n))

tracemalloc.start()
base = tracemalloc.get_traced_memory()[0]
as_list = build_list(200_000)
list_cost = tracemalloc.get_traced_memory()[0] - base

base = tracemalloc.get_traced_memory()[0]
as_gen = build_generator(200_000)
gen_cost = tracemalloc.get_traced_memory()[0] - base
peak = tracemalloc.get_traced_memory()[1]
tracemalloc.stop()

print(f"a list of 200,000 squares:      {list_cost / 1024 / 1024:.1f} MiB")
print(f"a generator of the same thing:  {gen_cost} bytes")
print(f"peak while both existed:        {peak / 1024 / 1024:.1f} MiB")
print()
print("the generator has not computed anything yet, and never holds more than one value.")

test("the list cost megabytes", list_cost > 1_000_000, True)
test("the generator cost almost nothing", gen_cost < 1000, True)
test("and both produce the same first value", as_list[0], next(as_gen))

a list of 200,000 squares:      7.7 MiB
a generator of the same thing:  408 bytes
peak while both existed:        7.7 MiB

the generator has not computed anything yet, and never holds more than one value.
ok   the list cost megabytes
ok   the generator cost almost nothing
ok   and both produce the same first value