The deep end all 2 written

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19class Constant:
    """The smallest possible descriptor."""
    def __get__(self, obj, objtype=None):
        return 42

class Circle:
    answer = Constant()

c = Circle()
print("c.answer:      ", c.answer)
print("what is actually stored in the class:", type(Circle.__dict__["answer"]).__name__)
print("the instance dict is empty:", c.__dict__)
print()
print("calling the protocol by hand:", Circle.__dict__["answer"].__get__(c, Circle))

test("attribute access ran __get__", c.answer, 42)
test("nothing was stored on the instance", c.__dict__, {})
test("what the class holds is the descriptor, not the value",
     type(Circle.__dict__["answer"]).__name__, "Constant")

c.answer:       42
what is actually stored in the class: Constant
the instance dict is empty: {}

calling the protocol by hand: 42
ok   attribute access ran __get__
ok   nothing was stored on the instance
ok   what the class holds is the descriptor, not the value

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30class Account:
    def __init__(self, balance):
        self._balance = balance

    @property
    def balance(self):
        return self._balance

    @balance.setter
    def balance(self, value):
        if value < 0:
            raise ValueError("balance cannot be negative")
        self._balance = value

held = Account.__dict__["balance"]

print("what @property produced:", type(held).__name__)
print("defines __get__:", hasattr(held, "__get__"))
print("defines __set__:", hasattr(held, "__set__"))
print("defines __delete__:", hasattr(held, "__delete__"))
print()
a = Account(10)
print("a.balance:", a.balance)
print("by hand:  ", held.__get__(a, Account))

test("property is an ordinary object in the class dict", type(held).__name__, "property")
test("it implements all three hooks",
     (hasattr(held, "__get__"), hasattr(held, "__set__"), hasattr(held, "__delete__")),
     (True, True, True))
test("and the protocol can be driven manually", held.__get__(a, Account), 10)

what @property produced: property
defines __get__: True
defines __set__: True
defines __delete__: True

a.balance: 10
by hand:   10
ok   property is an ordinary object in the class dict
ok   it implements all three hooks
ok   and the protocol can be driven manually

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30class Data:
    def __get__(self, obj, objtype=None):
        return "from the data descriptor"
    def __set__(self, obj, value):
        obj.__dict__["stored"] = value

class NonData:
    def __get__(self, obj, objtype=None):
        return "from the non-data descriptor"

class Thing:
    a = Data()
    b = NonData()

t = Thing()
print("before anything is in the instance dict:")
print("  t.a ->", t.a)
print("  t.b ->", t.b)

t.__dict__["a"] = "from the instance dict"
t.__dict__["b"] = "from the instance dict"

print()
print("after writing both names straight into t.__dict__:")
print("  t.a ->", t.a)
print("  t.b ->", t.b)

test("a data descriptor cannot be shadowed by the instance", t.a, "from the data descriptor")
test("a non-data one can be", t.b, "from the instance dict")
test("and the instance dict really does hold both", sorted(t.__dict__), ["a", "b"])

before anything is in the instance dict:
  t.a -> from the data descriptor
  t.b -> from the non-data descriptor

after writing both names straight into t.__dict__:
  t.a -> from the data descriptor
  t.b -> from the instance dict
ok   a data descriptor cannot be shadowed by the instance
ok   a non-data one can be
ok   and the instance dict really does hold both

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24from functools import cached_property

computed = []

class Report:
    @cached_property
    def total(self):
        computed.append("ran")
        return sum(range(1000))

r = Report()
print("instance dict before any access:", r.__dict__)
print("first access: ", r.total)
print("instance dict now:", r.__dict__)
print("second access:", r.total)
print("times the body ran:", len(computed))
print()
print("cached_property defines __set__?", hasattr(Report.__dict__["total"], "__set__"))
print("so after the first access, the instance dict shadows it.")

test("the body ran exactly once", len(computed), 1)
test("because the value moved into the instance dict", r.__dict__["total"], 499500)
test("which works only because it is a non-data descriptor",
     hasattr(Report.__dict__["total"], "__set__"), False)

instance dict before any access: {}
first access:  499500
instance dict now: {'total': 499500}
second access: 499500
times the body ran: 1

cached_property defines __set__? False
so after the first access, the instance dict shadows it.
ok   the body ran exactly once
ok   because the value moved into the instance dict
ok   which works only because it is a non-data descriptor

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21class Greeter:
    def greet(self, name):
        return f"hello {name}, from {type(self).__name__}"

stored = Greeter.__dict__["greet"]
g = Greeter()

print("what the class dict holds:", type(stored).__name__)
print("does a function have __get__?", hasattr(stored, "__get__"))
print()
print("looked up on the instance:  ", type(g.greet).__name__)
print("what it is bound to:        ", type(g.greet.__self__).__name__)
print()
print("normal call:     ", g.greet("ada"))
print("binding by hand: ", stored.__get__(g)("ada"))
print("no binding at all:", stored(g, "ada"))

test("a plain function is stored in the class", type(stored).__name__, "function")
test("looking it up on an instance produces a bound method", type(g.greet).__name__, "method")
test("the binding is __get__, and you can do it yourself", stored.__get__(g)("ada"), g.greet("ada"))
test("or skip it and pass self explicitly", stored(g, "x"), g.greet("x"))

what the class dict holds: function
does a function have __get__? True

looked up on the instance:   method
what it is bound to:         Greeter

normal call:      hello ada, from Greeter
binding by hand:  hello ada, from Greeter
no binding at all: hello ada, from Greeter
ok   a plain function is stored in the class
ok   looking it up on an instance produces a bound method
ok   the binding is __get__, and you can do it yourself
ok   or skip it and pass self explicitly

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21class C:
    def instance_method(self): return "got an instance"
    @staticmethod
    def static_method(): return "got nothing"
    @classmethod
    def class_method(cls): return f"got {cls.__name__}"

for name in ("instance_method", "static_method", "class_method"):
    stored = C.__dict__[name]
    bound = stored.__get__(C(), C)
    print(f"{name:<16} stored as {type(stored).__name__:<13} -> binds to {type(bound).__name__}")

print()
print("staticmethod, bound through the class:", C.__dict__["static_method"].__get__(None, C)())
print("classmethod, bound through the class: ", C.__dict__["class_method"].__get__(None, C)())

test("a staticmethod's __get__ gives back a plain function",
     type(C.__dict__["static_method"].__get__(None, C)).__name__, "function")
test("a classmethod's gives a method bound to the class",
     type(C.__dict__["class_method"].__get__(None, C)).__name__, "method")
test("and calling it receives the class", C.class_method(), "got C")

instance_method  stored as function      -> binds to method
static_method    stored as staticmethod  -> binds to function
class_method     stored as classmethod   -> binds to method

staticmethod, bound through the class: got nothing
classmethod, bound through the class:  got C
ok   a staticmethod's __get__ gives back a plain function
ok   a classmethod's gives a method bound to the class
ok   and calling it receives the class

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30class Broken:
    """Stores the value on the descriptor. Looks fine with one instance."""
    def __init__(self):
        self.value = None
    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        return self.value
    def __set__(self, obj, value):
        self.value = value          # <- on the DESCRIPTOR, not the instance

class Item:
    name = Broken()

one, two = Item(), Item()
one.name = "first"
two.name = "second"

print("one.name:", one.name)
print("two.name:", two.name)
print()
print("there is exactly one Broken object, living on the class:")
print("  ", Item.__dict__["name"])
print("so both instances have been writing to the same attribute.")

test("the second write overwrote the first", one.name, "second")
test("both instances read the same value", one.name == two.name, True)
test("the value ended up on the descriptor, not on either instance",
     Item.__dict__["name"].value, "second")
test("and both instance dicts are empty", (one.__dict__, two.__dict__), ({}, {}))

one.name: second
two.name: second

there is exactly one Broken object, living on the class:
   <__main__.Broken object at 0x105784590>
so both instances have been writing to the same attribute.
ok   the second write overwrote the first
ok   both instances read the same value
ok   the value ended up on the descriptor, not on either instance
ok   and both instance dicts are empty

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52class Positive:
    """A validated numeric field, reusable across attributes and classes."""

    def __set_name__(self, owner, name):
        # Called once, at class creation, with the attribute name.
        self.public = name
        self.private = "_" + name

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self                      # accessed on the class
        return getattr(obj, self.private)

    def __set__(self, obj, value):
        if not isinstance(value, (int, float)):
            raise TypeError(f"{self.public} must be a number, got {type(value).__name__}")
        if value <= 0:
            raise ValueError(f"{self.public} must be positive, got {value}")
        setattr(obj, self.private, value)    # on the INSTANCE

class LineItem:
    price = Positive()
    quantity = Positive()

    def __init__(self, price, quantity):
        self.price = price
        self.quantity = quantity

    @property
    def total(self):
        return self.price * self.quantity

book = LineItem(12.50, 3)
pen = LineItem(1.00, 10)

print("names bound by __set_name__:", LineItem.price.public, "and", LineItem.quantity.public)
print("book:", book.__dict__, "-> total", book.total)
print("pen: ", pen.__dict__, "-> total", pen.total)
print()
for bad in (-5, 0, "free"):
    try:
        book.price = bad
    except (TypeError, ValueError) as exc:
        print(f"  price = {bad!r:<8} {type(exc).__name__}: {exc}")

test("the two instances keep separate values", (book.price, pen.price), (12.5, 1.0))
test("values are stored on the instance, under the private name",
     sorted(book.__dict__), ["_price", "_quantity"])
test("validation names the attribute it rejected", LineItem.quantity.public, "quantity")
test("and the rejected write did not land", book.price, 12.5)
test("one descriptor class covers both fields",
     type(LineItem.price) is type(LineItem.quantity), True)

names bound by __set_name__: price and quantity
book: {'_price': 12.5, '_quantity': 3} -> total 37.5
pen:  {'_price': 1.0, '_quantity': 10} -> total 10.0

  price = -5       ValueError: price must be positive, got -5
  price = 0        ValueError: price must be positive, got 0
  price = 'free'   TypeError: price must be a number, got str
ok   the two instances keep separate values
ok   values are stored on the instance, under the private name
ok   validation names the attribute it rejected
ok   and the rejected write did not land
ok   one descriptor class covers both fields

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33class Tracked:
    def __set_name__(self, owner, name):
        self.public, self.private = name, "_" + name
    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        return getattr(obj, self.private)
    def __set__(self, obj, value):
        setattr(obj, self.private, value)
    def __delete__(self, obj):
        print(f"  __delete__ ran for {self.public}")
        delattr(obj, self.private)

class Row:
    label = Tracked()

r = Row()
r.label = "x"
after_set = dict(r.__dict__)
print("after assignment:", after_set)
del r.label
print("after del:       ", r.__dict__)

try:
    r.label
    missing = None
except AttributeError as exc:
    missing = str(exc)
print("reading it now:  ", missing)

test("__set__ stored it under the private name", after_set, {"_label": "x"})
test("del ran __delete__ and removed it", r.__dict__, {})
test("after which reading raises", "has no attribute" in missing, True)

after assignment: {'_label': 'x'}
  __delete__ ran for label
after del:        {}
reading it now:   'Row' object has no attribute '_label'
ok   __set__ stored it under the private name
ok   del ran __delete__ and removed it
ok   after which reading raises

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16class Rectangle:
    sides = 4

print("an instance's type:", type(Rectangle()).__name__)
print("the class's type:  ", type(Rectangle).__name__)
print("type's own type:   ", type(type).__name__)
print()
print("so a class is an instance of type:", isinstance(Rectangle, type))
print("and can be passed around like any value:")
for cls in (Rectangle, int, str):
    print("  ", cls.__name__, "->", cls.__mro__[-1].__name__)

test("instances are instances of their class", isinstance(Rectangle(), Rectangle), True)
test("and the class is an instance of type", isinstance(Rectangle, type), True)
test("type is its own type", type(type) is type, True)
test("every class inherits from object", Rectangle.__mro__[-1], object)

an instance's type: Rectangle
the class's type:   type
type's own type:    type

so a class is an instance of type: True
and can be passed around like any value:
   Rectangle -> object
   int -> object
   str -> object
ok   instances are instances of their class
ok   and the class is an instance of type
ok   type is its own type
ok   every class inherits from object

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24def area(self):
    return self.width * self.height

# Built by calling type directly.
Made = type("Made", (object,), {"width": 2, "height": 3, "area": area})

# Built with the statement.
class Written:
    width, height = 2, 3
    def area(self):
        return self.width * self.height

print("Made().area():   ", Made().area())
print("Written().area():", Written().area())
print()
print("both are classes:      ", isinstance(Made, type), isinstance(Written, type))
print("both report type type: ", type(Made).__name__, type(Written).__name__)
print("Made's body came from a dict passed at runtime.")

test("the two are equivalent in behaviour", Made().area(), Written().area())
test("and indistinguishable in kind", type(Made) is type(Written), True)
test("the name is whatever you passed", Made.__name__, "Made")
test("a method built this way still binds self",
     type(Made().area).__name__, "method")

Made().area():    6
Written().area(): 6

both are classes:       True True
both report type type:  type type
Made's body came from a dict passed at runtime.
ok   the two are equivalent in behaviour
ok   and indistinguishable in kind
ok   the name is whatever you passed
ok   a method built this way still binds self

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36class Recording(type):
    @classmethod
    def __prepare__(mcls, name, bases, **kwargs):
        print(f"  1. __prepare__({name}) - hands back the mapping the body fills in")
        return {}

    def __new__(mcls, name, bases, namespace, **kwargs):
        body = [k for k in namespace if not k.startswith("__")]
        print(f"  2. __new__({name}) - body contains {body}")
        cls = super().__new__(mcls, name, bases, namespace)
        cls.made_by = mcls.__name__          # inject something
        return cls

    def __init__(cls, name, bases, namespace, **kwargs):
        print(f"  3. __init__({name}) - the class now exists")
        super().__init__(name, bases, namespace)

print("defining the class:")
class Widget(metaclass=Recording):
    size = 1
    def draw(self):
        return "drawn"

print()
print("injected attribute:", Widget.made_by)
print("type of Widget:    ", type(Widget).__name__)
print()
print("making instances:")
first, second = Widget(), Widget()
print("  nothing above re-ran.")

test("the metaclass injected an attribute", Widget.made_by, "Recording")
test("Widget's type is the metaclass, not type", type(Widget).__name__, "Recording")
test("but Widget is still a class", isinstance(Widget, type), True)
test("and instances are ordinary", first.draw(), "drawn")
test("creation hooks run per class, not per instance", type(first).made_by, "Recording")

defining the class:
  1. __prepare__(Widget) - hands back the mapping the body fills in
  2. __new__(Widget) - body contains ['size', 'draw']
  3. __init__(Widget) - the class now exists

injected attribute: Recording
type of Widget:     Recording

making instances:
  nothing above re-ran.
ok   the metaclass injected an attribute
ok   Widget's type is the metaclass, not type
ok   but Widget is still a class
ok   and instances are ordinary
ok   creation hooks run per class, not per instance

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30class RegisteringMeta(type):
    registry = {}

    def __new__(mcls, name, bases, namespace, **kwargs):
        cls = super().__new__(mcls, name, bases, namespace)
        if bases:                            # skip the base class itself
            # The key has to be guessed from the class name, because the class
            # statement had no way to pass one in.
            mcls.registry[name.removesuffix("Exporter").lower()] = cls
        return cls

class Exporter(metaclass=RegisteringMeta):
    def export(self, rows):
        raise NotImplementedError

class CsvExporter(Exporter):
    def export(self, rows):
        return "\n".join(",".join(map(str, r)) for r in rows)

class TsvExporter(Exporter):
    def export(self, rows):
        return "\n".join("\t".join(map(str, r)) for r in rows)

print("registry:", {k: v.__name__ for k, v in RegisteringMeta.registry.items()})
chosen = RegisteringMeta.registry["csv"]()
print("csv export:", repr(chosen.export([[1, 2], [3, 4]])))

test("both subclasses registered themselves", sorted(RegisteringMeta.registry), ["csv", "tsv"])
test("the base class did not", "exporter" in RegisteringMeta.registry, False)
test("and the looked-up class works", chosen.export([[1, 2]]), "1,2")

registry: {'csv': 'CsvExporter', 'tsv': 'TsvExporter'}
csv export: '1,2\n3,4'
ok   both subclasses registered themselves
ok   the base class did not
ok   and the looked-up class works

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38class Exporter:
    registry = {}

    def __init_subclass__(cls, /, format=None, **kwargs):
        super().__init_subclass__(**kwargs)
        if format is None:
            raise TypeError(f"{cls.__name__} must declare format=")
        Exporter.registry[format] = cls

    def export(self, rows):
        raise NotImplementedError

class Csv(Exporter, format="csv"):
    def export(self, rows):
        return "\n".join(",".join(map(str, r)) for r in rows)

class Tsv(Exporter, format="tsv"):
    def export(self, rows):
        return "\n".join("\t".join(map(str, r)) for r in rows)

print("registry:", {k: v.__name__ for k, v in Exporter.registry.items()})

# Forgetting the keyword is caught at class definition time, not at first use.
try:
    class Broken(Exporter):
        pass
    complaint = None
except TypeError as exc:
    complaint = str(exc)
print("forgetting format=:", complaint)
print()
print("__init_subclass__ is implicitly a classmethod:",
      type(Exporter.__dict__["__init_subclass__"]).__name__)

test("the registry is keyed by the declared format", sorted(Exporter.registry), ["csv", "tsv"])
test("the base class is not in it", Exporter in Exporter.registry.values(), False)
test("a missing keyword fails at definition time", complaint, "Broken must declare format=")
test("and no metaclass was involved", type(Csv) is type, True)

registry: {'csv': 'Csv', 'tsv': 'Tsv'}
forgetting format=: Broken must declare format=

__init_subclass__ is implicitly a classmethod: classmethod
ok   the registry is keyed by the declared format
ok   the base class is not in it
ok   a missing keyword fails at definition time
ok   and no metaclass was involved

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41from abc import ABC, abstractmethod

class Store(ABC):
    @abstractmethod
    def get(self, key): ...

    @abstractmethod
    def put(self, key, value): ...

    def get_or(self, key, default):        # concrete, built on the abstract pair
        try:
            return self.get(key)
        except KeyError:
            return default

class Memory(Store):
    def __init__(self):
        self.data = {}
    def get(self, key):
        return self.data[key]
    def put(self, key, value):
        self.data[key] = value

m = Memory()
m.put("a", 1)
print("the complete subclass works:", m.get("a"), "|", m.get_or("b", "missing"))
print("metaclass of Store:", type(Store).__name__)
print("abstract methods:  ", sorted(Store.__abstractmethods__))
print()

for cls, label in ((Store, "the abstract base"), (type("Half", (Store,), {"get": lambda s, k: None}), "a half-finished subclass")):
    try:
        cls()
        print(f"{label}: instantiated")
    except TypeError as exc:
        print(f"{label}: TypeError: {exc}")

test("ABC brings a metaclass with it", type(Store).__name__, "ABCMeta")
test("it records what is unimplemented", sorted(Store.__abstractmethods__), ["get", "put"])
test("a complete subclass has nothing left abstract", Memory.__abstractmethods__, frozenset())
test("and inherits the concrete helper", m.get_or("nope", 0), 0)

the complete subclass works: 1 | missing
metaclass of Store: ABCMeta
abstract methods:   ['get', 'put']

the abstract base: TypeError: Can't instantiate abstract class Store without an implementation for abstract methods 'get', 'put'
a half-finished subclass: TypeError: Can't instantiate abstract class Half without an implementation for abstract method 'put'
ok   ABC brings a metaclass with it
ok   it records what is unimplemented
ok   a complete subclass has nothing left abstract
ok   and inherits the concrete helper

1
2
3
4
5
6
7
8class Serialisable(type): pass
class Observable(type): pass

class Model(metaclass=Serialisable): pass
class Widget(metaclass=Observable): pass

# A class that wants to be both.
class Form(Model, Widget): pass

Traceback (most recent call last):
  File "example.py", line 8, in <module>
    class Form(Model, Widget): pass
TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24class Single(type):
    _instances = {}
    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super().__call__(*args, **kwargs)
        return cls._instances[cls]

class Settings(metaclass=Single):
    def __init__(self):
        self.loaded = True

first, second = Settings(), Settings()
print("same object:", first is second)
print("Settings() calls type(Settings).__call__:", type(Settings).__call__ is Single.__call__)

# What the language already gives you, for the same purpose:
#   a module. Imported once, cached in sys.modules, shared by every importer.
import sys
print()
print("sys is the same object everywhere it is imported:", sys is sys.modules["sys"])

test("the metaclass made it a singleton", first is second, True)
test("calling the class goes through the metaclass", type(Settings).__call__ is Single.__call__, True)
test("while a module is already a singleton, for free", sys is sys.modules["sys"], True)

same object: True
Settings() calls type(Settings).__call__: True

sys is the same object everywhere it is imported: True
ok   the metaclass made it a singleton
ok   calling the class goes through the metaclass
ok   while a module is already a singleton, for free