Mastering Python Second Edition Release Code

CH_12_performance/README.rst

@@ -0,0 +1,4 @@
+Chapter 12, Performance
+##############################################################################
+
+| Tracking and Reducing your Memory and CPU Usage shows several methods of measuring and improving CPU and memory usage.

CH_12_performance/T_00_timeit.py

@@ -0,0 +1,60 @@
+import timeit
+
+
+def test_list():
+    return list(range(10000))
+
+
+def test_list_comprehension():
+    return [i for i in range(10000)]
+
+
+def test_append():
+    x = []
+    for i in range(10000):
+        x.append(i)
+
+    return x
+
+
+def test_insert():
+    x = []
+    for i in range(10000):
+        x.insert(0, i)
+
+    return x
+
+
+def benchmark(function, number=100, repeat=10):
+    # Measure the execution times. Passing the globals() is an
+    # easy way to make the functions available.
+    times = timeit.repeat(function, number=number,
+                          globals=globals())
+    # The repeat function gives `repeat` results so we take the
+    # min() and divide it by the number of runs
+    time = min(times) / number
+    print(f'{number} loops, best of {repeat}: {time:9.6f}s :: ',
+          function.__name__)
+
+
+def autorange_benchmark(function):
+
+    def print_result(number, time_taken):
+        # The autorange function keeps trying until the total
+        # runtime (time_taken) reaches 0.2 seconds. To get the
+        # time per run we need to divide it by the number of runs
+        time = time_taken / number
+        name = function.__name__
+        print(f'{number} loops, average: {time:9.6f}s :: {name}')
+
+    # Measure the execution times. Passing the globals() is an
+    # easy way to make the functions available.
+    timer = timeit.Timer(function, globals=globals())
+    timer.autorange(print_result)
+
+
+if __name__ == '__main__':
+    benchmark(test_list)
+    benchmark(test_list_comprehension)
+    benchmark(test_append)
+    benchmark(test_insert)

CH_12_performance/T_01_timeit_main.py

@@ -0,0 +1,3 @@
+import timeit
+
+timeit.main(args=['[x for x in range(1000000)]'])

CH_12_performance/T_02_custom_timeit.py

@@ -0,0 +1,56 @@
+import gc
+import time
+import functools
+
+
+assert time
+
+TIMEIT_TEMPLATE = '''
+def run(number):
+    {setup}
+    start = time.perf_counter()
+    for i in range(number):
+        {statement}
+    stop = time.perf_counter()
+    return stop - start
+'''
+
+
+def timeit(statement, setup='', number=1000000, globals_=None):
+    # Get or create globals
+    globals_ = globals() if globals_ is None else globals_
+
+    # Create the test code so we can separate the namespace
+    src = TIMEIT_TEMPLATE.format(
+        statement=statement,
+        setup=setup,
+        number=number,
+    )
+    # Compile the source
+    code = compile(src, '<source>', 'exec')
+
+    # Define locals for the benchmarked code
+    locals_ = {}
+
+    # Execute the code so we can get the benchmark fuction
+    exec(code, globals_, locals_)
+
+    # Get the run function from locals() which was added by `exec`
+    run = functools.partial(locals_['run'], number=number)
+
+    # Disable garbage collection to prevent skewing results
+    gc.disable()
+    try:
+        result = run()
+    finally:
+        gc.enable()
+
+    return result
+
+
+if __name__ == '__main__':
+    statement = '[x for x in range(100)]'
+    print('{:.7f}'.format(timeit(statement, number=1)))
+    print('{:.7f}'.format(timeit(statement) / 1000000))
+    print('{:.7f}'.format(timeit(statement, number=1)))
+    print('{:.7f}'.format(timeit(statement) / 1000000))

CH_12_performance/T_03_profile_fibonacci.py

@@ -0,0 +1,26 @@
+import sys
+import functools
+
+
+@functools.lru_cache()
+def fibonacci_cached(n):
+    if n < 2:
+        return n
+    else:
+        return fibonacci_cached(n - 1) + fibonacci_cached(n - 2)
+
+
+def fibonacci(n):
+    if n < 2:
+        return n
+    else:
+        return fibonacci(n - 1) + fibonacci(n - 2)
+
+
+if __name__ == '__main__':
+    n = 30
+    if sys.argv[-1] == 'cache':
+        fibonacci_cached(n)
+    else:
+        fibonacci(n)
+

CH_12_performance/T_04_profiler_calibration.py

@@ -0,0 +1,21 @@
+import profile
+
+if __name__ == '__main__':
+    profiler = profile.Profile()
+    for i in range(10):
+        print(profiler.calibrate(100000))
+
+##############################################################################
+
+import profile
+
+
+# The number here is bias calculated earlier
+profile.Profile.bias = 9.809351906482531e-07
+
+##############################################################################
+
+import profile
+
+
+profiler = profile.Profile(bias=9.809351906482531e-07)

CH_12_performance/T_05_profiler_large_bias.py

@@ -0,0 +1,41 @@
+import sys
+import pstats
+import profile
+import functools
+
+
+@functools.lru_cache()
+def fibonacci_cached(n):
+    if n < 2:
+        return n
+    else:
+        return fibonacci_cached(n - 1) + fibonacci_cached(n - 2)
+
+
+def fibonacci(n):
+    if n < 2:
+        return n
+    else:
+        return fibonacci(n - 1) + fibonacci(n - 2)
+
+
+if __name__ == '__main__':
+    profiler = profile.Profile(bias=9.809351906482531e-07)
+    n = 30
+
+    if sys.argv[-1] == 'cache':
+        profiler.runcall(fibonacci_cached, n)
+    else:
+        profiler.runcall(fibonacci, n)
+
+    stats = pstats.Stats(profiler).sort_stats('calls')
+    stats.print_stats()
+
+##############################################################################
+
+import profile
+
+
+if __name__ == '__main__':
+    profiler = profile.Profile()
+    profiler.bias = profiler.calibrate(100000)

CH_12_performance/T_06_selective_profiling.py

@@ -0,0 +1,51 @@
+import cProfile
+import datetime
+import functools
+
+
+def timer(function):
+    @functools.wraps(function)
+    def _timer(*args, **kwargs):
+        start = datetime.datetime.now()
+        try:
+            return function(*args, **kwargs)
+        finally:
+            end = datetime.datetime.now()
+            print(f'{function.__name__}: {end - start}')
+    return _timer
+
+
+def profiler(function):
+    @functools.wraps(function)
+    def _profiler(*args, **kwargs):
+        profiler = cProfile.Profile()
+        try:
+            profiler.enable()
+            return function(*args, **kwargs)
+        finally:
+            profiler.disable()
+            profiler.print_stats()
+    return _profiler
+
+
+@profiler
+def profiled_fibonacci(n):
+    return fibonacci(n)
+
+
+@timer
+def timed_fibonacci(n):
+    return fibonacci(n)
+
+
+def fibonacci(n):
+    if n < 2:
+        return n
+    else:
+        return fibonacci(n - 1) + fibonacci(n - 2)
+
+
+if __name__ == '__main__':
+    timed_fibonacci(32)
+    profiled_fibonacci(32)
+

CH_12_performance/T_07_profile_statistics.py

@@ -0,0 +1,30 @@
+import sys
+import pathlib
+import pstats
+import cProfile
+
+import pyperformance
+
+# pyperformance doesn't expose the benchmarks anymore so we need
+# to manually add the path
+pyperformance_path = pathlib.Path(pyperformance.__file__).parent
+sys.path.append(str(pyperformance_path / 'data-files'))
+
+# Now we can import the benchmark
+from benchmarks.bm_float import run_benchmark as bm_float  # noqa
+
+
+def benchmark():
+    for i in range(10):
+        bm_float.benchmark(bm_float.POINTS)
+
+
+if __name__ == '__main__':
+    profiler = cProfile.Profile()
+    profiler.runcall(benchmark)
+    profiler.dump_stats('bm_float.profile')
+
+    stats = pstats.Stats('bm_float.profile')
+    stats.strip_dirs()
+    stats.sort_stats('calls', 'cumtime')
+    stats.print_stats(10)

CH_12_performance/T_08_line_profiler.py

@@ -0,0 +1,25 @@
+import itertools
+
+
+@profile
+def primes():
+    n = 2
+    primes = set()
+    while True:
+        for p in primes:
+            if n % p == 0:
+                break
+        else:
+            primes.add(n)
+            yield n
+        n += 1
+
+
+if __name__ == '__main__':
+    total = 0
+    n = 2000
+    for prime in itertools.islice(primes(), n):
+        total += prime
+
+    print('The sum of the first %d primes is %d' % (n, total))
+

CH_12_performance/T_09_lists_versus_generators.rst

@@ -0,0 +1,15 @@
+>>> import itertools
+
+# Without itertools.tee:
+
+>>> generator = itertools.count()
+>>> list(itertools.islice(generator, 5))
+[0, 1, 2, 3, 4]
+>>> list(itertools.islice(generator, 5))
+[5, 6, 7, 8, 9]
+
+>>> generator_a, generator_b = itertools.tee(itertools.count())
+>>> list(itertools.islice(generator_a, 5))
+[0, 1, 2, 3, 4]
+>>> list(itertools.islice(generator_b, 5))
+[0, 1, 2, 3, 4]

CH_12_performance/T_10_slots_performance.py

@@ -0,0 +1,39 @@
+import timeit
+import pytest
+import functools
+
+
+class WithSlots:
+    __slots__ = 'eggs',
+
+
+class WithoutSlots:
+    pass
+
+
+with_slots = WithSlots()
+no_slots = WithoutSlots()
+
+
+@pytest.mark.skip()
+def test_set(obj):
+    obj.eggs = 5
+
+
+@pytest.mark.skip()
+def test_get(obj):
+    return obj.eggs
+
+
+timer = functools.partial(
+    timeit.timeit,
+    number=20000000,
+    setup='\n'.join((
+        f'from {__name__} import with_slots, no_slots',
+        f'from {__name__} import test_get, test_set',
+    )),
+)
+for function in 'test_set', 'test_get':
+    print(function)
+    print('with slots', timer(f'{function}(with_slots)'))
+    print('with slots', timer(f'{function}(no_slots)'))

CH_12_performance/T_11_tracemalloc.py

@@ -0,0 +1,23 @@
+import tracemalloc
+
+
+if __name__ == '__main__':
+    tracemalloc.start()
+
+    # Reserve some memory
+    x = list(range(1000000))
+
+    # Import some modules
+    import os
+    import sys
+    import asyncio
+
+    assert os
+    assert sys
+    assert asyncio
+
+    # Take a snapshot to calculate the memory usage
+    snapshot = tracemalloc.take_snapshot()
+    for statistic in snapshot.statistics('lineno')[:10]:
+        print(statistic)
+

CH_12_performance/T_12_memory_profiler.py

@@ -0,0 +1,21 @@
+try:
+    import memory_profiler
+except ImportError:
+    print('Please install the memory profiler to run this example')
+    print('pip install -U memory-profiler')
+else:
+    @memory_profiler.profile
+    def main():
+        n = 100000
+        a = [i for i in range(n)]
+        b = [i for i in range(n)]
+        c = list(range(n))
+        d = list(range(n))
+        e = dict.fromkeys(a, b)
+        f = dict.fromkeys(c, d)
+        assert e
+        assert f
+
+    if __name__ == '__main__':
+        main()
+

CH_12_performance/T_13_memory_leaks.py

@@ -0,0 +1,36 @@
+import tracemalloc
+
+
+class SomeClass:
+    pass
+
+
+if __name__ == '__main__':
+    # Initialize some variables to ignore them from the leak
+    # detection
+    n = 100000
+
+    tracemalloc.start()
+    # Your application should initialize here
+
+    snapshot_a = tracemalloc.take_snapshot()
+    instances = []
+
+    # This code should be the memory leaking part
+    for i in range(n):
+        a = SomeClass()
+        b = SomeClass()
+        # Circular reference. a references b, b references a
+        a.b = b
+        b.a = a
+        # Force Python to keep the object in memory for now
+        instances.append(a)
+
+    # Clear the list of items again. Now all memory should be
+    # released, right?
+    del instances
+    snapshot_b = tracemalloc.take_snapshot()
+
+    statistics = snapshot_b.compare_to(snapshot_a, 'lineno')
+    for statistic in statistics[:10]:
+        print(statistic)

CH_12_performance/T_14_garbage_collection.py

@@ -0,0 +1,37 @@
+import gc
+
+
+class SomeClass(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__}: {self.name}'
+
+
+# Create the objects
+a = SomeClass('a')
+b = SomeClass('b')
+
+# Add some circular references
+a.b = a
+b.a = b
+
+# Remove the objects
+del a
+del b
+
+# See if the objects are still there
+print('Before manual collection:')
+for object_ in gc.get_objects():
+    if isinstance(object_, SomeClass):
+        print('\t', object_, gc.get_referents(object_))
+
+print('After manual collection:')
+gc.collect()
+for object_ in gc.get_objects():
+    if isinstance(object_, SomeClass):
+        print('\t', object_, gc.get_referents(object_))
+
+print('Thresholds:', gc.get_threshold())

CH_12_performance/T_15_garbage_collector_viewing.py

@@ -0,0 +1,12 @@
+import gc
+import collections
+
+
+if __name__ == '__main__':
+    objects = collections.Counter()
+    for object_ in gc.get_objects():
+        objects[type(object_)] += 1
+
+    print(f'Different object count: {len(objects)}')
+    for object_, count in objects.most_common(10):
+        print(f'{count}: {object_}')

CH_12_performance/T_16_weak_references.py

@@ -0,0 +1,36 @@
+import gc
+import weakref
+
+
+class SomeClass(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.name)
+
+
+def print_mem(message):
+    print(message)
+    for object_ in gc.get_objects():
+        if isinstance(object_, SomeClass):
+            print('\t', object_, gc.get_referents(object_))
+
+
+# Create the objects
+a = SomeClass('a')
+b = SomeClass('b')
+
+# Add some weak circular references
+a.b = weakref.ref(a)
+b.a = weakref.ref(b)
+
+print_mem('Objects in memory before del:')
+
+# Remove the objects
+del a
+del b
+
+# See if the objects are still there
+print_mem('Objects in memory after del:')

CH_12_performance/T_17_weakref_pitfalls.rst

@@ -0,0 +1,42 @@
+>>> import weakref
+
+>>> weakref.ref(dict(a=123))
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: cannot create weak reference to 'dict' object
+>>> weakref.ref([1, 2, 3])
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: cannot create weak reference to 'list' object
+>>> weakref.ref('test')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: cannot create weak reference to 'str' object
+>>> weakref.ref(b'test')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: cannot create weak reference to 'bytes' object
+>>> a = weakref.WeakValueDictionary(a=123)
+Traceback (most recent call last):
+    ...
+TypeError: cannot create weak reference to 'int' object
+
+>>> class CustomDict(dict):
+...     pass
+
+>>> weakref.ref(CustomDict())
+<weakref at 0x...; dead>
+
+>>> class SomeClass:
+...     def __init__(self, name):
+...         self.name = name
+
+>>> a = SomeClass('a')
+>>> b = weakref.proxy(a)
+>>> b.name
+'a'
+>>> del a
+>>> b.name
+Traceback (most recent call last):
+    ...
+ReferenceError: weakly-referenced object no longer exists

CH_12_performance/T_18_freeing_memory.py

@@ -0,0 +1,22 @@
+import os
+import psutil
+
+
+def print_usage(message):
+    process = psutil.Process(os.getpid())
+    usage = process.memory_info().rss / (1 << 20)
+    print(f'Memory usage {message}: {usage:.1f} MiB')
+
+
+def allocate_and_release():
+    # Allocate large block of memory
+    large_list = list(range(1000000))
+    print_usage('after allocation')
+
+    del large_list
+    print_usage('after releasing')
+
+
+print_usage('initial')
+allocate_and_release()
+allocate_and_release()

CH_12_performance/T_19_slots.py

@@ -0,0 +1,33 @@
+
+class Slots(object):
+    __slots__ = 'index', 'name', 'description'
+
+    def __init__(self, index):
+        self.index = index
+        self.name = 'slot %d' % index
+        self.description = 'some slot with index %d' % index
+
+
+class NoSlots(object):
+
+    def __init__(self, index):
+        self.index = index
+        self.name = 'slot %d' % index
+        self.description = 'some slot with index %d' % index
+
+
+try:
+    import memory_profiler
+except ImportError:
+    print('Please install the memory profiler to run this example')
+    print('pip install -U memory-profiler')
+else:
+    @memory_profiler.profile
+    def main():
+        slots = [Slots(i) for i in range(25000)]
+        no_slots = [NoSlots(i) for i in range(25000)]
+        return slots, no_slots
+
+    if __name__ == '__main__':
+        main()
+