Mastering Python Second Edition Release Code

2022-05-05 18:25:55 +02:00
commit 3223a43fe3
454 changed files with 20230 additions and 0 deletions
@@ -0,0 +1,4 @@
+Chapter 14, C/C++ Extensions
+##############################################################################
+
+| System Calls and C/C++ Libraries covers the calling of C/C++ functions for both interoperability and performance using Ctypes, CFFI and native C/C++.
@@ -0,0 +1,104 @@
+# Windows
+
+>>> import ctypes
+
+>>> ctypes.cdll
+<ctypes.LibraryLoader object at 0x...>
+>>> libc = ctypes.cdll.msvcrt
+>>> libc
+<CDLL 'msvcrt', handle ... at ...>
+>>> libc.printf
+<_FuncPtr object at 0x...>
+
+------------------------------------------------------------------------------
+
+# Linux
+
+>>> import ctypes
+
+>>> ctypes.cdll
+<ctypes.LibraryLoader object at 0x...>
+>>> libc = ctypes.cdll.LoadLibrary('libc.so.6')
+>>> libc
+<CDLL 'libc.so.6', handle ... at ...>
+>>> libc.printf
+<_FuncPtr object at 0x...>
+
+------------------------------------------------------------------------------
+
+# OS X
+
+>>> import ctypes
+
+>>> libc = ctypes.cdll.LoadLibrary('libc.dylib')
+>>> libc
+<CDLL 'libc.dylib', handle ... at 0x...>
+>>> libc.printf
+<_FuncPtr object at 0x...>
+
+------------------------------------------------------------------------------
+
+# OS X
+
+>>> from ctypes import util
+>>> from ctypes import cdll
+
+>>> library = util.find_library('libc')
+>>> library
+'/usr/lib/libc.dylib'
+
+# Load the library
+
+>>> libc = cdll.LoadLibrary(library)
+>>> libc
+<CDLL '/usr/lib/libc.dylib', handle ... at 0x...>
+
+------------------------------------------------------------------------------
+
+>>> c_string = ctypes.create_string_buffer(b'some bytes')
+>>> ctypes.sizeof(c_string)
+11
+>>> c_string.raw
+b'some bytes\x00'
+>>> c_string.value
+b'some bytes'
+>>> libc.printf(c_string)
+10
+some bytes>>>
+
+------------------------------------------------------------------------------
+
+| >>> libc.printf(123)
+| segmentation fault (core dumped)  python3
+
+------------------------------------------------------------------------------
+
+>>> format_string = b'Number: %d\n'
+>>> libc.printf(format_string, 123)
+Number: 123
+12
+>>> x = ctypes.c_int(123)
+>>> libc.printf(format_string, x)
+Number: 123
+12
+
+------------------------------------------------------------------------------
+
+>>> format_string = b'Number: %.3f\n'
+>>> libc.printf(format_string, 123.45)
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+ctypes.ArgumentError: argument 2: <class 'TypeError'>: Don't know how to convert parameter 2
+>>> x = ctypes.c_double(123.45)
+>>> libc.printf(format_string, x)
+Number: 123.450
+16
+
+------------------------------------------------------------------------------
+
+>>> x = ctypes.c_double(123.45)
+>>> x.value
+123.45
+>>> x.value = 456
+>>> x
+c_double(456.0)
@@ -0,0 +1,20 @@
+>>> from _libc import libc
+>>> import ctypes
+
+>>> class ComplexStructure(ctypes.Structure):
+...     _fields_ = [
+...         ('some_int', ctypes.c_int),
+...         ('some_double', ctypes.c_double),
+...         ('some_char', ctypes.c_char),
+...         ('some_string', ctypes.c_char_p),
+...     ]
+... 
+>>> structure = ComplexStructure(123, 456.789, b'x', b'abc')
+>>> structure.some_int
+123
+>>> structure.some_double
+456.789
+>>> structure.some_char
+b'x'
+>>> structure.some_string
+b'abc'
@@ -0,0 +1,42 @@
+>>> import ctypes
+
+
+>>> TenNumbers = 10 * ctypes.c_double
+>>> numbers = TenNumbers()
+>>> numbers[0]
+0.0
+
+------------------------------------------------------------------------------
+
+>>> class ComplexStructure(ctypes.Structure):
+...     _fields_ = [
+...         ('some_int', ctypes.c_int),
+...         ('some_double', ctypes.c_double),
+...         ('some_char', ctypes.c_char),
+...         ('some_string', ctypes.c_char_p),
+...     ]
+
+
+>>> GrossComplexStructures = 144 * ComplexStructure 
+>>> complex_structures = GrossComplexStructures()
+
+>>> complex_structures[10].some_double = 123
+>>> complex_structures[10]
+<__main__.ComplexStructure object at ...>
+>>> complex_structures
+<__main__.ComplexStructure_Array_144 object at ...>
+
+------------------------------------------------------------------------------
+
+>>> TenNumbers = 10 * ctypes.c_double
+>>> numbers = TenNumbers()
+>>> ctypes.resize(numbers, 11 * ctypes.sizeof(ctypes.c_double))
+>>> ctypes.resize(numbers, 10 * ctypes.sizeof(ctypes.c_double))
+>>> ctypes.resize(numbers, 9 * ctypes.sizeof(ctypes.c_double))
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+ValueError: minimum size is 80
+>>> numbers[:5] = range(5)
+>>> numbers[:]
+[0.0, 1.0, 2.0, 3.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+
@@ -0,0 +1,66 @@
+>>> import ctypes
+
+>>> class Point(ctypes.Structure):
+...     _fields_ = ('x', ctypes.c_int), ('y', ctypes.c_int)
+
+>>> class Vertex(ctypes.Structure):
+...     _fields_ = ('c', Point), ('d', Point)
+
+>>> a = Point(0, 1)
+>>> b = Point(2, 3)
+>>> a.x, a.y, b.x, b.y
+(0, 1, 2, 3)
+
+# Swap points a and b
+
+>>> a, b = b, a
+>>> a.x, a.y, b.x, b.y
+(2, 3, 0, 1)
+
+>>> v = Vertex()
+>>> v.c = Point(0, 1)
+>>> v.d = Point(2, 3)
+>>> v.c.x, v.c.y, v.d.x, v.d.y
+(0, 1, 2, 3)
+
+# Swap points c and d
+
+>>> v.c, v.d = v.d, v.c
+>>> v.c.x, v.c.y, v.d.x, v.d.y
+(2, 3, 2, 3)
+
+
+# Regular Python version to illustrate the difference:
+
+>>> import dataclasses
+
+>>> @dataclasses.dataclass
+... class Point:
+...     x: int
+...     y: int
+
+>>> @dataclasses.dataclass
+... class Vertex:
+...     c: Point
+...     d: Point
+
+>>> a = Point(0, 1)
+>>> b = Point(2, 3)
+>>> a.x, a.y, b.x, b.y
+(0, 1, 2, 3)
+
+# Swap points a and b
+
+>>> a, b = b, a
+>>> a.x, a.y, b.x, b.y
+(2, 3, 0, 1)
+
+>>> v = Vertex(c = Point(0, 1), d = Point(2, 3))
+>>> v.c.x, v.c.y, v.d.x, v.d.y
+(0, 1, 2, 3)
+
+# Swap points c and d
+
+>>> v.c, v.d = v.d, v.c
+>>> v.c.x, v.c.y, v.d.x, v.d.y
+(2, 3, 0, 1)
@@ -0,0 +1,9 @@
+>>> import cffi
+
+>>> ffi = cffi.FFI()
+>>> ffi.cdef('int printf(const char* format, ...);')
+>>> libc = ffi.dlopen(None)
+>>> arg = ffi.new('char[]', b'Printing using CFFI\n')
+>>> libc.printf(arg)
+20
+
@@ -0,0 +1,32 @@
+>>> from ctypes import util
+>>> import cffi
+
+# Initialize the FFI builder
+
+>>> ffi = cffi.FFI()
+
+# Find the libc library on OS X. Look back at the ctypes examples
+for other platforms.
+
+>>> library = util.find_library('libc.dylib')
+>>> library
+'/usr/lib/libc.dylib'
+
+# Load the library
+
+>>> libc = ffi.dlopen(library)
+>>> libc
+<cffi.api._make_ffi_library.<locals>.FFILibrary object at ...>
+
+# We do have printf available, but CFFI requires a signature
+
+>>> libc.printf
+Traceback (most recent call last):
+  ...
+AttributeError: printf
+
+# Define the printf signature and call printf
+
+>>> ffi.cdef('int printf(const char* format, ...);')
+>>> libc.printf
+<cdata 'int(*)(char *, ...)' ...>
@@ -0,0 +1,38 @@
+>>> import cffi
+
+>>> ffi = cffi.FFI()
+
+# Create the structures as C structs
+
+>>> ffi.cdef('''
+... typedef struct {
+...     int x;
+...     int y;
+... } point;
+...
+... typedef struct {
+...     point a;
+...     point b;
+... } vertex;
+... ''')
+
+# Create a vertex and return the pointer
+
+>>> v = ffi.new('vertex*')
+
+# Set the data
+
+>>> v.a.x, v.a.y, v.b.x, v.b.y = (0, 1, 2, 3)
+
+# Print before change
+
+>>> v.a.x, v.a.y, v.b.x, v.b.y
+(0, 1, 2, 3)
+
+>>> v.a, v.b = v.b, v.a
+
+# Print after change
+
+>>> v.a.x, v.a.y, v.b.x, v.b.y
+(2, 3, 2, 3)
+
@@ -0,0 +1,28 @@
+>>> import cffi
+
+>>> ffi = cffi.FFI()
+
+# Create arrays of size 10:
+
+>>> x = ffi.new('int[10]')
+>>> y = ffi.new('int[]', 10)
+
+>>> x
+<cdata 'int[10]' owning 40 bytes>
+>>> y
+<cdata 'int[]' owning 40 bytes>
+
+>>> x[0:10] = range(10)
+>>> list(x)
+[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+>>> y[:] = range(10)
+Traceback (most recent call last):
+    ...
+IndexError: slice start must be specified
+
+>>> x[0:100] = range(100)
+Traceback (most recent call last):
+    ...
+IndexError: index too large (expected 100 <= 10)
+
@@ -0,0 +1,39 @@
+>>> import cffi
+
+>>> ffi = cffi.FFI()
+
+# In API mode we can in-line the actual C code
+
+>>> ffi.set_source('_sum', '''
+... int sum(int* input, int n){
+...     int result = 0;
+...     while(n--)result += input[n];
+...     return result;
+... }
+... ''')
+
+>>> ffi.cdef('int sum(int*, int);')
+
+>>> library = ffi.compile()
+
+# Now we can import the library
+
+>>> import _sum
+
+# Or use `ffi.dlopen()` with the results from the compile step
+
+>>> _sum_lib = ffi.dlopen(library)
+
+# Create an array with 5 items
+
+>>> N = 5
+>>> array = ffi.new('int[]', N)
+>>> array[0:N] = range(N)
+
+# Call our C function from either the import or the dlopen
+
+>>> _sum.lib.sum(array, N)
+10
+
+>>> _sum_lib.sum(array, N)
+10
@@ -0,0 +1,7 @@
+import sys
+import pathlib
+
+# Little hack to add the current directory to sys.path so we can
+# find the imports
+path = pathlib.Path(__file__).parent
+sys.path.append(str(path.resolve()))
@@ -0,0 +1,15 @@
+
+long sum_of_squares(long n){
+    long total = 0;
+
+    /* The actual summing code */
+    for(int i=0; i<n; i++){
+        if((i * i) < n){
+            total += i * i;
+        }else{
+            break;
+        }
+    }
+
+    return total;
+}
@@ -0,0 +1,18 @@
+import pathlib
+import setuptools
+
+# Get the current directory
+PROJECT_PATH = pathlib.Path(__file__).parent
+
+sum_of_squares = setuptools.Extension('sum_of_squares', sources=[
+    # Get the relative path to sum_of_squares.c
+    str(PROJECT_PATH / 'sum_of_squares.c'),
+])
+
+if __name__ == '__main__':
+    setuptools.setup(
+        name='SumOfSquares',
+        version='1.0',
+        ext_modules=[sum_of_squares],
+    )
+
@@ -0,0 +1,49 @@
+#include <Python.h>
+
+static PyObject* sum_of_squares(PyObject *self, PyObject
+        *args){
+    /* Declare the variables */
+    int n;
+    int total = 0;
+
+    /* Parse the arguments */
+    if(!PyArg_ParseTuple(args, "i", &n)){
+        return NULL;
+    }
+
+    /* The actual summing code */
+    for(int i=0; i<n; i++){
+        if((i * i) < n){
+            total += i * i;
+        }else{
+            break;
+        }
+    }
+
+    /* Return the number but convert it to a Python object first
+     */
+    return PyLong_FromLong(total);
+}
+
+static PyMethodDef methods[] = {
+    /* Register the function */
+    {"sum_of_squares", sum_of_squares, METH_VARARGS,
+     "Sum the perfect squares below n"},
+    /* Indicate the end of the list */
+    {NULL, NULL, 0, NULL},
+};
+
+static struct PyModuleDef module = {
+    PyModuleDef_HEAD_INIT,
+    "sum_of_squares", /* Module name */
+    NULL, /* Module documentation */
+    -1, /* Module state, -1 means global. This parameter is
+           for sub-interpreters */
+    methods,
+};
+
+/* Initialize the module */
+PyMODINIT_FUNC PyInit_sum_of_squares(void){
+    return PyModule_Create(&module);
+}
+
@@ -0,0 +1,11 @@
+def sum_of_squares(n):
+    total = 0
+
+    for i in range(n):
+        if i * i < n:
+            total += i * i
+        else:
+            break
+
+    return total
+
@@ -0,0 +1,35 @@
+import sys
+import timeit
+import argparse
+import functools
+
+from sum_of_squares_py import sum_of_squares as sum_py
+
+try:
+    from sum_of_squares import sum_of_squares as sum_c
+except ImportError:
+    print('Please run "python setup.py build install" first')
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('repetitions', type=int)
+    parser.add_argument('maximum', type=int)
+    args = parser.parse_args()
+
+    timer = functools.partial(
+        timeit.timeit, number=args.repetitions, globals=globals())
+
+    print(f'Testing {args.repetitions} repetitions with maximum: '
+          f'{args.maximum}')
+
+    result = sum_c(args.maximum)
+    duration_c = timer('sum_c(args.maximum)')
+    print(f'C: {result} took {duration_c:.3f} seconds')
+
+    result = sum_py(args.maximum)
+    duration_py = timer('sum_py(args.maximum)')
+    print(f'Py: {result} took {duration_py:.3f} seconds')
+
+    print(f'C was {duration_py / duration_c:.1f} times faster')
@@ -0,0 +1,15 @@
+
+long sum_of_squares(long n){
+    long total = 0;
+
+    /* The actual summing code */
+    for(int i=0; i<n; i++){
+        if((i * i) < n){
+            total += i * i;
+        }else{
+            break;
+        }
+    }
+
+    return total;
+}
@@ -0,0 +1,17 @@
+import pathlib
+import setuptools
+
+# Get the current directory
+PROJECT_PATH = pathlib.Path(__file__).parent
+
+sum_of_large_squares = setuptools.Extension(
+    'sum_of_large_squares',
+    sources=[str(PROJECT_PATH / 'sum_of_large_squares.c')])
+
+if __name__ == '__main__':
+    setuptools.setup(
+        name='SumOfSquares',
+        version='1.0',
+        ext_modules=[sum_of_large_squares],
+    )
+
@@ -0,0 +1,49 @@
+#include <Python.h>
+
+typedef unsigned long long int bigint;
+
+static PyObject* sum_of_large_squares(PyObject *self, PyObject *args){
+    /* Declare the variables */
+    bigint n;
+    bigint total = 0;
+
+    /* Parse the arguments */
+    if(!PyArg_ParseTuple(args, "K", &n)){
+        return NULL;
+    }
+
+    /* The actual summing code */
+    for(bigint i=0; i<n; i++){
+        if((i * i) < n){
+            total += i * i;
+        }else{
+            break;
+        }
+    }
+
+    /* Return the number but convert it to a Python object first */
+    return PyLong_FromUnsignedLongLong(total);
+}
+
+static PyMethodDef methods[] = {
+    /* Register the function */
+    {"sum_of_large_squares", sum_of_large_squares, METH_VARARGS,
+     "Sum the perfect squares below n"},
+    /* Indicate the end of the list */
+    {NULL, NULL, 0, NULL},
+};
+
+static struct PyModuleDef module = {
+    PyModuleDef_HEAD_INIT,
+    "sum_of_large_squares", /* Module name */
+    NULL, /* Module documentation */
+    -1, /* Module state, -1 means global. This parameter is
+           for sub-interpreters */
+    methods,
+};
+
+/* Initialize the module */
+PyMODINIT_FUNC PyInit_sum_of_large_squares(void){
+    return PyModule_Create(&module);
+}
+
@@ -0,0 +1,11 @@
+def sum_of_squares(n):
+    total = 0
+
+    for i in range(n):
+        if i * i < n:
+            total += i * i
+        else:
+            break
+
+    return total
+
@@ -0,0 +1,35 @@
+import sys
+import timeit
+import argparse
+import functools
+
+from sum_of_squares_py import sum_of_squares as sum_py
+
+try:
+    from sum_of_large_squares import sum_of_large_squares as sum_c
+except ImportError:
+    print('Please run "python setup.py build install" first')
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('repetitions', type=int)
+    parser.add_argument('maximum', type=int)
+    args = parser.parse_args()
+
+    timer = functools.partial(
+        timeit.timeit, number=args.repetitions, globals=globals())
+
+    print(f'Testing {args.repetitions} repetitions with maximum: '
+          f'{args.maximum}')
+
+    result = sum_c(args.maximum)
+    duration_c = timer('sum_c(args.maximum)')
+    print(f'C: {result} took {duration_c:.3f} seconds')
+
+    result = sum_py(args.maximum)
+    duration_py = timer('sum_py(args.maximum)')
+    print(f'Py: {result} took {duration_py:.3f} seconds')
+
+    print(f'C was {duration_py / duration_c:.1f} times faster')
@@ -0,0 +1,27 @@
+static PyObject* function(
+        PyObject *self,
+        PyObject *args,
+        PyObject *kwargs){
+    /* Declare the variables */
+    PyObject* callback;
+    int n;
+
+    static char* keywords[] = {"callback", "n", NULL};
+
+    /* Parse the arguments */
+    if(!PyArg_ParseTupleAndKeywords(args, kwargs, "Oi", keywords,
+                &callback, &n)){
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef methods[] = {
+    /* Register the function with kwargs */
+    {"function", function, METH_VARARGS | METH_KEYWORDS,
+     "Some kwargs function"},
+    /* Indicate the end of the list */
+    {NULL, NULL, 0, NULL},
+};
+
@@ -0,0 +1,4 @@
+static PyObject* count_eggs(PyObject *self, PyObject *args){
+    PyErr_SetString(PyExc_RuntimeError, "Too many eggs!");
+    return NULL;
+}
@@ -0,0 +1,95 @@
+#include <Python.h>
+
+static PyObject* custom_sum(PyObject* self, PyObject* args){
+    /* Declare all variables, note that the values for total and
+     * callback are defaults in the case these arguments are not
+     * specified */
+    long long int total = 0;
+    int overflow = 0;
+    PyObject* iterator;
+    PyObject* iterable;
+    PyObject* callback = NULL;
+    PyObject* value;
+    PyObject* item;
+
+    /* Now we parse a PyObject* followed by, optionally
+     * (the | character), a PyObject* and a long long int */
+    if(!PyArg_ParseTuple(args, "O|OL", &iterable, &callback,
+                &total)){
+        return NULL;
+    }
+
+    /* See if we can create an iterator from the iterable. This is
+     * effectively the same as doing iter(iterable) in Python */
+    iterator = PyObject_GetIter(iterable);
+    if(iterator == NULL){
+        PyErr_SetString(PyExc_TypeError,
+                "Argument is not iterable");
+        return NULL;
+    }
+
+    /* Check if the callback exists or wasn't specified. If it was
+     * specified check whether it's callable or not */
+    if(callback != NULL && !PyCallable_Check(callback)){
+        PyErr_SetString(PyExc_TypeError,
+                "Callback is not callable");
+        return NULL;
+    }
+
+    /* Loop through all items of the iterable */
+    while((item = PyIter_Next(iterator))){
+        /* If we have a callback available, call it. Otherwise
+         * just return the item as the value */
+        if(callback == NULL){
+            value = item;
+        }else{
+            value = PyObject_CallFunction(callback, "O", item);
+        }
+
+        /* Add the value to total and check for overflows */
+        total += PyLong_AsLongLongAndOverflow(value, &overflow);
+        if(overflow > 0){
+            PyErr_SetString(PyExc_RuntimeError,
+                    "Integer overflow");
+            return NULL;
+        }else if(overflow < 0){
+            PyErr_SetString(PyExc_RuntimeError,
+                    "Integer underflow");
+            return NULL;
+        }
+
+        /* If we were indeed using the callback, decrease the
+         * reference count to the value because it is a separate
+         * object now */
+        if(callback != NULL){
+            Py_DECREF(value);
+        }
+        Py_DECREF(item);
+    }
+    Py_DECREF(iterator);
+
+    return PyLong_FromLongLong(total);
+}
+
+static PyMethodDef methods[] = {
+    /* Register the function */
+    {"custom_sum", custom_sum, METH_VARARGS,
+     "Sum the given numbers"},
+    /* Indicate the end of the list */
+    {NULL, NULL, 0, NULL},
+};
+
+static struct PyModuleDef module = {
+    PyModuleDef_HEAD_INIT,
+    "custom_sum", /* Module name */
+    NULL, /* Module documentation */
+    -1, /* Module state, -1 means global. This parameter is
+           for sub-interpreters */
+    methods,
+};
+
+/* Initialize the module */
+PyMODINIT_FUNC PyInit_custom_sum(void){
+    return PyModule_Create(&module);
+}
+
@@ -0,0 +1,11 @@
+import platform
+from ctypes import cdll
+
+__all__ = 'libc'
+
+if platform.system() == 'Windows':
+    libc = cdll.msvcrt
+elif platform.system() == 'Darwin':
+    libc = cdll.LoadLibrary('libc.dylib')
+else:
+    libc = cdll.LoadLibrary('libc.so.6')
@@ -0,0 +1,7 @@
+import sys
+import pathlib
+
+# Little hack to add the current directory to sys.path so we can
+# find the imports
+path = pathlib.Path(__file__).parent
+sys.path.append(str(path.resolve()))