Mastering Python Second Edition Release Code

CH_15_scientific_python/T_00_numpy.rst

@@ -0,0 +1,73 @@
+# A commonly used shorthand for numpy is np
+
+>>> import numpy as np
+
+# Generate an array of numbers from 0 up to 1 million
+
+>>> a = np.arange(1000000)
+>>> a
+array([        0,         1, ..., 999998, 999999])
+
+# Change the shape (still references the same data) to a
+2-dimensional 1000x1000 array
+
+>>> b = a.reshape((1000, 1000))
+>>> b
+array([[     0,      1,      2, ...,    997,    998,    999],
+       [  1000,   1001,   1002, ...,   1997,   1998,   1999],
+       ...,
+       [998000, 998001, 998002, ..., 998997, 998998, 998999],
+       [999000, 999001, 999002, ..., 999997, 999998, 999999]])
+
+# The first row of the matrix
+
+>>> b[0]
+array([  0,   1,   2,   3, ..., 995, 996, 997, 998, 999])
+
+# The first column of the matrix
+
+>>> b[:, 0]
+array([     0,   1000,   2000,   ..., 997000, 998000, 999000])
+
+# Row 10 up to 12, the even columns between 20 and 30
+
+>>> b[10:12, 20:30:2]
+array([[10020, 10022, 10024, 10026, 10028],
+       [11020, 11022, 11024, 11026, 11028]])
+
+# Row 10, columns 5 up to 10:
+
+>>> b[10, 5:10]
+array([10005, 10006, 10007, 10008, 10009])
+
+# Alternative syntax for the last slice
+
+>>> b[10][5:10]
+array([10005, 10006, 10007, 10008, 10009])
+
+##################################################################
+
+>>> b[0] *= 10
+>>> b[:, 0] *= 20
+
+>>> a
+array([     0,     10,     20, ..., 999997, 999998, 999999])
+>>> b[0:2]
+array([[    0,    10,    20, ...,  9970,  9980,  9990],
+       [20000,  1001,  1002, ...,  1997,  1998,  1999]])
+
+##################################################################
+
+>>> a = list(range(10000))
+
+>>> def dot(xs, ys):
+...     total = 0
+...     for x, y in zip(xs, ys):
+...         total += x * y
+...     return total
+
+>>> dot(a, a)
+333283335000
+
+>>> np.dot(a, a)
+333283335000

CH_15_scientific_python/T_01_numba.rst

@@ -0,0 +1,15 @@
+>>> import numpy
+>>> import numba
+
+>>> numbers = numpy.arange(500, dtype=numpy.int64)
+
+>>> @numba.vectorize([numba.int64(numba.int64)])
+... def add_one(x):
+...     return x + 1
+
+>>> numbers
+array([  0,   1,   2, ..., 498, 499])
+
+>>> add_one(numbers)
+array([  1,   2,   3, ..., 499, 500])
+

CH_15_scientific_python/T_02_scipy.rst

@@ -0,0 +1,20 @@
+>>> import numpy
+
+>>> from scipy import sparse
+
+>>> x = numpy.identity(10000)
+>>> y = sparse.identity(10000)
+
+>>> x.data.nbytes
+800000000
+
+# Summing the memory usage of scipy.sparse objects require summing
+of all internal arrays. We can test for these arrays using the
+nbytes attribute.
+
+>>> arrays = [a for a in vars(y).values() if hasattr(a, 'nbytes')]
+
+# sum the bytes from all arrays
+
+>>> sum(a.nbytes for a in arrays)
+80004

CH_15_scientific_python/T_03_pandas.rst

@@ -0,0 +1,211 @@
+# A commonly used shorthand for pandas is pd
+
+>>> import pandas as pd
+>>> import re
+>>> import io
+
+>>> data = '''
+... Version\tLatest micro version\tRelease date\tEnd of full support\tEnd of security fixes
+... 0.9\t0.9.9[2]\t1991-02-20[2]\t1993-07-29[a][2]
+... 1.0\t1.0.4[2]\t1994-01-26[2]\t1994-02-15[a][2]
+... 1.1\t1.1.1[2]\t1994-10-11[2]\t1994-11-10[a][2]
+... 1.2\t\t1995-04-13[2]\tUnsupported
+... 1.3\t\t1995-10-13[2]\tUnsupported
+... 1.4\t\t1996-10-25[2]\tUnsupported
+... 1.5\t1.5.2[39]\t1998-01-03[2]\t1999-04-13[a][2]
+... 1.6\t1.6.1[39]\t2000-09-05[40]\t2000-09[a][39]
+... 2.0\t2.0.1[41]\t2000-10-16[42]\t2001-06-22[a][41]
+... 2.1\t2.1.3[41]\t2001-04-15[43]\t2002-04-09[a][41]
+... 2.2\t2.2.3[41]\t2001-12-21[44]\t2003-05-30[a][41]
+... 2.3\t2.3.7[41]\t2003-06-29[45]\t2008-03-11[a][41]
+... 2.4\t2.4.6[41]\t2004-11-30[46]\t2008-12-19[a][41]
+... 2.5\t2.5.6[41]\t2006-09-19[47]\t2011-05-26[a][41]
+... 2.6\t2.6.9[26]\t2008-10-01[26]\t2010-08-24[b][26]\t2013-10-29[26]
+... 2.7\t2.7.18[31]\t2010-07-03[31]\t2020-01-01[c][31]
+... 3.0\t3.0.1[41]\t2008-12-03[26]\t2009-06-27[48]
+... 3.1\t3.1.5[49]\t2009-06-27[49]\t2011-06-12[50]\t2012-06[49]
+... 3.2\t3.2.6[51]\t2011-02-20[51]\t2013-05-13[b][51]\t2016-02-20[51]
+... 3.3\t3.3.7[52]\t2012-09-29[52]\t2014-03-08[b][52]\t2017-09-29[52]
+... 3.4\t3.4.10[53]\t2014-03-16[53]\t2017-08-09[54]\t2019-03-18[a][53]
+... 3.5\t3.5.10[55]\t2015-09-13[55]\t2017-08-08[56]\t2020-09-30[55]
+... 3.6\t3.6.13[57]\t2016-12-23[57]\t2018-12-24[b][57]\t2021-12[57]
+... 3.7\t3.7.10[58]\t2018-06-27[58]\t2020-06-27[b][58]\t2023-06[58]
+... 3.8\t3.8.10[59]\t2019-10-14[59]\t2021-05-03[59]\t2024-10[59]
+... 3.9\t3.9.5[60]\t2020-10-05[60]\t2022-05[61]\t2025-10[60][61]
+... 3.10\t\t2021-10-04[62]\t2023-05[62]\t2026-10[62]
+... '''.strip()
+
+# Slightly cleanup data by removing references
+
+>>> data = re.sub(r'\[.+?\]', '', data)
+
+# df is often used as a shorthand for pandas.DataFrame
+
+>>> df = pd.read_table(io.StringIO(data))
+
+# List the columns
+
+>>> df.columns
+Index(['Version', ..., 'Release date', ...], dtype='object')
+
+# List the versions:
+
+>>> df['Version']
+0     0.9
+...
+25    3.9
+26    3.1
+Name: Version, dtype: float64
+
+# Oops... where did Python 3.10 go in the output above? The
+conversion to float trimmed the 0 so we need to disable that.
+
+>>> df = pd.read_table(io.StringIO(data), dtype=dict(Version=str))
+
+# Much better, we didn't lose the version info this time
+
+>>> df['Version']
+0      0.9
+...
+25     3.9
+26     3.10
+Name: Version, dtype: object
+
+# The release date is read as a string by default, so we convert
+it to a datetime:
+
+>>> df['Release date'] = pd.to_datetime(df['Release date'])
+
+>>> df['Release date']
+0      1991-02-20
+...
+26     2021-10-04
+Name: Release date, dtype: datetime64[ns]
+
+# Let's see which month is the most popular for Python releases.
+First we run groupby() on the release month and after that we
+run a count() on the version:
+
+>>> df.groupby([df['Release date'].dt.month])['Version'].count()
+Release date
+1     2
+2     2
+3     1
+4     2
+6     3
+7     1
+9     4
+10    8
+11    1
+12    3
+Name: Version, dtype: int64
+
+#################################################################
+
+>>> import pandas as pd
+>>> import numpy as np
+
+>>> df = pd.DataFrame(dict(
+...     building=['x', 'x', 'y', 'x', 'x', 'y', 'z', 'z', 'z'],
+...     rooms=['a', 'a', 'a', 'b', 'b', 'b', 'c', 'c', 'c'],
+...     hours=[10, 11, 12, 10, 11, 12, 10, 11, 12],
+...     
+...     temperatures=np.arange(0.0, 9.0),
+... ))
+
+>>> df
+  building rooms  hours  temperatures
+0        x     a     10           0.0
+1        x     a     11           1.0
+...
+7        z     c     11           7.0
+8        z     c     12           8.0
+
+>>> pd.pivot_table(
+...     df, values='temperatures', index=['rooms'],
+...     columns=['hours'], aggfunc=np.mean)
+hours   10   11   12
+rooms
+a      0.0  1.0  2.0
+b      3.0  4.0  5.0
+c      6.0  7.0  8.0
+
+>>> pd.pivot_table(
+...     df, values='temperatures', index=['building', 'rooms'],
+...     columns=['hours'], aggfunc=np.mean)
+hours            10   11   12
+building rooms
+x        a      0.0  1.0  NaN
+         b      3.0  4.0  NaN
+y        a      NaN  NaN  2.0
+         b      NaN  NaN  5.0
+z        c      6.0  7.0  8.0
+
+>>> df.groupby(pd.Grouper(key='hours')).mean()
+       temperatures
+hours
+10              3.0
+11              4.0
+12              5.0
+
+#################################################################
+
+>>> import pandas as pd
+>>> import numpy as np
+
+>>> df = pd.DataFrame(dict(
+...     building=['x', 'x', 'y', 'x', 'x', 'y', 'z', 'z', 'z'],
+...     rooms=['a', 'a', 'a', 'b', 'b', 'b', 'c', 'c', 'c'],
+...     hours=[10, 11, 12, 10, 11, 12, 10, 11, 12],
+...     
+...     temperatures=np.arange(0.0, 9.0),
+... ))
+
+>>> df
+  building rooms  hours  temperatures
+0        x     a     10           0.0
+1        x     a     11           1.0
+...
+7        z     c     11           7.0
+8        z     c     12           8.0
+
+>>> pd.pivot_table(
+...     df, values='temperatures', index=['rooms'],
+...     columns=['hours'], aggfunc=np.mean)
+hours   10   11   12
+rooms
+a      0.0  1.0  2.0
+b      3.0  4.0  5.0
+c      6.0  7.0  8.0
+
+>>> pd.pivot_table(
+...     df, values='temperatures', index=['building', 'rooms'],
+...     columns=['hours'], aggfunc=np.mean)
+hours            10   11   12
+building rooms
+x        a      0.0  1.0  NaN
+         b      3.0  4.0  NaN
+y        a      NaN  NaN  2.0
+         b      NaN  NaN  5.0
+z        c      6.0  7.0  8.0
+
+#################################################################
+
+>>> import pandas as pd
+>>> import numpy as np
+
+>>> pd_series = pd.Series(np.arange(100))  # [0, 1, 2, ... 99]
+
+# Create a  rolling Window with size 10
+
+>>> window = pd_series.rolling(10)
+
+# Calculate the running mean and ignore the N/A values at the
+beginning before the window is full:
+
+>>> window.mean().dropna()
+9      4.5
+10     5.5
+      ...
+99    94.5
+Length: 91, dtype: float64

CH_15_scientific_python/T_04_statsmodels.rst

@@ -0,0 +1,22 @@
+# The common shorthand for statsmodels is sm
+
+>>> import statsmodels.api as sm
+>>> import numpy as np
+
+>>> Y = np.arange(8)
+>>> X = np.ones(8)
+
+# Create the weighted-least-squares model
+
+>>> model = sm.WLS(Y, X)
+
+# Fit the model and generate the regression results
+
+>>> fit = model.fit()
+
+# Show the estimated parameters and the t-values:
+
+>>> fit.params
+array([3.5])
+>>> fit.tvalues
+array([4.04145188])

CH_15_scientific_python/T_05_xarray.rst

@@ -0,0 +1,77 @@
+>>> import re
+>>> import io
+>>> import pandas as pd
+
+>>> data = '''
+... Version\tLatest micro version\tRelease date\tEnd of full support\tEnd of security fixes
+... 0.9\t0.9.9[2]\t1991-02-20[2]\t1993-07-29[a][2]
+... 1.0\t1.0.4[2]\t1994-01-26[2]\t1994-02-15[a][2]
+... 1.1\t1.1.1[2]\t1994-10-11[2]\t1994-11-10[a][2]
+... 1.2\t\t1995-04-13[2]\tUnsupported
+... 1.3\t\t1995-10-13[2]\tUnsupported
+... 1.4\t\t1996-10-25[2]\tUnsupported
+... 1.5\t1.5.2[39]\t1998-01-03[2]\t1999-04-13[a][2]
+... 1.6\t1.6.1[39]\t2000-09-05[40]\t2000-09[a][39]
+... 2.0\t2.0.1[41]\t2000-10-16[42]\t2001-06-22[a][41]
+... 2.1\t2.1.3[41]\t2001-04-15[43]\t2002-04-09[a][41]
+... 2.2\t2.2.3[41]\t2001-12-21[44]\t2003-05-30[a][41]
+... 2.3\t2.3.7[41]\t2003-06-29[45]\t2008-03-11[a][41]
+... 2.4\t2.4.6[41]\t2004-11-30[46]\t2008-12-19[a][41]
+... 2.5\t2.5.6[41]\t2006-09-19[47]\t2011-05-26[a][41]
+... 2.6\t2.6.9[26]\t2008-10-01[26]\t2010-08-24[b][26]\t2013-10-29[26]
+... 2.7\t2.7.18[31]\t2010-07-03[31]\t2020-01-01[c][31]
+... 3.0\t3.0.1[41]\t2008-12-03[26]\t2009-06-27[48]
+... 3.1\t3.1.5[49]\t2009-06-27[49]\t2011-06-12[50]\t2012-06[49]
+... 3.2\t3.2.6[51]\t2011-02-20[51]\t2013-05-13[b][51]\t2016-02-20[51]
+... 3.3\t3.3.7[52]\t2012-09-29[52]\t2014-03-08[b][52]\t2017-09-29[52]
+... 3.4\t3.4.10[53]\t2014-03-16[53]\t2017-08-09[54]\t2019-03-18[a][53]
+... 3.5\t3.5.10[55]\t2015-09-13[55]\t2017-08-08[56]\t2020-09-30[55]
+... 3.6\t3.6.13[57]\t2016-12-23[57]\t2018-12-24[b][57]\t2021-12[57]
+... 3.7\t3.7.10[58]\t2018-06-27[58]\t2020-06-27[b][58]\t2023-06[58]
+... 3.8\t3.8.10[59]\t2019-10-14[59]\t2021-05-03[59]\t2024-10[59]
+... 3.9\t3.9.5[60]\t2020-10-05[60]\t2022-05[61]\t2025-10[60][61]
+... 3.10\t\t2021-10-04[62]\t2023-05[62]\t2026-10[62]
+... '''.strip()
+
+>>> data = re.sub(r'\[.+?\]', '', data)
+>>> df = pd.read_table(io.StringIO(data), dtype=dict(Version=str))
+>>> df['Release date'] = pd.to_datetime(df['Release date'])
+
+# The code above is copied from T_03_pandas.rst
+
+# The common shorthand for xarray is xr
+
+>>> import xarray as xr
+
+>>> ds = xr.Dataset.from_dataframe(df)
+
+# For reference, the pandas version of the groupby
+df.groupby([df['Release date'].dt.month])['Version'].count()
+
+>>> ds.groupby('Release date.month').count()['Version']
+<xarray.DataArray 'Version' (month: 10)>
+array([2, 2, 1, 2, 3, 1, 4, 8, 1, 3])
+Coordinates:
+  * month    (month) int64 1 2 3 4 6 7 9 10 11 12
+
+#################################################################
+
+>>> import xarray as xr
+>>> import numpy as np
+
+>>> points = np.arange(27).reshape((3, 3, 3))
+>>> triangles = np.arange(27).reshape((3, 3, 3))
+>>> ds = xr.Dataset(dict(
+...     triangles=(['p0', 'p1', 'p2'], triangles),
+... ), coords=dict(
+...     points=(['x', 'y', 'z'], points),
+... ))
+
+>>> ds
+<xarray.Dataset>
+Dimensions:    (p0: 3, p1: 3, p2: 3, x: 3, y: 3, z: 3)
+Coordinates:
+    points     (x, y, z) int64 0 1 2 3 4 5 ... 21 22 23 24 25 26
+Dimensions without coordinates: p0, p1, p2, x, y, z
+Data variables:
+    triangles  (p0, p1, p2) int64 0 1 2 3 4 ... 21 22 23 24 25 26

CH_15_scientific_python/T_06_stumpy.rst

@@ -0,0 +1,35 @@
+>>> import numpy as np
+>>> import stumpy
+
+>>> temperatures = np.array([22., 21., 22., 21., 22., 23.])
+
+>>> window_size = 3
+
+# Calculate a Euclidean distance matrix between the windows
+
+>>> stump = stumpy.stump(temperatures, window_size)
+
+# Show the distance matrix. The row number is the index in the
+input array. The first column is the distance, the next columns
+are the indices of the nearest match, the left match and the
+right match.
+
+>>> stump
+array([[0.0, 2, -1, 2],
+      [2.449489742783178, 3, -1, 3],
+      [0.0, 0, 0, -1],
+      [2.449489742783178, 1, 1, -1]], dtype=object)
+
+# As we can see in the matrix above, the first window has a
+distance of 0 to the window at index 2 meaning that they are
+identical. We can easily verify that by showing both windows:
+
+# The first window:
+
+>>> temperatures[0:window_size]
+array([22., 21., 22.])
+
+# The window at index 2:
+
+>>> temperatures[2:2 + window_size]
+array([22., 21., 22.])

CH_15_scientific_python/T_07_gmpy2.rst

@@ -0,0 +1,4 @@
+>>> import gmpy2
+
+>>> gmpy2.const_pi(1000)
+mpfr('3.14159265358979...33936072602491412736',1000)

CH_15_scientific_python/T_08_sagemath.sage

@@ -0,0 +1,2 @@
+sage: x, y, z = var('x, y, z')
+sage: solve([x + y == 10, x - y == 5, x + y + z == 1], x, y, z)

CH_15_scientific_python/T_09_mpmath.rst

@@ -0,0 +1,48 @@
+>>> N = 10
+>>> x = 0.1
+
+# Regular addition
+
+>>> a = 0.0
+>>> for _ in range(N):
+...     a += x
+
+>>> a
+0.9999999999999999
+
+# Using sum, the same result as addition
+
+>>> sum(x for _ in range(N))
+0.9999999999999999
+
+#################################################################
+
+# Sum using Python's optimized fsum:
+
+>>> import math
+
+>>> math.fsum(x for _ in range(N))
+1.0
+
+#################################################################
+
+>>> import mpmath
+
+# Increase the mpmath precision to 100 decimal places
+
+>>> mpmath.mp.dps = 100
+>>> y = mpmath.mpf('0.1')
+
+# Using mpmath with addition:
+
+>>> b = mpmath.mpf('0.0')
+>>> for _ in range(N):
+...     b += y
+
+>>> b
+mpf('1.00000000000000000000000000...00000000000000000000000014')
+
+# Or a regular sum with mpmath:
+
+>>> sum(y for _ in range(N))
+mpf('1.00000000000000000000000000...00000000000000000000000014')

CH_15_scientific_python/T_10_sympy.rst

@@ -0,0 +1,12 @@
+>>> from sympy import *
+
+>>> init_printing(use_unicode=True)
+>>> x, y, z = symbols('x y z')
+
+>>> integral = Integral(x * cos(x), x)
+>>> integral
+⌠
+⎮ x⋅cos(x) dx
+⌡
+>>> integral.doit()
+x⋅sin(x) + cos(x)

CH_15_scientific_python/T_11_patsy.rst

@@ -0,0 +1,18 @@
+>>> import patsy
+>>> import numpy as np
+
+>>> array = np.arange(2, 6)
+
+>>> data = dict(a=array, b=array, c=array)
+>>> patsy.dmatrix('a + np.square(b) + np.power(c, 3)', data)
+DesignMatrix with shape (4, 4)
+  Intercept  a  np.square(b)  np.power(c, 3)
+          1  2             4               8
+          1  3             9              27
+          1  4            16              64
+          1  5            25             125
+  Terms:
+    'Intercept' (column 0)
+    'a' (column 1)
+    'np.square(b)' (column 2)
+    'np.power(c, 3)' (column 3)