Adding source code for chapter 8

Chapter8/rdd/rddaction1.py

@@ -0,0 +1,24 @@
+#rddaction1.py: rdd action functions
+
+#please ignore next 2 statements if running directly in PySpark shell
+import time
+
+from pyspark.sql import SparkSession
+spark = SparkSession.builder.master("local[*]")\
+    .appName("RDD Test app")\
+    .getOrCreate()
+
+data = [5, 4, 6, 3, 2, 8, 9, 2, 8, 7,
+        8, 4, 4, 8, 2, 7, 8, 9, 6, 9]
+rdd1 = spark.sparkContext.parallelize(data)
+
+print("RDD contents with partitions: "+rdd1.glom().collect())
+print("Count by values: "+rdd1.countByValue())
+print("reduce function"+rdd1.reduce(lambda a,b: a+b))
+print("Sum of RDD contents"+rdd1.sum())
+print(""+rdd1.top(5))
+print(rdd1.count())
+print(rdd1.max())
+print(rdd1.min())
+
+time.sleep(60)

Chapter8/rdd/rddcreate.py

@@ -0,0 +1,16 @@
+#rddcreate.py: to create rdd from a collection and from a file
+
+#please ignore next 2 statements if running directly in PySpark shell
+from pyspark.sql import SparkSession
+spark = SparkSession.builder.master("local[*]")\
+    .appName("RDD Test app")\
+    .getOrCreate()
+
+data = [5, 4, 6, 3, 2, 8, 9, 2, 8, 7,
+        8, 4, 4, 8, 2, 7, 8, 9, 6, 9]
+rdd1 = spark.sparkContext.parallelize(data)
+print(rdd1.getNumPartitions())
+
+
+rdd2 = spark.sparkContext.textFile('sample.txt')
+print(rdd2.getNumPartitions())

Chapter8/rdd/rddtranform1.py

@@ -0,0 +1,19 @@
+#rddtransform1.py: rdd tranformation function
+
+#please ignore next 2 statements if running directly in PySpark shell
+import time
+
+from pyspark.sql import SparkSession
+spark = SparkSession.builder.master("local[*]")\
+    .appName("RDD Test app")\
+    .getOrCreate()
+
+rdd1 = spark.sparkContext.textFile('sample.txt')
+#print(rdd1.getNumPartitions())
+rdd2 = rdd1.map(lambda lines: lines.lower())
+rdd3 = rdd1.map(lambda lines: lines.upper())
+
+print(rdd2.collect())
+print(rdd3.collect())
+
+time.sleep(60)

Chapter8/rdd/rddtransform2.py

@@ -0,0 +1,13 @@
+#rddtransform2.py: rdd tranformation function-map
+
+#please ignore next 2 statements if running directly in PySpark shell
+from pyspark.sql import SparkSession
+spark = SparkSession.builder.master("local[*]")\
+    .appName("RDD Test app")\
+    .getOrCreate()
+
+data = [5, 4, 6, 3, 2, 8, 9, 2, 8, 7,
+        8, 4, 4, 8, 2, 7, 8, 9, 6, 9]
+rdd1 = spark.sparkContext.parallelize(data)
+rdd2 = rdd1.filter(lambda x: x % 2 !=0 )
+print(rdd2.collect())

Chapter8/rdd/sample.txt

@@ -0,0 +1,4 @@
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...