Adding source code for chapter 8

This commit is contained in:
muassif
2021-05-31 18:04:55 +04:00
committed by GitHub
parent f88527eef4
commit ac10d74a74
13 changed files with 360 additions and 0 deletions
+19
View File
@@ -0,0 +1,19 @@
#rddtransform1.py: rdd tranformation function
#please ignore next 2 statements if running directly in PySpark shell
import time
from pyspark.sql import SparkSession
spark = SparkSession.builder.master("local[*]")\
.appName("RDD Test app")\
.getOrCreate()
rdd1 = spark.sparkContext.textFile('sample.txt')
#print(rdd1.getNumPartitions())
rdd2 = rdd1.map(lambda lines: lines.lower())
rdd3 = rdd1.map(lambda lines: lines.upper())
print(rdd2.collect())
print(rdd3.collect())
time.sleep(60)