Adding source code for chapter 8
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
#casestudy2.py: word count application
|
||||
import matplotlib.pyplot as plt
|
||||
from pyspark.sql import SparkSession
|
||||
from wordcloud import WordCloud
|
||||
|
||||
spark = SparkSession.builder.master("local[*]")\
|
||||
.appName("word cloud app")\
|
||||
.getOrCreate()
|
||||
|
||||
wc_threshold = 1
|
||||
wl_threshold = 3
|
||||
textRDD = spark.sparkContext.textFile('wordcloud.txt',3)
|
||||
flatRDD = textRDD.flatMap(lambda x: x.split(' '))
|
||||
wcRDD = flatRDD.map(lambda word: (word, 1)).\
|
||||
reduceByKey(lambda v1, v2: v1 + v2)
|
||||
|
||||
# filter out words with fewer than threshold occurrences
|
||||
filteredRDD = wcRDD.filter(lambda pair: pair[1] >= wc_threshold)
|
||||
filteredRDD2 = filteredRDD.filter(lambda pair:
|
||||
len(pair[0]) > wl_threshold)
|
||||
|
||||
word_freq = dict(filteredRDD2.collect())
|
||||
|
||||
# Create the wordcloud object
|
||||
wordcloud = WordCloud(width=480, height=480, margin=0).\
|
||||
generate_from_frequencies(word_freq)
|
||||
|
||||
# Display the generated cloud image
|
||||
plt.imshow(wordcloud, interpolation='bilinear')
|
||||
plt.axis("off")
|
||||
plt.margins(x=0, y=0)
|
||||
plt.show()
|
||||
Reference in New Issue
Block a user