Karan Solanki
33 lines
1007 B
Python
33 lines
1007 B
Python
#casestudy2.py: word count application
|
|
import matplotlib.pyplot as plt
|
|
from pyspark.sql import SparkSession
|
|
from wordcloud import WordCloud
|
|
|
|
spark = SparkSession.builder.master("local[*]")\
|
|
.appName("word cloud app")\
|
|
.getOrCreate()
|
|
|
|
wc_threshold = 1
|
|
wl_threshold = 3
|
|
textRDD = spark.sparkContext.textFile('wordcloud.txt',3)
|
|
flatRDD = textRDD.flatMap(lambda x: x.split(' '))
|
|
wcRDD = flatRDD.map(lambda word: (word, 1)).\
|
|
reduceByKey(lambda v1, v2: v1 + v2)
|
|
|
|
# filter out words with fewer than threshold occurrences
|
|
filteredRDD = wcRDD.filter(lambda pair: pair[1] >= wc_threshold)
|
|
filteredRDD2 = filteredRDD.filter(lambda pair:
|
|
len(pair[0]) > wl_threshold)
|
|
|
|
word_freq = dict(filteredRDD2.collect())
|
|
|
|
# Create the wordcloud object
|
|
wordcloud = WordCloud(width=480, height=480, margin=0).\
|
|
generate_from_frequencies(word_freq)
|
|
|
|
# Display the generated cloud image
|
|
plt.imshow(wordcloud, interpolation='bilinear')
|
|
plt.axis("off")
|
|
plt.margins(x=0, y=0)
|
|
plt.show()
|