uploading chapter 9 source code

Chapter9/dataflow/my_wordcount.py

@@ -0,0 +1,61 @@
+# wordcount.py: count words in a text file
+
+import argparse
+import os
+import re
+
+import apache_beam as beam
+from apache_beam.io import ReadFromText
+from apache_beam.io import WriteToText
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.options.pipeline_options import SetupOptions
+
+def run(argv=None, save_main_session=True):
+  os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/Users/muasif/gcd-projs/gcp-key/word-count-316612-f22f7ffcc2dd.json"
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--input',
+      dest='input',
+      default='gs://muasif/input/sample.txt',
+      help='Input file to process.')
+  parser.add_argument(
+      '--output',
+      dest='output',
+      default='gs://muasif/output/result',
+      help='Output file to write results to.')
+  known_args, pipeline_args = parser.parse_known_args(argv)
+  pipeline_args.extend([
+      '--runner=DataflowRunner',
+      '--project=word-count-316612',
+      '--region=us-central1',
+      '--staging_location=gs://muasif/staging',
+      '--temp_location=gs://muasif/temp',
+      '--job_name=my-wordcount-job',
+  ])
+
+  pipeline_options = PipelineOptions(pipeline_args)
+  pipeline_options.view_as(SetupOptions).\
+      save_main_session = save_main_session
+  with beam.Pipeline(options=pipeline_options) as p:
+
+    lines = p | ReadFromText(known_args.input)
+    # Count the occurrences of each word.
+    counts = (
+        lines
+        | 'Split words' >> (
+            beam.FlatMap(
+                lambda x: re.findall(r'[A-Za-z\']+', x)).
+                with_output_types(str))
+        | 'Pair with 1' >> beam.Map(lambda x: (x, 1))
+        | 'Group & Sum' >> beam.CombinePerKey(sum))
+
+    # Format the word counts into a PCollection of strings.
+    def format_result(word_count):
+      (word, count) = word_count
+      return '%s: %s' % (word, count)
+
+    output = counts | 'Format' >> beam.Map(format_result)
+    output | WriteToText(known_args.output)
+
+if __name__ == '__main__':
+  run()

Chapter9/pipeline/pipeline1.py

@@ -0,0 +1,12 @@
+#pipeline1.py: Separate strings from a PCollection
+import apache_beam as beam
+
+with beam.Pipeline() as pipeline:
+  plants = (
+      pipeline
+      | 'Subjects' >> beam.Create([
+          'English Maths Science',
+          'French Arts',
+      ])
+      | 'Split subjects' >> beam.FlatMap(str.split)
+      | beam.Map(print))

Chapter9/pipeline/pipeline2.py

@@ -0,0 +1,18 @@
+#pipeline2.py: Separate subject with grade from a PCollection
+import apache_beam as beam
+
+def my_format(sub, marks):
+    yield '{}\t{}'.format(sub,marks)
+
+with beam.Pipeline() as pipeline:
+  plants = (
+      pipeline
+      | 'Subjects' >> beam.Create([
+      ('English','A'),
+      ('Maths', 'B+'),
+      ('Science', 'A-'),
+      ('French', 'A'),
+      ('Arts', 'A+'),
+      ])
+      | 'Format subjects with marks' >> beam.FlatMapTuple(my_format)
+      | beam.Map(print))

Chapter9/pipeline/pipeline3.py

@@ -0,0 +1,14 @@
+#pipeline3.py: Read data from a file and give results back to another file
+import apache_beam as beam
+from apache_beam.io import WriteToText, ReadFromText
+
+with beam.Pipeline() as pipeline:
+    lines = pipeline | ReadFromText('sample1.txt')
+
+    subjects = (
+      lines
+      | 'Subjects' >> beam.FlatMap(str.split))
+
+    subjects | WriteToText(file_path_prefix='subjects',
+                           file_name_suffix='.txt',
+                           shard_name_template='')

Chapter9/pipeline/pipeline4.py

@@ -0,0 +1,49 @@
+#pipeline4.py: Using argument for a pipeline
+import re
+
+import apache_beam as beam
+import argparse
+from apache_beam.io import WriteToText, ReadFromText
+from apache_beam.options.pipeline_options import PipelineOptions
+
+class WordParsingDoFn(beam.DoFn):
+  def process(self, element):
+    return re.findall(r'[\w\']+', element, re.UNICODE)
+
+def run(argv=None, save_main_session=True):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--input',
+        dest='input',
+        default='sample1.txt',
+        help='Input file to process.')
+    parser.add_argument(
+        '--output',
+        dest='output',
+        default='subjects',
+        help='Output file to write results to.')
+
+    parser.add_argument(
+        '--extension',
+        dest='ext',
+        default='.txt',
+        help='Output file extension to use.')
+
+    known_args, pipeline_args = parser.parse_known_args(argv)
+
+    pipeline_args.extend([
+        '--runner=DirectRunner',
+        '--job_name=demo-local-job',
+    ])
+    pipeline_options = PipelineOptions(pipeline_args)
+    with beam.Pipeline(options=pipeline_options) as pipeline:
+        lines = pipeline | ReadFromText(known_args.input)
+        subjects = (
+                lines
+                | 'Subjects' >> beam.ParDo(WordParsingDoFn()).
+                with_output_types(str))
+
+        subjects | WriteToText(known_args.output, known_args.ext)
+
+if __name__ == '__main__':
+    run()

Chapter9/pipeline/sample.txt

@@ -0,0 +1,57 @@
+
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
+
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...

Chapter9/pipeline/sample1.txt

@@ -0,0 +1,2 @@
+
+English Maths Science French Arts

Chapter9/time-wsproj/app.yaml

@@ -0,0 +1,15 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+runtime: python39

Chapter9/time-wsproj/main.py

@@ -0,0 +1,30 @@
+
+from flask import Flask
+from datetime import date, datetime
+
+
+# If `entrypoint` is not defined in app.yaml, App Engine will look for an app
+# called `app` in file `main.py`. This is the case in our yaml file
+app = Flask(__name__)
+
+
+@app.route('/')
+def welcome():
+    return 'Welcome Python Geek! Use appropriate URI for date and time'
+
+
+@app.route('/date')
+def today():
+    today = date.today()
+    return "{date:" + today.strftime("%B %d, %Y") + '}'
+
+
+@app.route('/time')
+def time():
+    now = datetime.now()
+    return "{time:" + now.strftime("%H:%M:%S") + '}'
+
+
+if __name__ == '__main__':
+    # For local testing
+    app.run(host='127.0.0.1', port=8080, debug=True)

Chapter9/time-wsproj/requirements.txt

@@ -0,0 +1 @@
+Flask==2.0.1