uploading chapter 9 source code

2021-06-15 23:10:30 +04:00
parent ac10d74a74
commit 360b7f1420
10 changed files with 259 additions and 0 deletions
@@ -0,0 +1,12 @@
+#pipeline1.py: Separate strings from a PCollection
+import apache_beam as beam
+
+with beam.Pipeline() as pipeline:
+  plants = (
+      pipeline
+      | 'Subjects' >> beam.Create([
+          'English Maths Science',
+          'French Arts',
+      ])
+      | 'Split subjects' >> beam.FlatMap(str.split)
+      | beam.Map(print))
@@ -0,0 +1,18 @@
+#pipeline2.py: Separate subject with grade from a PCollection
+import apache_beam as beam
+
+def my_format(sub, marks):
+    yield '{}\t{}'.format(sub,marks)
+
+with beam.Pipeline() as pipeline:
+  plants = (
+      pipeline
+      | 'Subjects' >> beam.Create([
+      ('English','A'),
+      ('Maths', 'B+'),
+      ('Science', 'A-'),
+      ('French', 'A'),
+      ('Arts', 'A+'),
+      ])
+      | 'Format subjects with marks' >> beam.FlatMapTuple(my_format)
+      | beam.Map(print))
@@ -0,0 +1,14 @@
+#pipeline3.py: Read data from a file and give results back to another file
+import apache_beam as beam
+from apache_beam.io import WriteToText, ReadFromText
+
+with beam.Pipeline() as pipeline:
+    lines = pipeline | ReadFromText('sample1.txt')
+
+    subjects = (
+      lines
+      | 'Subjects' >> beam.FlatMap(str.split))
+
+    subjects | WriteToText(file_path_prefix='subjects',
+                           file_name_suffix='.txt',
+                           shard_name_template='')
@@ -0,0 +1,49 @@
+#pipeline4.py: Using argument for a pipeline
+import re
+
+import apache_beam as beam
+import argparse
+from apache_beam.io import WriteToText, ReadFromText
+from apache_beam.options.pipeline_options import PipelineOptions
+
+class WordParsingDoFn(beam.DoFn):
+  def process(self, element):
+    return re.findall(r'[\w\']+', element, re.UNICODE)
+
+def run(argv=None, save_main_session=True):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--input',
+        dest='input',
+        default='sample1.txt',
+        help='Input file to process.')
+    parser.add_argument(
+        '--output',
+        dest='output',
+        default='subjects',
+        help='Output file to write results to.')
+
+    parser.add_argument(
+        '--extension',
+        dest='ext',
+        default='.txt',
+        help='Output file extension to use.')
+
+    known_args, pipeline_args = parser.parse_known_args(argv)
+
+    pipeline_args.extend([
+        '--runner=DirectRunner',
+        '--job_name=demo-local-job',
+    ])
+    pipeline_options = PipelineOptions(pipeline_args)
+    with beam.Pipeline(options=pipeline_options) as pipeline:
+        lines = pipeline | ReadFromText(known_args.input)
+        subjects = (
+                lines
+                | 'Subjects' >> beam.ParDo(WordParsingDoFn()).
+                with_output_types(str))
+
+        subjects | WriteToText(known_args.output, known_args.ext)
+
+if __name__ == '__main__':
+    run()
@@ -0,0 +1,57 @@
+
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
+
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
+Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
+Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
+
+Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
+scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
+
+Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
+You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
+
+Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
+9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
@@ -0,0 +1,2 @@
+
+English Maths Science French Arts