uploading chapter 9 source code
This commit is contained in:
muassif
@@ -0,0 +1,61 @@
|
||||
# wordcount.py: count words in a text file
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
|
||||
import apache_beam as beam
|
||||
from apache_beam.io import ReadFromText
|
||||
from apache_beam.io import WriteToText
|
||||
from apache_beam.options.pipeline_options import PipelineOptions
|
||||
from apache_beam.options.pipeline_options import SetupOptions
|
||||
|
||||
def run(argv=None, save_main_session=True):
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/Users/muasif/gcd-projs/gcp-key/word-count-316612-f22f7ffcc2dd.json"
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--input',
|
||||
dest='input',
|
||||
default='gs://muasif/input/sample.txt',
|
||||
help='Input file to process.')
|
||||
parser.add_argument(
|
||||
'--output',
|
||||
dest='output',
|
||||
default='gs://muasif/output/result',
|
||||
help='Output file to write results to.')
|
||||
known_args, pipeline_args = parser.parse_known_args(argv)
|
||||
pipeline_args.extend([
|
||||
'--runner=DataflowRunner',
|
||||
'--project=word-count-316612',
|
||||
'--region=us-central1',
|
||||
'--staging_location=gs://muasif/staging',
|
||||
'--temp_location=gs://muasif/temp',
|
||||
'--job_name=my-wordcount-job',
|
||||
])
|
||||
|
||||
pipeline_options = PipelineOptions(pipeline_args)
|
||||
pipeline_options.view_as(SetupOptions).\
|
||||
save_main_session = save_main_session
|
||||
with beam.Pipeline(options=pipeline_options) as p:
|
||||
|
||||
lines = p | ReadFromText(known_args.input)
|
||||
# Count the occurrences of each word.
|
||||
counts = (
|
||||
lines
|
||||
| 'Split words' >> (
|
||||
beam.FlatMap(
|
||||
lambda x: re.findall(r'[A-Za-z\']+', x)).
|
||||
with_output_types(str))
|
||||
| 'Pair with 1' >> beam.Map(lambda x: (x, 1))
|
||||
| 'Group & Sum' >> beam.CombinePerKey(sum))
|
||||
|
||||
# Format the word counts into a PCollection of strings.
|
||||
def format_result(word_count):
|
||||
(word, count) = word_count
|
||||
return '%s: %s' % (word, count)
|
||||
|
||||
output = counts | 'Format' >> beam.Map(format_result)
|
||||
output | WriteToText(known_args.output)
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
@@ -0,0 +1,12 @@
|
||||
#pipeline1.py: Separate strings from a PCollection
|
||||
import apache_beam as beam
|
||||
|
||||
with beam.Pipeline() as pipeline:
|
||||
plants = (
|
||||
pipeline
|
||||
| 'Subjects' >> beam.Create([
|
||||
'English Maths Science',
|
||||
'French Arts',
|
||||
])
|
||||
| 'Split subjects' >> beam.FlatMap(str.split)
|
||||
| beam.Map(print))
|
||||
@@ -0,0 +1,18 @@
|
||||
#pipeline2.py: Separate subject with grade from a PCollection
|
||||
import apache_beam as beam
|
||||
|
||||
def my_format(sub, marks):
|
||||
yield '{}\t{}'.format(sub,marks)
|
||||
|
||||
with beam.Pipeline() as pipeline:
|
||||
plants = (
|
||||
pipeline
|
||||
| 'Subjects' >> beam.Create([
|
||||
('English','A'),
|
||||
('Maths', 'B+'),
|
||||
('Science', 'A-'),
|
||||
('French', 'A'),
|
||||
('Arts', 'A+'),
|
||||
])
|
||||
| 'Format subjects with marks' >> beam.FlatMapTuple(my_format)
|
||||
| beam.Map(print))
|
||||
@@ -0,0 +1,14 @@
|
||||
#pipeline3.py: Read data from a file and give results back to another file
|
||||
import apache_beam as beam
|
||||
from apache_beam.io import WriteToText, ReadFromText
|
||||
|
||||
with beam.Pipeline() as pipeline:
|
||||
lines = pipeline | ReadFromText('sample1.txt')
|
||||
|
||||
subjects = (
|
||||
lines
|
||||
| 'Subjects' >> beam.FlatMap(str.split))
|
||||
|
||||
subjects | WriteToText(file_path_prefix='subjects',
|
||||
file_name_suffix='.txt',
|
||||
shard_name_template='')
|
||||
@@ -0,0 +1,49 @@
|
||||
#pipeline4.py: Using argument for a pipeline
|
||||
import re
|
||||
|
||||
import apache_beam as beam
|
||||
import argparse
|
||||
from apache_beam.io import WriteToText, ReadFromText
|
||||
from apache_beam.options.pipeline_options import PipelineOptions
|
||||
|
||||
class WordParsingDoFn(beam.DoFn):
|
||||
def process(self, element):
|
||||
return re.findall(r'[\w\']+', element, re.UNICODE)
|
||||
|
||||
def run(argv=None, save_main_session=True):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--input',
|
||||
dest='input',
|
||||
default='sample1.txt',
|
||||
help='Input file to process.')
|
||||
parser.add_argument(
|
||||
'--output',
|
||||
dest='output',
|
||||
default='subjects',
|
||||
help='Output file to write results to.')
|
||||
|
||||
parser.add_argument(
|
||||
'--extension',
|
||||
dest='ext',
|
||||
default='.txt',
|
||||
help='Output file extension to use.')
|
||||
|
||||
known_args, pipeline_args = parser.parse_known_args(argv)
|
||||
|
||||
pipeline_args.extend([
|
||||
'--runner=DirectRunner',
|
||||
'--job_name=demo-local-job',
|
||||
])
|
||||
pipeline_options = PipelineOptions(pipeline_args)
|
||||
with beam.Pipeline(options=pipeline_options) as pipeline:
|
||||
lines = pipeline | ReadFromText(known_args.input)
|
||||
subjects = (
|
||||
lines
|
||||
| 'Subjects' >> beam.ParDo(WordParsingDoFn()).
|
||||
with_output_types(str))
|
||||
|
||||
subjects | WriteToText(known_args.output, known_args.ext)
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
@@ -0,0 +1,57 @@
|
||||
|
||||
Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
|
||||
Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
|
||||
|
||||
Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
|
||||
scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
|
||||
|
||||
Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
|
||||
You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
|
||||
|
||||
Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
|
||||
9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
|
||||
|
||||
Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
|
||||
Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
|
||||
|
||||
Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
|
||||
scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
|
||||
|
||||
Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
|
||||
You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
|
||||
|
||||
Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
|
||||
9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
|
||||
Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
|
||||
Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
|
||||
|
||||
Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
|
||||
scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
|
||||
|
||||
Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
|
||||
You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
|
||||
|
||||
Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
|
||||
9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
|
||||
Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
|
||||
Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
|
||||
|
||||
Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
|
||||
scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
|
||||
|
||||
Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
|
||||
You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
|
||||
|
||||
Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
|
||||
9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
|
||||
Spark Read Text File | RDD | DataFrame — SparkByExampleshttps://sparkbyexamples.com › spark › spark-read-text-...
|
||||
Complete example — txt files, for example, sparkContext.textFile() and sparkContext.wholeTextFiles() methods to read into RDD and spark.read.text() ...
|
||||
|
||||
Quick Start - Spark 2.2.1 Documentation - Apache Sparkhttps://spark.apache.org › docs › quick-start
|
||||
scala> val textFile = spark.read.textFile("README.md") textFile: org.apache.spark.sql. ... For example, we can easily call functions declared elsewhere. We'll use ...
|
||||
|
||||
Examples | Apache Spark - The Apache Software Foundation!https://spark.apache.org › examples
|
||||
You create a dataset from external data, then apply parallel operations to it. ... Creates a DataFrame having a single column named "line" df = textFile.map(lambda r: ... In this example, we read a table stored in a database and calculate the ...
|
||||
|
||||
Spark read Text file into Dataframe - datanebhttps://www.dataneb.com › post › spark-read-text-file-i...
|
||||
9 Nov 2019 — Blog has four sections: Spark read Text File Spark read CSV with ... I am using squid logs as sample data for this example. ... Each library has its significance, I have commented when it's used import org.apache.spark._ import ...
|
||||
@@ -0,0 +1,2 @@
|
||||
|
||||
English Maths Science French Arts
|
||||
@@ -0,0 +1,15 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
runtime: python39
|
||||
@@ -0,0 +1,30 @@
|
||||
|
||||
from flask import Flask
|
||||
from datetime import date, datetime
|
||||
|
||||
|
||||
# If `entrypoint` is not defined in app.yaml, App Engine will look for an app
|
||||
# called `app` in file `main.py`. This is the case in our yaml file
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
@app.route('/')
|
||||
def welcome():
|
||||
return 'Welcome Python Geek! Use appropriate URI for date and time'
|
||||
|
||||
|
||||
@app.route('/date')
|
||||
def today():
|
||||
today = date.today()
|
||||
return "{date:" + today.strftime("%B %d, %Y") + '}'
|
||||
|
||||
|
||||
@app.route('/time')
|
||||
def time():
|
||||
now = datetime.now()
|
||||
return "{time:" + now.strftime("%H:%M:%S") + '}'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# For local testing
|
||||
app.run(host='127.0.0.1', port=8080, debug=True)
|
||||
@@ -0,0 +1 @@
|
||||
Flask==2.0.1
|
||||
Reference in New Issue
Block a user