View define_function_af.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from collections import Counter | |
# define the python function | |
def my_function(): | |
# get the variable value | |
file_path = Variable.get("data_path") | |
# open the file | |
file_ = open(file_path) | |
# read the file and calculate the word count | |
data = Counter(file_.read().split()) |
View airflow_python_operator.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
### importing the required libraries | |
from datetime import timedelta | |
from airflow import DAG | |
from airflow.operators.python_operator import PythonOperator | |
from airflow.utils.dates import days_ago |
View airflow_import.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import timedelta | |
# The DAG object; we'll need this to instantiate a DAG | |
from airflow import DAG | |
# Operators; we need this to operate! | |
from airflow.operators.bash_operator import BashOperator | |
from airflow.utils.dates import days_ago |
View dataframe.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
spark.createDataFrame( | |
[ | |
(1, 'Lakshay'), # create your data here, make sure to be consistent in the types. | |
(2, 'Aniruddha'), | |
. | |
. | |
. | |
. | |
(100, 'Siddhart') | |
], |
View rdd_23.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# parallelizing data collection | |
my_list = [1, 2, 3, 4, 5] | |
my_list_rdd = sc.parallelize(my_list) | |
## 2. Referencing to external data file | |
file_rdd = sc.textFile("path_of_file") |
View collection_1.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# create weekly demand collection | |
database.create_collection("weekly_demand") |
View 1_ag.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
result_1 = weekly_demand_collection.aggregate([ | |
## stage 1 | |
{ | |
"$match" : { | |
"center_id" : { | |
"$eq" : 11 | |
} | |
} | |
}, | |
## stage 2 |
View find_one.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
weekly_demand_collection.find_one() |
View add_sheet.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# add a sheet with 20 rows and 2 columns | |
sheet.add_worksheet(rows=20,cols=2,title='runs') | |
# get the instance of the second sheet | |
sheet_runs = sheet.get_worksheet(1) |
View generate_html.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# define function to add the image in the html file with the class name | |
def get_picture_html(path, tag): | |
image_html = """<p> {tag_name} </p> <picture> <img src= "../{path_name}" height="300" width="400"> </picture>""" | |
return image_html.format(tag_name=tag, path_name=path) | |
# define function to add the list element in the html file | |
def get_count_html(category, count): | |
count_html = """<li> {category_name} : {count_} </li>""" | |
return count_html.format(category_name = category, count_ = count) |
NewerOlder