kaysush

main:
    params: [event]
    steps:
        - init:
            assign:
                - project_id: ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
                - location: "us-central1"
                - zone: "asia-south2-a"
                - job_name: "df-word-count"
                - template_path: "gs://dataflow-templates-us-central1/latest/Word_Count"

kaysush

from flask import Flask,render_template,Response
from unittest import result
import numpy as np
import cv2
import time 
import tensorflow as tf
import tensorflow_hub as hub
from matplotlib import pyplot as plt
import win32api
import pyttsx3

kaysush

FROM node:16.13.1-stretch as build

WORKDIR /app

COPY package*.json /app/

# Install production dependencies
RUN npm install --only=production
RUN cp -a node_modules /tmp/prod_node_modules

kaysush

FROM node:18.7.0-alpine3.16
RUN mkdir $HOME/app
WORKDIR $HOME/app
COPY main.js .
CMD ["node", "main.js"]

kaysush

from pyspark.sql import SparkSession

spark = SparkSession.builder.enableHiveSupport().getOrCreate()

print('Storing random numbers in a GCS bucket')
spark.range(100).write.mode("overwrite").parquet("gs://apt-task-314904/random")
print('complete')

kaysush

from airflow import DAG
from datetime import timedelta
from airflow.utils.dates import days_ago
from airflow.providers.google.cloud.operators.dataproc import  DataprocCreateClusterOperator
from airflow.providers.google.cloud.operators.dataproc import DataprocSubmitJobOperator
from airflow.providers.google.cloud.operators.dataproc import DataprocDeleteClusterOperator

default_args = {
    'depends_on_past': False   
}

kaysush

from datetime import timedelta
from textwrap import dedent

# The DAG object; we'll need this to instantiate a DAG
from airflow import DAG

# Operators; we need this to operate!
from airflow.operators.bash import BashOperator
from airflow.utils.dates import days_ago
# These args will get passed on to each operator

kaysush

import base64
import json
import requests

def filter_rule(event, context):
    """Triggered from a message on a Cloud Pub/Sub topic.
    Args:
         event (dict): Event payload.
         context (google.cloud.functions.Context): Metadata for the event.
    """

kaysush

import json
import pandas as pd
from os import listdir
from os.path import isfile, join

results_base_path = 'jsonl_results'

onlyfiles = [f for f in listdir(results_base_path) if isfile(join(results_base_path, f))]

results = []

kaysush

import json
from google.cloud import storage

BUCKET='apt-task-314904-us'
DELIMITER='/'
TEST_PREFIX='test1/test1/'

TEST_BASE_PATH = f'gs://{BUCKET}/{TEST_PREFIX}'