Taiwo O. Adetiloye taiwotman

## docker-compose up
docker compose up run

## docker build
$ docker build -t <image-name> .

## Dockerfile
FROM python:latest

WORKDIR app
COPY . /app

RUN python3 -m pip install -r requirements.txt

## requirements.txt
psycopg2==2.9.3
pytest== 7.1.2

## docker-compose.yml
version: "3.9"
services:
    postgres:
      image: postgres:10.5
      restart: always
      environment:
        - POSTGRES_USER=<username>
        - POSTGRES_PASSWORD=<password>
      ports:
        - '5433:5432'

## docker commands
## Key Docker Commands

docker build --pull --no-cache --tag=<image-name> .

docker-compose up

docker-compose down

## capacity-optimization-heuristics-approach-for-the-bus-loading-problem.ipynb

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                taiwotman
                / capacity-optimization-heuristics-approach-for-the-bus-loading-problem.ipynb
            
            
              Last active
              January 13, 2022 14:27
            
              
                Capacity Optimization Heuristic for the Bus Loading Problem
              
          
      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
      
    
## aws_redshift_pyspark2.py
#Transformation
df = df.withColumn("latitude", col("latitude").cast(DoubleType()))\
       .withColumn("longitude", col("longitude").cast(DoubleType()))

df.printSchema()

df.show(10)


#More Insight

## aws_redshift_pyspark.py
from pyspark.sql import SparkSession
from pyspark.sql.types import DoubleType
from pyspark.sql.functions import col

jar_path = 'rds_jar_driver.jar'
spark = SparkSession \
    .builder \
    .appName("AWS REDSHIFT PYSPARK APP") \
    .config("spark.jars", jar_path)\
    .config('spark.driver.extraClassPath', jar_path) \

## rds_sql2.sql
COPY seattle_emergence_calls from 's3://seattle-realtime-emergence-fire-call/Seattle_Real_Time_Fire_911_Calls.csv'  FORMAT CSV
        IGNOREHEADER 1
        CREDENTIALS 'aws_iam_role=your_iam_role'
        REGION 'us-east-1';
	FROM python:latest

	WORKDIR app
	COPY . /app

	RUN python3 -m pip install -r requirements.txt
	version: "3.9"
	services:
	postgres:
	image: postgres:10.5
	restart: always
	environment:
	- POSTGRES_USER=<username>
	- POSTGRES_PASSWORD=<password>
	ports:
	- '5433:5432'
	## Key Docker Commands

	docker build --pull --no-cache --tag=<image-name> .

	docker-compose up

	docker-compose down
	#Transformation
	df = df.withColumn("latitude", col("latitude").cast(DoubleType()))\
	.withColumn("longitude", col("longitude").cast(DoubleType()))

	df.printSchema()

	df.show(10)


	#More Insight
	from pyspark.sql import SparkSession
	from pyspark.sql.types import DoubleType
	from pyspark.sql.functions import col

	jar_path = 'rds_jar_driver.jar'
	spark = SparkSession \
	.builder \
	.appName("AWS REDSHIFT PYSPARK APP") \
	.config("spark.jars", jar_path)\
	.config('spark.driver.extraClassPath', jar_path) \
	COPY seattle_emergence_calls from 's3://seattle-realtime-emergence-fire-call/Seattle_Real_Time_Fire_911_Calls.csv' FORMAT CSV
	IGNOREHEADER 1
	CREDENTIALS 'aws_iam_role=your_iam_role'
	REGION 'us-east-1';