~/LargeApp # Name of Application
|__ /app # Our Application Module
|__ /common
|-- database.py
|-- __init__.py
|-- /example # Module like album, song, video ...
|-- __init__.py
|-- views.py
|-- models.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import random | |
import string | |
import time | |
import uuid | |
import boto3 | |
client = boto3.client('kinesis') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import logging | |
import os | |
from pyspark.sql import SparkSession | |
from pyspark.streaming import StreamingContext | |
from pyspark.streaming.kinesis import KinesisUtils, InitialPositionInStream | |
os.environ['PYSPARK_SUBMIT_ARGS'] = "--packages org.apache.spark:spark-streaming-kinesis-asl_2.12:3.0.1," | |
"org.apache.spark:spark-avro_2.12:3.0.1 pyspark-shell" # Adicionando as bibliotecas necessárias para o consumo dos dados |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
import requests | |
from requests_aws4auth import AWS4Auth | |
host = '' # include https:// and trailing / | |
region = '' # e.g. us-west-1 | |
service = 'es' | |
credentials = boto3.Session().get_credentials() | |
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"Version": "2012-10-17", | |
"Statement": [ | |
{ | |
"Effect": "Allow", | |
"Action": "iam:PassRole", | |
"Resource": "arn:aws:iam::123456789012:role/es-assume-role-backup" | |
}, | |
{ | |
"Effect": "Allow", |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"Version": "2012-10-17", | |
"Statement": [{ | |
"Action": [ | |
"s3:ListBucket" | |
], | |
"Effect": "Allow", | |
"Resource": [ | |
"arn:aws:s3:::s3-bucket-name" | |
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"Version": "2012-10-17", | |
"Statement": [{ | |
"Sid": "", | |
"Effect": "Allow", | |
"Principal": { | |
"Service": "es.amazonaws.com" | |
}, | |
"Action": "sts:AssumeRole" | |
}] |