coingraham

"""
This script is used to create pre-signed URLS used in AMS CFT deployments.

You can read more about pre-signed URLs here:
https://docs.aws.amazon.com/AmazonS3/latest/dev/PresignedUrlUploadObject.html
"""

# You will need to install python, and install the boto3 package.  Once python is installed
# you should be able to install it with "pip install boto3"
import boto3

coingraham

[version, accountid, interfaceid, srcaddr, dstaddr, srcport, distport=25, protocol, packets, bytes, start, end, action, logstatus]

coingraham

date_created_at = datetime.strptime(image.creation_date, "%Y-%m-%dT%H:%M:%S.000Z")

coingraham

Import-Module ADFS
Add-ADFSRelyingPartyTrust -Name "Amazon Web Services & AD Groups" -MetadataURL "https://signin.aws.amazon.com/static/saml-metadata.xml" -MonitoringEnabled:$true -AutoUpdateEnabled:$true
$ruleSet = New-AdfsClaimRuleSet -ClaimRuleFile ((pwd).Path + "\claims-AD-Groups.txt")
$authSet = New-AdfsClaimRuleSet -ClaimRuleFile ((pwd).Path + "\auth.txt")
Set-AdfsRelyingPartyTrust -TargetName "Amazon Web Services & AD Groups" -IssuanceTransformRules $ruleSet.ClaimRulesString -IssuanceAuthorizationRules $authSet.ClaimRulesString 

coingraham

import boto3

profile = "myprofile"
region = "us-east-1"
expiration = 36000  # one hour in seconds
bucket = "mybucket"
key = "myobjectkey"

session = boto3.session.Session(profile_name=profile, region_name=region)

coingraham

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
 
## @params: [JOB_NAME]
args = getResolvedOptions(sys.argv, ['JOB_NAME'])
 

coingraham

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
from awsglue.dynamicframe import DynamicFrame
 
## @params: [JOB_NAME]
args = getResolvedOptions(sys.argv, ['JOB_NAME'])

coingraham

from pyspark.context import SparkContext
from pyspark.sql import SQLContext
from pyspark.sql import SparkSession

if __name__ == "__main__":
  
    # Create the spark session (include Hive Support)
    spark = SparkSession\
        .builder\
        .appName("SparkEMRUsingGlueCatalot")\

coingraham

from pyspark.context import SparkContext
from pyspark.sql import SparkSession

if __name__ == "__main__":

    # Create the spark session
    spark = SparkSession\
        .builder\
        .appName("SparkEMR")\
        .getOrCreate()

coingraham

# Example uses GDELT dataset found here: https://aws.amazon.com/public-datasets/gdelt/
# Column headers found here: http://gdeltproject.org/data/lookups/CSV.header.dailyupdates.txt

# Load RDD
lines = sc.textFile("s3://gdelt-open-data/events/2016*") # Loads 73,385,698 records from 2016
# Split lines into columns; change split() argument depending on deliminiter e.g. '\t'
parts = lines.map(lambda l: l.split('\t'))
# Convert RDD into DataFrame
from urllib import urlopen
html = urlopen("http://gdeltproject.org/data/lookups/CSV.header.dailyupdates.txt").read().rstrip()