Skip to content

Instantly share code, notes, and snippets.

@adam-phillipps
Last active August 15, 2019 23:28
Show Gist options
  • Save adam-phillipps/60d254add8602838ba4089af2c9c3d6b to your computer and use it in GitHub Desktop.
Save adam-phillipps/60d254add8602838ba4089af2c9c3d6b to your computer and use it in GitHub Desktop.
#!/bin/bash
cd /home/ubuntu
export AWS_DEFAULT_REGION=us-west-2
export QUEUE=backlog_crawlBot
export WIP_Q=wip_crawlBot
export ART_NAME=SmashEndurance.jar
export ART_BKT=endurance-crawl-artifacts
export OUT_BKT=endurance-crawl-output
sudo usermod -a -G docker ec2-user
sudo service docker restart
pip install boto3
# start wappalyzer cli
docker run --rm wappalyzer/cli https://www.wappalyzer.com
# create wrapper script
cat << WPR >> wrapper.py
import boto3
import os
import pdb
s3 = boto3.resource('s3')
sqs = boto3.client('sqs', region_name='us-west-2')
jar_art = s3.Object(os.getenv('ART_BKT'), os.getenv('ART_NAME')).download_file(os.getenv('ART_NAME'))
# count the backlog messages
def msg_count():
n = sqs.get_queue_attributes(QueueUrl=os.getenv('QUEUE'),AttributeNames=['ApproximateNumberOfMessages'])['Attributes']['ApproximateNumberOfMessages']
print("{} msgs in queue".format(n))
return int(n)
# get a message
# run the jar on the message
# put the results in the results bucket
# delete the message from the backlog because we just finished it
def work():
print("working")
try:
msg = sqs.receive_message(QueueUrl=os.getenv('QUEUE'), WaitTimeSeconds=20, MaxNumberOfMessages=1, VisibilityTimeout=120)['Messages'][0]
except Exception as e:
print("Exception getting message: {}".format(e))
return
sid, surl = msg['Body'].replace("'",'').split(',')
res_name = "{}.json".format(sid)
invoke_jar_command = "java -jar SmashEndurance.jar {} {}".format(sid, surl)
print("invoked jar with {}".format(invoke_jar_command))
ex_code = os.system(invoke_jar_command)
if ex_code == 0:
s3.Object(os.getenv('OUT_BKT'), res_name).upload_file(res_name)
sqs.delete_message(QueueUrl=os.getenv('QUEUE'), ReceiptHandle=msg['ReceiptHandle'])
os.remove(res_name)
else:
print("jar error->{}->{}->{}".format(ex_code, sid, surl))
# always stay busy and break if there is no work left
while True:
if msg_count() > 0:
work()
else:
break
WPR
# make wrapper runnable
chmod +x wrapper.py
# run the wrapper
python wrapper.py
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment