This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
BUCKET = 'deeplens-sagemaker-poopinator' | |
PREFIX = 'working' # root path working space | |
labeling_job_name = 'dog-obj2' | |
training_job_name = 'poopinator-detection-resnet' | |
local_working_dir = 'working' | |
local_manifest_dir = local_working_dir + '/manifests' | |
!aws s3 sync s3://$BUCKET/images images |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
client = boto3.client('sagemaker') | |
s3_output = client.describe_labeling_job(LabelingJobName=labeling_job_name)['OutputConfig']['S3OutputPath'] + labeling_job_name | |
augmented_manifest_url = f'{s3_output}/manifests/output/output.manifest' | |
import os | |
import shutil | |
try: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import matplotlib.pyplot as plt | |
import matplotlib.patches as patches | |
from PIL import Image | |
import numpy as np | |
from itertools import cycle | |
def show_annotated_image(img_path, bboxes, prec): | |
im = np.array(Image.open(img_path), dtype=np.uint8) | |
# Create figure and axes |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
!pip -q install --upgrade pip | |
!pip -q install jsonlines | |
import jsonlines | |
from itertools import islice | |
with jsonlines.open(augmented_manifest_file, 'r') as reader: | |
for desc in islice(reader, 10): | |
img_url = desc['source-ref'] | |
img_file = "images/source/" + os.path.basename(img_url) | |
file_exists = os.path.isfile(img_file) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
augmented_manifest_filename_output = local_manifest_dir + '/output.manifest' | |
with jsonlines.open(augmented_manifest_filename_output, 'r') as reader: | |
lines = list(reader) | |
# Shuffle data in place. | |
np.random.shuffle(lines) | |
dataset_size = len(lines) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
pfx_training = PREFIX + '/training' if PREFIX else 'training' | |
# Defines paths for use in the training job request. | |
s3_train_data_path = 's3://{}/{}/{}'.format(BUCKET, pfx_training, augmented_manifest_filename_train) | |
s3_validation_data_path = 's3://{}/{}/{}'.format(BUCKET, pfx_training, augmented_manifest_filename_validation) | |
!aws s3 cp $augmented_manifest_filename_train s3://$BUCKET/$pfx_training/ | |
!aws s3 cp $augmented_manifest_filename_validation s3://$BUCKET/$pfx_training/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import time | |
import sagemaker | |
role = sagemaker.get_execution_role() | |
sess = sagemaker.Session() | |
training_image = sagemaker.amazon.amazon_estimator.get_image_uri( | |
boto3.Session().region_name, 'object-detection', repo_version='latest') | |
s3_output_path = 's3://{}/{}/output'.format(BUCKET, pfx_training) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import time | |
timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime()) | |
model_name = training_job_name + '-model' + timestamp | |
training_image = training_info['AlgorithmSpecification']['TrainingImage'] | |
model_data = training_info['ModelArtifacts']['S3ModelArtifacts'] | |
primary_container = { | |
'Image': training_image, | |
'ModelDataUrl': model_data, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime()) | |
endpoint_config_name = training_job_name + '-epc' + timestamp | |
endpoint_config_response = client.create_endpoint_config( | |
EndpointConfigName = endpoint_config_name, | |
ProductionVariants=[{ | |
'InstanceType':'ml.t2.medium', | |
'InitialInstanceCount':1, | |
'ModelName':model_name, | |
'VariantName':'AllTraffic'}]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime()) | |
endpoint_name = training_job_name + '-ep' + timestamp | |
print('Endpoint name: {}'.format(endpoint_name)) | |
endpoint_params = { | |
'EndpointName': endpoint_name, | |
'EndpointConfigName': endpoint_config_name, | |
} | |
endpoint_response = client.create_endpoint(**endpoint_params) | |
print('EndpointArn = {}'.format(endpoint_response['EndpointArn'])) |
OlderNewer