ctodd

BUCKET = 'deeplens-sagemaker-poopinator'
PREFIX = 'working' # root path working space
labeling_job_name = 'dog-obj2'
training_job_name = 'poopinator-detection-resnet'
local_working_dir = 'working'
local_manifest_dir = local_working_dir + '/manifests'

!aws s3 sync s3://$BUCKET/images images

ctodd

import boto3
client = boto3.client('sagemaker')

s3_output = client.describe_labeling_job(LabelingJobName=labeling_job_name)['OutputConfig']['S3OutputPath'] + labeling_job_name
augmented_manifest_url = f'{s3_output}/manifests/output/output.manifest'

import os
import shutil

try:

ctodd

import matplotlib.pyplot as plt
import matplotlib.patches as patches
from PIL import Image
import numpy as np
from itertools import cycle

def show_annotated_image(img_path, bboxes, prec):
    im = np.array(Image.open(img_path), dtype=np.uint8)
    
    # Create figure and axes

ctodd

!pip -q install --upgrade pip
!pip -q install jsonlines
import jsonlines
from itertools import islice
    
with jsonlines.open(augmented_manifest_file, 'r') as reader:
    for desc in islice(reader, 10):
        img_url = desc['source-ref']
        img_file = "images/source/" + os.path.basename(img_url)
        file_exists = os.path.isfile(img_file)

ctodd

import json

augmented_manifest_filename_output = local_manifest_dir + '/output.manifest'

with jsonlines.open(augmented_manifest_filename_output, 'r') as reader:
    lines = list(reader)
    # Shuffle data in place.
    np.random.shuffle(lines)
    
dataset_size = len(lines)

ctodd

pfx_training = PREFIX + '/training' if PREFIX else 'training'
# Defines paths for use in the training job request.
s3_train_data_path = 's3://{}/{}/{}'.format(BUCKET, pfx_training, augmented_manifest_filename_train)
s3_validation_data_path = 's3://{}/{}/{}'.format(BUCKET, pfx_training, augmented_manifest_filename_validation)

!aws s3 cp $augmented_manifest_filename_train s3://$BUCKET/$pfx_training/
!aws s3 cp $augmented_manifest_filename_validation s3://$BUCKET/$pfx_training/

ctodd

import time
import sagemaker

role = sagemaker.get_execution_role()
sess = sagemaker.Session()

training_image = sagemaker.amazon.amazon_estimator.get_image_uri(
    boto3.Session().region_name, 'object-detection', repo_version='latest')
s3_output_path = 's3://{}/{}/output'.format(BUCKET, pfx_training)

ctodd

import time
timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime())
model_name = training_job_name + '-model' + timestamp

training_image = training_info['AlgorithmSpecification']['TrainingImage']
model_data = training_info['ModelArtifacts']['S3ModelArtifacts']

primary_container = {
    'Image': training_image,
    'ModelDataUrl': model_data,

ctodd

timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime())
endpoint_config_name = training_job_name + '-epc' + timestamp
endpoint_config_response = client.create_endpoint_config(
    EndpointConfigName = endpoint_config_name,
    ProductionVariants=[{
        'InstanceType':'ml.t2.medium',
        'InitialInstanceCount':1,
        'ModelName':model_name,
        'VariantName':'AllTraffic'}])

ctodd

timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime())
endpoint_name = training_job_name + '-ep' + timestamp
print('Endpoint name: {}'.format(endpoint_name))

endpoint_params = {
    'EndpointName': endpoint_name,
    'EndpointConfigName': endpoint_config_name,
}
endpoint_response = client.create_endpoint(**endpoint_params)
print('EndpointArn = {}'.format(endpoint_response['EndpointArn']))