Manual creation of a basic docker image with image and repository manifest generation

app.data

This is my configuration file

app.go

package main

import (
	"fmt"
	"io/ioutil"
	"os"
)

var fileName = "./app.data"

func main() {
	fmt.Println("Hello World...")

	file, err := os.OpenFile(fileName, os.O_RDONLY, 400)
	defer file.Close()
	if err != nil {
		fmt.Printf("There was an issue opening the file %s. Check your settings and trying again.\n", fileName)
		os.Exit(1)
	}

	contents, err := ioutil.ReadAll(file)
	if err != nil {
		fmt.Printf("There was an issue reading the contents of file %s.\n", fileName)
		os.Exit(2)
	}

	fmt.Printf("The configuration is:\n%s\n", contents)
}

jsonGeneration.sh

#!/bin/sh

function layerJSONGeneration {
  for i in $(head -$(expr $(wc -cl $LAYER_CACHE | cut -d" " -f3) - 1) $LAYER_CACHE); do
    echo \
'{
  "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
  "size": 0,
  "digest": "placeholder"
}' | jq --argjson size $(echo $i | cut -d, -f1) --arg digest  $(echo $i | cut -d, -f2)  '.size = $size | .digest = $digest'
  done | jq  -cs '.'
}

function imageManifestGeneration {
  #TODO change things to work better, such as entrypoint and check whats necessary and remove the rest for a valid image. Fix logic for dynamic number of layers vs the 2 set ones currently.
  echo \
'{
  "architecture": "amd64",
  "config": {
    "Hostname": "",
    "Domainname": "",
    "User": "",
    "AttachStdin": false,
    "AttachStdout": false,
    "AttachStderr": false,
    "Tty": false,
    "OpenStdin": false,
    "StdinOnce": false,
    "Env": [
      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ],
    "Cmd": [
      "/files/entrypoint.sh"
    ],
    "ArgsEscaped": true,
    "Image": "sha256:126bb391fa3e3fe122a094b31adc63e2f69dcb3dac25558da46ce35f05b43519",
    "Volumes": null,
    "WorkingDir": "",
    "Entrypoint": null,
    "OnBuild": null,
    "Labels": null
  },
  "container": "5db797ebf14d7bbd5c4924ecece61ae929ad254b42dc0faa2ef2f10d16020f29",
  "container_config": {
    "Hostname": "5db797ebf14d",
    "Domainname": "",
    "User": "",
    "AttachStdin": false,
    "AttachStdout": false,
    "AttachStderr": false,
    "Tty": false,
    "OpenStdin": false,
    "StdinOnce": false,
    "Env": [
      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ],
    "Cmd": [
      "/bin/sh",
      "-c",
      "#(nop) ",
      "CMD [\"/files/entrypoint.sh\"]"
    ],
    "ArgsEscaped": true,
    "Image": "sha256:126bb391fa3e3fe122a094b31adc63e2f69dcb3dac25558da46ce35f05b43519",
    "Volumes": null,
    "WorkingDir": "",
    "Entrypoint": null,
    "OnBuild": null,
    "Labels": {}
  },
  "created": "2019-01-03T00:19:12.7589019Z",
  "docker_version": "18.09.0",
  "history": [
    {
      "created": "2019-01-03T00:19:12.7589019Z",
      "created_by": "/bin/sh -c #(nop)  CMD [\"/files/entrypoint.sh\"]",
      "empty_layer": true
    }
  ],
  "os": "linux",
  "rootfs": {
    "type": "layers"
  }
}' |  jq --argjson details '["'$(sed '1q;d' $LAYER_CACHE | cut -d, -f3)'","'$(sed '2q;d' $LAYER_CACHE | cut -d, -f3)'"]' -c '.rootfs.diff_ids += $details'
}

function repositoryManifestGeneration {
  echo \
'{
  "schemaVersion": 2,
  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
  "config": {
    "mediaType": "application/vnd.docker.container.image.v1+json",
    "size": 0,
    "digest": "placeholder"
  },
  "layers": []
}' | jq --argjson size $(tail -1 $LAYER_CACHE | cut -d, -f1) --arg digest $(tail -1 $LAYER_CACHE | cut -d, -f2) --argjson layers $(layerJSONGeneration) -c '.config.size = $size | .config.digest = $digest | .layers += $layers'
}

getopts ":lir" flag
case "$flag" in
  l) layerJSONGeneration ;;
  i) imageManifestGeneration ;;
  r) repositoryManifestGeneration ;;
esac

Makefile.manual-docker-image

.DEFAULT_GOAL := helper
BUILD_TIME ?= $(shell date '+%s')

# Useful for colour coding outputs
TEXT_RED = \033[0;31;1m
TEXT_BLUE = \033[0;34;1m
TEXT_GREEN = \033[0;32;1m
TEXT_PURPLE = \033[0;35;1m
TEXT_NOCOLOR = \033[0m

# Configurations for the below run
REPOSITORY_NAME := test
LAYERS_FOLDER="./layers/"
LAYER_CACHE ="${LAYERS_FOLDER}/layers.cache"

helper: # Adapted from: https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
	@echo "Available targets..." # @ will not output shell command part to stdout that Makefiles normally do but will execute and display the output.
	@grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

all: build run ## Runs through all required steps

build: ## Builds the application
	go build -o app app.go
	echo "This is my configuration file" > app.data

run: ## Runs through the example workflow with two layers
	-mkdir ${LAYERS_FOLDER}

	# Layer 1 - Binary
	# TODO fix this section as it generates a new tar and gzip causing a new shasum for the repo layer digest even though the layer digest is the same.
	tar cvf ${LAYERS_FOLDER}/binary_layer.tar app && gzip -fk ${LAYERS_FOLDER}/binary_layer.tar
	$(MAKE) layer MANIFEST_FILE=${LAYERS_FOLDER}/binary_layer.tar.gz LOCAL_FILE=${LAYERS_FOLDER}/binary_layer.tar

	# # Layer 2 - Configuration file
	tar cvf ${LAYERS_FOLDER}/data_layer.tar app.data && gzip -fk ${LAYERS_FOLDER}/data_layer.tar
	$(MAKE) layer MANIFEST_FILE=${LAYERS_FOLDER}/data_layer.tar.gz LOCAL_FILE=${LAYERS_FOLDER}/data_layer.tar

  # Layer 3- Image manifest file. Links the different layers above together to make the image. Note: needs to be last in the file for tracking. Hacky but works.
	LAYER_CACHE=${LAYER_CACHE} bash jsonGeneration.sh -i > ${LAYERS_FOLDER}/manifest.json
	$(MAKE) layer MANIFEST_FILE=${LAYERS_FOLDER}/manifest.json LOCAL_FILE=${LAYERS_FOLDER}/manifest.json

  # Using the details from the layers create the required registry/repository manifest
	$(MAKE) repoManifest
	$(MAKE) valuesToCompare

	rm ${LAYER_CACHE}

layer: ## Uploads the indivdual layers to ECR
	#FIXME this calculation as it appears to be an issue with whats transmitted compared to whats local
	$(eval SIZE=$(shell expr $(shell du -sb ${MANIFEST_FILE} | awk '{print $$1}') - 1))
	$(eval MANIFEST_FILE_DIGEST=$(shell sha256sum ${MANIFEST_FILE} | awk '{print $$1}'))
	$(eval TAR_FILE_DIGEST=$(shell sha256sum ${LOCAL_FILE} | awk '{print $$1}'))
	$(eval ILUID=$(shell aws ecr initiate-layer-upload --repository-name ${REPOSITORY_NAME} --query uploadId | jq -r .))
	aws ecr upload-layer-part --repository-name ${REPOSITORY_NAME} --upload-id ${ILUID} --part-first-byte 0 --part-last-byte ${SIZE} --layer-part-blob fileb://${MANIFEST_FILE}
	-aws ecr complete-layer-upload --repository-name ${REPOSITORY_NAME} --upload-id ${ILUID} --layer-digests sha256:${MANIFEST_FILE_DIGEST}
	echo "${SIZE},sha256:${MANIFEST_FILE_DIGEST},sha256:${TAR_FILE_DIGEST}" >> ${LAYER_CACHE}
	$(MAKE) checkLayerAvailability SUM="sha256:${MANIFEST_FILE_DIGEST}"

repoManifest: ## Generates the manifest required for a put-iamge in ECR
	LAYER_CACHE=${LAYER_CACHE} bash jsonGeneration.sh -r > ${LAYERS_FOLDER}/repositoryManifest.json
	aws ecr put-image --repository-name ${REPOSITORY_NAME} --image-tag customUpload --image-manifest file://${LAYERS_FOLDER}/repositoryManifest.json
	echo "Image is located at your registry under the image/tag of: ${REPOSITORY_NAME}:customUpload"

checkLayerAvailability: ## Checks individual layer availability from ECR's perspective
	@if [ $(shell aws ecr batch-check-layer-availability --output text --repository-name ${REPOSITORY_NAME} --layer-digests ${SUM} --query layers[0].layerAvailability) != "AVAILABLE" ] ; then echo "Image Availability failed"; return 1; fi

valuesToCompare: ## Using the sha256 values for the individual components will show the equivalent docker vaules
	@echo "Repo digest:\n\
	  Local file: ${TEXT_GREEN}$(shell sha256sum ${LAYERS_FOLDER}/repositoryManifest.json | awk '{print $1}')${TEXT_NOCOLOR}\n\
	  From docker: ${TEXT_BLUE}docker inspect --format "{{.RepoDigests}}" <image>${TEXT_NOCOLOR}"
ifdef IMAGE
	@echo "  Execution: ${TEXT_PURPLE}$(shell docker inspect --format "{{.RepoDigests}}" ${IMAGE})${TEXT_NOCOLOR}"
endif
	@echo "\nImage digest:\n\
	  Local file: ${TEXT_GREEN}$(shell sha256sum ${LAYERS_FOLDER}/manifest.json | awk '{print $1}')${TEXT_NOCOLOR}\n\
	  From docker: ${TEXT_BLUE}docker inspect --format "{{.Id}}" <image>${TEXT_NOCOLOR}"
ifdef IMAGE
	@echo "  Execution: ${TEXT_PURPLE}$(shell docker inspect --format "{{.Id}}" ${IMAGE})${TEXT_NOCOLOR}"
endif
	@echo "\nFrom the above sample layers:\n\
	  Local file - Layer 1: ${TEXT_GREEN}$(shell sha256sum ${LAYERS_FOLDER}/binary_layer.tar | awk '{print $1}')${TEXT_NOCOLOR}\n\
	  Local file - Layer 2: ${TEXT_GREEN}$(shell sha256sum ${LAYERS_FOLDER}/data_layer.tar | awk '{print $1}')${TEXT_NOCOLOR}\n\
	  From docker: ${TEXT_BLUE}docker inspect --format "{{.RootFS.Layers}}" <image>${TEXT_NOCOLOR}"
ifdef IMAGE
	@echo "  Execution: ${TEXT_PURPLE}$(shell docker inspect --format "{{.RootFS.Layers}}" ${IMAGE})${TEXT_NOCOLOR}"
endif

test: ## Tests against the application

clean: ## Cleans up any old/unneeded items
	-rm -r ${LAYERS_FOLDER}
	-rm app && rm app.data