Script to build TFLite benchmark_model tool and label_image demo for Android (arm64) including patches for FP16 support and optional RUY support

0001-tflite-allow-fp16-for-fp32-models.patch

From b5a99e2174f115f87c0cb3de9d2a2479a0193db2 Mon Sep 17 00:00:00 2001
From: Koan-Sin Tan <koansin.tan@gmail.com>
Date: Sun, 18 Nov 2018 13:11:38 +0800
Subject: [PATCH 1/3] add cmdline option to allow running fp32 models with fp16

Add an option for TFLite benchmark_model to allow running fp32
models with fp16. Useful when testing NNPAI accelerators with
fp16.
---
 .../lite/tools/benchmark/benchmark_tflite_model.cc       | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
index 83e0ff1f872ef7..2d8b3a198c2967 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
@@ -200,6 +200,7 @@ BenchmarkParams BenchmarkTfLiteModel::DefaultParams() {
   default_params.AddParam("input_layer_shape",
                           BenchmarkParam::Create<std::string>(""));
   default_params.AddParam("use_nnapi", BenchmarkParam::Create<bool>(false));
+  default_params.AddParam("allow_fp16", BenchmarkParam::Create<bool>(false));
   return default_params;
 }
 
@@ -219,7 +220,8 @@ std::vector<Flag> BenchmarkTfLiteModel::GetFlags() {
       CreateFlag<std::string>("input_layer", &params_, "input layer names"),
       CreateFlag<std::string>("input_layer_shape", &params_,
                               "input layer shape"),
-      CreateFlag<bool>("use_nnapi", &params_, "use nnapi api")};
+      CreateFlag<bool>("use_nnapi", &params_, "use nnapi api"),
+      CreateFlag<bool>("allow_fp16", &params_, "allow fp16")};
 
   flags.insert(flags.end(), specific_flags.begin(), specific_flags.end());
   return flags;
@@ -233,6 +235,7 @@ void BenchmarkTfLiteModel::LogParams() {
   TFLITE_LOG(INFO) << "Input shapes: ["
                    << params_.Get<std::string>("input_layer_shape") << "]";
   TFLITE_LOG(INFO) << "Use nnapi : [" << params_.Get<bool>("use_nnapi") << "]";
+  TFLITE_LOG(INFO) << "Allow fp16 : [" << params_.Get<bool>("allow_fp16") << "]";
 }
 
 bool BenchmarkTfLiteModel::ValidateParams() {
@@ -328,6 +331,10 @@ void BenchmarkTfLiteModel::Init() {
   interpreter->UseNNAPI(use_nnapi);
   ApplyDelegates();
 
+  bool allow_fp16 = params_.Get<bool>("allow_fp16");
+
+  interpreter->SetAllowFp16PrecisionForFp32(allow_fp16);
+
   auto interpreter_inputs = interpreter->inputs();
 
   if (!inputs.empty()) {

From dc4b9745b41ca7b2a00efb142bb0ab1e9895b4e0 Mon Sep 17 00:00:00 2001
From: Koan-Sin Tan <koansin.tan@gmail.com>
Date: Sun, 18 Nov 2018 13:31:02 +0800
Subject: [PATCH 2/3] add option for label_image to allow running fp32 models
 with fp16

Add an option for TFLite label_image to allow running fp32
models with fp16. Useful when testing NNPAI accelerators with
fp16.
---
 tensorflow/lite/examples/label_image/label_image.cc | 6 ++++++
 tensorflow/lite/examples/label_image/label_image.h  | 1 +
 2 files changed, 7 insertions(+)

diff --git a/tensorflow/lite/examples/label_image/label_image.cc b/tensorflow/lite/examples/label_image/label_image.cc
index b8dc2840dfb49f..b83563280ec173 100644
--- a/tensorflow/lite/examples/label_image/label_image.cc
+++ b/tensorflow/lite/examples/label_image/label_image.cc
@@ -113,6 +113,7 @@ void RunInference(Settings* s) {
   }
 
   interpreter->UseNNAPI(s->accel);
+  interpreter->SetAllowFp16PrecisionForFp32(s->allow_fp16);
 
   if (s->verbose) {
     LOG(INFO) << "tensors size: " << interpreter->tensors_size() << "\n";
@@ -253,6 +254,7 @@ void RunInference(Settings* s) {
 void display_usage() {
   LOG(INFO) << "label_image\n"
             << "--accelerated, -a: [0|1], use Android NNAPI or not\n"
+            << "--allow_fp16, -f: [0|1], allow running fp32 models with fp16 not\n"
             << "--count, -c: loop interpreter->Invoke() for certain times\n"
             << "--input_mean, -b: input mean\n"
             << "--input_std, -s: input standard deviation\n"
@@ -273,6 +275,7 @@ int Main(int argc, char** argv) {
   while (1) {
     static struct option long_options[] = {
         {"accelerated", required_argument, nullptr, 'a'},
+        {"allow_fp16", required_argument, nullptr, 'f'},
         {"count", required_argument, nullptr, 'c'},
         {"verbose", required_argument, nullptr, 'v'},
         {"image", required_argument, nullptr, 'i'},
@@ -305,6 +308,9 @@ int Main(int argc, char** argv) {
         s.loop_count =
             strtol(optarg, nullptr, 10);  // NOLINT(runtime/deprecated_fn)
         break;
+      case 'f':
+        s.allow_fp16 = strtol(optarg, nullptr, 10);  // NOLINT(runtime/deprecated_fn)
+        break;
       case 'i':
         s.input_bmp_name = optarg;
         break;
diff --git a/tensorflow/lite/examples/label_image/label_image.h b/tensorflow/lite/examples/label_image/label_image.h
index 88b047fecc4b3e..cc46e56b64a9dc 100644
--- a/tensorflow/lite/examples/label_image/label_image.h
+++ b/tensorflow/lite/examples/label_image/label_image.h
@@ -26,6 +26,7 @@ struct Settings {
   bool accel = false;
   bool input_floating = false;
   bool profiling = false;
+  bool allow_fp16 = false;
   int loop_count = 1;
   float input_mean = 127.5f;
   float input_std = 127.5f;

From ab7b554405dc4bf5b023522095a9b4a318327e0b Mon Sep 17 00:00:00 2001
From: Koan-Sin Tan <koansin.tan@gmail.com>
Date: Tue, 12 Feb 2019 16:15:22 +0800
Subject: [PATCH 3/3] rebase and run clang-format

---
 .../lite/examples/label_image/label_image.cc  | 32 ++++++++++---------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/tensorflow/lite/examples/label_image/label_image.cc b/tensorflow/lite/examples/label_image/label_image.cc
index b83563280ec173..340fbab5c6fcc9 100644
--- a/tensorflow/lite/examples/label_image/label_image.cc
+++ b/tensorflow/lite/examples/label_image/label_image.cc
@@ -252,20 +252,21 @@ void RunInference(Settings* s) {
 }
 
 void display_usage() {
-  LOG(INFO) << "label_image\n"
-            << "--accelerated, -a: [0|1], use Android NNAPI or not\n"
-            << "--allow_fp16, -f: [0|1], allow running fp32 models with fp16 not\n"
-            << "--count, -c: loop interpreter->Invoke() for certain times\n"
-            << "--input_mean, -b: input mean\n"
-            << "--input_std, -s: input standard deviation\n"
-            << "--image, -i: image_name.bmp\n"
-            << "--labels, -l: labels for the model\n"
-            << "--tflite_model, -m: model_name.tflite\n"
-            << "--profiling, -p: [0|1], profiling or not\n"
-            << "--num_results, -r: number of results to show\n"
-            << "--threads, -t: number of threads\n"
-            << "--verbose, -v: [0|1] print more information\n"
-            << "\n";
+  LOG(INFO)
+      << "label_image\n"
+      << "--accelerated, -a: [0|1], use Android NNAPI or not\n"
+      << "--allow_fp16, -f: [0|1], allow running fp32 models with fp16 not\n"
+      << "--count, -c: loop interpreter->Invoke() for certain times\n"
+      << "--input_mean, -b: input mean\n"
+      << "--input_std, -s: input standard deviation\n"
+      << "--image, -i: image_name.bmp\n"
+      << "--labels, -l: labels for the model\n"
+      << "--tflite_model, -m: model_name.tflite\n"
+      << "--profiling, -p: [0|1], profiling or not\n"
+      << "--num_results, -r: number of results to show\n"
+      << "--threads, -t: number of threads\n"
+      << "--verbose, -v: [0|1] print more information\n"
+      << "\n";
 }
 
 int Main(int argc, char** argv) {
@@ -309,7 +310,8 @@ int Main(int argc, char** argv) {
             strtol(optarg, nullptr, 10);  // NOLINT(runtime/deprecated_fn)
         break;
       case 'f':
-        s.allow_fp16 = strtol(optarg, nullptr, 10);  // NOLINT(runtime/deprecated_fn)
+        s.allow_fp16 =
+            strtol(optarg, nullptr, 10);  // NOLINT(runtime/deprecated_fn)
         break;
       case 'i':
         s.input_bmp_name = optarg;

0002-tflite-label-image-log-timing-dist.patch

diff --git a/tensorflow/lite/examples/label_image/label_image.cc b/tensorflow/lite/examples/label_image/label_image.cc
index 7c6f523..e458423 100644
--- a/tensorflow/lite/examples/label_image/label_image.cc
+++ b/tensorflow/lite/examples/label_image/label_image.cc
@@ -24,6 +24,8 @@ limitations under the License.
 #include <string>
 #include <unordered_set>
 #include <vector>
+#include <algorithm>
+#include <functional>
 
 #include <fcntl.h>      // NOLINT(build/include_order)
 #include <getopt.h>     // NOLINT(build/include_order)
@@ -45,7 +47,7 @@ limitations under the License.
 namespace tflite {
 namespace label_image {
 
-double get_us(struct timeval t) { return (t.tv_sec * 1000000 + t.tv_usec); }
+double get_us(struct timeval t) { return (t.tv_sec * 1000000.0 + (double)t.tv_usec); }
 
 // Takes a file name, and loads a list of labels from it, one per line, and
 // returns a vector of the strings. It pads with empty strings so the length
@@ -87,6 +89,41 @@ void PrintProfilingInfo(const profiling::ProfileEvent* e, uint32_t op_index,
             << "\n";
 }
 
+void LogTimingDistribution(std::vector<double> & timings) {
+  int timing_count = timings.size();
+  int p5_count = timing_count / 20;
+  std::sort(timings.begin(), timings.end());
+
+  int i = 0;
+  int c = 0;
+  double avg = 0;
+  for(auto t : timings) {
+    double ms = t / 1000.0;
+    avg += ms;
+    ++i;
+    ++c;
+    if (i > p5_count || c == timing_count) {
+      LOG(INFO) << " => " << (c*100/timing_count) << "% is " << ms <<
+          " ms. avg within bucket: " << (avg/i) << " ms\n";
+      i = 0;
+      avg = 0;
+    }
+  }
+}
+
+void LogTimings(std::vector<double> & timings) {
+  int timing_count = timings.size();
+
+  int i = 0;
+  int c = 0;
+  double avg = 0;
+  for(auto t : timings) {
+    double ms = t / 1000.0;
+    LOG(INFO) << "Iteration_" << c << "=" << ms <<" ms\n";
+    ++c;
+  }
+}
+
 void RunInference(Settings* s) {
   if (!s->model_name.c_str()) {
     LOG(ERROR) << "no model file name\n";
@@ -188,19 +226,26 @@ void RunInference(Settings* s) {
 
   if (s->profiling) profiler->StartProfiling();
 
-  struct timeval start_time, stop_time;
+  struct timeval start_time, stop_time, act_time;
   gettimeofday(&start_time, nullptr);
+  std::vector<double> timings;
+  double prev_time_us = get_us(start_time);
   for (int i = 0; i < s->loop_count; i++) {
     if (interpreter->Invoke() != kTfLiteOk) {
       LOG(FATAL) << "Failed to invoke tflite!\n";
     }
+    gettimeofday(&act_time, NULL);
+    double act_time_us = get_us(act_time);
+    timings.push_back(act_time_us - prev_time_us);
+    prev_time_us = act_time_us;
   }
   gettimeofday(&stop_time, nullptr);
   LOG(INFO) << "invoked \n";
   LOG(INFO) << "average time: "
             << (get_us(stop_time) - get_us(start_time)) / (s->loop_count * 1000)
             << " ms \n";
-
+  LogTimings(timings);
+  LogTimingDistribution(timings);
   if (s->profiling) {
     profiler->StopProfiling();
     auto profile_events = profiler->GetProfileEvents();

build-tflite-1.13.sh

#!/usr/bin/env bash

set -eo pipefail

# Release tag or branch name
TF_VERSION=${TF_VERSION:-"v1.13.1"}

# Clone a fresh copy
if [ -n "${TF_CLONE}" ]; then
    TF_SRCDIR=/tmp/build-tflite/tensorflow_src
    rm -rf ${TF_SRCDIR}
    mkdir -p ${TF_SRCDIR}
    if [ ! -d "${TF_SRCDIR}" ]; then
        git clone --branch ${TF_VERSION} --depth 1 https://github.com/tensorflow/tensorflow ${TF_SRCDIR}
    fi

    # Download patches
    TF_PATCH_DIR=/tmp/build-tflite/tf_patches
    TF_APPLY_PATCH="yes"
    TF_PATCHES=("0001-tflite-allow-fp16-for-fp32-models.patch" "0002-tflite-label-image-log-timing-dist.patch")
    for patch in "${TF_PATCHES[@]}"; do
        wget -q -nc --show-progress https://gist.github.com/muendelezaji/47b32812a15b622f05019b0e433de9b1/raw/${patch} -P ${TF_PATCH_DIR}
    done
fi

# Alternatively, provide path to cloned TensorFlow repo
TF_SRCDIR=${1:-"$TF_SRCDIR"}
TF_SRCDIR="$(readlink -f ${1:-"$TF_SRCDIR"})" # get absolute
# Folder with patches
TF_PATCH_DIR="$(readlink -f ${TF_PATCH_DIR:-"patches"})"
TF_APPLY_PATCH=${TF_APPLY_PATCH:-"yes"}
# Output folder
TF_OUTDIR="$(readlink -f ${TF_OUTDIR:-"out"})"

# Sanity checks
if [ ! -d "${TF_SRCDIR}" ]; then
    echo "ERROR: Cannot find TensorFlow source directory. Ensure it exists or set TF_CLONE to do a new clone."
    exit 1
elif [ ! -f "${TF_SRCDIR}/configure" -o ! -f "${TF_SRCDIR}/BUILD" ]; then
    echo "ERROR: Missing configure or BUILD file in $TF_SRCDIR. Ensure it is a valid Bazel workspace."
    exit 1
elif [ "${TF_APPLY_PATCH}" = "yes" -a ! -d "${TF_PATCH_DIR}" ]; then
    echo "ERROR: Patches directory was not found. It must exist when TF_APPLY_PATCH is set to 'yes'."
    exit 1
elif [ -z "${ANDROID_NDK_HOME}" -o -z "${ANDROID_SDK_HOME}" ]; then
    echo "ERROR: Cannot locate Android NDK or SDK. Ensure both ANDROID_NDK_HOME and ANDROID_SDK_HOME are set."
    exit 1
fi

# Apply required patches
if [ "${TF_APPLY_PATCH}" = "yes" ]; then
    pushd ${TF_SRCDIR}
    for patch in $(find "${TF_PATCH_DIR}" -type f -name "*.patch" | sort); do
        git apply ${patch}
    done
    git config --local user.name temp
    git config --local user.email temp@example.com
    git commit -am "[TFLite] Add fp16 support and log timing distribution"
    popd
fi

# Select features
TF_BUILD_VARS="TF_NEED_CUDA=0"
TF_BUILD_VARS+=" TF_NEED_OPENCL_SYCL=0"
TF_BUILD_VARS+=" TF_NEED_ROCM=0"
TF_BUILD_VARS+=" TF_NEED_JEMALLOC=1"
TF_BUILD_VARS+=" TF_NEED_MPI=0"
TF_BUILD_VARS+=" TF_ENABLE_XLA=0"
TF_BUILD_VARS+=" TF_DOWNLOAD_CLANG=0"
TF_BUILD_VARS+=" TF_SET_ANDROID_WORKSPACE=1"
export ${TF_BUILD_VARS}

# Bazel settings
BAZEL_BIN=${BAZEL_BIN:-"$(command -v bazel || true)"}
BAZEL_JOBS=${BAZEL_JOBS:-"4"}
# BAZEL_PATCH="yes"
# BAZEL_AVAIL_RAM=1536 # MB
# BAZEL_AVAIL_CPU=4.0  # number of cpu cores (1.0 representing a single full core)
# BAZEL_AVAIL_IO=1.0   # workstation I/O capability (with 1.0 representing average workstation)
# if [ -n "$BAZEL_AVAIL_RAM" -a -n "$BAZEL_AVAIL_CPU" -a -n "$BAZEL_AVAIL_IO" ]; then
#     BAZEL_RESOURCES="--local_resources ${BAZEL_AVAIL_RAM},${BAZEL_AVAIL_CPU},${BAZEL_AVAIL_IO}"
# fi

# Download if not present - use portable binary to avoid installation
if [ ! -x "${BAZEL_BIN}" ]; then
    BAZEL_VERSION="0.20.0" # was 0.19.2
    BAZEL_FILE="bazel-${BAZEL_VERSION}-linux-x86_64"
    BAZEL_DIR="/tmp/build-tflite/bazel_bin/${BAZEL_VERSION}"
    mkdir -p ${BAZEL_DIR}
    wget -q -nc --show-progress https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/${BAZEL_FILE} -P ${BAZEL_DIR}
    chmod +x ${BAZEL_DIR}/${BAZEL_FILE}
    ln -sf ${BAZEL_DIR}/${BAZEL_FILE} ${BAZEL_DIR}/bazel
    export PATH="${BAZEL_DIR}:$PATH"
    BAZEL_BIN="$(command -v bazel)"
fi

# Build config
# Disable default-on TensorFlow cloud features
BAZEL_COPT_FLAGS=" --config=noaws --config=nogcp --config=nohdfs --config=nokafka --config=noignite --config=nonccl --define tensorflow_mkldnn_contraction_kernel=0"
# Android "--config=android_arm64" is equivalent to:
# --cpu=arm64-v8a --fat_apk_cpu=arm64-v8a --crosstool_top=//external:android/crosstool --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
BAZEL_COPT_FLAGS+=" --copt=-DTFLITE_PROFILING_ENABLED --cxxopt=-std=c++11 --config=monolithic --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=28" # was 21
# TFLite targets for Android
BAZEL_EXTRA_FLAGS=" //tensorflow/lite/examples/label_image:label_image"
BAZEL_EXTRA_FLAGS+=" //tensorflow/lite/tools/benchmark:benchmark_model"

# Python
TF_PYTHON_VERSION=${TF_PYTHON_VERSION:-"2"}
export PYTHON_BIN_PATH="$(command -v python${TF_PYTHON_VERSION})"
export USE_DEFAULT_PYTHON_LIB_PATH="1"

# GCC optimisations
# For more info, see: https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html
export CC_OPT_FLAGS="-march=armv8-a"
# Add suffix to TF_OUTDIR for more specialised builds
if [ "${TF_GCC_OPTS}" = "simd" ]; then
    CC_OPT_FLAGS="-march=armv8-a+simd"
    TF_OUTDIR="${TF_OUTDIR}-simd"
elif [ "${TF_GCC_OPTS}" = "fp16" ]; then
    CC_OPT_FLAGS="-march=armv8.2-a+fp16"
    TF_OUTDIR="${TF_OUTDIR}-fp16"
fi
BAZEL_COPT_FLAGS+=" --copt=${CC_OPT_FLAGS}"

# Android
export NDK_ROOT="${ANDROID_NDK_HOME}"
export ANDROID_API_LEVEL="28" # was 26
export ANDROID_BUILD_TOOLS_VERSION="28.0.3"

# Configure and build
pushd ${TF_SRCDIR} && ./configure
# Clean if not an incremental rebuild
if [ -z "${TF_REBUILD}" ]; then
    ${BAZEL_BIN} clean
fi

${BAZEL_BIN} build ${BAZEL_RESOURCES} -c opt --verbose_failures --jobs=${BAZEL_JOBS} ${BAZEL_COPT_FLAGS} ${BAZEL_EXTRA_FLAGS}

mkdir -p ${TF_OUTDIR}
cp -a --remove-destination -t ${TF_OUTDIR} \
    bazel-bin/tensorflow/lite/examples/label_image/label_image \
    bazel-bin/tensorflow/lite/tools/benchmark/benchmark_model

echo "Builds saved to: ${TF_OUTDIR}"

build-tflite-for-arm64.jenkinsfile

#!/usr/bin/env groovy

pipeline {
    agent any

    parameters {
        choice name: 'BAZEL_VERSION', choices: ['0.19.2', '0.21.0', '0.24.1'], description: 'Bazel version to use when building'
        choice name: 'NDK_VERSION', choices: ['13', '14', '18', '19'], description: 'NDK version to use when building'
        choice name: 'SDK_VERSION', choices: ['21', '28', '29'], description: 'SDK version to use when building'
        choice name: 'TF_VERSION', choices: ['v1.13.1', 'v1.14.0', 'v2.0.0-beta1'], description: 'Tagged version of TensorFlow to download'
    }

    options {
        buildDiscarder logRotator(numToKeepStr: '10')
        skipDefaultCheckout()
        skipStagesAfterUnstable()
        timeout time: 30, unit: 'MINUTES'
        timestamps()
    }

    environment {
        ANDROID_NDK_HOME = "${env.BUILD_TOOLS}/android/android-ndk-r${params.NDK_VERSION}"
        ANDROID_SDK_HOME = "${env.BUILD_TOOLS}/android/android-sdk"
        BAZEL_HOME = "${env.BUILD_TOOLS}/bazel/bazel-${params.BAZEL_VERSION}"
        VENV_HOME  = "${env.HOME}/.local"

        // Set env from params (used in build shell script)
        BAZEL_VERSION = "${params.BAZEL_VERSION}"
        NDK_VERSION   = "${params.NDK_VERSION}"
        SDK_VERSION   = "${params.SDK_VERSION}"
        TF_VERSION    = "${params.TF_VERSION}"

        TF_SRCDIR    = 'src'
        TF_OUTDIR    = 'out'
        TF_PATCH_DIR = 'patches'
        TF_PATCH     = (params.TF_VERSION ==~ '.*1.13.*') ? 'yes' : 'no'
    }

    stages {
        stage('Checkout repo') {
            steps {
                dir("${env.TF_SRCDIR}") {
                    checkout([$class: 'GitSCM', branches: [[name: "refs/tags/${params.TF_VERSION}"]], userRemoteConfigs: [[url: 'https://github.com/tensorflow/tensorflow']]])
                }
            }
        }

        stage('Build TFLite') {
            steps {
                echo '====++++ executing Build TFLite ++++===='
                withEnv([
                    "BAZEL_BIN=${env.BAZEL_HOME}/bin/bazel",
                    "PATH+BAZEL=${env.BAZEL_HOME}/bin",
                    "PATH+VENV=${env.VENV_HOME}/bin",
                ]) {
                    sh '''
                        # Create and activate Python venv
                        virtualenv .
                        virtualenv --relocatable .
                        . ./bin/activate

                        # Install pip requirements
                        # From REQUIRED_PACKAGES in tensorflow/tools/pip_package/setup.py
                        pip install -U pip six numpy wheel setuptools mock future>=0.17.1
                        pip install -U keras_applications==1.0.6 --no-deps
                        pip install -U keras_preprocessing==1.0.5 --no-deps

                        # Download patches
                        GIST_URL=https://gist.github.com/muendelezaji/47b32812a15b622f05019b0e433de9b1/raw
                        mkdir -p $TF_PATCH_DIR
                        for patch in '0001-tflite-allow-fp16-for-fp32-models.patch' '0002-tflite-label-image-log-timing-dist.patch'; do
                            curl -fsL $GIST_URL/$patch > $TF_PATCH_DIR/$patch
                        done

                        # Get build script and run it
                        BUILD_SCRIPT='build-tflite-1.13.sh'
                        curl -fsL $GIST_URL/$BUILD_SCRIPT > $BUILD_SCRIPT
                        chmod +x $BUILD_SCRIPT
                        bash -x ./$BUILD_SCRIPT

                        deactivate
                    '''
                }
            }

            post {
                success {
                    echo '====++++ Build TFLite executed succesfully ++++===='
                    archiveArtifacts artifacts: "${env.TF_OUTDIR}/*", fingerprint: true
                }

                failure {
                    echo '====++++ Build TFLite execution failed ++++===='
                }
            }
        }

        stage('Run TFLite') {
            steps {
                echo '====++++ executing Run TFLite ++++===='
            }
        }
    }

    post {
        always {
            echo '====++++ Pipeline finished ++++===='
            deleteDir() // clean up the workspace
        }
    }
}

build-tflite-for-arm64.sh

#!/bin/sh

# Configurable build tool parameters
BAZEL_VERSION=0.19.2
NDK_VERSION=18
SDK_VERSION=28
TF_VERSION=v1.13.1
TF_PATCH=yes
TF_PATCH_DIR=patches

# Setup build tools - use global config or custom tools plugin
ANDROID_NDK_HOME=$BUILD_TOOLS/android/android-ndk-r$NDK_VERSION
ANDROID_SDK_HOME=$BUILD_TOOLS/android/android-sdk
BAZEL_HOME=$BUILD_TOOLS/bazel/bazel-$BAZEL_VERSION
VENV_HOME=$HOME/.local
# Add them to path
PATH=$BAZEL_HOME/bin:$VENV_HOME/bin:$PATH

# Create and activate Python venv
virtualenv .
virtualenv --relocatable .
. ./bin/activate

# Install pip requirements
# From REQUIRED_PACKAGES in tensorflow/tools/pip_package/setup.py
pip install -U pip six numpy wheel setuptools mock future>=0.17.1
pip install -U keras_applications==1.0.6 --no-deps
pip install -U keras_preprocessing==1.0.5 --no-deps

# Download patches
GIST_URL=https://gist.github.com/muendelezaji/47b32812a15b622f05019b0e433de9b1/raw
mkdir -p $TF_PATCH_DIR
for patch in '0001-tflite-allow-fp16-for-fp32-models.patch'; do
    curl -fsL $GIST_URL/$patch > $TF_PATCH_DIR/$patch
done

# Get build script and run it
BUILD_SCRIPT='build-tflite.sh'
curl -fsL $GIST_URL/$BUILD_SCRIPT > $BUILD_SCRIPT
chmod +x $BUILD_SCRIPT
bash -x ./$BUILD_SCRIPT

deactivate

build-tflite.sh

#!/usr/bin/env bash

set -eo pipefail

if [ "${BUILD_CONFIG}" = "2.0" ]; then
    TF_VERSION=v2.0.0-beta1
    TF_PATCH=0
    TF_CLONE=1
    BAZEL_VERSION=0.24.1
    BAZEL_BIN=none # force download
    BAZEL_COPT_FLAGS+=" --config=v2"
elif [ "${BUILD_CONFIG}" = "1.14" ]; then
    TF_VERSION=v1.14.0
    TF_PATCH=0
    TF_CLONE=1
    BAZEL_VERSION=0.24.1
    BAZEL_BIN=none
fi

# Release tag or branch name
TF_VERSION=${TF_VERSION:-"v1.13.1"}
TF_PATCH=${TF_PATCH:-"1"}
# Shorthand
# : ${TF_VERSION:="v1.13.1"}
# : ${TF_PATCH:="1"}

# Clone a fresh copy
if [ -n "${TF_CLONE}" ]; then
    TF_SRCDIR="/tmp/build-tflite/tensorflow/${TF_VERSION}/src"
    TF_OUTDIR="/tmp/build-tflite/tensorflow/${TF_VERSION}/out"
    if [ ! -d "${TF_SRCDIR}" ]; then
        git clone --branch ${TF_VERSION} --depth 1 https://github.com/tensorflow/tensorflow ${TF_SRCDIR}
    fi

    # Download patches
    if [ "${TF_PATCH}" = "1" ]; then
        TF_PATCH_DIR="/tmp/build-tflite/tensorflow/${TF_VERSION}/patches"
        TF_PATCHES=("0001-tflite-allow-fp16-for-fp32-models.patch" "0002-tflite-label-image-log-timing-dist.patch")
        for patch in "${TF_PATCHES[@]}"; do
            wget -q -nc --show-progress https://gist.github.com/muendelezaji/47b32812a15b622f05019b0e433de9b1/raw/${patch} -P ${TF_PATCH_DIR}
        done
    fi
fi

# Alternatively, provide path to cloned TensorFlow repo
TF_SRCDIR="${TF_SRCDIR:-$(readlink -f "src")}"
TF_OUTDIR="${TF_OUTDIR:-$(readlink -f "out")}"           # Build outputs folder
TF_PATCH_DIR="${TF_PATCH_DIR:-$(readlink -f "patches")}" # Folder with patches
# Shorthand
# : ${TF_SRCDIR:=$(readlink -f "src")}
# : ${TF_OUTDIR:=$(readlink -f "out")}
# : ${TF_PATCH_DIR:=$(readlink -f "patches")}

# Sanity checks
if [ ! -d "${TF_SRCDIR}" ]; then
    echo "ERROR: Cannot find TensorFlow source directory. Ensure it exists or set TF_CLONE to do a new clone."
    exit 1
elif [ ! -f "${TF_SRCDIR}/configure" -o ! -f "${TF_SRCDIR}/BUILD" ]; then
    echo "ERROR: Missing configure or BUILD file in ${TF_SRCDIR}. Ensure it is a valid Bazel workspace."
    exit 1
elif [ "${TF_PATCH}" = "1" -a ! -d "${TF_PATCH_DIR}" ]; then
    echo "ERROR: Patches directory was not found. It must exist when TF_PATCH is set to 'yes'."
    exit 1
elif [ -z "${ANDROID_NDK_HOME}" -o -z "${ANDROID_SDK_HOME}" ]; then
    echo "ERROR: Cannot locate Android NDK or SDK. Ensure both ANDROID_NDK_HOME and ANDROID_SDK_HOME are set."
    exit 1
fi

pushd ${TF_SRCDIR}

# Apply required patches
if [ "${TF_PATCH}" = "1" ]; then
    for patch in $(find ${TF_PATCH_DIR} -type f -name "*.patch" | sort); do
        git apply ${patch}
    done
    git config --local user.name temp
    git config --local user.email temp@example.com
    git commit -am "[TFLite] All patched up and ready to go"
fi

# Bazel
BAZEL_BIN=${BAZEL_BIN:-"$(command -v bazel || true)"}
BAZEL_JOBS=${BAZEL_JOBS:-"4"}
# BAZEL_PATCH="no"
# BAZEL_AVAIL_RAM=2048 # MB
# BAZEL_AVAIL_CPU=4.0  # number of cpu cores (1.0 representing a single full core)
# BAZEL_AVAIL_IO=1.0   # workstation I/O capability (with 1.0 representing average workstation)
# if [ -n "${BAZEL_AVAIL_RAM}" -a -n "${BAZEL_AVAIL_CPU}" -a -n "${BAZEL_AVAIL_IO}" ]; then
#     BAZEL_RESOURCES="--local_resources ${BAZEL_AVAIL_RAM},${BAZEL_AVAIL_CPU},${BAZEL_AVAIL_IO}"
# fi
if [ -n "${BAZEL_DRYRUN}" ]; then
    BAZEL_DEBUG_FLAGS+=" --nobuild"
fi
if [ -n "${BAZEL_VERBOSE}" ]; then
    BAZEL_DEBUG_FLAGS+=" -s --explain=${TF_OUTDIR}/explain.log"
fi
if [ -n "${BAZEL_DEBUG_FLAGS}" ]; then
    set -x
fi

# Download portable binary if not present
if [ ! -x "${BAZEL_BIN}" ]; then
    BAZEL_VERSION=${BAZEL_VERSION:-"0.24.1"}
    BAZEL_FILE="bazel-${BAZEL_VERSION}-linux-x86_64"
    BAZEL_DIR="/tmp/build-tflite/bazel/${BAZEL_VERSION}"

    wget -q -nc --show-progress https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/${BAZEL_FILE} -P ${BAZEL_DIR}
    chmod +x ${BAZEL_DIR}/${BAZEL_FILE}
    ln -sf ${BAZEL_DIR}/${BAZEL_FILE} ${BAZEL_DIR}/bazel
    export PATH="${BAZEL_DIR}:${PATH}"
    BAZEL_BIN="$(command -v bazel)"
fi

# Python
PYTHON_VERSION=${PYTHON_VERSION:-"2"}
export PYTHON_BIN_PATH="$(command -v python${PYTHON_VERSION})"
export USE_DEFAULT_PYTHON_LIB_PATH="1"

# Android
export ANDROID_BUILD_TOOLS_VERSION="28.0.3"
export ANDROID_API_LEVEL="28"
export ANDROID_NDK_API_LEVEL=${ANDROID_API_LEVEL}
NDK_VERSION="$(sed -En 's/Pkg.Revision = ([0-9]+).*/\1/p' ${ANDROID_NDK_HOME}/source.properties)"

# Select TensorFlow features
TF_BUILD_VARS+=" TF_NEED_CUDA=0"
TF_BUILD_VARS+=" TF_NEED_OPENCL_SYCL=0"
TF_BUILD_VARS+=" TF_NEED_ROCM=0"
TF_BUILD_VARS+=" TF_NEED_JEMALLOC=1"
TF_BUILD_VARS+=" TF_NEED_MPI=0"
TF_BUILD_VARS+=" TF_ENABLE_XLA=0"
TF_BUILD_VARS+=" TF_DOWNLOAD_CLANG=0"
TF_BUILD_VARS+=" TF_SET_ANDROID_WORKSPACE=1"
export ${TF_BUILD_VARS}

# Build options
# Disable unused default-on features
BAZEL_COPT_FLAGS+=" --config=noaws --config=nogcp --config=nohdfs --config=nokafka --config=noignite --config=nonccl --define tensorflow_mkldnn_contraction_kernel=0"
# Android config is equivalent to:
# --cpu=arm64-v8a --fat_apk_cpu=arm64-v8a --crosstool_top=//external:android/crosstool --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
BAZEL_COPT_FLAGS+=" --config=android_arm64 --config=monolithic --config=opt --cxxopt=-std=c++11"
# Enable RUY
if [ "${TF_ENABLE_RUY}" = "1" ]; then
    BAZEL_COPT_FLAGS+=" --define tflite_with_ruy=true"
else
    BAZEL_COPT_FLAGS+=" --define tflite_with_ruy=false"
fi

# TFLite targets for Android
BAZEL_TARGETS+=" //tensorflow/lite:libtensorflowlite.so"
BAZEL_TARGETS+=" //tensorflow/lite/examples/label_image:label_image"
BAZEL_TARGETS+=" //tensorflow/lite/tools/benchmark:benchmark_model"

build_variant() {
    local variant="${1}"
    local ccflags="${2}"
    # Output path
    local variant_name="${TF_VERSION}_ruy${TF_ENABLE_RUY}_${variant}"
    local variant_path="${TF_OUTDIR}/${variant_name}"

    # GCC build optimisations - autoloaded during Bazel config step
    # For more info, see https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html
    export CC_OPT_FLAGS=" -O3 -DTFLITE_PROFILING_ENABLED ${ccflags}"

    # Configure and build
    if [ "${BAZEL_REBUILD}" != "1" ]; then
        ${BAZEL_BIN} clean
    fi
    ./configure
    ${BAZEL_BIN} build ${BAZEL_DEBUG_FLAGS} ${BAZEL_RESOURCES} -c opt --verbose_failures --jobs=${BAZEL_JOBS} ${BAZEL_COPT_FLAGS} ${BAZEL_TARGETS}

    if [ "${BAZEL_DRYRUN}" != "1" ]; then
        # Move to output dir
        mkdir -p ${variant_path}
        mv -f -t ${variant_path} \
            bazel-bin/tensorflow/lite/libtensorflowlite.so \
            bazel-bin/tensorflow/lite/examples/label_image/label_image \
            bazel-bin/tensorflow/lite/tools/benchmark/benchmark_model
        echo "Builds saved to: ${variant_path}"
    fi
}

# Generic v8-a with SIMD instructions
build_variant "v8a" "-march=armv8-a+simd"
# Optimised FP16 instructions
build_variant "v8.2a" "-march=armv8.2-a+fp16+dotprod"

run-tflite.sh

#!/usr/bin/env bash

# README FIRST:
# ============
# Script to run the TFLite tools benchmark_model or label_image and log results to a file.
#
# Environment prerequisites:
#   1. Export ADB_SERIAL=<your_device_id> in your shell
#   2. Copy files to the device:
#       - binaries to /data/local/tmp/tflite/bin/
#       - models to /data/local/tmp/tflite/models/
#       - test data (sample images, labels file) to /data/local/tmp/tflite/data/
#   3. Setup device CPU/GPU/etc frequencies as appropriate
#
# Saved results location:
#   By default run logs are saved to '[LOGDIR]/[tool_name]-[run_date]/<model_name>.log' where:
#     - [LOGDIR] is set as an env variable, or defaults to working dir
#     - [tool_name] is one of 'benchmark_model' or 'label_image'
#     - [run_date] is the start date-time in the format 'YYYYmmdd_HHMMSS'
#
#   Depending on selected config, the log directory and file names may include one or more of the
#   run options such as - fp16, nnapi, ruy.

set -o pipefail

README_FIRST=yes  # Set this to "yes" disable readme prompt

show_info() {
    echo "
Script to run the TFLite tools benchmark_model or label_image and log results to a file.

Environment prerequisites:
  1. Export ADB_SERIAL=<your_device_id> in your shell
  2. Copy files to the device:
      - binaries to /data/local/tmp/tflite/bin/
      - models to /data/local/tmp/tflite/models/
      - test data (sample images, labels file) to /data/local/tmp/tflite/data/
  3. Setup device CPU/GPU/etc frequencies as appropriate

Saved results location:
  By default run logs are saved to '[LOGDIR]/[tool_name]-[run_date]/<model_name>.log' where:
    - [LOGDIR] is set as an env variable, or defaults to working dir
    - [tool_name] is one of 'benchmark_model' or 'label_image'
    - [run_date] is the start date-time in the format 'YYYYmmdd_HHMMSS'

  Depending on selected config, the log directory and file names may include one or more of the
  run options such as - fp16, nnapi, ruy.
"
}

if [ "$1" = "-h" -o "$1" = "--help" ]; then
    show_info
    exit 0
fi

if [ "$README_FIRST" != "yes" ]; then
    printf "README FIRST:\n============\n"
    show_info
    read -p "Press ENTER key to continue..."
fi

if [ -z "$ADB_SERIAL" ]; then
    printf "\nERROR: Environment ADB_SERIAL must be set!\n"
    exit 1
fi

# Use taskset instead of cpufreq to select processor affinity
if [ "$TASKSET" = "big" ]; then
    taskset_cmd="taskset f0"
elif [ "$TASKSET" = "little" ]; then
    taskset_cmd="taskset 0f"
fi

# Host to which device is connected
ADB_SERVER=${ADB_SERVER:-localhost}
ADB_CMD=(adb -H $ADB_SERVER -s $ADB_SERIAL shell)

warmups=${WARMUPS:-5} # benchmark_model only
iters=${ITERS:-50}
threads=${THREADS:-4}
prof=${PROFILE:-0} # label_image only
nnapi=${NNAPI:-0}
fp16=${FP16:-0}

# Directory for saved logs
[[ "$RUNTOOL" == label* ]] && tool=label_image || tool=benchmark_model
rundate=${RUNDATE:-$(date +%Y%m%d_%H%M%S)}
logpath=${LOGDIR:-$PWD}/$tool-$rundate

# Append config options where necessary
[ $nnapi = 1 ] && logpath=$logpath-nnapi
[ $fp16 = 1 ] && logpath=$logpath-fp16
# [ ! -d "$logpath" ] && mkdir -p $logpath
mkdir -p $(dirname $logpath) && logpath=$logpath.log

# Log runtime config
printf "\nDevice: $(${ADB_CMD[@]} getprop | grep -i ro.product.device)\n" | tee -a $logpath
printf "
Running TFLite with the following params:
  - taskset = ${taskset_cmd:-no}
  - warmups = $warmups runs (benchmark_model)
  - iterations = $iters runs
  - threads = $threads
  - profiling = $prof (label_image)
  - nnapi = $nnapi
  - fp16 = $fp16
  - logpath = $logpath
" | tee -a $logpath

# Folder with tflite installation (tools and data)
BASEDIR=/data/local/tmp/tflite

run_tool() {
    local tool=$1
    local model=$2
    if [ "$tool" = "label_image" ]; then
        # e.g. label_image --labels data/imagenet_labels.txt --image data/grace_hopper.bmp --threads 4 --count 50 --profiling 0 --accelerated 1 --allow_fp16 0 --tflite_model models/float/squeezenet.tflite
        ${ADB_CMD[@]} $taskset_cmd $BASEDIR/bin/label_image -l $BASEDIR/data/imagenet_labels.txt -i $BASEDIR/data/grace_hopper.bmp -t $threads -p $prof -c $iters -a $nnapi -f $fp16 -m $model
    else
        # e.g. benchmark_model --num_threads=4 --warmup_runs=5 --num_runs=50 --use_nnapi=0 --allow_fp16=0 --graph=models/float/squeezenet.tflite
        ${ADB_CMD[@]} $taskset_cmd $BASEDIR/bin/benchmark_model --num_threads=$threads --warmup_runs=$warmups --num_runs=$iters --use_nnapi=$nnapi --allow_fp16=$fp16 --graph=$model
    fi
}

for model in $(${ADB_CMD[@]} find $BASEDIR/models -type f -name "*.tflite"); do
    # logfile=$logpath/$(echo ${model##*/} | sed 's/.tflite//').log # use model name minus path dirname and extension
    run_tool $tool $model 2>&1 | tee -a $logpath
    sleep 1
done

printf "\nRUN COMPLETE: Saved runtime logs to: $logpath\n"