Tensorflow meets C# Azure function

.gitignore

bin/
.vscode/
obj/

azure_function.png

function.json

{
  "scriptFile": "./bin/TensorflowServerlessAzureLib.dll",
  "entryPoint": "TensorflowServerlessAzureLib.TensorflowImageClassification.Run",
  "bindings": [
    {
      "authLevel": "function",
      "name": "req",
      "type": "httpTrigger",
      "direction": "in"
    },
    {
      "name": "$return",
      "type": "http",
      "direction": "out"
    }
  ],
  "disabled": false
}

kudu.png

TensorflowImageClassification.cs

//
// Based on the code from: https://github.com/migueldeicaza/TensorFlowSharp/tree/master/Examples/ExampleInceptionInference
//
// An example for using the TensorFlow C# API for image recognition
// using a pre-trained inception model (http://arxiv.org/abs/1512.00567).
//
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using TensorFlow;

namespace TensorflowServerlessAzureLib
{
    public class TensorflowImageClassification
    {
        const string modelsDir = @"D:\home\site\wwwroot\models";

        private static IDictionary<string, TFGraph> graphCache = new ConcurrentDictionary<string, TFGraph>();

        private static IDictionary<string, string[]> labelsCache = new ConcurrentDictionary<string, string[]>();

        public static async Task<HttpResponseMessage> Run(HttpRequestMessage req, TextWriter log)
        {
            var queryParameters = PrepareQueryParameters(req); 
            log.WriteLine("start");
            var imageUrl = queryParameters["q"];
            var result = await TensorflowImageClassification.ExecuteIntercaption(imageUrl);
            log.WriteLine(result);
            var response = new HttpResponseMessage(HttpStatusCode.OK);
            
            response.Content = new StringContent(result, Encoding.UTF8, "application/text");
            return response;
        }

        public static async Task<HttpResponseMessage> RunIntercaption(HttpRequestMessage req)
        {
            var input = PrepareQueryParameters(req) ["q"];
            var res = await ExecuteIntercaption(input);
            return new HttpResponseMessage() { Content = new StringContent(res) };
        }

        private static Dictionary<string, string> PrepareQueryParameters(HttpRequestMessage req)
        {
            return req.RequestUri.PathAndQuery.Split('?') [1].Split('&').Select(x => { var v = x.Split('='); return new { Key = v[0], Value = v[1] }; }).ToDictionary(x => x.Key, x => x.Value);
        }

        public static async Task<string> ExecuteIntercaption(string input)
        {
            var tensor = await PrepareImageTensor(input);
            Console.WriteLine($"Image tensor prepared");
            tensor = CreateTensorFromImageFile(tensor);
            Console.WriteLine($"Image processed");
            return ExecuteModel(tensor, "input", "output", "interception");
        }

        private static async Task<TFTensor> PrepareImageTensor(string input)
        {
            var contents = await PrepareImage(input);
            var tensor = TFTensor.CreateString(contents);
            return tensor;
        }

        private static string ExecuteModel(TFTensor tensor, string inputName, string outputName, string modelKey)
        {
            ModelFiles(modelsDir);
            var graph = PrepareGraph(modelKey);
            Console.WriteLine($"Graph {modelKey} prepared");
            using(var session = new TFSession(graph))
            {
                var runner = session.GetRunner();
                runner.AddInput(graph[inputName][0], tensor).Fetch(graph[outputName][0]);
                var output = runner.Run();
                Console.WriteLine($"Classification finished");
                return ReadResult(output, modelKey);
            }
        }

        private static async Task<byte[]> PrepareImage(string input)
        {
            byte[] contents;
            using(HttpClient client = new HttpClient())
            {
                contents = await client.GetByteArrayAsync(input);
            }

            return contents;
        }

        private static string ReadResult(TFTensor[] output, string modelKey)
        {
            var result = output[0];
            var rshape = result.Shape;
            if (result.NumDims != 2 || rshape[0] != 1)
            {
                var shape = "";
                foreach (var d in rshape)
                {
                    shape += $"{d} ";
                }
                shape = shape.Trim();
                Console.WriteLine($"Error: expected to produce a [1 N] shaped tensor where N is the number of labels, instead it produced one with shape [{shape}]");
                Environment.Exit(1);
            }
            // You can get the data in two ways, as a multi-dimensional array, or arrays of arrays, 
            // code can be nicer to read with one or the other, pick it based on how you want to process
            // it
            bool jagged = true;
            var bestIdx = 0;
            float best = 0;
            if (jagged)
            {
                var probabilities = ((float[][]) result.GetValue(jagged: true)) [0];
                for (int i = 0; i < probabilities.Length; i++)
                {
                    if (probabilities[i] > best)
                    {
                        bestIdx = i;
                        best = probabilities[i];
                    }
                }
            }
            else
            {
                var val = (float[, ]) result.GetValue(jagged: false);
                // Result is [1,N], flatten array
                for (int i = 0; i < val.GetLength(1); i++)
                {
                    if (val[0, i] > best)
                    {
                        bestIdx = i;
                        best = val[0, i];
                    }
                }
            }
            var res = $"Image best match: [{bestIdx}] {best * 100.0}% {PrepareLabels(modelKey)[bestIdx]}";
            Console.WriteLine(res);
            return res;
        }

        static TFTensor CreateTensorFromImageFile(TFTensor tensor)
        {
            TFOutput input, output;
            // Construct a graph to normalize the image
            using(var graph = ConstructGraphToNormalizeImage(out input, out output))
            {
                // Execute that graph to normalize this one image
                using(var session = new TFSession(graph))
                {
                    var normalized = session.Run(
                        inputs: new [] { input },
                        inputValues: new [] { tensor },
                        outputs: new [] { output });
                    return normalized[0];
                }
            }
        }

        static TFGraph ConstructGraphToNormalizeImage(out TFOutput input, out TFOutput output)
        {
            // Some constants specific to the pre-trained model at:
            // https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip
            //
            // - The model was trained after with images scaled to 224x224 pixels.
            // - The colors, represented as R, G, B in 1-byte each were converted to
            //   float using (value - Mean)/Scale.
            const int W = 224;
            const int H = 224;
            const float Mean = 117;
            const float Scale = 1;
            var graph = new TFGraph();
            input = graph.Placeholder(TFDataType.String);
            output = graph.Div(
                x: graph.Sub(
                    x : graph.ResizeBilinear(
                        images: graph.ExpandDims(
                            input : graph.Cast(
                                graph.DecodeJpeg(contents: input, channels: 3), DstT: TFDataType.Float),
                            dim: graph.Const(0, "make_batch")),
                        size : graph.Const(new int[] { W, H }, "size")),
                    y : graph.Const(Mean, "mean")),
                y : graph.Const(Scale, "scale"));

            return graph;
        }

        private static TFGraph PrepareGraph(string modelKey)
        {
            TFGraph graph = new TFGraph();
            if (!graphCache.TryGetValue(modelKey, out graph))
            {
                graph = new TFGraph();
                var model = File.ReadAllBytes($"{modelsDir}/tensorflow_inception_graph.pb");
                graph.Import(model, "");
                graphCache[modelKey] = graph;
            }

            return graph;
        }

        private static string[] PrepareLabels(string modelKey)
        {
            string[] labels;
            if (!labelsCache.TryGetValue(modelKey, out labels))
            {
                labels = File.ReadAllLines($"{modelsDir}/imagenet_comp_graph_label_strings.txt");
                labelsCache[modelKey] = labels;
            }

            return labels;
        }

        private static void ModelFiles(string dir)
        {
            string url = "https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip";

            var modelFile = Path.Combine(dir, "tensorflow_inception_graph.pb");
            var labelsFile = Path.Combine(dir, "imagenet_comp_graph_label_strings.txt");
            var zipfile = Path.Combine(dir, "inception5h.zip");

            if (File.Exists(modelFile) && File.Exists(labelsFile))
                return;

            Directory.CreateDirectory(dir);
            var wc = new WebClient();
            wc.DownloadFile(url, zipfile);
            ZipFile.ExtractToDirectory(zipfile, dir);
            File.Delete(zipfile);
        }
    }
}

TensorflowServerlessAzureLib.csproj

<Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
    <TargetFramework>netstandard2.0</TargetFramework>
    <LangVersion>7.1</LangVersion>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="TensorFlowSharp" Version="1.9.0" />
  </ItemGroup>

</Project>