do.py

#!/usr/bin/python3
import os
import base64
from flask import Flask, request, redirect, url_for, send_from_directory, render_template, flash
#from werkzeug.utils import secure_filename

from collections import Counter

import sys

import uuid

# process.py
import numpy as np
from medium_facenet_tutorial.align_dlib import AlignDlib
import cv2

# train_classifier.py
import tensorflow as tf
from sklearn.svm import SVC
from tensorflow.python.platform import gfile
from medium_facenet_tutorial.lfw_input import filter_dataset, split_dataset, get_dataset
import medium_facenet_tutorial.lfw_input
from medium_facenet_tutorial.train_classifier import _load_images_and_labels, _load_model, _create_embeddings, _evaluate_classifier
from medium_facenet_tutorial import train_classifier

import pickle

# curl -F "file=@image.jpg" http://localhost:8080/


CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
UPLOAD_FOLDER = CURRENT_DIR + '/uploadedimages/auth/'

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['CURRENT_DIR'] = sys.path[0]
align_dlib = AlignDlib(CURRENT_DIR + '/medium_facenet_tutorial/' + 'shape_predictor_68_face_landmarks.dat')

# def allowed_file(filename):
#     return '.' in filename and \
#         filename.rsplit('.',1)[1].lower() in ALLOWED_EXTENSION

@app.route('/',methods=['POST','GET'])
def upload():
    
    if request.method == 'POST':

        if 'imageToForm1' not in request.values or \
           'imageToForm2' not in request.values or \
           'imageToForm3' not in request.values or \
           'imageToForm4' not in request.values or \
           'imageToForm5' not in request.values or \
           'imageToForm6' not in request.values or \
           'imageToForm7' not in request.values or \
           'imageToForm8' not in request.values or \
           'imageToForm9' not in request.values or \
           'imageToForm10' not in request.values:
            for x in request.values.keys(): print(x)
            return "NO FILES"

        for sendimage in ['imageToForm' + str(x) for x in range(1,11)]:
            image_array = np.fromstring(base64.b64decode(request.values[sendimage]),
                                        np.uint8)
            image = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
            # cv2.imwrite(app.config['CURRENT_DIR'] + '/' + sendimage + '.jpg', image)
            bb = align_dlib.getLargestFaceBoundingBox(image)
            aligned = align_dlib.align(128,
                                       image,
                                       bb,
                                       landmarkIndices=AlignDlib.INNER_EYES_AND_BOTTOM_LIP)
            cv2.imwrite(os.path.join(app.config['UPLOAD_FOLDER'],
                                     sendimage + '.jpg'),
                        image)
            
        with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess:

            batch_size=12
            num_threads=2
            classifier_output_path = app.config['CURRENT_DIR'] + '/output/classifier.pkl'
            model_path = app.config['CURRENT_DIR'] + '/etc/20170511-185253/20170511-185253.pb'
#            print("modpath", model_path)
            test_set = get_dataset(app.config['CURRENT_DIR'] + '/uploadedimages/')
#            test_set = get_dataset(app.config['CURRENT_DIR'] + '/output/intermediate2/')
            images, labels, class_names = _load_images_and_labels(test_set,image_size=160,batch_size=batch_size,num_threads=num_threads,num_epochs=1)

            _load_model(model_filepath=model_path)

#            print("testset",len(test_set))
            init_op = tf.group(tf.global_variables_initializer(),
                                   tf.local_variables_initializer())
            sess.run(init_op)

            images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
            embedding_layer = tf.get_default_graph().get_tensor_by_name("embeddings:0")
            phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")

            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord, sess=sess)

            emb_array, label_array = _create_embeddings(embedding_layer,
                                                        images,
                                                        labels,
                                                        images_placeholder,
                                                        phase_train_placeholder,
                                                        sess)
#            print("embarr", len(emb_array))
#            print("labarr", len(label_array))
            coord.request_stop()
            coord.join(threads=threads)

            classifier_filename = classifier_output_path

            with open(classifier_filename, 'rb') as f:
                model, class_names = pickle.load(f)
                predictions = model.predict_proba(emb_array, )
                best_class_indices = np.argmax(predictions, axis=1)
                best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]
                print("CN: ", class_names, len(class_names))
                print("PR: ", predictions)
                print("BCN: ", best_class_indices)
                print("BCP: ", best_class_probabilities)
                mvp = max(set(best_class_indices),key=list(best_class_indices).count)
                print("MPV", mvp)
                return class_names[mvp]

    return render_template('upload.html')

@app.route('/uploads/<filename>')
def uploaded_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'],
                               filename)

if __name__ == '__main__':
    app.run(debug=True,host='0.0.0.0',port=8080)

makefile

mnt_dir = /home/sendai/projects/medium-facenet-tutorial
images_dir = data
output_dir = output/intermediate
model_dir = etc
docker_cmd = docker run -v $(mnt_dir):/medium-facenet-tutorial -e PYTHONPATH=$$PYTHONPATH:/medium-facenet-tutorial -it colemurray/medium-facenet-tutorial

fetch:
	curl -O http://vis-www.cs.umass.edu/lfw/lfw.tgz
	tar -xzvf lfw.tgz
	mkdir -p data/
	mv lfw/* data/

# preprocess.py
preprocess:
	$(docker_cmd) python3 /medium-facenet-tutorial/medium_facenet_tutorial/preprocess.py \
               --input-dir /medium-facenet-tutorial/$(images_dir) \
               --output-dir /medium-facenet-tutorial/$(output_dir) \
               --crop-dim 180


model:  
	$(docker_cmd) python3 /medium-facenet-tutorial/medium_facenet_tutorial/download_and_extract_model.py \
               --model-dir /medium-facenet-tutorial/$(model_dir)


# train_classifier.py
# --is-train
train:
	$(docker_cmd) python3 /medium-facenet-tutorial/medium_facenet_tutorial/train_classifier.py \
               --input-dir /medium-facenet-tutorial/output/intermediate \
               --model-path /medium-facenet-tutorial/etc/20170511-185253/20170511-185253.pb \
               --classifier-path /medium-facenet-tutorial/output/classifier.pkl \
               --num-threads 16 \
               --num-epochs 25 \
               --min-num-images-per-class 10 \
               --is-train  

# train_classifier.py
run:
	$(docker_cmd) python3 /medium-facenet-tutorial/medium_facenet_tutorial/train_classifier.py \
               --input-dir /medium-facenet-tutorial/output/intermediate \
               --model-path /medium-facenet-tutorial/etc/20170511-185253/20170511-185253.pb \
               --classifier-path /medium-facenet-tutorial/output/classifier.pkl \
               --num-threads 16 \
               --num-epochs 5 \
               --min-num-images-per-class 10

eval:
	$(docker_cmd) python3 /medium-facenet-tutorial/medium_facenet_tutorial/train_classifier.py \
               --input-dir /medium-facenet-tutorial/output/intermediate2 \
               --model-path /medium-facenet-tutorial/etc/20170511-185253/20170511-185253.pb \
               --classifier-path /medium-facenet-tutorial/output/classifier.pkl \
               --num-threads 16 \
               --num-epochs 5 \
               --min-num-images-per-class 10 \
               --eval-all

train_classifier.py

import argparse
import logging
import os
import pickle
import sys
import time

import numpy as np
import tensorflow as tf
from sklearn.svm import SVC
from tensorflow.python.platform import gfile

from lfw_input import filter_dataset, split_dataset, get_dataset
from medium_facenet_tutorial import lfw_input

logger = logging.getLogger(__name__)


def main(input_directory, model_path, classifier_output_path, batch_size, num_threads, num_epochs,
         min_images_per_labels, split_ratio, is_train=True):
    """
    Loads images from :param input_dir, creates embeddings using a model defined at :param model_path, and trains
     a classifier outputted to :param output_path
     
    :param input_directory: Path to directory containing pre-processed images
    :param model_path: Path to protobuf graph file for facenet model
    :param classifier_output_path: Path to write pickled classifier
    :param batch_size: Batch size to create embeddings
    :param num_threads: Number of threads to utilize for queuing
    :param num_epochs: Number of epochs for each image
    :param min_images_per_labels: Minimum number of images per class
    :param split_ratio: Ratio to split train/test dataset
    :param is_train: bool denoting if training or evaluate
    :param eval_all: bool
    """

    start_time = time.time()
    with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess:

        if eval_all:
            test_set = get_dataset(input_directory)
        else:
            train_set, test_set = _get_test_and_train_set(input_directory,
                                                          min_num_images_per_label=min_images_per_labels,
                                                          split_ratio=split_ratio)
        # 158 158
        print(len(train_set),len(test_set))
        # lfw_input.ImageClass object at 0x7f59ffef6860
        # print(train_set)
        # print(test_set)
        if is_train:
            images, labels, class_names = _load_images_and_labels(train_set,
                                                                  image_size=160,
                                                                  batch_size=batch_size,
                                                                  num_threads=num_threads,
                                                                  num_epochs=num_epochs,
                                                                  random_flip=True,
                                                                  random_brightness=True,
                                                                  random_contrast=True)
        else:
            images, labels, class_names = _load_images_and_labels(test_set,
                                                                  image_size=160,
                                                                  batch_size=batch_size,
                                                                  num_threads=num_threads,
                                                                  num_epochs=1)

        _load_model(model_filepath=model_path)

        init_op = tf.group(tf.global_variables_initializer(),
                           tf.local_variables_initializer())
        sess.run(init_op)

        images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
        embedding_layer = tf.get_default_graph().get_tensor_by_name("embeddings:0")
        phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord, sess=sess)

        emb_array, label_array = _create_embeddings(embedding_layer,
                                                    images,
                                                    labels,
                                                    images_placeholder,
                                                    phase_train_placeholder,
                                                    sess)

        coord.request_stop()
        coord.join(threads=threads)
        logger.info('Created {} embeddings'.format(len(emb_array)))

        classifier_filename = classifier_output_path

        if is_train:
            _train_and_save_classifier(emb_array,
                                       label_array,
                                       class_names,
                                       classifier_filename)
        else:
            _evaluate_classifier(emb_array,
                                 label_array,
                                 classifier_filename)

        logger.info('Completed in {} seconds'.format(time.time() - start_time))


def _get_test_and_train_set(input_dir, min_num_images_per_label, split_ratio=0.7):
    """
    Load train and test dataset. Classes with < :param min_num_images_per_label will be filtered out.
    :param input_dir: 
    :param min_num_images_per_label: 
    :param split_ratio: 
    :return: 
    """
    dataset = get_dataset(input_dir)
    # AJ_Cook, 1 images
    # Aaron_Sorkin, 2 images
    # for d in dataset:
    #     print(str(d),"\n")
    #print(min_num_images_per_label,"\n") # 10
    dataset = filter_dataset(dataset, min_images_per_label=min_num_images_per_label)
    # Kofi_Annan, 32 images
    # for d in dataset:
    #     print(str(d),"\n")
    train_set, test_set = split_dataset(dataset, split_ratio=split_ratio)

    return train_set, test_set

def _load_images_and_labels(dataset, image_size, batch_size, num_threads, num_epochs, random_flip=False,
                            random_brightness=False, random_contrast=False, eval_all=False):
    class_names = [cls.name for cls in dataset]
    image_paths, labels = lfw_input.get_image_paths_and_labels(dataset)
    images, labels = lfw_input.read_data(image_paths,
                                         labels,
                                         image_size,
                                         batch_size,
                                         num_epochs,
                                         num_threads,
                                         shuffle=False,
                                         random_flip=random_flip,
                                         random_brightness=random_brightness,
                                         random_contrast=random_contrast)
    return images, labels, class_names


def _load_model(model_filepath):
    """
    Load frozen protobuf graph
    :param model_filepath: Path to protobuf graph
    :type model_filepath: str
    """
    model_exp = os.path.expanduser(model_filepath)
    if os.path.isfile(model_exp):
        logging.info('Model filename: %s' % model_exp)
        with gfile.FastGFile(model_exp, 'rb') as f:
            graph_def = tf.GraphDef()
            graph_def.ParseFromString(f.read())
            tf.import_graph_def(graph_def, name='')
    else:
        logger.error('Missing model file. Exiting')
        sys.exit(-1)


def _create_embeddings(embedding_layer, images, labels, images_placeholder, phase_train_placeholder, sess):
    """
    Uses model to generate embeddings from :param images.
    :param embedding_layer: 
    :param images: 
    :param labels: 
    :param images_placeholder: 
    :param phase_train_placeholder: 
    :param sess: 
    :return: (tuple): image embeddings and labels
    """
    emb_array = None
    label_array = None
    try:
        i = 0
        while True:
            batch_images, batch_labels = sess.run([images, labels])
            logger.info('Processing iteration {} batch of size: {}'.format(i, len(batch_labels)))
            emb = sess.run(embedding_layer,
                           feed_dict={images_placeholder: batch_images, phase_train_placeholder: False})

            emb_array = np.concatenate([emb_array, emb]) if emb_array is not None else emb
            label_array = np.concatenate([label_array, batch_labels]) if label_array is not None else batch_labels
            i += 1

    except tf.errors.OutOfRangeError:
        pass

    return emb_array, label_array


def _train_and_save_classifier(emb_array, label_array, class_names, classifier_filename_exp):
    logger.info('Training Classifier')
    model = SVC(kernel='linear', probability=True, verbose=False)
    model.fit(emb_array, label_array)

    with open(classifier_filename_exp, 'wb') as outfile:
        pickle.dump((model, class_names), outfile)
    logging.info('Saved classifier model to file "%s"' % classifier_filename_exp)


def _evaluate_classifier(emb_array, label_array, classifier_filename):
    logger.info('Evaluating classifier on {} images'.format(len(emb_array)))
    if not os.path.exists(classifier_filename):
        raise ValueError('Pickled classifier not found, have you trained first?')

    with open(classifier_filename, 'rb') as f:
        model, class_names = pickle.load(f)

        predictions = model.predict_proba(emb_array, )
        best_class_indices = np.argmax(predictions, axis=1)
        best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]

        for i in range(len(best_class_indices)):
            print('%4d  %s: %.3f' % (i, class_names[best_class_indices[i]], best_class_probabilities[i]))

        accuracy = np.mean(np.equal(best_class_indices, label_array))
        print('Accuracy: %.3f' % accuracy)


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    parser = argparse.ArgumentParser(add_help=True)
    parser.add_argument('--model-path', type=str, action='store', dest='model_path',
                        help='Path to model protobuf graph')
    parser.add_argument('--input-dir', type=str, action='store', dest='input_dir',
                        help='Input path of data to train on')
    parser.add_argument('--batch-size', type=int, action='store', dest='batch_size',
                        help='Input path of data to train on', default=128)
    parser.add_argument('--num-threads', type=int, action='store', dest='num_threads', default=16,
                        help='Number of threads to utilize for queue')
    parser.add_argument('--num-epochs', type=int, action='store', dest='num_epochs', default=3,
                        help='Path to output trained classifier model')
    parser.add_argument('--split-ratio', type=float, action='store', dest='split_ratio', default=0.7,
                        help='Ratio to split train/test dataset')
    parser.add_argument('--min-num-images-per-class', type=int, action='store', default=10,
                        dest='min_images_per_class', help='Minimum number of images per class')
    parser.add_argument('--classifier-path', type=str, action='store', dest='classifier_path',
                        help='Path to output trained classifier model')
    parser.add_argument('--is-train', action='store_true', dest='is_train', default=False,
                        help='Flag to determine if train or evaluate')
    parser.add_argument('--eval-all', action='store_true', dest='eval_all', default=False,
                        help='Flag to determine if we use all input to evaluate')

    args = parser.parse_args()

    main(input_directory=args.input_dir, model_path=args.model_path, classifier_output_path=args.classifier_path,
         batch_size=args.batch_size, num_threads=args.num_threads, num_epochs=args.num_epochs,
         min_images_per_labels=args.min_images_per_class, split_ratio=args.split_ratio, is_train=args.is_train, eval_all=args.eval_all)