OpenCV for UnityとDlib FaceLandmark Detector で異なる画像上の顔同士を入れ替えるサンプル

MultiImageFaceSwap.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using DlibFaceLandmarkDetector;
using FaceSwapperExample;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.FaceSwap;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.ObjdetectModule;
using OpenCVForUnity.UnityUtils;
using UnityEngine;
using Rect = OpenCVForUnity.CoreModule.Rect;

public class MultiImageFaceSwap : MonoBehaviour
{
    public bool useDlibDetecter = true;
    public Texture2D SwapedFace1 { get; private set; }
    public Texture2D SwapedFace2 { get; private set; }
    [SerializeField] private Texture2D _faceTex1;
    [SerializeField] private Texture2D _faceTex2;
    [SerializeField, Range(0f, 1f)] private float _frontalFaceRateLowerLimit = .75f;

    private CascadeClassifier _cascade;
    private FaceLandmarkDetector _faceLandmarkDetector;
    private string _haarcascadeFrontalFaceAltXMLFilePath;
    private string _spHumanFace68DatFilepath;
    private const string HaarcascadeFrontalFaceAltXMLFilePath = "haarcascade_frontalface_alt.xml";
    private const string HumanFace68DatFilePath = "sp_human_face_68.dat";


    private void Start()
    {
        Init();

        // テクスチャ読み込み
        using var face1Mat = new Mat(_faceTex1.height, _faceTex1.width, CvType.CV_8UC4);
        using var face2Mat = new Mat(_faceTex2.height, _faceTex2.width, CvType.CV_8UC4);
        Utils.texture2DToMat(_faceTex1, face1Mat, true);
        Utils.texture2DToMat(_faceTex2, face2Mat, true);

        // サイズを揃える
        Imgproc.resize(face1Mat, face1Mat, new Size(face2Mat.width(), face2Mat.height()));
        // 2枚の画像を1枚に連結する
        using var concatMat = new Mat();
        Core.vconcat(new List<Mat> {face1Mat, face2Mat}, concatMat);

        // 顔の位置を検出
        var detectResult = DetectFace(concatMat);
        
        // 顔の形状を検出
        var landmarkPoints = DetectFaceLandmarkPoints(concatMat, detectResult);
        
        // 正面を向いていない顔を除外
        var frontalFaceChecker = new FrontalFaceChecker(concatMat.width(), concatMat.height());
        FilterNonFrontalFaces(landmarkPoints, frontalFaceChecker, detectResult);

        // 顔を入れ替え
        SwapFaces(landmarkPoints, concatMat);

        // 連結した部分を再分割
        using var trimUpMat = new Mat(concatMat, new Rect(0, 0, concatMat.width(), (int) (concatMat.height() * .5f)));
        using var trimDownMat = new Mat(concatMat, new Rect(0, (int) (concatMat.height() * .5f),
            concatMat.width(), (int) (concatMat.height() * .5f)));
        
        SwapedFace1 = new Texture2D(trimUpMat.cols(), trimUpMat.rows(), TextureFormat.RGBA32, false);
        SwapedFace2 = new Texture2D(trimDownMat.cols(), trimDownMat.rows(), TextureFormat.RGBA32, false);
        Utils.matToTexture2D(trimUpMat, SwapedFace1);
        Utils.matToTexture2D(trimDownMat, SwapedFace2);

        frontalFaceChecker.Dispose();
    }

    private void Init()
    {
        _haarcascadeFrontalFaceAltXMLFilePath =
            OpenCVForUnity.UnityUtils.Utils.getFilePath(HaarcascadeFrontalFaceAltXMLFilePath);
        _spHumanFace68DatFilepath = DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePath(HumanFace68DatFilePath);
        _cascade ??= new CascadeClassifier(_haarcascadeFrontalFaceAltXMLFilePath);
        _faceLandmarkDetector ??= new FaceLandmarkDetector(_spHumanFace68DatFilepath);
    }

    private List<Rect> DetectFace(Mat srcMat)
    {
        var detectResult = new List<Rect>();
        if (useDlibDetecter)
        {
            DetectByDlib(srcMat, detectResult);
        }
        else
        {
            DetectByCascade(srcMat, out detectResult);
        }

        return detectResult;
    }

    private void DetectByDlib(Mat srcMat, List<Rect> detectResult)
    {
        OpenCVForUnityUtils.SetImage(_faceLandmarkDetector, srcMat);
        var result = _faceLandmarkDetector.Detect();
        detectResult.AddRange(result.Select(unityRect =>
            new Rect((int) unityRect.x, (int) unityRect.y, (int) unityRect.width, (int) unityRect.height)));
    }

    private void DetectByCascade(Mat src, out List<Rect> detectResult)
    {
        // convert image to greyscale.
        var gray = new Mat();
        Imgproc.cvtColor(src, gray, Imgproc.COLOR_RGBA2GRAY);

        var faces = new MatOfRect();
        Imgproc.equalizeHist(gray, gray);
        _cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE,
            new Size(gray.cols() * 0.05, gray.cols() * 0.05),
            new Size());
        //Debug.Log ("faces " + faces.dump ());

        detectResult = faces.toList();

        // correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
        foreach (var r in detectResult)
        {
            r.y += (int) (r.height * 0.1f);
        }

        gray.Dispose();
    }

    private List<List<Vector2>> DetectFaceLandmarkPoints(Mat rgbaMat, IEnumerable<Rect> detectResult)
    {
        OpenCVForUnityUtils.SetImage(_faceLandmarkDetector, rgbaMat);
        var landmarkPoints = new List<List<Vector2>>();
        foreach (var rect in detectResult.Select(openCvRect =>
            new UnityEngine.Rect(openCvRect.x, openCvRect.y, openCvRect.width, openCvRect.height)))
        {
            //Debug.Log("face : " + rect);
            //OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);

            var points = _faceLandmarkDetector.DetectLandmark(rect);
            //OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
            landmarkPoints.Add(points);
        }

        return landmarkPoints;
    }

    private void FilterNonFrontalFaces(List<List<Vector2>> landmarkPoints, FrontalFaceChecker frontalFaceChecker,
        IList detectResult)
    {
        for (var i = 0; i < landmarkPoints.Count; i++)
        {
            if (!(frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < _frontalFaceRateLowerLimit)) continue;

            detectResult.RemoveAt(i);
            landmarkPoints.RemoveAt(i);
            i--;
        }
    }

    private int[] SwapFaces(IReadOnlyList<List<Vector2>> landmarkPoints, Mat rgbaMat)
    {
        var faceNums = new int[landmarkPoints.Count];
        for (var i = 0; i < faceNums.Length; i++)
        {
            faceNums[i] = i;
        }

        faceNums = faceNums.OrderBy(i => Guid.NewGuid()).ToArray();
        if (landmarkPoints.Count >= 2)
        {
            var faceSwapper = new DlibFaceSwapper
            {
                useSeamlessCloneForPasteFaces = true,
                isShowingDebugFacePoints = false
            };

            for (var i = 0; i < faceNums.Length - 1; i += 2)
            {
                var ann = faceNums[i];
                var bob = faceNums[i + 1];

                faceSwapper.SwapFaces(rgbaMat, landmarkPoints[ann], landmarkPoints[bob], 1);
            }

            faceSwapper.Dispose();
        }

        return faceNums;
    }
}