Sanity Hotspot Generation based on face

sanity-face-hotspots.ts

import {
  RekognitionClient,
  DetectFacesCommand,
  DetectFacesCommandInput,
} from "@aws-sdk/client-rekognition";
import imageUrlBuilder from "@sanity/image-url";
import client, { SanityImageAssetDocument, Transaction } from "@sanity/client";
import { default as nodeFetch } from "node-fetch";

const SANITY_DATASET = "<my project dataset>";
const SANITY_PROJECT_ID = "<my projectId>";
const sanityImageBuilder = imageUrlBuilder({
  dataset: SANITY_DATASET,
  projectId: SANITY_PROJECT_ID,
});

const SANITY_TOKEN = "*****";
export const sanityClient = client({
  projectId: SANITY_PROJECT_ID,
  dataset: SANITY_DATASET,
  token: SANITY_TOKEN,
  useCdn: false,
  apiVersion: "2021-03-25",
});

const AWS_KEY = "*****";
const AWS_SECRET = "**********";
const awsClient = new RekognitionClient({
  region: "us-west-2",
  credentials: {
    accessKeyId: AWS_KEY,
    secretAccessKey: AWS_SECRET,
  },
});
type UserRecord = {
  _id: string;
  firstName: string;
  image: { asset: SanityImageAssetDocument };
};

/*
  Finds documents of type "User" without a hotspot set.
*/
const lookupUsersWithoutHotspot = async (start: number, end: number) => {
  return await sanityClient.fetch(`
    *[_type=="user" && !defined(image.hotspot)] {
      _id,
      firstName,
      image {
        asset->
      }
    }[${start}...${end}]
  `);
};
const fetchImage = async (imageUrl: string): Promise<Buffer> => {
  const response = await nodeFetch(imageUrl);
  return await response.buffer();
};

/*
  Fetches user's image, and passes to Amazon Rekognition to detect face.  If face is detected, sanity hotspot is set with those coordinates.
*/
const findHotspot = async (transaction: Transaction, user: UserRecord) => {
  const imageUrl = user.image.asset?.url;
  if (imageUrl) {
    const imageBufferBytes = await fetchImage(imageUrl);
    const params: DetectFacesCommandInput = {
      Image: {
        Bytes: imageBufferBytes,
      },
    };

    try {
      const result = await awsClient.send(new DetectFacesCommand(params));
      if (result.$metadata.httpStatusCode === 200) {
        const boundingBox = result.FaceDetails?.[0].BoundingBox;
        if (
          boundingBox?.Height &&
          boundingBox.Width &&
          boundingBox.Left &&
          boundingBox.Top
        ) {
          // Amazon identifies the face to JUST the face.
          // If you want to expand the selection a bit, you may
          // increase these values:
          const widthFactor = 1;
          const heightFactor = 1;
          const width = boundingBox.Width * widthFactor;
          const height = boundingBox.Height * heightFactor;
          console.log(`Updating ${user._id} with hotspot...`);
          const patch = sanityClient.patch(user._id).set({
            image: {
              asset: {
                _type: "reference",
                _ref: user.image.asset._id,
              },
              hotspot: {
                height,
                width,
                x: boundingBox.Left + width / (2 * widthFactor),
                y: boundingBox.Top + height / (2 * heightFactor),
              },
            },
          });
          transaction.patch(patch);
        }
      }
    } catch (e) {
      console.log(`Error occurred with user ${user._id}`);
      console.warn(e);
    }
  }
};
export const findFaceHotspots = async () => {
  let transaction;

  const PAGE_SIZE = 40;
  let start = 0;
  let end = PAGE_SIZE;

  const count = await sanityClient.fetch(
    `count(*[_type=="user" && !defined(image.hotspot)])`
  );

  while (true) {
    transaction = sanityClient.transaction();
    const users = await lookupUsersWithoutHotspot(start, end);
    for (const i in users) {
      const user = users[i];
      await findHotspot(transaction, user);
      start += 1;
    }
    transaction.commit();
    if (start >= count) {
      break;
    }
    end = start + PAGE_SIZE;
  }
};

findFaceHotspots();

/*
  For debugging, you can use this to log the images for a given user:
*/
const logImageForUser = async (_id: string) => {
  const items = await sanityClient.fetch(
    `*[_type=="user" && _id=="${_id}"]{ _id, image{ hotspot{...}, asset-> }}`
  );
  const user = items[1];
  const square = sanityImageBuilder
    .image(user.image)
    .height(400)
    .width(400)
    .url();
  const rectangle = sanityImageBuilder
    .image(user.image)
    .height(225)
    .width(400)
    .url();

  const pano = sanityImageBuilder
    .image(user.image)
    .height(200)
    .width(600)
    .url();

  console.log("Square - ", square);
  console.log("16:9 - ", rectangle);
  console.log("Pano -", pano);
};