Smart avatar cropping in rails: high-performance face detection with YuNet, ONNX runtime and ActiveStorage

application_helper.rb

module ApplicationHelper
  def face_cropped_picture_tag(record, attachment_name, size: 300, html_classes: nil, alt: nil, **options)
    # Ensure the attachment actually exists before trying to render it
    return nil unless record.public_send(attachment_name).attached?

    variant_method = "#{attachment_name}_headshot"
    variant_retina_size = size * 2

    # Generate 1x variants (Standard)
    avif_1x = record.public_send(variant_method, format: :avif, size: size)
    webp_1x = record.public_send(variant_method, format: :webp, size: size)
    jpeg_1x = record.public_send(variant_method, format: :jpeg, size: size)

    # Generate 2x variants (Retina)
    avif_2x = record.public_send(variant_method, format: :avif, size: variant_retina_size)
    webp_2x = record.public_send(variant_method, format: :webp, size: variant_retina_size)
    jpeg_2x = record.public_send(variant_method, format: :jpeg, size: variant_retina_size)

    # Set default HTML attributes for the img tag
    html_options = options.dup
    html_options[:class] = html_classes

    html_options[:width] ||= size
    html_options[:height] ||= size
    html_options[:loading] ||= 'lazy'
    html_options[:alt] = alt.presence ||
                         html_options[:alt].presence ||
                         "#{record.class.name} #{attachment_name.to_s.humanize}"

    content_tag(:picture) do
      safe_join([
                  tag.source(
                    srcset: "#{url_for(avif_1x)} 1x, #{url_for(avif_2x)} 2x",
                    type: 'image/avif'
                  ),
                  tag.source(
                    srcset: "#{url_for(webp_1x)} 1x, #{url_for(webp_2x)} 2x",
                    type: 'image/webp'
                  ),
                  image_tag(
                    url_for(jpeg_1x),
                    **html_options,
                    srcset: "#{url_for(jpeg_1x)} 1x, #{url_for(jpeg_2x)} 2x"
                  )
                ])
    end
  end
end

assets_face_crop_box.rb

# frozen_string_literal: true

class AssetsFaceCropBox
  # Model & Threshold Configs
  FACE_THRESHOLD = 0.6
  YUNNET_IMAGE_PADDING = 32
  FACE_PADDING_PERCENTAGE = 0.3

  # Tensor Dimensions
  BATCH_SIZE = 1
  RGB_CHANNELS = 3

  # Minimum size to run inference (saves CPU on tiny placeholder images)
  MIN_IMAGE_DIMENSION = 32
  # Prevent memory exhaustion (OOM) on massive images
  MAX_IMAGE_DIMENSION = 4096

  # YuNet Architecture Settings
  MODEL_STRIDES = [8, 16, 32].freeze
  PRIMARY_VALUE_INDEX = 0

  # Normalization & Math Constraints
  MIN_PROBABILITY = 0.0
  MAX_PROBABILITY = 1.0
  PADDING_BOTH_SIDES_FACTOR = 2 # Applies padding to both sides (e.g., left + right)
  HALF_FACTOR = 2.0             # Used for centering offsets (radius / half-width)

  @model_mutex = Mutex.new

  class << self
    def call(vips_image)
      # Early return for images that are too small to contain a face
      return nil if vips_image.width < MIN_IMAGE_DIMENSION || vips_image.height < MIN_IMAGE_DIMENSION

      # Convert grayscale images (1 or 2 bands) to standard sRGB (which has 3 bands)
      vips_image = vips_image.colourspace(:srgb) if vips_image.bands < 3
      # Drop the alpha channel if the image happens to be RGBA
      vips_image = vips_image.extract_band(0, n: 3) if vips_image.bands > 3

      # Dynamic Scaling for ML Inference
      scale = 1.0
      ml_image = vips_image

      if vips_image.width > MAX_IMAGE_DIMENSION || vips_image.height > MAX_IMAGE_DIMENSION
        # Calculate how much to shrink it (e.g., scale = 0.5 for an 8192px image)
        scale = MAX_IMAGE_DIMENSION.to_f / [vips_image.width, vips_image.height].max
        ml_image = vips_image.resize(scale)
      end

      padded_image = model_padded_image(ml_image)

      # Convert to NCHW Tensor
      nchw_tensor = nchw_tensor(padded_image)

      # Inference
      input_name = onnx_model.inputs[0][:name]
      result = onnx_model.predict({ input_name => nchw_tensor })

      # GC to cleanup
      nchw_tensor = nil # rubocop:disable Lint/UselessAssignment

      # Find Best Face
      best_score, best_bbox, best_stride, best_grid_x, best_grid_y = find_best_face(padded_image, result)

      # Help GC
      padded_image = nil # rubocop:disable Lint/UselessAssignment
      result = nil # rubocop:disable Lint/UselessAssignment

      # Return nil if no face is found (confidence threshold)
      return nil if best_score < FACE_THRESHOLD

      crop_rect = calculate_face_box(vips_image: ml_image, best_bbox:, best_stride:, best_grid_x:, best_grid_y:)

      # Map coordinates back to the original high-resolution image space
      if scale < 1.0
        inverse_scale = 1.0 / scale
        crop_rect = crop_rect.map { |val| (val * inverse_scale).round }
      end

      crop_rect
    end

    private

    def model_padded_image(img)
      pad_w = (YUNNET_IMAGE_PADDING - (img.width % YUNNET_IMAGE_PADDING)) % YUNNET_IMAGE_PADDING
      pad_h = (YUNNET_IMAGE_PADDING - (img.height % YUNNET_IMAGE_PADDING)) % YUNNET_IMAGE_PADDING
      img.embed(
        0,
        0,
        img.width + pad_w,
        img.height + pad_h,
        extend: :black
      )
    end

    def nchw_tensor(img)
      bytes = img.write_to_memory
      shape = [img.height, img.width, img.bands]

      narray_img = Numo::UInt8.from_binary(bytes, shape)

      # Free the raw string bytes immediately
      bytes = nil # rubocop:disable Lint/UselessAssignment

      narray_img.cast_to(Numo::SFloat).transpose(
        2, 0, 1
      ).reshape(
        BATCH_SIZE, RGB_CHANNELS, img.height, img.width
      )
    end

    def find_best_face(padded_image, result)
      best_score = -1.0
      best_bbox = nil
      best_stride = nil
      best_grid_x = nil
      best_grid_y = nil

      MODEL_STRIDES.each do |stride|
        cls_data = result["cls_#{stride}"].first
        obj_data = result["obj_#{stride}"].first
        bbox_data = result["bbox_#{stride}"].first

        feature_w = (padded_image.width.to_f / stride).ceil

        cls_data.each_with_index do |cls_arr, idx|
          cls = cls_arr[PRIMARY_VALUE_INDEX].clamp(MIN_PROBABILITY, MAX_PROBABILITY)
          obj = obj_data[idx][PRIMARY_VALUE_INDEX].clamp(MIN_PROBABILITY, MAX_PROBABILITY)
          score = Math.sqrt(cls * obj)

          next unless score > best_score

          best_score = score
          best_bbox = bbox_data[idx]
          best_stride = stride
          best_grid_y = idx / feature_w
          best_grid_x = idx % feature_w
        end
      end

      [
        best_score,
        best_bbox,
        best_stride,
        best_grid_x,
        best_grid_y
      ]
    end

    def calculate_face_box(vips_image:, best_bbox:, best_stride:, best_grid_x:, best_grid_y:)
      # Decode Coordinates
      face_cx = (best_grid_x + best_bbox[0]) * best_stride
      face_cy = (best_grid_y + best_bbox[1]) * best_stride
      face_w  = Math.exp(best_bbox[2]) * best_stride
      face_h  = Math.exp(best_bbox[3]) * best_stride

      # Calculate Tight Square Crop Area
      face_max_dim = [face_w, face_h].max
      target_size = face_max_dim * (1.0 + (FACE_PADDING_PERCENTAGE * PADDING_BOTH_SIDES_FACTOR))
      crop_size = [target_size, vips_image.width, vips_image.height].min

      crop_x = face_cx - (crop_size / HALF_FACTOR)
      crop_x = crop_x.clamp(0, vips_image.width - crop_size)

      crop_y = face_cy - (crop_size / HALF_FACTOR)
      crop_y = crop_y.clamp(0, vips_image.height - crop_size)

      # ActiveStorage image_processing expects: [left, top, width, height]
      [crop_x.round, crop_y.round, crop_size.round, crop_size.round]
    end

    # Cache the model at the class level so it only loads into memory once
    def onnx_model
      return @onnx_model if @onnx_model

      @model_mutex.synchronize do
        # Check again inside the lock in case another thread just loaded it
        @onnx_model ||= OnnxRuntime::Model.new(
          Rails.root.join("lib/assets/models/vision/face_detection_yunet_2026may.onnx").to_s
        )
      end
    end
  end
end

face_crop_analyzer.rb

# frozen_string_literal: true

class FaceCropAnalyzer < ActiveStorage::Analyzer::ImageAnalyzer::Vips
  def self.accept?(blob)
    # Ensure it is an image that Vips can process
    return false unless super

    blob.attachments.pluck(:record_type, :name).any? do |record_type, name|
      record_class = record_type.safe_constantize

      record_class.respond_to?(:face_croppable_attachments) &&
        record_class.face_croppable_attachments.include?(name)
    end
  end

  def metadata
    base_metadata = super

    if base_metadata[:face_crop_rect].blank? && base_metadata[:face_detection_run].blank?
      read_image do |image|
        crop_rect = AssetsFaceCropBox.call(image)
        base_metadata[:face_crop_rect] = crop_rect if crop_rect.present?
      rescue StandardError => e
        Sentry.capture_exception(e)
        Rails.logger.error("[FaceCropAnalyzer] Failed to process image: #{e.message}")
      ensure
        # Always mark it as run so we don't infinitely retry failed images
        base_metadata[:face_detection_run] = true
      end
    end

    base_metadata
  end
end

face_croppable_attachments.rb

# frozen_string_literal: true

module FaceCroppableAttachments
  extend ActiveSupport::Concern

  SAVER_OPTIONS = {
    strip: true,
    lossless: false,
    quality: 95
  }.freeze

  included do
    # Creates an inheritable array on the model to track our attachments
    class_attribute :face_croppable_attachments, default: []
  end

  class_methods do
    def generates_face_cropped_headshot(attachment_name)
      self.face_croppable_attachments |= [attachment_name.to_s]

      define_method("#{attachment_name}_headshot") do |format: :webp, size: 300|
        attachment = public_send(attachment_name)
        return nil unless attachment.attached?

        metadata = attachment.blob.metadata

        unless metadata[:face_detection_run]
          return attachment.variant(
            format: format,
            resize_to_fill: [size, size],
            saver: SAVER_OPTIONS
          )
        end

        transformations = {
          format: format,
          saver: SAVER_OPTIONS
        }

        if metadata[:face_crop_rect].present?
          transformations[:crop] = metadata[:face_crop_rect]
          transformations[:resize_to_limit] = [size, size]
        else
          transformations[:resize_to_fill] = [size, size]
        end

        attachment.variant(transformations)
      end
    end
  end
end

Gemfile

# Face detection ONNX model runner
gem 'onnxruntime'

# Fast multidimensional matrix/array calculations
gem 'numo-narray-alt'

https://leopard.in.ua/2026/09/25/smart-avatar-cropping-in-rails - article