Example of custom Unity Recorder encoder that outputs HVEC (H.265).

HVECEncoder.cs

using System.Diagnostics;
using Unity.Collections;
using UnityEditor.Media;
using UnityEditor.Recorder.Encoder;

class HVECEncoder : IEncoder
{
    Process process;

    public void OpenStream(IEncoderSettings settings, RecordingContext ctx)
    {
        var hvecEncoderSettings = settings as HVECEncoderSettings;

        process = new Process
        {
            StartInfo = new ProcessStartInfo(hvecEncoderSettings.ffmpegPath)
            {
                Arguments = string.Join(' ',
                    "-y", // Overwrite existing file

                    // Input options:
                    "-f rawvideo",
                    "-framerate", (float)ctx.fps.numerator / ctx.fps.denominator,
                    "-pixel_format rgb24", // Match output of HVECEncoderSettings.GetTextureFormat
                    "-s", $"{ctx.width}x{ctx.height}",
                    "-vcodec rawvideo",
                    "-i -", // Read from stdin

                    // Output options:
                    "-codec:v libx265",
                    "-pix_fmt yuv420p",
                    "-vtag hvc1", // Tell QuickTime that it can play this file
                    ctx.path),
                CreateNoWindow = true,
                RedirectStandardInput = true,
                UseShellExecute = false,
            },
        };

        process.Start();
    }

    public void CloseStream()
    {
        process.StandardInput.Close();
        process.WaitForExit();
        process.Close();
        process.Dispose();
    }

    public void AddVideoFrame(NativeArray<byte> bytes, MediaTime time)
    {
        var stream = process.StandardInput.BaseStream;
        stream.Write(bytes.ToArray(), 0, bytes.Length);
        stream.Flush();
    }

    public void AddAudioFrame(NativeArray<float> bytes)
    {
        // TODO: implement
    }
}

HVECEncoderSettings.cs

using System.Collections.Generic;
using System.ComponentModel;
using UnityEditor.Recorder.Encoder;
using UnityEngine;

[DisplayName("HVEC (H.265) Encoder")]
[EncoderSettings(typeof(HVECEncoder))]
class HVECEncoderSettings : IEncoderSettings
{
    public bool CanCaptureAlpha => false;
    public bool CanCaptureAudio => false;
    public string Extension => "mp4";
    public TextureFormat GetTextureFormat(bool inputContainsAlpha) => TextureFormat.RGB24;
    public bool SupportsCurrentPlatform() => true;
    public void ValidateRecording(RecordingContext ctx, List<string> errors, List<string> warnings) {}

    public string ffmpegPath = "/usr/local/bin/ffmpeg";
}