TestServiceProvider.php

<?php

namespace Drupal\my_custom;

use Drupal\my_custom\Normalizer\FileEntityNormalizer;
use Drupal\Core\DependencyInjection\ContainerBuilder;
use Drupal\Core\DependencyInjection\ServiceProviderBase;
use Symfony\Component\DependencyInjection\Definition;
use Symfony\Component\DependencyInjection\Reference;

/**
 * Remove file normalizers that are causing problems.
 */
class TestServiceProvider extends ServiceProviderBase {

  /**
   * {@inheritdoc}
   */
  public function alter(ContainerBuilder $container) {

    // Make sure definition exists before we replace. It might not in certain
    // testing conditions.
    if ($container->has('serializer.normalizer.entity.file_entity')) {
      // Replace file normalizer with one that doesn't include actual files.
      $service_definition = new Definition(FileEntityNormalizer::class, [
        new Reference('hal.link_manager'),
        new Reference('entity.manager'),
        new Reference('module_handler'),
      ]);
      // The priority must be higher than that of
      // serializer.normalizer.file_entity.hal in hal.services.yml.
      $service_definition->addTag('normalizer', ['priority' => 30]);
      $container->setDefinition('serializer.normalizer.entity.file_entity', $service_definition);
    }
  }

}

There are two d9 upgrade failures here. I don't know what of them are testable its an interesting edge case for upgrade failures.

1. entity.manager deprecated then removed.
2. argument list doesn't match the constructor argument list in d9.