Field formatter for entity reference label to set link to custom controller or views. Used with terms, !!! not tested for another entities.

EntityReferenceLabelCustomPathFormatter.php

<?php

namespace Drupal\tools_ext\Plugin\Field\FieldFormatter;

use Drupal\Core\Field\FieldItemListInterface;
use Drupal\Core\Form\FormStateInterface;
use Drupal\Core\Field\Plugin\Field\FieldFormatter\EntityReferenceLabelFormatter;
use Drupal\Core\Url;
use Drupal\Component\Serialization\Json;

/**
 * Plugin implementation of the 'entity reference label custom path' formatter.
 *
 * @FieldFormatter(
 *   id = "entity_reference_label_custom_path",
 *   label = @Translation("Label custom path"),
 *   description = @Translation("Display the label (custom path) of the referenced entities."),
 *   field_types = {
 *     "entity_reference"
 *   }
 * )
 */
class EntityReferenceLabelCustomPathFormatter extends EntityReferenceLabelFormatter {

  /**
   * {@inheritdoc}
   */
  public static function defaultSettings() {
    return array(
      'custom_path' => '/custom-path',
      'add_entity_id' => TRUE,
      'open_in_dialog' => TRUE,
      'query_var_name' => '',
    ) + parent::defaultSettings();
  }

  /**
   * {@inheritdoc}
   */
  public function settingsForm(array $form, FormStateInterface $form_state) {    
    $elements['custom_path'] = array(
      '#title' => t('Custom path uri'),
      '#type' => 'textfield',
      '#default_value' => $this->getSetting('custom_path'),
      '#required' => TRUE
    );
    $elements['add_entity_id'] = [
      '#title' => t('Add entity_id as next parameter'),
      '#type' => 'checkbox',
      '#default_value' => $this->getSetting('add_entity_id'),
    ];
    $elements['open_in_dialog'] = [
      '#title' => t('Open link in dialog'),
      '#type' => 'checkbox',
      '#default_value' => $this->getSetting('open_in_dialog'),
    ];
    
    $elements['query_var_name'] = array(
      '#title' => t('Uri query variable name'),
      '#type' => 'textfield',
      '#default_value' => $this->getSetting('query_var_name'),
      '#required' => FALSE
    );

    return $elements;
  }

  /**
   * {@inheritdoc}
   */
  public function settingsSummary() {
    $summary = array();
    $summary[] = $this->getSetting('custom_path') ? t('Custom path uri: '.$this->getSetting('custom_path')) : t('Custom path uri: None');
    $summary[] = $this->getSetting('add_entity_id') ? t('Add entity_id as next parameter: '.$this->getSetting('add_entity_id')) : t('Add entity_id: None');
    $summary[] = $this->getSetting('open_in_dialog') ? t('Open in dialog: '.$this->getSetting('open_in_dialog')) : t('Open in dialog: None');
    $summary[] = $this->getSetting('query_var_name') ? t('Query variable name: '.$this->getSetting('query_var_name')) : t('Query variable: None');
    return $summary;
  }

  /**
   * {@inheritdoc}
   */
  public function viewElements(FieldItemListInterface $items, $langcode) {
    $elements = array();
    
    foreach ($this->getEntitiesToView($items, $langcode) as $delta => $entity) {
      $label = $entity->label();
      $custom_path = $this->getSettings('custom_path');
      $add_entity_id = $this->getSetting('add_entity_id');
      $open_in_dialog = $this->getSetting('open_in_dialog');
      $query_var_name = $this->getSettings('query_var_name');
      $options = '';
      // 
      if (!$entity->isNew()) {
        
        if ($add_entity_id) {
          $custom_path['custom_path'] .= '/'.$entity->id();
        }
        
        if ($query_var_name['query_var_name']) {
          $options = '?' . $query_var_name['query_var_name'] . '=' . $entity->id();
        }
        // this will result in /custom_path?query_var_name=123
        $uri = Url::fromUserInput($custom_path['custom_path'] . $options);
      }

      if (isset($uri) && !$entity->isNew()) {
        $elements[$delta] = [
          '#type' => 'link',
          '#title' => $label,
          '#url' => $uri,
        ];

        if (!empty($items[$delta]->_attributes)) {
          $elements[$delta]['#options'] += array('attributes' => array());
          $elements[$delta]['#options']['attributes'] += $items[$delta]->_attributes;
          // 
          unset($items[$delta]->_attributes);
        }
        
        if ($open_in_dialog) {
          $elements[$delta]['#options']['attributes']['class'] = 'use-ajax';
          $elements[$delta]['#options']['attributes']['data-dialog-type'] = 'modal';
          $elements[$delta]['#options']['attributes']['data-dialog-options'] = Json::encode([
            'width' => 700,
          ]);
        }
      }
      else {
        $elements[$delta] = array('#plain_text' => $label);
      }
      $elements[$delta]['#cache']['tags'] = $entity->getCacheTags();
    }

    return $elements;
  }

}