Drupal 8 modal link in links.aciton.yml

MODULENAME.DIRECT_ROUTE:
  route_name: MODULENAME.DIRECT_ROUTE
  title: 'In modal action link'
  appears_on:
    - MODULENAME.ROUTE
  options:
    attributes:
      class:
        - 'use-ajax'
      data-dialog-type: 'modal'
      data-dialog-options: '{"height":"auto","width":"auto"}'

Open in popup existing action link:

/**
 * Implements HOOK_menu_local_actions_alter().
 */
function MODULE_NAME_menu_local_actions_alter(&$local_actions) {
  if (isset($local_actions['entity.commerce_payment_method.add_form'])) {
    $local_actions['entity.commerce_payment_method.add_form']['options']['attributes']['class'][] = 'use-ajax';
    $local_actions['entity.commerce_payment_method.add_form']['options']['attributes']['data-dialog-type'] = 'modal';
    $local_actions['entity.commerce_payment_method.add_form']['options']['attributes']['data-dialog-options'] = '{"height":"auto","width":"auto"}';
  }
}