Filters and example code for Yoast SEO robots or WP robots.txt

wordpress_robots_custom.php

<?php
/********* DO NOT COPY THE PARTS ABOVE THIS LINE *********/

/*
 * Replace Disallow with Allow Generated Robots.txt
 * Credit: Unknown
 * Last Tested: June 09 2020 using WordPress 5.4.1
 */

add_filter('robots_txt','custom_robots');

function custom_robots($output) {
     $public = get_option( 'blog_public' );
     if ( '0' != $public )
     return str_replace('Disallow','Allow',$output);
}

yoast_seo_robots_callback_hooks.php

<?php
/********* DO NOT COPY THE PARTS ABOVE THIS LINE *********/

/* Check which functions have been used to hook into `wp_robots`
 * Credit: Yoast team
 * Last Tested: Unknown 
 */

\add_filter( 'wp_robots', 'list_hooks' , \PHP_INT_MAX);

function list_hooks( $robots ) {
	global $wp_filter;

	echo "<!-- This is a list of callback functions hooked into the 'wp_robots' filter:";
	echo json_encode($wp_filter['wp_robots'], JSON_PRETTY_PRINT);
	echo "-->";

	return $robots;
}

yoast_seo_robots_change_author_archive.php

/*
 * Change meta robots to index all author archive pages
 * Credit: Yoast development team
 * Last Tested: Jul 20 2023 using Yoast SEO 20.11 on WordPress 6.2.2
 */

add_filter( 'wpseo_robots', 'yoast_seo_robots_change_author_archive' );

function yoast_seo_robots_change_author_archive( $robots ) {
  if ( is_author() ) {
    return 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1';
  } else {
    return $robots;
  }
}

yoast_seo_robots_remove.php

<?php
/********* DO NOT COPY THE PARTS ABOVE THIS LINE *********/

/*
 * Remove robots meta tags from Yoast SEO
 * Credit: Yoast development team
 * Last Tested: May 22 2024 using Yoast SEO 22.7 on WordPress 6.5.3
 */

add_filter( 'wpseo_robots', '__return_empty_array' ); // Removes Yoast ouptut and returns to WP core output
add_filter( 'wp_robots', '__return_false' ); // Disable core output. Use with above to disable completely.

// Older versions
add_filter( 'wpseo_robots', '__return_false' ); // Yoast SEO 19.8 or older
add_filter( 'wpseo_googlebot', '__return_false' ); // Yoast SEO 14.x - Deprecated
add_filter( 'wpseo_bingbot', '__return_false' ); // Yoast SEO 14.x - Deprecated

yoast_seo_robots_remove_search.php

<?php
/********* DO NOT COPY THE PARTS ABOVE THIS LINE *********/

/*
 * Change meta robots using Yoast SEO
 * Credit: Yoast development team
 * Last Tested: Nov 1 2021 using Yoast SEO 17.4 on WordPress 5.8.1
 */

add_filter( 'wpseo_robots', 'yoast_seo_robots_remove_search' );

function yoast_seo_robots_remove_search( $robots ) {
  if ( is_search() ) {
    return array();
  } else {
    return $robots;
  }
}

yoast_seo_robots_remove_single.php

<?php
/********* DO NOT COPY THE PARTS ABOVE THIS LINE *********/

/*
 * Change meta robots using Yoast SEO
 * Credit: Yoast development team
 * Last Tested: Dec 12 2017 using Yoast SEO 9.2.1 on WordPress 5.0
 *********
 * DIFFERENT POST TYPES
 * Post: Change 123456 to the post ID
 * Page: Change is_single to is_page and 123456 to the page ID
 * Custom Post Type: Change is_single to is_singular and 123456 to the 'post_type_slug'
    Example: is_singular( 'cpt_slug' )
 *********
 * MULTIPLE ITEMS
 * Multiple of the same type can use an array.
    Example: is_single( array( 123456, 234567, 345678 ) )
 * Multiple of different types can repeat the if statement
 *********
 * The return empty array removes the robots tag on the page
 * Or you can return index/noindex follow/nofollow like
 * return 'noindex, follow';
 * Or
 * return 'noindex, nofollow';
 */

add_filter( 'wpseo_robots', 'yoast_seo_robots_remove_single' );

function yoast_seo_robots_remove_single( $robots ) {
  if ( is_single ( 123456 ) ) {
    return array();
  } else {
    return $robots;
  }
}

@LoganStrike Thanks for bringing this up again. I updated the code to remove the robots output. The first removes the Yoast changes and the second one removes the WordPress robots output. Use together to remove the robots meta tag completely.

Could you test the new code and let me know how it works in a real-world environment, preferably a non-production site?

@amboutwe The updated code works in my tests. May I recommend also updating yoast_seo_robots_remove_search.php and yoast_seo_robots_remove_single.php?

@LoganStrike Done!