Add a meta box to WordPress edit posts and list only authors

author-filter-meta-box.php

<?php
/**
 * Plugin Name: Author Meta Box only with authors
 * Plugin URI:  http://wordpress.stackexchange.com/questions/60429/stop-loading-collaborators-users-on-add-new-post-or-page
 * Description: 
 * Author:      Frank Bültge
 * Author URI:  http://bueltge.de
 * License:     GPLv3
 */
add_action( 'admin_menu', 'fb_remove_author_meta_boxes' );
function fb_remove_author_meta_boxes() {
	
	remove_meta_box('authordiv', 'post', 'normal');
	add_meta_box('fb_authordiv', __('Author'), 'fb_post_author_meta_box', 'post', 'normal', 'core');
}

function fb_post_author_meta_box( $post ) {
	global $user_ID;
	
	// get all authors
	$wp_user_search = new WP_User_Search($usersearch = '', $userspage = '', 'author');
	$authors = join( ', ', $wp_user_search->get_results() );
	?>
	<label class="screen-reader-text" for="post_author_override"><?php _e('Author'); ?></label>
	<?php
	wp_dropdown_users( array(
		'include' => $authors,
		'who' => 'authors',
		'name' => 'post_author_override',
		'selected' => empty($post->ID) ? $user_ID : $post->post_author,
		'include_selected' => true
	) );
}

Updated to use the newer WP_UserQuery, and allow for more flexible querying.

https://gist.github.com/benklocek/8cae13427ed7760cb76c