From http://wpsnipp.com/index.php/cat/add-nofollow-attribute-to-category-specific-links/ Uses http://codex.wordpress.org/Function_Reference/wp_rel_nofollow

nofollow-specific-category-link.php

<?php
function nofollow_cat_posts($text) {
global $post;
        if( in_category(1) ) { // SET CATEGORY ID HERE
                $text = stripslashes(wp_rel_nofollow($text));
        }
        return $text;
}
add_filter('the_content', 'nofollow_cat_posts');