A simple proof-of-concept for modularizing the CSS for each block, and linking it "just in time".

modular-stylesheets.php

<?php
// this snippet requires PHP 5.3+
add_action( 'wp_enqueue_scripts', function() {
	wp_register_style( 'atomic-blocks/ab-cta', '/path/to/atomic-blocks/css/ab-cta.css', array(), 1.0.0 );
} );

add_filter( 'render_block', function( $block_content, $block ) {
	if ( 'atomic-blocks/ab-cta' === $block['blockName'] ) {
		ob_start();
		wp_print_styles( $block['blockName'] );
		$block_content = ob_get_clean() . $block_content;
	}

	return $block_content;
}, 10, 2 );