Skip to content

Instantly share code, notes, and snippets.

@stuartsan stuartsan/analyze.py Secret
Last active May 2, 2019

Embed
What would you like to do?
Terraform vs CloudFormation new resources added 2018-04-01 to 2019-04-30. See stuartsandine.com/terraform-vs-cloudformation-aws-resource-support
import datetime
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from wordcloud import WordCloud
import squarify
df = pd.read_csv('./tf-vs-cfn.csv', parse_dates=[1,2], infer_datetime_format=True)
cfn_added_support_during_year = df[df['CFN date'] >= datetime.datetime(2018, 4, 1)]
tf_added_support_during_year = df[df['TF date'] >= datetime.datetime(2018, 4, 1)]
cfn_and_not_tf = cfn_added_support_during_year[cfn_added_support_during_year['TF date'].isnull()]
tf_and_not_cfn = tf_added_support_during_year[tf_added_support_during_year['CFN date'].isnull()]
# add a new column `delta` which represents the time difference between CFN and
# CFN support being added for a given resource.
# a negative number means CFN support came first, positive means TF support came first.
resources_both_tools_support = df[df['CFN date'].notnull() & df['TF date'].notnull()]
delta = resources_both_tools_support['CFN date'] - resources_both_tools_support['TF date']
resources_both_tools_support['delta'] = delta
same_day_support_added = resources_both_tools_support[resources_both_tools_support['delta'].dt.days == 0]
cfn_supported_first = resources_both_tools_support[resources_both_tools_support['delta'].dt.days < 0]
tf_supported_first = resources_both_tools_support[resources_both_tools_support['delta'].dt.days > 0]
cfn_only_services = [ r.split('::')[1] for r in cfn_and_not_tf['resource'].values ]
tf_only_services = [ r.split('_')[1] for r in tf_and_not_cfn['resource'].values ]
def save_wordcloud(filename, words):
wordcloud = WordCloud(regexp=r'\w+', width=600, height=400, background_color='white', collocations=False).generate(' '.join(words))
image = wordcloud.to_image()
image.save(filename)
def save_treemap(filename, df):
# filter out the zeroes
# df = df[df['delta'].dt.days != 0]
# colors = np.where(df['delta'].dt.days < 0, 'red', 'blue')
# squarify.plot(sizes=df['delta'].dt.days.values / 85, label=df['resource'].values, color=colors)
squarify.plot(sizes=df['delta'].dt.days.values, label=df['resource'].values)
plt.axis('off')
plt.savefig(filename)
plt.clf()
def save_diverging_bar(filename, df):
df['colors'] = ['orange' if d < 0 else 'purple' for d in df['delta'].dt.days]
df.sort_values('resource', inplace=True)
df.reset_index(inplace=True)
# colors = np.where(df['delta'].dt.days < 0, 'red', 'blue')
# Draw plot
plt.figure(figsize=(18,18))
plt.hlines(y=df.index, xmin=0, xmax=df.delta.dt.days.values)
for x, y, tex in zip(df.delta.dt.days.values, df.index, df.delta.dt.days.values):
t = plt.text(x, y, abs(round(tex, 2)), horizontalalignment='right' if x < 0 else 'left', verticalalignment='center', fontdict={'color':'orange' if x < 0 else 'purple', 'size':14})
# Decorations
plt.yticks(df.index, df.resource, fontsize=10)
plt.grid(linestyle='--', alpha=0.5)
# plt.xlim(-2.5, 5)
plt.subplots_adjust(left=0.23)
plt.savefig(filename)
plt.clf()
if __name__ == '__main__':
print(f'total number of AWS resources for which at least one tool added support: {len(df)}')
print(f'CFN added support for {len(cfn_added_support_during_year)} AWS resources')
print(f'TF added support for {len(tf_added_support_during_year)} AWS resources')
print(f'CFN added support for {len(cfn_and_not_tf)} AWS resources that TF does not support')
print(f'TF added support for {len(tf_and_not_cfn)} AWS resources that CFN does not support')
print(f'CFN and TF added support for {len(same_day_support_added)} AWS resources on the same day')
print(f'both tools support and CFN added support first:\n{cfn_supported_first.describe()}')
print(f'both tools support and TF added support first:\n{tf_supported_first.describe()}')
save_wordcloud('cfn-wordcloud.png', cfn_only_services)
save_wordcloud('tf-wordcloud.png', tf_only_services)
save_wordcloud('cfn-first-wordcloud.png', cfn_supported_first['resource'].values)
save_wordcloud('tf-first-wordcloud.png', tf_supported_first['resource'].values)
# save_treemap('tf-treemap.png', cfn_supported_first)
save_diverging_bar('diverging.png', resources_both_tools_support)
resource CFN date TF date Note
AWS::Config::AggregationAuthorization Apr 04, 2018 Jun 05, 2018
AWS::Config::ConfigurationAggregator Apr 04, 2018 Jun 05, 2018
AWS::AppSync::ApiKey Apr 10, 2018 Jul 11, 2018
AWS::AppSync::DataSource Apr 10, 2018 Mar 28, 2018
AWS::AppSync::GraphQLApi Apr 10, 2018 Feb 24, 2018
AWS::AppSync::GraphQLSchema Apr 10, 2018 Feb 24, 2018
AWS::AppSync::Resolver Apr 10, 2018 Mar 29, 2019
AWS::GuardDuty::Filter May 08, 2018 TF: unimplemented
AWS::AutoScalingPlans::ScalingPlan May 09, 2018 TF: unimplemented
AWS::Budgets::Budget May 22, 2018 May 02, 2018
AWS::ServiceCatalog::AcceptedPortfolioShare May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::CloudFormationProduct May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::LaunchNotificationConstraint May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::LaunchRoleConstraint May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::LaunchTemplateConstraint May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::Portfolio May 24, 2018 Oct 31, 2017
AWS::ServiceCatalog::PortfolioPrincipalAssociation May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::PortfolioProductAssociation May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::PortfolioShare May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::TagOption May 24, 2018 TF: unimplemented
AWS::ServiceCatalog::TagOptionAssociation May 24, 2018 TF: unimplemented
AWS::Neptune::DBCluster May 30, 2018 Jul 18, 2018
AWS::Neptune::DBClusterParameterGroup May 30, 2018 Jun 21, 2018
AWS::Neptune::DBInstance May 30, 2018 Aug 02, 2018
AWS::Neptune::DBParameterGroup May 30, 2018 Jun 14, 2018
AWS::Neptune::DBSubnetGroup May 30, 2018 Jun 21, 2018
AWS::SageMaker::Endpoint May 31, 2018 Apr 05, 2019
AWS::SageMaker::EndpointConfig May 31, 2018 Mar 29, 2019
AWS::SageMaker::Model May 31, 2018 Feb 08, 2019
AWS::SageMaker::NotebookInstance May 31, 2018 Jan 16, 2019
AWS::SageMaker::NotebookInstanceLifecycleConfig May 31, 2018 Apr 05, 2019
AWS::EKS::Cluster Jun 05, 2018 Jun 05, 2018
AWS::SSM::ResourceDataSync Jun 11, 2018 Nov 16, 2017
AWS::AmazonMQ::Broker Jun 14, 2018 Nov 29, 2017
AWS::AmazonMQ::Configuration Jun 14, 2018 Nov 29, 2017
AWS::EC2::VPCEndpointConnectionNotification Jun 21, 2018 Feb 09, 2018
AWS::EC2::VPCEndpointService Jun 21, 2018 Feb 09, 2018
AWS::CodePipeline::Webhook Jul 05, 2018 Oct 18, 2018
AWS::IAM::ServiceLinkedRole Jul 19, 2018 Apr 18, 2018
AWS::EC2::VPCEndpointServicePermissions Aug 09, 2018 Feb 09, 2018 TF: attribute of aws_vpc_endpoint_service
AWS::IoT1Click::Device Sep 20, 2018 TF: unimplemented
AWS::IoT1Click::Placement Sep 20, 2018 TF: unimplemented
AWS::IoT1Click::Project Sep 20, 2018 TF: unimplemented
AWS::Events::EventBusPolicy Oct 18, 2018 Jan 12, 2018 TF: called aws_cloudwatch_event_permission
AWS::AppStream::DirectoryConfig Oct 25, 2018 TF: unimplemented
AWS::AppStream::Fleet Oct 25, 2018 TF: unimplemented
AWS::AppStream::ImageBuilder Oct 25, 2018 TF: unimplemented
AWS::AppStream::Stack Oct 25, 2018 TF: unimplemented
AWS::AppStream::StackFleetAssociation Oct 25, 2018 TF: unimplemented
AWS::AppStream::StackUserAssociation Oct 25, 2018 TF: unimplemented
AWS::AppStream::User Oct 25, 2018 TF: unimplemented
AWS::DLM::LifecyclePolicy Nov 09, 2018 Nov 07, 2018
AWS::SecretsManager::ResourcePolicy Nov 09, 2018 Jul 26, 2018
AWS::SecretsManager::RotationSchedule Nov 09, 2018 Apr 25, 2018
AWS::SecretsManager::Secret Nov 09, 2018 Apr 25, 2018
AWS::SecretsManager::SecretTargetAttachment Nov 09, 2018 TF: unimplemented (?)
Alexa::ASK::Skill Nov 20, 2018 TF: unimplemented
AWS::AppSync::FunctionConfiguration Nov 20, 2018 TF: unimplemented (?)
AWS::EC2::EC2Fleet Nov 20, 2018 Oct 03, 2018
AWS::Kinesis::StreamConsumer Nov 20, 2018 TF: unimplemented
AWS::Route53Resolver:ResolverEndpoint Nov 20, 2018 Mar 07, 2019
AWS::Route53Resolver::ResolverRule Nov 20, 2018 Mar 15, 2019
AWS::EC2::TransitGateway Nov 26, 2018 Nov 27, 2018
AWS::EC2::TransitGatewayAttachment Nov 26, 2018 Nov 27, 2018
AWS::EC2::TransitGatewayRoute Nov 26, 2018 Nov 27, 2018
AWS::EC2::TransitGatewayRouteTable Nov 26, 2018 Nov 27, 2018
AWS::EC2::TransitGatewayRouteTableAssociation Nov 26, 2018 Nov 27, 2018
AWS::EC2::TransitGatewayRouteTablePropagation Nov 26, 2018 Nov 27, 2018
AWS::ServiceDiscovery::HttpNamespace Nov 28, 2018 Dec 20, 2018
AWS::Lambda::LayerVersion Nov 29, 2018 Jan 16, 2019
AWS::Lambda::LayerVersionPermission Nov 29, 2018 TF: unimplemented
AWS::AmazonMQ::ConfigurationAssociation Dec 13, 2018 TF: unimplemented
AWS::IoTAnalytics::Channel Dec 13, 2018 TF: unimplemented
AWS::IoTAnalytics::Dataset Dec 13, 2018 TF: unimplemented
AWS::IoTAnalytics::Datastore Dec 13, 2018 TF: unimplemented
AWS::IoTAnalytics::Pipeline Dec 13, 2018 TF: unimplemented
AWS::Route53Resolver::ResolverRuleAssociation Jan 03, 2019 Mar 15, 2019
AWS::DocDB::DBCluster Jan 10, 2019 Feb 08, 2019
AWS::DocDB::DBClusterParameterGroup Jan 10, 2019 Jan 10, 2019
AWS::DocDB::DBInstance Jan 10, 2019 Feb 14, 2019
AWS::DocDB::DBSubnetGroup Jan 10, 2019 Jan 16, 2019
AWS::OpsWorksCM::Server Jan 24, 2019 TF: unimplemented
AWS::ApiGatewayV2::Api Feb 08, 2019 TF: unimplemented
AWS::ApiGatewayV2::Authorizer Feb 08, 2019 TF: unimplemented
AWS::ApiGatewayV2::Deployment Feb 08, 2019 TF: unimplemented
AWS::ApiGatewayV2::Integration Feb 08, 2019 TF: unimplemented
AWS::ApiGatewayV2::IntegrationResponse Feb 08, 2019 TF: unimplemented
AWS::ApiGatewayV2::Model Feb 08, 2019 TF: unimplemented
AWS::ApiGatewayV2::Route Feb 08, 2019 TF: unimplemented
AWS::ApiGatewayV2::RouteResponse Feb 08, 2019 TF: unimplemented
AWS::ApiGatewayV2::Stage Feb 08, 2019 TF: unimplemented
AWS::FSx::FileSystem Feb 15, 2019 TF: unimplemented
AWS::KinesisAnalyticsv2::Application Feb 15, 2019 TF: unimplemented (?)
AWS::KinesisAnalyticsv2::ApplicationCloudWatchLoggingOption Feb 15, 2019 TF: unimplemented (?)
AWS::KinesisAnalyticsv2::ApplicationOutput Feb 15, 2019 TF: unimplemented (?)
AWS::KinesisAnalyticsv2::ApplicationReferenceDataSource Feb 15, 2019 TF: unimplemented (?)
AWS::RAM::ResourceShare Feb 21, 2019 Jan 16, 2019
AWS::RoboMaker::Fleet Feb 21, 2019 TF: unimplemented
AWS::RoboMaker::Robot Feb 21, 2019 TF: unimplemented
AWS::RoboMaker::RobotApplication Feb 21, 2019 TF: unimplemented
AWS::RoboMaker::RobotApplicationVersion Feb 21, 2019 TF: unimplemented
AWS::RoboMaker::SimulationApplication Feb 21, 2019 TF: unimplemented
AWS::RoboMaker::SimulationApplicationVersion Feb 21, 2019 TF: unimplemented
AWS::Greengrass::ConnectorDefinition Mar 15, 2019 TF: unimplemented
AWS::Greengrass::ConnectorDefinitionVersion Mar 15, 2019 TF: unimplemented
AWS::Greengrass::CoreDefinition Mar 15, 2019 TF: unimplemented
AWS::Greengrass::CoreDefinitionVersion Mar 15, 2019 TF: unimplemented
AWS::Greengrass::DeviceDefinition Mar 15, 2019 TF: unimplemented
AWS::Greengrass::DeviceDefinitionVersion Mar 15, 2019 TF: unimplemented
AWS::Greengrass::FunctionDefinition Mar 15, 2019 TF: unimplemented
AWS::Greengrass::FunctionDefinitionVersion Mar 15, 2019 TF: unimplemented
AWS::Greengrass::Group Mar 15, 2019 TF: unimplemented
AWS::Greengrass::GroupVersion Mar 15, 2019 TF: unimplemented
AWS::Greengrass::LoggerDefinition Mar 15, 2019 TF: unimplemented
AWS::Greengrass::LoggerDefinitionVersion Mar 15, 2019 TF: unimplemented
AWS::Greengrass::ResourceDefinition Mar 15, 2019 TF: unimplemented
AWS::Greengrass::ResourceDefinitionVersion Mar 15, 2019 TF: unimplemented
AWS::Greengrass::SubscriptionDefinition Mar 15, 2019 TF: unimplemented
AWS::Greengrass::SubscriptionDefinitionVersion Mar 15, 2019 TF: unimplemented
AWS::AppMesh::Mesh Mar 27, 2019 Dec 05, 2018
AWS::AppMesh::Route Mar 27, 2019 Dec 13, 2018
AWS::AppMesh::VirtualNode Mar 27, 2019 Dec 13, 2018
AWS::AppMesh::VirtualRouter Mar 27, 2019 Dec 05, 2018
AWS::AppMesh::VirtualService Mar 27, 2019 Mar 21, 2019
AWS::ServiceCatalog::ResourceUpdateConstraint Apr 04, 2019 TF: unimplemented
AWS::EC2::CapacityReservation Apr 18, 2019 Oct 31, 2018
aws_organizations_account Apr 06, 2018 CFN: unimplemented
aws_ses_identity_notification_topic Apr 06, 2018 CFN: unimplemented
aws_directory_service_conditional_forwarder Apr 18, 2018 CFN: unimplemented
aws_glue_connection Oct 24, 2017 Apr 18, 2018
aws_glue_job Oct 24, 2017 Apr 18, 2018
aws_launch_template Mar 29, 2018 Apr 18, 2018
aws_ses_domain_identity_verification Apr 18, 2018 CFN: unimplemented
aws_dax_parameter_group Aug 22, 2017 Apr 25, 2018
aws_dax_subnet_group Aug 22, 2017 Apr 25, 2018
aws_organizations_policy Apr 25, 2018 CFN: unimplemented
aws_organizations_policy_attachment Apr 25, 2018 CFN: unimplemented
aws_iam_user_group_membership Feb 25, 2011 May 02, 2018
aws_vpc_peering_connection_options May 02, 2018 CFN: unimplemented
aws_acmpca_certificate_authority May 10, 2018 CFN: unimplemented
aws_glue_catalog_table Oct 24, 2017 May 10, 2018
aws_sns_sms_preferences May 16, 2018 CFN: unimplemented
aws_glue_trigger Oct 24, 2017 May 23, 2018
aws_codebuild_webhook Feb 19, 2018 May 31, 2018
aws_cognito_identity_provider Apr 28, 2017 May 31, 2018 this interpretation is generous; CFN doesn't provide all the same
aws_cognito_resource_server May 31, 2018 CFN: unimplemented
aws_glue_classifier Oct 24, 2017 May 31, 2018
aws_dx_gateway Jun 21, 2018 CFN: unimplemented
aws_dx_gateway_association Jun 21, 2018 CFN: unimplemented
aws_glue_crawler Oct 24, 2017 Jun 21, 2018
aws_dx_hosted_private_virtual_interface Jun 27, 2018 CFN: unimplemented
aws_dx_hosted_private_virtual_interface_accepter Jun 27, 2018 CFN: unimplemented
aws_dx_hosted_public_virtual_interface Jun 27, 2018 CFN: unimplemented
aws_dx_hosted_public_virtual_interface_accepter Jun 27, 2018 CFN: unimplemented
aws_dx_private_virtual_interface Jun 27, 2018 CFN: unimplemented
aws_dx_public_virtual_interface Jun 27, 2018 CFN: unimplemented
aws_media_store_container_policy Jun 27, 2018 CFN: unimplemented
aws_s3_bucket_inventory Feb 25, 2011 Jul 04, 2018
aws_vpc_ipv4_cidr_block_association Dec 01, 2016 Jul 04, 2018
aws_swf_domain Jul 11, 2018 CFN: unimplemented
aws_macie_s3_bucket_association Jul 18, 2018 CFN: unimplemented
aws_storagegateway_gateway Jul 18, 2018 CFN: unimplemented
aws_macie_member_account_association Aug 02, 2018 CFN: unimplemented
aws_storagegateway_nfs_file_share Aug 02, 2018 CFN: unimplemented
aws_storagegateway_upload_buffer Aug 02, 2018 CFN: unimplemented
aws_storagegateway_working_storage Aug 02, 2018 CFN: unimplemented
aws_db_cluster_snapshot Aug 09, 2018 CFN: unimplemented
aws_neptune_event_subscription Aug 09, 2018 CFN: unimplemented
aws_storagegateway_cache Aug 09, 2018 CFN: unimplemented
aws_storagegateway_smb_file_share Aug 09, 2018 CFN: unimplemented
aws_neptune_cluster_snapshot Aug 16, 2018 CFN: unimplemented
aws_storagegateway_cached_iscsi_volume Aug 16, 2018 CFN: unimplemented
aws_cloudfront_public_key Sep 13, 2018 CFN: unimplemented
aws_dx_bgp_peer Sep 19, 2018 CFN: unimplemented
aws_pinpoint_app Oct 03, 2018 CFN: unimplemented
aws_ebs_snapshot_copy Oct 10, 2018 CFN: unimplemented
aws_pinpoint_adm_channel Oct 10, 2018 CFN: unimplemented
aws_pinpoint_baidu_channel Oct 10, 2018 CFN: unimplemented
aws_pinpoint_email_channel Oct 10, 2018 CFN: unimplemented
aws_pinpoint_event_stream Oct 10, 2018 CFN: unimplemented
aws_pinpoint_gcm_channel Oct 10, 2018 CFN: unimplemented
aws_pinpoint_sms_channel Oct 10, 2018 CFN: unimplemented
aws_redshift_snapshot_copy_grant Oct 10, 2018 CFN: unimplemented
aws_cloudhsm_v2_cluster Oct 18, 2018 CFN: unimplemented
aws_cloudhsm_v2_hsm Oct 18, 2018 CFN: unimplemented
aws_pinpoint_apns_channel Oct 18, 2018 CFN: unimplemented
aws_redshift_event_subscription Oct 18, 2018 CFN: unimplemented
aws_glue_security_configuration Oct 31, 2018 CFN: unimplemented
aws_iot_policy_attachment Jul 20, 2016 Oct 31, 2018
aws_iot_thing_principal_attachment Jul 20, 2016 Oct 31, 2018
aws_pinpoint_apns_sandbox_channel Oct 31, 2018 CFN: unimplemented
aws_pinpoint_apns_voip_channel Oct 31, 2018 CFN: unimplemented
aws_pinpoint_apns_voip_sandbox_channel Oct 31, 2018 CFN: unimplemented
aws_dlm_lifecycle_policy Nov 09, 2018 Nov 07, 2018
aws_kinesis_analytics_application Jul 28, 2017 Nov 07, 2018
aws_gamelift_game_session_queue Nov 14, 2018 CFN: unimplemented
aws_glacier_vault_lock Nov 14, 2018 CFN: unimplemented
aws_datasync_agent Nov 26, 2018 CFN: unimplemented
aws_datasync_location_efs Nov 26, 2018 CFN: unimplemented
aws_datasync_location_nfs Nov 26, 2018 CFN: unimplemented
aws_datasync_location_s3 Nov 26, 2018 CFN: unimplemented
aws_datasync_task Nov 26, 2018 CFN: unimplemented
aws_rds_cluster_endpoint Dec 13, 2018 CFN: unimplemented
aws_securityhub_account Dec 13, 2018 CFN: unimplemented
aws_transfer_server Dec 13, 2018 CFN: unimplemented
aws_licensemanager_license_configuration Dec 20, 2018 CFN: unimplemented
aws_rds_global_cluster Sep 20, 2018 Dec 20, 2018
aws_s3_account_public_access_block Dec 20, 2018 CFN: unimplemented
aws_securityhub_standards_subscription Dec 20, 2018 CFN: unimplemented
aws_transfer_ssh_key Dec 20, 2018 CFN: unimplemented
aws_transfer_user Dec 20, 2018 CFN: unimplemented
aws_licensemanager_association Dec 21, 2018 CFN: unimplemented
aws_s3_bucket_public_access_block Nov 19, 2018 Dec 21, 2018
aws_securityhub_product_subscription Dec 21, 2018 CFN: unimplemented
aws_media_package_channel Jan 10, 2019 CFN: unimplemented
aws_resourcegroups_group Jan 10, 2019 CFN: unimplemented
aws_globalaccelerator_accelerator Jan 16, 2019 CFN: unimplemented
aws_backup_vault Feb 08, 2019 CFN: unimplemented
aws_docdb_cluster_snapshot Feb 08, 2019 CFN: unimplemented
aws_ec2_client_vpn_endpoint Feb 08, 2019 CFN: unimplemented
aws_ec2_client_vpn_network_association Feb 08, 2019 CFN: unimplemented
aws_cur_report_definition Feb 14, 2019 CFN: unimplemented
aws_ram_principal_association Feb 21, 2019 Feb 14, 2019
aws_ram_resource_association Feb 21, 2019 Feb 14, 2019
aws_globalaccelerator_listener Mar 15, 2019 CFN: unimplemented
aws_guardduty_invite_accepter Mar 15, 2019 CFN: unimplemented
aws_backup_plan Mar 21, 2019 CFN: unimplemented
aws_kms_ciphertext Mar 29, 2019 CFN: unimplemented
aws_kms_external_key Mar 29, 2019 CFN: unimplemented
aws_backup_selection Apr 05, 2019 CFN: unimplemented
aws_worklink_website_certificate_authority_association Apr 10, 2019 CFN: unimplemented
aws_dx_gateway_association_proposal Apr 26, 2019 CFN: unimplemented
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.