gistfile1.txt

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"

	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/Azure/go-autorest/autorest/azure/auth"
	"github.com/Azure/go-autorest/autorest/to"
	"github.com/sirupsen/logrus"

	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
	mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"

	"github.com/Azure/ARO-RP/pkg/api"
	"github.com/Azure/ARO-RP/pkg/util/arm"
	"github.com/Azure/ARO-RP/pkg/util/azureclient"
	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features"
	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
	"github.com/Azure/ARO-RP/pkg/util/azureerrors"
	utillog "github.com/Azure/ARO-RP/pkg/util/log"
	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
)

var (
	infraPostFix   = "kwc7m" // random postfix
	deploymentName = "aro-deploy"
)

func main() {
	ctx := context.Background()
	log := utillog.GetLogger()
	err := run(ctx, log)
	if err != nil {
		panic(err)
	}
}

type reprodusable struct {
	log *logrus.Entry

	location          *string
	subscriptionID    string
	resourceGroupName string
	infraID           string

	loadbalancers    network.LoadBalancersClient
	groups           features.ResourceGroupsClient
	deployments      features.DeploymentsClient
	privateendpoints network.PrivateEndpointsClient
}

func run(ctx context.Context, log *logrus.Entry) error {
	authorizer, err := auth.NewAuthorizerFromCLI()
	if err != nil {
		return err
	}
	r := reprodusable{
		log:               log,
		subscriptionID:    os.Getenv("AZURE_SUBSCRIPTION_ID"),
		location:          to.StringPtr(os.Getenv("LOCATION")),
		resourceGroupName: os.Getenv("RESOURCEGROUP"),
		infraID:           os.Getenv("CLUSTER") + "-" + infraPostFix,
	}

	r.loadbalancers = network.NewLoadBalancersClient(r.subscriptionID, authorizer)
	r.groups = features.NewResourceGroupsClient(r.subscriptionID, authorizer)
	r.deployments = features.NewDeploymentsClient(r.subscriptionID, authorizer)
	r.privateendpoints = network.NewPrivateEndpointsClient(r.subscriptionID, authorizer)

	err = r.init(ctx)
	if err != nil {
		return err
	}

	r.log.Infof("GET LB %s from %s in %s", r.infraID+"-internal", r.resourceGroupName, *r.location)
	ilb, err := r.loadbalancers.Get(ctx, r.resourceGroupName, r.infraID+"-internal", "")
	if err != nil {
		return err
	}

	r.log.Infof("Add new FE to LB  %s", r.infraID+"-internal")
	fe := mgmtnetwork.FrontendIPConfiguration{
		FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{
			PrivateIPAllocationMethod: mgmtnetwork.Dynamic,
			Subnet: &mgmtnetwork.Subnet{
				ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/vnet/subnets/s1", r.subscriptionID, r.resourceGroupName)),
			},
		},
		Name: to.StringPtr("a2a79c077817949a6a3310ad1654a13c"),
	}

	*ilb.FrontendIPConfigurations = append(*ilb.FrontendIPConfigurations, fe)

	r.log.Infof("Update LB with new FE %s", r.infraID+"-internal")
	return r.loadbalancers.CreateOrUpdateAndWait(ctx, r.resourceGroupName, r.infraID+"-internal", ilb)
}

func (r *reprodusable) init(ctx context.Context) error {
	r.log.Infof("create rg %s in %s", r.resourceGroupName, *r.location)
	_, err := r.groups.CreateOrUpdate(ctx, r.resourceGroupName, mgmtfeatures.ResourceGroup{
		Location: r.location})
	if err != nil {
		return err
	}

	r.log.Infof("create rg %s in %s", r.resourceGroupName+"-management", *r.location)
	_, err = r.groups.CreateOrUpdate(ctx, r.resourceGroupName+"-management", mgmtfeatures.ResourceGroup{
		Location: r.location})
	if err != nil {
		return err
	}

	r.log.Infof("create ilb %s-internal %s in %s", r.infraID, r.resourceGroupName, *r.location)
	t := &arm.Template{
		Schema:         "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
		ContentVersion: "1.0.0.0",
		Parameters: map[string]*arm.TemplateParameter{
			"sas": {
				Type: "object",
			},
		},
		Resources: []*arm.Resource{
			r.vnet(),
			r.securityGroup(),
			r.networkInternalLoadBalancer(),
			r.networkPrivateLinkService(),
		},
	}

	err = r.deployARMTemplate(ctx, r.resourceGroupName, "resources", t, map[string]interface{}{
		"sas": map[string]interface{}{
			"value": map[string]interface{}{
				"signedPermission":    "rl",
				"signedResourceTypes": "o",
				"signedServices":      "b",
				"signedProtocol":      "https",
			},
		},
	})

	r.log.Infof("create management %s-internal %s-management in %s", r.infraID, r.resourceGroupName, *r.location)
	t = &arm.Template{
		Schema:         "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
		ContentVersion: "1.0.0.0",
		Parameters: map[string]*arm.TemplateParameter{
			"sas": {
				Type: "object",
			},
		},
		Resources: []*arm.Resource{
			r.vnet(),
			r.securityGroup(),
		},
	}

	err = r.deployARMTemplate(ctx, r.resourceGroupName+"-management", "resources", t, map[string]interface{}{
		"sas": map[string]interface{}{
			"value": map[string]interface{}{
				"signedPermission":    "rl",
				"signedResourceTypes": "o",
				"signedServices":      "b",
				"signedProtocol":      "https",
			},
		},
	})

	r.log.Infof("finish PE ")
	return r.privateendpoints.CreateOrUpdateAndWait(ctx, r.resourceGroupName+"-management", r.infraID, mgmtnetwork.PrivateEndpoint{
		PrivateEndpointProperties: &mgmtnetwork.PrivateEndpointProperties{
			Subnet: &mgmtnetwork.Subnet{
				ID: to.StringPtr("/subscriptions/" + r.subscriptionID + "/resourceGroups/" + r.resourceGroupName + "-management/providers/Microsoft.Network/virtualNetworks/vnet/subnets/s1"),
			},
			ManualPrivateLinkServiceConnections: &[]mgmtnetwork.PrivateLinkServiceConnection{
				{
					Name: to.StringPtr("rp-plsconnection"),
					PrivateLinkServiceConnectionProperties: &mgmtnetwork.PrivateLinkServiceConnectionProperties{
						PrivateLinkServiceID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/privateLinkServices/%s", r.subscriptionID, r.resourceGroupName, r.infraID+"-pls")),
					},
				},
			},
		},
		Location: r.location,
	})

}

func (r *reprodusable) deployARMTemplate(ctx context.Context, rg string, tName string, t *arm.Template, params map[string]interface{}) error {
	r.log.Printf("deploying %s template", tName)

	err := r.deployments.CreateOrUpdateAndWait(ctx, rg, deploymentName, mgmtfeatures.Deployment{
		Properties: &mgmtfeatures.DeploymentProperties{
			Template:   t,
			Parameters: params,
			Mode:       mgmtfeatures.Incremental,
		},
	})

	if azureerrors.IsDeploymentActiveError(err) {
		r.log.Printf("waiting for %s template to be deployed", tName)
		err = r.deployments.Wait(ctx, rg, deploymentName)
	}

	if azureerrors.HasAuthorizationFailedError(err) ||
		azureerrors.HasLinkedAuthorizationFailedError(err) {
		return err
	}

	serviceErr, _ := err.(*azure.ServiceError) // futures return *azure.ServiceError directly

	// CreateOrUpdate() returns a wrapped *azure.ServiceError
	if detailedErr, ok := err.(autorest.DetailedError); ok {
		serviceErr, _ = detailedErr.Original.(*azure.ServiceError)
	}

	if serviceErr != nil {
		b, _ := json.Marshal(serviceErr)

		return &api.CloudError{
			StatusCode: http.StatusBadRequest,
			CloudErrorBody: &api.CloudErrorBody{
				Code:    api.CloudErrorCodeDeploymentFailed,
				Message: "Deployment failed.",
				Details: []api.CloudErrorBody{
					{
						Message: string(b),
					},
				},
			},
		}
	}

	return err
}

// resources

func (r *reprodusable) networkPrivateLinkService() *arm.Resource {
	return &arm.Resource{
		Resource: &mgmtnetwork.PrivateLinkService{
			PrivateLinkServiceProperties: &mgmtnetwork.PrivateLinkServiceProperties{
				LoadBalancerFrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
					{
						ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", r.infraID)),
					},
				},
				IPConfigurations: &[]mgmtnetwork.PrivateLinkServiceIPConfiguration{
					{
						PrivateLinkServiceIPConfigurationProperties: &mgmtnetwork.PrivateLinkServiceIPConfigurationProperties{
							Subnet: &mgmtnetwork.Subnet{
								ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/vnet/subnets/s1", r.subscriptionID, r.resourceGroupName)),
							},
						},
						Name: to.StringPtr(r.infraID + "-pls-nic"),
					},
				},
				Visibility: &mgmtnetwork.PrivateLinkServicePropertiesVisibility{
					Subscriptions: &[]string{
						r.subscriptionID,
					},
				},
				AutoApproval: &mgmtnetwork.PrivateLinkServicePropertiesAutoApproval{
					Subscriptions: &[]string{
						r.subscriptionID,
					},
				},
			},
			Name:     to.StringPtr(r.infraID + "-pls"),
			Type:     to.StringPtr("Microsoft.Network/privateLinkServices"),
			Location: r.location,
		},
		APIVersion: azureclient.APIVersions["Microsoft.Network"],
		DependsOn: []string{
			"Microsoft.Network/loadBalancers/" + r.infraID + "-internal",
		},
	}
}

func (r *reprodusable) securityGroup() *arm.Resource {
	nsg := &mgmtnetwork.SecurityGroup{
		SecurityGroupPropertiesFormat: &mgmtnetwork.SecurityGroupPropertiesFormat{
			SecurityRules: &[]mgmtnetwork.SecurityRule{
				{
					SecurityRulePropertiesFormat: &mgmtnetwork.SecurityRulePropertiesFormat{
						Protocol:                 mgmtnetwork.SecurityRuleProtocolTCP,
						SourcePortRange:          to.StringPtr("*"),
						DestinationPortRange:     to.StringPtr("443"),
						SourceAddressPrefix:      to.StringPtr("*"),
						DestinationAddressPrefix: to.StringPtr("*"),
						Access:                   mgmtnetwork.SecurityRuleAccessAllow,
						Priority:                 to.Int32Ptr(120),
						Direction:                mgmtnetwork.SecurityRuleDirectionInbound,
					},
					Name: to.StringPtr("nsg"),
				},
				{
					SecurityRulePropertiesFormat: &mgmtnetwork.SecurityRulePropertiesFormat{
						Protocol:                 mgmtnetwork.SecurityRuleProtocolTCP,
						SourcePortRange:          to.StringPtr("*"),
						DestinationPortRange:     to.StringPtr("22"),
						SourceAddressPrefix:      to.StringPtr("*"),
						DestinationAddressPrefix: to.StringPtr("*"),
						Access:                   mgmtnetwork.SecurityRuleAccessAllow,
						Priority:                 to.Int32Ptr(125),
						Direction:                mgmtnetwork.SecurityRuleDirectionInbound,
					},
					Name: to.StringPtr("ssh_in"),
				},
			},
		},
		Name:     to.StringPtr("nsg"),
		Type:     to.StringPtr("Microsoft.Network/networkSecurityGroups"),
		Location: to.StringPtr("[resourceGroup().location]"),
	}

	return &arm.Resource{
		Resource:   nsg,
		APIVersion: azureclient.APIVersions["Microsoft.Network"],
	}
}

func (r *reprodusable) vnet() *arm.Resource {
	subnet := mgmtnetwork.Subnet{
		SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{
			AddressPrefix: to.StringPtr("10.0.0.0/24"),
			NetworkSecurityGroup: &mgmtnetwork.SecurityGroup{
				ID: to.StringPtr("[resourceId('Microsoft.Network/networkSecurityGroups', 'nsg')]"),
			},
			PrivateLinkServiceNetworkPolicies: to.StringPtr("Disabled"),
			PrivateEndpointNetworkPolicies:    to.StringPtr("Disabled"),
		},
		Name: to.StringPtr("s1"),
	}

	return &arm.Resource{
		Resource: &mgmtnetwork.VirtualNetwork{
			VirtualNetworkPropertiesFormat: &mgmtnetwork.VirtualNetworkPropertiesFormat{
				AddressSpace: &mgmtnetwork.AddressSpace{
					AddressPrefixes: &[]string{
						"10.0.0.0/24",
					},
				},
				Subnets: &[]mgmtnetwork.Subnet{
					subnet,
				},
			},
			Name:     to.StringPtr("vnet"),
			Type:     to.StringPtr("Microsoft.Network/virtualNetworks"),
			Location: to.StringPtr("[resourceGroup().location]"),
		},
		APIVersion: azureclient.APIVersions["Microsoft.Network"],
		DependsOn: []string{
			"[resourceId('Microsoft.Network/networkSecurityGroups', 'nsg')]",
		},
	}
}

func (r *reprodusable) networkInternalLoadBalancer() *arm.Resource {
	return &arm.Resource{
		Resource: &mgmtnetwork.LoadBalancer{
			Sku: &mgmtnetwork.LoadBalancerSku{
				Name: mgmtnetwork.LoadBalancerSkuNameStandard,
			},
			LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
				FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
					{
						FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{
							PrivateIPAllocationMethod: mgmtnetwork.Dynamic,
							Subnet: &mgmtnetwork.Subnet{
								ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/vnet/subnets/s1", r.subscriptionID, r.resourceGroupName)),
							},
						},
						Name: to.StringPtr("internal-lb-ip-v4"),
					},
				},
				BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
					{
						Name: to.StringPtr(r.infraID),
					},
				},
				LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
					{
						LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
							FrontendIPConfiguration: &mgmtnetwork.SubResource{
								ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", r.infraID)),
							},
							BackendAddressPool: &mgmtnetwork.SubResource{
								ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', '%[1]s')]", r.infraID)),
							},
							Probe: &mgmtnetwork.SubResource{
								ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'api-internal-probe')]", r.infraID)),
							},
							Protocol:             mgmtnetwork.TransportProtocolTCP,
							LoadDistribution:     mgmtnetwork.LoadDistributionDefault,
							FrontendPort:         to.Int32Ptr(6443),
							BackendPort:          to.Int32Ptr(6443),
							IdleTimeoutInMinutes: to.Int32Ptr(30),
							DisableOutboundSnat:  to.BoolPtr(true),
						},
						Name: to.StringPtr("api-internal-v4"),
					},
					{
						LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
							FrontendIPConfiguration: &mgmtnetwork.SubResource{
								ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', '%s-internal', 'internal-lb-ip-v4')]", r.infraID)),
							},
							BackendAddressPool: &mgmtnetwork.SubResource{
								ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', '%s-internal', '%[1]s')]", r.infraID)),
							},
							Probe: &mgmtnetwork.SubResource{
								ID: to.StringPtr(fmt.Sprintf("[resourceId('Microsoft.Network/loadBalancers/probes', '%s-internal', 'sinternal-probe')]", r.infraID)),
							},
							Protocol:             mgmtnetwork.TransportProtocolTCP,
							LoadDistribution:     mgmtnetwork.LoadDistributionDefault,
							FrontendPort:         to.Int32Ptr(22623),
							BackendPort:          to.Int32Ptr(22623),
							IdleTimeoutInMinutes: to.Int32Ptr(30),
						},
						Name: to.StringPtr("sinternal-v4"),
					},
				},
				Probes: &[]mgmtnetwork.Probe{
					{
						ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
							Protocol:          mgmtnetwork.ProbeProtocolHTTPS,
							Port:              to.Int32Ptr(6443),
							IntervalInSeconds: to.Int32Ptr(10),
							NumberOfProbes:    to.Int32Ptr(3),
							RequestPath:       to.StringPtr("/readyz"),
						},
						Name: to.StringPtr("api-internal-probe"),
					},
					{
						ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
							Protocol:          mgmtnetwork.ProbeProtocolHTTPS,
							Port:              to.Int32Ptr(22623),
							IntervalInSeconds: to.Int32Ptr(10),
							NumberOfProbes:    to.Int32Ptr(3),
							RequestPath:       to.StringPtr("/healthz"),
						},
						Name: to.StringPtr("sinternal-probe"),
					},
				},
			},
			Name:     to.StringPtr(r.infraID + "-internal"),
			Type:     to.StringPtr("Microsoft.Network/loadBalancers"),
			Location: r.location,
		},
		APIVersion: azureclient.APIVersions["Microsoft.Network"],
		DependsOn: []string{
			"[resourceId('Microsoft.Network/virtualNetworks', 'vnet')]",
		},
	}
}