AzureVnetRessourceGroup

azure-machinedeplyoment.yaml

apiVersion: v1
kind: Secret
metadata:
  name: machine-controller-azure
  namespace: kube-system
type: Opaque
stringData:
  tenantID: "<< AZURE_TENANT_ID >>"
  clientID: "<< AZURE_CLIENT_ID >>"
  clientSecret: "<< AZURE_CLIENT_SECRET >>"
  subscriptionID: "<< AZURE_SUBSCRIPTION_ID >>"
---
apiVersion: "cluster.k8s.io/v1alpha1"
kind: MachineDeployment
metadata:
  name: azure-machinedeployment
  namespace: kube-system
spec:
  paused: false
  replicas: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  minReadySeconds: 0
  selector:
    matchLabels:
      foo: bar
  template:
    metadata:
      labels:
        foo: bar
    spec:
      providerSpec:
        value:
          sshPublicKeys:
            - "<< YOUR_PUBLIC_KEY >>"
          cloudProvider: "azure"
          cloudProviderSpec:
            # Can also be set via the env var 'AZURE_TENANT_ID' on the machine-controller
            tenantID:
              secretKeyRef:
                namespace: kube-system
                name: machine-controller-azure
                key: tenantID
            # Can also be set via the env var 'AZURE_CLIENT_ID' on the machine-controller
            clientID:
              secretKeyRef:
                namespace: kube-system
                name: machine-controller-azure
                key: clientID
            # Can also be set via the env var 'AZURE_CLIENT_SECRET' on the machine-controller
            clientSecret:
              secretKeyRef:
                namespace: kube-system
                name: machine-controller-azure
                key: clientSecret
            # Can also be set via the env var 'AZURE_SUBSCRIPTION_ID' on the machine-controller
            subscriptionID:
              secretKeyRef:
                namespace: kube-system
                name: machine-controller-azure
                key: subscriptionID
            location: "westeurope"
            resourceGroup: "<< YOUR_RESOURCE_GROUP >>"
            vnetResourceGroup: "<< YOUR_VNET_RESOURCE_GROUP >>"
            vmSize: "Standard_F2"
            # optional disk size values in GB. If not set, the defaults for the vmSize will be used.
            osDiskSize: 30
            dataDiskSize: 30
            vnetName: "<< VNET_NAME >>"
            subnetName: "<< SUBNET_NAME >>"
            routeTableName: "<< ROUTE_TABLE_NAME >>"
            imageID: "myImageID"
            assignPublicIP: false
            securityGroupName: my-security-group
            # zones is an optional field and it represents Availability Zones is a high-availability offering
            # that protects your applications and data from datacenter failures.
            zones:
              - "1"
          # Can be 'ubuntu', 'coreos' ,'centos' or 'rhel'
          operatingSystem: "coreos"
          operatingSystemSpec:
            distUpgradeOnBoot: false
            disableAutoUpdate: true
            # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER`
            rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>"
            # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD`
            rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>"
            # 'rhsmOfflineToken' if it was provided red hat systems subscriptions will be removed upon machines deletions, and if wasn't
            # provided the rhsm will be disabled and any created subscription won't be removed automatically
            rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>"
      versions:
        kubelet: 1.9.6

cloud-provider.md

Cloud providers

Digitalocean

machine.spec.providerConfig.cloudProviderSpec

# your digitalocean token
token: "<< YOUR_DO_TOKEN >>"
# droplet region
region: "fra1"
# droplet size
size: "2gb"
# enable backups for the droplet
backups: false
# enable ipv6 for the droplet
ipv6: false- Add operating system config
# enable private networking for the droplet
private_networking: true
# enable monitoring for the droplet
monitoring: true
# add the following tags to the droplet
tags:
- "machine-controller"

AWS

machine.spec.providerConfig.cloudProviderSpec

# your aws access key id
accessKeyId: "<< YOUR_ACCESS_KEY_ID >>"
# your aws secret access key id
secretAccessKey: "<< YOUR_SECRET_ACCESS_KEY_ID >>"
# region for the instance
region: "eu-central-1"
# avaiability zone for the instance
availabilityZone: "eu-central-1a"
# vpc id for the instance
vpcId: "vpc-819f62e9"
# subnet id for the instance
subnetId: "subnet-2bff4f43"
# enable public IP assignment, default is true
assignPublicIP: true
# instance type
instanceType: "t2.micro"
# enable provisioning as spot instance machine, default false
isSpotInstance: false
# size of the root disk in gb
diskSize: 50
# root disk type (gp2, io1, st1, sc1, or standard)
diskType: "gp2"
# IOPS for EBS volumes, required with diskType: io1
diskIops: 500
# enable EBS volume encryption
ebsVolumeEncrypted: false
# optional! the ami id to use. Needs to fit to the specified operating system
ami: ""
# optional! The security group ids for the instance.
# When not set a 'kubernetes-v1' security gruop will get created
securityGroupIDs:
- ""
# name of the instance profile to use.
# When not set a 'kubernetes-v1' instance profile will get created
instanceProfile : ""

# instance tags ("KubernetesCluster": "my-cluster" is a required tag.
# If not set, the kubernetes controller-manager will delete the nodes)
tags:
  "KubernetesCluster": "my-cluster"

Openstack

machine.spec.providerConfig.cloudProviderSpec

# identity endpoint of your openstack installation
identityEndpoint: ""
# your openstack username
username: ""
# your openstack password
password: ""
# the openstack domain
domainName: "default"
# tenant name
tenantName: ""
# image to use (currently only ubuntu & coreos are supported)
image: "Ubuntu 18.04 amd64"
# instance flavor
flavor: ""
# additional security groups.
# a default security group will be created which node-to-node communication
securityGroups:
- "external-ssh"
# the name of the subnet to use
subnet: ""
# [not implemented] the floating ip pool to use. When set a floating ip will be assigned o the instance
floatingIpPool: ""
# the availability zone to create the instance in
availabilityZone: ""
# the region to operate in
region: ""
# the name of the network to use
network: ""
# set trust-device-path flag for kubelet
trustDevicePath: false
# set root disk size
rootDiskSizeGB: 50
# set node-volume-attach-limit flag for cloud-config
nodeVolumeAttachLimit: 20
# the list of tags you would like to attach to the instance
tags:
  tagKey: tagValue

Google Cloud Platform

machine.spec.providerConfig.cloudProviderSpec

serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>"
# See https://cloud.google.com/compute/docs/regions-zones/
zone: "europe-west3-a"
# See https://cloud.google.com/compute/docs/machine-types
machineType: "n1-standard-2"
# See https://cloud.google.com/compute/docs/instances/preemptible
preemptible: false
# In GB
diskSize: 25
# Can be 'pd-standard' or 'pd-ssd'
diskType: "pd-standard"
# The name or self_link of the network and subnetwork to attach this interface to;
# either of both can be provided, otherwise default network will taken
# in case if both empty — default network will be used
network: "my-cool-network"
subnetwork: "my-cool-subnetwork"
# assign a public IP Address. Required for Internet access
assignPublicIPAddress: true
# set node labels
labels:
    "kubernetesCluster": "my-cluster"

Hetzner cloud

machine.spec.providerConfig.cloudProviderSpec

token: "<< HETZNER_API_TOKEN >>"
serverType: "cx11"
datacenter: ""
location: "fsn1"
# Optional: network IDs or names
networks:
  - "<< YOUR_NETWORK >>"
# set node labels
labels:
  "kubernetesCluster": "my-cluster"

Linode

machine.spec.providerConfig.cloudProviderSpec

# your linode token
token: "<< YOUR_LINODE_TOKEN >>"
# linode region
region: "eu-west"
# linode size
type: "g6-standard-2"
# enable backups for the linode
backups: false
# enable private networking for the linode
private_networking: true
# add the following tags to the linode
tags:
- "machine-controller"

Alibaba

machine.spec.providerConfig.cloudProviderSpec

# If empty, can be set via ALIBABA_ACCESS_KEY_ID env var
accessKeyID: "<< YOUR ACCESS ID >>"
accessKeySecret: "<< YOUR ACCESS SECRET >>"
# instance type
instanceType: "ecs.t1.xsmall"
# instance name
instanceName: "alibaba-instance"
# region
regionID: eu-central-1
# image id
imageID: "aliyun_2_1903_64_20G_alibase_20190829.vhd"
# disk type
diskType: "cloud_efficiency"
# disk size in GB
diskSize: "40"
# set an existing vSwitch ID to use, VPC default is used if not set.
vSwitchID:
labels:
  "kubernetesCluster": "my-cluster"

Azure

machine.spec.providerConfig.cloudProviderSpec

# Can also be set via the env var 'AZURE_TENANT_ID' on the machine-controller
tenantID: "<< AZURE_TENANT_ID >>"
# Can also be set via the env var 'AZURE_CLIENT_ID' on the machine-controller
clientID: "<< AZURE_CLIENT_ID >>"
# Can also be set via the env var 'AZURE_CLIENT_SECRET' on the machine-controller
clientSecret: "<< AZURE_CLIENT_SECRET >>"
# Can also be set via the env var 'AZURE_SUBSCRIPTION_ID' on the machine-controller
subscriptionID: "<< AZURE_SUBSCRIPTION_ID >>"
# Azure location
location: "westeurope"
# Azure resource group
resourceGroup: "<< YOUR_RESOURCE_GROUP >>"
# Azure resource group of the vnet
vnetResourceGroup: "<< YOUR_VNET_RESOURCE_GROUP >>"
# Azure availability set
availabilitySet: "<< YOUR AVAILABILITY SET >>"
# VM size
vmSize: "Standard_B1ms"
# optional OS and Data disk size values in GB. If not set, the defaults for the vmSize will be used.
osDiskSize: 30
dataDiskSize: 30
# network name
vnetName: "<< VNET_NAME >>"
# subnet name
subnetName: "<< SUBNET_NAME >>"
# route able name
routeTableName: "<< ROUTE_TABLE_NAME >>"
# assign public IP addresses for nodes, required for Internet access
assignPublicIP: true
# security group
securityGroupName: my-security-group
# node tags
tags:
  "kubernetesCluster": "my-cluster"

KubeVirt

machine.spec.providerConfig.cloudProviderSpec

# kubeconfig to access KubeVirt cluster
kubeconfig: '<< KUBECONFIG >>'
# KubeVirt namespace
namespace: kube-system
# kubernetes storage class
storageClassName: kubermatic-fast
# storage PVC size
pvcSize: "10Gi"
# OS Image URL
sourceURL: http://10.109.79.210/<< OS_NAME >>.img
# instance resources
cpus: "1"
memory: "2048M"

Packet

machine.spec.providerConfig.cloudProviderSpec

# If empty, can be set via PACKET_API_KEY env var
apiKey: "<< PACKET_API_KEY >>"
# instance type
instanceType: "t1.small.x86"
# packet project ID
projectID: "<< PROJECT_ID >>"
# packet facilities
facilities:
  - "ewr1"
# packet billingCycle
billingCycle: ""
# node tags
tags:
  "kubernetesCluster": "my-cluster"

vSphere

Refer to the VSphere specific documentation.

cloudconfig.go

/*
Copyright 2019 The Machine Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package types

import (
	"encoding/json"
	"fmt"
)

type CloudConfig struct {
	Cloud           string `json:"cloud"`
	TenantID        string `json:"tenantId"`
	SubscriptionID  string `json:"subscriptionId"`
	AADClientID     string `json:"aadClientId"`
	AADClientSecret string `json:"aadClientSecret"`

	ResourceGroup              string  `json:"resourceGroup"`
	VNetResourceGroup          string  `json:"vnetResourceGroup"`
	Location                   string  `json:"location"`
	VNetName                   string  `json:"vnetName"`
	SubnetName                 string  `json:"subnetName"`
	RouteTableName             string  `json:"routeTableName"`
	SecurityGroupName          string  `json:"securityGroupName" yaml:"securityGroupName"`
	PrimaryAvailabilitySetName string  `json:"primaryAvailabilitySetName"`
	VnetResourceGroup          *string `json:"vnetResourceGroup,omitempty"`
	UseInstanceMetadata        bool    `json:"useInstanceMetadata"`
}

func CloudConfigToString(c *CloudConfig) (string, error) {
	b, err := json.Marshal(c)
	if err != nil {
		return "", fmt.Errorf("failed to unmarshal config: %v", err)
	}

	return string(b), nil
}

create_delete_resources.go

/*
Copyright 2019 The Machine Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
	"context"
	"fmt"

	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
	"github.com/Azure/go-autorest/autorest/azure/auth"
	"github.com/Azure/go-autorest/autorest/to"

	"k8s.io/apimachinery/pkg/types"
	"k8s.io/klog"
)

// deleteInterfacesByMachineUID will remove all network interfaces tagged with the specific machine's UID.
// The machine has to be deleted or disassociated with the interfaces beforehand, since Azure won't allow
// us to remove interfaces connected to a VM.
func deleteInterfacesByMachineUID(ctx context.Context, c *config, machineUID types.UID) error {
	ifClient, err := getInterfacesClient(c)
	if err != nil {
		return fmt.Errorf("failed to create interfaces client: %v", err)
	}

	list, err := ifClient.List(ctx, c.ResourceGroup)
	if err != nil {
		return fmt.Errorf("failed to list interfaces in resource group %q", c.ResourceGroup)
	}

	var allInterfaces []network.Interface

	for list.NotDone() {
		allInterfaces = append(allInterfaces, list.Values()...)
		if err = list.Next(); err != nil {
			return fmt.Errorf("failed to iterate the result list: %s", err)
		}
	}

	for _, iface := range allInterfaces {
		if iface.Tags != nil && iface.Tags[machineUIDTag] != nil && *iface.Tags[machineUIDTag] == string(machineUID) {
			future, err := ifClient.Delete(ctx, c.ResourceGroup, *iface.Name)
			if err != nil {
				return err
			}

			if err = future.WaitForCompletionRef(ctx, ifClient.Client); err != nil {
				return err
			}
		}
	}

	return nil
}

// deleteIPAddressesByMachineUID will remove public IP addresses tagged with the specific machine's UID.
// Their respective network interfaces have to be deleted or disassociated with the IPs beforehand, since
// Azure won't allow us to remove IPs connected to NICs.
func deleteIPAddressesByMachineUID(ctx context.Context, c *config, machineUID types.UID) error {
	ipClient, err := getIPClient(c)
	if err != nil {
		return fmt.Errorf("failed to create IP addresses client: %v", err)
	}

	list, err := ipClient.List(ctx, c.ResourceGroup)
	if err != nil {
		return fmt.Errorf("failed to list public IP addresses in resource group %q", c.ResourceGroup)
	}

	var allIPs []network.PublicIPAddress

	for list.NotDone() {
		allIPs = append(allIPs, list.Values()...)
		if err = list.Next(); err != nil {
			return fmt.Errorf("failed to iterate the result list: %s", err)
		}
	}

	for _, ip := range allIPs {
		if ip.Tags != nil && ip.Tags[machineUIDTag] != nil && *ip.Tags[machineUIDTag] == string(machineUID) {
			future, err := ipClient.Delete(ctx, c.ResourceGroup, *ip.Name)
			if err != nil {
				return err
			}

			if err = future.WaitForCompletionRef(ctx, ipClient.Client); err != nil {
				return err
			}
		}
	}

	return nil
}

func deleteVMsByMachineUID(ctx context.Context, c *config, machineUID types.UID) error {
	vmClient, err := getVMClient(c)
	if err != nil {
		return err
	}

	list, err := vmClient.ListAll(ctx)
	if err != nil {
		return err
	}

	var allServers []compute.VirtualMachine

	for list.NotDone() {
		allServers = append(allServers, list.Values()...)
		if err = list.Next(); err != nil {
			return fmt.Errorf("failed to iterate the result list: %s", err)
		}
	}

	for _, vm := range allServers {
		if vm.Tags != nil && vm.Tags[machineUIDTag] != nil && *vm.Tags[machineUIDTag] == string(machineUID) {
			future, err := vmClient.Delete(ctx, c.ResourceGroup, *vm.Name)
			if err != nil {
				return err
			}

			if err = future.WaitForCompletionRef(ctx, vmClient.Client); err != nil {
				return err
			}
		}
	}

	return nil
}

func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UID) error {
	disksClient, err := getDisksClient(c)
	if err != nil {
		return fmt.Errorf("failed to get disks client: %v", err)
	}

	matchingDisks, err := getDisksByMachineUID(ctx, disksClient, c, machineUID)
	if err != nil {
		return err
	}

	for _, disk := range matchingDisks {
		future, err := disksClient.Delete(ctx, c.ResourceGroup, *disk.Name)
		if err != nil {
			return fmt.Errorf("failed to delete disk %s: %v", *disk.Name, err)
		}

		if err = future.WaitForCompletionRef(ctx, disksClient.Client); err != nil {
			return fmt.Errorf("failed to wait for deletion of disk %s: %v", *disk.Name, err)
		}
	}

	return nil
}

func getDisksByMachineUID(ctx context.Context, disksClient *compute.DisksClient, c *config, UID types.UID) ([]compute.Disk, error) {

	list, err := disksClient.List(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to list disks: %v", err)
	}

	var allDisks, matchingDisks []compute.Disk
	for list.NotDone() {
		allDisks = append(allDisks, list.Values()...)
		if err = list.Next(); err != nil {
			return nil, fmt.Errorf("failed to iterate the result list: %s", err)
		}
	}

	for _, disk := range allDisks {
		if disk.Tags != nil && disk.Tags[machineUIDTag] != nil && *disk.Tags[machineUIDTag] == string(UID) {
			matchingDisks = append(matchingDisks, disk)
		}
	}

	return matchingDisks, nil
}

func createOrUpdatePublicIPAddress(ctx context.Context, ipName string, machineUID types.UID, c *config) (*network.PublicIPAddress, error) {
	klog.Infof("Creating public IP %q", ipName)
	ipClient, err := getIPClient(c)
	if err != nil {
		return nil, err
	}

	ipParams := network.PublicIPAddress{
		Name:     to.StringPtr(ipName),
		Location: to.StringPtr(c.Location),
		PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
			PublicIPAddressVersion:   network.IPv4,
			PublicIPAllocationMethod: network.Static,
		},
		Tags:  map[string]*string{machineUIDTag: to.StringPtr(string(machineUID))},
		Zones: &c.Zones,
	}
	future, err := ipClient.CreateOrUpdate(ctx, c.ResourceGroup, ipName, ipParams)
	if err != nil {
		return nil, fmt.Errorf("failed to create public IP address: %v", err)
	}

	err = future.WaitForCompletionRef(ctx, ipClient.Client)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve public IP address creation result: %v", err)
	}

	if _, err = future.Result(*ipClient); err != nil {
		return nil, fmt.Errorf("failed to create public IP address: %v", err)
	}

	klog.Infof("Fetching info for IP address %q", ipName)
	ip, err := getPublicIPAddress(ctx, ipName, c.ResourceGroup, ipClient)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch info about public IP %q: %v", ipName, err)
	}

	return ip, nil
}

func getPublicIPAddress(ctx context.Context, ipName string, resourceGroup string, ipClient *network.PublicIPAddressesClient) (*network.PublicIPAddress, error) {
	ip, err := ipClient.Get(ctx, resourceGroup, ipName, "")
	if err != nil {
		return nil, err
	}

	return &ip, nil
}

func getSubnet(ctx context.Context, c *config) (network.Subnet, error) {
	subnetsClient, err := getSubnetsClient(c)
	if err != nil {
		return network.Subnet{}, fmt.Errorf("failed to create subnets client: %v", err)
	}

	return subnetsClient.Get(ctx, c.VNetResourceGroup, c.VNetName, c.SubnetName, "")
}

func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, error) {
	virtualNetworksClient, err := getVirtualNetworksClient(c)
	if err != nil {
		return network.VirtualNetwork{}, err
	}

	return virtualNetworksClient.Get(ctx, c.VNetResourceGroup, c.VNetName, "")
}

func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP *network.PublicIPAddress) (*network.Interface, error) {
	ifClient, err := getInterfacesClient(config)
	if err != nil {
		return nil, fmt.Errorf("failed to create interfaces client: %v", err)
	}

	subnet, err := getSubnet(ctx, config)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch subnet: %v", err)
	}

	ifSpec := network.Interface{
		Name:     to.StringPtr(ifName),
		Location: &config.Location,
		InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
			IPConfigurations: &[]network.InterfaceIPConfiguration{
				{
					Name: to.StringPtr("ip-config-1"),
					InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
						Subnet:                    &subnet,
						PrivateIPAllocationMethod: network.Dynamic,
						PublicIPAddress:           publicIP,
					},
				},
			},
		},
		Tags: map[string]*string{machineUIDTag: to.StringPtr(string(machineUID))},
	}
	if config.SecurityGroupName != "" {
		authorizer, err := auth.NewClientCredentialsConfig(config.ClientID, config.ClientSecret, config.TenantID).Authorizer()
		if err != nil {
			return nil, fmt.Errorf("failed to create authorizer for security groups: %v", err)
		}
		secGroupClient := network.NewSecurityGroupsClient(config.SubscriptionID)
		secGroupClient.Authorizer = authorizer
		secGroup, err := secGroupClient.Get(ctx, config.ResourceGroup, config.SecurityGroupName, "")
		if err != nil {
			return nil, fmt.Errorf("failed to get securityGroup %q: %v", config.SecurityGroupName, err)
		}
		ifSpec.NetworkSecurityGroup = &secGroup
	}
	klog.Infof("Creating/Updating public network interface %q", ifName)
	future, err := ifClient.CreateOrUpdate(ctx, config.ResourceGroup, ifName, ifSpec)
	if err != nil {
		return nil, fmt.Errorf("failed to create interface: %v", err)
	}

	err = future.WaitForCompletionRef(ctx, ifClient.Client)
	if err != nil {
		return nil, fmt.Errorf("failed to get interface creation response: %v", err)
	}

	_, err = future.Result(*ifClient)
	if err != nil {
		return nil, fmt.Errorf("failed to get interface creation result: %v", err)
	}

	klog.Infof("Fetching info about network interface %q", ifName)
	iface, err := ifClient.Get(ctx, config.ResourceGroup, ifName, "")
	if err != nil {
		return nil, fmt.Errorf("failed to fetch info about interface %q: %v", ifName, err)
	}

	return &iface, nil
}

provider.go

/*
Copyright 2019 The Machine Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
	"github.com/Azure/go-autorest/autorest/to"

	"github.com/kubermatic/machine-controller/pkg/apis/cluster/common"
	"github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1"
	"github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh"
	cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors"
	"github.com/kubermatic/machine-controller/pkg/cloudprovider/instance"
	azuretypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/azure/types"
	cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types"
	"github.com/kubermatic/machine-controller/pkg/cloudprovider/util"
	kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes"
	"github.com/kubermatic/machine-controller/pkg/providerconfig"
	providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/klog"
	"k8s.io/utils/pointer"
)

const (
	machineUIDTag = "Machine-UID"

	finalizerPublicIP = "kubermatic.io/cleanup-azure-public-ip"
	finalizerNIC      = "kubermatic.io/cleanup-azure-nic"
	finalizerDisks    = "kubermatic.io/cleanup-azure-disks"
	finalizerVM       = "kubermatic.io/cleanup-azure-vm"
)

const (
	envClientID       = "AZURE_CLIENT_ID"
	envClientSecret   = "AZURE_CLIENT_SECRET"
	envTenantID       = "AZURE_TENANT_ID"
	envSubscriptionID = "AZURE_SUBSCRIPTION_ID"
)

type provider struct {
	configVarResolver *providerconfig.ConfigVarResolver
}

type config struct {
	SubscriptionID string
	TenantID       string
	ClientID       string
	ClientSecret   string

	Location          string
	ResourceGroup     string
	VNetResourceGroup string
	VMSize            string
	VNetName          string
	SubnetName        string
	RouteTableName    string
	AvailabilitySet   string
	SecurityGroupName string
	ImageID           string
	Zones             []string
	ImagePlan         *compute.Plan

	OSDiskSize   int32
	DataDiskSize int32

	AssignPublicIP bool
	Tags           map[string]string
}

type azureVM struct {
	vm          *compute.VirtualMachine
	ipAddresses map[string]v1.NodeAddressType
	status      instance.Status
}

func (vm *azureVM) Addresses() map[string]v1.NodeAddressType {
	return vm.ipAddresses
}

func (vm *azureVM) ID() string {
	return *vm.vm.ID
}

func (vm *azureVM) Name() string {
	return *vm.vm.Name
}

func (vm *azureVM) Status() instance.Status {
	return vm.status
}

var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageReference{
	providerconfigtypes.OperatingSystemCoreos: {
		Publisher: to.StringPtr("CoreOS"),
		Offer:     to.StringPtr("CoreOS"),
		Sku:       to.StringPtr("Stable"),
		Version:   to.StringPtr("latest"),
	},
	providerconfigtypes.OperatingSystemCentOS: {
		Publisher: to.StringPtr("OpenLogic"),
		Offer:     to.StringPtr("CentOS"),
		Sku:       to.StringPtr("7-CI"), // https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init
		Version:   to.StringPtr("latest"),
	},
	providerconfigtypes.OperatingSystemUbuntu: {
		Publisher: to.StringPtr("Canonical"),
		Offer:     to.StringPtr("UbuntuServer"),
		// FIXME We'd like to use Ubuntu 18.04 eventually, but the docker's release
		// deb repo for `bionic` is empty, and we use `$RELEASE` in userdata.
		// Either Docker needs to fix their repo, or we need to hardcode `xenial`.
		Sku:     to.StringPtr("18.04-LTS"),
		Version: to.StringPtr("latest"),
	},
	providerconfigtypes.OperatingSystemRHEL: {
		Publisher: to.StringPtr("RedHat"),
		Offer:     to.StringPtr("RHEL"),
		Sku:       to.StringPtr("7-RAW-CI"),
		Version:   to.StringPtr("7.7.2019081601"),
	},
	providerconfigtypes.OperatingSystemFlatcar: {
		Publisher: to.StringPtr("kinvolk"),
		Offer:     to.StringPtr("flatcar-container-linux"),
		Sku:       to.StringPtr("stable"),
		Version:   to.StringPtr("2345.3.0"),
	},
}

var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{
	providerconfigtypes.OperatingSystemFlatcar: {
		Name:      pointer.StringPtr("stable"),
		Publisher: pointer.StringPtr("kinvolk"),
		Product:   pointer.StringPtr("flatcar-container-linux"),
	},
}

func getOSImageReference(imageID string, os providerconfigtypes.OperatingSystem) (*compute.ImageReference, error) {
	if imageID != "" {
		return &compute.ImageReference{
			ID: to.StringPtr(imageID),
		}, nil
	}

	ref, supported := imageReferences[os]
	if !supported {
		return nil, fmt.Errorf("operating system %q not supported", os)
	}

	return &ref, nil
}

// New returns a digitalocean provider
func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider {
	return &provider{configVarResolver: configVarResolver}
}

func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*config, *providerconfigtypes.Config, error) {
	if s.Value == nil {
		return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil")
	}
	pconfig := providerconfigtypes.Config{}
	err := json.Unmarshal(s.Value.Raw, &pconfig)
	if err != nil {
		return nil, nil, err
	}
	rawCfg := azuretypes.RawConfig{}
	err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawCfg)
	if err != nil {
		return nil, nil, err
	}

	c := config{}
	c.SubscriptionID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.SubscriptionID, envSubscriptionID)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"subscriptionID\" field, error = %v", err)
	}

	c.TenantID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.TenantID, envTenantID)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"tenantID\" field, error = %v", err)
	}

	c.ClientID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.ClientID, envClientID)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"clientID\" field, error = %v", err)
	}

	c.ClientSecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.ClientSecret, envClientSecret)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"clientSecret\" field, error = %v", err)
	}

	c.ResourceGroup, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.ResourceGroup)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"resourceGroup\" field, error = %v", err)
	}

    c.VNetResourceGroup, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VNetResourceGroup)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"VNetResourceGroup\" field, error = %v", err)
	}

	c.Location, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.Location)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"location\" field, error = %v", err)
	}

	c.VMSize, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VMSize)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"vmSize\" field, error = %v", err)
	}

	c.VNetName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VNetName)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"vnetName\" field, error = %v", err)
	}

	c.SubnetName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.SubnetName)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"subnetName\" field, error = %v", err)
	}

	c.RouteTableName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.RouteTableName)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"routeTableName\" field, error = %v", err)
	}

	c.AssignPublicIP, err = p.configVarResolver.GetConfigVarBoolValue(rawCfg.AssignPublicIP)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"assignPublicIP\" field, error = %v", err)
	}

	c.AvailabilitySet, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.AvailabilitySet)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"availabilitySet\" field, error = %v", err)
	}

	c.SecurityGroupName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.SecurityGroupName)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get the value of \"securityGroupName\" field, error = %v", err)
	}

	c.Zones = rawCfg.Zones
	c.Tags = rawCfg.Tags
	c.OSDiskSize = rawCfg.OSDiskSize
	c.DataDiskSize = rawCfg.DataDiskSize

	if rawCfg.ImagePlan != nil {
		c.ImagePlan = &compute.Plan{
			Name:      pointer.StringPtr(rawCfg.ImagePlan.Name),
			Publisher: pointer.StringPtr(rawCfg.ImagePlan.Publisher),
			Product:   pointer.StringPtr(rawCfg.ImagePlan.Product),
		}
	}

	c.ImageID, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.ImageID)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get image id: %v", err)
	}

	return &c, &pconfig, nil
}

func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine) (map[string]v1.NodeAddressType, error) {
	var (
		ipAddresses = map[string]v1.NodeAddressType{}
		err         error
	)

	if vm.VirtualMachineProperties == nil {
		return nil, fmt.Errorf("machine is missing properties")
	}

	if vm.VirtualMachineProperties.NetworkProfile == nil {
		return nil, fmt.Errorf("machine has no network profile")
	}

	if vm.NetworkProfile.NetworkInterfaces == nil {
		return nil, fmt.Errorf("machine has no network interfaces data")
	}

	for n, iface := range *vm.NetworkProfile.NetworkInterfaces {
		if iface.ID == nil || len(*iface.ID) == 0 {
			return nil, fmt.Errorf("interface %d has no ID", n)
		}

		splitIfaceID := strings.Split(*iface.ID, "/")
		ifaceName := splitIfaceID[len(splitIfaceID)-1]
		ipAddresses, err = getNICIPAddresses(ctx, c, ifaceName)
		if vm.NetworkProfile.NetworkInterfaces == nil {
			return nil, fmt.Errorf("failed to get addresses for interface %q: %v", ifaceName, err)
		}
	}

	return ipAddresses, nil
}

func getNICIPAddresses(ctx context.Context, c *config, ifaceName string) (map[string]v1.NodeAddressType, error) {
	ifClient, err := getInterfacesClient(c)
	if err != nil {
		return nil, fmt.Errorf("failed to create interfaces client: %v", err)
	}

	netIf, err := ifClient.Get(ctx, c.ResourceGroup, ifaceName, "")
	if err != nil {
		return nil, fmt.Errorf("failed to get interface %q: %v", ifaceName, err.Error())
	}

	ipAddresses := map[string]v1.NodeAddressType{}

	if netIf.IPConfigurations != nil {
		for _, conf := range *netIf.IPConfigurations {
			var name string
			if conf.Name != nil {
				name = *conf.Name
			} else {
				klog.Warningf("IP configuration of NIC %q was returned with no name, trying to dissect the ID.", ifaceName)
				if conf.ID == nil || len(*conf.ID) == 0 {
					return nil, fmt.Errorf("IP configuration of NIC %q was returned with no ID", ifaceName)
				}
				splitConfID := strings.Split(*conf.ID, "/")
				name = splitConfID[len(splitConfID)-1]
			}

			if c.AssignPublicIP {
				publicIPName := ifaceName + "-pubip"
				publicIPs, err := getIPAddressStrings(ctx, c, publicIPName)
				if err != nil {
					return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %v", name, err)
				}
				for _, ip := range publicIPs {
					ipAddresses[ip] = v1.NodeExternalIP
				}
			}

			internalIPs, err := getInternalIPAddresses(ctx, c, ifaceName, name)
			if err != nil {
				return nil, fmt.Errorf("failed to retrieve internal IP string for IP %q: %v", name, err)
			}
			for _, ip := range internalIPs {
				ipAddresses[ip] = v1.NodeInternalIP
			}

		}
	}

	return ipAddresses, nil
}

func getIPAddressStrings(ctx context.Context, c *config, addrName string) ([]string, error) {
	ipClient, err := getIPClient(c)
	if err != nil {
		return nil, fmt.Errorf("failed to create IP address client: %v", err)
	}

	ip, err := ipClient.Get(ctx, c.ResourceGroup, addrName, "")
	if err != nil {
		return nil, fmt.Errorf("failed to get IP %q: %v", addrName, err)
	}

	if ip.IPConfiguration == nil {
		return nil, fmt.Errorf("IP %q has nil IPConfiguration", addrName)
	}

	var ipAddresses []string
	if ip.IPAddress != nil {
		ipAddresses = append(ipAddresses, *ip.IPAddress)
	}

	return ipAddresses, nil
}

func getInternalIPAddresses(ctx context.Context, c *config, inetface, ipconfigName string) ([]string, error) {
	var ipAddresses []string
	ipConfigClient, err := getIPConfigClient(c)
	if err != nil {
		return nil, fmt.Errorf("failed to create IP config client: %v", err)
	}

	internalIP, err := ipConfigClient.Get(ctx, c.ResourceGroup, inetface, ipconfigName)
	if err != nil {
		return nil, fmt.Errorf("failed to get IP config %q: %v", inetface, err)
	}

	if internalIP.ID == nil {
		return nil, fmt.Errorf("private IP %q has nil IPConfiguration", inetface)
	}
	if internalIP.PrivateIPAddress != nil {
		ipAddresses = append(ipAddresses, *internalIP.PrivateIPAddress)
	}

	return ipAddresses, nil
}

func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) {
	return spec, nil
}

func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) (*compute.StorageProfile, error) {
	osRef, err := getOSImageReference(config.ImageID, providerCfg.OperatingSystem)
	if err != nil {
		return nil, fmt.Errorf("failed to get OSImageReference: %v", err)
	}
	// initial default storage profile, this will use the VMSize default storage profile
	sp := &compute.StorageProfile{
		ImageReference: osRef,
	}
	if config.OSDiskSize != 0 {
		sp.OsDisk = &compute.OSDisk{
			DiskSizeGB:   pointer.Int32Ptr(config.OSDiskSize),
			CreateOption: compute.DiskCreateOptionTypesFromImage,
		}
	}

	if config.DataDiskSize != 0 {
		sp.DataDisks = &[]compute.DataDisk{
			{
				// this should be in range 0-63 and should be unique per datadisk, since we have only one datadisk, this should be fine
				Lun:          new(int32),
				DiskSizeGB:   pointer.Int32Ptr(config.DataDiskSize),
				CreateOption: compute.DiskCreateOptionTypesEmpty,
			},
		}
	}
	return sp, nil
}

func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) {
	config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec)
	if err != nil {
		return nil, cloudprovidererrors.TerminalError{
			Reason:  common.InvalidConfigurationMachineError,
			Message: fmt.Sprintf("failed to parse MachineSpec, due to %v", err),
		}
	}

	vmClient, err := getVMClient(config)
	if err != nil {
		return nil, fmt.Errorf("failed to create VM client: %v", err)
	}

	// We genete a random SSH key, since Azure won't let us create a VM without an SSH key or a password
	key, err := ssh.NewKey()
	if err != nil {
		return nil, fmt.Errorf("failed to generate ssh key: %v", err)
	}

	ifaceName := machine.Name + "-netiface"
	publicIPName := ifaceName + "-pubip"
	var publicIP *network.PublicIPAddress
	if config.AssignPublicIP {
		if err = data.Update(machine, func(updatedMachine *v1alpha1.Machine) {
			if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerPublicIP) {
				updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerPublicIP)
			}
		}); err != nil {
			return nil, err
		}
		publicIP, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPName, machine.UID, config)
		if err != nil {
			return nil, fmt.Errorf("failed to create public IP: %v", err)
		}
	}

	if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) {
		if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerNIC) {
			updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerNIC)
		}
	}); err != nil {
		return nil, err
	}
	iface, err := createOrUpdateNetworkInterface(context.TODO(), ifaceName, machine.UID, config, publicIP)
	if err != nil {
		return nil, fmt.Errorf("failed to generate main network interface: %v", err)
	}

	tags := make(map[string]*string, len(config.Tags)+1)
	for k, v := range config.Tags {
		tags[k] = to.StringPtr(v)
	}
	tags[machineUIDTag] = to.StringPtr(string(machine.UID))

	osPlane := osPlans[providerCfg.OperatingSystem]
	if config.ImagePlan != nil {
		osPlane = config.ImagePlan
	}

	adminUserName := getOSUsername(providerCfg.OperatingSystem)
	storageProfile, err := getStorageProfile(config, providerCfg)
	if err != nil {
		return nil, fmt.Errorf("failed to get StorageProfile: %v", err)
	}
	vmSpec := compute.VirtualMachine{
		Location: &config.Location,
		Plan:     osPlane,
		VirtualMachineProperties: &compute.VirtualMachineProperties{
			HardwareProfile: &compute.HardwareProfile{VMSize: compute.VirtualMachineSizeTypes(config.VMSize)},
			NetworkProfile: &compute.NetworkProfile{
				NetworkInterfaces: &[]compute.NetworkInterfaceReference{
					{
						ID:                                  iface.ID,
						NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{Primary: to.BoolPtr(true)},
					},
				},
			},
			OsProfile: &compute.OSProfile{
				AdminUsername: to.StringPtr(adminUserName),
				ComputerName:  &machine.Name,
				LinuxConfiguration: &compute.LinuxConfiguration{
					DisablePasswordAuthentication: to.BoolPtr(true),
					SSH: &compute.SSHConfiguration{
						PublicKeys: &[]compute.SSHPublicKey{
							{
								Path:    to.StringPtr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", adminUserName)),
								KeyData: &key.PublicKey,
							},
						},
					},
				},
				CustomData: to.StringPtr(base64.StdEncoding.EncodeToString([]byte(userdata))),
			},
			StorageProfile: storageProfile,
		},
		Tags:  tags,
		Zones: &config.Zones,
	}

	if config.AvailabilitySet != "" {
		// Azure expects the full path to the resource
		asURI := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/%s", config.SubscriptionID, config.ResourceGroup, config.AvailabilitySet)
		vmSpec.VirtualMachineProperties.AvailabilitySet = &compute.SubResource{ID: to.StringPtr(asURI)}
	}

	klog.Infof("Creating machine %q", machine.Name)
	if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) {
		if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerDisks) {
			updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerDisks)
		}
		if !kuberneteshelper.HasFinalizer(machine, finalizerVM) {
			updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerVM)
		}
	}); err != nil {
		return nil, err
	}

	future, err := vmClient.CreateOrUpdate(context.TODO(), config.ResourceGroup, machine.Name, vmSpec)
	if err != nil {
		return nil, fmt.Errorf("trying to create a VM: %v", err)
	}

	err = future.WaitForCompletionRef(context.TODO(), vmClient.Client)
	if err != nil {
		return nil, fmt.Errorf("waiting for operation returned: %v", err.Error())
	}

	vm, err := future.Result(*vmClient)
	if err != nil {
		return nil, fmt.Errorf("decoding result: %v", err.Error())
	}

	// get the actual VM object filled in with additional data
	vm, err = vmClient.Get(context.TODO(), config.ResourceGroup, machine.Name, "")
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve updated data for VM %q: %v", machine.Name, err)
	}

	ipAddresses, err := getVMIPAddresses(context.TODO(), config, &vm)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve IP addresses for VM %q: %v", machine.Name, err.Error())
	}

	status, err := getVMStatus(context.TODO(), config, machine.Name)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve status for VM %q: %v", machine.Name, err.Error())
	}

	return &azureVM{vm: &vm, ipAddresses: ipAddresses, status: status}, nil
}

func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) {
	config, _, err := p.getConfig(machine.Spec.ProviderSpec)
	if err != nil {
		return false, fmt.Errorf("failed to parse MachineSpec: %v", err)
	}

	_, err = p.get(machine)
	// If a defunct VM got created, the `Get` call returns an error - But not because the request
	// failed but because the VM has an invalid config hence always delete except on err == cloudprovidererrors.ErrInstanceNotFound
	if err != nil {
		if err == cloudprovidererrors.ErrInstanceNotFound {
			return util.RemoveFinalizerOnInstanceNotFound(finalizerVM, machine, data)
		}
		return false, err
	}

	klog.Infof("deleting VM %q", machine.Name)
	if err = deleteVMsByMachineUID(context.TODO(), config, machine.UID); err != nil {
		return false, fmt.Errorf("failed to delete instance for  machine %q: %v", machine.Name, err)
	}

	if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) {
		updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerVM)
	}); err != nil {
		return false, err
	}

	klog.Infof("deleting disks of VM %q", machine.Name)
	if err := deleteDisksByMachineUID(context.TODO(), config, machine.UID); err != nil {
		return false, fmt.Errorf("failed to remove disks of machine %q: %v", machine.Name, err)
	}
	if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) {
		updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerDisks)
	}); err != nil {
		return false, err
	}

	klog.Infof("deleting network interfaces of VM %q", machine.Name)
	if err := deleteInterfacesByMachineUID(context.TODO(), config, machine.UID); err != nil {
		return false, fmt.Errorf("failed to remove network interfaces of machine %q: %v", machine.Name, err)
	}
	if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) {
		updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerNIC)
	}); err != nil {
		return false, err
	}

	klog.Infof("deleting public IP addresses of VM %q", machine.Name)
	if err := deleteIPAddressesByMachineUID(context.TODO(), config, machine.UID); err != nil {
		return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %v", machine.Name, err)
	}
	if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) {
		updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerPublicIP)
	}); err != nil {
		return false, err
	}

	return true, nil
}

func getVMByUID(ctx context.Context, c *config, uid types.UID) (*compute.VirtualMachine, error) {
	vmClient, err := getVMClient(c)
	if err != nil {
		return nil, err
	}

	list, err := vmClient.ListAll(ctx)
	if err != nil {
		return nil, err
	}

	var allServers []compute.VirtualMachine

	for list.NotDone() {
		allServers = append(allServers, list.Values()...)
		if err := list.Next(); err != nil {
			return nil, fmt.Errorf("failed to iterate the result list: %s", err)
		}
	}

	for _, vm := range allServers {
		if vm.Tags != nil && vm.Tags[machineUIDTag] != nil && *vm.Tags[machineUIDTag] == string(uid) {
			return &vm, nil
		}
	}

	return nil, cloudprovidererrors.ErrInstanceNotFound
}

func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status, error) {
	vmClient, err := getVMClient(c)
	if err != nil {
		return instance.StatusUnknown, err
	}

	iv, err := vmClient.InstanceView(ctx, c.ResourceGroup, vmName)
	if err != nil {
		return instance.StatusUnknown, fmt.Errorf("failed to get instance view for machine %q: %v", vmName, err)
	}

	if iv.Statuses == nil {
		return instance.StatusUnknown, nil
	}

	// it seems that this field should contain two entries: a provisioning status and a power status
	if len(*iv.Statuses) < 2 {
		provisioningStatus := (*iv.Statuses)[0]
		if provisioningStatus.Code == nil {
			klog.Warningf("azure provisioning status has missing code")
			return instance.StatusUnknown, nil
		}

		switch *provisioningStatus.Code {
		case "":
			return instance.StatusUnknown, nil
		case "ProvisioningState/deleting":
			return instance.StatusDeleting, nil
		default:
			klog.Warningf("unknown Azure provisioning status %q", *provisioningStatus.Code)
			return instance.StatusUnknown, nil
		}
	}

	// the second field is supposed to be the power status
	// https://docs.microsoft.com/en-us/azure/virtual-machines/windows/tutorial-manage-vm#vm-power-states
	powerStatus := (*iv.Statuses)[1]
	if powerStatus.Code == nil {
		klog.Warningf("azure power status has missing code")
		return instance.StatusUnknown, nil
	}

	switch *powerStatus.Code {
	case "":
		return instance.StatusUnknown, nil
	case "PowerState/running":
		return instance.StatusRunning, nil
	case "PowerState/starting":
		return instance.StatusCreating, nil
	default:
		klog.Warningf("unknown Azure power status %q", *powerStatus.Code)
		return instance.StatusUnknown, nil
	}
}

func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) {
	return p.get(machine)
}

func (p *provider) get(machine *v1alpha1.Machine) (*azureVM, error) {
	config, _, err := p.getConfig(machine.Spec.ProviderSpec)
	if err != nil {
		return nil, fmt.Errorf("failed to parse MachineSpec: %v", err)
	}

	vm, err := getVMByUID(context.TODO(), config, machine.UID)
	if err != nil {
		if err == cloudprovidererrors.ErrInstanceNotFound {
			return nil, cloudprovidererrors.ErrInstanceNotFound
		}

		return nil, fmt.Errorf("failed to find machine %q by its UID: %v", machine.UID, err)
	}

	ipAddresses, err := getVMIPAddresses(context.TODO(), config, vm)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve IP addresses for VM %v: %v", vm.Name, err)
	}

	status, err := getVMStatus(context.TODO(), config, machine.Name)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve status for VM %v: %v", vm.Name, err)
	}

	return &azureVM{vm: vm, ipAddresses: ipAddresses, status: status}, nil
}

func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) {
	c, _, err := p.getConfig(spec.ProviderSpec)
	if err != nil {
		return "", "", fmt.Errorf("failed to parse config: %v", err)
	}

	cc := &azuretypes.CloudConfig{
		Cloud:                      "AZUREPUBLICCLOUD",
		TenantID:                   c.TenantID,
		SubscriptionID:             c.SubscriptionID,
		AADClientID:                c.ClientID,
		AADClientSecret:            c.ClientSecret,
		ResourceGroup:              c.ResourceGroup,
		VNetResourceGroup:          c.VNetResourceGroup,
		Location:                   c.Location,
		VNetName:                   c.VNetName,
		SubnetName:                 c.SubnetName,
		RouteTableName:             c.RouteTableName,
		PrimaryAvailabilitySetName: c.AvailabilitySet,
		SecurityGroupName:          c.SecurityGroupName,
		UseInstanceMetadata:        true,
	}

	s, err := azuretypes.CloudConfigToString(cc)
	if err != nil {
		return "", "", fmt.Errorf("failed to convert cloud-config to string: %v", err)
	}

	return s, "azure", nil
}

func (p *provider) Validate(spec v1alpha1.MachineSpec) error {
	c, providerCfg, err := p.getConfig(spec.ProviderSpec)
	if err != nil {
		return fmt.Errorf("failed to parse config: %v", err)
	}

	if c.SubscriptionID == "" {
		return errors.New("subscriptionID is missing")
	}

	if c.TenantID == "" {
		return errors.New("tenantID is missing")
	}

	if c.ClientID == "" {
		return errors.New("clientID is missing")
	}

	if c.ClientSecret == "" {
		return errors.New("clientSecret is missing")
	}

	if c.ResourceGroup == "" {
		return errors.New("resourceGroup is missing")
	}

	if c.VNetResourceGroup == "" {
	    c.VNetResourceGroup = c.ResourceGroup
	}

	if c.VNetResourceGroup == "" {
		return errors.New("resourceGroup is missing")
	}

	if c.VMSize == "" {
		return errors.New("vmSize is missing")
	}

	if c.VNetName == "" {
		return errors.New("vnetName is missing")
	}

	if c.SubnetName == "" {
		return errors.New("subnetName is missing")
	}

	vmClient, err := getVMClient(c)
	if err != nil {
		return fmt.Errorf("failed to (create) vm client: %v", err.Error())
	}

	_, err = vmClient.ListAll(context.TODO())
	if err != nil {
		return fmt.Errorf("failed to list all: %v", err.Error())
	}

	if _, err := getVirtualNetwork(context.TODO(), c); err != nil {
		return fmt.Errorf("failed to get virtual network: %v", err)
	}

	if _, err := getSubnet(context.TODO(), c); err != nil {
		return fmt.Errorf("failed to get subnet: %v", err)
	}

	_, err = getOSImageReference(c.ImageID, providerCfg.OperatingSystem)
	return err
}

func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	config, _, err := p.getConfig(machine.Spec.ProviderSpec)
	if err != nil {
		return cloudprovidererrors.TerminalError{
			Reason:  common.InvalidConfigurationMachineError,
			Message: fmt.Sprintf("failed to parse MachineSpec, due to %v", err),
		}
	}

	vmClient, err := getVMClient(config)
	if err != nil {
		return fmt.Errorf("failed to create VM client: %v", err)
	}

	ifaceName := machine.Name + "-netiface"
	publicIPName := ifaceName + "-pubip"
	var publicIP *network.PublicIPAddress

	if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) {
		_, err = createOrUpdatePublicIPAddress(ctx, publicIPName, new, config)
		if err != nil {
			return fmt.Errorf("failed to update UID on public IP: %v", err)
		}
	}

	if kuberneteshelper.HasFinalizer(machine, finalizerNIC) {
		_, err = createOrUpdateNetworkInterface(ctx, ifaceName, new, config, publicIP)
		if err != nil {
			return fmt.Errorf("failed to update UID on main network interface: %v", err)
		}
	}

	if kuberneteshelper.HasFinalizer(machine, finalizerDisks) {
		disksClient, err := getDisksClient(config)
		if err != nil {
			return fmt.Errorf("failed to get disks client: %v", err)
		}

		disks, err := getDisksByMachineUID(ctx, disksClient, config, machine.UID)
		if err != nil {
			return fmt.Errorf("failed to get disks: %v", err)
		}

		for _, disk := range disks {
			disk.Tags[machineUIDTag] = to.StringPtr(string(new))
			future, err := disksClient.CreateOrUpdate(ctx, config.ResourceGroup, *disk.Name, disk)
			if err != nil {
				return fmt.Errorf("failed to update UID for disk %s: %v", *disk.Name, err)
			}
			if err := future.WaitForCompletionRef(ctx, disksClient.Client); err != nil {
				return fmt.Errorf("failed waiting for completion of update UID operation for disk %s: %v", *disk.Name, err)
			}
		}
	}

	tags := map[string]*string{}
	for k, v := range config.Tags {
		tags[k] = to.StringPtr(v)
	}
	tags[machineUIDTag] = to.StringPtr(string(new))

	vmSpec := compute.VirtualMachine{Location: &config.Location, Tags: tags}
	future, err := vmClient.CreateOrUpdate(ctx, config.ResourceGroup, machine.Name, vmSpec)
	if err != nil {
		return fmt.Errorf("failed to update UID of the instance: %v", err)
	}

	if err := future.WaitForCompletionRef(ctx, vmClient.Client); err != nil {
		return fmt.Errorf("error waiting for instance to have the updated UID: %v", err)
	}

	return nil
}

func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) {
	labels := make(map[string]string)

	c, _, err := p.getConfig(machine.Spec.ProviderSpec)
	if err == nil {
		labels["size"] = c.VMSize
		labels["location"] = c.Location
	}

	return labels, err
}

func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error {
	return nil
}

func getOSUsername(os providerconfigtypes.OperatingSystem) string {
	switch os {
	case providerconfigtypes.OperatingSystemFlatcar:
		return "core"
	case providerconfigtypes.OperatingSystemCoreos:
		return "core"
	default:
		return string(os)
	}
}

types.go

/*
Copyright 2019 The Machine Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package types

import (
	providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types"
)

// RawConfig is a direct representation of an Azure machine object's configuration
type RawConfig struct {
	SubscriptionID providerconfigtypes.ConfigVarString `json:"subscriptionID,omitempty"`
	TenantID       providerconfigtypes.ConfigVarString `json:"tenantID,omitempty"`
	ClientID       providerconfigtypes.ConfigVarString `json:"clientID,omitempty"`
	ClientSecret   providerconfigtypes.ConfigVarString `json:"clientSecret,omitempty"`

	Location          providerconfigtypes.ConfigVarString `json:"location"`
	ResourceGroup     providerconfigtypes.ConfigVarString `json:"resourceGroup"`
	VNetResourceGroup providerconfigtypes.ConfigVarString `json:"vnetResourceGroup"`
	VMSize            providerconfigtypes.ConfigVarString `json:"vmSize"`
	VNetName          providerconfigtypes.ConfigVarString `json:"vnetName"`
	SubnetName        providerconfigtypes.ConfigVarString `json:"subnetName"`
	RouteTableName    providerconfigtypes.ConfigVarString `json:"routeTableName"`
	AvailabilitySet   providerconfigtypes.ConfigVarString `json:"availabilitySet"`
	SecurityGroupName providerconfigtypes.ConfigVarString `json:"securityGroupName"`
	Zones             []string                            `json:"zones"`
	ImagePlan         *ImagePlan                          `json:"imagePlan"`

	ImageID        providerconfigtypes.ConfigVarString `json:"imageID"`
	OSDiskSize     int32                               `json:"osDiskSize"`
	DataDiskSize   int32                               `json:"dataDiskSize"`
	AssignPublicIP providerconfigtypes.ConfigVarBool   `json:"assignPublicIP"`
	Tags           map[string]string                   `json:"tags,omitempty"`
}

// ImagePlan contains azure OS Plan fields for the marketplace images
type ImagePlan struct {
	Name      string `json:"name,omitempty"`
	Publisher string `json:"publisher,omitempty"`
	Product   string `json:"product,omitempty"`
}