Kubernetes Resource Extractor

Kubernetes Resource Extractor.md

Resource Extractor

This Go application parses the output of a helm template or kubectl kustomize command and extracts resource information, generating a CSV file with the extracted data.

Prerequisites

• Go (version 1.17 or later)
• Helm (version 3 or later) (if using Helm)
• kubectl (if using Kustomize)

Installation

1. Install Go dependencies:

   go mod init metrics
   go mod tidy

2. Save the code to a file:

   Save the provided Go code to a file named resource_extractor.go.

Usage

1. Prepare your source:

   • For Helm: Use helm pull to download a Helm chart. For example, to download the Prometheus chart:

     helm pull prometheus-community/prometheus --untar

   • For Kustomize: Ensure you have a Kustomize directory ready.

2. Run the application:

   Run the application with the source path and type as arguments:

   go run resource_extractor.go --source <path-to-source> --type <helm|kustomize>

   Replace <path-to-source> with the path to your Helm chart or Kustomize directory, and <helm|kustomize> with the appropriate type. For example:

   go run resource_extractor.go --source prometheus --type helm

   The application will parse the output for the specified source and create a CSV file with the extracted resources and volume claim templates. The CSV file will be named <source-name>.csv.

3. Check the generated CSV file:

   After running the application, you should see a CSV file named <source-name>.csv in the current directory. This file contains the extracted resource information.

Example

1. Create a directory for the Helm chart:

   mkdir -p my-charts
   cd my-charts

2. Pull the Prometheus Helm chart:

   helm pull prometheus-community/prometheus --untar

3. Run the application:

   go run ../resource_extractor.go --source prometheus --type helm

4. Check the generated CSV file:

   ls prometheus.csv

Tip: Building the Application

To build the application and place it in a bin directory for easier use:

1. Build the application:

   go build -o bin/resource_extractor resource_extractor.go

2. Add the bin directory to your PATH:

   export PATH=$PATH:$(pwd)/bin

3. Run the application:

   resource_extractor --source prometheus --type helm

   This will allow you to run the application from anywhere without needing to specify the full path to the Go file.

resource_extractor.go

package main

import (
	"bytes"
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/util/yaml"
)

type Resource struct {
	Name          string
	Image         string
	CPURequest    string
	MemoryRequest string
	CPULimit      string
	MemoryLimit   string
}

type VolumeClaimTemplate struct {
	Name      string
	AccessMode string
	Storage   string
}

func main() {
	// Define flags
	sourcePath := flag.String("source", "", "Path to the Helm chart or Kustomize directory")
	inputType := flag.String("type", "helm", "Input type: helm or kustomize")
	flag.Parse()

	// Validate flags
	if *sourcePath == "" {
		log.Fatalf("Usage: %s --source <path> --type <helm|kustomize>", os.Args[0])
	}

	// Run the appropriate command and get the output
	output, err := runCommand(*inputType, *sourcePath)
	if err != nil {
		log.Fatalf("Failed to run %s command: %v", *inputType, err)
	}

	// Parse the YAML output
	decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(output), 4096)
	scheme := runtime.NewScheme()
	appsv1.AddToScheme(scheme)
	corev1.AddToScheme(scheme)
	rbacv1.AddToScheme(scheme)
	codecs := serializer.NewCodecFactory(scheme)
	deserializer := codecs.UniversalDeserializer()

	var resources []Resource
	var volumeClaimTemplates []VolumeClaimTemplate
	var totalCPURequest, totalMemoryRequest, totalCPULimit, totalMemoryLimit int64

	for {
		var rawObj runtime.RawExtension
		if err := decoder.Decode(&rawObj); err != nil {
			break
		}
		obj, _, err := deserializer.Decode(rawObj.Raw, nil, nil)
		if err != nil {
			log.Printf("Failed to decode object: %v", err)
			continue
		}
		switch obj := obj.(type) {
		case *appsv1.Deployment:
			for _, container := range obj.Spec.Template.Spec.Containers {
				resource := Resource{
					Name:          container.Name,
					Image:         container.Image,
					CPURequest:    getResourceString(container.Resources.Requests.Cpu(), "0"),
					MemoryRequest: getResourceString(container.Resources.Requests.Memory(), "0"),
					CPULimit:      getResourceString(container.Resources.Limits.Cpu(), "unlimited"),
					MemoryLimit:   getResourceString(container.Resources.Limits.Memory(), "unlimited"),
				}
				resources = append(resources, resource)
				totalCPURequest += getResourceMilliValue(container.Resources.Requests.Cpu())
				totalMemoryRequest += getResourceBytes(container.Resources.Requests.Memory())
				if resource.CPULimit != "unlimited" {
					totalCPULimit += getResourceMilliValue(container.Resources.Limits.Cpu())
				}
				if resource.MemoryLimit != "unlimited" {
					totalMemoryLimit += getResourceBytes(container.Resources.Limits.Memory())
				}
			}
		case *appsv1.StatefulSet:
			for _, container := range obj.Spec.Template.Spec.Containers {
				resource := Resource{
					Name:          container.Name,
					Image:         container.Image,
					CPURequest:    getResourceString(container.Resources.Requests.Cpu(), "0"),
					MemoryRequest: getResourceString(container.Resources.Requests.Memory(), "0"),
					CPULimit:      getResourceString(container.Resources.Limits.Cpu(), "unlimited"),
					MemoryLimit:   getResourceString(container.Resources.Limits.Memory(), "unlimited"),
				}
				resources = append(resources, resource)
				totalCPURequest += getResourceMilliValue(container.Resources.Requests.Cpu())
				totalMemoryRequest += getResourceBytes(container.Resources.Requests.Memory())
				if resource.CPULimit != "unlimited" {
					totalCPULimit += getResourceMilliValue(container.Resources.Limits.Cpu())
				}
				if resource.MemoryLimit != "unlimited" {
					totalMemoryLimit += getResourceBytes(container.Resources.Limits.Memory())
				}
			}
			for _, vct := range obj.Spec.VolumeClaimTemplates {
				volumeClaimTemplate := VolumeClaimTemplate{
					Name:      vct.Name,
					AccessMode: getAccessModeString(vct.Spec.AccessModes),
					Storage:   getResourceStringFromQuantity(vct.Spec.Resources.Requests[corev1.ResourceStorage], "0"),
				}
				volumeClaimTemplates = append(volumeClaimTemplates, volumeClaimTemplate)
			}
		case *corev1.Pod:
			for _, container := range obj.Spec.Containers {
				resource := Resource{
					Name:          container.Name,
					Image:         container.Image,
					CPURequest:    getResourceString(container.Resources.Requests.Cpu(), "0"),
					MemoryRequest: getResourceString(container.Resources.Requests.Memory(), "0"),
					CPULimit:      getResourceString(container.Resources.Limits.Cpu(), "unlimited"),
					MemoryLimit:   getResourceString(container.Resources.Limits.Memory(), "unlimited"),
				}
				resources = append(resources, resource)
				totalCPURequest += getResourceMilliValue(container.Resources.Requests.Cpu())
				totalMemoryRequest += getResourceBytes(container.Resources.Requests.Memory())
				if resource.CPULimit != "unlimited" {
					totalCPULimit += getResourceMilliValue(container.Resources.Limits.Cpu())
				}
				if resource.MemoryLimit != "unlimited" {
					totalMemoryLimit += getResourceBytes(container.Resources.Limits.Memory())
				}
			}
		}
	}

	// Create the CSV file
	csvFileName := fmt.Sprintf("%s.csv", *sourcePath)
	csvFile, err := os.Create(csvFileName)
	if err != nil {
		log.Fatalf("Failed to create CSV file: %v", err)
	}
	defer csvFile.Close()
	writer := csv.NewWriter(csvFile)
	defer writer.Flush()

	// Write the header
	header := []string{"Name", "Image", "CPURequest (m)", "MemoryRequest (b)", "CPULimit (m)", "MemoryLimit (b)"}
	if err := writer.Write(header); err != nil {
		log.Fatalf("Failed to write header to CSV file: %v", err)
	}

	// Write the resources to the CSV file
	for _, resource := range resources {
		cpuRequest, _ := resource.ParseQuantity(resource.CPURequest)
		memoryRequest, _ := resource.ParseQuantity(resource.MemoryRequest)
		cpuLimit, _ := resource.ParseQuantity(resource.CPULimit)
		memoryLimit, _ := resource.ParseQuantity(resource.MemoryLimit)
		record := []string{
			resource.Name,
			resource.Image,
			strconv.FormatInt(cpuRequest.MilliValue(), 10),
			strconv.FormatInt(memoryRequest.Value(), 10), // Keep bytes as is
			strconv.FormatInt(cpuLimit.MilliValue(), 10),
			strconv.FormatInt(memoryLimit.Value(), 10), // Keep bytes as is
		}
		if err := writer.Write(record); err != nil {
			log.Fatalf("Failed to write record to CSV file: %v", err)
		}
	}

	// Write the totals to the CSV file
	totalRecord := []string{
		"Total", "",
		strconv.FormatInt(totalCPURequest, 10),
		strconv.FormatInt(totalMemoryRequest, 10), // Keep bytes as is
		strconv.FormatInt(totalCPULimit, 10),
		strconv.FormatInt(totalMemoryLimit, 10), // Keep bytes as is
	}
	if err := writer.Write(totalRecord); err != nil {
		log.Fatalf("Failed to write total record to CSV file: %v", err)
	}

	fmt.Printf("CSV file %s created successfully\n", csvFileName)
	fmt.Printf("Total CPU Request: %dm\n", totalCPURequest)
	fmt.Printf("Total Memory Request: %db\n", totalMemoryRequest)
	fmt.Printf("Total CPU Limit: %dm\n", totalCPULimit)
	fmt.Printf("Total Memory Limit: %db\n", totalMemoryLimit)
}

func runCommand(inputType, sourcePath string) ([]byte, error) {
	var cmd *exec.Cmd
	switch inputType {
	case "helm":
		cmd = exec.Command("helm", "template", sourcePath)
	case "kustomize":
		cmd = exec.Command("kubectl", "kustomize", sourcePath)
	default:
		return nil, fmt.Errorf("invalid type: %s. Use 'helm' or 'kustomize'", inputType)
	}

	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return nil, err
	}
	return out.Bytes(), nil
}

func getResourceString(q *resource.Quantity, defaultValue string) string {
	if q != nil && !q.IsZero() {
		return q.String()
	}
	return defaultValue
}

func getResourceStringFromQuantity(q resource.Quantity, defaultValue string) string {
	if !q.IsZero() {
		return q.String()
	}
	return defaultValue
}

func getAccessModeString(modes []corev1.PersistentVolumeAccessMode) string {
	if len(modes) > 0 {
		return string(modes[0])
	}
	return ""
}

func getResourceMilliValue(q *resource.Quantity) int64 {
	if q != nil && !q.IsZero() {
		return q.MilliValue()
	}
	return 0
}

func getResourceBytes(q *resource.Quantity) int64 {
	if q != nil && !q.IsZero() {
		bytes, ok := q.AsInt64()
		if ok {
			return bytes
		}
		log.Printf("Warning: Quantity %s is too large to fit in an int64", q.String())
	}
	return 0
}

func (r *Resource) ParseQuantity(quantityStr string) (resource.Quantity, error) {
	return resource.ParseQuantity(quantityStr)
}