Search / Download Retrieve all secrets from HashiCorp's Vault in Golang

vault-scanner.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"reflect"
	"strings"
	"sync"
	"time"

	vault "github.com/hashicorp/vault/api"
)

const maxConcurrency = 4

func main() {
	logsDir := "."
	logFile, err := setupLoggingFile(logsDir)
	if err != nil {
		log.Fatalf("Failed to set up logging: %v\n", err)
	}
	defer logFile.Close()

	client, err := createVaultClient()
	if err != nil {
		log.Fatalf("Error creating Vault client: %s", err)
	}

	var wg sync.WaitGroup
	limitCh := make(chan struct{}, maxConcurrency)
	done := make(chan struct{})
	secretPathCh := make(chan string, 20000)

	secretPathCh <- "applications/metadata/"
	// secretPathCh <- "secret/devops/metadata/"
	wg.Add(1)
	// wg.Add(2)

	go func() {
		for path := range secretPathCh {
			limitCh <- struct{}{}

			go func(path string) {
				defer wg.Done()
				if strings.HasSuffix(path, "/") {
					log.Printf("Queue length is %d, List secrets at: %s", len(secretPathCh), path)
					ListSecret(client, path, secretPathCh, &wg)
				} else {
					log.Printf("Queue length is %d, Get secret at: %s", len(secretPathCh), path)
					SaveSecretToFile(client, path)
				}

				<-limitCh
			}(path)
		}

		done <- struct{}{}
	}()

	wg.Wait()
	close(secretPathCh)

	<-done
	log.Println("Done")
}

func SaveSecretToFile(client *vault.Client, name string) {
	secret, err := client.Logical().Read(name)
	if err != nil || secret == nil || secret.Data == nil {
		log.Printf("Error reading secrets at path %s: %v, secret: %#v", name, err, secret)
		return
	}

	filename := "output/" + strings.ReplaceAll(name, "/", "__") + ".json"
	err = writeToJsonFile(secret.Data, filename)
	if err != nil {
		log.Fatalf("Failed to write to json file: %s", name)
	}
}

// ListSecret returns a list of secrets from Vault
func ListSecret(vaultCli *vault.Client, path string, secretPathCh chan string, wg *sync.WaitGroup) {
	secretList, err := vaultCli.Logical().List(path)
	if err != nil || isNil(secretList) {
		log.Fatalf("Error listing secret at %s, error:%v, secret: %#v", path, err, secretList)
	}

	secrets, ok := secretList.Data["keys"].([]interface{})
	if !ok {
		log.Fatalf("Secret is not valid at: %s", path)
	}

	// log.Printf("There are %d keys at: %s", len(secrets), path)
	for _, secret := range secrets {
		p, ok := secret.(string)
		if !ok {
			log.Fatalf("Secret is not string at: %s", path)
		}

		wg.Add(1)
		if strings.HasSuffix(p, "/") {
			secretPathCh <- path + p
		} else {
			secretPathCh <- strings.Replace(path, "metadata", "data", -1) + p
		}
	}
}

func createVaultClient() (*vault.Client, error) {
	vaultAddr, vaultToken := os.Getenv("VAULT_ADDR"), os.Getenv("VAULT_TOKEN")
	if vaultAddr == "" || vaultToken == "" {
		log.Fatalf("Please set VAULT_ADDR and VAULT_TOKEN environment variables.")
	}

	config := &vault.Config{
		Address: vaultAddr,
		HttpClient: &http.Client{
			Timeout: time.Second * 10, // adjust as necessary
		},
	}

	client, err := vault.NewClient(config)
	if err != nil {
		return nil, err
	}

	client.SetToken(vaultToken)
	return client, nil
}

func writeToJsonFile(v any, fileName string) error {
	content, err := json.MarshalIndent(v, "", "  ")
	if err != nil {
		return err
	}

	err = os.WriteFile(fileName, content, 0644)
	if err != nil {
		return err
	}
	return nil
}

// setupLogging sets up the default `log` logger to log to both stdout and a log file.
func setupLoggingFile(logsDir string) (*os.File, error) {
	logFileName := time.Now().Format("2006-01-02T15-04-05") + ".log"
	logFile, err := os.OpenFile(path.Join(logsDir, logFileName), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		return nil, fmt.Errorf("error creating log file '%s': %v", logFileName, err)
	}

	logWriter := io.MultiWriter(os.Stdout, logFile)
	log.SetOutput(logWriter)
	return logFile, nil
}

func isNil(v interface{}) bool {
	return v == nil || (reflect.ValueOf(v).Kind() == reflect.Ptr && reflect.ValueOf(v).IsNil())
}