fileconcat.go

package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"strings"
)

func ConcatenateFiles(directory, pattern, outputFile string, loadToMemory bool) error {
	// Read the directory for files
	files, err := os.ReadDir(directory)
	if err != nil {
		return fmt.Errorf("error reading the directory: %w", err)
	}

	if loadToMemory {
		var filesData [][]byte // to store file data if loadToMemory is true

		// Process each file for memory load
		for _, file := range files {
			name := file.Name()
			if strings.HasPrefix(name, pattern) {
				fmt.Println("Processing and deleting", name)
				filePath := directory + "/" + name

				// Read and store file content in memory
				data, readErr := os.ReadFile(filePath)
				if readErr != nil {
					return fmt.Errorf("error reading file %s: %w", name, readErr)
				}
				filesData = append(filesData, data)

				// Delete the file after reading
				delErr := os.Remove(filePath)
				if delErr != nil {
					return fmt.Errorf("error deleting file %s: %w", name, delErr)
				}
			}
		}

		// Write all data to outputFile at once
		err := os.WriteFile(outputFile, bytes.Join(filesData, nil), 0644)
		if err != nil {
			return fmt.Errorf("error writing to output file: %w", err)
		}
	} else {
		// Process each file for direct file append without loading in memory
		for _, file := range files {
			name := file.Name()
			if strings.HasPrefix(name, pattern) {
				fmt.Println("Processing", name)
				filePath := directory + "/" + name
				inFile, err := os.Open(filePath)
				if err != nil {
					return fmt.Errorf("error opening file %s: %w", name, err)
				}
				defer inFile.Close()

				outFile, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
				if err != nil {
					return fmt.Errorf("error opening the output file: %w", err)
				}
				defer outFile.Close()

				if _, err := io.Copy(outFile, inFile); err != nil {
					return fmt.Errorf("error copying from %s to output file: %w", name, err)
				}
			}
		}
	}

	fmt.Println("Files have been processed successfully.")
	return nil
}

func main() {
	directory := flag.String("dir", ".", "The directory to read files from")
	pattern := flag.String("pattern", "prefix", "The file pattern to match")
	outputFile := flag.String("out", "output.txt", "The file to output the concatenated content to")
	loadToMemory := flag.Bool("mem", false, "Load all files to memory before concatenation")

	flag.Parse()

	// Perform concatenation based on the loadToMemory flag
	if err := ConcatenateFiles(*directory, *pattern, *outputFile, *loadToMemory); err != nil {
		fmt.Println("Error during file processing:", err)
		os.Exit(1)
	}
}