[Go: recursively walk tree looking for go files and analysing their imports] #go #golang #ast

parse_go.go

package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

func main() {
	// Create a new file set
	fs := token.NewFileSet()

	// Create a map to track imported packages
	importedPackages := make(map[string]bool)

	// Create a slice to store the import paths
	var importPaths []string

	// Walk through the current directory and its subdirectories
	root := "/Users/integralist/Code/fastly/cli" // You can change this to the desired directory path
	err := filepath.Walk(root, func(path string, _ os.FileInfo, _ error) error {
		// Check if the file is a Go source file
		if strings.HasSuffix(path, ".go") {
			// Read the content of the Go file
			goCode, err := os.ReadFile(path)
			if err != nil {
				log.Printf("Error reading file %s: %v", path, err)
				return nil
			}

			// Parse the Go code into an AST
			node, err := parser.ParseFile(fs, path, string(goCode), parser.AllErrors)
			if err != nil {
				log.Printf("Error parsing file %s: %v", path, err)
				return nil
			}

			// Extract and store import declarations in the slice
			for _, decl := range node.Decls {
				if gd, isGenDecl := decl.(*ast.GenDecl); isGenDecl && gd.Tok == token.IMPORT {
					for _, spec := range gd.Specs {
						if ispec, isImportSpec := spec.(*ast.ImportSpec); isImportSpec {
							importPath := strings.TrimSpace(ispec.Path.Value)
							if !importedPackages[importPath] {
								importPaths = append(importPaths, importPath)
								importedPackages[importPath] = true
							}
						}
					}
				}
			}
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error walking directory: %v", err)
	}

	// Print the unique import paths
	sort.Strings(importPaths)
	for _, path := range importPaths {
		fmt.Println("Import:", path)
	}
}

parse_js.go

// IMCOMPLETE (started but never finished).
// Two formats to account for (CommonJS and ES Modules).
//
// CommonJS...
//
// const {add, subtract} = require('./util')
// const {
//   add, 
//   subtract
// } = require('./util')
// 
// ES Modules...
//
// import {add, subtract} from './util.mjs'
// import {
//   add, 
//   subtract
// } from './util.mjs' 
// import defaultExport from "module-name";
// import * as name from "module-name";
// import { 
//   export1 
// } from "module-name";
// import { export1 as alias1 } from "module-name";
// import { default as alias } from "module-name";
// import { export1, export2 } from "module-name";
// import { export1, export2 as alias2, /* … */ } from "module-name";
// import { "string name" as alias } from "module-name";
// import defaultExport, { export1, /* … */ } from "module-name";
// import defaultExport, * as name from "module-name";
// import "module-name";

// Variables used as part of parsing imports from JavaScript source files.
var (
	importSingleLineBlockPattern = regexp.MustCompile(`^import (\{ [^;]+);`)
	importAsPattern              = regexp.MustCompile(`as [^\s]+\s*`)
)

// Imports returns all source code imported packages.
func (j *JavaScript) Imports() []string {
	importedPackages := make(map[string]bool)

	var importPaths []string

	root := "."
	_ = filepath.Walk(root, func(path string, _ os.FileInfo, _ error) error {
		if strings.HasSuffix(path, ".js") {
			if strings.Contains(path, "node_modules") {
				return nil
			}

			f, err := os.Open(path)
			if err != nil {
				return nil
			}
			defer f.Close()

			scanner := bufio.NewScanner(f)

			for scanner.Scan() {
				line := scanner.Text()
				match := importSingleLineBlockPattern.FindStringSubmatch(line)
				if len(match) >= 2 {
					item := importAsPattern.ReplaceAllString(match[1], "")
					if !importedPackages[item] {
						importPaths = append(importPaths, item)
						importedPackages[item] = true
					}
				}
			}
		}
		return nil
	})

	sort.Strings(importPaths)
	return importPaths
}

parse_rust.go

/*
Works with...

use wasi_common::I32Exit;
use fastly::http::{header, Method, StatusCode}

use fastly::http::{header, Method, StatusCode};
use fastly::{mime, Error, Request, Response};

use {
    fastly::http::header,
    fastly::http::Method,
    fastly::http::StatusCode,
    fastly::{mime, Error, Request, Response},
};

use {
    fastly::http::header,
    fastly::http::Method,
    fastly::http::StatusCode,
    fastly::{
        mime, Error, Request, Response,
    },
};

use {
    fastly::http::{
      header,
    },
    fastly::http::Method,
    fastly::http::StatusCode,
    fastly::{
        mime, Error, Request, Response,
    },
};

use {
    fastly::http::*,
    fastly::{
        mime, Error, Request, Response,
    },
};
*/

package main

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

var (
	useSinglePattern               = regexp.MustCompile(`^\s*use\s+([^;]+);`)
	useMultilineStartPattern       = regexp.MustCompile(`^\s*use\s+\{$`)
	useMultilineEndPattern         = regexp.MustCompile(`^\s*};$`)
  	useMultilineNestedStartPattern = regexp.MustCompile(`^\s*((\w+::)+)\{$`)
	useMultilineNestedEndPattern   = regexp.MustCompile(`^\s*}$`)
	multilineNested                bool
	multilineNestedPrefix          string
)

func main() {
	importedPackages := make(map[string]bool)

	var importPaths []string

	root := "/Users/integralist/Code/test-projects/testing-fastly-cli"
	_ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if strings.HasSuffix(path, ".rs") {
			if strings.Contains(path, "/target/") {
				return nil
			}
			file, err := os.Open(path)
			if err != nil {
				return nil
			}
			defer file.Close()

			scanner := bufio.NewScanner(file)

			var multilineUse bool

			for scanner.Scan() {
				line := scanner.Text()

				// Parse single `use` declaration
				match := useSinglePattern.FindStringSubmatch(line)
				if len(match) >= 2 {
					usePath := strings.TrimSpace(match[1])
					var cont bool
					importPaths, cont = parseUseDeclarations(usePath, importPaths, importedPackages)
					if cont {
						continue
					}
				}

				// Parse multiline `use` declaration
				if useMultilineStartPattern.MatchString(line) {
					multilineUse = true
					continue
				}
				if useMultilineEndPattern.MatchString(line) {
					multilineUse = false
					continue
				}
				if multilineUse && !useMultilineEndPattern.MatchString(line) {
					usePath := strings.TrimSpace(line)
					usePath = strings.TrimSuffix(line, ",")
					var cont bool
					importPaths, cont = parseUseDeclarations(usePath, importPaths, importedPackages)
					if cont {
						continue // TODO: is this needed?
					}
				}
			}
		}
		return nil
	})

	sort.Strings(importPaths)
	for _, path := range importPaths {
		fmt.Println(path)
	}
}

func parseUseDeclarations(
	usePath string,
	importPaths []string,
	importedPackages map[string]bool,
) ([]string, bool) {
	// Parse a nested multiline crate declaration
	//
	// e.g.
	// use {
	//     fastly::http::header,
	//     fastly::http::Method,
	//     fastly::http::StatusCode,
	//     fastly::{                           <<< this
	//         mime, Error, Request, Response, <<< this
	//     },                                  <<< this
	// };
	match := useMultilineNestedStartPattern.FindStringSubmatch(usePath)
	if len(match) >= 2 {
		multilineNested = true
		multilineNestedPrefix = strings.TrimSpace(match[1])
		return importPaths, true
	}
	if useMultilineNestedEndPattern.MatchString(usePath) {
		multilineNested = false
		multilineNestedPrefix = ""
		return importPaths, true
	}
	if multilineNested && !useMultilineNestedEndPattern.MatchString(usePath) {
		usePath := strings.TrimSpace(usePath)
		usePath = strings.TrimSuffix(usePath, ",")
		for _, v := range strings.Split(usePath, ",") {
			item := fmt.Sprintf("%s%s", multilineNestedPrefix, strings.TrimSpace(v))
			if !importedPackages[item] {
				importPaths = append(importPaths, item)
				importedPackages[item] = true
			}
		}
		return importPaths, true
	}

	// Find the position of the opening and closing curly braces
	openBraceIndex := strings.Index(usePath, "{")
	closeBraceIndex := strings.Index(usePath, "}")

	// Parse `use <path>::{<path>, <path>, <path>}`
	if openBraceIndex != -1 && closeBraceIndex != -1 {
		// Extract the prefix before the opening curly brace
		prefix := usePath[:openBraceIndex]
		// Extract the contents inside the curly braces
		contents := usePath[openBraceIndex+1 : closeBraceIndex]

		for _, item := range strings.Split(contents, ",") {
			item = fmt.Sprintf("%s%s", strings.TrimSpace(prefix), strings.TrimSpace(item))
			if !importedPackages[item] {
				importPaths = append(importPaths, item)
				importedPackages[item] = true
			}
		}
		return importPaths, true
	}

	// Parse `use <path>;`
	usePath = strings.TrimSpace(usePath)
	if !importedPackages[usePath] {
		importPaths = append(importPaths, usePath)
		importedPackages[usePath] = true
	}

	return importPaths, false
}