find shell deps in a file or filesystem tree

README.md

find shell deps

The idea here is just to "parse" files to

1. see if they are shell
2. find the external commands/utilities that they use

A more advanced version of this would help identify dependencies.

there are bugs in the fs traversal and I'm not finding everything :-(.

example

d=$(mktemp -d); 
crane export cgr.dev/chainguard/jre | tar -C $d -xvf -
go run find-shell-deps.go $d

find-shell-deps.go

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var Shells = map[string]bool{"bash": true, "dash": true, "sh": true, "ash": true}

var Builtins = map[string]bool{
	"[": true, "[[": true, "echo": true, "printf": true, "set": true,
}

var AllCommands = []string{
	"add-shell", "addgroup", "adduser", "adjtimex", "arch", "arping", "ash", "awk",
	"base64", "basename", "bbconfig", "bc", "beep", "bunzip2", "bzcat", "bzip2",
	"cal", "cat", "chattr", "chgrp", "chmod", "chown", "chpasswd", "chroot",
	"chrt", "cksum", "clear", "cmp", "comm", "cp", "cpio", "cryptpw", "cut",
	"date", "dc", "dd", "delgroup", "deluser", "df", "diff", "dirname", "dmesg",
	"dnsdomainname", "dos2unix", "du", "echo", "ed", "egrep", "env", "expand",
	"expr", "factor", "fallocate", "false", "fgrep", "find", "findfs", "flock",
	"fold", "free", "fsync", "fuser", "getopt", "getty", "grep", "groups",
	"gunzip", "gzip", "hd", "head", "hexdump", "hostid", "hostname", "id",
	"inotifyd", "install", "ionice", "iostat", "ipcrm", "ipcs", "kill", "killall",
	"killall5", "less", "link", "linux32", "linux64", "ln", "logger", "login",
	"ls", "lsattr", "lsof", "lzcat", "lzma", "lzop", "lzopcat", "md5sum",
	"microcom", "mkdir", "mkfifo", "mknod", "mkpasswd", "mktemp", "more", "mount",
	"mountpoint", "mpstat", "mv", "nc", "netcat", "netstat", "nice", "nl",
	"nmeter", "nohup", "nologin", "nproc", "nsenter", "od", "passwd", "paste",
	"pgrep", "pidof", "ping", "ping6", "pipe_progress", "pivot_root", "pkill",
	"pmap", "printenv", "printf", "ps", "pstree", "pwd", "pwdx", "rdev",
	"readahead", "readlink", "realpath", "remove-shell", "renice", "reset",
	"resize", "rev", "rm", "rmdir", "run-parts", "sed", "seq", "setpriv",
	"setserial", "setsid", "sh", "sha1sum", "sha256sum", "sha3sum", "sha512sum",
	"shred", "shuf", "sleep", "sort", "split", "stat", "strings", "stty", "su",
	"sum", "sync", "sysctl", "tac", "tail", "tar", "tee", "telnet", "telnetd",
	"test", "time", "timeout", "top", "touch", "tr", "traceroute", "traceroute6",
	"tree", "true", "truncate", "tsort", "tty", "ttysize", "tunctl", "umount",
	"uname", "unexpand", "uniq", "unix2dos", "unlink", "unlzma", "unlzop", "unxz",
	"unzip", "uptime", "usleep", "uudecode", "uuencode", "vconfig", "vi", "vlock",
	"watch", "wc", "wget", "which", "who", "whoami", "xargs", "xxd", "xzcat",
	"yes", "zcat",
}

/*
	if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}

		if !strings.HasPrefix(path, "usr/bin/") && !strings.HasPrefix(path, "bin/") {
			return nil
		}

		if d.Type().IsDir() {
			return nil
		}

		if fp, err := fsys.Open(path); err == nil {
			shbang, err := getShbang(fp)
			if err != nil {
				log.Warnf("Error reading shbang from %s: %v", path, err)
			} else if shbang != "" {
				cmds[filepath.Base(shbang)] = path
			}
			fp.Close()
		} else {
			log.Infof("Failed to open %s: %v", path, err)
		}
		return nil
	}); err != nil {
		return err
	}
*/

func getShbang(fp fs.File) (string, error) {
	// python3 and sh are symlinks and generateCmdProviders currently only considers
	// regular files. Since nothing will fulfill such a depend, do not generate one.
	buf := make([]byte, 80)
	blen, err := io.ReadFull(fp, buf)
	if err == io.EOF {
		return "", nil
	} else if err == io.ErrUnexpectedEOF {
		if blen < 2 {
			return "", nil
		}
	} else if err != nil {
		return "", err
	}

	if !bytes.HasPrefix(buf, []byte("#!")) {
		return "", nil
	}

	toks := strings.Fields(string(buf[2 : blen-2]))
	bin := toks[0]

	// if #! is '/usr/bin/env foo', then use next arg as the dep
	if bin == "/usr/bin/env" {
		if len(toks) == 1 {
			return "", fmt.Errorf("a shbang of only '/usr/bin/env'")
		} else if len(toks) == 2 {
			bin = toks[1]
		} else if len(toks) >= 3 && toks[1] == "-S" && !strings.HasPrefix(toks[2], "-") {
			// we really need a env argument parser to figure out what the next cmd is.
			// special case handle /usr/bin/env -S prog [arg1 [arg2 [...]]]
			bin = toks[2]
		} else {
			return "", fmt.Errorf("a shbang of only '/usr/bin/env' with multiple arguments (%d %s)", len(toks), strings.Join(toks, " "))
		}
	}

	return filepath.Base(bin), nil
}

func isShell(fpath string) (bool, error) {
	for _, suff := range []string{".sh", ".bash"} {
		if strings.HasSuffix(fpath, suff) {
			return true, nil
		}
	}
	fp, err := os.Open(fpath)
	defer fp.Close()
	if err != nil {
		return false, err
	}

	bin, err := getShbang(fp)
	if err != nil {
		return false, err
	}

	return Shells[bin], nil
}

func addCommands(cmds map[string]bool, words []string) {
	for _, c := range words {
		if !Builtins[c] {
			cmds[c] = true
		}
	}
}

func findCommands(fpath string, commands map[string]bool) ([]string, error) {
	contentB, err := os.ReadFile(fpath)
	if err != nil {
		return []string{}, err
	}

	found := map[string]bool{}
	for _, tok := range strings.Fields(string(contentB)) {
		if found[tok] {
			continue
		}
		if strings.Contains(tok, "/") {
			tok = filepath.Base(tok)
		}
		if commands[tok] {
			found[tok] = true
		}
	}

	keys := make([]string, len(found))
	i := 0
	for k := range found {
		keys[i] = k
		i++
	}

	sort.Strings(keys)
	return keys, nil
}

func findShellFiles(path string) ([]string, error) {
	shellFiles := []string{}
	file, err := os.Open(path)
	if err != nil {
		return shellFiles, err
	}
	fInfo, err := file.Stat()
	if err != nil {
		return shellFiles, err
	}
	if !fInfo.IsDir() {
		shell, err := isShell(path)
		if shell && err == nil {
			return []string{path}, nil
		}
		return shellFiles, err
	}

	myFs := os.DirFS(path)

	myErr := fs.WalkDir(myFs, ".", func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return nil
		}
		if d.IsDir() {
			return nil
		}
		fp := filepath.Join(path, p)
		shell, err := isShell(fp)
		fmt.Printf(":: %t %s\n", err != nil, fp)
		if shell && err == nil {
			fmt.Printf("adding %s\n", fp)
			shellFiles = append(shellFiles, fp)
		}
		return err
	})

	fmt.Printf("len(shellFiels)=%d\n", len(shellFiles))
	return shellFiles, myErr
}

func main() {
	cmdlistFile := flag.String("words-file", "", "file with list of words to consider commands")
	commands := map[string]bool{}

	flag.Parse()
	words := AllCommands
	if *cmdlistFile != "" {
		content, err := os.ReadFile(*cmdlistFile)
		if err != nil {
			panic(fmt.Sprintf("Failed to open %s", *cmdlistFile))
		}
		words = strings.Fields(string(content))
	}

	addCommands(commands, words)

	shFiles := []string{}
	for _, p := range flag.Args() {
		found, err := findShellFiles(p)
		fmt.Printf("ok, found %d in %s err=%v\n", len(found), p, err)
		if err != nil {
			shFiles = append(shFiles, found...)
		}
	}

	fmt.Printf("found %d shell files from %d arguments\n", len(shFiles), len(flag.Args()))
	for _, shFile := range shFiles {
		found, err := findCommands(shFile, commands)
		if err != nil {
			panic(fmt.Sprintf("bogus: %s: %v", shFile, err))
		}
		fmt.Printf("%s: %v\n", shFile, found)
	}
}