Summary of google/osv-scanner#397
commit f32a63194dd9f1564d738c7b07ccaaa0e37d8dd8
Author: Dana Sherson <robot@dana.sh>
Date: Fri May 26 17:28:23 2023 +1200
Don't traverse gitignored dirs for gitignore files
this seems to be an upstream issue in go-git, and i'll prepare a PR for
them soon, but for now this copies in the affected method and fixes it
by checking the accumulated patterns while walking the fs looking for
gitignore files
fixes: #389
diff --git a/pkg/osvscanner/gitignore_dir.go b/pkg/osvscanner/gitignore_dir.go
new file mode 100644
index 0000000..23e2a6d
--- /dev/null
+++ b/pkg/osvscanner/gitignore_dir.go
@@ -0,0 +1,84 @@
+// based on https://github.com/go-git/go-git/blob/v5.7.0/plumbing/format/gitignore/dir.go
+// and skips ignored directories while traversing for gitignore files
+
+package osvscanner
+
+import (
+ "bufio"
+ "os"
+ "strings"
+
+ "github.com/go-git/go-billy/v5"
+ "github.com/go-git/go-git/v5/plumbing/format/gitignore"
+)
+
+const (
+ commentPrefix = "#"
+ gitDir = ".git"
+ gitignoreFile = ".gitignore"
+ infoExcludeFile = gitDir + "/info/exclude"
+)
+
+// readIgnoreFile reads a specific git ignore file.
+func readIgnoreFile(fs billy.Filesystem, path []string, ignoreFile string) (ps []gitignore.Pattern, err error) {
+ f, err := fs.Open(fs.Join(append(path, ignoreFile)...))
+ if err == nil {
+ defer f.Close()
+
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ s := scanner.Text()
+ if !strings.HasPrefix(s, commentPrefix) && len(strings.TrimSpace(s)) > 0 {
+ ps = append(ps, gitignore.ParsePattern(s, path))
+ }
+ }
+ } else if !os.IsNotExist(err) {
+ return nil, err
+ }
+
+ return
+}
+
+func readPatterns(fs billy.Filesystem, path []string, accumulatedPs []gitignore.Pattern) (ps []gitignore.Pattern, err error) {
+ ps, _ = readIgnoreFile(fs, path, gitignoreFile)
+
+ var fis []os.FileInfo
+ fis, err = fs.ReadDir(fs.Join(path...))
+ if err != nil {
+ return
+ }
+
+ accumulatedPs = append(accumulatedPs, ps...)
+ matcherForThisDir := gitignore.NewMatcher(accumulatedPs)
+
+ for _, fi := range fis {
+ if fi.IsDir() && fi.Name() != gitDir {
+ childPath := path
+ childPath = append(childPath, fi.Name())
+ if !matcherForThisDir.Match(childPath, fi.IsDir()) {
+ var subps []gitignore.Pattern
+ subps, err = readPatterns(fs, childPath, accumulatedPs)
+ if err != nil {
+ return
+ }
+
+ if len(subps) > 0 {
+ ps = append(ps, subps...)
+ }
+ }
+ }
+ }
+
+ return ps, err
+}
+
+// ReadPatterns reads the .git/info/exclude and then the gitignore patterns
+// recursively traversing through the directory structure. The result is in
+// the ascending order of priority (last higher).
+func ReadPatterns(fs billy.Filesystem, path []string) (ps []gitignore.Pattern, err error) {
+ ps, _ = readIgnoreFile(fs, path, infoExcludeFile)
+ subps, err := readPatterns(fs, path, ps)
+ ps = append(ps, subps...)
+
+ return
+}
diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go
index 586b258..ea9b6aa 100644
--- a/pkg/osvscanner/osvscanner.go
+++ b/pkg/osvscanner/osvscanner.go
@@ -152,7 +152,7 @@ func parseGitIgnores(path string) (*gitIgnoreMatcher, error) {
}
}
- patterns, err := gitignore.ReadPatterns(fs, []string{"."})
+ patterns, err := ReadPatterns(fs, []string{"."})
if err != nil {
return nil, err
}
--- upstreamd-dir.go 2024-01-17 12:51:38
+++ pr-dir.go 2024-01-17 12:51:16
@@ -1,31 +1,27 @@
-// https://github.com/go-git/go-git/blob/v5.7.0/plumbing/format/gitignore/dir.go
-package gitignore
+// FROM: https://github.com/google/osv-scanner/pull/397/files
+// based on https://github.com/go-git/go-git/blob/v5.7.0/plumbing/format/gitignore/dir.go
+// and skips ignored directories while traversing for gitignore files
+package osvscanner
+
import (
"bufio"
- "bytes"
- "io"
"os"
"strings"
"github.com/go-git/go-billy/v5"
- "github.com/go-git/go-git/v5/plumbing/format/config"
- gioutil "github.com/go-git/go-git/v5/utils/ioutil"
+ "github.com/go-git/go-git/v5/plumbing/format/gitignore"
)
const (
commentPrefix = "#"
- coreSection = "core"
- excludesfile = "excludesfile"
gitDir = ".git"
gitignoreFile = ".gitignore"
- gitconfigFile = ".gitconfig"
- systemFile = "/etc/gitconfig"
infoExcludeFile = gitDir + "/info/exclude"
)
// readIgnoreFile reads a specific git ignore file.
-func readIgnoreFile(fs billy.Filesystem, path []string, ignoreFile string) (ps []Pattern, err error) {
+func readIgnoreFile(fs billy.Filesystem, path []string, ignoreFile string) (ps []gitignore.Pattern, err error) {
f, err := fs.Open(fs.Join(append(path, ignoreFile)...))
if err == nil {
defer f.Close()
@@ -34,7 +30,7 @@
for scanner.Scan() {
s := scanner.Text()
if !strings.HasPrefix(s, commentPrefix) && len(strings.TrimSpace(s)) > 0 {
- ps = append(ps, ParsePattern(s, path))
+ ps = append(ps, gitignore.ParsePattern(s, path))
}
}
} else if !os.IsNotExist(err) {
@@ -44,98 +40,46 @@
return
}
-// ReadPatterns reads the .git/info/exclude and then the gitignore patterns
-// recursively traversing through the directory structure. The result is in
-// the ascending order of priority (last higher).
-func ReadPatterns(fs billy.Filesystem, path []string) (ps []Pattern, err error) {
- ps, _ = readIgnoreFile(fs, path, infoExcludeFile)
+func readPatterns(fs billy.Filesystem, path []string, accumulatedPs []gitignore.Pattern) (ps []gitignore.Pattern, err error) {
+ ps, _ = readIgnoreFile(fs, path, gitignoreFile)
- subps, _ := readIgnoreFile(fs, path, gitignoreFile)
- ps = append(ps, subps...)
-
var fis []os.FileInfo
fis, err = fs.ReadDir(fs.Join(path...))
if err != nil {
return
}
+ accumulatedPs = append(accumulatedPs, ps...)
+ matcherForThisDir := gitignore.NewMatcher(accumulatedPs)
+
for _, fi := range fis {
if fi.IsDir() && fi.Name() != gitDir {
- var subps []Pattern
- subps, err = ReadPatterns(fs, append(path, fi.Name()))
- if err != nil {
- return
- }
+ childPath := path
+ childPath = append(childPath, fi.Name())
+ if !matcherForThisDir.Match(childPath, fi.IsDir()) {
+ var subps []gitignore.Pattern
+ subps, err = readPatterns(fs, childPath, accumulatedPs)
+ if err != nil {
+ return
+ }
- if len(subps) > 0 {
- ps = append(ps, subps...)
+ if len(subps) > 0 {
+ ps = append(ps, subps...)
+ }
}
}
}
- return
+ return ps, err
}
-func loadPatterns(fs billy.Filesystem, path string) (ps []Pattern, err error) {
- f, err := fs.Open(path)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, nil
- }
- return nil, err
- }
+// ReadPatterns reads the .git/info/exclude and then the gitignore patterns
+// recursively traversing through the directory structure. The result is in
+// the ascending order of priority (last higher).
+func ReadPatterns(fs billy.Filesystem, path []string) (ps []gitignore.Pattern, err error) {
+ ps, _ = readIgnoreFile(fs, path, infoExcludeFile)
+ subps, err := readPatterns(fs, path, ps)
+ ps = append(ps, subps...)
- defer gioutil.CheckClose(f, &err)
-
- b, err := io.ReadAll(f)
- if err != nil {
- return
- }
-
- d := config.NewDecoder(bytes.NewBuffer(b))
-
- raw := config.New()
- if err = d.Decode(raw); err != nil {
- return
- }
-
- s := raw.Section(coreSection)
- efo := s.Options.Get(excludesfile)
- if efo == "" {
- return nil, nil
- }
-
- ps, err = readIgnoreFile(fs, nil, efo)
- if os.IsNotExist(err) {
- return nil, nil
- }
-
return
}
-
-// LoadGlobalPatterns loads gitignore patterns from from the gitignore file
-// declared in a user's ~/.gitconfig file. If the ~/.gitconfig file does not
-// exist the function will return nil. If the core.excludesfile property
-// is not declared, the function will return nil. If the file pointed to by
-// the core.excludesfile property does not exist, the function will return nil.
-//
-// The function assumes fs is rooted at the root filesystem.
-func LoadGlobalPatterns(fs billy.Filesystem) (ps []Pattern, err error) {
- home, err := os.UserHomeDir()
- if err != nil {
- return
- }
-
- return loadPatterns(fs, fs.Join(home, gitconfigFile))
-}
-
-// LoadSystemPatterns loads gitignore patterns from from the gitignore file
-// declared in a system's /etc/gitconfig file. If the /etc/gitconfig file does
-// not exist the function will return nil. If the core.excludesfile property
-// is not declared, the function will return nil. If the file pointed to by
-// the core.excludesfile property does not exist, the function will return nil.
-//
-// The function assumes fs is rooted at the root filesystem.
-func LoadSystemPatterns(fs billy.Filesystem) (ps []Pattern, err error) {
- return loadPatterns(fs, systemFile)
-}