Skip to content

Instantly share code, notes, and snippets.

@AlexanderBrevig
Created November 27, 2019 05:04
Show Gist options
  • Save AlexanderBrevig/aa96f36a40610f3c71cca6266c9579d5 to your computer and use it in GitHub Desktop.
Save AlexanderBrevig/aa96f36a40610f3c71cca6266c9579d5 to your computer and use it in GitHub Desktop.
Gitscan aggregate to TAB separated stream on stdout
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"path/filepath"
"strings"
)
func main() {
folders, err := ioutil.ReadDir("./")
if err != nil {
log.Fatal(err)
}
allTeams := uint32(0)
totalLeaks := uint32(0)
totalUnique := uint32(0)
for _, teamName := range folders {
if teamName.IsDir() {
allTeams += 1
totalTeamLeaks := uint32(0)
totalTeamUnique := uint32(0)
files, listerr := ioutil.ReadDir("./" + teamName.Name())
if listerr != nil {
log.Fatal(err)
}
for _, reportFile := range files {
filename := teamName.Name() + "/" + reportFile.Name()
filePath := "./" + filename
content, err := ioutil.ReadFile(filePath)
if err != nil {
log.Fatal(err)
}
strcontent := string(content)
if len(strcontent) > 0 {
switch filepath.Ext(reportFile.Name()) {
case ".txt":
unique := make(map[string]uint32)
lines := strings.Split(strcontent, "\n")
for _, line := range lines {
parts := strings.Split(line, ":")
line = strings.Join(parts[1:], ":")
unique[line]++
}
// because a few of the secrets are made with custom grep, they store to different files
// lets unwrap and simply remember the type of secret that is leaked
source := ""
if strings.Contains(reportFile.Name(), "artifactory") {
source = "artifactory"
} else {
source = "client_secret"
}
repoName := strings.ReplaceAll(reportFile.Name(), "_artifactory.txt", "")
repoName = strings.ReplaceAll(repoName, "_client_secret.txt", "")
repoName = strings.ReplaceAll(repoName, ".gitty", "")
for k, v := range unique {
if len(k) > 0 {
fmt.Printf("%s\t%s\t%s\t%s\t%s\t%d\n", teamName.Name(), repoName, source, reportFile.Name(), k, v)
totalTeamLeaks += v
totalTeamUnique += 1
}
}
break
case ".json":
var jsonData []map[string]interface{}
json.Unmarshal([]byte(strcontent), &jsonData)
unique := make(map[string]uint32)
for item := range jsonData {
line := fmt.Sprintf("%s\t%s\t%s\t%s\t%s", teamName.Name(), jsonData[item]["repo"], jsonData[item]["rule"], reportFile.Name(), jsonData[item]["offender"])
unique[line]++
}
for k, v := range unique {
if len(k) > 0 {
fmt.Printf("%s\t%d\n", k, v)
totalTeamLeaks += v
totalTeamUnique += 1
}
}
break
case ".gitty":
unique := make(map[string]uint32)
lines := strings.Split(strcontent, "\n")
for _, line := range lines {
if line != "No matches." {
unique[line]++
}
}
repoName := strings.ReplaceAll(reportFile.Name(), "_artifactory.txt", "")
repoName = strings.ReplaceAll(repoName, "_client_secret.txt", "")
repoName = strings.ReplaceAll(repoName, ".gitty", "")
for k, v := range unique {
if len(k) > 0 {
fmt.Printf("%s\t%s\t%s\t%s\t%s\t%d\n", teamName.Name(), repoName, "gitty", reportFile.Name(), k, v)
totalTeamLeaks += v
totalTeamUnique += 1
}
}
break
}
}
}
//fmt.Printf("TEAM: %s\tTOTAL: %d\tUNIQUE: %d\n", teamName.Name(), totalTeamLeaks, totalTeamUnique)
totalLeaks += totalTeamLeaks
totalUnique += totalTeamUnique
}
}
//fmt.Printf("ALL %d TEAMS\tTOTAL: %d\tUNIQUE: %d\n", allTeams, totalLeaks, totalUnique)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment