Skip to content

Instantly share code, notes, and snippets.

@tzaffi
Created February 3, 2022 19:11
Show Gist options
  • Save tzaffi/6a56dba837a048955073a89caef10417 to your computer and use it in GitHub Desktop.
Save tzaffi/6a56dba837a048955073a89caef10417 to your computer and use it in GitHub Desktop.
Using AST's in Go together with git submodules to Detect Incoming Structs
package future
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"sort"
"strings"
"testing"
)
type missing struct {
shouldMiss bool
msg string
}
func (hm *missing) handleMissing(isMissing bool) error {
if hm.shouldMiss != isMissing {
return fmt.Errorf(hm.msg)
}
return nil
}
type expectation struct {
File string
MissingFile missing
Name string
MissingStruct missing
Fields []string
MissingFields missing
}
func getProblematicExpectations(t *testing.T, expectations []expectation) (problems [][]error) {
problems = [][]error{}
for i, expected := range expectations {
fmt.Printf(`
____________________________________________
%d. Expectations for file %s
Expecting: %#v`, i+1, expected.File, expected)
prob := []error{}
fset := token.NewFileSet()
root, err := parser.ParseFile(fset, expected.File, nil, parser.ParseComments)
missingFile := false
if err != nil {
fmt.Printf("\nHeads up! ParseFile failed on [%v]", err)
missingFile = true
}
fileExpectationError := expected.MissingFile.handleMissing(missingFile)
if fileExpectationError != nil {
prob = append(prob, fileExpectationError)
}
if missingFile {
problems = append(problems, prob)
continue
}
fileStructs := getFileStructs(root)
fmt.Printf("\nStructs found in %s: %s", expected.File, fileStructs)
foundStruct, ok := fileStructs[expected.Name]
structExpectationError := expected.MissingStruct.handleMissing(!ok)
if structExpectationError != nil {
prob = append(prob, structExpectationError)
}
if !ok {
problems = append(problems, prob)
continue
}
missingSomeField := false
for _, field := range expected.Fields {
_, ok := foundStruct[field]
if !ok {
missingSomeField = true
break
}
}
fieldsExpectationError := expected.MissingFields.handleMissing(missingSomeField)
if fieldsExpectationError != nil {
prob = append(prob, fieldsExpectationError)
}
problems = append(problems, prob)
}
fmt.Printf("\nAll Problems: %+v", problems)
return
}
func getTopLevelTypeNodes(root *ast.File) (typeSpecs []*ast.TypeSpec) {
for _, decl := range root.Decls {
if decl == nil {
continue
}
gDecl, ok := decl.(*ast.GenDecl)
if !ok || gDecl == nil {
continue
}
for _, spec := range gDecl.Specs {
tSpec, ok := spec.(*ast.TypeSpec)
if !ok || tSpec == nil {
continue
}
typeSpecs = append(typeSpecs, tSpec)
}
}
return
}
// first elmt of stct is the struct's name, the rest are fields
func getStructFieldNames(tSpec *ast.TypeSpec) (stct []string) {
if tSpec == nil || tSpec.Type == nil {
return
}
stct = append(stct, tSpec.Name.Name)
sType, ok := tSpec.Type.(*ast.StructType)
if !ok {
return
}
if sType.Fields == nil || sType.Fields.List == nil {
return
}
for _, field := range sType.Fields.List {
if field == nil || field.Names == nil {
continue
}
for _, name := range field.Names {
stct = append(stct, name.Name)
}
}
return
}
type fStructs map[string]map[string]bool
func (fs fStructs) String() string {
parts := make([]string, len(fs))
for stct, fset := range fs {
fields := []string{}
for field := range fset {
fields = append(fields, field)
}
sort.Strings(fields)
parts = append(parts, fmt.Sprintf("%s: %s", stct, fields))
}
return strings.Join(parts, "\n")
}
// returns map from struct's name, to its set of field names
func getFileStructs(root *ast.File) fStructs {
fileStructs := map[string]map[string]bool{}
for _, tSpec := range getTopLevelTypeNodes(root) {
structInfo := getStructFieldNames(tSpec)
if len(structInfo) > 0 {
fields := map[string]bool{}
for _, field := range structInfo[1:] {
fields[field] = true
}
fileStructs[structInfo[0]] = fields
}
}
return fileStructs
}
package future
import (
"testing"
"github.com/stretchr/testify/assert"
)
var expectations = []expectation{
{
// we should NOT see the file third_party/go-algorand/ledger/ledgercore/accountdata.go:
"../third_party/go-algorand/ledger/ledgercore/accountdata.go",
missing{shouldMiss: true, msg: "\n!!!path ledger/ledgercore/accountdata.go detected ==> unlimited assets has arrived\n"},
// AND struct AccountBaseData should not be available either:
"AccountBaseData",
missing{shouldMiss: true, msg: "\n!!!struct AccountBaseData is detected ==> unlimited assets has arrived\n"},
// AND finally, AccountBaseData.TotalAssets also should not exist:
[]string{"TotalAssets"},
missing{shouldMiss: true, msg: "\n!!!field TotalAssets is detected ==> unlimited assets has arrived\n"},
},
}
func TestTheFuture(t *testing.T) {
problems := getProblematicExpectations(t, expectations)
for i, prob := range problems {
for _, err := range prob {
assert.NoError(t, err, "error in expectation #%d (%s):\n%v", i+1, expectations[i].File, err)
}
}
}
package future
type msg struct {
Subject string
Body []byte
}
package future
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var fixtureExpectations = []expectation{
// missing DNE.go and happy about that
{
"./DNE.go", missing{shouldMiss: true, msg: "Not ready to handle DNE.go"},
"msg", missing{shouldMiss: true, msg: "Wow, we already have the struct msg"},
[]string{"Subject", "Body"}, missing{shouldMiss: false, msg: "why don't we have Subject and Body"},
},
// missing DNE.go and un-happy about that
{
"./DNE.go", missing{shouldMiss: false, msg: "I really need DNE.go"},
"ReallyImportStruct", missing{shouldMiss: false, msg: "I'm missing the ReallyImportStruct"},
[]string{"field1", "field2"}, missing{shouldMiss: false, msg: "why don't we have field1 and field2"},
},
// have fixture.go and everything else, so super happy
{
"./fixture.go", missing{shouldMiss: false, msg: "should have fixture.go"},
"msg", missing{shouldMiss: false, msg: "why don't we have struct msg?"},
[]string{"Subject", "Body"}, missing{shouldMiss: false, msg: "why don't we have a Subjeee and Body"},
},
// have fixture.go, and un-happy about that only because fields messed up
{
"./fixture.go", missing{shouldMiss: false, msg: "should have fixture.go"},
"msg", missing{shouldMiss: false, msg: "why don't we have struct msg?"},
[]string{"Subjeee", "Body"}, missing{shouldMiss: false, msg: "why don't we have a Subjeee and Body"},
},
// have fixture.go, and un-happy about the fact that struct is missing
{
"./fixture.go", missing{shouldMiss: false, msg: "should have fixture.go"},
"notMSG", missing{shouldMiss: false, msg: "why don't we have struct notMSG?"},
[]string{"Subject", "Body"}, missing{shouldMiss: false, msg: "why don't we have a Subjeee and Body"},
},
// have fixture.go, and un-happy about that and everything else
{
"./fixture.go", missing{shouldMiss: true, msg: "Not ready to handle fixture.go"},
"msg", missing{shouldMiss: true, msg: "Wow, we already have the struct msg"},
[]string{"Subject", "Body"}, missing{shouldMiss: true, msg: "...and having Subject and Body makes life super difficult"},
},
// have fixture.go, and un-happy! But other stuff is just fine
{
"./fixture.go", missing{shouldMiss: true, msg: "Not ready to handle fixture.go"},
"msg", missing{shouldMiss: false, msg: "Might as well have msg"},
[]string{"Subject", "Body"}, missing{shouldMiss: false, msg: "...and having Subject and Body is fine too"},
},
}
func TestFixture(t *testing.T) {
problems := getProblematicExpectations(t, fixtureExpectations)
require.Equal(t, 7, len(problems))
require.Equal(t, 0, len(problems[0]))
require.Equal(t, 1, len(problems[1]))
require.Equal(t, 0, len(problems[2]))
require.Equal(t, 1, len(problems[3]))
require.Equal(t, 1, len(problems[4]))
require.Equal(t, 3, len(problems[5]))
require.Equal(t, 1, len(problems[6]))
}
// TestForLint -as the name suggests- is a function meant to get `make lint` to pass
func TestForLint(t *testing.T) {
m := msg{"hello", nil}
assert.Equal(t, "hello", m.Subject)
}
# includes indexer-v-algod (cf. my json_diff gist)
nightly-setup:
cd third_party/go-algorand && git pull origin master
nightly-teardown:
git submodule update
indexer-v-algod-swagger:
pytest -sv parity
future-nosync:
go test -run "^(TestTheFuture|TestFixture)$$" github.com/algorand/indexer/future
indexer-v-algod: nightly-setup indexer-v-algod-swagger nightly-teardown
future: nightly-setup future-nosync nightly-teardown
nightly: nightly-setup indexer-v-algod-swagger future-nosync nightly-teardown
.PHONY: indexer-v-algod future
@tzaffi
Copy link
Author

tzaffi commented Feb 3, 2022

Notable

  • ast_expectations.go
    • type expectation struct - together with type missing struct and the handleMissing() method, allows defining what should be expected in a go project and an indifidual script's AST
    • getFileStructs() parses out all the structs inside an AST and outputs a map from struct name to set of struct fields
  • ast_expectations_test.go - example usage
  • Makefile
    • make future
      1. gets the latest from go-algorand repo in the submodule
      2. runs the tests
      3. restores the go-algorand submodule to its former state

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment