Created
December 3, 2020 15:26
-
-
Save cuducos/0ee2d20134a21fc63f1ab87c00eb7883 to your computer and use it in GitHub Desktop.
validate.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Package schema uses Frictionless Data's table schema to create the JSON | |
// files describing the data expected in the compressed CSV files. | |
package schema | |
import ( | |
"fmt" | |
"log" | |
"os" | |
"sync" | |
"github.com/frictionlessdata/tableschema-go/csv" | |
"github.com/frictionlessdata/tableschema-go/schema" | |
) | |
func validate(c chan<- result, wg *sync.WaitGroup, r resource) { | |
defer wg.Done() | |
result := result{resource: r} | |
t, err := csv.NewTable(sourceFrom(r.csv()), csv.LoadHeaders()) | |
if err != nil { | |
result.err = err | |
c <- result | |
return | |
} | |
s, err := schema.LoadFromFile(r.schema()) | |
if err != nil { | |
panic(err) | |
// result.err = err | |
// c <- result | |
// return | |
} | |
iter, err := t.Iter() | |
if err != nil { | |
result.err = err | |
c <- result | |
return | |
} | |
defer iter.Close() | |
var l int | |
result.valid = true | |
for { | |
if !iter.Next() { | |
break | |
} | |
l++ | |
row := iter.Row() | |
for i, f := range s.Fields { | |
_, err = f.Cast(row[i]) | |
if err != nil { | |
result.valid = false | |
c <- result | |
log.Output(2, fmt.Sprintf("Error:%s:row %d:col %d: %s (%s expected to be %s)", r.csv(), l, i, err, f.Name, f.Type)) | |
} | |
} | |
} | |
result.valid = true | |
c <- result | |
} | |
// Validate uses the JSON files to valide the compressed CSV files. | |
func Validate(d string) { | |
c := make(chan result) | |
var wg sync.WaitGroup | |
for _, n := range names { | |
wg.Add(1) | |
go validate(c, &wg, resource{n, d}) | |
} | |
go func(wg *sync.WaitGroup) { | |
wg.Wait() | |
close(c) | |
}(&wg) | |
v := true | |
for r := range c { | |
if r.err != nil { | |
log.Output(2, fmt.Sprintf("Error while validating schema for %s:\n%s", r.resource.csv(), r.err)) | |
v = false | |
} | |
} | |
if v { | |
fmt.Println("OK") | |
} else { | |
os.Exit(1) | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment