Last active
April 15, 2020 18:33
-
-
Save guillermo-menjivar/7ea3108714e68c305af03332119aed62 to your computer and use it in GitHub Desktop.
load and search
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 4+ basic_file_upload.go + !/b/zsh [running] X | |
| ~/lab/analysis $ cat basic_file_upload.go | |
| package main | |
| import ( | |
| "bufio" | |
| "context" | |
| "encoding/json" | |
| "fmt" | |
| "net/http" | |
| "path/filepath" | |
| "regexp" | |
| "time" | |
| "github.com/gin-gonic/gin" | |
| "github.com/olivere/elastic" | |
| ) | |
| const ( | |
| indexName = "gnql" | |
| ) | |
| type Event struct { | |
| IP string `json:"ip"` | |
| FirstSeen string `json:"first_seen"` | |
| LastSeen string `json:"last_seen"` | |
| Metadata struct { | |
| Asn string `json:"asn"` | |
| City string `json:"city"` | |
| Country string `json:"country"` | |
| CountryCode string `json:"country_code"` | |
| Organization string `json:"organization"` | |
| Rdns string `json:"rdns"` | |
| Category string `json:"category"` | |
| Tor bool `json:"tor"` | |
| } `json:"metadata"` | |
| Actor string `json:"actor"` | |
| Tags []string `json:"tags"` | |
| Spoofable bool `json:"spoofable"` | |
| Classification string `json:"classification"` | |
| Seen bool `json:"seen"` | |
| RawData struct { | |
| Web struct { | |
| } `json:"web"` | |
| Scan []struct { | |
| Port int `json:"port"` | |
| Protocol string `json:"protocol"` | |
| } `json:"scan"` | |
| } `json:"raw_data"` | |
| } | |
| func query(service *elastic.SearchService, ipaddrs []string) *elastic.SearchService { | |
| q := elastic.NewBoolQuery().MinimumNumberShouldMatch(1) | |
| for _, ip := range ipaddrs { | |
| q = q.Should(elastic.NewMatchQuery("ip", ip)) | |
| } | |
| //q = q.Should(elastic.NewMatchQuery("ip", "31.56.96.195")) | |
| service = service.Query(q) | |
| return service | |
| } | |
| func aggs(service *elastic.SearchService) *elastic.SearchService { | |
| //countries | |
| agg := elastic.NewTermsAggregation().Field("metadata.country") | |
| service = service.Aggregation("countries", agg) | |
| //actors | |
| agg = elastic.NewTermsAggregation().Field("actor") | |
| service = service.Aggregation("actors", agg) | |
| //classifications | |
| agg = elastic.NewTermsAggregation().Field("classification") | |
| service = service.Aggregation("classifications", agg) | |
| //tags | |
| agg = elastic.NewTermsAggregation().Field("tags") | |
| service = service.Aggregation("tags", agg) | |
| //os | |
| agg = elastic.NewTermsAggregation().Field("os") | |
| service = service.Aggregation("os", agg) | |
| return service | |
| } | |
| func findRecords(ctx context.Context, client *elastic.Client, ips []string) (int, error) { | |
| search := client.Search().Index(indexName).Pretty(true) | |
| //search = search.DocvalueFields("classification", "ip", "metadata.organization", "actor", "last_seen") | |
| search = query(search, ips) | |
| search = aggs(search) | |
| sr, err := search.Do(ctx) | |
| if err != nil { | |
| return -1, err | |
| } | |
| for _, hit := range sr.Hits.Hits { | |
| var event Event | |
| err := json.Unmarshal(*hit.Source, &event) | |
| if err != nil { | |
| fmt.Println("we failed to unmarshal event") | |
| } | |
| fmt.Println("thise are the eventss=====") | |
| fmt.Println("") | |
| fmt.Println(event) | |
| data, err := json.Marshal(event) | |
| fmt.Println(string(data)) | |
| } | |
| fmt.Println(sr.Hits.Hits) | |
| // retrieve all countries aggregates | |
| var countries map[string]int64 | |
| if agg, found := sr.Aggregations.Terms("countries"); found { | |
| countries = make(map[string]int64) | |
| for _, bucket := range agg.Buckets { | |
| countries[bucket.Key.(string)] = bucket.DocCount | |
| } | |
| } | |
| fmt.Println("this is the aggregates") | |
| fmt.Println(countries) | |
| // retreive all actors aggregates | |
| var actors map[string]int64 | |
| if agg, found := sr.Aggregations.Terms("countries"); found { | |
| actors = make(map[string]int64) | |
| for _, bucket := range agg.Buckets { | |
| actors[bucket.Key.(string)] = bucket.DocCount | |
| } | |
| } | |
| fmt.Println("this is the aggregates") | |
| fmt.Println(actors) | |
| return 0, nil | |
| } | |
| func extrapulateIp(record string) []string { | |
| ips := []string{} | |
| engine := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) | |
| if engine.MatchString(record) { | |
| submatchall := engine.FindAllString(record, -1) | |
| for _, ip := range submatchall { | |
| ips = append(ips, ip) | |
| } | |
| } | |
| return ips | |
| } | |
| func submitQueryToElastic(ips []string) { | |
| client, err := elastic.NewClient(elastic.SetURL("http://localhost:9200"), elastic.SetSniff(false), elastic.SetHealthcheck(false)) | |
| if err != nil { | |
| panic(err) | |
| } | |
| //create a dummy context | |
| ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) | |
| defer cancel() | |
| response, err := findRecords(ctx, client, ips) | |
| fmt.Println(response) | |
| fmt.Println(err) | |
| } | |
| func main() { | |
| router := gin.Default() | |
| // Set a lower memory limit for multipart forms (default is 32 MiB) | |
| router.MaxMultipartMemory = 8 << 20 // 8 MiB | |
| router.Static("/", "./public") | |
| router.POST("/upload", func(c *gin.Context) { | |
| name := c.PostForm("name") | |
| email := c.PostForm("email") | |
| // Source | |
| _, h, err := c.Request.FormFile("file") | |
| if err != nil { | |
| c.String(http.StatusBadRequest, fmt.Sprintf("get form err: %s", err.Error())) | |
| return | |
| } | |
| filename := filepath.Base(h.Filename) | |
| raw, err := h.Open() | |
| scanner := bufio.NewScanner(raw) | |
| scanner.Split(bufio.ScanLines) | |
| totalIPs := []string{} | |
| for scanner.Scan() { | |
| ips := extrapulateIp(scanner.Text()) | |
| totalIPs = append(totalIPs, ips...) | |
| } | |
| fmt.Println(totalIPs) | |
| fmt.Println(len(totalIPs)) | |
| submitQueryToElastic(totalIPs) | |
| if err := c.SaveUploadedFile(h, filename); err != nil { | |
| c.String(http.StatusBadRequest, fmt.Sprintf("upload file err: %s", err.Error())) | |
| return | |
| } | |
| c.String(http.StatusOK, fmt.Sprintf("File %s uploaded successfully with fields name=%s and email=%s.", h.Filename, name, email)) | |
| }) | |
| router.Run(":8080") | |
| } |
curl -F 'file=@/Users/gmo/lab/analysis/file.txt' localhost:8080/upload
this is the query I want
{
"_source": ["classification", "ip", "metadata.organization", "actor", "last_seen"],
"query": {
"bool": {
"minimum_should_match": 1,
"should": [
{
"match": {
"ip": "5.125.235.56"
}
},
{"match": {
"ip": "31.56.96.195"
}}
]
}
},
"aggs": {
"countries": {
"terms": {"field": "metadata.country"}
},
"actors": {
"terms": {"field": "actor"}
},
"classification": {
"terms": {"field": "classification"}
},
"tags": {
"terms": {"field": "tags"}
},
"os": {
"terms": {"field": "metadata.os"}
}
}
}
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
this is the test file