Last active
July 6, 2019 05:20
-
-
Save gaurav-gogia/1ff0ebacd3b9ae3a4ce5a79e2589ef67 to your computer and use it in GitHub Desktop.
processing big files in go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"bufio" | |
"log" | |
"os" | |
) | |
const BUFF = 1024 * 1024 | |
func read(exp, path string) ([]string, error) { | |
var data []string | |
file, err := os.Open(path) | |
defer file.Close() | |
if err != nil { | |
return nil, err | |
} | |
scanner := bufio.NewScanner(file) | |
buf := make([]byte, 0, BUFF) | |
scanner.Buffer(buf, 10*BUFF) | |
if err != nil { | |
go log.Println(err) | |
return err | |
} | |
for scanner.Scan() { | |
data = append(data, match([]string{scanner.Text()}, exp)...) | |
} | |
return data, nil | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment