Skip to content

Instantly share code, notes, and snippets.

@gaurav-gogia
Last active July 6, 2019 05:20
Show Gist options
  • Save gaurav-gogia/1ff0ebacd3b9ae3a4ce5a79e2589ef67 to your computer and use it in GitHub Desktop.
Save gaurav-gogia/1ff0ebacd3b9ae3a4ce5a79e2589ef67 to your computer and use it in GitHub Desktop.
processing big files in go
package main
import (
"bufio"
"log"
"os"
)
const BUFF = 1024 * 1024
func read(exp, path string) ([]string, error) {
var data []string
file, err := os.Open(path)
defer file.Close()
if err != nil {
return nil, err
}
scanner := bufio.NewScanner(file)
buf := make([]byte, 0, BUFF)
scanner.Buffer(buf, 10*BUFF)
if err != nil {
go log.Println(err)
return err
}
for scanner.Scan() {
data = append(data, match([]string{scanner.Text()}, exp)...)
}
return data, nil
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment