Skip to content

Instantly share code, notes, and snippets.

@BenMcH
Created June 13, 2018 17:22
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save BenMcH/faa948e6cc2f525d9c15ec0b5f239bce to your computer and use it in GitHub Desktop.
Save BenMcH/faa948e6cc2f525d9c15ec0b5f239bce to your computer and use it in GitHub Desktop.
package main
import "io/ioutil"
import "regexp"
import "log"
import "strings"
import "encoding/json"
func main() {
size := 4
mapping := make(map[string]int)
bytes, err := ioutil.ReadFile("input.txt");
if err != nil {
log.Fatal("Error reading file!")
}
value := string(bytes)
value = standardizeSpaces(value)
reg, err := regexp.Compile("[^a-zA-Z!?. ]+")
if err != nil {
log.Fatal(err)
}
processedString := reg.ReplaceAllString(strings.ToLower(value), "")
sentences := regexp.MustCompile("[!.?]").Split(processedString, -1)
for _, str := range sentences {
stringSlice := append([]string{"*BEGIN*"}, strings.Fields(str)...)
stringSlice = append(stringSlice, "*END*")
for i := 0; i <= len(stringSlice) - (size * 2); i += 1 {
key := strings.Join(stringSlice[i:i+size], "_") + "#" + strings.Join(stringSlice[i+size:i+size*2], "_")
mapping[key] += 1
}
}
if jsonBytes, err := json.Marshal(mapping); err == nil {
ioutil.WriteFile("output.json", jsonBytes, 0644)
}
}
func standardizeSpaces(s string) string {
return strings.Join(strings.Fields(s), " ")
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment