Skip to content

Instantly share code, notes, and snippets.

@ambroff
Last active August 29, 2015 14:18
Show Gist options
  • Save ambroff/430be29f6acd313dc3e3 to your computer and use it in GitHub Desktop.
Save ambroff/430be29f6acd313dc3e3 to your computer and use it in GitHub Desktop.
Proper decoding
diff --git a/api/iteration.go b/api/iteration.go
index 49b833a..01eea02 100644
--- a/api/iteration.go
+++ b/api/iteration.go
@@ -5,6 +5,13 @@ import (
"io/ioutil"
"path/filepath"
"strings"
+
+ "golang.org/x/net/html/charset"
+ "golang.org/x/text/transform"
+)
+
+const (
+ mimeType = "text/plain"
)
var (
@@ -53,12 +60,13 @@ func NewIteration(dir string, filenames []string) (*Iteration, error) {
iter.Problem = segments[2]
for _, filename := range filenames {
- b, err := ioutil.ReadFile(filename)
+ fileContents, err := readFileAsUTF8String(filename)
if err != nil {
return nil, err
}
+
path := filename[len(iter.RelativePath()):]
- iter.Solution[path] = string(b)
+ iter.Solution[path] = *fileContents
}
return iter, nil
}
@@ -73,3 +81,21 @@ func (iter *Iteration) isValidFilepath(path string) bool {
}
return strings.HasPrefix(strings.ToLower(path), strings.ToLower(iter.Dir))
}
+
+func readFileAsUTF8String(filename string) (*string, error) {
+ b, err := ioutil.ReadFile(filename)
+ if err != nil {
+ return nil, err
+ }
+
+ encoding, _, _ := charset.DetermineEncoding(b, mimeType)
+ decoder := encoding.NewDecoder()
+
+ decodedBytes, _, err := transform.Bytes(decoder, b)
+ if err != nil {
+ return nil, err
+ }
+
+ s := string(decodedBytes)
+ return &s, nil
+}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment