csv2jsonl.go

package transformer

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"os"
)

type Column struct {
	Name  string
	Order int
	Type  string
}

// CSV2JSONL from CSV to  Newline delimited JSON
func CSV2JSONL(skipRows int, columns []Column, outJSONFile string, inCSVFiles ...string) error {
	writeFile, err := os.Create(outJSONFile)
	if err != nil {
		return fmt.Errorf("write file open err: %v", err)
	}
	w := bufio.NewWriter(writeFile)
	defer w.Flush()

	for _, readFile := range inCSVFiles {
		reader, err := os.Open(readFile)
		if err != nil {
			return fmt.Errorf("read file open err: %v", err)
		}
		r := csv.NewReader(reader)
		rowNum := 0
		for {
			rowNum++
			record, err := r.Read()
			if err == io.EOF {
				break // EOF
			}
			if err != nil {
				return fmt.Errorf("csv read err: %v", err)
			}
			if rowNum <= skipRows {
				continue // skip
			}
			jsonMap := map[string]interface{}{}
			for _, col := range columns {
				idx := col.Order - 1
				if idx < 0 || idx >= len(record) {
					return fmt.Errorf("record index err: %v", col)
				}
				key := col.Name
				val := record[idx]
				jsonMap[key] = val
			}

			// TODO(u110): meta情報としてファイル、行番号をのせるべきか検討
			// jsonMap["META_FILE"] = readFile
			// jsonMap["META_FLINE"] = rowNum

			jsonBuf, err := json.Marshal(jsonMap)
			if err != nil {
				return fmt.Errorf("json.Marshal err: %v", jsonMap)
			}
			i, err := fmt.Fprintln(w, string(jsonBuf))
			if err != nil {
				return fmt.Errorf("Write ERROR: n: %v err: %v", i, err)
			}
		}
	}
	return nil
}

todo?

• Typeを元に変換する？
• error時は w.Flush()すべき？