orcimport.go

package main

import (
	"bufio"
	"client"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var (
	c = client.NewClient("550002c1-63fa-428b-a04a-8d35e64d05b7")
	w sync.WaitGroup
)

func main() {
	fileGlob := os.Args[1] + "/*"

	if files, err := filepath.Glob(fileGlob); err != nil {
		panic(err)
	} else {
		for _, file := range files {
			go importFile(file)
			w.Add(1)
		}

		w.Wait()
	}
}

func importFile(filename string) {
	defer w.Done()

	file, err := os.Open(filename)

	if err != nil {
		log.Printf("Error: %v\n", err)
		return
	}
	defer file.Close()

	log.Printf("Importing %v ...", filename)

	reader := bufio.NewReaderSize(file, 1024*1024)

	var indexingConflicts int64 = 0

	for err == nil {
		var line string
		line, err = reader.ReadString('\n')

		if err != nil {
			break
		}

		decoder := json.NewDecoder(strings.NewReader(line))
		json := new(map[string]interface{})
		decoder.Decode(json)

		if err := c.Put(filepath.Base(filename), fmt.Sprintf("%v", (*json)["_id"].(map[string]interface{})["$oid"]), json); err != nil && err.(*client.OrchestrateError).StatusCode != 409 {
			log.Panicf("Error: %#v\n", err)
		} else if err != nil && err.(*client.OrchestrateError).StatusCode == 409 {
			indexingConflicts++
		}
	}

	if err != nil && err != io.EOF {
		log.Panicf("Scanner error: %v\n", err)
	}

	log.Printf("Done importing %v (with %v indexing conflicts)", filename, indexingConflicts)
}