Simple extractor for JSON, trying various high performance alternatives.

.gitignore

62f64de2016dc38186e21270715e8016

example.json

{
  "publisher": "Elsevier BV",
  "DOI": "10.1016/0039-9140(81)80126-9",
  "member": "http://id.crossref.org/member/78"
}

go.mod

module github.com/miku/62f64de2016dc38186e21270715e8016

go 1.12

require (
	github.com/json-iterator/go v1.1.6
	github.com/miku/parallel v0.0.0-20170719114747-dd55491c9ac3
	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
	github.com/modern-go/reflect2 v1.0.1 // indirect
)

go.sum

github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/miku/parallel v0.0.0-20170719114747-dd55491c9ac3 h1:zOnp8cZ+YxulefnHy2OnpHj9tXLPir5JSUnoYagIGis=
github.com/miku/parallel v0.0.0-20170719114747-dd55491c9ac3/go.mod h1:m4hVixrXwk3DUp5cQ1j661BsHpjqSc/SfXE0uUMxmAw=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=

main.go

package main

import (
	"log"
	"os"

	jsoniter "github.com/json-iterator/go"
	"github.com/miku/parallel"
)

type Doc struct {
	DOI       string
	Member    string `json:"member"`
	Publisher string `json:"publisher"`
}

func main() {

	var json = jsoniter.ConfigCompatibleWithStandardLibrary

	p := parallel.NewProcessor(os.Stdin, os.Stdout, func(b []byte) ([]byte, error) {
		var doc Doc
		if err := json.Unmarshal(b, &doc); err != nil {
			return nil, err
		}
		b, err := json.Marshal(doc)
		b = append(b, '\n')
		return b, err
	})
	p.BatchSize = 25000

	if err := p.Run(); err != nil {
		log.Fatal(err)
	}
}