Convert big GeoJSON FeatureCollections into a line by line features without loading the full file into memory

main.go

package main

import (
	"bufio"
	"encoding/json"
	"io"
	"os"

	"github.com/peterstace/simplefeatures/geom"
)

func main() {
	r := bufio.NewReader(os.Stdin)

	decoder := json.NewDecoder(r)

	var f geom.GeoJSONFeature

	for {
		token, err := decoder.Token()
		if err == io.EOF {
			break
		}

		if err != nil {
			panic(err)
		}

		if token == "features" {
			_, err = decoder.Token()
			if err != nil {
				panic(err)
			}

			break
		}
	}

	enc := json.NewEncoder(os.Stdout)

	for decoder.More() {
		if err := decoder.Decode(&f); err != nil {
			panic(err)
		}

		if err := enc.Encode(f); err != nil {
			panic(err)
		}
	}
}

jq is great but the command cat big.geoson | jq -c '.features[]' > big.geojsonseq is using a lot of memory.

Use this snippet like this.

mkdir jsonparser
cd jsonparser
# copy this gist into main.go
go mod init jsonparser
go build

cat big.geoson |./jsonparser > big.geojsonseq