wikipediaのダンプXMLのpages-articles.xmlをpageごとにjsonファイルとして出力する

wikipedia_dump_xml_to_json.go

package main

import (
	"bufio"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

type Text struct {
	Space string `xml:"space,attr"`
	Text  string `xml:",chardata"`
}

type Redirect struct {
	Title string `xml:"title,attr"`
}

type Revision struct {
	Id        int    `xml:"id"`
	Parentid  int    `xml:"parentid"`
	Timestamp string `xml:"timestamp"`
	Minor     string `xml:"minor"`
	Comment   string `xml:"comment"`
	Model     string `xml:"model"`
	Format    string `xml:"format"`
	Text      Text   `xml:"text"`
	Sha1      string `xml:"sha1"`
}

type XmlPage struct {
	Title        string   `xml:"title"`
	Ns           int      `xml:"ns"`
	Id           int      `xml:"id"`
	Redirect     Redirect `xml:"redirect"`
	Restrictions string   `xml:"restrictions"`
	Revision     Revision `xml:"revision"`
}

func main() {
	xmlfile, err := os.Open("./test.xml")
	if err != nil {
		log.Println("Error opening file:", err)
		os.Exit(1)
	}
	defer xmlfile.Close()

	scanner := bufio.NewScanner(xmlfile)

	strSection := ""
	for scanner.Scan() {
		steLine := scanner.Text()
		if "  <page>" == steLine {
			strSection = ""
		}

		strSection = strSection + steLine + "\n"

		if "  </page>" == steLine {
			convertJson(strSection)
		}
	}
}

func convertJson(xmlData string) {

	var page XmlPage

	err := xml.Unmarshal([]byte(xmlData), &page)
	if err != nil {
		fmt.Printf("error A: %v", err)
	}

	jsonBytes, err := json.Marshal(page)
	if err != nil {
		log.Println("JSON Marshal error:", err)
		return
	}

	filename := fmt.Sprintf("%s_%08d.json", "page", page.Id)
	jsonfilename := filepath.Join("./pages", filename)
	ioutil.WriteFile(jsonfilename, jsonBytes, os.ModePerm)

	log.Println(jsonfilename, page.Title)
}