Simple script to parse the lines in the wikipedia page title dump file and output a list of all the english page titles (deduplicated). See https://dumps.wikimedia.org/enwiki/20220801/ for more files

parse_wiki_titles.go

package wikipediatitles

import (
	_ "embed"
	"sort"
	"strings"
	"unicode"
)

// https://dumps.wikimedia.org/enwiki/20220801/enwiki-20220801-all-titles-in-ns0.gz
//go:embed enwiki-20220801-all-titles-in-ns0
var allTitlesString string


func Parse(minSize int) []string {
	start := 0
	m := map[string]struct{}{}
	for index, r := range allTitlesString {

		if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '-' || r == ' ' { // || r == '\'' ?
			continue
		}

		if index-start >= minSize {
			text := allTitlesString[start:index]
			text = strings.ToLower(strings.ReplaceAll(text, "_", " "))
			text = strings.Trim(text, "- ")
			if len(text) >= minSize {
				m[text] = struct{}{}
			}
		}

		start = index + len(string(r))
	}

	var results []string
	for k := range m {
		results = append(results, k)
	}
	sort.Strings(results)

	return results
}

parse_wiki_titles_test.go

package wikipediatitles

import (
	"fmt"
	"os"
	"strings"
	"testing"
)

func TestParse(t *testing.T) {

	titles := Parse(5)

	for i := 0; i < 100; i++ {
		fmt.Println(titles[i])
	}

	f, err := os.Create("titles.txt")
	if err != nil {
		t.Fatal(err)
	}

	_, err = f.Write([]byte(strings.Join(titles, "\n")))
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}
}