cgiosy/gen.js

## gen.js
const fs = require('fs');

const filenum = 30; // or 100
const lines = 2000000; // or 1000000
const errors = 100000;

const ratio = lines / errors | 0;
const count = lines / ratio | 0;

for (let i = 1; i <= filenum; i += 1) {
        const stream = fs.createWriteStream(`test${i}-log.xml`, { flags: 'a' });

        let text = `<error><!\[CDATA\[${i}#${Math.random()}\]\]><\/error>\n`;
        for (let j = 2; j <= ratio; j += 1)
                text += `<info><!\[CDATA\[${i}#${Math.random()}\]\]><\/info>\n`;
        text | 0;

        for (let j = 1; j <= count; j += 1)
                stream.write(text);
        stream.end();
}

## log.js
const { resolve } = require('path');
const { writeFile, readFile, readdir } = require('fs').promises;

const walkDir = (fn, startingDir = '.') => {
        const _walkDir = (currentDir) => readdir(currentDir, { withFileTypes: true }).then((children) => Promise.all(
                children.map((child) => {
                        const res = resolve(currentDir, child.name);
                        if (child.isDirectory()) return _walkDir(res);
                        return Promise.resolve(fn(res));
                })
        ));
        return _walkDir(startingDir);
};

const flatten = (str) => {
        str | 0;
        return str;
};

const extractErrors = async () => {
        const filenames = [];
        await walkDir((filename) => {
                if (!filename.toLowerCase().endsWith('-log.xml')) return;
                filenames.push(filename);
        });

        let errors = [];
        for (const filename of filenames) {
                console.log(filename);
                const text = await readFile(filename, { encoding: 'utf-8' }); // Out Of Memory! mem usage 97~99%
                errors = errors.concat(
                        (text.match(/<error><!\[CDATA\[(.+?)\]\]><\/error>/g) || [])
                                .filter(str => !str.includes(' '))
                                .map(str => str.slice(16, -11))
                                // .map(flatten)
                );
        }
        await writeFile('errors.txt', JSON.stringify([...new Set(errors)]));
};

setTimeout(extractErrors, 0 * 1000);
	const fs = require('fs');

	const filenum = 30; // or 100
	const lines = 2000000; // or 1000000
	const errors = 100000;

	const ratio = lines / errors \| 0;
	const count = lines / ratio \| 0;

	for (let i = 1; i <= filenum; i += 1) {
	const stream = fs.createWriteStream(`test${i}-log.xml`, { flags: 'a' });

	let text = `<error><!\[CDATA\[${i}#${Math.random()}\]\]><\/error>\n`;
	for (let j = 2; j <= ratio; j += 1)
	text += `<info><!\[CDATA\[${i}#${Math.random()}\]\]><\/info>\n`;
	text \| 0;

	for (let j = 1; j <= count; j += 1)
	stream.write(text);
	stream.end();
	}
	const { resolve } = require('path');
	const { writeFile, readFile, readdir } = require('fs').promises;

	const walkDir = (fn, startingDir = '.') => {
	const _walkDir = (currentDir) => readdir(currentDir, { withFileTypes: true }).then((children) => Promise.all(
	children.map((child) => {
	const res = resolve(currentDir, child.name);
	if (child.isDirectory()) return _walkDir(res);
	return Promise.resolve(fn(res));
	})
	));
	return _walkDir(startingDir);
	};

	const flatten = (str) => {
	str \| 0;
	return str;
	};

	const extractErrors = async () => {
	const filenames = [];
	await walkDir((filename) => {
	if (!filename.toLowerCase().endsWith('-log.xml')) return;
	filenames.push(filename);
	});

	let errors = [];
	for (const filename of filenames) {
	console.log(filename);
	const text = await readFile(filename, { encoding: 'utf-8' }); // Out Of Memory! mem usage 97~99%
	errors = errors.concat(
	(text.match(/<error><!\[CDATA\[(.+?)\]\]><\/error>/g) \|\| [])
	.filter(str => !str.includes(' '))
	.map(str => str.slice(16, -11))
	// .map(flatten)
	);
	}
	await writeFile('errors.txt', JSON.stringify([...new Set(errors)]));
	};

	setTimeout(extractErrors, 0 * 1000);