extract text from .sql file

dencr.php

#!/usr/bin/php
<?php

function doFile ($filename) {
	foreach(file($filename) as $line) {
// copied from gettextsbylanguage1() in lcwo/inc/functions.php
		echo preg_replace_callback( '/&#(\d+);/', function ($m) { return mb_chr($m[1]); }, $line);
	}
}

if($argc < 2) {
	doFile('php://stdin');
} else {
	for($i=1; $i<$argc; ++$i) {
		doFile($argv[$i]);
	}
}

?>

simplify.php

#!/usr/bin/php
<?php

// copied from lcwo/inc/functions.php
function simplify ($lang, $text) {
	switch ($lang) {
		case 'fi':
		case 'de':
			$replace['ä'] = 'ae';
			$replace['ö'] = 'oe';
			$replace['ü'] = 'ue';
			$replace['Ä'] = 'ae';
			$replace['Ö'] = 'oe';
			$replace['Ü'] = 'ue';
			$replace['ß'] = 'ss';
			break;
		case 'fr':
			$replace['ô'] = 'o';
			$replace['ù'] = 'u';
			$replace['û'] = 'u';
			$replace['á'] = 'a';
			$replace['à'] = 'a';
			$replace['â'] = 'a';
			$replace['ç'] = 'c';
			$replace['é'] = 'e';
			$replace['è'] = 'e';
			$replace['ê'] = 'e';
			$replace['ë'] = 'e';
			$replace['ï'] = 'i';
			$replace['î'] = 'i';
			break;
		case 'es':
			$replace['à'] = 'a';
			$replace['á'] = 'a';
			$replace['é'] = 'e';
			$replace['è'] = 'e';
			$replace['í'] = 'i';
			$replace['ñ'] = 'n';
			$replace['ó'] = 'o';
			break;
		case 'pt':
			$replace['á'] = 'a';
			$replace['à'] = 'a';
			$replace['â'] = 'a';
			$replace['ã'] = 'a';
			$replace['ç'] = 'c';
			$replace['é'] = 'e';
			$replace['ê'] = 'e';
			$replace['í'] = 'i';
			$replace['ó'] = 'o';
			$replace['ô'] = 'o';
			$replace['õ'] = 'o';
			$replace['ú'] = 'u';
			$replace['ü'] = 'u';
			break;
        case 'pl':
			$replace['ą'] = 'a';
			$replace['ć'] = 'c';
			$replace['ę'] = 'e';
			$replace['ł'] = 'l';
			$replace['ń'] = 'n';
			$replace['ó'] = 'o';
			$replace['ś'] = 's';
			$replace['ź'] = 'z';
			$replace['ż'] = 'z';
            break;
		default:
			$replace['a'] = 'a';
			$replace["'"] = '';
	}

	foreach ($replace as $a => $b) {
		$text = preg_replace("/$a/", "$b", $text);
	}

	return $text;
}

function doFile ($filename, $lang) {
	foreach(file($filename) as $line) {
		echo simplify($lang, $line);
	}
}

$lang = "fr";
$c = 0;

for($i=1; $i<$argc; $i++) {
	$l = $argv[$i];
	if($l[0] == "-") {
		$lang = substr($l, 1); continue;
	}
	doFile($l, $lang); $c++;
}
if($c == 0) doFile("php://stdin", $lang);

?>

sqltsv.php

#!/usr/bin/php
<?php

function doString($str, &$idx)
{
	$ret = '';
	while(($c = $str[$idx++]) !== '') {
		if($c === '\'')
			break;
		if($c === '\\' && $str[$idx] === '\'' ) {
			$c = $str[$idx++];
		}
		$ret .= $c;
	}
	return $ret;
}

function doNumber($str, &$idx)
{
	$ret = '';
	while(($c = $str[$idx++]) !== '') {
		if($c === ',' || $c === ')') {
			--$idx; break;
		}
		$ret .= $c;
	}
	return $ret;
}

function doRow($str, &$idx)
{
	while(($c = $str[$idx++]) !== '') {
		if($c === ')') {
			echo "\n"; break;
		}
		if($c === '\'') {
			echo doString($str, $idx);
		} else if($c === ',') {
			echo "\t";
		} else {
			--$idx;
			echo doNumber($str, $idx);
		}
	}
}

function doDevide($str, &$idx)
{
	while(($c = $str[$idx++]) !== '') {
		if($c === '(') {
			doRow($str, $idx);
		} else if($c === ';') {
			break;
		}
	}
}

function doFile ($filename) {
	foreach(file($filename) as $line) {
		$pos = strpos($line, 'INSERT INTO');
		if($pos !== false && $pos == 0) {
			$idx = strpos($line, '(');
			if($idx !== false) {
				doDevide($line, $idx);
			}
		}
	}
}

if($argc < 2) {
	doFile('php://stdin');
} else {
	for($i=1; $i<$argc; ++$i) {
		doFile($argv[$i]);
	}
}

?>

sqltsv.php devide SQL "INSERT INTO" statements into Tab Separated Values so that good old awk can handle
dencr.php DEcode Numeric Character Reference é -> é
simplify.php ä -> ae, etc.

% git clone https://gist.github.com/b8a9f8dd0552be94bcd7b21d57bf3bf5.git foo
% cd foo
% chmod +x *.php
% curl https://raw.githubusercontent.com/dj1yfk/lcwo/master/db/lcwo_words.sql | ./sqltsv.php | ./dencr.php > words.txt
% cat words.txt | awk -F \t '$2=="de" {print $3}' | ./simplify.php -de > words.de.txt
% cat words.txt | awk -F \t '$2=="fr" {print $3}' | ./simplify.php -fr > words.fr.txt