PHP UTF-16 Multibyte Regex for TexVC php

AllRunner.php

<?php
namespace MediaWiki\Extension\Math\TexVC;
require_once __DIR__ . '/vendor/autoload.php';

use MediaWiki\Extension\Math\TexVC\Parser;
use MediaWiki\Extension\Math\TexVC\TexVC;

/**
input:
'\\text {-0-9a-zA-Z+*,=():/;?.!\'` \x80-\xFF} ' +
'\\mbox {-0-9a-zA-Z+*,=():/;?.!\'` \x80-\xFF} ' +
'\\hbox {-0-9a-zA-Z+*,=():/;?.!\'` \x80-\xFF} ' +
'\\vbox {-0-9a-zA-Z+*,=():/;?.!\'` \x80-\xFF} ',

output:
'{\\text{-0-9a-zA-Z+*,=():/;?.!\'` \x80-\xFF}}' +
'{\\mbox{-0-9a-zA-Z+*,=():/;?.!\'` \x80-\xFF}}' +
'{\\hbox{-0-9a-zA-Z+*,=():/;?.!\'` \x80-\xFF}}' +
'{\\vbox{-0-9a-zA-Z+*,=():/;?.!\'` \x80-\xFF}}'
 */

$DELIMITERS1 =     [ "(", ")", "[", "]", "\\{", "\\}", "|" ];
$delim2 = "\\backslash\\downarrow\\Downarrow\\langle\\lbrace\\lceil\\lfloor" .
    "\\llcorner\\lrcorner\\rangle\\rbrace\\rceil\\rfloor\\rightleftharpoons" .
    "\\twoheadleftarrow\\twoheadrightarrow\\ulcorner\\uparrow\\Uparrow" .
    "\\updownarrow\\Updownarrow\\urcorner\\Vert\\vert\\lbrack\\rbrack";
$DELIMITERS2 = array_map( function ($f) { return "\\" . $f; }, explode("\\", $delim2));
array_shift($DELIMITERS2);

$delim3 = "\\darr\\dArr\\Darr\\lang\\rang\\uarr\\uArr\\Uarr";
$DELIMITERS3 = array_map( function ($f) { return "\\" . $f; }, explode("\\", $delim3));
array_shift($DELIMITERS3);


print("asd");
$testcases = [
    "Unicode"=> [
        //         "input"=> "{\\mbox{💩\uD83D\uDCA9}}" fails
       // "input"=> "{\\mbox{💩💩}}"  //fails
        //"input"=> "{\\mbox{\u{D83D}\u{DCA9}\u{D83D}\u{DCA9}}}" fails
        //"input"=> "{\\mbox{💩\u{D83D}\u{DCA9}}}"
        "input" => "{\\mbox{💩" . unescape_utf16( "\uD83D\uDCA9" ) . "}}"

    ]


];


function peg_regex_test($pattern, $string) {
    $matches = [];
    if (substr($pattern, -1) == "i") {
        $subst = substr($pattern, 1, -2);
        $res = mb_eregi($subst, $string, $matches);
    } else {
        $subst = substr($pattern, 1, -1);
        $res =  mb_ereg($subst, $string, $matches);
    }

    return $res;
}

function unescape_utf16($string) {
    /* go for possible surrogate pairs first */
    $string = preg_replace_callback(
        '/\\\\u(D[89ab][0-9a-f]{2})\\\\u(D[c-f][0-9a-f]{2})/i',
        function ($matches) {
            $d = pack("H*", $matches[1].$matches[2]);
            return mb_convert_encoding($d, "UTF-8", "UTF-16BE");
        }, $string);
    /* now the rest */
    $string = preg_replace_callback('/\\\\u([0-9a-f]{4})/i',
        function ($matches) {
            $d = pack("H*", $matches[1]);
            return mb_convert_encoding($d, "UTF-8", "UTF-16BE");
        }, $string);
    return $string;
}




try {
    $texVC = new TexVC();
    $testcases = [$testcases["Unicode"]];

    //$testcases = [bigs($DELIMITERS1,$DELIMITERS2)];
    // Filter by erronous testcases
    $testcases[0]["input"] = "{\\mbox{💩" . unescape_utf16( "\uD83D\uDCA9" ) . "}}"; // Fails
    $testcases[0]["input"] = "{\\mbox{💩" . "\uD83D\uDCA9" . "}}";
    $testcases[0]["input"] = unescape_utf16( "\uD83D\uDCA9" );
    $testcases[0]["input"] = "\\mbox{💩}";
    //mb_check_encoding("utf-16");


    //mb_regex_encoding('UTF-16');
    $str = "\xC4\xA2";
// The string is valid in all three encodings, so the first one listed will be returned
    var_dump(mb_detect_encoding($str, ['UTF-8', 'UTF-16', 'ISO-8859-1', 'ISO-8859-5']));
    $str = "💩"; //  unescape_utf16( "\uD83D\uDCA9" ); same
    $peg_c40 = "/^[\x{0000}\x{0000}-\x{FFFF}\x{FFFF}]+/u"; //fails
    //$peg_c40 = "/^[\x{0000}-\x{FFFF}]+/u"; //fails
    $peg_c40 = "/[\x{0000}\x{0000}-\x{FFFF}\x{FFFF}]/"; //fails
    $peg_c40 = "[\x{0000}\x{0000}-\x{FFFF}\x{FFFF}]"; //fails

    // mb_regex_encoding('UTF-8');

    $res = peg_regex_test($peg_c40, "💩");
    $res2 = mb_ereg($peg_c40, $str);
    if(!$res){
        print("\nerr");
    }else{
        print("\nsucc");
    }

    if(mb_ereg('[\x{0001}-\x{FFFF}]', $str)) // arabic range
//if(mb_ereg('[\x{0590}-\x{05FF}]', $text)) // hebrew range
    {
        echo "Text has some arabic/hebrew characters.";
    }

    foreach ($testcases as $title => $tc) {
        print("\n Checking: " . $title);
        $tc["output"] = $tc["output"] ?? $tc["input"];
        $message = "output should be correct";
        if(!key_exists("skipJs", $tc)  || !$tc["skipJs"] ) {
            $result = $texVC->check($tc["input"], array(
                "debug" => true,
                "usemathrm" => $tc["usemathrm"] ?? false,
                "oldtexvc" => $tc["oldtexvc"] ?? false
            ));
            print("\n output (1): " . $result["output"] );
            print("\n outputW(1): " . $tc["output"] );
            print("\n" );
            if($result["status"] !== "+"){
                print("\n err status ");
            }
            if($result["output"] !== $tc["output"]){
                print("\n err output");
            };
            //print("\n" . $result["output"]);

        }
        if (!key_exists("skipReparse", $tc) || !$tc["skipReparse"]){
            // verify that the output doesn't change if we feed it
            // through again.
            $message = "should parse its own output";
            $result1 = $texVC->check($tc["output"],  ["debug"=> true]);
            $result2 = $texVC->check($result1["output"], ["debug"=> true]);
            if($result2["status"] !== "+"){
                print("\n err status ");
            }
            if($result2["output"] !== $result1["output"]){
                print("\n err output");
            };
        }
        if (!key_exists("skipOcaml", $tc) || !$tc["skipOcaml"] ){
            $message = "should match ocaml output";
            //nyi tryocaml($tc["input"],$tc["output"], done # is async)

        } elseif ($tc["skipOcaml"] === "double spacing") {
            $message = "should match ocaml output (except for spacing)";
            //nyi tryocaml($tc["input"],$tc["output"], done # is async, true)
        }
    }



}
catch ( PhpPegJs\SyntaxError $ex ) {
    $message = "Syntax error: " . $ex->getMessage() . ' at line ' . $ex->grammarLine . ' column ' . $ex->grammarColumn . ' offset ' . $ex->grammarOffset;
}