Skip to content

Instantly share code, notes, and snippets.

@hakre
Created April 26, 2012 09:12
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save hakre/2497942 to your computer and use it in GitHub Desktop.
Save hakre/2497942 to your computer and use it in GitHub Desktop.
Tokenizer Example of Mirc Color and Style (bold) Codes.
<?php
/**
* Tokenizer Example of Mirc Color and Style (bold) Codes.
*
* @link http://stackoverflow.com/q/10329443/367456
*/
$mirc = "\x034this text is red\x033this text is green\x03 \x02bold text\x02
\x034,3this text is red with a green background\x03";
$tokenizer = new Tokenizer($mirc);
while(list($token, $data) = $tokenizer->getNext())
{
switch($token)
{
case 'color-fgbg':
printf('<%s:%d,%d>', $token, $data[1], $data[2]);
break;
case 'color-fg':
printf('<%s:%d>', $token, $data[1]);
break;
case 'color-reset':
case 'style-bold';
printf('<%s>', $token);
break;
case 'catch-all':
echo $data[0];
break;
default:
throw new Exception(sprintf('Unknown token <%s>.', $token));
}
}
/**
* regular expression based tokenizer,
* first token wins.
*/
class Tokenizer
{
private $subject;
private $offset = 0;
private $tokens = array(
'color-fgbg' => '\x03(\d{1,2}),(\d{1,2})',
'color-fg' => '\x03(\d{1,2})',
'color-reset' => '\x03',
'style-bold' => '\x02',
'catch-all' => '.|\n',
);
public function __construct($subject)
{
$this->subject = (string) $subject;
}
public function setOffset($offset)
{
$this->offset = max(0, $offset);
}
public function getOffset()
{
return $this->offset;
}
/**
* @return array|null
*/
public function getNext()
{
if ($this->offset >= strlen($this->subject))
return NULL;
foreach($this->tokens as $name => $token)
{
if (FALSE === $r = preg_match("~$token~", $this->subject, $matches, PREG_OFFSET_CAPTURE, $this->offset))
throw new RuntimeException('Pattern for token %s failed (regex error).', $name);
if ($r === 0)
continue;
if (!isset($matches[0])) {
var_dump(substr($this->subject, $this->offset));
$c = 1;
}
if ($matches[0][1] !== $this->offset)
continue;
$data = array();
foreach($matches as $match)
{
list($data[]) = $match;
}
$this->offset += strlen($data[0]);
return array($name, $data);
}
return NULL;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment