token.pl

package CIF::Smrt::Parser::Default;

use strict;
use warnings;
use namespace::autoclean;

use Moose;
use MooseX::FollowPBP;
use String::Tokenizer;
use CIF qw(observable_type);

has 'handle' => (
    is      => 'ro',
    isa     => 'String::Tokenizer',
    default => sub { String::Tokenizer->new() },
);

has 'skip_comments' => (
    is      => 'ro',
    isa     => 'Bool',
    default => 1,
);

has 'ignore'    => (
    is      => 'ro',
    isa     => 'ArrayRef',
);

has 'replace'   => (
    is      => 'ro',
    isa     => 'HashRef',
    default => sub { {
        'hxxp://'   => 'http://',
    }},
);

with 'CIF::Smrt::Parser';

sub understands {
    my $self = shift;
    my $args = shift;
    
    return 1 unless($args->{'parser'});
    return 0;
}


sub process {
    my $self = shift;
    my $args = shift;

    my ($rv,$type);
    foreach (@{$args->{'content'}}){
        # skip comments
        next if($self->get_skip_comments() && $_ =~ $self->get_comments());
        $self->get_handle()->tokenize($_);
        foreach my $t ($self->get_handle()->getTokens()){
            next if($self->_ignore($t));
            $t = $self->_replace($t);
            push(@$rv,{ observable => $t }) if(observable_type($t));  
        }      
    }
    
    die ::Dumper($rv);
    return $rv;
}

sub _replace {
    my $self = shift;
    my $arg = shift;
    
    foreach (keys %{$self->get_replace()}){
        next unless($arg =~ $_);
        my $r = $self->get_replace->{$_};
        $arg =~ s/$_/$r/;
    }
    return $arg;
}

sub _ignore {
    my $self = shift;
    my $arg = shift;
 
    foreach (@{$self->get_ignore()}){
        return 1 if($arg =~ $_);
    }
}

__PACKAGE__->meta->make_immutable;

1;