Skip to content

Instantly share code, notes, and snippets.

@fuzzysteve
Created October 2, 2013 13:51
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save fuzzysteve/6794129 to your computer and use it in GitHub Desktop.
Save fuzzysteve/6794129 to your computer and use it in GitHub Desktop.
kill loader for solr
#!/usr/bin/perl -w
use strict;
use JSON;
use Data::Dumper;
use WebService::Solr;
use Time::HiRes qw(time);
my $solr = WebService::Solr->new( undef, { autocommit => 0 } );
my $filename = $ARGV[0];
print $filename."\n";
my $json_text = do {
open(my $json_fh, "<:encoding(UTF-8)", $filename)
or die("Can't open \$filename\": $!\n");
local $/;
<$json_fh>
};
my $json = JSON->new;
my $data = $json->decode($json_text);
my $x=0;
my $stime = time;
print STDERR "Starting\n";
foreach my $array ( @{$data} ) {
my $kill=$json->decode($array);
$x++;
if(!($x%100))
{
# print STDERR $x."\n";
}
my $start_time = time;
if(!($x%1000))
{
print STDERR time - $stime."\n";
print STDERR $x."\n";
$start_time = time;
$solr->commit();
print STDERR time - $start_time."\n";
$stime= time;
}
my $doc = WebService::Solr::Document->new;
$doc->add_fields((WebService::Solr::Field->new( killid => $kill->{'killID'} )));
my $time=$kill->{'killTime'};
$time=~ s/ /T/;
$time=$time."Z";
$doc->add_fields((WebService::Solr::Field->new( killTime => $time )));
$doc->add_fields((WebService::Solr::Field->new( documentType => '0' )));
$doc->add_fields((WebService::Solr::Field->new( shiptype => $kill->{'victim'}->{'shipTypeID'} )));
$doc->add_fields((WebService::Solr::Field->new( characterid => $kill->{'victim'}->{'characterID'} )));
$doc->add_fields((WebService::Solr::Field->new( corporationid => $kill->{'victim'}->{'corporationID'} )));
$doc->add_fields((WebService::Solr::Field->new( allianceid => $kill->{'victim'}->{'allianceID'} )));
$doc->add_fields((WebService::Solr::Field->new( damage => $kill->{'victim'}->{'damageTaken'} )));
foreach my $item (@{$kill->{'items'}})
{
$doc->add_fields((WebService::Solr::Field->new( items => $item->{'typeID'} )));
}
foreach my $attacker (@{$kill->{'attackers'}})
{
$doc->add_fields((WebService::Solr::Field->new( attackerdetails => $attacker->{'characterID'}."|".$attacker->{'shipTypeID'}."|".$attacker->{'weaponTypeID'} )));
$doc->add_fields((WebService::Solr::Field->new( attackerid => $attacker->{'characterID'} )));
$doc->add_fields((WebService::Solr::Field->new( attackercorporationid => $attacker->{'corporationID'} )));
$doc->add_fields((WebService::Solr::Field->new( attackerallianceid => $attacker->{'allianceID'} )));
if ($attacker->{'finalBlow'}==1)
{
$doc->add_fields((WebService::Solr::Field->new( finalBlow => $attacker->{'characterID'} )));
}
}
$solr->add( $doc );
}
$solr->commit();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment