Custom.pm

package Redis::Custom;
use strict;
use warnings;

# Redis::Custom->setup_custom_command( $redis, 'sjaccardstore', $lua_script, 1 );

sub setup_custom_command {
    my ($package, $redis, $command_name, $lua_script, $key_count, @binded_values) = @_;

    my ($sha1) = $redis->script_load( $lua_script );

    {
        no strict 'refs';
        no warnings 'redefine';
        *{ "Redis::$command_name" } = sub {
            my $self = shift;
            $self->evalsha( $sha1, $key_count, @binded_values, @_ );
        };
    }
}

1;

redis-sinter-sunion.pl

#!/usr/bin/env perl
use strict;
use warnings;
use FindBin::libs;

use Benchmark qw/:hireswallclock :all/;

use Test::RedisServer;
use Redis;
use Redis::SJaccardStore;

my $server = Test::RedisServer->new;
my $redis  = Redis->new( $server->connect_info );

my ($keys, $scard) = @ARGV;

sub prepare {
    for my $key (1..$keys) {
        for (1..$scard) {
            $redis->sadd( "u:$key" => int( rand() * $scard ) );
        }
    }
    Redis::SJaccardStore->setup( $redis );
}

sub sunion {
    for (my $index = 1; ($index < $keys); $index++ ) {
        $redis->sunion( "u:1" => "u:$index" );
    }
}

sub sinter {
    for (my $index = 1; ($index < $keys); $index++ ) {
        $redis->sinter( "u:1" => "u:$index" );
    }
}

sub jaccard {
    for (my $index = 1; ($index < $keys); $index++ ) {
        # $redis->sinter( "u:1" => "u:$index" );
        my @inter = $redis->sinter( "u:1" => "u:$index" );
        my @union = $redis->sunion( "u:1" => "u:$index" );
        my $jaccard = ((scalar @inter) || (scalar @union)) ? (scalar @inter) / (scalar @union)
                                                           : 0;
        $redis->zadd( "j:u:1" => $jaccard => "u:$index" );
    }
}

sub lua {
    for (my $index = 1; ($index < $keys); $index++ ) {
        $redis->sjaccardstore( "j:u:1", "u:1", "u:$index", "u:$index" );
    }
}

prepare();
printf( "env: %d sets, %d members for each set\n", $keys, $scard );

my $r = timethese( 0, {
    sunion  => \&sunion,
    sinter  => \&sinter,
    jaccard => \&jaccard,
    lua     => \&lua,
});
# use wallclock time
for my $v (values %$r) {
  $v->[1] = $v->[3] = $v->[0];
  $v->[2] = $v->[4] = 0;
}
cmpthese($r);

__DATA__

perl-5.12.4
Redis.pm 1.961
redis 2.6.1
MacOSX

Rhodos% perl script/benchmarks/redis-sinter-sunion.pl 100 10
env: 100 sets, 10 members for each set
Benchmark: running jaccard, lua, sinter, sunion for at least 3 CPU seconds...
   jaccard: 3.35703 wallclock secs ( 2.94 usr +  0.17 sys =  3.11 CPU) @ 25.40/s (n=79)
       lua: 5.08822 wallclock secs ( 2.83 usr +  0.33 sys =  3.16 CPU) @ 131.65/s (n=416)
    sinter: 3.22002 wallclock secs ( 2.84 usr +  0.16 sys =  3.00 CPU) @ 73.33/s (n=220)
    sunion: 3.15851 wallclock secs ( 2.90 usr +  0.12 sys =  3.02 CPU) @ 53.31/s (n=161)
          Rate jaccard  sunion  sinter     lua
jaccard 11.8/s      --    -54%    -66%    -71%
sunion  25.5/s    117%      --    -25%    -38%
sinter  34.2/s    190%     34%      --    -16%
lua     40.9/s    247%     60%     20%      --

Rhodos% perl script/benchmarks/redis-sinter-sunion.pl 100 100
env: 100 sets, 100 members for each set
Benchmark: running jaccard, lua, sinter, sunion for at least 3 CPU seconds...
   jaccard: 3.41807 wallclock secs ( 2.96 usr +  0.05 sys =  3.01 CPU) @  4.65/s (n=14)
       lua: 8.35508 wallclock secs ( 2.83 usr +  0.31 sys =  3.14 CPU) @ 132.48/s (n=416)
    sinter: 3.50854 wallclock secs ( 3.04 usr +  0.04 sys =  3.08 CPU) @ 15.26/s (n=47)
    sunion: 3.24783 wallclock secs ( 2.97 usr +  0.04 sys =  3.01 CPU) @  7.64/s (n=23)
          Rate jaccard  sunion  sinter     lua
jaccard 2.05/s      --    -42%    -69%    -92%
sunion  3.54/s     73%      --    -47%    -86%
sinter  6.70/s    227%     89%      --    -73%
lua     24.9/s   1116%    603%    272%      --

Rhodos% perl script/benchmarks/redis-sinter-sunion.pl 100 1000
env: 100 sets, 1000 members for each set
Benchmark: running jaccard, lua, sinter, sunion for at least 3 CPU seconds...
   jaccard: 3.84865 wallclock secs ( 3.66 usr +  0.02 sys =  3.68 CPU) @  0.54/s (n=2)
            (warning: too few iterations for a reliable count)
       lua: 47.3268 wallclock secs ( 2.92 usr +  0.39 sys =  3.31 CPU) @ 85.50/s (n=283)
    sinter: 3.55866 wallclock secs ( 3.36 usr +  0.01 sys =  3.37 CPU) @  1.78/s (n=6)
    sunion: 3.7364 wallclock secs ( 3.58 usr +  0.00 sys =  3.58 CPU) @  0.84/s (n=3)
            (warning: too few iterations for a reliable count)
        s/iter jaccard  sunion  sinter     lua
jaccard   3.85      --    -35%    -69%    -91%
sunion    2.49     55%      --    -52%    -87%
sinter    1.19    224%    110%      --    -72%
lua      0.334   1051%    645%    255%      --
Rhodos%

SJaccardStore.pm

package Redis::SJaccardStore;
use strict;
use warnings;
use Redis::Custom;
use Data::Section::Simple qw(get_data_section);

# SJACCARDSTORE( destination1, set1, set2, member1 )
#  $jaccard = SINTER( set1, set2 ) / SUNION( set1, set2 )
#  ZADD( destination1, $jaccard, member1 )

# SJACCARDSTOREBOTHWAYS( destination1, destination2, set1, set2, member1, member2 )
#  $jaccard = SINTER( set1, set2 ) / SUNION( set1, set2 )
#  ZADD( destination1, $jaccard, member1 )
#  ZADD( destination2, $jaccard, member2 )

sub setup {
    my ($package, $redis) = @_;

    my $common                = get_data_section( 'common.lua' );
    my $sjaccardstore         = get_data_section( 'sjaccardstore.lua' );
    my $sjaccardstorebothways = get_data_section( 'sjaccardstorebothways.lua' );
    Redis::Custom->setup_custom_command( $redis,
                                         'sjaccardstore',
                                         join( "\n",
                                               $common,
                                               $sjaccardstore ),
                                         0 );
    Redis::Custom->setup_custom_command( $redis,
                                         'sjaccardstorebothways',
                                         join( "\n",
                                               $common,
                                               $sjaccardstorebothways ),
                                         0 );
}

1;

__DATA__
@@ common.lua

-- sjaccardstore( 'j:sim:u:1', 'sim:u:1', 'sim:u:2', 'u:2' )
local function sjaccardstore( destination1, set1, set2, member1 )
    local sinter_count = 0
    local sunion_count = 0
    do
        -- minimize scope for tables
        local sinter = redis.call( 'sinter', set1, set2 )
        sinter_count = #sinter
    end
    do
        local sunion = redis.call( 'sunion', set1, set2 )
        sunion_count = #sunion
    end

    local jaccard = 0
    if sunion_count ~= 0 then
        jaccard = sinter_count / sunion_count
    end

    if jaccard ~= 0 then
        redis.call( 'zadd', destination1, jaccard, member1 )
    else
        redis.call( 'zrem', destination1, member1 )
    end

    -- lua number converts to redis integers
    -- we want float, so return string
    return jaccard
end

@@ sjaccardstore.lua

return tostring( sjaccardstore( ARGV[1], ARGV[2], ARGV[3], ARGV[4] ) )

@@ sjaccardstorebothways.lua

-- sjaccardstorebothways( 'j:sim:u:1', 'j:sim:u:2', 'sim:u:1', 'sim:u:2', 'u:2', 'u:1' )
local function sjaccardstorebothways( destination1, destination2, set1, set2, member1, member2 )
    local jaccard = sjaccardstore( destination1, set1, set2, member1 )
    if jaccard ~= 0 then
        redis.call( 'zadd', destination2, jaccard, member2 )
    else
        redis.call( 'zrem', destination2, member2 )
    end

    return jaccard
end

return tostring( sjaccardstorebothways( ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5], ARGV[6] ) )