Testing Solr Indexing

run_tests_single_threaded.sh

#!/bin/bash

NUM_USERS=100
THREADS=1
# variable to get us back to the parent director of nakamura & oae builder
BASE=../
BRANCHES=('content-indexing' 'master')
PROCS=('conns' 'msgs')
WINDOW_NAME="Java Monitoring & Management Console"

pushd $BASE
# move any existing app jar
[[ -e "../nakamura/app/target/org.sakaiproject.nakamura.app-1.1-SNAPSHOT.jar" ]] && mv ../nakamura/app/target/org.sakaiproject.nakamura.app-1.1-SNAPSHOT.jar ../nakamura/app/target/org.sakaiproject.nakamura.app-1.1-SNAPSHOT.jar.bak

# checkout out the branch to build in nakamura, solr and sparse
for BRANCH in ${BRANCHES[@]}
do
    # clean out any old servers
    rake clean

    # symlink the appropriate jar to the expected app jar name for other
    # scripts to use
    ln -s `pwd`/solr-test/org.sakaiproject.nakamura.app-$BRANCH-1.1-SNAPSHOT.jar ../nakamura/app/target/org.sakaiproject.nakamura.app-1.1-SNAPSHOT.jar

    # clear out any old db, create a new one and load up the 100 users we use
    # will also setup the config file for using mysql
    pushd solr-test > /dev/null
    ./test-solr-changes-setup.sh
    popd > /dev/null # solr-test -> OAE-Builder

    # start the server
    rake run

    # wait for the server to come up
    CODE=404
    while [ $CODE -ne 200 ]
    do
        echo -n .
        sleep 10
        CODE=`curl -I --silent http://localhost:8080/index | grep HTTP | awk '{print $2}'`
    done
    echo up

    # continue testing as more data is loaded. this will build up more
    # data in the storage and index to give us something of a view into
    # how things look as they get bigger.
    echo "" >> oae_solr_test_results.txt
    pushd solr-test > /dev/null
    for PROC in ${PROCS[@]}
    do
        jconsole -interval=1 `ps ax | grep nakamura | grep java | awk '{print $1}'` &
        sleep 5 # give jconsole a bit to come up
        ./test-solr-changes.rb -n $NUM_USERS -p $PROC >> oae_solr_test_results.txt
        import -window "$WINDOW_NAME" solr-test-$BRANCH-$PROC.png
        killall jconsole
    done
    popd > /dev/null # solr-test -> OAE-Builder

    # stop the server and remove the symlink
    rake kill
    rm -f ../nakamura/app/target/org.sakaiproject.nakamura.app-1.1-SNAPSHOT.jar
done

# if we moved a jar, put it back
[[ -e "../nakamura/app/target/org.sakaiproject.nakamura.app-1.1-SNAPSHOT.jar.bak" ]] && mv ../nakamura/app/target/org.sakaiproject.nakamura.app-1.1-SNAPSHOT.jar.bak ../nakamura/app/target/org.sakaiproject.nakamura.app-1.1-SNAPSHOT.jar

test-solr-changes-setup.sh

# drop, recreate and populate the database
mysqladmin -f -u root -pmtrpls12 drop oae_solr_test create oae_solr_test
#mysqladmin -u root -pmtrpls12 create oae_solr_test
mysql -u root -pmtrpls12 oae_solr_test < oae_solr_test_users.sql

DIR=../sling/config/org/sakaiproject/nakamura/lite/storage/jdbc
FILE=JDBCStorageClientPool.config

mkdir -p $DIR
pushd $DIR > /dev/null
echo 'service.pid="org.sakaiproject.nakamura.lite.storage.jdbc.JDBCStorageClientPool"' > $FILE
echo 'jdbc-driver="com.mysql.jdbc.Driver"' >> $FILE
echo 'jdbc-url="jdbc:mysql://localhost/oae_solr_test?autoReconnectForPools=true"' >> $FILE
echo 'password="mtrpls12"' >> $FILE
echo 'username="root"' >> $FILE
popd > /dev/null # $DIR

test-solr-changes.rb

#!/usr/bin/env ruby
require 'logger'
require 'optparse'
require 'net/http'
require 'benchmark'

def create_users(num_users = 0, first_user = 1)
  count = 0
  last_user = first_user + num_users - 1
  (first_user..last_user).each do |i|
    @log.info "Creating User #{i}"
    req = Net::HTTP::Post.new("/system/userManager/user.create.html")
    req.set_form_data({
      ":name" => "user#{i}",
      "pwd" => "test",
      "pwdConfirm" => "test",
      "email" => "user#{i}@sakaiproject.invalid",
      "firstName" => "User",
      "lastName" => "#{i}",
      "locale" => "en_US",
      "timezone" => "America/Los_Angeles",
      "_charset_" => "utf-8",
      ":sakai:profile-import" => "{'basic': {'access': 'everybody', 'elements': {'email': {'value': 'user#{i}@sakaiproject.invalid'}, 'firstName': {'value': 'User'}, 'lastName': {'value': '#{i}'}}}}"
    })
    req.basic_auth("admin", "admin")
    response = @localinstance.request(req)
    if response.code == '200' then
      count += 1
    else
      @log.info response.inspect
    end
  end
  return count
end

def create_connections(num_users = 0, first_user = 1)
  count = 0
  last_user = num_users + first_user - 1
  (first_user..last_user-1).each do |i|
    (i+1).upto(last_user).each do |j|
      @log.info "Requesting connection between User #{i} and User #{j}"
      req = Net::HTTP::Post.new("/~user#{i}/contacts.invite.html")
      req.set_form_data({
        "fromRelationships" => "Classmate",
        "toRelationships" => "Classmate",
        "targetUserId" => "user#{j}",
        "_charset_" => "utf-8"
      })
      req.basic_auth("user#{i}", "test")
      response = @localinstance.request(req)
      if response.code == '200'
        @log.info "Accepting connection between User #{i} and User #{j}"
        req = Net::HTTP::Post.new("/~user#{j}/contacts.accept.html")
        req.set_form_data({
          "targetUserId" => "user#{i}",
          "_charset_" => "utf-8"
        })
        req.basic_auth("user#{j}", "test")
        response = @localinstance.request(req)
        if response.code == '200'
          count += 1
        end
      else
        @log.info response.inspect
      end
    end
  end
  return count
end

def send_messages(num_users = 0, first_user = 0)
  count = 0
  last_user = num_users + first_user - 1
  (first_user..last_user).each do |i|
    (first_user..last_user).each do |j|
      if i == j then next end

      @log.info "Sending internal message: user#{i} => user#{j}"
      response = send_internal_message "user#{i}", "user#{j}", "test #{i} => #{j}", "test body #{i} => #{j}"
      if response.code == '200'
        count += 1
      else
        @log.info response.inspect
      end

      @log.info "Sending internal message: user#{j} => user#{i}"
      response = send_internal_message "user#{j}", "user#{i}", "test #{j} => #{i}", "test body #{j} => #{i}"
      if response.code == '200'
        count += 1
      else
        @log.info response.inspect
      end
    end
  end
  return count
end

def send_internal_message(to, from, subject, body)
  req = Net::HTTP::Post.new("/~#{from}/message.create.html")
  req.set_form_data({
    "_charset_" => "utf-8",
    "sakai:body" => "test body #{from} => #{to}",
    "sakai:category" => "message",
    "sakai:from" => "#{from}",
    "sakai:messagebox" => "outbox",
    "sakai:sendstate" => "pending",
    "sakai:subject" => "test #{from} => #{to}",
    "sakai:to" => "internal:#{to}",
    "sakai:type" => "internal"
  })
  req.basic_auth("#{from}", "test")
  @localinstance.request(req)
end

if $PROGRAM_NAME.include? __FILE__
  @log = Logger.new('solr-test.log')
  @log.level = Logger::DEBUG
  @log.info "\n==== start processing #{Time.now}"
  
  options = {}
  OptionParser.new do |opts|
    opts.banner = "Usage: #{__FILE__} [options]"
    
    options[:procs] = []
    opts.on('-p', '--procs all,users,conns,msgs', Array, 'Run various processing functions (all, users, connections, messages)') do |procs|
      options[:procs] = procs
    end
    
    opts.on('-n', '--num_users NUM_USERS', 'Number of users to process') do |num_users|
      options[:num_users] = num_users.to_i
    end
    
    options[:start] = 1
    opts.on('-s', '--start_num [START_NUM]', 'User number to start processing') do |start|
      options[:start] = start.to_i
    end
  end.parse!
  
  @log.info "num_users = #{options[:num_users]}"
  @log.info "procs = #{options[:procs]}"
  raise OptionParser::MissingArgument, '-n' if options[:num_users].nil?
  raise OptionParser::MissingArgument, '-p' if options[:procs].nil?
  
  @uri = URI.parse("http://localhost:8080")
  @localinstance = Net::HTTP.new(@uri.host, @uri.port)
  
  users = 0
  conns = 0
  msgs = 0
  
  Benchmark.bm(7) do |x|
    if options[:procs].include? 'all' or options[:procs].include? 'users'
      x.report('users:') { users = create_users options[:num_users], options[:start] }
    end
    if options[:procs].include? 'all' or options[:procs].include? 'conns'
      x.report('connections:') { conns = create_connections options[:num_users], options[:start] }
    end
    if options[:procs].include? 'all' or options[:procs].include? 'msgs'
      x.report('messages:') { msgs = send_messages options[:num_users], options[:start] }
    end
  end
  
  @log.info "Created: users: #{users}; connections: #{conns}; messages: #{msgs}"
  puts "Created: users: #{users}; connections: #{conns}; messages: #{msgs}"
end