freq_user.rb

module FreqUser
  class Mapper < Wukong::Streamer::StructStreamer
    #
    # extract just the word
    #
    def process thing, &block
      next unless thing.is_a? TweetToken
      yield [thing.user_id, thing.word]
    end
  end

  class Reducer < Wukong::Streamer::CountLines
  end

  # Execute the script
  Wukong::Script.new(
    Mapper,
    Reducer,
    :partition_fields => 2,
    :sort_fields      => 2
    ).run
end

freq_whole_corpus.rb

module FreqWholeCorpus
  class Mapper < Wukong::Streamer::StructStreamer
    #
    # extract just the word
    #
    def process thing, &block
      next unless thing.is_a? TweetToken
      yield thing.word
    end
  end

  # Execute the script
  Wukong::Script.new(
    Mapper,
    Wukong::Streamer::CountKeys
    ).run
end

word_count.rb

module WordCount
  class Mapper < Wukong::Streamer::StructStreamer
    #
    # Extract all the semantic items (smilies, hashtags, etc)
    # and all the remaining words from each tweet
    #
    def process thing, &block
      next unless thing.is_a? Tweet
      thing.tokenize(true).each do |token|
        yield token
      end
    end
  end

  # Execute the script
  Wukong::Script.new(
    Mapper,
    nil # WordFreq::Reducer,
   :reduce_tasks => 0
  ).run
end