clusterer.rb

require 'zlib'
require 'pp'
require 'logger'
# original seen on http://www.igvita.com/2011/04/20/intuition-data-driven-machine-learning/
$log = Logger.new(STDOUT)
$log.level = Logger::WARN

class CompareFiles

  attr_reader :pairs

  def initialize
    @files = Dir[ARGV[0] + '/*']
    @pairs = process_files(@files)
    return @pairs
  end

  def deflate(*files)
    z = Zlib::Deflate.new
    z.deflate(files.collect {|f|
      open(f).read
    }.join("\\n"), Zlib::FINISH).size
  end

  private :deflate

  def process_files(files)
    pairs = files.combination(2).collect { |f1, f2|
      a, b = deflate(f1), deflate(f2)
      both = deflate(f1, f2)
      {:files => [f1, f2], :score => (a+b)-both}
    }
    return pairs
  end

  private :process_files

end

begin
  pairwise = CompareFiles.new.pairs
  pp pairwise.sort {|a,b| b[:score] <=> a[:score]}[0,20]
rescue => ex
  $log.fatal("Caught exception....")
  $log.fatal(ex)
end