wiki_crawl.rb

require 'uri'
require 'mechanize'

module Wiki
  class Crawl

    attr_reader :start, :domain, :mech, :exclude

    def initialize(start)
      @start  = start
      @domain = get_domain()
      @mech   = Mechanize.new
      @exclude = /main_page|talk:|category:|template:|special:|user:|wikipedia:|help:|portal:|file:/i
    end

    def rand_link(page, num)
      return if num.zero?

      link = mech
               .get(page)
               .links_with(href: /^\/wiki\//i)
               .map{|l| l.href}
               .select{ |l| l !~ exclude}
               .sample

      puts domain + link
      sleep(3)
      rand_link(domain + link, num -= 1)
    end

    def get_domain()
      uri = URI.parse(start)
      "#{uri.scheme}://#{uri.host}"
    end

    def init(num)
      rand_link(start, num)
    end
  end
end

Wiki::Crawl.new('https://en.wikipedia.org/wiki/Main_Page').init(100)