anemoneで作った雑なクローラ

readme.md

使い方はrun.shを参照のこと

とりあえず使いたい人は下記のコマンドをコピペしてください

git clone https://gist.github.com/cb7d754c4ac26322fb0e1805da206a72.git
cd cb7d754c4ac26322fb0e1805da206a72
bundle install --path vendor/bundle --jobs 4
chmod +x run.sh
./run.sh

crawl.rb

require 'bundler/setup'
require 'anemone'

url = ARGV[0]
site_name = ARGV[1]
Dir.mkdir("./#{site_name}")
count = 0
Anemone.crawl(url) do |anemone|
  anemone.on_every_page do |page|
    puts page.url
    File.write("./#{site_name}/#{count}.html", page.body)
    count += 1
  end
end

Gemfile

# frozen_string_literal: true
source "https://rubygems.org"

gem "anemone"

run.sh

ruby sample.rb https://www.engadget.com/ engadget