atnd `ruby main.rb > atnd.json`

.gitignore

atnd.json

README.md

USAGE

ruby main.rb > atnd.json

Gemfile

# frozen_string_literal: true

source "https://rubygems.org"

git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }

gem 'nokogiri'

Gemfile.lock

GEM
  remote: https://rubygems.org/
  specs:
    mini_portile2 (2.4.0)
    nokogiri (1.10.8)
      mini_portile2 (~> 2.4.0)

PLATFORMS
  ruby

DEPENDENCIES
  nokogiri

BUNDLED WITH
   1.17.2

main.rb

require 'pp'
require 'pry'
require 'net/http'
require 'nokogiri'
require 'json'
require 'open-uri'

MAX_COUNT = 100

EVENT_SERACH_API = 'http://api.atnd.org/events/'
EVENT_USERS_API = 'http://api.atnd.org/events/users/'

OWNER_ID = 63450

def fetch_all(base_uri)
  1.step(by: MAX_COUNT).lazy.map do |start|
    uri = URI("#{base_uri}?owner_id=#{OWNER_ID}&count=#{MAX_COUNT}&start=#{start}&format=json")
    json = Net::HTTP.get uri
    JSON.parse json
  end.take_while{|data| data['results_returned'].to_i > 0 }.flat_map do |data|
    data['events'].map{|e| [e['event']['event_id'], e['event']] }
  end.force.to_h
end

events      = fetch_all(EVENT_SERACH_API)
event_users = fetch_all(EVENT_USERS_API)

events = events.map{|id, event| event.merge(event_users[id] || {}) }

# クローリング
events = events.map do |event|
  event_url = URI('http://atnd.org/events/16069')
  doc = Nokogiri::HTML.parse(open(event_url))

  comments = doc.search('.comments-entry').map do |c|
    {
      user_name: c.search('.header-name').first.text,
      datetime: DateTime.parse(c.search('.comments-date').first.text.strip[1..-2]),
      body: c.search('dd').first.text.strip,
    }
  end
  event['comments'] = comments

  sleep 0.5

  event
end

puts JSON.generate(events)