scraper.rb file wrote for the first episode/tutorial on how to find the cheapest openpilot compatible car in the UK (code in procedural style - for learning ruby / scraping)

scraper.rb

require 'nokogiri'
require 'net/http'
require 'json'

def get(url)
  uri  = URI url
  resp = Net::HTTP.get_response uri
  body = resp.body
  JSON.parse body
end

API_HOST = "https://www.autotrader.co.uk"
API_PATH = "/results-car-search"

def build_params(page:)
  make = "TOYOTA"
  year = 2016
  postcode = "E145AB"
  distance = 10
  # distance = 50
  distance = 5

  params =  "page=#{page}&make=#{make}&year-from=#{year}"
  params += "&postcode=#{postcode}&radius=#{distance}"
  params += "&transmission=Automatic"
end

def search(params:)
  url    = "#{API_HOST}#{API_PATH}?#{params}"
  puts "URL: #{url}"

  resp   = get url
  html   = resp.fetch "html"
  dom    = Nokogiri::HTML html

  regex = /\/classified\/advert\/20/
  links = dom.search("a").select { |li| li["href"] =~ regex }

  links.uniq! { |link| link["href"].gsub(/#.+/, '') }
  links
end

def start_search
  results = []
  1.upto(30) do |page|
    params = build_params page: page
    links  = search params: params
    results += links
    puts "links in page: #{links.size}"
  end
  results
rescue JSON::ParserError
  results
end

results = start_search

puts "Results: #{results.size}"

# File.open("index.html", "w"){ |f| f.write html }

code from tutorial video - https://youtu.be/lx6-xE8UQCM How to build a ruby scraper (to find the cheapest OP car in the uk!!1!!11)