Crawl rel="me" links

.ruby-version

3.2.1

00readme.md

$ bundle install
$ bundle exec ruby draw-relme-links.rb https://example.com/your-profile

generates relme.png

draw-relme-links.rb

#!/usr/bin/ruby
#
# usage: ruby draw-relme-links.rb start-url
#
require "nokogiri"
require "http"
require "ruby-graphviz"

def normalized_url(url)
  unless url
    return nil
  end
  r = url.encode("ASCII").chomp
  if r.empty?
    return nil
  end
  return r
end

def compact_url(url)
  return url.sub(/\Ahttps?:\/\//i, "").sub(/\/\z/, "").sub(/\//, "\n/").chomp
end

urls = [URI.parse(normalized_url(ARGV.shift))]
links = {}

while not urls.empty?
  src = urls.shift
  $stderr.puts "Checking #{src}"
  res = HTTP.headers(:accept => "text/html").get(src)
  targets = Nokogiri::HTML5(res.body).xpath(
    '//a[contains(concat(" ", normalize-space(@rel), " "), " me ")]|//link[contains(concat(" ", normalize-space(@rel), " "), " me ")]'
  ).map{|link|
    dst = URI.parse(normalized_url(link['href']))
    dst = src.merge(dst.to_s) unless dst.absolute?
    dst
  }.uniq.compact
  links[src] = targets
  urls += targets - urls - links.keys
end

links.values.flatten.sort_by{_1.to_s}.uniq.each do |dst|
  puts "#{dst} <-"
  links.keys.sort_by{_1.to_s}.each do |src|
    puts "\t#{src}" if links[src].include?(dst)
  end
end

nodes = []
edges = Hash.new{|h,k| h[k] = []}
links.keys.each do |src|
  sc = compact_url(src.to_s)
  nodes << sc
  dcs = links[src].map{|dst| compact_url(dst.to_s)}
  edges[sc] << dcs
  nodes += dcs
end

nodes.uniq!
edges.each_pair do |sc, dcs|
  edges[sc] = dcs.flatten.uniq
end

g = GraphViz.new( :G, :type => :digraph )
gn = nodes.map{|url| [url, g.add_nodes(url)]}.to_h

require 'pp'
pp edges
edges.keys.each do |src|
  edges[src].each do |dst|
    e = g.add_edges(gn[src], gn[dst])
    if edges[dst].include?(src)
      edges[dst].delete(src)
      e.set do |_e|
        _e.dir = "both"
        _e.color = "forestgreen"
      end
    else
      e.set do |_e|
        _e.color = "darkgray"
      end
    end
  end
end

g.output(:png => "relme.png")

Gemfile

# frozen_string_literal: true
source "https://rubygems.org"
gem "nokogiri", "~> 1.14"
gem "http", "~> 5.1"
gem "ruby-graphviz", "~> 1.2"

Gemfile.lock

GEM
  remote: https://rubygems.org/
  specs:
    addressable (2.8.4)
      public_suffix (>= 2.0.2, < 6.0)
    domain_name (0.5.20190701)
      unf (>= 0.0.5, < 1.0.0)
    ffi (1.15.5)
    ffi-compiler (1.0.1)
      ffi (>= 1.0.0)
      rake
    http (5.1.1)
      addressable (~> 2.8)
      http-cookie (~> 1.0)
      http-form_data (~> 2.2)
      llhttp-ffi (~> 0.4.0)
    http-cookie (1.0.5)
      domain_name (~> 0.5)
    http-form_data (2.3.0)
    llhttp-ffi (0.4.0)
      ffi-compiler (~> 1.0)
      rake (~> 13.0)
    nokogiri (1.14.3-arm64-darwin)
      racc (~> 1.4)
    nokogiri (1.14.3-x86_64-linux)
      racc (~> 1.4)
    public_suffix (5.0.1)
    racc (1.6.2)
    rake (13.0.6)
    rexml (3.2.5)
    ruby-graphviz (1.2.5)
      rexml
    unf (0.1.4)
      unf_ext
    unf_ext (0.0.8.2)

PLATFORMS
  arm64-darwin-22
  x86_64-linux

DEPENDENCIES
  http (~> 5.1)
  nokogiri (~> 1.14)
  ruby-graphviz (~> 1.2)

BUNDLED WITH
   2.4.6