Last active
November 27, 2019 08:01
-
-
Save suchowan/68a2ae48474959a93da0604abd20ec14 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf-8 -*- | |
=begin | |
This script was written by Takashi SUGA on January 2017-November 2019 | |
You may use and/or modify this file according to the license described in the MIT LICENSE.txt file https://raw.githubusercontent.com/suchowan/watson-api-client/master/LICENSE. | |
=end | |
require 'open-uri' | |
require 'openssl' | |
require 'json' | |
require 'rest-client' | |
Encoding.default_external = 'UTF-8' | |
Encoding.default_internal = 'UTF-8' | |
GEMS = 'https://rubygems.org/profiles/suchowan' | |
HUB = 'https://github.com/suchowan?tab=repositories' | |
GIST = 'https://gist.github.com/suchowan' | |
LOD = 'http://user.linkdata.org/search?keyword=%E9%A0%88%E8%B3%80%E9%9A%86&category=all' | |
QIITA = 'https://qiita.com/suchowan' | |
QIITA_API = RestClient::Resource.new('http://qiita.com/api/v2') | |
def stocks(id) | |
STDERR.puts " stocks - #{id}" | |
JSON.parse(QIITA_API["/items/#{id}/stockers"].get.body) | |
end | |
def crawl(url) | |
STDERR.puts ' crawl' | |
args = [url, 'r:utf-8'] | |
args << {:ssl_verify_mode=>OpenSSL::SSL::VERIFY_NONE} if url =~ /^https:/ | |
open(*args) do |source| | |
contents = source.read | |
# STDERR.puts [contents.length, contents[0...80]] | |
return contents | |
end | |
end | |
Contents = <<Contents | |
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> | |
<html> | |
<head> | |
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> | |
<title>Related Links - suchowan's Home Pag</title> | |
</head> | |
<style type="text/css"> | |
<!-- | |
body{ | |
background-image : url("wall.png"); | |
background-repeat : no-repeat; | |
background-attachment : fixed; | |
} | |
--> | |
</style> | |
<body bgcolor="#F0F0F0" text="Black" link="Blue" alink="Fuchsia" vlink="Green"> | |
<small> | |
<table width='100%%'> | |
<tr><td colspan='2' align='right'><small>Updated: %s</small></td></tr> | |
<tr> | |
<td width='44%%' align='center'><a href="http://hosi.org:3000" target="_top" title="今日の暦 - 暦の相互変換"><font color='Green'><h1>hosi.org</h1></font><font color='Orange'>暦の相互変換</font></a> <font color='Orange'>( <a href="http://www2u.biglobe.ne.jp/~suchowan/when_exe_wiki.html" target='_top' title='when.exe Wiki の入口です'>Wiki</a>, <a href="https://suchowan.at.webry.info/" target='_blank' title="suchowan's blog を別タブで表示します">Blog</a> )</font></td> | |
<td width='56%%'> | |
<table border='1' width='100%%'> | |
<tr><th><a href='#{GEMS}' target='_RubyGems'>RubyGems</a></th><th>Version</th><th>Download</th></tr> | |
%s | |
</table> | |
</td></tr> | |
<tr><td colspan='2'> | |
<table border='1' width='100%%'> | |
<tr><th><a href='#{HUB}' target='_GitHub'>GitHub</a> (<a href='#{GIST}' target='_Gist'>Gist</a>)</th><th>Description</th><th>Star</th><th>Fork</th></tr> | |
%s | |
</table> | |
</td></tr> | |
<tr><td colspan='2'> | |
<table border='1' width='100%%'> | |
<tr><th><a href='#{QIITA}' target='_Qiita'>Qiita</a></th><th>Like</th><th>Stock</th><th>Posted</th></tr> | |
%s | |
</table> | |
</td></tr> | |
<tr><td colspan='2'> | |
<table border='1' width='100%%'> | |
<tr><th><a href='#{LOD.gsub('%','%%')}' target='_Lod'>Linked Data</a></th><th>Award</th><th>Updated</th></tr> | |
%s | |
</table> | |
</td></tr> | |
<tr><td colspan='2' align='center'><font color='Maroon'><b><big>hosi.org - service ports</big></b></font></td></tr> | |
<tr><td colspan='2'> | |
<table border='1' width='100%%'> | |
<tr><th>Port</th><th>Service</th><th>Port</th><th>Service</th></tr> | |
<tr><td align='right'>80</td><td><a href='http://hosi.org:80' target='_top' title='具体的なURLに応じてhosi.org代表サーバが処理を振り分けます。ユーザは意識しないでよいでしょう'>代表ポート</a>(転送など)</td> | |
<td align='right'>4000</td><td><a href='http://hosi.org:4000/bookmarks' target='_top' title='GitHub の bookmarks のうち sig.url.txt に基づいてブックマーク・ツリーを表示します'>ブックマーク</a></td></tr> | |
<tr><td align='right'>3000</td><td><a href='http://hosi.org:3000' target='_top' title='「今日の暦」などを表示するWhen.exe テストサーバです'>When.exe テスト(メイン)</a></td> | |
<td align='right'>8080</td><td><a href='http://hosi.org:8080/cgi-bin/conv.cgi?m=01000' target='_top' title='GitHub の a_converter をオンラインで動かしています'>進法&単位変換サービス</a></td></tr> | |
<tr><td align='right'>3030</td><td><a href='http://www2u.biglobe.ne.jp/~suchowan/wiki/query.html' target='_top' title='「日本暦注データセット」の SPARQL エンドポイントです。SPARQLでアクセスしてください'>日本暦注データセット<br/>問合せ</a></td> | |
<td align='right'>8080</td><td><a href='http://hosi.org:8080/disk' target='_top' title='暦の会会員限定の様々な情報を置いています'>暦の会関連 諸々データ置き場</a><br/>(ID,パスワード要)</td></tr> | |
<tr><td align='right'>4000</td><td><a href='http://hosi.org:4000' target='_top' title='「今日の暦」などを表示するWhen.exe テストサーバ(予備)です。Rails のバージョンが古く少し動作が遅いです'>When.exe テスト(サブ)</a></td> | |
<td align='right'>8090</td><td><a href='http://hosi.org:8090' target='_top' title='Qiitaの「機械学習関連情報の収集と分類(構想)」に基づいて Fess により情報収集した結果を提供しています'>FESS による機械学習関連情報検索</a></td></tr> | |
</table> | |
</td></tr> | |
<tr> | |
<td align='center' colspan='2'> | |
<table><tr> | |
<td><a href="https://independent.academia.edu/TakashiSUGA" target='_top' title='Academia に置いた論文'><font color=#FF0000>Academia</font></a></td> | |
<td><a href="https://twitter.com/suchowan/with_replies" target='_top' title='Blog は Twitter と連動しています'><font color=#E00000>Twitter</font></a></td> | |
<td><a href="http://hosi.org/cgi-bin/links.cgi" target='mainFrame' title='大昔の私のブックマークを公開します。リンク切れになったサイトも多いです'><font color=#c000c0>Links</font></a></td> | |
<td><a href="http://calndr-l.10958.n7.nabble.com/template/NamlServlet.jtp?macro=search_page&node=2&query=Suga" target='_Sig' title='Calndr-Lの関連投稿です'><font color=#A000A0>Calndr-L</font></a></td> | |
<td><a href="http://suchowan.at.webry.info/" target='mainFrame' title="suchowan's blogです"><font color=#008040>Blog</font></a></td> | |
<td><a href="http://www2u.biglobe.ne.jp/~suchowan/when_exe_wiki.html" target='_top' title='Wiki の入口です'><font color=#008060>Wiki</font></a></td> | |
<td align='Center'><small><a href="http://www.asahi-net.or.jp/~dd6t-sg/book_a.html" target='mainFrame' title='私の本棚の写真を蔵書目録がわりに作ってみました'><font color=Teal>Book<br/>Shelves</font></a><br/> | |
<a href="http://hosi.org/cgi-bin/links.cgi?New_Room" target='mainFrame' title='転居後の写真です'><font color=Blue>(New)</font></a></small></td> | |
<td align='Center'><small><a href="http://www.asahi-net.or.jp/~dd6t-sg/simulation.html" target='mainFrame' title='その他自作の文章・データやプログラムを紹介します'><font color=#400080>Over<br/>decades<br/>ago</font></a></small></td> | |
<td align='Center'><small><a href="http://www.asahi-net.or.jp/~dd6t-sg/whenhome_contents.html" target="mainFrame" title='When.exe MS-DOS版'><font color=#300060>When.exe<br/>MS-DOS<br/>version</font></a></small></td> | |
<td align='Center'><small><a href="http://www.asahi-net.or.jp/~dd6t-sg/index.htm" target="_top" title='旧ホームページです'><font color=#200040>Old<br/>Home<br/>Page</font></a></small></td> | |
</tr></table> | |
</td> | |
</tr> | |
<tr> | |
<td align='center' colspan='2'> | |
<a target="_blank" href="http://www.mebius.co.jp/" title='「Mebuis」へ行きます'><img width="120" src="http://www.mebius.co.jp/img/mebiuslogo.png"/></a> | |
<a target="_blank" href="http://www.asahi-net.or.jp/~nr8c-ab/" title='旧ホームページのバナーを作ってくださった Apollon さんのサイトです'><img width="120" src="http://www.asahi-net.or.jp/~nr8c-ab/bna2.gif"/></a> | |
<!-- <a target="_blank" href="http://www.mojikyo.org/" title='本サイトで特殊な漢字を表記する際、文字鏡のフォントを使用しています'><img width="120" src="http://www.mojikyo.org/img/mologo.gif"/></a> --> | |
<a target="_blank" href="http://www.mojikyo.org/" title='本サイトで特殊な漢字を表記する際、文字鏡のフォントを使用しています'><img width="120" src="http://www.asahi-net.or.jp/~dd6t-sg/mologo.gif"/></a> | |
<br/>Since 1998-09-20<br/>Copyright <a href='http://hosi.org:3000/TakashiSuga.rdf' target='_reference'>Takashi SUGA</a></td></tr> | |
</table> | |
</small> | |
</body> | |
</html> | |
Contents | |
# | |
# RubyGems | |
# | |
STDERR.puts 'RubyGems...' | |
state = nil | |
gem = nil | |
gems = [] | |
crawl(GEMS).each_line do |line| | |
case line | |
when /\/([^\/]+?)" class="gems__gem__name"/ | |
gems << gem if gem | |
state = nil | |
gem = {:title=>$1} | |
when /class="gems__gem__version">(.*)</ | |
state = nil | |
gem[:version] = $1 | |
when /class="gems__gem__downloads__count"/ | |
state = :count | |
when /^\s*([,\d]+)\s*$/ | |
gem[state] = $1.gsub(',','').to_i if state | |
end | |
end | |
gems << gem if gem | |
gems_table = gems.map {|gem| | |
"<tr><td><a href='https://rubygems.org/gems/#{gem[:title]}' target='_RubyGems'>#{gem[:title]}</a></td><td align='center'>#{gem[:version]}</td><td align='right'><a href='http://bestgems.org/gems/#{gem[:title]}' target='_RubyGems' title='BestGems'>#{gem[:count].to_s.gsub(/(\d)(?=(\d{3})+(?!\d))/, '\1,')}</a></td></tr>" | |
}.join("\n") | |
# | |
# GitHub | |
# | |
STDERR.puts | |
STDERR.puts 'GitHub...' | |
state = nil | |
repo = nil | |
repos = [] | |
crawl(HUB).each_line do |line| | |
case line | |
when /\/([^\/]+?)" itemprop="name/ | |
repos << repo if repo | |
state = nil | |
repo = {:title=>$1} | |
when /itemprop="description">/ | |
state = :desc | |
when /aria-label="stars?"/ | |
state = :stars | |
when /aria-label="forks?"/ | |
state = :forks | |
when /aria-label="Contribution"/ | |
state = nil | |
when /^\s*([,\d]+)\s*$/ | |
repo[state] = $1.gsub(',','').to_i if state && repo | |
state = nil | |
else | |
if state == :desc | |
repo[:desc] = line.chomp.strip | |
repo[:desc] = repo[:desc][0...45] + '...' if repo[:desc].length > 45 | |
state = nil | |
end | |
end | |
end | |
repos << repo if repo | |
repos_table = repos.sort_by {|repo| -(repo[:stars]||0)}.map {|repo| | |
"<tr><td><a href='https://github.com/suchowan/#{repo[:title]}' target='_GitHub'>#{repo[:title]}</a></td><td>#{repo[:desc]}</td><td align='center'>#{repo[:stars]||0}</td><td align='center'>#{repo[:forks]||0}</td></tr>" | |
}.join("\n") | |
# | |
# Qiita | |
# | |
STDERR.puts | |
STDERR.puts 'Qiita...' | |
item = nil | |
items = [] | |
crawl(QIITA).scan(/posted on (.+?)\s*<|href="\/suchowan\/items\/([a-z0-9]+?)">\s*(.+?)<\/a>|<li><span class="fa fa-like"><\/span>\s*([,\d]+)\s*<\/li>/) do | |
date, url, title, like = $~[1..4] | |
case | |
when date | |
items << item if item && (item[:like] || item[:stock] > 0) | |
item = {:date=>Date.strptime(date, '%B %d, %Y').to_s} | |
when item && title =~ /[^\d]/ | |
item[:url] = url | |
item[:title] = title | |
item[:stock] = stocks(url[/[a-z0-9]+$/i]).length | |
when item && title =~ /^[,\d]+$/ | |
item[:comment] = title.gsub(',','').to_i | |
when item && like | |
item[:like] = like.gsub(',','').to_i | |
end | |
end | |
items << item if item | |
items_table = items.map {|item| | |
"<tr><td><a href='https://qiita.com/suchowan/items/#{item[:url]}' target='_Qiita'>#{item[:title]}</a></td><td align='center'>#{item[:like]||0}</td><td align='center'>#{item[:stock]||0}</td><td align='Center'>#{item[:date].sub('20','')}</td></tr>" | |
}.join("\n") | |
# | |
# Linked Data | |
# | |
STDERR.puts | |
STDERR.puts 'Linked Data...' | |
state = nil | |
lod = nil | |
lods = [{:url =>'https://web.archive.org/web/20181129223238/http://lod.sfc.keio.ac.jp/challenge2014/show_status.php?id=b015', | |
:title=>'When.exe Ruby版', :date=>'2015-01-18'}, | |
{:url =>'https://web.archive.org/web/20181129223238/http://lod.sfc.keio.ac.jp/challenge2014/show_status.php?id=d070', | |
:title=>'When.exe Ruby版による日本暦注データセット', :date=>'2015-01-18', | |
:usage=>"<br/>(<a href='https://web.archive.org/web/20181129223238/http://2016.lodc.jp/resource.html#d080' target='_Lod'>データセットの概要と使い方</a>)", | |
:award=>"<a href='https://web.archive.org/web/20181129223238/http://lod.sfc.keio.ac.jp/blog/?p=2586#dataset' target='_Lod'>データセット<br/>部門優秀賞</a>"}] | |
crawl(LOD).each_line do |line| | |
case line | |
when /class="title" href="(.+?)"/ | |
lods << lod if lod | |
state = :url | |
lod = {:url=>$1} | |
when /(Updated Date|更新)\s*:\s*([\/\d]+)/ | |
state = nil | |
lod[:date] = $2.gsub('/', '-') | |
when /^\s*(.+)\s$/ | |
if state | |
lod[:title] = $1 | |
state = nil | |
end | |
end | |
end | |
lods << lod if lod | |
lods.each do |lod| | |
lod[:usage] = "(<a href='http://hosi.org:8080/z/books/bunko/index.htm' target='_Books'>文庫一覧</a>)" if lod[:title] =~ /蔵書/ | |
end | |
lods_table = lods.sort {|a,b| b[:date] <=> a[:date]}.map {|lod| | |
"<tr><td><a href='#{lod[:url]}' target='_Lod'>#{lod[:title]}</a>#{lod[:usage]}</td><td align='center'>#{lod[:award]}</td><td align='Center'>#{lod[:date].sub('20','')}</td></tr>" | |
}.join("\n") | |
STDERR.puts | |
STDERR.puts 'Generating...' | |
puts Contents % [Time.now.iso8601, gems_table, repos_table, items_table, lods_table] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment