Skip to content

Instantly share code, notes, and snippets.

@suchowan
Last active November 27, 2019 08:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save suchowan/68a2ae48474959a93da0604abd20ec14 to your computer and use it in GitHub Desktop.
Save suchowan/68a2ae48474959a93da0604abd20ec14 to your computer and use it in GitHub Desktop.
# -*- coding: utf-8 -*-
=begin
This script was written by Takashi SUGA on January 2017-November 2019
You may use and/or modify this file according to the license described in the MIT LICENSE.txt file https://raw.githubusercontent.com/suchowan/watson-api-client/master/LICENSE.
=end
require 'open-uri'
require 'openssl'
require 'json'
require 'rest-client'
Encoding.default_external = 'UTF-8'
Encoding.default_internal = 'UTF-8'
GEMS = 'https://rubygems.org/profiles/suchowan'
HUB = 'https://github.com/suchowan?tab=repositories'
GIST = 'https://gist.github.com/suchowan'
LOD = 'http://user.linkdata.org/search?keyword=%E9%A0%88%E8%B3%80%E9%9A%86&category=all'
QIITA = 'https://qiita.com/suchowan'
QIITA_API = RestClient::Resource.new('http://qiita.com/api/v2')
def stocks(id)
STDERR.puts " stocks - #{id}"
JSON.parse(QIITA_API["/items/#{id}/stockers"].get.body)
end
def crawl(url)
STDERR.puts ' crawl'
args = [url, 'r:utf-8']
args << {:ssl_verify_mode=>OpenSSL::SSL::VERIFY_NONE} if url =~ /^https:/
open(*args) do |source|
contents = source.read
# STDERR.puts [contents.length, contents[0...80]]
return contents
end
end
Contents = <<Contents
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Related Links - suchowan's Home Pag</title>
</head>
<style type="text/css">
<!--
body{
background-image : url("wall.png");
background-repeat : no-repeat;
background-attachment : fixed;
}
-->
</style>
<body bgcolor="#F0F0F0" text="Black" link="Blue" alink="Fuchsia" vlink="Green">
<small>
<table width='100%%'>
<tr><td colspan='2' align='right'><small>Updated: %s</small></td></tr>
<tr>
<td width='44%%' align='center'><a href="http://hosi.org:3000" target="_top" title="今日の暦 - 暦の相互変換"><font color='Green'><h1>hosi.org</h1></font><font color='Orange'>暦の相互変換</font></a> <font color='Orange'>( <a href="http://www2u.biglobe.ne.jp/~suchowan/when_exe_wiki.html" target='_top' title='when.exe Wiki の入口です'>Wiki</a>, <a href="https://suchowan.at.webry.info/" target='_blank' title="suchowan's blog を別タブで表示します">Blog</a> )</font></td>
<td width='56%%'>
<table border='1' width='100%%'>
<tr><th><a href='#{GEMS}' target='_RubyGems'>RubyGems</a></th><th>Version</th><th>Download</th></tr>
%s
</table>
</td></tr>
<tr><td colspan='2'>
<table border='1' width='100%%'>
<tr><th><a href='#{HUB}' target='_GitHub'>GitHub</a> (<a href='#{GIST}' target='_Gist'>Gist</a>)</th><th>Description</th><th>Star</th><th>Fork</th></tr>
%s
</table>
</td></tr>
<tr><td colspan='2'>
<table border='1' width='100%%'>
<tr><th><a href='#{QIITA}' target='_Qiita'>Qiita</a></th><th>Like</th><th>Stock</th><th>Posted</th></tr>
%s
</table>
</td></tr>
<tr><td colspan='2'>
<table border='1' width='100%%'>
<tr><th><a href='#{LOD.gsub('%','%%')}' target='_Lod'>Linked Data</a></th><th>Award</th><th>Updated</th></tr>
%s
</table>
</td></tr>
<tr><td colspan='2' align='center'><font color='Maroon'><b><big>hosi.org - service ports</big></b></font></td></tr>
<tr><td colspan='2'>
<table border='1' width='100%%'>
<tr><th>Port</th><th>Service</th><th>Port</th><th>Service</th></tr>
<tr><td align='right'>80</td><td><a href='http://hosi.org:80' target='_top' title='具体的なURLに応じてhosi.org代表サーバが処理を振り分けます。ユーザは意識しないでよいでしょう'>代表ポート</a>(転送など)</td>
<td align='right'>4000</td><td><a href='http://hosi.org:4000/bookmarks' target='_top' title='GitHub の bookmarks のうち sig.url.txt に基づいてブックマーク・ツリーを表示します'>ブックマーク</a></td></tr>
<tr><td align='right'>3000</td><td><a href='http://hosi.org:3000' target='_top' title='「今日の暦」などを表示するWhen.exe テストサーバです'>When.exe テスト(メイン)</a></td>
<td align='right'>8080</td><td><a href='http://hosi.org:8080/cgi-bin/conv.cgi?m=01000' target='_top' title='GitHub の a_converter をオンラインで動かしています'>進法&単位変換サービス</a></td></tr>
<tr><td align='right'>3030</td><td><a href='http://www2u.biglobe.ne.jp/~suchowan/wiki/query.html' target='_top' title='「日本暦注データセット」の SPARQL エンドポイントです。SPARQLでアクセスしてください'>日本暦注データセット<br/>問合せ</a></td>
<td align='right'>8080</td><td><a href='http://hosi.org:8080/disk' target='_top' title='暦の会会員限定の様々な情報を置いています'>暦の会関連 諸々データ置き場</a><br/>(ID,パスワード要)</td></tr>
<tr><td align='right'>4000</td><td><a href='http://hosi.org:4000' target='_top' title='「今日の暦」などを表示するWhen.exe テストサーバ(予備)です。Rails のバージョンが古く少し動作が遅いです'>When.exe テスト(サブ)</a></td>
<td align='right'>8090</td><td><a href='http://hosi.org:8090' target='_top' title='Qiitaの「機械学習関連情報の収集と分類(構想)」に基づいて Fess により情報収集した結果を提供しています'>FESS による機械学習関連情報検索</a></td></tr>
</table>
</td></tr>
<tr>
<td align='center' colspan='2'>
<table><tr>
<td><a href="https://independent.academia.edu/TakashiSUGA" target='_top' title='Academia に置いた論文'><font color=#FF0000>Academia</font></a></td>
<td><a href="https://twitter.com/suchowan/with_replies" target='_top' title='Blog は Twitter と連動しています'><font color=#E00000>Twitter</font></a></td>
<td><a href="http://hosi.org/cgi-bin/links.cgi" target='mainFrame' title='大昔の私のブックマークを公開します。リンク切れになったサイトも多いです'><font color=#c000c0>Links</font></a></td>
<td><a href="http://calndr-l.10958.n7.nabble.com/template/NamlServlet.jtp?macro=search_page&node=2&query=Suga" target='_Sig' title='Calndr-Lの関連投稿です'><font color=#A000A0>Calndr-L</font></a></td>
<td><a href="http://suchowan.at.webry.info/" target='mainFrame' title="suchowan's blogです"><font color=#008040>Blog</font></a></td>
<td><a href="http://www2u.biglobe.ne.jp/~suchowan/when_exe_wiki.html" target='_top' title='Wiki の入口です'><font color=#008060>Wiki</font></a></td>
<td align='Center'><small><a href="http://www.asahi-net.or.jp/~dd6t-sg/book_a.html" target='mainFrame' title='私の本棚の写真を蔵書目録がわりに作ってみました'><font color=Teal>Book<br/>Shelves</font></a><br/>
<a href="http://hosi.org/cgi-bin/links.cgi?New_Room" target='mainFrame' title='転居後の写真です'><font color=Blue>(New)</font></a></small></td>
<td align='Center'><small><a href="http://www.asahi-net.or.jp/~dd6t-sg/simulation.html" target='mainFrame' title='その他自作の文章・データやプログラムを紹介します'><font color=#400080>Over<br/>decades<br/>ago</font></a></small></td>
<td align='Center'><small><a href="http://www.asahi-net.or.jp/~dd6t-sg/whenhome_contents.html" target="mainFrame" title='When.exe MS-DOS版'><font color=#300060>When.exe<br/>MS-DOS<br/>version</font></a></small></td>
<td align='Center'><small><a href="http://www.asahi-net.or.jp/~dd6t-sg/index.htm" target="_top" title='旧ホームページです'><font color=#200040>Old<br/>Home<br/>Page</font></a></small></td>
</tr></table>
</td>
</tr>
<tr>
<td align='center' colspan='2'>
<a target="_blank" href="http://www.mebius.co.jp/" title='「Mebuis」へ行きます'><img width="120" src="http://www.mebius.co.jp/img/mebiuslogo.png"/></a>
<a target="_blank" href="http://www.asahi-net.or.jp/~nr8c-ab/" title='旧ホームページのバナーを作ってくださった Apollon さんのサイトです'><img width="120" src="http://www.asahi-net.or.jp/~nr8c-ab/bna2.gif"/></a>
<!-- <a target="_blank" href="http://www.mojikyo.org/" title='本サイトで特殊な漢字を表記する際、文字鏡のフォントを使用しています'><img width="120" src="http://www.mojikyo.org/img/mologo.gif"/></a> -->
<a target="_blank" href="http://www.mojikyo.org/" title='本サイトで特殊な漢字を表記する際、文字鏡のフォントを使用しています'><img width="120" src="http://www.asahi-net.or.jp/~dd6t-sg/mologo.gif"/></a>
<br/>Since 1998-09-20<br/>Copyright <a href='http://hosi.org:3000/TakashiSuga.rdf' target='_reference'>Takashi SUGA</a></td></tr>
</table>
</small>
</body>
</html>
Contents
#
# RubyGems
#
STDERR.puts 'RubyGems...'
state = nil
gem = nil
gems = []
crawl(GEMS).each_line do |line|
case line
when /\/([^\/]+?)" class="gems__gem__name"/
gems << gem if gem
state = nil
gem = {:title=>$1}
when /class="gems__gem__version">(.*)</
state = nil
gem[:version] = $1
when /class="gems__gem__downloads__count"/
state = :count
when /^\s*([,\d]+)\s*$/
gem[state] = $1.gsub(',','').to_i if state
end
end
gems << gem if gem
gems_table = gems.map {|gem|
"<tr><td><a href='https://rubygems.org/gems/#{gem[:title]}' target='_RubyGems'>#{gem[:title]}</a></td><td align='center'>#{gem[:version]}</td><td align='right'><a href='http://bestgems.org/gems/#{gem[:title]}' target='_RubyGems' title='BestGems'>#{gem[:count].to_s.gsub(/(\d)(?=(\d{3})+(?!\d))/, '\1,')}</a></td></tr>"
}.join("\n")
#
# GitHub
#
STDERR.puts
STDERR.puts 'GitHub...'
state = nil
repo = nil
repos = []
crawl(HUB).each_line do |line|
case line
when /\/([^\/]+?)" itemprop="name/
repos << repo if repo
state = nil
repo = {:title=>$1}
when /itemprop="description">/
state = :desc
when /aria-label="stars?"/
state = :stars
when /aria-label="forks?"/
state = :forks
when /aria-label="Contribution"/
state = nil
when /^\s*([,\d]+)\s*$/
repo[state] = $1.gsub(',','').to_i if state && repo
state = nil
else
if state == :desc
repo[:desc] = line.chomp.strip
repo[:desc] = repo[:desc][0...45] + '...' if repo[:desc].length > 45
state = nil
end
end
end
repos << repo if repo
repos_table = repos.sort_by {|repo| -(repo[:stars]||0)}.map {|repo|
"<tr><td><a href='https://github.com/suchowan/#{repo[:title]}' target='_GitHub'>#{repo[:title]}</a></td><td>#{repo[:desc]}</td><td align='center'>#{repo[:stars]||0}</td><td align='center'>#{repo[:forks]||0}</td></tr>"
}.join("\n")
#
# Qiita
#
STDERR.puts
STDERR.puts 'Qiita...'
item = nil
items = []
crawl(QIITA).scan(/posted on (.+?)\s*<|href="\/suchowan\/items\/([a-z0-9]+?)">\s*(.+?)<\/a>|<li><span class="fa fa-like"><\/span>\s*([,\d]+)\s*<\/li>/) do
date, url, title, like = $~[1..4]
case
when date
items << item if item && (item[:like] || item[:stock] > 0)
item = {:date=>Date.strptime(date, '%B %d, %Y').to_s}
when item && title =~ /[^\d]/
item[:url] = url
item[:title] = title
item[:stock] = stocks(url[/[a-z0-9]+$/i]).length
when item && title =~ /^[,\d]+$/
item[:comment] = title.gsub(',','').to_i
when item && like
item[:like] = like.gsub(',','').to_i
end
end
items << item if item
items_table = items.map {|item|
"<tr><td><a href='https://qiita.com/suchowan/items/#{item[:url]}' target='_Qiita'>#{item[:title]}</a></td><td align='center'>#{item[:like]||0}</td><td align='center'>#{item[:stock]||0}</td><td align='Center'>#{item[:date].sub('20','')}</td></tr>"
}.join("\n")
#
# Linked Data
#
STDERR.puts
STDERR.puts 'Linked Data...'
state = nil
lod = nil
lods = [{:url =>'https://web.archive.org/web/20181129223238/http://lod.sfc.keio.ac.jp/challenge2014/show_status.php?id=b015',
:title=>'When.exe Ruby版', :date=>'2015-01-18'},
{:url =>'https://web.archive.org/web/20181129223238/http://lod.sfc.keio.ac.jp/challenge2014/show_status.php?id=d070',
:title=>'When.exe Ruby版による日本暦注データセット', :date=>'2015-01-18',
:usage=>"<br/>(<a href='https://web.archive.org/web/20181129223238/http://2016.lodc.jp/resource.html#d080' target='_Lod'>データセットの概要と使い方</a>)",
:award=>"<a href='https://web.archive.org/web/20181129223238/http://lod.sfc.keio.ac.jp/blog/?p=2586#dataset' target='_Lod'>データセット<br/>部門優秀賞</a>"}]
crawl(LOD).each_line do |line|
case line
when /class="title" href="(.+?)"/
lods << lod if lod
state = :url
lod = {:url=>$1}
when /(Updated Date|更新)\s*:\s*([\/\d]+)/
state = nil
lod[:date] = $2.gsub('/', '-')
when /^\s*(.+)\s$/
if state
lod[:title] = $1
state = nil
end
end
end
lods << lod if lod
lods.each do |lod|
lod[:usage] = "(<a href='http://hosi.org:8080/z/books/bunko/index.htm' target='_Books'>文庫一覧</a>)" if lod[:title] =~ /蔵書/
end
lods_table = lods.sort {|a,b| b[:date] <=> a[:date]}.map {|lod|
"<tr><td><a href='#{lod[:url]}' target='_Lod'>#{lod[:title]}</a>#{lod[:usage]}</td><td align='center'>#{lod[:award]}</td><td align='Center'>#{lod[:date].sub('20','')}</td></tr>"
}.join("\n")
STDERR.puts
STDERR.puts 'Generating...'
puts Contents % [Time.now.iso8601, gems_table, repos_table, items_table, lods_table]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment