.gitignore

*~
/local/
/perl
/prove
/plackup
/Makefile.setupenv
/cin
/config/perl/libs.txt
/config/
/modules/
/deps/

README

はてなIDとパスワードを入力すると、自分の投稿したデータを生データ
(はてなの JSON API 形式
<http://developer.hatena.ne.jp/ja/documents/nano/timeline/nativelist#json>)
で保存します。遡れるだけ遡って取得します。

はてなハイク2、はてなココ、はてなOneの投稿は全部取れそうです。
ただしはてなココの古いコメントは取れなそうです。

はてなハイク、はてなダイアリー、はてなブックマーク、はてなフォトライフ、
人力検索はてなの投稿は最近のもの (はてなOne以降?) は取れそうです。

画像は URL だけで、データは含まれていません。
はてなブログは本文データが入っていなそうです。

取得できるのは自分の投稿だけです。

* 使い方

  $ git clone https://gist.github.com/11209675bd8ff46a1e98.git
  $ make deps
  $ ./perl get-timeline.pl
  User: ...
  Password: ...
  $ ls activities.json*

* ライセンス

Copyright 2013-2014 Wakaba <wakaba@suikawiki.org>.
 
This library is free software; you can redistribute it and/or modify
it under the same terms as Perl itself.

get-timeline.pl

use strict;
use warnings;
use URI::Escape;
use Web::UserAgent::Functions;
use JSON::PS;

sub login ($$) {
  my ($user, $pass) = @_;
  my (undef, $res) = http_post
      url => q<https://www.hatena.ne.jp/login>,
      max_redirect => 0,
      params => {
        name => $user,
        password => $pass,
      };
  my $cookie = $res->header ('Set-Cookie') || '';
  if ($cookie =~ /\brk=([^;]+)/) {
    return $1;
  } else {
    return undef;
  }
} # login

sub save ($$) {
  my ($rk, $url) = @_;
  my $file_name = $url;
  $file_name =~ s/\?.*$//s;
  $file_name =~ /([0-9A-Za-z_.-]+)$/;
  $file_name = $1;
  if ($url =~ /reftime=(?:%2B|-)([0-9]+)/) {
    $file_name .= '.' . $1;
  }
  print "<$url> => $file_name...\n";
  my %param;
  if ($url =~ s{\?(.*)$}{}) {
    %param = map { uri_unescape $_ } map { split /=/, $_, 2 } split /&/, $1;
  }
  {
    my (undef, $res) = http_get
        url => $url, cookies => {rk => $rk}, params => \%param,
        timeout => 600;
    redo if $res->is_error;
    open my $file, '>', $file_name or die "$0: $file_name: $!";
    print $file $res->content;
    defined wantarray ? json_bytes2perl $res->content : undef;
  }
} # save

sub save_tl ($$) {
  my ($user, $rk) = @_;

  my $next_url = qq<http://h.hatena.ne.jp/$user/activities.json?per_page=1000&location=http://h2.hatena.ne.jp/>;
  while (1) {
    my $json = save $rk, $next_url;
    last unless @{$json->{items}};
    $next_url = $json->{older_url};
  }
} # save_tl

print "User: ";
my $user = <>;
chomp $user;
print "Password: ";
my $pass = <>;
chomp $pass;

my $rk = login $user, $pass or die "Login failed";
save_tl $user, $rk;

=head1 LICENSE

Copyright 2013-2014 Wakaba <wakaba@suikawiki.org>.

This library is free software; you can redistribute it and/or modify
it under the same terms as Perl itself.

=cut

Makefile

all:

WGET = wget
CURL = curl
GIT = git

## ------ Setup ------

deps: git-submodules pmbp-install

git-submodules:
	$(GIT) submodule update --init
	$(GIT) clone git://github.com/wakaba/perl-web-useragent-functions modules/web-useragent-functions || (cd modules/web-useragent-functions && $(GIT) checkout master && $(GIT) pull)
	$(GIT) clone git://github.com/wakaba/perl-json-ps modules/json-ps || (cd modules/json-ps && $(GIT) checkout master && $(GIT) pull)
	mkdir -p config/perl
	echo '- "../../modules/json-ps" tests' > config/perl/pmbp-exclusions.txt
	echo '- "../../modules/web-useragent-functions" tests anyevent anyeventsocks oauth cli testws wsse' > config/perl/pmbp-exclusions.txt

local/bin/pmbp.pl:
	mkdir -p local/bin
	$(WGET) -O $@ https://raw.github.com/wakaba/perl-setupenv/master/bin/pmbp.pl
pmbp-upgrade: local/bin/pmbp.pl
	perl local/bin/pmbp.pl --update-pmbp-pl
pmbp-update: git-submodules pmbp-upgrade
	perl local/bin/pmbp.pl --update
pmbp-install: pmbp-upgrade
	perl local/bin/pmbp.pl --install \
	    --install-module URI::Escape \
            --create-perl-command-shortcut perl \
            --create-perl-command-shortcut prove