Skip to content

Instantly share code, notes, and snippets.

@neetsdkasu

neetsdkasu/.gitignore Secret

Last active July 29, 2020 10:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save neetsdkasu/1c306b3504d8597aab28f913eb22ca6a to your computer and use it in GitHub Desktop.
Save neetsdkasu/1c306b3504d8597aab28f913eb22ca6a to your computer and use it in GitHub Desktop.
Download ZIP
Tweet.js解析
Raw
.gitignore
*.json
*.exe
*.js
*.xml
*.ico
*.txt
*.mt
*.html
Raw
def.go
package main
import (
"strings"
)
type HashTag struct {
Indices []string
Text string
}
type SizeObject struct {
H string
Resize string
W string
}
type SizeSetObject struct {
Large *SizeObject
Medium *SizeObject
Small *SizeObject
Thumb *SizeObject
}
type AdditionalMediaInfo struct {
Description string
Embeddable bool
Monetizable bool
Title string
}
type VideoVariantObject struct {
Bitrate string
Content_type string
Url string
}
type VideoInfo struct {
Aspect_ratio []string
Duration_millis string
Variants []*VideoVariantObject
}
type MediaObject struct {
Additional_media_info *AdditionalMediaInfo
Display_url string
Expanded_url string
Id string
Id_str string
Indices []string
Media_url string
Media_url_https string
Sizes *SizeSetObject
Source_status_id string
Source_status_id_str string
Source_user_id string
Source_user_id_str string
Type string
Url string
Video_info *VideoInfo
}
type SymbolObject struct {
Indices []string
Text string
}
type UrlObject struct {
Display_url string
Expanded_url string
Indices []string
Url string
}
type MentionObject struct {
Id string
Id_str string
Indices []string
Name string
Screen_name string
}
type Entities struct {
Hashtags []*HashTag
Media []*MediaObject // おそらく画像付きツイートしたときの1枚目の画像
Symbols []*SymbolObject // キャッシュタグ $ のやつ
Urls []*UrlObject // おそらくメディア以外の
User_mentions []*MentionObject // メンションもリンプライも含む
}
type ExtendedEntities struct {
Media []*MediaObject
}
type TweetObject struct {
Contributors []string // チームアカウントとしてツイートした場合のツイートしたユーザID
Created_at string // tweet date
Display_text_range []string
Entities *Entities
Extended_entities *ExtendedEntities // 画像付きツイートした場合の画像リンク
Favorite_count string
Favorited bool // all false
Full_text string // ツイートのテキスト
Id string // tweet status id(number)
Id_str string
In_reply_to_screen_name string // 多くのユーザがIDと誤認してるやつ (unieque name)
In_reply_to_status_id string // リプでここしか値が入ってない場合があるぽい？
In_reply_to_status_id_str string
In_reply_to_user_id string // 本来のユーザID(数字)
In_reply_to_user_id_str string
Lang string
Possibly_sensitive bool
Retweet_count string
Retweeted bool // all false
Source string // application for tweeting
Truncated bool
}
type MainObject struct {
Tweet TweetObject
}
func (this *MainObject) IsRetweet() bool {
if this == nil {
return false
}
return strings.HasPrefix(this.Tweet.Full_text, "RT @")
}
func (this *MainObject) IsReply() bool {
if this == nil {
return false
}
return len(this.Tweet.In_reply_to_screen_name) > 0 ||
len(this.Tweet.In_reply_to_status_id) > 0 ||
len(this.Tweet.In_reply_to_user_id) > 0
}
func (this *MainObject) HasEntities() bool {
if this == nil {
return false
}
return this.Tweet.Entities != nil
}
func (this *Entities) HasMedia() bool {
if this == nil {
return false
}
return len(this.Media) > 0
}
Raw
frommastodon.go
package main
import (
"strings"
)
func IsMyToot(u string) bool {
if strings.Contains(u, "mstdn.jp") ||
strings.Contains(u, "pawoo.net") ||
strings.Contains(u, "friends.nico") {
if !strings.Contains(u, "neetsdkasu") {
return false
}
if strings.HasSuffix(u, "neetsdkasu") {
return false
}
return true
}
return false
}
var QuoteFromMastodon = map[string]string{
"https://pawoo.net/@neetsdkasu/1675189": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/04/19/235858#pawoo1675189\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-04-19T09:05:08+09:00\">2017年04月19日 09:05:08</time>&nbsp;@neetsdkasu@pawoo.net</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://pawoo.net/@neetsdkasu/1675189\" rel=\"noopener noreferrer nofollow external\" title=\"https://pawoo.net/@neetsdkasu/1675189\">#</a></div><p>「トゥ」は「twu」で変換できる、これ豆な☝</p></blockquote>",
"https://mstdn.jp/@neetsdkasu/26310591": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/07/03/235858#mstdn26310591\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-07-03T04:27:25+09:00\">2017年07月03日 04:27:25</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/26310591\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/26310591\">#</a></div><p>Wow! Coder Leonardone competed in Codeforces Round #422 (Div. 2) and gained 125 rating points taking place 621 <a href=\"http://codeforces.com/bestRatingChanges/1172297\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">http://</span><span class=\"ellipsis\">codeforces.com/bestRatingChang</span><span class=\"invisible\">es/1172297</span></a></p></blockquote>",
"https://mstdn.jp/users/neetsdkasu/updates/12709682": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/07/12/235858#mstdn28326747\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-07-12T06:17:58+09:00\">2017年07月12日 06:17:58</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/28326747\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/28326747\">#</a></div><p>Wow! Coder Leonardone competed in Codeforces Round #423 (Div. 2, rated, based on VK Cup Finals) and gained 11 rating points taking place 1,561 <a href=\"http://codeforces.com/bestRatingChanges/1177405\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">http://</span><span class=\"ellipsis\">codeforces.com/bestRatingChang</span><span class=\"invisible\">es/1177405</span></a></p></blockquote>",
"https://mstdn.jp/@neetsdkasu/32285368": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/07/31/235858#mstdn32285368\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-07-31T05:41:59+09:00\">2017年07月31日 05:41:59</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/32285368\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/32285368\">#</a></div><p>Codeforces Hot News!<br />Wow! Coder Leonardone competed in Codeforces Round #426 (Div. 2) and gained +8 rating points taking place 2,490 <br /><a href=\"http://codeforces.com/bestRatingChanges/1195706\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">http://</span><span class=\"ellipsis\">codeforces.com/bestRatingChang</span><span class=\"invisible\">es/1195706</span></a></p></blockquote>",
"https://mstdn.jp/users/neetsdkasu/updates/14059113": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/08/01/235858#mstdn32486901\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-08-01T04:47:43+09:00\">2017年08月01日 04:47:43</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/32486901\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/32486901\">#</a></div><p>Codeforces Hot News!<br />Wow! Coder Leonardone competed in Codeforces Round #427 (Div. 2) and gained +64 rating points taking place 1,493 <br /><a href=\"http://codeforces.com/bestRatingChanges/1199557\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">http://</span><span class=\"ellipsis\">codeforces.com/bestRatingChang</span><span class=\"invisible\">es/1199557</span></a></p></blockquote>",
"https://mstdn.jp/@neetsdkasu/36999549": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/08/25/235858#mstdn36999549\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-08-25T05:48:25+09:00\">2017年08月25日 05:48:25</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/36999549\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/36999549\">#</a></div><p>初１番乗りをＧＥＴしました！！😄</p><p>第154回「今週のアルゴリズム：トーナメントでの想定対戦数は？」正解者発表｜CodeIQ MAGAZINE <a href=\"https://codeiq.jp/magazine/2017/08/53644/\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">codeiq.jp/magazine/2017/08/536</span><span class=\"invisible\">44/</span></a> @codeiqさんから</p></blockquote>",
"https://mstdn.jp/@neetsdkasu/38015247": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/08/30/235858#mstdn38015247\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-08-30T03:45:11+09:00\">2017年08月30日 03:45:11</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/38015247\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/38015247\">#</a></div><p>Codeforces Hot News!<br />Wow! Coder Leonardone competed in Codeforces Round #430 (Div. 2) and gained +50 rating points taking place 1,141 </p><p><a href=\"http://codeforces.com/bestRatingChanges/1228045\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">http://</span><span class=\"ellipsis\">codeforces.com/bestRatingChang</span><span class=\"invisible\">es/1228045</span></a></p></blockquote>",
"https://mstdn.jp/@neetsdkasu/39262835": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/09/05/235858#mstdn39262835\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-09-05T05:12:57+09:00\">2017年09月05日 05:12:57</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/39262835\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/39262835\">#</a></div><p>Codeforces Hot News!<br />Wow! Coder Leonardone competed in Codeforces Round #432 (Div. 2, based on IndiaHacks Final Round 2017) and gained +112 rating points taking place 612 <br /><a href=\"http://codeforces.com/bestRatingChanges/1237279\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">http://</span><span class=\"ellipsis\">codeforces.com/bestRatingChang</span><span class=\"invisible\">es/1237279</span></a></p></blockquote>",
"https://mstdn.jp/@neetsdkasu/44433381": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/10/01/235858#mstdn44433381\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-10-01T17:55:59+09:00\">2017年10月01日 17:55:59</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/44433381\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/44433381\">#</a></div><p>Codeforces Hot News!<br />Wow! Coder Leonardone competed in Codeforces Round #437 (Div. 2, based on MemSQL Start[c]UP 3.0 - Round 2) and gained +5 rating points taking place 1,071 </p><p><a href=\"http://codeforces.com/bestRatingChanges/1263621\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">http://</span><span class=\"ellipsis\">codeforces.com/bestRatingChang</span><span class=\"invisible\">es/1263621</span></a></p></blockquote>",
"https://pawoo.net/@neetsdkasu/11218455": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/05/13/235858#pawoo11218455\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-05-13T09:58:28+09:00\">2017年05月13日 09:58:28</time>&nbsp;@neetsdkasu@pawoo.net</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://pawoo.net/@neetsdkasu/11218455\" rel=\"noopener noreferrer nofollow external\" title=\"https://pawoo.net/@neetsdkasu/11218455\">#</a></div><p>謎めいた評価を受けた </p><a target=\"_blank\" href=\"https://img.pawoo.net/media_attachments/files/000/794/822/original/360d8eb15f7dce1f.png\" rel=\"noopener\"><img src=\"https://img.pawoo.net/media_attachments/files/000/794/822/original/360d8eb15f7dce1f.png\" width=\"100\" alt=\"image-0\"></a></blockquote>",
"https://mstdn.jp/@neetsdkasu/98879873964228083": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/10/24/235858#mstdn98879873964228083\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-10-24T04:32:19+09:00\">2017年10月24日 04:32:19</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/98879873964228083\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/98879873964228083\">#</a></div><p>Codeforces Hot News!<br />Wow! Coder Leonardone competed in Codeforces Round #442 (Div. 2) and gained +149 rating points taking place 608 <a href=\"http://codeforces.com/bestRatingChanges/1280719\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">http://</span><span class=\"ellipsis\">codeforces.com/bestRatingChang</span><span class=\"invisible\">es/1280719</span></a></p></blockquote>",
"https://mstdn.jp/@neetsdkasu/99026974050479636": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/11/19/235858#mstdn99026974050479636\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-11-19T04:01:48+09:00\">2017年11月19日 04:01:48</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/99026974050479636\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/99026974050479636\">#</a></div><p>TopCoderのSRM(div2)で２位を取りますた！！！😄 <br /> </p><a target=\"_blank\" href=\"https://media.mstdn.jp/media_attachments/files/003/084/821/original/a74eefdfcc6fcd79.png\" rel=\"noopener\"><img src=\"https://media.mstdn.jp/media_attachments/files/003/084/821/original/a74eefdfcc6fcd79.png\" width=\"100\" alt=\"image-0\"></a><a target=\"_blank\" href=\"https://media.mstdn.jp/media_attachments/files/003/084/822/original/c462e726ac525375.png\" rel=\"noopener\"><img src=\"https://media.mstdn.jp/media_attachments/files/003/084/822/original/c462e726ac525375.png\" width=\"100\" alt=\"image-1\"></a></blockquote>",
"https://mstdn.jp/@neetsdkasu/99029607761690441": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2017/11/19/235858#mstdn99029607761690441\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2017-11-19T15:11:35+09:00\">2017年11月19日 15:11:35</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/99029607761690441\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/99029607761690441\">#</a></div><p>TopCoderのSRMのレーティングが 710 → 917 に上がって灰色から緑色になったお！！！😄 </p><p> </p><a target=\"_blank\" href=\"https://media.mstdn.jp/media_attachments/files/003/089/754/original/bbaaff1931197d0b.png\" rel=\"noopener\"><img src=\"https://media.mstdn.jp/media_attachments/files/003/089/754/original/bbaaff1931197d0b.png\" width=\"100\" alt=\"image-0\"></a><a target=\"_blank\" href=\"https://media.mstdn.jp/media_attachments/files/003/089/755/original/6f9627270441e2e3.png\" rel=\"noopener\"><img src=\"https://media.mstdn.jp/media_attachments/files/003/089/755/original/6f9627270441e2e3.png\" width=\"100\" alt=\"image-1\"></a></blockquote>",
"https://pawoo.net/@neetsdkasu/99457669660041098": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2018/02/03/235858#pawoo99457669660041098\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2018-02-03T05:33:21+09:00\">2018年02月03日 05:33:21</time>&nbsp;@neetsdkasu@pawoo.net</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://pawoo.net/@neetsdkasu/99457669660041098\" rel=\"noopener noreferrer nofollow external\" title=\"https://pawoo.net/@neetsdkasu/99457669660041098\">#</a></div><p>:pawoo:</p></blockquote>",
"https://mstdn.jp/@neetsdkasu/102734731464706809": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2019/09/04/235858#mstdn102734731464706809\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2019-09-04T23:33:16+09:00\">2019年09月04日 23:33:16</time>&nbsp;@neetsdkasu@mstdn.jp</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/102734731464706809\" rel=\"noopener noreferrer nofollow external\" title=\"https://mstdn.jp/@neetsdkasu/102734731464706809\">#</a></div><p>Solved By neetsdkasu<br />TopCoder: 49<br />CodeForces: 204<br />AtCoder: 471<br />AOJ: 77<br />yukicoder: 199<br />Sum: 1000<br /> <a href=\"https://rating-history.herokuapp.com/index.html?handle_topcoder=neetsdkasu&amp;handle_codeforces=Leonardone&amp;handle_atcoder=neetsdkasu&amp;handle_aoj=neetsdkasu&amp;handle_yukicoder=Leonardone\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">rating-history.herokuapp.com/i</span><span class=\"invisible\">ndex.html?handle_topcoder=neetsdkasu&amp;handle_codeforces=Leonardone&amp;handle_atcoder=neetsdkasu&amp;handle_aoj=neetsdkasu&amp;handle_yukicoder=Leonardone</span></a></p></blockquote>",
"https://pawoo.net/@neetsdkasu/102745981851453414": "<blockquote><div style=\"font-size: 0.8em; color: grey;\"><a href=\"/entry/2019/09/06/235858#pawoo102745981851453414\" style=\"text-decoration: none; color: unset;\"><time datetime=\"2019-09-06T23:14:24+09:00\">2019年09月06日 23:14:24</time>&nbsp;@neetsdkasu@pawoo.net</a>&nbsp;<a style=\"color: black;\" target=\"_blank\" href=\"https://pawoo.net/@neetsdkasu/102745981851453414\" rel=\"noopener noreferrer nofollow external\" title=\"https://pawoo.net/@neetsdkasu/102745981851453414\">#</a></div><p>他のインスタンスの画像取得しなくなったのか<br /></p><blockquote><a target=\"_blank\" href=\"https://mstdn.jp/@neetsdkasu/102745045206930497\" rel=\"noopener noreferrer nofollow external\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">mstdn.jp/@neetsdkasu/102745045</span><span class=\"invisible\">206930497</span></a></blockquote></blockquote>",
}
Raw
loader.go
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"sort"
"strings"
"time"
)
func loadFile(name, file string) ([]*Tweet, error) {
jsonBlob, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
for i := range jsonBlob {
if jsonBlob[i] == '[' {
jsonBlob = jsonBlob[i:]
break
}
}
var tweets []*MainObject
err = json.Unmarshal(jsonBlob, &tweets)
if err != nil {
return nil, err
}
ret := make([]*Tweet, 0, len(tweets))
for _, o := range tweets {
if o.IsRetweet() {
continue
}
if o.IsReply() {
if !okReply(o) {
continue
}
}
tm, err := time.Parse(time.RubyDate, o.Tweet.Created_at)
if err != nil {
return nil, err
}
t := &Tweet{
name,
tm.In(time.Local),
o,
}
ret = append(ret, t)
}
return ret, nil
}
func loadAll() (tweets, idsearch []*Tweet, ok bool) {
tweets = []*Tweet{}
list := []struct {
name string
file string
}{
{neetsdkasu, "n_tweet.js"},
{usakdsteen, "u_tweet.js"},
{enodranoeL, "e_tweet.js"},
}
for _, tg := range list {
fmt.Print("load... ", tg.name, ", ", tg.file, ", ")
tmp, err := loadFile(tg.name, tg.file)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(len(tmp))
tweets = append(tweets, tmp...)
}
fmt.Println("sort...")
sort.Slice(tweets, func(i, j int) bool {
return tweets[i].Time.Before(tweets[j].Time)
})
idsearch = make([]*Tweet, len(tweets))
copy(idsearch, tweets)
fmt.Println("sort...")
sort.Slice(idsearch, func(i, j int) bool {
return strings.Compare(idsearch[i].M.Tweet.Id_str, idsearch[j].M.Tweet.Id_str) < 0
})
ok = true
return
}
Raw
main.go
package main
import (
"fmt"
"html"
"os"
"path/filepath"
"sort"
"strings"
)
var _ = strings.HasPrefix
var _ = html.EscapeString
func main() {
tweets, idsearch, ok := loadAll()
if !ok {
return
}
fmt.Println("count: ", len(tweets))
fmt.Println()
fmt.Println()
for i := range tweets[:1] {
if !tweets[i].M.HasEntities() {
continue
}
if len(tweets[i].M.Tweet.Entities.Media) == 0 {
continue
}
found := false
for _, m := range tweets[i].M.Tweet.Entities.Media {
if m.Video_info != nil {
found = true
break
}
}
if !found && tweets[i].M.Tweet.Extended_entities != nil {
for _, m := range tweets[i].M.Tweet.Extended_entities.Media {
if m.Video_info != nil {
found = true
break
}
}
}
if !found {
continue
}
fmt.Println(tweets[i].Time.Format("2006-01-02T15:04:05-07:00"))
fmt.Println(tweets[i].Time.Format("2006年01月02日 15:04:05"))
fmt.Printf("%+v", tweets[i].M.Tweet)
fmt.Println()
fmt.Printf("%+v", tweets[i].M.Tweet.Entities)
fmt.Println()
fmt.Printf("%+v", tweets[i].M.Tweet.Extended_entities)
fmt.Println()
for _, s := range tweets[i].M.Tweet.Entities.Media {
if s.Video_info != nil {
fmt.Printf("%+v", s)
fmt.Println()
fmt.Printf("%+v", s.Video_info)
fmt.Println()
for _, v := range s.Video_info.Variants {
fmt.Printf("%+v", v)
fmt.Println()
}
}
}
if tweets[i].M.Tweet.Extended_entities != nil {
for _, s := range tweets[i].M.Tweet.Extended_entities.Media {
if s.Video_info != nil {
fmt.Printf("%+v", s)
fmt.Println()
fmt.Printf("%+v", s.Video_info)
fmt.Println()
for _, v := range s.Video_info.Variants {
fmt.Printf("%+v", v)
fmt.Println()
}
}
}
}
fmt.Println()
}
curDate := 0
counts := []int{}
count := 0
flags := uint(0)
theDate := 0
theDateMinT := 0
theDateCount := 0
for _, t := range tweets {
date := t.Time.Year()*100*100 +
int(t.Time.Month())*100 + t.Time.Day()
if curDate != date {
counts = append(counts, count)
if flags+1 == (1 << 5) {
if theDate == 0 || count < theDateMinT {
theDate = curDate
theDateMinT = count
}
theDateCount++
}
count = 1
curDate = date
flags = 0
} else {
count++
}
if t.M.IsReply() && (flags&(1<<4) == 0) {
flags |= 1 << 4
} else if t.M.Tweet.Extended_entities != nil && (flags&(1<<3) == 0) {
flags |= 1 << 3
} else if len(t.M.Tweet.Entities.Media) > 0 && (flags&(1<<0) == 0) {
flags |= 1 << 0
} else if len(t.M.Tweet.Entities.Urls) > 0 && (flags&(1<<1) == 0) {
flags |= 1 << 1
} else if len(t.M.Tweet.Entities.User_mentions) > 0 && (flags&(1<<2) == 0) {
if !t.M.IsReply() {
flags |= 1 << 2
}
}
}
counts = append(counts, count)
counts = counts[1:]
if flags+1 == (1 << 5) {
theDateCount++
if theDate == 0 || count < theDateMinT {
theDate = curDate
theDateMinT = count
}
}
fmt.Println("theDate", theDate)
fmt.Println("theDateMinT", theDateMinT)
fmt.Println("theDateCount", theDateCount)
fmt.Println("days:", len(counts))
fmt.Println("average:", float64(len(tweets))/float64(len(counts)))
sort.Ints(counts)
fmt.Println("min:", counts[0], "max:", counts[len(counts)-1])
fmt.Println("med:", counts[len(counts)/2])
fmt.Println("med3/4:", counts[len(counts)*3/4], len(counts)*3/4)
fmt.Println("med7/8:", counts[len(counts)*7/8], len(counts)*7/8)
fmt.Println("med15/16:", counts[len(counts)*15/16], len(counts)*15/16)
fmt.Println("med31/32:", counts[len(counts)*31/32], len(counts)*31/32)
fmt.Println()
for _, t := range tweets[:] {
date := t.Time.Year()*100*100 +
int(t.Time.Month())*100 + t.Time.Day()
_ = date
if t.M.Tweet.Extended_entities == nil {
continue
}
found := false
for _, m := range t.M.Tweet.Extended_entities.Media {
// if strings.Contains(m.Expanded_url, "video") {
if m.Video_info != nil {
found = true
break
}
}
if !found {
continue
}
// fmt.Printf("%+v", t.M.Tweet)
// fmt.Println()
// for _, m := range t.M.Tweet.Entities.Media {
// fmt.Printf("%+v", m)
// fmt.Println()
// }
// for _, m := range t.M.Tweet.Extended_entities.Media {
// fmt.Printf("%+v", m)
// fmt.Println()
// fmt.Printf("%+v", m.Video_info)
// fmt.Println()
// for _, v := range m.Video_info.Variants {
// fmt.Printf("%+v", v)
// fmt.Println()
// }
// }
// fmt.Println()
theDate = date
fmt.Println("theDate", theDate)
}
ids := map[string]string{}
for _, t := range tweets[:1] {
switch t.M.Tweet.In_reply_to_screen_name {
case neetsdkasu, usakdsteen, enodranoeL:
if len(t.M.Tweet.In_reply_to_user_id_str) > 0 {
ids[t.M.Tweet.In_reply_to_user_id_str] = t.M.Tweet.In_reply_to_screen_name
}
}
}
fmt.Println(ids)
for _, t := range tweets[:0] {
date := t.Time.Year()*100*100 +
int(t.Time.Month())*100 + t.Time.Day()
_ = date
for _, u := range t.M.Tweet.Entities.Urls {
if strings.Contains(u.Expanded_url, "mstdn.jp") ||
strings.Contains(u.Expanded_url, "pawoo.net") ||
strings.Contains(u.Expanded_url, ".nico/") {
if !strings.Contains(u.Expanded_url, "neetsdkasu") {
continue
}
if strings.HasSuffix(u.Expanded_url, "neetsdkasu") {
continue
}
} else {
continue
}
fmt.Println(u.Expanded_url)
fmt.Println("https://twitter.com/" + t.Screen_name + "/status/" + t.M.Tweet.Id_str)
if strings.Contains(u.Expanded_url, "upd") {
fmt.Println(t.M.Tweet.Full_text)
}
}
}
theDate = 20190907
targets := make([]*Tweet, 0, theDateMinT)
for _, t := range tweets[:] {
date := t.Time.Year()*100*100 +
int(t.Time.Month())*100 + t.Time.Day()
if date == theDate {
targets = append(targets, t)
}
}
tweetsByDay := [][]*Tweet{}
{
temp := []*Tweet{}
curDate = 0
for _, t := range tweets[:] {
date := t.Time.Year()*100*100 +
int(t.Time.Month())*100 + t.Time.Day()
if date != curDate {
if curDate != 0 {
tweetsByDay = append(tweetsByDay, temp)
temp = []*Tweet{}
}
curDate = date
}
temp = append(temp, t)
}
tweetsByDay = append(tweetsByDay, temp)
}
// output, err := os.Create("test.mt")
output, err := os.Create("tweet.mt")
if err != nil {
fmt.Println(err)
return
}
defer output.Close()
_ = idsearch
const Author = "Leonardone"
categories := []string{"from-tw"}
// fmt.Fprint(output, GetMT(10, "<p>UL ver.</p>", makeUL(idsearch, targets)))
// fmt.Fprint(output, GetTestMT(20, "<p>OL ver.</p>", makeOL(idsearch, targets)))
// fmt.Fprint(output, GetTestMT(30, "<p>DL ver.</p>", makeDL(idsearch, targets)))
// fmt.Fprint(output, GetTestMT(40, "<p>Table ver.</p>", makeTable(idsearch, targets)))
// for i, ts := range tweetsByDay[:15] {
for i, ts := range tweetsByDay {
tm := ts[0].Time
title := tm.Format("2006年01月02日") + "のTweets"
body := fmt.Sprintf(
`<span><time datetime="%s">%s</time>のTweets</span>`,
tm.Format("2006-01-02T15:04:05-07:00"),
tm.Format("2006年01月02日"),
)
if i > 0 {
ytm := tweetsByDay[i-1][0].Time
prev := fmt.Sprintf(
`&lt;&nbsp;<a href="/entry/%s"><span><time datetime="%s">%s</time>のTweets</span></a>&nbsp;|&nbsp;`,
tweetsByDay[i-1][0].MTBaseName(),
ytm.Format("2006-01-02T15:04:05-07:00"),
ytm.Format("2006年01月02日"),
)
body = prev + body
}
if i+1 < len(tweetsByDay) {
ttm := tweetsByDay[i+1][0].Time
body += fmt.Sprintf(
`&nbsp;|&nbsp;<a href="/entry/%s"><span><time datetime="%s">%s</time>のTweets</span></a>&nbsp;&gt;`,
tweetsByDay[i+1][0].MTBaseName(),
ttm.Format("2006-01-02T15:04:05-07:00"),
ttm.Format("2006年01月02日"),
)
}
body = "<div>" + body + "</div>"
ext := makeOL(idsearch, ts) + body
entry := &BlogEntry{
Author: Author,
Title: title,
Year: tm.Year(),
Month: int(tm.Month()),
Day: tm.Day(),
Hour: 23,
Minute: 59,
Second: 59,
Categories: categories,
Body: body,
ExtendedBody: ext,
}
fmt.Fprint(output, entry.String())
}
tweetsByMonth := [][][]*Tweet{}
{
curMonth := 0
temp := [][]*Tweet{}
for _, t := range tweetsByDay[:] {
month := t[0].Time.Year()*100*100 +
int(t[0].Time.Month())*100
if curMonth != month {
if curMonth != 0 {
tweetsByMonth = append(tweetsByMonth, temp)
temp = [][]*Tweet{}
}
curMonth = month
}
temp = append(temp, t)
}
tweetsByMonth = append(tweetsByMonth, temp)
}
fmt.Println("tweetsByMonth:", len(tweetsByMonth))
const monthly = "monthly"
if err = os.MkdirAll(monthly, os.ModeDir); err != nil {
println(err.Error())
return
}
links, err2 := os.Create(filepath.Join(monthly, "index.html"))
if err2 != nil {
println(err2.Error())
return
}
defer links.Close()
fmt.Fprintln(links, TestHtmlHead)
fmt.Fprintln(links, "<ol>")
for _, ms := range tweetsByMonth[:] {
tm := ms[0][0].Time
filename := filepath.Join(
monthly,
tm.Format("2006-01")+".html",
)
fmt.Fprintf(links,
`<li><a href="%[1]s" target="_blank" rel="noopener">%[1]s</a></li>`,
tm.Format("2006-01")+".html",
)
fmt.Println(filename, "...")
{
temp, err1 := os.Create(filename)
if err1 != nil {
println(err1.Error())
return
}
defer temp.Close()
fmt.Fprintln(temp, TestHtmlHead)
for _, ds := range ms {
date := ds[0].Time
entry := &BlogEntry{
Author: Author,
Title: date.Format("2006年01月02日"),
Year: date.Year(),
Month: int(date.Month()),
Day: date.Day(),
Hour: 23,
Minute: 59,
Second: 59,
Categories: categories,
Body: "<h2>" + date.Format("2006年01月02日") + "</h2>",
ExtendedBody: makeOL(idsearch, ds),
}
fmt.Fprint(temp, entry.String())
}
fmt.Fprintln(temp, TestHtmlTail)
}
}
fmt.Fprintln(links, "</ol>")
fmt.Fprintln(links, TestHtmlTail)
// do_for_masotdon(idsearch)
if false {
if anq := searchTweet(idsearch, "769057809606533120"); anq != nil {
fmt.Printf("%+v\n", anq)
fmt.Printf("%+v\n", anq.M.Tweet)
fmt.Printf("%+v\n", anq.M.Tweet.Entities.Hashtags)
fmt.Printf("%+v\n", anq.M.Tweet.Entities.Media)
fmt.Printf("%+v\n", anq.M.Tweet.Entities.Symbols)
fmt.Printf("%+v\n", anq.M.Tweet.Entities.Urls)
fmt.Printf("%+v\n", anq.M.Tweet.Entities.User_mentions)
}
}
}
func do_for_masotdon(idsearch []*Tweet) {
for_mastodon, err := os.Create("for_mastodon.txt")
if err != nil {
fmt.Println(err)
return
}
defer for_mastodon.Close()
mastodon := "985238337463107584"
if mt := searchTweet(idsearch, mastodon); mt != nil {
fmt.Fprint(for_mastodon, mt.Blockquote(idsearch, nil))
}
}
const TestHtmlHead = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>test</title>
<style>
.leonardonetweet > li:target {
background-color: rgba(255,255,0,0.3);
border: 5px solid rgba(0,255,255,0.3);
}
dt {
border-top: 1px solid rgba(127,127,127,0.5);
}
table, th, td {
border: 1px solid rgba(127,127,127,0.5);
}
blockquote {
border-left: 5px solid rgba(127,255,127,0.7);
padding-left: 5px;
}
</style>
</head>
<body>
`
const TestHtmlTail = `
<!--
<script async="" src="https://platform.twitter.com/widgets.js"></script>
-->
</body>
</html>
`
Raw
maker.go
package main
import (
"fmt"
"strings"
)
func makeUL(tweets, targets []*Tweet) string {
return makeList("<ul>", "</ul>", tweets, targets)
}
func makeOL(tweets, targets []*Tweet) string {
return makeList(`<ol class="leonardonetweet">`, "</ol>", tweets, targets)
}
func makeList(op, cl string, tweets, targets []*Tweet) string {
var b strings.Builder
fmt.Fprintln(&b, op)
for _, t := range targets {
rep := t.ReplyCode(tweets, t)
text, quote, img := t.BodyCode(tweets)
timeStr := t.TimeCode(false)
name := "@" + t.Screen_name
link := t.MakeLink(t)
fmt.Fprintf(
&b,
`<li id="%s"><div style="font-size: 0.8em; color: grey;">%s&nbsp;%s&nbsp;%s</div>%s<p>%s</p>%s%s</li>`,
makeLinkTargetId(t.M.Tweet.Id_str),
timeStr,
name,
link,
rep,
text,
quote,
img,
)
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, cl)
return b.String()
}
func makeDL(tweets, targets []*Tweet) string {
var b strings.Builder
fmt.Fprintln(&b, "<dl>")
for _, t := range targets {
rep := t.ReplyCode(tweets, t)
text, quote, img := t.BodyCode(tweets)
timeStr := t.TimeCode(false)
name := "@" + t.Screen_name
fmt.Fprintf(
&b,
`<dt id="%s" style="font-size: 0.8em; color: grey;">%s&nbsp;%s&nbsp;%s</dt><dd>%s<p>%s</p>%s%s</dd>`,
makeLinkTargetId(t.M.Tweet.Id_str),
timeStr,
name,
t.MakeLink(t),
rep,
text,
quote,
img,
)
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, "</dl>")
return b.String()
}
func makeTable(tweets, targets []*Tweet) string {
var b strings.Builder
fmt.Fprintln(&b, "<table>")
fmt.Fprintln(&b, "<thead><tr><th>time</th><th>account</th><th>status</th></tr></thead>")
fmt.Fprintln(&b, "<tbody>")
for _, t := range targets {
rep := t.ReplyCode(tweets, t)
text, quote, img := t.BodyCode(tweets)
timeStr := t.TimeCode(false)
name := "@" + t.Screen_name
fmt.Fprintf(
&b,
`<tr id="%s"><td>%s</td><td>%s</td><td>%s<p>%s</p>%s%s</td></tr>`,
makeLinkTargetId(t.M.Tweet.Id_str),
timeStr,
name,
rep,
text,
quote,
img,
)
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, "</tbody>")
fmt.Fprintln(&b, "</table>")
return b.String()
}
Raw
misc.go
package main
import (
"net/url"
"sort"
"strings"
)
const (
neetsdkasu = "neetsdkasu"
usakdsteen = "usakdsteen"
enodranoeL = "enodranoeL"
ms_rinna = "ms_rinna"
)
func makeLinkTargetId(s string) string {
return "tweet" + s
}
func okReply(o *MainObject) bool {
switch o.Tweet.In_reply_to_screen_name {
case neetsdkasu, usakdsteen, enodranoeL, ms_rinna:
for _, m := range o.Tweet.Entities.User_mentions {
switch m.Screen_name {
case neetsdkasu, usakdsteen, enodranoeL, ms_rinna:
default:
if strings.Contains(m.Screen_name, "McD") ||
strings.Contains(m.Screen_name, "bot") ||
strings.Contains(m.Screen_name, "Th") {
return false
}
}
}
return true
}
count := len(o.Tweet.Entities.User_mentions)
if count == 0 {
return false
}
for _, m := range o.Tweet.Entities.User_mentions {
switch m.Screen_name {
case neetsdkasu, usakdsteen, enodranoeL, ms_rinna:
count--
}
}
return count == 0
}
func isTwitterStatus(target_url string) (string, bool) {
u, err := url.Parse(target_url)
if err != nil {
return "", false
}
if u.Hostname() != "twitter.com" {
return "", false
}
es := strings.Split(u.EscapedPath(), "/")
if len(es) != 4 || es[2] != "status" {
return "", false
}
switch es[1] {
case neetsdkasu, usakdsteen, enodranoeL:
return es[3], true
default:
return "", true
}
}
func searchTweet(tweets []*Tweet, id_str string) *Tweet {
if tweets == nil || len(tweets) == 0 || len(id_str) == 0 {
return nil
}
pos := sort.Search(len(tweets), func(i int) bool {
return strings.Compare(tweets[i].M.Tweet.Id_str, id_str) >= 0
})
if pos == len(tweets) {
return nil
}
if t := tweets[pos]; t.M.Tweet.Id_str == id_str {
return t
} else {
return nil
}
}
Raw
mttemp.go
package main
import (
"strings"
"text/template"
)
const MT_TEMP = `AUTHOR: {{.Author}}
TITLE: {{.Title}}
BASENAME: {{printf "%04d/%02d/%02d/%02d%02d%02d" .Year .Month .Day .Hour .Minute .Second}}
STATUS: Publish
ALLOW COMMENTS: 1
ALLOW PINGS: 0
CONVERT BREAKS: {{.ConvertBreaks}}
PRIMARY CATEGORY: {{index .Categories 0}}{{range .Categories}}
CATEGORY: {{.}}{{end}}
DATE: {{printf "%02d/%02d/%04d %02d:%02d:%02d" .Month .Day .Year .Hour .Minute .Second}}
-----
BODY:
{{.Body}}
-----
EXTENDED BODY:{{if .ExtendedBody}}
{{.ExtendedBody}}{{end}}
-----
EXTENDED BODY PRIVATE:
-----
EXCERPT:
-----
KEYWORDS:
-----
--------
`
var MTtemplate = template.Must(template.New("MT").Parse(MT_TEMP))
type BlogEntry struct {
Author string
Title string
Year int
Month int
Day int
Hour int
Minute int
Second int
Categories []string
ConvertBreaks int
Body string
ExtendedBody string
}
func (this *BlogEntry) String() string {
var b strings.Builder
err := MTtemplate.Execute(&b, this)
if err != nil {
println(err.Error())
}
return b.String()
}
func GetTestMT(sec int, body, ext string) string {
entry := &BlogEntry{
Author: "Leonardone",
Title: "テスト投稿",
Year: 2020,
Month: 6,
Day: 1,
Hour: 11,
Minute: 22,
Second: sec,
Categories: []string{"testing"},
Body: body,
ExtendedBody: ext,
}
return entry.String()
}
Raw
regacy.go
package main
import (
"encoding/json"
"fmt"
)
func showTweet(x interface{}) {
tweet, ok := x.(map[string]interface{})
if !ok {
fmt.Println("parse error tweet")
fmt.Printf("%+v", x)
return
}
for e, o := range tweet {
fmt.Println("Entry: ", e)
data, ok := o.(map[string]interface{})
if !ok {
fmt.Println("%+v", o)
continue
}
for k, v := range data {
fmt.Println(" Key: ", k)
fmt.Printf(" Value: %+v", v)
fmt.Println()
}
}
}
func parseWithInterface(jsonBlob []byte) {
var values interface{}
err := json.Unmarshal(jsonBlob, &values)
if err != nil {
fmt.Println("error:", err)
return
}
tweets, ok := values.([]interface{})
if !ok {
fmt.Println("invalid")
return
}
pos := 0
for {
var cmd string
fmt.Print("Command? ")
fmt.Scanln(&cmd)
switch cmd {
case "exit", "quit", "bye":
return
case "len":
fmt.Println("len: ", len(tweets))
default:
fmt.Sscan(cmd, &pos)
if pos >= 0 && pos < len(tweets) {
showTweet(tweets[pos])
}
fmt.Println()
}
}
}
Raw
result.txt
Array Object {
"tweet": Object {
"contributors": Array String
"created_at": String
"display_text_range": Array String
"entities": Object {
"hashtags": Array Object {
"indices": Array String
"text": String
}
"media": Array Object {
"display_url": String
"expanded_url": String
"id": String
"id_str": String
"indices": Array String
"media_url": String
"media_url_https": String
"sizes": Object {
"large": Object {
"h": String
"resize": String
"w": String
}
"medium": Object {
"h": String
"resize": String
"w": String
}
"small": Object {
"h": String
"resize": String
"w": String
}
"thumb": Object {
"h": String
"resize": String
"w": String
}
}
"source_status_id": String
"source_status_id_str": String
"source_user_id": String
"source_user_id_str": String
"type": String
"url": String
}
"symbols": Array Object {
"indices": Array String
"text": String
}
"urls": Array Object {
"display_url": String
"expanded_url": String
"indices": Array String
"url": String
}
"user_mentions": Array Object {
"id": String
"id_str": String
"indices": Array String
"name": String
"screen_name": String
}
}
"extended_entities": Object {
"media": Array Object {
"additional_media_info": Object {
"description": String
"embeddable": Boolean
"monetizable": Boolean
"title": String
}
"display_url": String
"expanded_url": String
"id": String
"id_str": String
"indices": Array String
"media_url": String
"media_url_https": String
"sizes": Object {
"large": Object {
"h": String
"resize": String
"w": String
}
"medium": Object {
"h": String
"resize": String
"w": String
}
"small": Object {
"h": String
"resize": String
"w": String
}
"thumb": Object {
"h": String
"resize": String
"w": String
}
}
"source_status_id": String
"source_status_id_str": String
"source_user_id": String
"source_user_id_str": String
"type": String
"url": String
"video_info": Object {
"aspect_ratio": Array String
"duration_millis": String
"variants": Array Object {
"bitrate": String
"content_type": String
"url": String
}
}
}
}
"favorite_count": String
"favorited": Boolean
"full_text": String
"id": String
"id_str": String
"in_reply_to_screen_name": String
"in_reply_to_status_id": String
"in_reply_to_status_id_str": String
"in_reply_to_user_id": String
"in_reply_to_user_id_str": String
"lang": String
"possibly_sensitive": Boolean
"retweet_count": String
"retweeted": Boolean
"source": String
"truncated": Boolean
}
}
Raw
shower.go
package main
import (
"encoding/json"
"fmt"
"strings"
"time"
)
func parseMainObject(jsonBlob []byte) {
var tweets []*MainObject
err := json.Unmarshal(jsonBlob, &tweets)
if err != nil {
fmt.Println("error:", err)
return
}
fmt.Println("len: ", len(tweets))
pos := 0
for {
var cmd string
fmt.Print("Command? ")
fmt.Scanln(&cmd)
switch cmd {
case "exit", "quit", "bye":
return
case "len":
fmt.Println("len: ", len(tweets))
case "contri":
showContri(tweets)
case "reply":
showReply(tweets)
case "retweet":
showRetweet(tweets)
case "rt":
showRT(tweets)
case "crt":
showCRT(tweets)
case "media":
showMedia(tweets)
case "ami":
showAMI(tweets)
case "any":
showAny(tweets)
case "ent":
showEntities(tweets)
default:
fmt.Sscan(cmd, &pos)
if pos >= 0 && pos < len(tweets) {
fmt.Printf("%+v", tweets[pos])
fmt.Println()
fmt.Printf("%+v", tweets[pos].Tweet.Entities)
fmt.Println()
fmt.Printf("%+v", tweets[pos].Tweet.Extended_entities)
fmt.Println()
if tm, err := time.Parse(time.RubyDate, tweets[pos].Tweet.Created_at); err == nil {
tm = tm.In(time.Local)
fmt.Println(tm)
}
}
fmt.Println()
}
}
}
func showContri(tweets []*MainObject) {
count := map[string][]*MainObject{}
for _, t := range tweets {
if len(t.Tweet.Contributors) > 0 {
key := t.Tweet.Contributors[0]
if c, ok := count[key]; ok {
count[key] = append(c, t)
} else {
a := []*MainObject{}
count[key] = append(a, t)
}
}
}
fmt.Println("contri(>0) count: ", len(count))
for k, c := range count {
fmt.Println("Contributors: ", k)
fmt.Println("Count: ", len(c))
fmt.Println()
if len(c) < 10 {
for _, o := range c {
fmt.Printf("%+v", o)
fmt.Println()
fmt.Println()
}
}
}
fmt.Println()
}
func showReply(tweets []*MainObject) {
count := map[string]int{}
for _, t := range tweets {
if t.IsReply() {
key := t.Tweet.In_reply_to_screen_name + "$" + t.Tweet.In_reply_to_user_id
if c, ok := count[key]; ok {
count[key] = c + 1
} else {
count[key] = 1
fmt.Println(key)
fmt.Printf("%+v", t)
fmt.Println()
fmt.Println()
}
}
}
fmt.Println("reply count: ", len(count))
fmt.Printf("%+v", count)
fmt.Println()
}
func showRetweet(tweets []*MainObject) {
count := []*MainObject{}
for _, t := range tweets {
if len(t.Tweet.Retweet_count) > 0 && t.Tweet.Retweet_count != "0" {
count = append(count, t)
}
}
fmt.Println("retweet count: ", len(count))
var cmd string
for pos := 0; pos < len(count); pos += 10 {
fmt.Print("PRESS KEY? ")
fmt.Scanln(&cmd)
for i := 0; i < 10 && pos+i < len(count); i++ {
fmt.Printf("%+v", count[pos+i])
fmt.Println()
fmt.Println()
}
}
fmt.Println()
}
func showEntities(tweets []*MainObject) {
count := []*MainObject{}
for _, t := range tweets {
if !t.HasEntities() {
continue
}
if len(t.Tweet.Entities.Urls) == 0 {
continue
}
count = append(count, t)
}
fmt.Println("have entities count: ", len(count))
for i := 10; i < 30 && i < len(count); i++ {
fmt.Printf("%+v", count[i])
fmt.Println()
fmt.Printf("%+v", count[i].Tweet.Entities)
fmt.Println()
fmt.Printf("%+v", count[i].Tweet.Entities.Urls[0])
fmt.Println()
fmt.Println()
}
fmt.Println()
}
func showCRT(tweets []*MainObject) {
for _, t := range tweets {
switch t.Tweet.Id {
case "1126881310155231232", "1128382583564169217", "1122288816646004736", "1122283397349380096", "1121063106799476737", "1069665180013887488", "1067517647472943104":
fmt.Printf("%+v", t)
fmt.Println()
if t.HasEntities() {
fmt.Printf("%+v", t.Tweet.Entities)
fmt.Println()
if len(t.Tweet.Entities.Urls) > 0 {
for _, u := range t.Tweet.Entities.Urls {
fmt.Printf("%+v", u)
fmt.Println()
}
}
}
fmt.Println()
}
}
fmt.Println()
}
func showMedia(tweets []*MainObject) {
for _, t := range tweets {
switch t.Tweet.Id {
case "1077143041926787073", "1072870934820806656", "1155148055127003136", "1155442259036233734", "1132729950065156098", "1133115492187508736":
fmt.Printf("%+v", t)
fmt.Println()
if t.HasEntities() {
fmt.Printf("%+v", t.Tweet.Entities)
fmt.Println()
if len(t.Tweet.Entities.Urls) > 0 {
for _, u := range t.Tweet.Entities.Urls {
fmt.Printf("%+v", u)
fmt.Println()
}
}
if len(t.Tweet.Entities.Media) > 0 {
for _, u := range t.Tweet.Entities.Media {
fmt.Printf("%+v", u)
fmt.Println()
}
}
}
if t.Tweet.Extended_entities != nil {
fmt.Printf("%+v", t.Tweet.Extended_entities)
fmt.Println()
if len(t.Tweet.Extended_entities.Media) > 0 {
for _, u := range t.Tweet.Extended_entities.Media {
fmt.Printf("%+v", u)
fmt.Println()
}
}
}
fmt.Println()
}
}
fmt.Println()
}
func showAMI(tweets []*MainObject) {
count := []*MainObject{}
for _, t := range tweets {
found := false
if t.HasEntities() {
if len(t.Tweet.Entities.Media) > 0 {
for _, m := range t.Tweet.Entities.Media {
if m.Additional_media_info == nil {
continue
}
found = true
break
}
}
}
if t.Tweet.Extended_entities != nil {
if len(t.Tweet.Extended_entities.Media) > 0 {
for _, m := range t.Tweet.Extended_entities.Media {
if m.Additional_media_info == nil {
continue
}
found = true
break
}
}
}
if found {
count = append(count, t)
}
}
fmt.Println("count: ", len(count))
for i := 0; i < 10 && i < len(count); i++ {
t := count[i]
fmt.Printf("%+v", t)
fmt.Println()
if t.HasEntities() {
if len(t.Tweet.Entities.Media) > 0 {
for _, m := range t.Tweet.Entities.Media {
if m.Additional_media_info == nil {
continue
}
fmt.Printf("%+v", m.Additional_media_info)
fmt.Println()
}
}
}
if t.Tweet.Extended_entities != nil {
if len(t.Tweet.Extended_entities.Media) > 0 {
for _, m := range t.Tweet.Extended_entities.Media {
if m.Additional_media_info == nil {
continue
}
fmt.Printf("%+v", m.Additional_media_info)
fmt.Println()
}
}
}
fmt.Println()
}
}
func showAny(tweets []*MainObject) {
count := map[string][]*MainObject{}
dups := 0
for _, t := range tweets {
if !t.HasEntities() {
continue
}
if t.IsReply() {
continue
}
if t.IsRetweet() {
continue
}
if len(t.Tweet.Entities.User_mentions) == 0 {
continue
}
if len(t.Tweet.Entities.User_mentions) > 1 {
dups++
}
for _, um := range t.Tweet.Entities.User_mentions {
key := um.Name + "$" + um.Screen_name + "$" + um.Id
if c, ok := count[key]; ok {
count[key] = append(c, t)
} else {
a := []*MainObject{}
count[key] = append(a, t)
}
}
}
fmt.Println("count: ", len(count), " dups: ", dups)
for k, c := range count {
fmt.Println(k, ":", len(c), ", ")
}
fmt.Println()
fmt.Println()
}
func showRT(tweets []*MainObject) {
count := map[string][]*MainObject{}
total := 0
for _, t := range tweets {
if !t.IsRetweet() {
continue
}
s := strings.SplitN(t.Tweet.Full_text, ":", 2)
if len(s) == 0 {
continue
}
total++
key := strings.TrimPrefix(s[0], "RT ")
if c, ok := count[key]; ok {
count[key] = append(c, t)
} else {
a := []*MainObject{}
count[key] = append(a, t)
}
}
fmt.Println("RT total: ", total)
fmt.Println("RT account count: ", len(count))
for k, c := range count {
fmt.Print(k, ":", len(c), ", ")
}
fmt.Println()
fmt.Println()
}
Raw
tweet.go
package main
import (
"fmt"
"html"
"strings"
"time"
)
type Tweet struct {
Screen_name string
Time time.Time
M *MainObject
}
func (this *Tweet) Url() string {
if this == nil {
return ""
}
return fmt.Sprintf(
"https://twitter.com/%s/status/%s",
this.Screen_name,
this.M.Tweet.Id_str,
)
}
func (this *Tweet) MakeLink(dayTweet *Tweet) string {
if this == nil {
return ""
}
var href string
if this.Screen_name == enodranoeL {
href = "#" + makeLinkTargetId(this.M.Tweet.Id_str)
if !this.IsSameDate(dayTweet) {
href = "/entry/" + this.MTBaseName() + href
}
} else {
href = this.Url()
}
return fmt.Sprintf(
`<a style="color: black;" target="_blank" href="%s" title="%s" rel="noopener noreferrer external">#</a>`,
href,
this.Url(),
)
}
func (this *Tweet) MTBaseName() string {
if this == nil {
return ""
}
return this.Time.Format("2006/01/02") + "/235959"
}
func (this *Tweet) IsSameDate(other *Tweet) bool {
if this == nil || other == nil {
return false
}
return this.Time.Year() == other.Time.Year() &&
this.Time.Month() == other.Time.Month() &&
this.Time.Day() == other.Time.Day()
}
func (this *Tweet) TimeCode(full bool) string {
if this == nil {
return ""
}
var dt string
if full {
dt = this.Time.Format("2006年01月02日 15:04:05")
} else {
dt = this.Time.Format("15:04:05")
}
return fmt.Sprintf(
`<time datetime="%s">%s</time>`,
this.Time.Format("2006-01-02T15:04:05-07:00"),
dt,
)
}
func (this *Tweet) BodyCode(tweets []*Tweet) (string, string, string) {
if this == nil {
return "", "", ""
}
text := strings.ReplaceAll(
strings.ReplaceAll(
this.M.Tweet.Full_text,
"\r\n",
"<br>",
),
"\n",
"<br>",
)
reply := this.M.Tweet.In_reply_to_screen_name
if reply != "" {
reply = "@" + reply
if strings.HasPrefix(text, reply) {
text = strings.TrimPrefix(text, reply)
}
}
quote := ""
for _, u := range this.M.Tweet.Entities.Urls {
id_str, ok := isTwitterStatus(u.Expanded_url)
if !ok {
if mastodon, ok := QuoteFromMastodon[u.Expanded_url]; ok {
text = strings.ReplaceAll(
text,
u.Url,
"",
)
quote += mastodon
} else {
text = strings.ReplaceAll(
text,
u.Url,
fmt.Sprintf(
`<a target="_blank" href="%s" rel="nofollow noreferrer noopener external">%s</a>`,
html.EscapeString(u.Expanded_url),
html.EscapeString(u.Display_url),
),
)
}
} else if t := searchTweet(tweets, id_str); t != nil {
text = strings.ReplaceAll(
text,
u.Url,
"",
)
quote += t.Blockquote(tweets, this)
} else {
text = strings.ReplaceAll(
text,
u.Url,
"",
)
quote += fmt.Sprintf(
`<blockquote class="twitter-tweet" data-conversation="none" data-theme="dark"><a href="%s" target="_blank" rel="nofollow noreferrer noopener external">%s</a></blockquote>`,
html.EscapeString(u.Expanded_url),
html.EscapeString(u.Display_url),
)
}
}
img := ""
var media []*MediaObject
if this.M.Tweet.Extended_entities != nil {
media = this.M.Tweet.Extended_entities.Media
} else {
media = this.M.Tweet.Entities.Media
}
for i, m := range media {
text = strings.ReplaceAll(text, m.Url, "")
if m.Video_info != nil {
img += `<blockquote class="twitter-video" data-conversation="none" data-theme="dark" style="display: inline-block; margin-left: 0px;">`
img += fmt.Sprintf(
`<a target="_blank" href="%s" rel="nofollow noreferrer noopener external"><img src="%s" width="100" alt="image-%d"></a>`,
html.EscapeString(m.Expanded_url),
html.EscapeString(m.Media_url_https),
i,
)
img += "</blockquote>"
} else {
img += fmt.Sprintf(
`<a target="_blank" href="%s" rel="noopener"><img src="%s" width="100" alt="image-%d"></a>`,
html.EscapeString(m.Media_url_https),
html.EscapeString(m.Media_url_https),
i,
)
}
}
return text, quote, img
}
func (this *Tweet) ReplyCode(tweets []*Tweet, dayTweet *Tweet) string {
if this == nil {
return ""
}
if !this.M.IsReply() {
return ""
}
id_str := this.M.Tweet.In_reply_to_status_id_str
if t := searchTweet(tweets, id_str); t != nil {
href := "#" + makeLinkTargetId(id_str)
if !t.IsSameDate(dayTweet) {
href = "/entry/" + t.MTBaseName() + href
}
return fmt.Sprintf(
`<span><a href="%s">@%s</a></span>`,
href,
t.Screen_name,
)
}
if len(id_str) == 0 {
// 話しかけ？
if this.M.Tweet.In_reply_to_screen_name == "" {
return `<del><span title="Unknown account">@???????</span></del>`
}
return fmt.Sprintf(
"<span>@%s</span>",
this.M.Tweet.In_reply_to_screen_name,
)
}
switch this.M.Tweet.In_reply_to_screen_name {
case neetsdkasu, usakdsteen, enodranoeL:
return fmt.Sprintf(
`<blockquote class="twitter-tweet" data-conversation="none" data-theme="dark" style="display: inline-block; margin: unset; padding: unset; border: unset; background-color: unset; color: unset;"><del><a target="_blank" href="https://twitter.com/%[1]s/status/%[2]s" rel="nofollow noreferrer noopener external" title="twitter.com/%[1]s/status/%[2]s">@%[1]s</a></del></blockquote>`,
this.M.Tweet.In_reply_to_screen_name,
id_str,
)
case "":
return fmt.Sprintf(
`<del><span title="Unknown account: twitter.com/???????/status/%s">@???????</span></del>`,
id_str,
)
}
return fmt.Sprintf(
`<blockquote class="twitter-tweet" data-conversation="none" data-theme="dark" style="display: inline-block; margin: unset; padding: unset; border: unset; background-color: unset; color: unset;"><a target="_blank" href="https://twitter.com/%[1]s/status/%[2]s" rel="nofollow noreferrer noopener external" title="twitter.com/%[1]s/status/%[2]s">@%[1]s</a></blockquote>`,
this.M.Tweet.In_reply_to_screen_name,
id_str,
)
}
var EmptyTweets = []*Tweet{}
func (this *Tweet) Blockquote(tweets []*Tweet, dayTweet *Tweet) string {
if this == nil {
return ""
}
timeStr := this.TimeCode(!this.IsSameDate(dayTweet))
name := "@" + this.Screen_name
rep := this.ReplyCode(tweets, dayTweet)
text, quote, img := this.BodyCode(EmptyTweets)
href := "#" + makeLinkTargetId(this.M.Tweet.Id_str)
if !this.IsSameDate(dayTweet) {
href = "/entry/" + this.MTBaseName() + href
}
link := this.MakeLink(dayTweet)
return fmt.Sprintf(
`<blockquote><div style="font-size: 0.8em; color: grey;"><a href="%s" style="text-decoration: none; color: unset;">%s&nbsp;%s</a>&nbsp;%s</div>%s<p>%s</p>%s%s</blockquote>`,
href,
timeStr,
name,
link,
rep,
text,
quote,
img,
)
}
@neetsdkasu
Copy link
Author

neetsdkasu commented Jul 13, 2020 •
edited

F#で書いたJsonAnalyzer.exe(link)でresult.txtは生成
hogehoge.jsonは

sed -e "1c [{" tweet.js > hogehoge.json

とした

まぁmain.go内で[を検出すればいいのかもだが

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment