metion

SinaCorpInteractionCrawler.java

public void getOneUserMention(String userId, long lastMsgTime, String nickname) {
        if (sinaToken == null) {
            LOG.info(userId + ", userMention get token null!");
            return;
        }
        if (!isOwnToken) {
            LOG.info(userId + ", userMention no ownToken, exit!");
            return;
        }
        try {
            List<Status> list = new LinkedList<Status>();
            // 抓取第一页
            long totalCount = 0;
            long finalSize = 0;
            Pair<List<Status>, Long> res = fetchUserMention(userId, sinaToken, 1);
            if (res != null && res.first != null) {
                Pair<Pair<List<Status>, Boolean>, Pair<Integer, Long>> filterPair = replyFilter.filterSinaReplyWithYzCount(res.first, REPLY_TYPE.MENTION, nickname, lastMsgTime, false);
                boolean reachLastMsgTime = filterPair.first.second;
                if (!AppUtil.isEmpty(filterPair.first.first)) {
                    list.addAll(filterPair.first.first);
                }

                // 抓取剩余页
                totalCount = res.second;
                long pageCount = Math.min((totalCount + 199) / 200, CrawlerConsts.MIN_PAGE_LOAD);  //最多抓取50
                for (int page = 2; page <= pageCount; page++) {
                    if (page % 10 == 0) {
                        LOG.info("nickname: " + nickname + ", mentions finalSize: " + list.size() + ", page: " + page);
                        hbaseDao.insertSinaMention(list, userId, nickname, corpRepostMap);
                        finalSize += list.size();

                        list.clear();
                    }
                    if (reachLastMsgTime) { // 已经抓到sinceId，不需要再往前抓
                        LOG.info("reach lastMsgTime " + lastMsgTime + ", stop crawling");
                        break;
                    }
                    try {
                        res = fetchUserMention(userId, sinaToken, page);
                        if (res != null && res.first != null) {
                            filterPair = replyFilter.filterSinaReplyWithYzCount(res.first, REPLY_TYPE.MENTION, nickname, lastMsgTime, false);
                            reachLastMsgTime = filterPair.first.second;
                            if (!AppUtil.isEmpty(filterPair.first.first)) {
                                list.addAll(filterPair.first.first);
                            }
                        } else
                            break;
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }

            if (!list.isEmpty()) {
                LOG.info("nickname: " + nickname + ", mentions finalSize: " + list.size());
                hbaseDao.insertSinaMention(list, userId, nickname, corpRepostMap);
                finalSize += list.size();

                list.clear();
            }
            LOG.info("user=" + userId + ", mentionSize=" + totalCount + ", finalSize: " + finalSize);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }