Skip to content

Instantly share code, notes, and snippets.

Jae-cheol Kim forcemax

Block or report user

Report or block forcemax

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
@forcemax
forcemax / clean_dataset.py
Last active Oct 13, 2016
YFCC100M tag prediction clean dataset python
View clean_dataset.py
import os
import re
import collections
import urllib.parse
from time import time
from multiprocessing import Pool
KEEPWORDS_FILE = "keepwords.txt"
TRAIN_DATASET_DIR = "../yfcc100m/"
View PrinterBolt.java
public void execute(Tuple input) {
Object obj = input.getValueByField("rankings");
try {
Rankings ranking = (Rankings) obj;
LOGGER.debug("input ranking : {}", ranking);
int count = 1;
for(Rankable rank : ranking.getRankings()) {
LOGGER.info("RANK : {}, HashTag : {}, Count : {}", count++, rank.getObject(), rank.getCount());
}
} catch (Exception e) {
View AbstractRankerBolt.java
public final void execute(Tuple input) {
if (isTickTuple(input)) {
getLogger().debug("Received tick tuple, triggering emit of current rankings");
emitRankings(collector);
} else {
getLogger().debug("tuple : {}", input);
updateRankingsWithTuple(input);
}
collector.ack(input);
}
View RollingCountBolt.java
public void execute(Tuple input) {
if (isTickTuple(input)) {
LOGGER.debug("Received tick tuple, triggering emit of current window counts");
emitCurrentWindowCounts();
} else {
countObj(input);
}
collector.ack(input);
}
View ExtractHashTagBolt.java
public void execute(Tuple input) {
Object obj = input.getValueByField("tweet");
if (obj instanceof Status) {
try {
final Status status = (Status) obj;
LOGGER.debug("input tweet : {}", status);
final HashtagEntity[] hashtags = status.getHashtagEntities();
if (hashtags != null) {
for(HashtagEntity hashtag : hashtags) {
View TwitterSpout.java
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
queue = new LinkedBlockingQueue<Status>(32*1024);
_collector = collector;
StatusListener listener = new StatusListener() {
public void onStatus(Status status) {
queue.offer(status);
}
}
}
View StormTwitterHashtagTopologyRunner.java
private static TopologyBuilder wireTopology() {
TopologyBuilder builder = new TopologyBuilder();
// Twitter API Key
String consumerKey = "consumerKey";
String consumerSecret = "consumerSecret";
String accessToken = "accessToken";
String accessTokenSecret = "accessTokenSecret";
String[] keyWords = new String[0];
You can’t perform that action at this time.