Skip to content

Instantly share code, notes, and snippets.

View mananai's full-sized avatar

Mananai mananai

  • Thailand
View GitHub Profile
@mananai
mananai / TwitterSQLiteDB.java
Created March 30, 2023 15:15
Code to write tweets to the SQLite database
class TwitterSQLiteDB implements TwitterDAO {
private final Connection connection;
private Logger logger = Logger.getLogger(this.getClass().getName());
private int totalCount = 0;
TwitterSQLiteDB(String jdbcURL) throws SQLException {
connection = DriverManager.getConnection(jdbcURL);
connection.setAutoCommit(false);
createTweetTable();
}
@mananai
mananai / Tweets20230324.csv
Created March 30, 2023 12:39
Sample tweets csv files
We can make this file beautiful and searchable if this error is corrected: Unclosed quoted field in line 7.
id,author_id,created_at,lang,text,retweeted_id,quoted_id,replied_to_id,retweet_count,quote_count,reply_count,like_count
1639079716362817536,1179260191948197888,2023-03-24T08:40:31,th,,1638906045237178375,,,11065,0,0,0
1639079728916348928,4309068859,2023-03-24T08:40:34,th,,1638566271834013696,,,14697,0,0,0
1639079733119045634,3029991627,2023-03-24T08:40:35,th,,1638837179194699782,,,1699,0,0,0
1639079733123256323,1289515331925487617,2023-03-24T08:40:35,th,,1638603281667850242,,,27988,0,0,0
1639079737304977408,1586539597122908161,2023-03-24T08:40:36,th,,1638880145229611010,,,1024,0,0,0
1638880145229611010,832453615130271744,2023-03-23T19:27:30,th,"🪞 #BTS #Jimin เตรียมปล่อยเพลง Like Crazy พร้อมอัลบั้ม FACE เราสามารถซัพพอร์ตได้อย่างไรบ้าง? สรุปสั้น ๆ ครบ จบ ในเธรดนี้ที่เดียว!
⏰ ฟังพร้อมกันศุกร์ที่ 24 มี.ค. เวลา 11:00 น. (ไทย)
@mananai
mananai / TwitterDBWriter.java
Created March 30, 2023 11:30
A callable(task) to save tweets to an SQL database
class TwitterDBWriter implements Callable<Integer> {
private final static int TWEETS_BATCH_SIZE = 20;
private final BlockingQueue<Tweet[]> tweetsQueue;
private final Queue<Tweet[]> writingQueue = new LinkedList<>();
private final TwitterDAO twitterDAO ;
TwitterDBWriter(String jdbcURL, BlockingQueue<Tweet[]> tweetsQueue) throws SQLException {
super();
this.twitterDAO = TwitterDAOFactory.create(jdbcURL);
this.tweetsQueue = tweetsQueue;
}
@mananai
mananai / SampleStreamCollector.java
Last active March 30, 2023 10:51
A callable (task) to collect tweets from sample stream
class SampleStreamCollector implements Callable<Integer> {
private static final int TWITTER_API_FORBIDDEN_ERROR = 403;
private static final int MAX_RECONNECT_COUNT = 20;
private static final int SAMPLE_STREAM_REQUEST_DELAY = 15*60*1000/50;
private final Predicate<Tweet> predicate;
private final String[] bearerTokenArray;
private final BlockingQueue<Tweet[]> tweetsQueue;
private final AtomicBoolean isTerminated;
private Logger logger = Logger.getLogger(this.getClass().getName());
@mananai
mananai / SampleStream2DB.java
Last active March 30, 2023 10:48
Main class to save Twitter sample stream into SQL database
public class SampleStream2DB {
private static final int THREAD_POOL_SIZE = 2;
static Logger logger = Logger.getLogger(SampleStream2DB.class.getName());
public static void main(String[] args) throws SQLException {
if (args.length < 3)
throw new IllegalArgumentException();
final String jdbcURL = args[0];
final String lang = args[1];
final String[] bearerTokenArray = Arrays.copyOfRange(args, 2, args.length);
@mananai
mananai / twitter4j.properties
Created June 25, 2022 06:41
Sample twitter4j.properties
oauth.consumerKey = <API Key>
oauth.consumerSecret = <API Secret>
oauth.accessToken = <Access Token>
oauth.accessTokenSecret = <Access Token Secret>
MATCH (u:User)
WITH u.sccId AS sccId, count(u) AS size, COLLECT(id(u)) AS ids
RETURN sccId, size, ids[0..10] AS First_10_Members
ORDER BY size DESC LIMIT 10;
@mananai
mananai / LabelPropagationCommunityVisual.cypher
Created June 22, 2022 02:28
Label Propagation Community Visualization
MATCH (u:User)
WHERE u.labelId = 35964
RETURN u;
@mananai
mananai / User Recommendations.csv
Created June 21, 2022 12:35
User Recommendations
topUser.screen_name topUser.name topUser.followers_count score
LILITEAMTH327 LILITEAM 207503 0.2807962712389718
LISANATIONS_ LISANATIONS 335580 0.1620213024460489
LISABar_CN LISABar 304360 0.12581057943353507
BIGHIT_MUSIC BIGHIT MUSIC 25677826 0.09781302064695072
Bulgariofficial Bulgari 734334 0.06657324316599895
@mananai
mananai / User Recommendations.cypher
Last active June 21, 2022 12:34
User Recommendations
MATCH (u:User)
WHERE u.labelId = 4854
WITH collect(id(u)) AS ids
CALL gds.pageRank.stream('follows',{sourceNodes:ids })
YIELD nodeId, score
WITH gds.util.asNode(nodeId) AS topUser, score
WHERE topUser.followers_count > 100000
MATCH (u2:User)
WHERE id(u2)=2138
AND NOT ( (u2)-[:FOLLOWS]->(topUser) )