Skip to content

Instantly share code, notes, and snippets.

@soeirosantos
Last active July 2, 2020 19:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save soeirosantos/535b249d255fc077aade2fbec837b799 to your computer and use it in GitHub Desktop.
Save soeirosantos/535b249d255fc077aade2fbec837b799 to your computer and use it in GitHub Desktop.
plugins {
id 'java'
id 'org.jetbrains.kotlin.jvm' version '1.3.72'
}
group 'br.com.soeirosantos'
version '1.0.0-SNAPSHOT'
repositories {
mavenCentral()
}
dependencies {
compile group: 'org.twitter4j', name: 'twitter4j-core', version: '4.0.7'
compile group: 'org.twitter4j', name: 'twitter4j-stream', version: '4.0.7'
compile group: 'com.github.kittinunf.fuel', name: 'fuel', version: '2.2.3'
compile group: 'com.github.kittinunf.fuel', name: 'fuel-gson', version: '2.2.3'
compile group: 'com.google.code.gson', name: 'gson', version: '2.8.5'
compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.8.0-alpha2'
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
testCompile group: 'junit', name: 'junit', version: '4.12'
}
compileKotlin {
kotlinOptions.jvmTarget = "1.8"
}
compileTestKotlin {
kotlinOptions.jvmTarget = "1.8"
}
repositories {
maven {
url "https://dl.bintray.com/kittinunf/maven"
}
}
log4j.rootLogger = INFO, FILE, stdout
log4j.appender.FILE=org.apache.log4j.FileAppender
log4j.appender.FILE.File=./log.out
log4j.appender.FILE.ImmediateFlush=true
log4j.appender.FILE.Threshold=info
log4j.appender.FILE.Append=true
log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
log4j.appender.FILE.layout.conversionPattern=%m%n
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yy-MM-dd HH:mm:ss:SSS} %5p %t %c{2}:%L - %m%n
package br.com.soeirosantos.twitter
import com.github.kittinunf.fuel.Fuel
import com.github.kittinunf.fuel.core.ResponseDeserializable
import com.google.gson.Gson
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import twitter4j.*
import java.io.File
import kotlin.math.roundToInt
import kotlin.streams.toList
class TwitterEngagement
val log: Logger = LoggerFactory.getLogger(TwitterEngagement::class.java)
const val FILTER_PROPERTY = "filter"
const val LANGUAGE_FILTER = "en"
const val LOG_FILE = "./log.out"
const val TWITTER_URL = "https://twitter.com"
const val SENTIMENT_URL = "http://text-processing.com/api/sentiment/"
val FAVORITE_CACHE = HashSet<Long>()
val RETWEET_CACHE = HashSet<String>()
fun main() {
reloadCache()
val twitter = TwitterFactory.getSingleton()
val twitterStream: TwitterStream = TwitterStreamFactory().instance
twitterStream.onException { log.error("Boo!", it) }
twitterStream.onStatus {
if (!it.isRetweet) return@onStatus
if (it.retweetedStatus.id in FAVORITE_CACHE) return@onStatus
if (it.retweetedStatus.favoriteCount == 0) return@onStatus
if (it.retweetedStatus.retweetCount == 0) return@onStatus
if (favorite(it.retweetedStatus)) {
val sentiment = sentiment(it.retweetedStatus.text)
if (sentiment.label == "neg") return@onStatus
twitter.createFavorite(it.retweetedStatus.id)
FAVORITE_CACHE.add(it.retweetedStatus.id)
if (!(it.retweetedStatus.user.screenName in RETWEET_CACHE) && retweet(it.retweetedStatus)) {
twitter.retweetStatus(it.retweetedStatus.id)
RETWEET_CACHE.add(it.retweetedStatus.user.screenName)
}
log.info(
"favCount: ${it.retweetedStatus.favoriteCount} | " +
"retweetCount: ${it.retweetedStatus.retweetCount} | " +
"sentiment: ${sentiment.label} - " +
" $TWITTER_URL/${it.retweetedStatus.user.screenName}/status/${it.retweetedStatus.id}"
)
}
}
val query = FilterQuery()
query
.language(LANGUAGE_FILTER)
.track(
*System.getProperty(FILTER_PROPERTY, "devops")
.split(",").map { it.trim() }.toTypedArray()
)
twitterStream.filter(query)
}
private fun favorite(status: Status) = status.favoriteCount > favoriteEngagementFactor() ||
status.retweetCount > favoriteEngagementFactor()
private fun retweet(status: Status) = status.favoriteCount > retweetEngagementFactor() ||
status.retweetCount > retweetEngagementFactor()
/**
* This function is used to weight the engagement for favorite tweets
*
* The current implementation says that we reduce the engagement rate as
* the favorite engagement grows.
*/
private fun favoriteEngagementFactor(): Int {
return (5 + .5 * FAVORITE_CACHE.size).roundToInt() //customize the rule to obtain the favorite factor
}
/**
* This function is used to weight the engagement for retweets
*
* The current implementation says it's constant at 1000
*/
private fun retweetEngagementFactor(): Int {
return 1000 //customize the rule to obtain the retweet factor
}
/**
* The cache keeps track of tweets and users we have already engaged
* to avoid an excess of similar interactions.
*
* This function reloads the cache from the log file to make sure
* we are in a good shape across executions.
*/
private fun reloadCache() {
val file = File(LOG_FILE)
if (!file.exists()) return
file.forEachLine { line ->
if (line.contains(TWITTER_URL)) {
val split = line.split("/")
val tweetId = split[split.size - 1]
FAVORITE_CACHE.add(tweetId.toLong())
val userScreenName = split[split.size - 3]
RETWEET_CACHE.add(userScreenName)
}
}
}
data class Sentiment(var label: String) {
class Deserializer : ResponseDeserializable<Sentiment> {
override fun deserialize(content: String): Sentiment = Gson().fromJson(content, Sentiment::class.java)
}
}
fun sentiment(text: String): Sentiment {
val (_, _, result) = Fuel.post(SENTIMENT_URL)
.body("text=$text")
.responseObject(Sentiment.Deserializer())
return result.get()
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment