Skip to content

Instantly share code, notes, and snippets.

@B-R-P
Created April 2, 2023 13:21
Show Gist options
  • Save B-R-P/9f1086d6f756c043bcf0f56f59040169 to your computer and use it in GitHub Desktop.
Save B-R-P/9f1086d6f756c043bcf0f56f59040169 to your computer and use it in GitHub Desktop.
scala declarations in Twitter Algorithm
This file has been truncated, but you can view the full file.
.\ann\src\main\scala\com\twitter\ann\annoy\AnnoyCommon.scala
object AnnoyCommon
def apply(scalaParams: AnnoyRuntimeParams): ServiceRuntimeParams
def invert(thriftParams: ServiceRuntimeParams): Try[AnnoyRuntimeParams] =
thriftParams match
class AnnoyRuntimeParams(
/* Number of vectors to evaluate while searching. A larger value will give more accurate results, but will take longer time to return.
* Default value would be numberOfTrees*numberOfNeigboursRequested
*/
nodesToExplore: Option[Int])
extends RuntimeParams
.\ann\src\main\scala\com\twitter\ann\annoy\RawAnnoyIndexBuilder.scala
object RawAnnoyIndexBuilder
def apply[D <: Distance[D]](
dimension: Int,
numOfTrees: Int,
metric: Metric[D],
futurePool: FuturePool
): RawAppendable[AnnoyRuntimeParams, D] with Serialization
def annoyMetric(metric: Metric[_]): AnnoyLib.Metric
class RawAnnoyIndexBuilder[D <: Distance[D]](
dimension: Int,
numOfTrees: Int,
metric: Metric[D],
indexBuilder: AnnoyLib.Builder,
futurePool: FuturePool)
extends RawAppendable[AnnoyRuntimeParams, D]
with Serialization
def append(embedding: EmbeddingVector): Future[Long] =
semaphore.acquireAndRun(
def toQueryable: Queryable[Long, AnnoyRuntimeParams, D]
def toDirectory(directory: ResourceId): Unit
def toDirectory(directory: AbstractFile): Unit
def toDirectory(directory: IndexOutputFile): Unit
def saveIndex(indexFile: IndexOutputFile): Unit
def saveMetadata(metadataFile: IndexOutputFile): Unit
.\ann\src\main\scala\com\twitter\ann\annoy\RawAnnoyQueryIndex.scala
object RawAnnoyQueryIndex
def apply[D <: Distance[D]](
dimension: Int,
metric: Metric[D],
futurePool: FuturePool,
directory: AbstractFile
): Queryable[Long, AnnoyRuntimeParams, D]
def annoyMetric(metric: Metric[_]): IndexType
def loadIndex(
indexFile: AbstractFile,
dimension: Int,
indexType: IndexType
): ANNIndex
class RawAnnoyQueryIndex[D <: Distance[D]](
dimension: Int,
metric: Metric[D],
numOfTrees: Int,
index: ANNIndex,
futurePool: FuturePool)
extends Queryable[Long, AnnoyRuntimeParams, D]
with AutoCloseable
def query(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: AnnoyRuntimeParams
): Future[List[Long]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: AnnoyRuntimeParams
): Future[List[NeighborWithDistance[Long, D]]]
def neighboursToRequest(
numOfNeighbours: Int,
annoyParams: AnnoyRuntimeParams
): Int
.\ann\src\main\scala\com\twitter\ann\annoy\TypedAnnoyIndex.scala
object TypedAnnoyIndex
def indexBuilder[T, D <: Distance[D]](
dimension: Int,
numOfTrees: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
futurePool: FuturePool
): Appendable[T, AnnoyRuntimeParams, D] with Serialization
def loadQueryableIndex[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
futurePool: FuturePool,
directory: AbstractFile
): Queryable[T, AnnoyRuntimeParams, D]
.\ann\src\main\scala\com\twitter\ann\annoy\TypedAnnoyIndexBuilderWithFile.scala
object TypedAnnoyIndexBuilderWithFile
def apply[T, D <: Distance[D]](
dimension: Int,
numOfTrees: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
futurePool: FuturePool
): Appendable[T, AnnoyRuntimeParams, D] with Serialization
class TypedAnnoyIndexBuilderWithFile[T, D <: Distance[D]](
indexBuilder: RawAppendable[AnnoyRuntimeParams, D] with Serialization,
store: WritableIndexIdFileStore[T])
extends Appendable[T, AnnoyRuntimeParams, D]
with Serialization
def append(entity: EntityEmbedding[T]): Future[Unit]
def toDirectory(directory: ResourceId): Unit
def toDirectory(directory: AbstractFile): Unit
def toDirectory(directory: IndexOutputFile): Unit
def toQueryable: Queryable[T, AnnoyRuntimeParams, D]
.\ann\src\main\scala\com\twitter\ann\annoy\TypedAnnoyQueryIndexWithFile.scala
object TypedAnnoyQueryIndexWithFile
def apply[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
futurePool: FuturePool,
directory: AbstractFile
): Queryable[T, AnnoyRuntimeParams, D]
class TypedAnnoyQueryIndexWithFile[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
futurePool: FuturePool,
injection: Injection[T, Array[Byte]])
extends QueryableDeserialization[
T,
AnnoyRuntimeParams,
D,
Queryable[T, AnnoyRuntimeParams, D]
]
def fromDirectory(directory: AbstractFile): Queryable[T, AnnoyRuntimeParams, D]
.\ann\src\main\scala\com\twitter\ann\brute_force\BruteForceDeserialization.scala
class BruteForceDeserialization[T, D <: Distance[D]] @VisibleForTesting private[brute_force] (
metric: Metric[D],
embeddingInjection: PersistedEmbeddingInjection[T],
futurePool: FuturePool,
thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding],
factory: (Metric[D], FuturePool, Iterator[EntityEmbedding[T]]) => BruteForceIndex[T, D])
extends QueryableDeserialization[T, BruteForceRuntimeParams.type, D, BruteForceIndex[T, D]]
def this(
metric: Metric[D],
embeddingInjection: PersistedEmbeddingInjection[T],
futurePool: FuturePool,
thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding]
)
def fromDirectory(
serializationDirectory: AbstractFile
): BruteForceIndex[T, D]
.\ann\src\main\scala\com\twitter\ann\brute_force\BruteForceIndex.scala
object BruteForceRuntimeParams extends RuntimeParams
object BruteForceIndex
def apply[T, D <: Distance[D]](
metric: Metric[D],
futurePool: FuturePool,
initialEmbeddings: Iterator[EntityEmbedding[T]] = Iterator()
): BruteForceIndex[T, D]
class BruteForceIndex[T, D <: Distance[D]] private (
metric: Metric[D],
futurePool: FuturePool,
// visible for serialization
private[brute_force] val linkedQueue: ConcurrentLinkedQueue[EntityEmbedding[T]])
extends Appendable[T, BruteForceRuntimeParams.type, D]
with Queryable[T, BruteForceRuntimeParams.type, D]
def append(embedding: EntityEmbedding[T]): Future[Unit]
def toQueryable: Queryable[T, BruteForceRuntimeParams.type, D] = this
override def query(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: BruteForceRuntimeParams.type
): Future[List[T]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: BruteForceRuntimeParams.type
): Future[List[NeighborWithDistance[T, D]]]
object SerializableBruteForceIndex
def apply[T, D <: Distance[D]](
metric: Metric[D],
futurePool: FuturePool,
embeddingInjection: PersistedEmbeddingInjection[T],
thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding]
): SerializableBruteForceIndex[T, D]
class that wrapps a BruteForceIndex and provides a method for serialization.
*
* @param bruteForceIndex all queries and updates are sent to this index.
* @param embeddingInjection injection that can convert embeddings to thrift embeddings.
* @param thriftIteratorIO class that provides a way to write PersistedEmbeddings to disk
*/
class SerializableBruteForceIndex[T, D <: Distance[D]](
bruteForceIndex: BruteForceIndex[T, D],
embeddingInjection: PersistedEmbeddingInjection[T],
thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding])
extends Appendable[T, BruteForceRuntimeParams.type, D]
with Queryable[T, BruteForceRuntimeParams.type, D]
with Serialization
def append(entity: EntityEmbedding[T]): Future[Unit] =
bruteForceIndex.append(entity)
override def toQueryable: Queryable[T, BruteForceRuntimeParams.type, D] = this
override def query(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: BruteForceRuntimeParams.type
): Future[List[T]] =
bruteForceIndex.query(embedding, numOfNeighbours, runtimeParams)
override def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: BruteForceRuntimeParams.type
): Future[List[NeighborWithDistance[T, D]]] =
bruteForceIndex.queryWithDistance(embedding, numOfNeighbours, runtimeParams)
override def toDirectory(serializationDirectory: ResourceId): Unit
def toDirectory(serializationDirectory: AbstractFile): Unit
def toDirectory(serializationDirectory: IndexOutputFile): Unit
.\ann\src\main\scala\com\twitter\ann\common\AnnInjections.scala
object AnnInjections
.\ann\src\main\scala\com\twitter\ann\common\Api.scala
object EmbeddingType
class EntityEmbedding[T](id: T, embedding: EmbeddingVector)
// Query interface for ANN
trait Queryable[T, P <: RuntimeParams, D <: Distance[D]]
def query(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[T]]
/**
* ANN query for ids with distance.
* @param embedding: Embedding/Vector to be queried with.
* @param numOfNeighbors: Number of neighbours to be queried for.
* @param runtimeParams: Runtime params associated with index to control accuracy/latency etc.
* @return List of approximate nearest neighbour ids with distance from the query embedding.
*/
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[NeighborWithDistance[T, D]]]
}
// Query interface for ANN over indexes that are grouped
trait QueryableGrouped[T, P <: RuntimeParams, D <: Distance[D]] extends Queryable[T, P, D]
def query(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P,
key: Option[String]
): Future[List[T]]
/**
* ANN query for ids with distance.
* @param embedding: Embedding/Vector to be queried with.
* @param numOfNeighbors: Number of neighbours to be queried for.
* @param runtimeParams: Runtime params associated with index to control accuracy/latency etc.
* @param key: Optional key to lookup specific ANN index and perform query there
* @return List of approximate nearest neighbour ids with distance from the query embedding.
*/
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P,
key: Option[String]
): Future[List[NeighborWithDistance[T, D]]]
}
/**
* Runtime params associated with index to control accuracy/latency etc while querying.
*/
trait RuntimeParams
class NeighborWithDistance[T, D <: Distance[D]](neighbor: T, distance: D)
/**
* ANN query result with seed entity for which this neighbor was provided.
* @param seed: Seed Id for which ann query was called
* @param neighbor : Id of the neighbours
*/
case class NeighborWithSeed[T1, T2](seed: T1, neighbor: T2)
/**
* ANN query result with distance with seed entity for which this neighbor was provided.
* @param seed: Seed Id for which ann query was called
* @param neighbor : Id of the neighbours
* @param distance: Distance of neighbour from query ex: D: CosineDistance, L2Distance, InnerProductDistance
*/
case class NeighborWithDistanceWithSeed[T1, T2, D <: Distance[D]](
seed: T1,
neighbor: T2,
distance: D)
trait RawAppendable[P <: RuntimeParams, D <: Distance[D]]
def append(embedding: EmbeddingVector): Future[Long]
/**
* Convert an Appendable to Queryable interface to query an index.
*/
def toQueryable: Queryable[Long, P, D]
}
// Index building interface for ANN.
trait Appendable[T, P <: RuntimeParams, D <: Distance[D]]
def append(entity: EntityEmbedding[T]): Future[Unit]
/**
* Convert an Appendable to Queryable interface to query an index.
*/
def toQueryable: Queryable[T, P, D]
}
// Updatable index interface for ANN.
trait Updatable[T]
.\ann\src\main\scala\com\twitter\ann\common\EmbeddingProducer.scala
trait EmbeddingProducer[T]
.\ann\src\main\scala\com\twitter\ann\common\IndexOutputFile.scala
class creates a wrapper around GCS filesystem and HDFS filesystem for the index
* generation job. It implements the basic methods required by the index generation job and hides
* the logic around handling HDFS vs GCS.
*/
class IndexOutputFile(val abstractFile: AbstractFile, val resourceId: ResourceId)
def this(resourceId: ResourceId)
def this(abstractFile: AbstractFile)
def isAbstractFile(): Boolean
def createSuccessFile(): Unit
def isDirectory(): Boolean
def getPath(): String
def createFile(fileName: String): IndexOutputFile
def createDirectory(directoryName: String): IndexOutputFile
def getChild(fileName: String, isDirectory: Boolean = false): IndexOutputFile
def getOutputStream(): OutputStream
def getInputStream(): InputStream
def copyFrom(srcIn: InputStream): Unit
def writeIndexMetadata(annIndexMetadata: AnnIndexMetadata): Unit
def loadIndexMetadata(): AnnIndexMetadata
.\ann\src\main\scala\com\twitter\ann\common\IndexTransformer.scala
object IndexTransformer
def transformQueryable[T, P <: RuntimeParams, D <: Distance[D]](
index: Queryable[Long, P, D],
store: ReadableStore[Long, T]
): Queryable[T, P, D]
def query(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[T]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[NeighborWithDistance[T, D]]]
def transformAppendable[T, P <: RuntimeParams, D <: Distance[D]](
index: RawAppendable[P, D],
store: Store[Long, T]
): Appendable[T, P, D]
def append(entity: EntityEmbedding[T]): Future[Unit]
def toQueryable: Queryable[T, P, D]
def transform1[
Index <: RawAppendable[P, D] with Queryable[Long, P, D],
T,
P <: RuntimeParams,
D <: Distance[D]
](
index: Index,
store: Store[Long, T]
): Queryable[T, P, D] with Appendable[T, P, D]
.\ann\src\main\scala\com\twitter\ann\common\MemoizedInEpochs.scala
class MemoizedInEpochs[K, V](f: K => Try[V]) extends Logging
def epoch(keys: Seq[K]): Seq[V]
.\ann\src\main\scala\com\twitter\ann\common\Metric.scala
trait Distance[D] extends Any with Ordered[D]
def distance: Float
}
case class L2Distance(distance: Float) extends AnyVal with Distance[L2Distance]
def compare(that: L2Distance): Int =
Ordering.Float.compare(this.distance, that.distance)
}
case class CosineDistance(distance: Float) extends AnyVal with Distance[CosineDistance]
def compare(that: CosineDistance): Int =
Ordering.Float.compare(this.distance, that.distance)
}
case class InnerProductDistance(distance: Float)
extends AnyVal
with Distance[InnerProductDistance]
def compare(that: InnerProductDistance): Int =
Ordering.Float.compare(this.distance, that.distance)
}
case class EditDistance(distance: Float) extends AnyVal with Distance[EditDistance]
def compare(that: EditDistance): Int =
Ordering.Float.compare(this.distance, that.distance)
}
object Metric
def fromThrift(metric: DistanceMetric): Metric[_ <: Distance[_]]
def toThrift(metric: Metric[_ <: Distance[_]]): DistanceMetric
def fromString(metricName: String): Metric[_ <: Distance[_]]
with Injection[_, ServiceDistance]
trait Metric[D <: Distance[D]]
def distance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): D
def absoluteDistance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): Float
def fromAbsoluteDistance(distance: Float): D
}
case object L2 extends Metric[L2Distance] with Injection[L2Distance, ServiceDistance]
def distance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): L2Distance
def fromAbsoluteDistance(distance: Float): L2Distance
def absoluteDistance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): Float = distance(embedding1, embedding2).distance
override def apply(scalaDistance: L2Distance): ServiceDistance
def invert(serviceDistance: ServiceDistance): Try[L2Distance]
object Cosine extends Metric[CosineDistance] with Injection[CosineDistance, ServiceDistance]
def distance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): CosineDistance
def fromAbsoluteDistance(distance: Float): CosineDistance
def absoluteDistance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): Float = distance(embedding1, embedding2).distance
override def apply(scalaDistance: CosineDistance): ServiceDistance
def invert(serviceDistance: ServiceDistance): Try[CosineDistance]
object InnerProduct
extends Metric[InnerProductDistance]
with Injection[InnerProductDistance, ServiceDistance]
def distance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): InnerProductDistance
def fromAbsoluteDistance(distance: Float): InnerProductDistance
def absoluteDistance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): Float = distance(embedding1, embedding2).distance
override def apply(scalaDistance: InnerProductDistance): ServiceDistance
def invert(
serviceDistance: ServiceDistance
): Try[InnerProductDistance]
object Edit extends Metric[EditDistance] with Injection[EditDistance, ServiceDistance]
def intDistance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector,
pos1: Int,
pos2: Int,
precomputedDistances: scala.collection.mutable.Map[(Int, Int), Int]
): Int
def distance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): EditDistance
def fromAbsoluteDistance(distance: Float): EditDistance
def absoluteDistance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): Float = distance(embedding1, embedding2).distance
override def apply(scalaDistance: EditDistance): ServiceDistance
def invert(
serviceDistance: ServiceDistance
): Try[EditDistance]
object MetricUtil
def dot(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): Float
def l2distance(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): Double
def cosineSimilarity(
embedding1: EmbeddingVector,
embedding2: EmbeddingVector
): Float
def norm(
embedding: EmbeddingVector
): EmbeddingVector
.\ann\src\main\scala\com\twitter\ann\common\QueryableById.scala
trait that allows you to query for nearest neighbors given an arbitrary type T1. This is
* in contrast to a regular com.twitter.ann.common.Appendable, which takes an embedding as the input
* argument.
*
* This interface uses the Stitch API for batching. See go/stitch for details on how to use it.
*
* @tparam T1 type of the query.
* @tparam T2 type of the result.
* @tparam P runtime parameters supported by the index.
* @tparam D distance function used in the index.
*/
trait QueryableById[T1, T2, P <: RuntimeParams, D <: Distance[D]]
.\ann\src\main\scala\com\twitter\ann\common\QueryableByIdImplementation.scala
class QueryableByIdImplementation[T1, T2, P <: RuntimeParams, D <: Distance[D]](
embeddingProducer: EmbeddingProducer[T1],
queryable: Queryable[T2, P, D])
extends QueryableById[T1, T2, P, D]
def queryById(
id: T1,
numOfNeighbors: Int,
runtimeParams: P
): Stitch[List[T2]]
def queryByIdWithDistance(
id: T1,
numOfNeighbors: Int,
runtimeParams: P
): Stitch[List[NeighborWithDistance[T2, D]]]
def batchQueryById(
ids: Seq[T1],
numOfNeighbors: Int,
runtimeParams: P
): Stitch[List[NeighborWithSeed[T1, T2]]]
def batchQueryWithDistanceById(
ids: Seq[T1],
numOfNeighbors: Int,
runtimeParams: P
): Stitch[List[NeighborWithDistanceWithSeed[T1, T2, D]]]
.\ann\src\main\scala\com\twitter\ann\common\QueryableOperations.scala
object QueryableOperations
class Map[T, P <: RuntimeParams, D <: Distance[D]](
val q: Queryable[T, P, D])
def mapRuntimeParameters(f: P => P): Queryable[T, P, D]
.\ann\src\main\scala\com\twitter\ann\common\ReadWriteFuturePool.scala
trait ReadWriteFuturePool
def read[T](f: => T): Future[T]
def write[T](f: => T): Future[T]
}
object ReadWriteFuturePool
def apply(readPool: FuturePool, writePool: FuturePool): ReadWriteFuturePool
def apply(commonPool: FuturePool): ReadWriteFuturePool
class ReadWriteFuturePoolANN(readPool: FuturePool, writePool: FuturePool)
extends ReadWriteFuturePool
def read[T](f: => T): Future[T]
def write[T](f: => T): Future[T]
.\ann\src\main\scala\com\twitter\ann\common\Serialization.scala
trait Serialization
def toDirectory(
serializationDirectory: AbstractFile
): Unit
def toDirectory(
serializationDirectory: ResourceId
): Unit
}
/**
* Interface for reading a Queryable from a directory
* @tparam T the id of the embeddings
* @tparam Q type of the Queryable that is deserialized.
*/
trait QueryableDeserialization[T, P <: RuntimeParams, D <: Distance[D], Q <: Queryable[T, P, D]]
.\ann\src\main\scala\com\twitter\ann\common\ServiceClientQueryable.scala
class ServiceClientQueryable[T, P <: RuntimeParams, D <: Distance[D]](
service: Service[NearestNeighborQuery, NearestNeighborResult],
runtimeParamInjection: Injection[P, ServiceRuntimeParams],
distanceInjection: Injection[D, ServiceDistance],
idInjection: Injection[T, Array[Byte]])
extends Queryable[T, P, D]
def query(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[T]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[NeighborWithDistance[T, D]]] =
service
.apply(
NearestNeighborQuery(
embeddingSerDe.toThrift(embedding),
withDistance = true,
runtimeParamInjection(runtimeParams),
numOfNeighbors
)
)
.map
.\ann\src\main\scala\com\twitter\ann\common\ShardApi.scala
trait ShardFunction[T]
def apply(shards: Int, entity: EntityEmbedding[T]): Int
}
/**
* Randomly shards the embeddings based on number of total shards.
*/
class RandomShardFunction[T] extends ShardFunction[T]
def apply(shards: Int, entity: EntityEmbedding[T]): Int
class ShardedAppendable[T, P <: RuntimeParams, D <: Distance[D]](
indices: Seq[Appendable[T, P, D]],
shardFn: ShardFunction[T],
shards: Int)
extends Appendable[T, P, D]
def append(entity: EntityEmbedding[T]): Future[Unit]
def toQueryable: Queryable[T, P, D]
class ComposedQueryable[T, P <: RuntimeParams, D <: Distance[D]](
indices: Seq[Queryable[T, P, D]])
extends Queryable[T, P, D]
def query(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[T]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[NeighborWithDistance[T, D]]]
.\ann\src\main\scala\com\twitter\ann\common\ShardedSerialization.scala
object ShardConstants
class ShardedSerialization(
shards: Seq[Serialization])
extends Serialization
def toDirectory(directory: AbstractFile): Unit
def toDirectory(directory: ResourceId): Unit
def toDirectory(directory: IndexOutputFile): Unit
class ComposedQueryableDeserialization[T, P <: RuntimeParams, D <: Distance[D]](
deserializationFn: (AbstractFile) => Queryable[T, P, D])
extends QueryableDeserialization[T, P, D, Queryable[T, P, D]]
def fromDirectory(directory: AbstractFile): Queryable[T, P, D]
def accept(file: AbstractFile): Boolean =
file.getName.startsWith(ShardConstants.ShardPrefix)
})
.asScala
.toList
val indices = shardDirs
.map
class ShardedIndexBuilderWithSerialization[T, P <: RuntimeParams, D <: Distance[D]](
shardedIndex: ShardedAppendable[T, P, D],
shardedSerialization: ShardedSerialization)
extends Appendable[T, P, D]
with Serialization
def append(entity: EntityEmbedding[T]): Future[Unit]
def toDirectory(directory: AbstractFile): Unit
def toDirectory(directory: ResourceId): Unit
def toQueryable: Queryable[T, P, D]
.\ann\src\main\scala\com\twitter\ann\common\Task.scala
trait Task extends Closable
def run(): Future[Unit]
def task(): Future[Unit]
// Task interval
protected def taskInterval: Duration
/**
* Start the task after random jitter
*/
final def jitteredStart(): Unit = synchronized
def startImmediately(): Unit = synchronized
def close(deadline: Time): Future[Unit]
.\ann\src\main\scala\com\twitter\ann\dataflow\offline\ANNIndexBuilderBeamJob.scala
trait ANNOptions extends DateRangeOptions
def getOutputPath(): String
def setOutputPath(value: String): Unit
@Description("If set, the index is grouped")
@Default.Boolean(false)
def getGrouped: Boolean
def setGrouped(value: Boolean): Unit
@Description(
"If set, a segment will be registered for the provided DAL dataset module which will trigger " +
"DAL registration.")
@Default.Boolean(false)
def getEnableDalRegistration: Boolean
def setEnableDalRegistration(value: Boolean): Unit
@Description(
"Output GCS path for the generated index. The OutputPath should be of the format " +
"'gs://user.
def getOutputDALPath: String
def setOutputDALPath(value: String): Unit
@Description("Get ANN index dataset name")
def getDatasetModuleName: String
def setDatasetModuleName(value: String): Unit
@Description("Get ANN index dataset owner role")
def getDatasetOwnerRole: String
def setDatasetOwnerRole(value: String): Unit
@Description("If set, index is written in <output>/<timestamp>")
@Default.Boolean(false)
def getOutputWithTimestamp: Boolean
def setOutputWithTimestamp(value: Boolean): Unit
@Description("File which contains a SQL query to retrieve embeddings from BQ")
def getDatasetSqlPath: String
def setDatasetSqlPath(value: String): Unit
@Description("Dimension of embedding in the input data. See go/ann")
def getDimension: Int
def setDimension(value: Int): Unit
@Description("The type of entity ID that is used with the embeddings. See go/ann")
def getEntityKind: String
def setEntityKind(value: String): Unit
@Description("The kind of index you want to generate (HNSW/Annoy/Brute Force/faiss). See go/ann")
def getAlgo: String
def setAlgo(value: String): Unit
@Description("Distance metric (InnerProduct/Cosine/L2). See go/ann")
def getMetric: String
def setMetric(value: String): Unit
@Description("Specifies how many parallel inserts happen to the index. See go/ann")
def getConcurrencyLevel: Int
def setConcurrencyLevel(value: Int): Unit
@Description(
"Used by HNSW algo. Larger value increases build time but will give better recall. See go/ann")
def getEfConstruction: Int
def setEfConstruction(value: Int): Unit
@Description(
"Used by HNSW algo. Larger value increases the index size but will give better recall. " +
"See go/ann")
def getMaxM: Int
def setMaxM(value: Int): Unit
@Description("Used by HNSW algo. Approximate number of elements that will be indexed. See go/ann")
def getExpectedElements: Int
def setExpectedElements(value: Int): Unit
@Description(
"Used by Annoy. num_trees is provided during build time and affects the build time and the " +
"index size. A larger value will give more accurate results, but larger indexes. See go/ann")
def getAnnoyNumTrees: Int
def setAnnoyNumTrees(value: Int): Unit
@Description(
"FAISS factory string determines the ANN algorithm and compression. " +
"See https://github.com/facebookresearch/faiss/wiki/The-index-factory")
def getFAISSFactoryString: String
def setFAISSFactoryString(value: String): Unit
@Description("Sample rate for training during creation of FAISS index. Default is 0.05f")
@Default.Float(0.05f)
def getTrainingSampleRate: Float
def setTrainingSampleRate(value: Float): Unit
}
/**
* Builds ANN index.
*
* The input embeddings are read from BigQuery using the input SQL query. The output from this SQL
* query needs to have two columns, "entityID" [Long] and "embedding" [List[Double]]
*
* Output directory supported is GCS bucket
*/
object ANNIndexBuilderBeamJob extends ScioBeamJob[ANNOptions]
def configurePipeline(sc: ScioContext, opts: ANNOptions): Unit
def transformTableRowToKeyVal(
data: BaseEmbeddingData
): Option[KV[String, KV[Long, TEmbedding]]]
class OutputSink(
outDir: ResourceId,
isFaiss: Boolean,
outputDALPath: String,
enableDalRegistration: Boolean,
datasetModuleName: String,
datasetOwnerRole: String,
instant: Instant,
date: DateRange,
counterNameSpace: String)
extends PTransform[PCollection[Map[String, Iterable[KV[Long, TEmbedding]]]], PDone]
def expand(input: PCollection[Map[String, Iterable[KV[Long, TEmbedding]]]]): PDone
class BuildANNIndex(outDir: ResourceId, counterNameSpace: String)
extends DoFn[Map[String, Iterable[KV[Long, TEmbedding]]], Unit]
def transformKeyValToEmbeddingWithEntity[T <: EntityId](
entityKind: EntityKind[T]
)(
keyVal: KV[Long, TEmbedding]
): EntityEmbedding[T]
def processElement[T <: EntityId, D <: Distance[D]](
@Element dataGrouped: Map[String, Iterable[KV[Long, TEmbedding]]],
context: ProcessContext
): Unit
class BuildFaissANNIndex(outDir: ResourceId, counterNameSpace: String)
extends DoFn[Map[String, Iterable[KV[Long, TEmbedding]]], Unit]
def processElement[D <: Distance[D]](
@Element dataGrouped: Map[String, Iterable[KV[Long, TEmbedding]]],
context: ProcessContext
): Unit
.\ann\src\main\scala\com\twitter\ann\dataflow\offline\BaseEmbeddingData.scala
trait BaseEmbeddingData
.\ann\src\main\scala\com\twitter\ann\experimental\Runner.scala
object Runner
def main(args: Array[String]): Unit
def time[T](fn: => T): (T, Long)
.\ann\src\main\scala\com\twitter\ann\faiss\FaissCommon.scala
object FaissCommon
def apply(scalaParams: FaissParams): ServiceRuntimeParams
def invert(thriftParams: ServiceRuntimeParams): Try[FaissParams] =
thriftParams match
def isValidFaissIndex(path: AbstractFile): Boolean
.\ann\src\main\scala\com\twitter\ann\faiss\FaissIndex.scala
class FaissParams(
nprobe: Option[Int],
quantizerEf: Option[Int],
quantizerKFactorRF: Option[Int],
quantizerNprobe: Option[Int],
ht: Option[Int])
extends RuntimeParams
def toString: String = s"FaissParams($
def toLibraryString: String =
Seq(
nprobe.map
object FaissIndex
def loadIndex[T, D <: Distance[D]](
outerDimension: Int,
outerMetric: Metric[D],
directory: AbstractFile
): Queryable[T, FaissParams, D]
.\ann\src\main\scala\com\twitter\ann\faiss\FaissIndexer.scala
trait FaissIndexer extends Logging
def build[D <: Distance[D]](
pipe: TypedPipe[EntityEmbedding[Long]],
sampleRate: Float,
factoryString: String,
metric: Metric[D],
outputDirectory: AbstractFile
): Execution[Unit]
def buildAndWriteFaissIndex[D <: Distance[D]](
entities: Iterable[EntityEmbedding[Long]],
sampleRate: Float,
factoryString: String,
metricType: Metric[D],
outputDirectory: IndexOutputFile
): Unit
def copyToOutputAndCreateSuccess(
tmpFile: AbstractFile,
outputDirectory: IndexOutputFile
)
def toFloatVector(
dimensions: Int,
entities: Iterable[EntityEmbedding[Long]]
): FloatVector
def toIndexVector(embeddings: Iterable[EntityEmbedding[Long]]): LongVector
def parseMetric[D <: Distance[D]](metric: Metric[D]): MetricType = metric match
def l2Normalize[D <: Distance[D]](metric: Metric[D]): Boolean = metric match
object FaissIndexer extends FaissIndexer
.\ann\src\main\scala\com\twitter\ann\faiss\HourlyDirectoryWithSuccessFileListing.scala
object HourlyDirectoryWithSuccessFileListing extends Logging
def listHourlyIndexDirectories(
root: AbstractFile,
startingFrom: Time,
count: Int,
lookbackInterval: Int
): Seq[AbstractFile] = listingStep(root, startingFrom, count, lookbackInterval)
private def listingStep(
root: AbstractFile,
startingFrom: Time,
remainingDirectoriesToFind: Int,
remainingAttempts: Int
): List[AbstractFile]
def getSuccessfulDirectoryForDate(
root: AbstractFile,
date: Time
): Try[AbstractFile]
.\ann\src\main\scala\com\twitter\ann\faiss\HourlyShardedIndex.scala
object HourlyShardedIndex
def loadIndex[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
directory: AbstractFile,
shardsToLoad: Int,
shardWatchInterval: Duration,
lookbackInterval: Int,
statsReceiver: StatsReceiver
): HourlyShardedIndex[T, D]
class HourlyShardedIndex[T, D <: Distance[D]](
outerMetric: Metric[D],
outerDimension: Int,
directory: AbstractFile,
shardsToLoad: Int,
shardWatchInterval: Duration,
lookbackInterval: Int,
override protected val statsReceiver: StatsReceiver)
extends QueryableIndexAdapter[T, D]
with Logging
with Task
def index: Index
trait
protected def task(): Future[Unit] = Future.value(reloadShards())
protected def taskInterval: Duration = shardWatchInterval
private def loadIndex(directory: AbstractFile): Try[Index] =
Try(QueryableIndexAdapter.loadJavaIndex(directory))
private val shardsCache = new MemoizedInEpochs[AbstractFile, Index](loadIndex)
// Destroying original index invalidate casted index. Keep a reference to both.
private val originalIndex = new AtomicReference[IndexShards]()
private val castedIndex = new AtomicReference[Index]()
private def reloadShards(): Unit
.\ann\src\main\scala\com\twitter\ann\faiss\QueryableIndexAdapter.scala
object QueryableIndexAdapter extends Logging
def loadJavaIndex(directory: AbstractFile): Index
trait QueryableIndexAdapter[T, D <: Distance[D]] extends Queryable[T, FaissParams, D]
def index: Index
protected val metric: Metric[D]
protected val dimension: Int
private def maybeNormalizeEmbedding(embeddingVector: EmbeddingVector): EmbeddingVector
def maybeTranslateToCosineDistanceInplace(array: floatArray, len: Int): Unit
def ensuringParams[R](parameterString: String, f: () => R): R
def replaceIndex(f: () => Unit): Unit
def query(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: FaissParams
): Future[List[T]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: FaissParams
): Future[List[NeighborWithDistance[T, D]]]
def toFloatArray(emb: EmbeddingVector): floatArray
def toSeq(vector: LongVector, len: Long): Seq[Long]
def toSeq(array: floatArray, len: Int): Seq[Float]
.\ann\src\main\scala\com\twitter\ann\featurestore\FeatureStoreEmbeddingProducer.scala
object FeatureStoreEmbeddingProducer
def apply[T <: EntityId](
dataset: VersionedOnlineAccessDataset[T, TEmbedding],
version: Long,
boundFeature: BoundFeature[T, RawFloatTensor],
client: Client,
statsReceiver: StatsReceiver = new InMemoryStatsReceiver,
featureStoreAttributions: Seq[Attribution] = Seq.empty
): EmbeddingProducer[EntityWithId[T]]
class FeatureStoreEmbeddingProducer[T <: EntityId](
boundFeature: BoundFeature[T, RawFloatTensor],
featureStoreClient: FeatureStoreClient)
extends EmbeddingProducer[EntityWithId[T]]
def produceEmbedding(input: EntityWithId[T]): Stitch[Option[Embedding[Float]]]
.\ann\src\main\scala\com\twitter\ann\file_store\ReadableIndexIdFileStore.scala
object ReadableIndexIdFileStore
def apply[V](
file: AbstractFile,
injection: Injection[V, Array[Byte]]
): ReadableStore[Long, V]
def loadFile(file: AbstractFile): ByteBuffer
.\ann\src\main\scala\com\twitter\ann\file_store\WritableIndexIdFileStore.scala
object WritableIndexIdFileStore
def apply[V](
injection: Injection[V, Array[Byte]]
): WritableIndexIdFileStore[V]
class WritableIndexIdFileStore[V] private (
map: JConcurrentHashMap[Long, Option[V]],
injection: Injection[V, Array[Byte]])
extends Store[Long, V]
def get(k: Long): Future[Option[V]]
def put(kv: (Long, Option[V])): Future[Unit]
def save(file: IndexOutputFile): Unit
def getInjection: Injection[V, Array[Byte]] = injection
private[this] def toThrift(): FileBasedIndexIdStore
def saveThrift(thriftObj: FileBasedIndexIdStore, file: IndexOutputFile): Unit
.\ann\src\main\scala\com\twitter\ann\hnsw\DistanceFunctionGenerator.scala
object DistanceFunctionGenerator
def apply[T, D <: Distance[D]](
metric: Metric[D],
idToEmbeddingFn: (T) => EmbeddingVector
): DistanceFunctionGenerator[T]
def distance(id1: T, id2: T) =
updatedMetric.absoluteDistance(
idToEmbeddingFn(id1),
idToEmbeddingFn(id2)
)
}
val distFnQuery = new DistanceFunction[EmbeddingVector, T]
.\ann\src\main\scala\com\twitter\ann\hnsw\Hnsw.scala
object Hnsw
def apply[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
efConstruction: Int,
maxM: Int,
expectedElements: Int,
futurePool: ReadWriteFuturePool,
idEmbeddingMap: IdEmbeddingMap[T]
): Hnsw[T, D]
def get(): Random = ThreadLocalRandom.current()
}
val distFn =
DistanceFunctionGenerator(metric, (key: T) => idEmbeddingMap.get(key))
val internalIndex = new HnswIndex[T, EmbeddingVector](
distFn.index,
distFn.query,
efConstruction,
maxM,
expectedElements,
randomProvider
)
new Hnsw[T, D](
dimension,
metric,
internalIndex,
futurePool,
idEmbeddingMap,
distFn.shouldNormalize,
LockedAccess.apply(expectedElements)
)
}
}
private[hnsw] object LockedAccess
def apply[T](expectedElements: Int): LockedAccess[T] =
DefaultLockedAccess(new ConcurrentHashMap[T, Lock](expectedElements))
protected[hnsw] def apply[T](): LockedAccess[T] =
DefaultLockedAccess(new ConcurrentHashMap[T, Lock]())
}
private[hnsw] case class DefaultLockedAccess[T](locks: ConcurrentHashMap[T, Lock])
extends LockedAccess[T]
def lockProvider(item: T) = locks.computeIfAbsent(item, (_: T) => new ReentrantLock())
}
private[hnsw] trait LockedAccess[T]
def lockProvider(item: T): Lock
def lock[K](item: T)(fn: => K): K
class Hnsw[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
hnswIndex: HnswIndex[T, EmbeddingVector],
readWriteFuturePool: ReadWriteFuturePool,
idEmbeddingMap: IdEmbeddingMap[T],
shouldNormalize: Boolean,
lockedAccess: LockedAccess[T] = LockedAccess.apply[T]())
extends Appendable[T, HnswParams, D]
with Queryable[T, HnswParams, D]
with Updatable[T]
def append(entity: EntityEmbedding[T]): Future[Unit]
def toQueryable: Queryable[T, HnswParams, D] = this
override def query(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: HnswParams
): Future[List[T]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: HnswParams
): Future[List[NeighborWithDistance[T, D]]]
def updatedEmbedding(embedding: EmbeddingVector): EmbeddingVector
def getIndex: HnswIndex[T, EmbeddingVector] = hnswIndex
def getDimen: Int = dimension
def getMetric: Metric[D] = metric
def getIdEmbeddingMap: IdEmbeddingMap[T] = idEmbeddingMap
override def update(
entity: EntityEmbedding[T]
): Future[Unit]
.\ann\src\main\scala\com\twitter\ann\hnsw\HnswCommon.scala
object HnswCommon
def apply(scalaParams: HnswParams): ServiceRuntimeParams
def invert(thriftParams: ServiceRuntimeParams): Try[HnswParams] =
thriftParams match
def isValidHnswIndex(path: AbstractFile): Boolean
class HnswParams(ef: Int) extends RuntimeParams
.\ann\src\main\scala\com\twitter\ann\hnsw\HnswIOUtil.scala
object HnswIOUtil
def loadEmbeddings[T](
embeddingFile: AbstractFile,
injection: Injection[T, Array[Byte]],
idEmbeddingMap: IdEmbeddingMap[T],
): IdEmbeddingMap[T]
def saveEmbeddings[T](
stream: OutputStream,
injection: Injection[T, Array[Byte]],
iter: Iterator[(T, EmbeddingVector)]
): Unit
def saveIndexMetadata(
dimension: Int,
metric: Metric[_ <: Distance[_]],
numElements: Int,
metadataStream: OutputStream
): Unit
def loadIndexMetadata(
metadataFile: AbstractFile
): HnswIndexMetadata
.\ann\src\main\scala\com\twitter\ann\hnsw\IdEmbeddingMap.scala
trait IdEmbeddingMap[T]
.\ann\src\main\scala\com\twitter\ann\hnsw\JMapBasedIdEmbeddingMap.scala
object JMapBasedIdEmbeddingMap
def applyInMemory[T](expectedElements: Int): IdEmbeddingMap[T] =
new JMapBasedIdEmbeddingMap[T](
new ConcurrentHashMap[T, EmbeddingVector](expectedElements),
Option.empty
)
/**
* Creates in-memory concurrent hashmap based container that can be serialized to disk for storing id embedding mapping.
* @param expectedElements: Expected num of elements for sizing hint, need not be exact.
* @param injection : Injection for typed Id T to Array[Byte]
*/
def applyInMemoryWithSerialization[T](
expectedElements: Int,
injection: Injection[T, Array[Byte]]
): IdEmbeddingMap[T] =
new JMapBasedIdEmbeddingMap[T](
new ConcurrentHashMap[T, EmbeddingVector](expectedElements),
Some(injection)
)
/**
* Loads id embedding mapping in in-memory concurrent hashmap.
* @param embeddingFile: Local/Hdfs file path for embeddings
* @param injection : Injection for typed Id T to Array[Byte]
* @param numElements: Expected num of elements for sizing hint, need not be exact
*/
def loadInMemory[T](
embeddingFile: AbstractFile,
injection: Injection[T, Array[Byte]],
numElements: Option[Int] = Option.empty
): IdEmbeddingMap[T]
class JMapBasedIdEmbeddingMap[T](
map: java.util.concurrent.ConcurrentHashMap[T, EmbeddingVector],
injection: Option[Injection[T, Array[Byte]]])
extends IdEmbeddingMap[T]
def putIfAbsent(id: T, embedding: EmbeddingVector): EmbeddingVector
def put(id: T, embedding: EmbeddingVector): EmbeddingVector
def get(id: T): EmbeddingVector
def iter(): Iterator[(T, EmbeddingVector)] =
map
.entrySet()
.iterator()
.asScala
.map(e => (e.getKey, e.getValue))
override def size(): Int = map.size()
override def toDirectory(embeddingFile: OutputStream): Unit
.\ann\src\main\scala\com\twitter\ann\hnsw\MapDbBasedIdEmbeddingMap.scala
class currently only support querying and creates map db on fly from thrift serialized embedding mapping
* Implement index creation with this or altogether replace mapdb with some better performing solution as it takes a lot of time to create/query or precreate while serializing thrift embeddings
*/
private[hnsw] object MapDbBasedIdEmbeddingMap
def loadAsReadonly[T](
embeddingFile: AbstractFile,
injection: Injection[T, Array[Byte]]
): IdEmbeddingMap[T]
class MapDbBasedIdEmbeddingMap[T](
mapDb: HTreeMap[Array[Byte], Array[Float]],
injection: Injection[T, Array[Byte]])
extends IdEmbeddingMap[T]
def putIfAbsent(id: T, embedding: EmbeddingVector): EmbeddingVector
def put(id: T, embedding: EmbeddingVector): EmbeddingVector
def get(id: T): EmbeddingVector
def iter(): Iterator[(T, EmbeddingVector)]
def size(): Int = mapDb.size()
override def toDirectory(embeddingFile: OutputStream): Unit
.\ann\src\main\scala\com\twitter\ann\hnsw\SerializableHnsw.scala
object SerializableHnsw
def apply[T, D <: Distance[D]](
index: Hnsw[T, D],
injection: Injection[T, Array[Byte]]
): SerializableHnsw[T, D]
def loadMapBasedQueryableIndex[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
futurePool: ReadWriteFuturePool,
directory: AbstractFile
): SerializableHnsw[T, D]
def loadMMappedBasedQueryableIndex[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
futurePool: ReadWriteFuturePool,
directory: AbstractFile
): SerializableHnsw[T, D]
def loadIndex[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
futurePool: ReadWriteFuturePool,
directory: AbstractFile,
idEmbeddingMap: IdEmbeddingMap[T],
metadata: HnswIndexMetadata
): SerializableHnsw[T, D]
def get(): Random = ThreadLocalRandom.current()
}
val internalIndex = HnswIndex.loadHnswIndex[T, EmbeddingVector](
distFn.index,
distFn.query,
directory.getChild(InternalIndexDir),
injection,
randomProvider
)
val index = new Hnsw[T, D](
dimension,
metric,
internalIndex,
futurePool,
idEmbeddingMap,
distFn.shouldNormalize,
LockedAccess.apply(metadata.numElements)
)
new SerializableHnsw(index, injection)
}
private[this] def validateMetadata[D <: Distance[D]](
dimension: Int,
metric: Metric[D],
existingMetadata: HnswIndexMetadata
): Unit
class SerializableHnsw[T, D <: Distance[D]](
index: Hnsw[T, D],
injection: Injection[T, Array[Byte]])
extends Appendable[T, HnswParams, D]
with Queryable[T, HnswParams, D]
with Serialization
with Updatable[T]
def append(entity: EntityEmbedding[T]) = index.append(entity)
override def toQueryable: Queryable[T, HnswParams, D] = index.toQueryable
override def query(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: HnswParams
) = index.query(embedding, numOfNeighbours, runtimeParams)
override def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbours: Int,
runtimeParams: HnswParams
) = index.queryWithDistance(embedding, numOfNeighbours, runtimeParams)
def toDirectory(directory: ResourceId): Unit
def toDirectory(directory: AbstractFile): Unit
def toDirectory(indexFile: IndexOutputFile): Unit
def update(
entity: EntityEmbedding[T]
): Future[Unit]
.\ann\src\main\scala\com\twitter\ann\hnsw\TypedHnswIndex.scala
object TypedHnswIndex
def index[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
efConstruction: Int,
maxM: Int,
expectedElements: Int,
readWriteFuturePool: ReadWriteFuturePool
): Appendable[T, HnswParams, D] with Queryable[T, HnswParams, D] with Updatable[T]
def serializableIndex[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
efConstruction: Int,
maxM: Int,
expectedElements: Int,
injection: Injection[T, Array[Byte]],
readWriteFuturePool: ReadWriteFuturePool
): Appendable[T, HnswParams, D]
with Queryable[T, HnswParams, D]
with Updatable[T]
with Serialization
def loadIndex[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
readWriteFuturePool: ReadWriteFuturePool,
directory: AbstractFile
): Appendable[T, HnswParams, D]
with Queryable[T, HnswParams, D]
with Updatable[T]
with Serialization
def loadMMappedIndex[T, D <: Distance[D]](
dimension: Int,
metric: Metric[D],
injection: Injection[T, Array[Byte]],
readWriteFuturePool: ReadWriteFuturePool,
directory: AbstractFile
): Appendable[T, HnswParams, D]
with Queryable[T, HnswParams, D]
with Updatable[T]
with Serialization
.\ann\src\main\scala\com\twitter\ann\manhattan\ManhattanEmbeddingProducer.scala
class ManhattanEmbeddingProducer[T](
keyDescriptor: DescriptorP1L0.DKey[T],
valueDescriptor: ValueDescriptor.EmptyValue[EmbeddingVector],
manhattanEndpoint: ManhattanKVEndpoint)
extends EmbeddingProducer[T]
def produceEmbedding(input: T): Stitch[Option[EmbeddingVector]]
object ManhattanEmbeddingProducer
def keyDescriptor[T](
injection: Injection[T, Array[Byte]],
dataset: String
): DescriptorP1L0.DKey[T] =
ReadOnlyKeyDescriptor(injection.andThen(Bijections.BytesBijection))
.withDataset(dataset)
private[manhattan] val EmbeddingDescriptor: ValueDescriptor.EmptyValue[
EmbeddingType.EmbeddingVector
]
def apply[T](
dataset: String,
injection: Injection[T, Array[Byte]],
manhattanEndpoint: ManhattanKVEndpoint
): EmbeddingProducer[T]
.\ann\src\main\scala\com\twitter\ann\scalding\benchmark\Knn.scala
class com.twitter.ann.scalding.offline.com.twitter.ann.scalding.benchmark.KnnJob -- \
* --dalEnvironment Prod \
* --search_space_entity_type user \
* --user.feature_store_embedding ConsumerFollowEmbedding300Dataset \
* --user.feature_store_major_version 1569196895 \
* --user.date_range 2019-10-23 \
* --search_space.feature_store_embedding ConsumerFollowEmbedding300Dataset \
* --search_space.feature_store_major_version 1569196895 \
* --search_space.date_range 2019-10-23 \
* --date 2019-10-25 \
* --version "consumer_follower_test" \
* --reducers 10000 \
* --num_of_random_groups 20 \
* --num_replicas 1000 \
* --indexing_strategy.metric InnerProduct \
* --indexing_strategy.type hnsw \
* --indexing_strategy.dimension 300 \
* --indexing_strategy.ef_construction 30 \
* --indexing_strategy.max_m 10 \
* --indexing_strategy.ef_query 50 \
* --search_space_shards 3000 \
* --query_shards 3000 \
* --search_space.read_sample_ratio 0.038
*/
trait KnnJobBase
def getKnnDataset[B <: EntityId, D <: Distance[D]](
args: Args
)(
implicit uniqueID: UniqueID
): TypedPipe[Knn]
object KnnJob extends TwitterExecutionApp with KnnJobBase
def job: Execution[Unit] = Execution.withId
.\ann\src\main\scala\com\twitter\ann\scalding\offline\IndexingStrategy.scala
trait IndexingStrategy[D <: Distance[D]]
def buildIndex[T](
indexItems: TraversableOnce[EntityEmbedding[T]]
): ParameterlessQueryable[T, _, D]
}
object IndexingStrategy
def parse(
args: Args,
argumentName: String = "indexing_strategy"
): IndexingStrategy[_]
def metricArg[D <: Distance[D]] =
Metric.fromString(args(s"$argumentName.metric")).asInstanceOf[Metric[D]]
args(s"$argumentName.type") match
class BruteForceIndexingStrategy[D <: Distance[D]](metric: Metric[D])
extends IndexingStrategy[D]
def buildIndex[T](
indexItems: TraversableOnce[EntityEmbedding[T]]
): ParameterlessQueryable[T, _, D]
class HnswIndexingStrategy[D <: Distance[D]](
dimension: Int,
metric: Metric[D],
efConstruction: Int,
maxM: Int,
hnswParams: HnswParams,
concurrencyLevel: Int = 1)
extends IndexingStrategy[D]
def buildIndex[T](
indexItems: TraversableOnce[EntityEmbedding[T]]
): ParameterlessQueryable[T, _, D]
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnDebug.scala
class ConsumerAssoc(consumerId: UserId, assoc: List[String])
object KnnDebug
def getConsumerAssociations(
graph: TypedPipe[GraphEdge[UserId, UserId]],
usernames: TypedPipe[(UserId, String)],
reducers: Int
): TypedPipe[ConsumerAssoc]
def getDebugTable(
neighborsPipe: TypedPipe[(EntityKey, NearestNeighbors)],
shards: Int,
reducers: Int,
limit: Int = 10000,
userDataset: Option[TypedPipe[FlatUser]] = None,
followDataset: Option[TypedPipe[GraphEdge[UserId, UserId]]] = None,
consumerStatesDataset: Option[TypedPipe[CondensedUserState]] = None,
minFollows: Int = 25,
maxFollows: Int = 50
)(
implicit dateRange: DateRange
): TypedPipe[(String, String, String, String)]
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnEntityRecoDebugJob.scala
object KnnEntityRecoDebugJob extends TwitterExecutionApp
def job: Execution[Unit] = Execution.withId
def run[A <: EntityId, B <: EntityId, D <: Distance[D]](
uncastQueryEntityKind: EntityKind[_],
uncastSearchSpaceEntityKind: EntityKind[_],
uncastMetric: Metric[_],
args: Args
)(
implicit uniqueID: UniqueID
): Execution[Unit]
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnHelper.scala
class Index[T, D <: Distance[D]](
injection: Injection[T, Array[Byte]],
metric: Metric[D],
dimension: Int,
directory: AbstractFile)
object KnnHelper
def getFilteredUserEmbeddings(
args: Args,
filterPath: Option[String],
reducers: Int,
useHashJoin: Boolean
)(
implicit dateRange: DateRange
): TypedPipe[EmbeddingWithEntity[UserId]]
def getNeighborsPipe[T <: EntityId, D <: Distance[D]](
args: Args,
uncastEntityKind: EntityKind[_],
uncastMetric: Metric[_],
ef: Int,
consumerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]],
abstractFile: Option[AbstractFile],
reducers: Int,
numNeighbors: Int,
dimension: Int
)(
implicit dateRange: DateRange
): TypedPipe[(EntityKey, NearestNeighbors)]
def bruteForceNearestNeighbors(
consumerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]],
producerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]],
numNeighbors: Int,
reducers: Int
): TypedPipe[(EntityKey, NearestNeighbors)]
def findNearestNeighbours[A <: EntityId, B <: EntityId, D <: Distance[D]](
queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]],
searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]],
metric: Metric[D],
numNeighbors: Int = 10,
queryIdsFilter: Option[TypedPipe[A]] = Option.empty,
reducers: Int = 100,
mappers: Int = 100,
isSearchSpaceLarger: Boolean = true,
numOfSearchGroups: Int = 1,
numReplicas: Int = 1,
useCounters: Boolean = true
)(
implicit ordering: Ordering[A],
uid: UniqueID
): TypedPipe[(A, Seq[(B, D)])]
def findNearestNeighboursWithIndexingStrategy[A <: EntityId, B <: EntityId, D <: Distance[D]](
queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]],
searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]],
numNeighbors: Int,
numOfSearchGroups: Int,
indexingStrategy: IndexingStrategy[D],
numReplicas: Int = 1,
reducersOption: Option[Int] = None,
queryShards: Option[Int] = None,
searchSpaceShards: Option[Int] = None,
useCounters: Boolean = true
)(
implicit ordering: Ordering[A],
uid: UniqueID
): UnsortedGrouped[A, Seq[(B, D)]]
def shard[T](
pipe: TypedPipe[T],
numberOfShards: Option[Int]
): TypedPipe[T]
def findNearestNeighboursViaCross[A <: EntityId, B <: EntityId, D <: Distance[D]](
queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]],
searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]],
metric: Metric[D],
numNeighbors: Int,
reducers: Int,
mappers: Int,
isSearchSpaceLarger: Boolean
)(
implicit ordering: Ordering[A]
): TypedPipe[(A, Seq[(B, D)])]
def nearestNeighborsToString[A <: EntityId, B <: EntityId, D <: Distance[D]](
nearestNeighbors: (A, Seq[(B, D)]),
queryEntityKind: EntityKind[A],
neighborEntityKind: EntityKind[B],
idDistanceSeparator: String = ":",
neighborSeparator: String = "\t"
): String
class NNKey(
searchGroup: Int,
replica: Int,
maxReplica: Option[Int] = None)
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnOfflineJob.scala
object KnnOfflineJob extends TwitterExecutionApp
def job: Execution[Unit] = Execution.withId
object for this job
* @param abstractFile: An optional of producer embedding path
*/
def execute(
args: Args,
abstractFile: Option[AbstractFile]
)(
implicit uniqueID: UniqueID
): Execution[Unit]
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnTruthSetGenerator.scala
object KnnTruthSetGenerator extends TwitterExecutionApp
def job: Execution[Unit] = Execution.withId
def run[A <: EntityId, B <: EntityId, D <: Distance[D]](
uncastQueryEntityKind: EntityKind[_],
uncastIndexSpaceEntityKind: EntityKind[_],
uncastMetric: Metric[_],
args: Args
)(
implicit uniqueID: UniqueID
): Execution[Unit]
.\ann\src\main\scala\com\twitter\ann\scalding\offline\ParameterlessQueryable.scala
class ParameterlessQueryable[T, P <: RuntimeParams, D <: Distance[D]](
queryable: Queryable[T, P, D],
runtimeParamsForAllQueries: P)
.\ann\src\main\scala\com\twitter\ann\scalding\offline\faissindexbuilder\IndexBuilder.scala
object IndexBuilder extends FaissIndexer with Logging
def run[T <: UserId, D <: Distance[D]](
embeddingFormat: EmbeddingFormat[T],
embeddingLimit: Option[Int],
sampleRate: Float,
factoryString: String,
metric: Metric[D],
outputDirectory: AbstractFile,
numDimensions: Int
): Execution[Unit]
.\ann\src\main\scala\com\twitter\ann\scalding\offline\faissindexbuilder\IndexBuilderApp.scala
trait IndexBuilderExecutable extends Logging
def indexBuilderExecution[T <: UserId, D <: Distance[D]](
args: Args
): Execution[Unit]
object IndexBuilderApp extends TwitterExecutionApp with IndexBuilderExecutable
def job: Execution[Unit] = Execution.getArgs.flatMap
.\ann\src\main\scala\com\twitter\ann\scalding\offline\indexbuilder\IndexBuilder.scala
object IndexBuilder
def run[T <: EntityId, _, D <: Distance[D]](
embeddingFormat: EmbeddingFormat[T],
embeddingLimit: Option[Int],
index: Appendable[T, _, D] with Serialization,
concurrencyLevel: Int,
outputDirectory: AbstractFile,
numDimensions: Int
): Execution[Unit]
.\ann\src\main\scala\com\twitter\ann\scalding\offline\indexbuilder\IndexBuilderApp.scala
trait IndexBuilderExecutable
def indexBuilderExecution[T <: EntityId, D <: Distance[D]](
args: Args
): Execution[Unit]
object IndexBuilderApp extends TwitterExecutionApp with IndexBuilderExecutable
def job: Execution[Unit] = Execution.getArgs.flatMap
.\ann\src\main\scala\com\twitter\ann\scalding\offline\indexbuilderfrombq\IndexBuilderFromBQ.scala
object IndexBuilder
def run[T <: EntityId, _, D <: Distance[D]](
embeddingsPipe: TypedPipe[EmbeddingWithEntity[T]],
embeddingLimit: Option[Int],
index: Appendable[T, _, D] with Serialization,
concurrencyLevel: Int,
outputDirectory: AbstractFile,
numDimensions: Int
): Execution[Unit]
.\ann\src\main\scala\com\twitter\ann\scalding\offline\indexbuilderfrombq\IndexBuilderFromBQApp.scala
trait IndexBuilderFromBQExecutable
def indexBuilderExecution[T <: EntityId, D <: Distance[D]](
args: Args
): Execution[Unit]
def avroMapper(row: GenericRecord): KeyVal[Long, java.util.List[Double]]
def transform[T <: EntityId](
entityKind: EntityKind[T]
)(
bqRecord: KeyVal[Long, java.util.List[Double]]
): EmbeddingWithEntity[T]
object IndexBuilderFromBQApp extends TwitterExecutionApp with IndexBuilderFromBQExecutable
def job: Execution[Unit] = Execution.getArgs.flatMap
.\ann\src\main\scala\com\twitter\ann\serialization\DummyANNIndexInjection.scala
object DummyANNIndexInjection
.\ann\src\main\scala\com\twitter\ann\serialization\PersistedEmbeddingInjection.scala
class PersistedEmbeddingInjection[T](
idByteInjection: Injection[T, Array[Byte]])
extends Injection[EntityEmbedding[T], PersistedEmbedding]
def apply(entity: EntityEmbedding[T]): PersistedEmbedding
def invert(persistedEmbedding: PersistedEmbedding): Try[EntityEmbedding[T]]
.\ann\src\main\scala\com\twitter\ann\serialization\ThriftIteratorIO.scala
class can do things lazily so there is no need to have all the object into memory.
*/
class ThriftIteratorIO[T <: ThriftStruct](
codec: ThriftStructCodec[T])
def toOutputStream(
iterator: Iterator[T],
outputStream: OutputStream
): Unit
def fromInputStream(
inputStream: InputStream
): Iterator[T]
object ThriftIteratorIO
def getIterator[T <: ThriftStruct](
codec: ThriftStructCodec[T],
inputStream: InputStream
): Iterator[T]
def getNext: Option[T] =
try
.\ann\src\main\scala\com\twitter\ann\service\loadtest\AnnLoadTest.scala
class AnnIndexQueryLoadTest(
worker: AnnLoadTestWorker = new AnnLoadTestWorker())
def performQueries[T, P <: RuntimeParams, D <: Distance[D]](
queryable: Queryable[T, P, D],
qps: Int,
duration: Duration,
queries: Seq[Query[T]],
concurrencyLevel: Int,
runtimeConfigurations: Seq[QueryTimeConfiguration[T, P]]
): Future[Unit]
class Query[T](embedding: EmbeddingVector, trueNeighbours: Seq[T] = Seq.empty)
class AnnIndexBuildLoadTest(
buildRecorder: LoadTestBuildRecorder,
embeddingIndexer: EmbeddingIndexer = new EmbeddingIndexer())
def indexEmbeddings[T, P <: RuntimeParams, D <: Distance[D]](
appendable: Appendable[T, P, D],
indexSet: Seq[EntityEmbedding[T]],
concurrencyLevel: Int
): Future[Queryable[T, P, D]]
.\ann\src\main\scala\com\twitter\ann\service\loadtest\AnnLoadTestMain.scala
object AnnLoadTestMain extends TwitterServer
def start(): Unit
def getQueries[Q, I]: Seq[Query[I]]
def getQueryRuntimeConfig[
T,
P <: RuntimeParams
]: Seq[QueryTimeConfiguration[T, P]]
def toNonZeroOptional(x: Int): Option[Int] = if (x != 0) Some(x) else None
for
def buildQueryable[T, P <: RuntimeParams, D <: Distance[D]](
inMemoryBuildRecorder: InMemoryLoadTestBuildRecorder
): Future[Queryable[T, P, D]]
def indexEmbeddingsAndGetQueryable[T, P <: RuntimeParams, D <: Distance[D]](
buildRecorder: LoadTestBuildRecorder,
indexSet: Seq[EntityEmbedding[T]]
): Future[Queryable[T, P, D]]
def performQueries[T, P <: RuntimeParams, D <: Distance[D]](
queryable: Queryable[T, P, D],
queryTimeConfig: Seq[QueryTimeConfiguration[T, P]],
queries: Seq[Query[T]]
): Future[Unit]
def getIndexIdInjection[T]: Injection[T, Array[Byte]]
def getRuntimeParamInjection[
P <: RuntimeParams
]: Injection[P, ServiceRuntimeParams]
def getDistanceInjection[D <: Distance[D]]: Injection[D, ServiceDistance]
def getDistanceMetric[D <: Distance[D]]: Metric[D]
def buildQueryTimeConfig[T, P <: RuntimeParams](
numOfNeighbors: Int,
params: P,
config: Map[String, String]
): QueryTimeConfiguration[T, P]
.\ann\src\main\scala\com\twitter\ann\service\loadtest\AnnLoadTestWorker.scala
object QueryTimeConfiguration
class QueryTimeConfiguration[T, P <: RuntimeParams](
recorder: LoadTestQueryRecorder[T],
param: P,
numberOfNeighbors: Int,
private val results: InMemoryLoadTestQueryRecorder[T])
def toString: String =
s"QueryTimeConfiguration(param = $param, numberOfNeighbors = $numberOfNeighbors)"
def printResults: String
class AnnLoadTestWorker(
timer: Timer = DefaultTimer)
def runWithQps[T, P <: RuntimeParams, D <: Distance[D]](
queryable: Queryable[T, P, D],
queries: Seq[Query[T]],
qps: Int,
duration: Duration,
configuration: QueryTimeConfiguration[T, P],
concurrencyLevel: Int
): Future[Int]
def performQuery[T, P <: RuntimeParams, D <: Distance[D]](
configuration: QueryTimeConfiguration[T, P],
queryable: Queryable[T, P, D],
query: Query[T]
): Future[Try[Unit]]
.\ann\src\main\scala\com\twitter\ann\service\loadtest\EmbeddingIndexer.scala
class EmbeddingIndexer
def indexEmbeddings[T, P <: RuntimeParams, D <: Distance[D]](
appendable: Appendable[T, P, D],
recorder: LoadTestBuildRecorder,
indexSet: Seq[EntityEmbedding[T]],
concurrencyLevel: Int
): Future[Queryable[T, P, D]]
.\ann\src\main\scala\com\twitter\ann\service\loadtest\LoadTestRecorder.scala
trait LoadTestQueryRecorder[T]
def recordQueryResult(
trueNeighbors: Seq[T],
foundNeighbors: Seq[T],
queryLatency: Duration
): Unit
}
case class LoadTestQueryResults(
numResults: Int,
top1Recall: Float,
top10Recall: Option[Float],
overallRecall: Float)
private object LoadTestQueryRecorder
def recordQueryResult[T](
trueNeighbors: Seq[T],
foundNeighbors: Seq[T]
): LoadTestQueryResults
class StatsLoadTestQueryRecorder[T](
statsReceiver: StatsReceiver)
extends LoadTestQueryRecorder[T]
def recordQueryResult(
trueNeighbors: Seq[T],
foundNeighbors: Seq[T],
queryLatency: Duration
): Unit
trait LoadTestBuildRecorder
def recordIndexCreation(
indexSize: Int,
indexLatency: Duration,
toQueryableLatency: Duration
): Unit
}
class StatsLoadTestBuildRecorder(
statsReceiver: StatsReceiver)
extends LoadTestBuildRecorder
def recordIndexCreation(
indexSize: Int,
indexLatency: Duration,
toQueryableLatency: Duration
): Unit
class QueryRecorderSnapshot(snapshot: Snapshot)
def avgQueryLatencyMicros: Double = snapshot.average
def p50QueryLatencyMicros: Double =
snapshot.percentiles.find(_.quantile == .5).get.value
def p90QueryLatencyMicros: Double =
snapshot.percentiles.find(_.quantile == .9).get.value
def p99QueryLatencyMicros: Double =
snapshot.percentiles.find(_.quantile == .99).get.value
}
class InMemoryLoadTestQueryRecorder[T](
// You have to specify a name of the histogram even though it is not used
// Use latch period of bottom. We will compute a new snapshot every time we call computeSnapshot
private[this] val latencyHistogram: MetricsBucketedHistogram =
new MetricsBucketedHistogram("latencyhistogram", latchPeriod = Duration.Bottom))
extends LoadTestQueryRecorder[T]
def computeSnapshot(): QueryRecorderSnapshot
def recall: Double =
if (counter.get() != 0)
def top1Recall: Double =
if (counter.get() != 0)
def top10Recall: Double =
if (countMoreThan10Results.get() != 0)
def avgRPS: Double =
if (elapsedTime.get() != Duration.Zero)
def recordQueryResult(
trueNeighbors: Seq[T],
foundNeighbors: Seq[T],
queryLatency: Duration
): Unit
class InMemoryLoadTestBuildRecorder extends LoadTestBuildRecorder
def recordIndexCreation(
size: Int,
indexLatencyArg: Duration,
toQueryableLatencyArg: Duration
): Unit
class ComposedLoadTestQueryRecorder[T](
recorders: Seq[LoadTestQueryRecorder[T]])
extends LoadTestQueryRecorder[T]
def recordQueryResult(
trueNeighbors: Seq[T],
foundNeighbors: Seq[T],
queryLatency: Duration
): Unit = recorders.foreach
class ComposedLoadTestBuildRecorder(
recorders: Seq[LoadTestBuildRecorder])
extends LoadTestBuildRecorder
def recordIndexCreation(
indexSize: Int,
indexLatency: Duration,
toQueryableLatency: Duration
): Unit = recorders.foreach
.\ann\src\main\scala\com\twitter\ann\service\loadtest\LoadTestUtils.scala
object LoadTestUtils
def getTruthSetMap[Q, I](
directory: String,
queryIdType: String,
indexIdType: String
): Map[Q, Seq[I]]
def getLocalFileHandle(
directory: String
): AbstractFile
def getEmbeddingsSet[T](
directory: String,
idType: String
): Seq[EntityEmbedding[T]]
def loadKnnDirFileToMap[K, V](
directory: AbstractFile,
f: Array[String] => Seq[V],
converter: String => K
): Map[K, Seq[V]]
def accept(file: AbstractFile): Boolean =
file.getName != AbstractFile.SUCCESS_FILE_NAME
}).foreach
def getRandomQuerySet(
dimension: Int,
totalQueries: Int,
minValue: Float,
maxValue: Float
): Seq[EmbeddingVector]
def getKeyConverter[T](idType: String): String => T
def buildRemoteServiceQueryClient[T, P <: RuntimeParams, D <: Distance[D]](
destination: String,
clientId: String,
statsReceiver: StatsReceiver,
serviceIdentifier: ServiceIdentifier,
runtimeParamInjection: Injection[P, ServiceRuntimeParams],
distanceInjection: Injection[D, ServiceDistance],
indexIdInjection: Injection[T, Array[Byte]]
): Future[Queryable[T, P, D]]
def apply(request: NearestNeighborQuery): Future[NearestNeighborResult] =
client.query(request)
}
Future.value(
new ServiceClientQueryable[T, P, D](
service,
runtimeParamInjection,
distanceInjection,
indexIdInjection
)
)
}
// helper method to convert a line in KNN file output format into map
@VisibleForTesting
def addToMapFromKnnString[K, V](
line: String,
f: Array[String] => Seq[V],
map: mutable.HashMap[K, Seq[V]],
converter: String => K
): Unit
def printResults(
inMemoryBuildRecorder: InMemoryLoadTestBuildRecorder,
queryTimeConfigurations: Seq[QueryTimeConfiguration[_, _]]
): Seq[String]
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\BaseQueryIndexServer.scala
class provides most of the configuration needed for logging, stats, deciders etc.
*/
abstract class BaseQueryIndexServer extends ThriftServer with Mtls
def additionalModules: Seq[Module]
/**
* Override this method to add the controller to the thrift router. BaseQueryIndexServer takes
* care of most of the other configuration for you.
* @param router
*/
protected def addController(router: ThriftRouter): Unit
override protected final lazy val modules: Seq[Module] = Seq(
DeciderModule,
new MtlsThriftWebFormsModule[AnnQueryService.MethodPerEndpoint](this)
) ++ additionalModules
override protected final def configureThrift(router: ThriftRouter): Unit
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\Exceptions.scala
object RuntimeExceptionTransform extends ExceptionTransformer
def transform
def getStatName: PartialFunction[Exception, String]
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\FaissIndexPathProvider.scala
class FaissIndexPathProvider(
override val minIndexSizeBytes: Long,
override val maxIndexSizeBytes: Long,
override val statsReceiver: StatsReceiver)
extends BaseIndexPathProvider
def isValidIndex(dir: AbstractFile): Boolean
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\IndexPathProvider.scala
abstract class IndexPathProvider
def provideIndexPath(rootPath: AbstractFile, group: Boolean = false): Try[AbstractFile]
def provideIndexPathWithGroups(rootPath: AbstractFile): Try[Seq[AbstractFile]]
}
abstract class BaseIndexPathProvider extends IndexPathProvider
def accept(file: AbstractFile): Boolean
def findLatestTimeStampValidSuccessDirectory(
path: AbstractFile,
group: Boolean
): AbstractFile
def isValidIndex(index: AbstractFile): Boolean
override def provideIndexPath(
rootPath: AbstractFile,
group: Boolean = false
): Try[AbstractFile]
def provideIndexPathWithGroups(
rootPath: AbstractFile
): Try[Seq[AbstractFile]]
def accept(file: AbstractFile): Boolean =
file.isDirectory && file.hasSuccessFile
}).asScala.toSeq
}
}
}
case class ValidatedIndexPathProvider(
override val minIndexSizeBytes: Long,
override val maxIndexSizeBytes: Long,
override val statsReceiver: StatsReceiver)
extends BaseIndexPathProvider
def isValidIndex(dir: AbstractFile): Boolean
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\QueryableProvider.scala
trait QueryableProvider[T, P <: RuntimeParams, D <: Distance[D]]
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\QueryIndexThriftController.scala
class QueryIndexThriftController[T, P <: RuntimeParams, D <: Distance[D]] @Inject() (
statsReceiver: StatsReceiver,
queryable: Queryable[T, P, D],
runtimeParamInjection: Injection[P, ServiceRuntimeParams],
distanceInjection: Injection[D, ServiceDistance],
idInjection: Injection[T, Array[Byte]])
extends Controller(AnnQueryService)
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\QueryServerUtil.scala
object QueryServerUtil
abstract file (directory) size is within the defined limits.
* @param dir Hdfs/Local directory
* @param minIndexSizeBytes minimum size of file in bytes (Exclusive)
* @param maxIndexSizeBytes minimum size of file in bytes (Exclusive)
* @return true if file size within minIndexSizeBytes and maxIndexSizeBytes else false
*/
def isValidIndexDirSize(
dir: AbstractFile,
minIndexSizeBytes: Long,
maxIndexSizeBytes: Long
): Boolean
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\RefreshableQueryable.scala
class RefreshableQueryable[T, P <: RuntimeParams, D <: Distance[D]](
grouped: Boolean,
rootDir: AbstractFile,
queryableProvider: QueryableProvider[T, P, D],
indexPathProvider: IndexPathProvider,
statsReceiver: StatsReceiver,
updateInterval: Duration = 10.minutes)
extends QueryableGrouped[T, P, D]
def run(): Unit
def start(): Unit
def buildIndex(indexPath: AbstractFile): Queryable[T, P, D]
def innerLoad(): Unit
def computeRandomInitDelay(): Duration
def query(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P,
key: Option[String]
): Future[List[T]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P,
key: Option[String]
): Future[List[NeighborWithDistance[T, D]]]
def getGroupMapping(): Map[Option[String], Queryable[T, P, D]]
def query(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[T]]
def queryWithDistance(
embedding: EmbeddingVector,
numOfNeighbors: Int,
runtimeParams: P
): Future[List[NeighborWithDistance[T, D]]]
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\UnsafeQueryIndexServer.scala
class is used when you do not know the generic parameters of the Server at compile time.
* If you want compile time checks that your parameters are correct use QueryIndexServer instead.
* In particular, when you want to have these id, distance and the runtime params as cli options you
* should extend this class.
*/
abstract class UnsafeQueryIndexServer[R <: RuntimeParams] extends BaseQueryIndexServer
def addController(router: ThriftRouter): Unit
def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, R, D]
protected val runtimeInjection: Injection[R, ServiceRuntimeParams]
private[this] def queryIndexThriftController[
T,
D <: Distance[D]
]: QueryIndexThriftController[T, R, D]
def idInjection[T](): Injection[T, Array[Byte]]
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\throttling\AuroraCPUStatsReader.scala
class AuroraCPUStatsReader()
def throttledTimeNanos(): Option[Long] = cgroupsCpu.cpuStat.map
def cpuQuota: Double = cgroupsCpu.cfsPeriodMicros match
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\throttling\ThrottlingBasedQualityTask.scala
object ThrottlingBasedQualityTask
class ThrottlingBasedQualityTask(
override val statsReceiver: StatsReceiver,
// Parameters are taken from OverloadAdmissionController
instrument: ThrottlingInstrument = new WindowedThrottlingInstrument(SAMPLING_INTERVAL, 5,
new AuroraCPUStatsReader()))
extends Task
with Logging
def task(): Future[Unit]
def taskInterval: Duration = SAMPLING_INTERVAL
def discountParams[T <: RuntimeParams](params: T): T
def applyQualityFactor(param: Int) = math.max(1, math.ceil(param * qualityFactor).toInt)
params match
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\throttling\WindowedStats.scala
class WindowedStats(window: Int)
def add(v: Long): Unit
def avg: Double
def sum: Long
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\throttling\WindowedThrottlingInstrument.scala
trait ThrottlingInstrument
def sample(): Unit
def percentageOfTimeSpentThrottling(): Double
def disabled: Boolean
}
class WindowedThrottlingInstrument(
stepFrequency: Duration,
windowLengthInFrequencySteps: Int,
reader: AuroraCPUStatsReader)
extends ThrottlingInstrument
def disabled: Boolean = cpuQuota <= 0
def sample(): Unit = sampleThrottling() match
def sampleThrottling(): Option[Long] = reader.throttledTimeNanos().map
def percentageOfTimeSpentThrottling(): Double
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\warmup\Warmup.scala
trait Warmup extends Logging
def minSuccessfulTries: Int
protected def maxTries: Int
protected def randomQueryDimension: Int
protected def timeout: Duration
@tailrec
final protected def run(
iteration: Int = 0,
successes: Int = 0,
name: String,
f: => Future[_]
): Unit
.\ann\src\main\scala\com\twitter\ann\service\query_server\faiss\FaissQueryIndexServer.scala
object FaissQueryIndexServer extends FaissQueryableServer
class FaissQueryableServer extends UnsafeQueryIndexServer[FaissParams]
def queryableProvider[T, D <: Distance[D]]: QueryableProvider[T, FaissParams, D] =
new QueryableProvider[T, FaissParams, D]
def provideQueryable(
directory: AbstractFile
): Queryable[T, FaissParams, D]
def buildSimpleQueryable[T, D <: Distance[D]](
dir: AbstractFile
): Queryable[T, FaissParams, D]
def buildShardedQueryable[T, D <: Distance[D]](
dir: AbstractFile
): Queryable[T, FaissParams, D]
def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, FaissParams, D]
def warmup(): Unit =
if (warmup_enabled())
new FaissWarmup(unsafeQueryableMap, dimension()).warmup()
}
class FaissWarmup(faiss: Queryable[_, FaissParams, _], dimension: Int) extends Warmup
def minSuccessfulTries: Int = 100
protected def maxTries: Int = 1000
protected def timeout: Duration = 50.milliseconds
protected def randomQueryDimension: Int = dimension
def warmup(): Unit
.\ann\src\main\scala\com\twitter\ann\service\query_server\hnsw\HnswQueryIndexServer.scala
object HnswQueryIndexServer extends HnswQueryableServer
class HnswQueryableServer extends UnsafeQueryIndexServer[HnswParams]
def queryableProvider[T, D <: Distance[D]]: QueryableProvider[T, HnswParams, D] =
new QueryableProvider[T, HnswParams, D]
def provideQueryable(
dir: AbstractFile
): Queryable[T, HnswParams, D]
def buildQueryable[T, D <: Distance[D]](
dir: AbstractFile,
grouped: Boolean
): Queryable[T, HnswParams, D]
def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, HnswParams, D]
def warmup(): Unit =
if (warmup_enabled()) new HNSWWarmup(unsafeQueryableMap, dimension()).warmup()
}
class HNSWWarmup(hnsw: Queryable[_, HnswParams, _], dimension: Int) extends Warmup
def minSuccessfulTries: Int = 100
protected def maxTries: Int = 1000
protected def timeout: Duration = 50.milliseconds
protected def randomQueryDimension: Int = dimension
def warmup(): Unit
.\ann\src\main\scala\com\twitter\ann\util\IndexBuilderUtils.scala
object IndexBuilderUtils
def addToIndex[T](
appendable: Appendable[T, _, _],
embeddings: Seq[EntityEmbedding[T]],
concurrencyLevel: Int
): Future[Int]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\CrMixerHttpServerWarmupHandler.scala
class CrMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging
def handle(): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\CrMixerServer.scala
object CrMixerServerMain extends CrMixerServer
class CrMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls
def defaultMethodAccess: MethodOptions.Access
def configureThrift(router: ThriftRouter): Unit
def warmup(): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\CrMixerThriftServerWarmupHandler.scala
class CrMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup)
extends Handler
with Logging
def handle(): Unit
def warmupQuery(userId: Long): st.CrMixerTweetRequest
def assertWarmupResponse(
result: Try[Response[st.CrMixer.GetTweetRecommendations.SuccessType]]
): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\AdsBlender.scala
class AdsBlender @Inject() (globalStats: StatsReceiver)
def blend(
inputCandidates: Seq[Seq[InitialAdsCandidate]],
): Future[Seq[BlendedAdsCandidate]]
def buildBlendedAdsCandidate(
inputCandidates: Seq[Seq[InitialAdsCandidate]],
interleavedCandidates: Seq[InitialAdsCandidate]
): Seq[BlendedAdsCandidate]
def buildCandidateToCGInfosMap(
candidateSeq: Seq[Seq[InitialAdsCandidate]],
): Map[TweetId, Seq[CandidateGenerationInfo]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\BlendedCandidatesBuilder.scala
object BlendedCandidatesBuilder
def build(
inputCandidates: Seq[Seq[InitialCandidate]],
interleavedCandidates: Seq[InitialCandidate]
): Seq[BlendedCandidate]
def buildCandidateToCGInfosMap(
candidateSeq: Seq[Seq[InitialCandidate]],
): Map[TweetId, Seq[CandidateGenerationInfo]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\ContentSignalBlender.scala
class ContentSignalBlender @Inject() (globalStats: StatsReceiver)
def blend(
params: Params,
inputCandidates: Seq[Seq[InitialCandidate]],
): Future[Seq[BlendedCandidate]]
def removeDuplicates(candidates: Seq[InitialCandidate]): Seq[InitialCandidate]
def groupByEngineTypeOrFirstContribEngine(
candidates: Seq[InitialCandidate]
): Map[SimilarityEngineType, Seq[InitialCandidate]]
def flattenAndGroupByEngineTypeOrFirstContribEngine(
candidates: Seq[Seq[InitialCandidate]]
): Seq[Seq[InitialCandidate]]
def standardizeAndSortByScore(
candidates: Seq[Seq[InitialCandidate]]
): Seq[InitialCandidate]
def getSnowflakeTimeStamp(tweetId: Long): Time
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\CountWeightedInterleaveBlender.scala
class CountWeightedInterleaveBlender @Inject() (globalStats: StatsReceiver)
def blend(
query: CrCandidateGeneratorQuery,
inputCandidates: Seq[Seq[InitialCandidate]]
): Future[Seq[BlendedCandidate]]
def countWeightedInterleave(
query: WeightedBlenderQuery,
inputCandidates: Seq[Seq[InitialCandidate]],
): Future[Seq[BlendedCandidate]]
object CountWeightedInterleaveBlender
class WeightedBlenderQuery(
rankerWeightShrinkage: Double,
maxWeightAdjustments: Int)
def paramToQuery(params: Params): WeightedBlenderQuery
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\InterleaveBlender.scala
class InterleaveBlender @Inject() (globalStats: StatsReceiver)
def blend(
inputCandidates: Seq[Seq[InitialCandidate]],
): Future[Seq[BlendedCandidate]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\SourceTypeBackFillBlender.scala
class SourceTypeBackFillBlender @Inject() (globalStats: StatsReceiver)
def blend(
params: Params,
inputCandidates: Seq[Seq[InitialCandidate]],
): Future[Seq[BlendedCandidate]]
object ImplicitSignalBackFillBlender
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\SwitchBlender.scala
class SwitchBlender @Inject() (
defaultBlender: InterleaveBlender,
sourceTypeBackFillBlender: SourceTypeBackFillBlender,
adsBlender: AdsBlender,
contentSignalBlender: ContentSignalBlender,
globalStats: StatsReceiver)
def blend(
params: Params,
userState: UserState,
inputCandidates: Seq[Seq[InitialCandidate]],
): Future[Seq[BlendedCandidate]]
object SwitchBlender
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\AdsCandidateGenerator.scala
class AdsCandidateGenerator @Inject() (
ussSourceSignalFetcher: UssSourceSignalFetcher,
realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher,
adsCandidateSourceRouter: AdsCandidateSourcesRouter,
adsBlender: AdsBlender,
scribeLogger: AdsRecommendationsScribeLogger,
globalStats: StatsReceiver)
def get(query: AdsCandidateGeneratorQuery): Future[Seq[RankedAdsCandidate]]
def fetchSources(
query: AdsCandidateGeneratorQuery
): Future[Set[SourceInfo]]
def fetchCandidates(
query: AdsCandidateGeneratorQuery,
sourceSignals: Set[SourceInfo],
realGraphSeeds: Map[UserId, Double]
): Future[Seq[Seq[InitialAdsCandidate]]]
def fetchSeeds(
query: AdsCandidateGeneratorQuery
): Future[Map[UserId, Double]]
def interleave(
candidates: Seq[Seq[InitialAdsCandidate]]
): Future[Seq[BlendedAdsCandidate]]
def rank(
candidates: Seq[BlendedAdsCandidate],
enableScoreBoost: Boolean,
scoreBoostFactor: Double,
statsReceiver: StatsReceiver,
): Future[Seq[RankedAdsCandidate]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\AdsCandidateSourcesRouter.scala
class AdsCandidateSourcesRouter @Inject() (
activePromotedTweetStore: ReadableStore[TweetId, Seq[LineItemInfo]],
decider: CrMixerDecider,
@Named(ModuleNames.SimClustersANNSimilarityEngine) simClustersANNSimilarityEngine: StandardSimilarityEngine[
Query,
TweetWithScore
],
@Named(ModuleNames.TweetBasedUserAdGraphSimilarityEngine)
tweetBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[
TweetBasedUserAdGraphSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.ConsumersBasedUserAdGraphSimilarityEngine)
consumersBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[
ConsumersBasedUserAdGraphSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.ProducerBasedUserAdGraphSimilarityEngine)
producerBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[
ProducerBasedUserAdGraphSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine)
tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine,
@Named(ModuleNames.ConsumerEmbeddingBasedTwHINANNSimilarityEngine) consumerTwHINANNSimilarityEngine: HnswANNSimilarityEngine,
@Named(ModuleNames.ConsumerBasedWalsSimilarityEngine)
consumerBasedWalsSimilarityEngine: StandardSimilarityEngine[
ConsumerBasedWalsSimilarityEngine.Query,
TweetWithScore
],
globalStats: StatsReceiver,
)
def fetchCandidates(
requestUserId: UserId,
sourceSignals: Set[SourceInfo],
realGraphSeeds: Map[UserId, Double],
params: configapi.Params
): Future[Seq[Seq[InitialAdsCandidate]]]
def convertToInitialCandidates(
candidates: Seq[TweetWithCandidateGenerationInfo],
stats: StatsReceiver
): Future[Seq[InitialAdsCandidate]]
def getSimClustersANNCandidates(
requestUserId: UserId,
sourceInfo: Option[SourceInfo],
params: configapi.Params,
configId: String,
minScore: Double
)
def getProducerBasedUserAdGraphCandidates(
sourceInfo: Option[SourceInfo],
params: configapi.Params
)
def getTweetBasedUserAdGraphCandidates(
sourceInfo: Option[SourceInfo],
params: configapi.Params
)
def getRealGraphConsumersBasedUserAdGraphCandidates(
realGraphSeeds: Map[UserId, Double],
params: configapi.Params
)
def getTwHINAdsCandidates(
similarityEngine: HnswANNSimilarityEngine,
similarityEngineType: SimilarityEngineType,
requestUserId: UserId,
sourceInfo: Option[SourceInfo], // if none, then it's consumer-based similarity engine
model: String
): Future[Seq[TweetWithCandidateGenerationInfo]]
def getConsumerBasedWalsCandidates(
sourceSignals: Set[SourceInfo],
params: configapi.Params
): Future[Seq[TweetWithCandidateGenerationInfo]]
object AdsCandidateSourcesRouter
def getSimClustersANNEmbeddingType(
sourceInfo: SourceInfo
): EmbeddingType
def buildHnswANNQuery(internalId: InternalId, modelId: String): HnswANNEngineQuery
def getConsumerBasedWalsSourceInfo(
sourceSignals: Set[SourceInfo]
): Set[SourceInfo]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\CandidateSourcesRouter.scala
class CandidateSourcesRouter @Inject() (
customizedRetrievalCandidateGeneration: CustomizedRetrievalCandidateGeneration,
simClustersInterestedInCandidateGeneration: SimClustersInterestedInCandidateGeneration,
@Named(ModuleNames.TweetBasedUnifiedSimilarityEngine)
tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[
TweetBasedUnifiedSimilarityEngine.Query,
TweetWithCandidateGenerationInfo
],
@Named(ModuleNames.ProducerBasedUnifiedSimilarityEngine)
producerBasedUnifiedSimilarityEngine: StandardSimilarityEngine[
ProducerBasedUnifiedSimilarityEngine.Query,
TweetWithCandidateGenerationInfo
],
@Named(ModuleNames.ConsumerEmbeddingBasedTripSimilarityEngine)
consumerEmbeddingBasedTripSimilarityEngine: StandardSimilarityEngine[
TripEngineQuery,
TripTweetWithScore
],
@Named(ModuleNames.ConsumerEmbeddingBasedTwHINANNSimilarityEngine)
consumerBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine,
@Named(ModuleNames.ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine)
consumerBasedTwoTowerSimilarityEngine: HnswANNSimilarityEngine,
@Named(ModuleNames.ConsumersBasedUserVideoGraphSimilarityEngine)
consumersBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[
ConsumersBasedUserVideoGraphSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.UserTweetEntityGraphSimilarityEngine) userTweetEntityGraphSimilarityEngine: StandardSimilarityEngine[
UserTweetEntityGraphSimilarityEngine.Query,
TweetWithScoreAndSocialProof
],
@Named(ModuleNames.ConsumerBasedWalsSimilarityEngine)
consumerBasedWalsSimilarityEngine: StandardSimilarityEngine[
ConsumerBasedWalsSimilarityEngine.Query,
TweetWithScore
],
tweetInfoStore: ReadableStore[TweetId, TweetInfo],
globalStats: StatsReceiver,
)
def fetchCandidates(
requestUserId: UserId,
sourceSignals: Set[SourceInfo],
sourceGraphs: Map[String, Option[GraphSourceInfo]],
params: configapi.Params,
): Future[Seq[Seq[InitialCandidate]]]
def getGraphBasedCandidates[QueryType](
params: configapi.Params,
query: EngineQuery[QueryType],
engine: StandardSimilarityEngine[QueryType, TweetWithScore],
toSimilarityEngineInfo: Double => SimilarityEngineInfo,
graphSourceInfoOpt: Option[GraphSourceInfo] = None
): Future[Seq[InitialCandidate]]
def getCandidates[QueryType](
sourceSignals: Set[SourceInfo],
params: configapi.Params,
fromParams: (SourceInfo, configapi.Params) => QueryType,
getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]]
): Future[Seq[Seq[InitialCandidate]]]
def getConsumerBasedWalsCandidates(
sourceSignals: Set[SourceInfo],
params: configapi.Params
): Future[Seq[InitialCandidate]]
def getSimClustersTripCandidates(
params: configapi.Params,
query: TripEngineQuery,
engine: StandardSimilarityEngine[
TripEngineQuery,
TripTweetWithScore
],
): Future[Seq[InitialCandidate]]
def getHnswCandidates(
params: configapi.Params,
query: HnswANNEngineQuery,
engine: HnswANNSimilarityEngine,
): Future[Seq[InitialCandidate]]
def getCandidatesPerSimilarityEngineModel[QueryType](
requestUserId: UserId,
params: configapi.Params,
fromParams: (InternalId, configapi.Params) => QueryType,
getFunc: QueryType => Future[
Option[Seq[Seq[TweetWithCandidateGenerationInfo]]]
]
): Future[Seq[Seq[InitialCandidate]]]
def convertToInitialCandidates(
candidates: Seq[TweetWithCandidateGenerationInfo],
): Future[Seq[InitialCandidate]]
object CandidateSourcesRouter
def getGraphSourceInfoBySourceType(
sourceTypeStr: String,
sourceGraphs: Map[String, Option[GraphSourceInfo]]
): Option[GraphSourceInfo]
def getTweetBasedSourceInfo(
sourceSignals: Set[SourceInfo]
): Set[SourceInfo]
def getProducerBasedSourceInfo(
sourceSignals: Set[SourceInfo]
): Set[SourceInfo]
def getConsumerBasedWalsSourceInfo(
sourceSignals: Set[SourceInfo]
): Set[SourceInfo]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\CrCandidateGenerator.scala
class CrCandidateGenerator @Inject() (
sourceInfoRouter: SourceInfoRouter,
candidateSourceRouter: CandidateSourcesRouter,
switchBlender: SwitchBlender,
preRankFilterRunner: PreRankFilterRunner,
postRankFilterRunner: PostRankFilterRunner,
switchRanker: SwitchRanker,
crMixerScribeLogger: CrMixerScribeLogger,
timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver)
def get(query: CrCandidateGeneratorQuery): Future[Seq[RankedCandidate]]
def fetchSources(
query: CrCandidateGeneratorQuery
): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])]
def filterSourceInfo(
positiveSignals: Set[SourceInfo],
negativeSignals: Set[SourceInfo]
): Set[SourceInfo]
def fetchCandidates(
query: CrCandidateGeneratorQuery,
sourceSignals: Set[SourceInfo],
sourceGraphs: Map[String, Option[GraphSourceInfo]]
): Future[Seq[Seq[InitialCandidate]]]
def preRankFilter(
query: CrCandidateGeneratorQuery,
candidates: Seq[Seq[InitialCandidate]]
): Future[Seq[Seq[InitialCandidate]]]
def postRankFilter(
query: CrCandidateGeneratorQuery,
candidates: Seq[RankedCandidate]
): Future[Seq[RankedCandidate]]
def interleave(
query: CrCandidateGeneratorQuery,
candidates: Seq[Seq[InitialCandidate]]
): Future[Seq[BlendedCandidate]]
def rank(
query: CrCandidateGeneratorQuery,
candidates: Seq[BlendedCandidate],
): Future[Seq[RankedCandidate]]
def trackResultStats(
stats: StatsReceiver
)(
fn: => Future[Seq[RankedCandidate]]
): Future[Seq[RankedCandidate]]
def trackReasonChosenSourceTypeStats(
candidates: Seq[RankedCandidate],
stats: StatsReceiver
): Unit
def trackReasonChosenSimilarityEngineStats(
candidates: Seq[RankedCandidate],
stats: StatsReceiver
): Unit
def trackPotentialReasonsSourceTypeStats(
candidates: Seq[RankedCandidate],
stats: StatsReceiver
): Unit
def trackPotentialReasonsSimilarityEngineStats(
candidates: Seq[RankedCandidate],
stats: StatsReceiver
): Unit
def trackBlueVerifiedTweetStats(
candidates: Seq[RankedCandidate],
statsReceiver: StatsReceiver
): Unit
def trackTopKStats(
k: Int,
tweetCandidates: Seq[RankedCandidate],
isQueryK: Boolean,
statsReceiver: StatsReceiver
): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\CustomizedRetrievalCandidateGeneration.scala
class shall not be thought of as a
* Unified Similarity Engine. It is a CG that calls multiple singular Similarity Engines.
*/
@Singleton
case class CustomizedRetrievalCandidateGeneration @Inject() (
@Named(ModuleNames.TwhinCollabFilterSimilarityEngine)
twhinCollabFilterSimilarityEngine: LookupSimilarityEngine[
TwhinCollabFilterSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.DiffusionBasedSimilarityEngine)
diffusionBasedSimilarityEngine: LookupSimilarityEngine[
DiffusionBasedSimilarityEngine.Query,
TweetWithScore
],
statsReceiver: StatsReceiver)
extends CandidateSource[
Query,
Seq[TweetWithCandidateGenerationInfo]
]
def name: String = this.getClass.getSimpleName
private val stats = statsReceiver.scope(name)
private val fetchCandidatesStat = stats.scope("fetchCandidates")
/**
* For each Similarity Engine Model, return a list of tweet candidates
*/
override def get(
query: Query
): Future[Option[Seq[Seq[TweetWithCandidateGenerationInfo]]]]
def tweetAgeFilter(
candidates: Seq[TweetWithScore],
maxTweetAgeHours: Duration
): Seq[TweetWithScore]
def ageFilterWithStats(
offlineInterestedInCandidates: Seq[TweetWithScore],
maxTweetAgeHours: Duration,
scopedStatsReceiver: StatsReceiver
): Seq[TweetWithScore]
def getTwhinCollabCandidatesWithCGInfo(
tweetCandidates: Option[Seq[TweetWithScore]],
maxCandidateNumPerSourceKey: Int,
twhinCollabFilterQuery: LookupEngineQuery[
TwhinCollabFilterSimilarityEngine.Query
],
): Seq[TweetWithCandidateGenerationInfo]
def getDiffusionBasedCandidatesWithCGInfo(
tweetCandidates: Option[Seq[TweetWithScore]],
maxCandidateNumPerSourceKey: Int,
diffusionBasedSimilarityEngineQuery: LookupEngineQuery[
DiffusionBasedSimilarityEngine.Query
],
): Seq[TweetWithCandidateGenerationInfo]
object CustomizedRetrievalCandidateGeneration
class Query(
internalId: InternalId,
maxCandidateNumPerSourceKey: Int,
maxTweetAgeHours: Duration,
// twhinCollabFilter
enableTwhinCollabFilter: Boolean,
twhinCollabFilterFollowQuery: LookupEngineQuery[
TwhinCollabFilterSimilarityEngine.Query
],
twhinCollabFilterEngagementQuery: LookupEngineQuery[
TwhinCollabFilterSimilarityEngine.Query
],
// twhinMultiCluster
enableTwhinMultiCluster: Boolean,
twhinMultiClusterFollowQuery: LookupEngineQuery[
TwhinCollabFilterSimilarityEngine.Query
],
twhinMultiClusterEngagementQuery: LookupEngineQuery[
TwhinCollabFilterSimilarityEngine.Query
],
enableRetweetBasedDiffusion: Boolean,
diffusionBasedSimilarityEngineQuery: LookupEngineQuery[
DiffusionBasedSimilarityEngine.Query
],
)
def fromParams(
internalId: InternalId,
params: configapi.Params
): Query
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\FrsTweetCandidateGenerator.scala
class FrsTweetCandidateGenerator @Inject() (
@Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]],
frsBasedSimilarityEngine: EarlybirdSimilarityEngineRouter,
tweetInfoStore: ReadableStore[TweetId, TweetInfo],
timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver)
def get(
frsTweetCandidateGeneratorQuery: FrsTweetCandidateGeneratorQuery
): Future[Seq[FrsTweet]]
def fetchSeeds(
userId: UserId,
userDenyList: Set[UserId],
languageCodeOpt: Option[String],
countryCodeOpt: Option[String],
params: Params
): Future[Option[Map[UserId, FrsQueryResult]]]
def fetchCandidates(
searcherUserId: UserId,
seedAuthors: Seq[UserId],
impressedTweetList: Set[TweetId],
frsUserToScores: Map[UserId, Double],
params: Params
): Future[Option[Seq[TweetWithAuthor]]]
def filterCandidates(
candidates: Option[Seq[TweetWithAuthor]],
params: Params
): Future[Option[Seq[TweetWithAuthor]]]
def hydrateCandidates(
frsAuthorWithScores: Option[Map[UserId, FrsQueryResult]],
candidates: Option[Seq[TweetWithAuthor]]
): Future[Option[Seq[FrsTweet]]]
object FrsTweetCandidateGenerator
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\RelatedTweetCandidateGenerator.scala
class RelatedTweetCandidateGenerator @Inject() (
@Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[
TweetBasedUnifiedSimilarityEngine.Query,
TweetWithCandidateGenerationInfo
],
@Named(ModuleNames.ProducerBasedUnifiedSimilarityEngine) producerBasedUnifiedSimilarityEngine: StandardSimilarityEngine[
ProducerBasedUnifiedSimilarityEngine.Query,
TweetWithCandidateGenerationInfo
],
preRankFilterRunner: PreRankFilterRunner,
relatedTweetScribeLogger: RelatedTweetScribeLogger,
tweetInfoStore: ReadableStore[TweetId, TweetInfo],
globalStats: StatsReceiver)
def get(
query: RelatedTweetCandidateGeneratorQuery
): Future[Seq[InitialCandidate]]
def fetchCandidates(
query: RelatedTweetCandidateGeneratorQuery
): Future[Seq[Seq[InitialCandidate]]]
def getCandidatesFromSimilarityEngine[QueryType](
query: RelatedTweetCandidateGeneratorQuery,
fromParamsForRelatedTweet: (InternalId, configapi.Params) => QueryType,
getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]]
): Future[Seq[Seq[InitialCandidate]]]
def preRankFilter(
query: RelatedTweetCandidateGeneratorQuery,
candidates: Seq[Seq[InitialCandidate]]
): Future[Seq[Seq[InitialCandidate]]]
def convertToInitialCandidates(
candidates: Seq[TweetWithCandidateGenerationInfo],
): Future[Seq[InitialCandidate]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\RelatedVideoTweetCandidateGenerator.scala
class RelatedVideoTweetCandidateGenerator @Inject() (
@Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[
TweetBasedUnifiedSimilarityEngine.Query,
TweetWithCandidateGenerationInfo
],
preRankFilterRunner: PreRankFilterRunner,
tweetInfoStore: ReadableStore[TweetId, TweetInfo],
globalStats: StatsReceiver)
def get(
query: RelatedVideoTweetCandidateGeneratorQuery
): Future[Seq[InitialCandidate]]
def fetchCandidates(
query: RelatedVideoTweetCandidateGeneratorQuery
): Future[Seq[Seq[InitialCandidate]]]
def getCandidatesFromSimilarityEngine[QueryType](
query: RelatedVideoTweetCandidateGeneratorQuery,
fromParamsForRelatedVideoTweet: (InternalId, configapi.Params) => QueryType,
getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]]
): Future[Seq[Seq[InitialCandidate]]]
def preRankFilter(
query: RelatedVideoTweetCandidateGeneratorQuery,
candidates: Seq[Seq[InitialCandidate]]
): Future[Seq[Seq[InitialCandidate]]]
def convertToInitialCandidates(
candidates: Seq[TweetWithCandidateGenerationInfo],
): Future[Seq[InitialCandidate]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\SimClustersInterestedInCandidateGeneration.scala
class moving forward.
*
* After the abstraction improvement (apply SimilarityEngine trait)
* these CG will be subjected to change.
*/
@Singleton
case class SimClustersInterestedInCandidateGeneration @Inject() (
@Named(ModuleNames.SimClustersANNSimilarityEngine)
simClustersANNSimilarityEngine: StandardSimilarityEngine[
SimClustersANNSimilarityEngine.Query,
TweetWithScore
],
statsReceiver: StatsReceiver)
extends CandidateSource[
SimClustersInterestedInCandidateGeneration.Query,
Seq[TweetWithCandidateGenerationInfo]
]
def name: String = this.getClass.getSimpleName
private val stats = statsReceiver.scope(name)
private val fetchCandidatesStat = stats.scope("fetchCandidates")
override def get(
query: SimClustersInterestedInCandidateGeneration.Query
): Future[Option[Seq[Seq[TweetWithCandidateGenerationInfo]]]]
def simClustersCandidateMinScoreFilter(
simClustersAnnCandidates: Seq[TweetWithScore],
simClustersInterestedInMinScore: Double,
simClustersANNConfigId: String
): Seq[TweetWithScore]
def getInterestedInCandidateResult(
simClustersANNSimilarityEngine: StandardSimilarityEngine[
SimClustersANNSimilarityEngine.Query,
TweetWithScore
],
simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
simClustersInterestedInMinScore: Double,
): Future[Option[Seq[TweetWithCandidateGenerationInfo]]]
object SimClustersInterestedInCandidateGeneration
class Query(
internalId: InternalId,
enableUserInterestedIn: Boolean,
enableUserNextInterestedIn: Boolean,
enableAddressBookNextInterestedIn: Boolean,
enableProdSimClustersANNSimilarityEngine: Boolean,
enableExperimentalSimClustersANNSimilarityEngine: Boolean,
enableSimClustersANN1SimilarityEngine: Boolean,
enableSimClustersANN2SimilarityEngine: Boolean,
enableSimClustersANN3SimilarityEngine: Boolean,
enableSimClustersANN5SimilarityEngine: Boolean,
enableSimClustersANN4SimilarityEngine: Boolean,
simClustersInterestedInMinScore: Double,
simClustersNextInterestedInMinScore: Double,
simClustersAddressBookInterestedInMinScore: Double,
interestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
nextInterestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
addressbookInterestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
interestedInExperimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
nextInterestedInExperimentalSimClustersANNQuery: EngineQuery[
SimClustersANNSimilarityEngine.Query
],
addressbookInterestedInExperimentalSimClustersANNQuery: EngineQuery[
SimClustersANNSimilarityEngine.Query
],
interestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
nextInterestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
addressbookInterestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
interestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
nextInterestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
addressbookInterestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
interestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
nextInterestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
addressbookInterestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
interestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
nextInterestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
addressbookInterestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
interestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
nextInterestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
addressbookInterestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
)
def fromParams(
internalId: InternalId,
params: configapi.Params,
): Query
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\TopicTweetCandidateGenerator.scala
class TopicTweetCandidateGenerator @Inject() (
certoTopicTweetSimilarityEngine: CertoTopicTweetSimilarityEngine,
skitTopicTweetSimilarityEngine: SkitTopicTweetSimilarityEngine,
skitHighPrecisionTopicTweetSimilarityEngine: SkitHighPrecisionTopicTweetSimilarityEngine,
tweetInfoStore: ReadableStore[TweetId, TweetInfo],
timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver)
def get(
query: TopicTweetCandidateGeneratorQuery
): Future[Map[Long, Seq[TopicTweet]]]
def fetchCandidates(
query: TopicTweetCandidateGeneratorQuery
): Future[Map[TopicId, Option[Seq[TopicTweetWithScore]]]]
def convertToInitialCandidates(
candidatesMap: Map[TopicId, Option[Seq[TopicTweetWithScore]]]
): Future[Map[TopicId, Seq[InitialCandidate]]]
def filterCandidates(
topicTweetMap: Map[TopicId, Seq[InitialCandidate]],
maxTweetAge: Duration,
isVideoOnly: Boolean,
excludeTweetIds: Set[TweetId]
): Future[Map[TopicId, Seq[InitialCandidate]]]
def rankCandidates(
tweetCandidatesMap: Map[TopicId, Seq[InitialCandidate]]
): Map[TopicId, Seq[InitialCandidate]]
def hydrateCandidates(
topicCandidatesMap: Map[TopicId, Seq[InitialCandidate]]
): Map[Long, Seq[TopicTweet]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\UtegTweetCandidateGenerator.scala
class UtegTweetCandidateGenerator @Inject() (
@Named(ModuleNames.UserTweetEntityGraphSimilarityEngine) userTweetEntityGraphSimilarityEngine: StandardSimilarityEngine[
UserTweetEntityGraphSimilarityEngine.Query,
TweetWithScoreAndSocialProof
],
utegTweetScribeLogger: UtegTweetScribeLogger,
tweetInfoStore: ReadableStore[TweetId, TweetInfo],
realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher,
utegFilterRunner: UtegFilterRunner,
globalStats: StatsReceiver)
def get(
query: UtegTweetCandidateGeneratorQuery
): Future[Seq[TweetWithScoreAndSocialProof]]
def utegFilter(
query: UtegTweetCandidateGeneratorQuery,
candidates: Seq[InitialCandidate]
): Future[Seq[InitialCandidate]]
def fetchSeeds(
query: UtegTweetCandidateGeneratorQuery
): Future[Map[UserId, Double]]
def rankCandidates(
query: UtegTweetCandidateGeneratorQuery,
filteredCandidates: Seq[InitialCandidate],
): Future[Seq[RankedCandidate]]
def fetchCandidates(
query: UtegTweetCandidateGeneratorQuery,
realGraphSeeds: Map[UserId, Double],
): Future[Seq[TweetWithScoreAndSocialProof]]
def convertToInitialCandidates(
candidates: Seq[TweetWithScoreAndSocialProof],
): Future[Seq[InitialCandidate]]
def convertToTweets(
candidates: Seq[RankedCandidate],
tweetMap: Map[TweetId, TweetWithScoreAndSocialProof]
): Seq[TweetWithScoreAndSocialProof]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\config\SimClustersANNConfig.scala
class SimClustersANNConfig(
maxNumResults: Int,
minScore: Double,
candidateEmbeddingType: EmbeddingType,
maxTopTweetsPerCluster: Int,
maxScanClusters: Int,
maxTweetCandidateAge: Duration,
minTweetCandidateAge: Duration,
annAlgorithm: ScoringAlgorithm)
object SimClustersANNConfig
def getConfig(
embeddingType: String,
modelVersion: String,
id: String
): SimClustersANNConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\controller\CrMixerThriftController.scala
class CrMixerThriftController @Inject() (
crCandidateGenerator: CrCandidateGenerator,
relatedTweetCandidateGenerator: RelatedTweetCandidateGenerator,
relatedVideoTweetCandidateGenerator: RelatedVideoTweetCandidateGenerator,
utegTweetCandidateGenerator: UtegTweetCandidateGenerator,
frsTweetCandidateGenerator: FrsTweetCandidateGenerator,
topicTweetCandidateGenerator: TopicTweetCandidateGenerator,
crMixerScribeLogger: CrMixerScribeLogger,
relatedTweetScribeLogger: RelatedTweetScribeLogger,
utegTweetScribeLogger: UtegTweetScribeLogger,
adsRecommendationsScribeLogger: AdsRecommendationsScribeLogger,
adsCandidateGenerator: AdsCandidateGenerator,
decider: CrMixerDecider,
paramsBuilder: ParamsBuilder,
endpointLoadShedder: EndpointLoadShedder,
signalTimestampStatsUtil: SignalTimestampStatsUtil,
tweetRecommendationResultsStore: ReadableWritableStore[UserId, CrMixerTweetResponse],
userStateStore: ReadableStore[UserId, UserState],
statsReceiver: StatsReceiver)
extends Controller(t.CrMixer)
def logErrMessage(endpoint: String, e: Throwable): Unit
def generateRequestUUID(): Long
def getRelatedTweets(
endpointName: String,
request: RelatedTweetRequest
): Future[RelatedTweetResponse]
def getRelatedVideoTweets(
endpointName: String,
request: RelatedVideoTweetRequest
): Future[RelatedVideoTweetResponse]
def buildCrCandidateGeneratorQuery(
thriftRequest: CrMixerTweetRequest,
requestUUID: Long,
userId: Long
): Future[CrCandidateGeneratorQuery]
def buildRelatedTweetQuery(
thriftRequest: RelatedTweetRequest,
requestUUID: Long
): Future[RelatedTweetCandidateGeneratorQuery]
def buildAdsCandidateGeneratorQuery(
thriftRequest: AdsRequest
): Future[AdsCandidateGeneratorQuery]
def buildRelatedVideoTweetQuery(
thriftRequest: RelatedVideoTweetRequest,
requestUUID: Long
): Future[RelatedVideoTweetCandidateGeneratorQuery]
def buildUtegTweetQuery(
thriftRequest: UtegTweetRequest,
requestUUID: Long
): Future[UtegTweetCandidateGeneratorQuery]
def buildTopicTweetQuery(
thriftRequest: TopicTweetRequest,
requestUUID: Long
): TopicTweetCandidateGeneratorQuery
def buildFrsBasedTweetQuery(
thriftRequest: FrsTweetRequest,
requestUUID: Long
): Future[FrsTweetCandidateGeneratorQuery]
def buildThriftResponse(
candidates: Seq[RankedCandidate]
): CrMixerTweetResponse
def scribeTweetScoreFunnelSeries(
candidates: Seq[RankedCandidate]
): Seq[RankedCandidate]
def buildRelatedTweetResponse(candidates: Seq[InitialCandidate]): RelatedTweetResponse
def buildRelatedVideoTweetResponse(
candidates: Seq[InitialCandidate]
): RelatedVideoTweetResponse
def buildUtegTweetResponse(
candidates: Seq[TweetWithScoreAndSocialProof]
): UtegTweetResponse
def buildAdsResponse(
candidates: Seq[RankedAdsCandidate]
): AdsResponse
def cacheTweetRecommendationResults(
request: CrMixerTweetRequest,
response: Future[CrMixerTweetResponse]
): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\featureswitch\CrMixerLoggingABDecider.scala
class to be used the [[SetImpressedBucketsLocalContextFilter]] must be applied
* at the beginning of the request, to initialize a concurrent map used to store impressed buckets.
*
* Whenever we get an a/b impression, the bucket information is logged to the concurrent hashmap.
*/
case class CrMixerLoggingABDecider(
loggingAbDecider: LoggingABDecider,
statsReceiver: StatsReceiver)
extends LoggingABDecider
def impression(
experimentName: String,
recipient: Recipient
): Option[Bucket]
def track(
experimentName: String,
eventName: String,
recipient: Recipient
): Unit
def bucket(
experimentName: String,
recipient: Recipient
): Option[Bucket]
def experiments: Seq[String] = loggingAbDecider.experiments
override def experiment(experimentName: String) =
loggingAbDecider.experiment(experimentName)
}
object CrMixerImpressedBuckets
def getAllImpressedBuckets: Option[List[Bucket]]
def recordImpressedBucket(bucket: Bucket)
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\featureswitch\ParamsBuilder.scala
object for building [[Params]] to override */
@Singleton
class ParamsBuilder @Inject() (
globalStats: StatsReceiver,
abDecider: LoggingABDecider,
featureContextBuilder: FeatureContextBuilder,
config: Config)
def buildFromClientContext(
clientContext: ClientContext,
product: t.Product,
userState: UserState,
userRoleOverride: Option[Set[String]] = None,
featureOverrides: Map[String, FeatureValue] = Map.empty,
): Params
def buildFeatureSwitchRecipientWithGuestId(
clientContext: ClientContext,
product: t.Product,
userState: UserState
): FeatureSwitchRecipient
def buildFeatureSwitchRecipient(
userId: Long,
userRolesOverride: Option[Set[String]],
clientContext: ClientContext,
product: t.Product,
userState: UserState
): FeatureSwitchRecipient
object ParamsBuilder
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\featureswitch\SetImpressedBucketsLocalContextFilter.scala
class SetImpressedBucketsLocalContextFilter @Inject() () extends Filter.TypeAgnostic
def toFilter[Req, Rep]: Filter[Req, Rep, Req, Rep] =
(request: Req, service: Service[Req, Rep]) =>
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\FilterBase.scala
trait FilterBase
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\ImpressedTweetlistFilter.scala
class ImpressedTweetlistFilter() extends FilterBase
def filter(
candidates: Seq[Seq[InitialCandidate]],
config: FilterConfig
): Future[Seq[Seq[InitialCandidate]]]
def requestToConfig[CGQueryType <: CandidateGeneratorQuery](
request: CGQueryType
): FilterConfig
object ImpressedTweetlistFilter
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\InNetworkFilter.scala
class InNetworkFilter @Inject() (
@Named(ModuleNames.RealGraphInStore) realGraphStoreMh: ReadableStore[UserId, CandidateSeq],
globalStats: StatsReceiver)
extends FilterBase
def filter(
candidates: Seq[Seq[InitialCandidate]],
filterConfig: FilterConfig,
): Future[Seq[Seq[InitialCandidate]]]
def filterCandidates(
candidates: Seq[Seq[InitialCandidate]],
filterConfig: FilterConfig,
): Future[Seq[Seq[InitialCandidate]]]
def requestToConfig[CGQueryType <: CandidateGeneratorQuery](
request: CGQueryType
): FilterConfig
object InNetworkFilter
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\PostRankFilterRunner.scala
class PostRankFilterRunner @Inject() (
globalStats: StatsReceiver)
def run(
query: CrCandidateGeneratorQuery,
candidates: Seq[RankedCandidate]
): Future[Seq[RankedCandidate]]
def removeBadRecentNotificationCandidates(
candidates: Seq[RankedCandidate]
): Seq[RankedCandidate]
def isBadQualityRecentNotificationCandidate(candidate: RankedCandidate): Boolean
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\PreRankFilterRunner.scala
class PreRankFilterRunner @Inject() (
impressedTweetListFilter: ImpressedTweetlistFilter,
tweetAgeFilter: TweetAgeFilter,
videoTweetFilter: VideoTweetFilter,
tweetReplyFilter: ReplyFilter,
globalStats: StatsReceiver)
def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery](
request: CGQueryType,
candidates: Seq[Seq[InitialCandidate]],
): Future[Seq[Seq[InitialCandidate]]]
object PreRankFilterRunner
def recordCandidateStatsBeforeFilter(
candidates: Seq[Seq[InitialCandidate]],
statsReceiver: StatsReceiver
): Unit
def recordCandidateStatsAfterFilter(
candidates: Seq[Seq[InitialCandidate]],
statsReceiver: StatsReceiver
): Unit
def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery](
request: CGQueryType,
candidates: Seq[Seq[InitialCandidate]],
filters: Seq[FilterBase],
statsReceiver: StatsReceiver
): Future[Seq[Seq[InitialCandidate]]] =
filters.foldLeft(Future.value(candidates))
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\ReplyFilter.scala
class ReplyFilter @Inject() () extends FilterBase
def name: String = this.getClass.getCanonicalName
override type ConfigType = Boolean
override def filter(
candidates: Seq[Seq[InitialCandidate]],
config: ConfigType
): Future[Seq[Seq[InitialCandidate]]]
def requestToConfig[CGQueryType <: CandidateGeneratorQuery](
query: CGQueryType
): ConfigType
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\RetweetFilter.scala
class RetweetFilter @Inject() () extends FilterBase
def name: String = this.getClass.getCanonicalName
override type ConfigType = Boolean
override def filter(
candidates: Seq[Seq[InitialCandidate]],
config: ConfigType
): Future[Seq[Seq[InitialCandidate]]]
def requestToConfig[CGQueryType <: CandidateGeneratorQuery](
query: CGQueryType
): ConfigType
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\TweetAgeFilter.scala
class TweetAgeFilter() extends FilterBase
def filter(
candidates: Seq[Seq[InitialCandidate]],
maxTweetAge: Duration
): Future[Seq[Seq[InitialCandidate]]]
def requestToConfig[CGQueryType <: CandidateGeneratorQuery](
query: CGQueryType
): Duration
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\TweetInfoHealthFilterBase.scala
trait TweetInfoHealthFilterBase extends FilterBase
def name: String = this.getClass.getCanonicalName
override type ConfigType = HealthThreshold.Enum.Value
def thresholdToPropertyMap: Map[HealthThreshold.Enum.Value, TweetInfo => Option[Boolean]]
def getFilterParamFn: CandidateGeneratorQuery => HealthThreshold.Enum.Value
override def filter(
candidates: Seq[Seq[InitialCandidate]],
config: HealthThreshold.Enum.Value
): Future[Seq[Seq[InitialCandidate]]]
def requestToConfig[CGQueryType <: CandidateGeneratorQuery](
query: CGQueryType
): HealthThreshold.Enum.Value
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\UtegFilterRunner.scala
class UtegFilterRunner @Inject() (
inNetworkFilter: InNetworkFilter,
utegHealthFilter: UtegHealthFilter,
retweetFilter: RetweetFilter,
globalStats: StatsReceiver)
def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery](
request: CGQueryType,
candidates: Seq[Seq[InitialCandidate]],
): Future[Seq[Seq[InitialCandidate]]]
object UtegFilterRunner
def recordCandidateStatsBeforeFilter(
candidates: Seq[Seq[InitialCandidate]],
statsReceiver: StatsReceiver
): Unit
def recordCandidateStatsAfterFilter(
candidates: Seq[Seq[InitialCandidate]],
statsReceiver: StatsReceiver
): Unit
def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery](
request: CGQueryType,
candidates: Seq[Seq[InitialCandidate]],
filters: Seq[FilterBase],
statsReceiver: StatsReceiver
): Future[Seq[Seq[InitialCandidate]]] =
filters.foldLeft(Future.value(candidates))
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\UtegHealthFilter.scala
class UtegHealthFilter @Inject() () extends FilterBase
def name: String = this.getClass.getCanonicalName
override type ConfigType = Boolean
override def filter(
candidates: Seq[Seq[InitialCandidate]],
config: ConfigType
): Future[Seq[Seq[InitialCandidate]]]
def requestToConfig[CGQueryType <: CandidateGeneratorQuery](
query: CGQueryType
): ConfigType
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\VideoTweetFilter.scala
class VideoTweetFilter() extends FilterBase
def filter(
candidates: Seq[Seq[InitialCandidate]],
config: ConfigType
): Future[Seq[Seq[InitialCandidate]]]
def checkTweetInfoAttribute(attributeOpt: => Option[Boolean]): Boolean
def requestToConfig[CGQueryType <: CandidateGeneratorQuery](
query: CGQueryType
): FilterConfig
object VideoTweetFilter
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\AdsRecommendationsScribeLogger.scala
class AdsRecommendationsScribeLogger @Inject() (
@Named(ModuleNames.AdsRecommendationsLogger) adsRecommendationsScribeLogger: Logger,
decider: CrMixerDecider,
statsReceiver: StatsReceiver)
def scribeInitialAdsCandidates(
query: AdsCandidateGeneratorQuery,
getResultFn: => Future[Seq[Seq[InitialAdsCandidate]]],
enableScribe: Boolean // controlled by feature switch so that we can scribe for certain DDG
): Future[Seq[Seq[InitialAdsCandidate]]]
def scribeGetAdsRecommendations(
request: AdsRequest,
startTime: Long,
scribeMetadata: ScribeMetadata,
getResultFn: => Future[AdsResponse],
enableScribe: Boolean
): Future[AdsResponse]
def convertFetchCandidatesResult(
candidatesSeq: Seq[Seq[InitialAdsCandidate]],
requestUserId: UserId
): AdsRecommendationsResult
def buildScribeMessage(
result: AdsRecommendationsResult,
scribeMetadata: ScribeMetadata,
latencyMs: Long,
traceId: Long
): GetAdsRecommendationsScribe
def scribeResult(
scribeMsg: GetAdsRecommendationsScribe
): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\CrMixerScribeLogger.scala
class CrMixerScribeLogger @Inject() (
decider: CrMixerDecider,
statsReceiver: StatsReceiver,
@Named(ModuleNames.TweetRecsLogger) tweetRecsScribeLogger: Logger,
@Named(ModuleNames.BlueVerifiedTweetRecsLogger) blueVerifiedTweetRecsScribeLogger: Logger,
@Named(ModuleNames.TopLevelApiDdgMetricsLogger) ddgMetricsLogger: Logger,
kafkaProducer: KafkaProducerBase[String, GetTweetsRecommendationsScribe])
def scribeSignalSources(
query: CrCandidateGeneratorQuery,
getResultFn: => Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])]
): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])]
def scribeInitialCandidates(
query: CrCandidateGeneratorQuery,
getResultFn: => Future[Seq[Seq[InitialCandidate]]]
): Future[Seq[Seq[InitialCandidate]]]
def scribePreRankFilterCandidates(
query: CrCandidateGeneratorQuery,
getResultFn: => Future[Seq[Seq[InitialCandidate]]]
): Future[Seq[Seq[InitialCandidate]]]
def scribeInterleaveCandidates(
query: CrCandidateGeneratorQuery,
getResultFn: => Future[Seq[BlendedCandidate]]
): Future[Seq[BlendedCandidate]]
def scribeRankedCandidates(
query: CrCandidateGeneratorQuery,
getResultFn: => Future[Seq[RankedCandidate]]
): Future[Seq[RankedCandidate]]
def scribeGetTweetRecommendations(
request: CrMixerTweetRequest,
startTime: Long,
scribeMetadata: ScribeMetadata,
getResultFn: => Future[CrMixerTweetResponse]
): Future[CrMixerTweetResponse]
def scribeGetTweetRecommendationsForBlueVerified(
scribeMetadata: ScribeMetadata,
getResultFn: => Future[Seq[RankedCandidate]]
): Future[Seq[RankedCandidate]]
def scribeResultsAndPerformanceMetrics[T](
scribeMetadata: ScribeMetadata,
getResultFn: => Future[T],
convertToResultFn: (T, UserId) => Result,
enableKafkaScribe: Boolean = false
): Future[T]
def convertTopLevelAPIResult(
request: CrMixerTweetRequest,
response: CrMixerTweetResponse,
startTime: Long
): Result
def convertFetchSignalSourcesResult(
sourceInfoSetTuple: (Set[SourceInfo], Map[String, Option[GraphSourceInfo]]),
requestUserId: UserId
): Result
def convertFetchCandidatesResult(
candidatesSeq: Seq[Seq[InitialCandidate]],
requestUserId: UserId
): Result
def convertPreRankFilterResult(
candidatesSeq: Seq[Seq[InitialCandidate]],
requestUserId: UserId
): Result
def convertInterleaveResult(
blendedCandidates: Seq[BlendedCandidate],
requestUserId: UserId
): Result
def convertRankResult(
rankedCandidates: Seq[RankedCandidate],
requestUserId: UserId
): Result
def buildScribeMessage(
result: Result,
scribeMetadata: ScribeMetadata,
latencyMs: Long,
traceId: Long
): GetTweetsRecommendationsScribe
def scribeResult(
scribeMsg: GetTweetsRecommendationsScribe
): Unit
def shouldScribeKafkaMessage(
userId: UserId,
product: Product
): Boolean
def downsampleKafkaMessage(
scribeMsg: GetTweetsRecommendationsScribe
): Seq[GetTweetsRecommendationsScribe]
def publishTopLevelDdgMetrics(
logger: Logger,
topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata,
candidateSize: Long,
latencyMs: Long,
): Unit
def getClientData(
topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata
): ClientDataProvider =
MinimalClientDataProvider(
userId = topLevelDdgMetricsMetadata.userId,
guestId = None,
clientApplicationId = topLevelDdgMetricsMetadata.clientApplicationId,
countryCode = topLevelDdgMetricsMetadata.countryCode
)
private def getNamespace(
topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata,
label: (String, String)
): Map[String, String]
object CrMixerScribeLogger
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\RelatedTweetScribeLogger.scala
class RelatedTweetScribeLogger @Inject() (
decider: CrMixerDecider,
statsReceiver: StatsReceiver,
@Named(ModuleNames.RelatedTweetsLogger) relatedTweetsScribeLogger: Logger)
def scribeInitialCandidates(
query: RelatedTweetCandidateGeneratorQuery,
getResultFn: => Future[Seq[Seq[InitialCandidate]]]
): Future[Seq[Seq[InitialCandidate]]]
def scribePreRankFilterCandidates(
query: RelatedTweetCandidateGeneratorQuery,
getResultFn: => Future[Seq[Seq[InitialCandidate]]]
): Future[Seq[Seq[InitialCandidate]]]
def scribeGetRelatedTweets(
request: RelatedTweetRequest,
startTime: Long,
relatedTweetScribeMetadata: RelatedTweetScribeMetadata,
getResultFn: => Future[RelatedTweetResponse]
): Future[RelatedTweetResponse]
def scribeResultsAndPerformanceMetrics[T](
relatedTweetScribeMetadata: RelatedTweetScribeMetadata,
getResultFn: => Future[T],
convertToResultFn: (T, UserId) => RelatedTweetResult
): Future[T]
def convertTopLevelAPIResult(
request: RelatedTweetRequest,
response: RelatedTweetResponse,
startTime: Long
): RelatedTweetResult
def convertFetchCandidatesResult(
candidatesSeq: Seq[Seq[InitialCandidate]],
requestUserId: UserId
): RelatedTweetResult
def convertPreRankFilterResult(
candidatesSeq: Seq[Seq[InitialCandidate]],
requestUserId: UserId
): RelatedTweetResult
def buildScribeMessage(
relatedTweetResult: RelatedTweetResult,
relatedTweetScribeMetadata: RelatedTweetScribeMetadata,
latencyMs: Long,
traceId: Long
): GetRelatedTweetsScribe
def scribeResult(
scribeMsg: GetRelatedTweetsScribe
): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\ScribeLoggerUtils.scala
object ScribeLoggerUtils
def publish[T <: ThriftStruct](
logger: Logger,
codec: ThriftStructCodec[T],
message: T
): Unit
def getImpressedBuckets(
scopedStats: StatsReceiver
): Option[List[ImpressesedBucketInfo]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\ScribeMetadata.scala
class ScribeMetadata(
requestUUID: Long,
userId: UserId,
product: Product)
object ScribeMetadata
def from(query: CrCandidateGeneratorQuery): ScribeMetadata
def from(query: UtegTweetCandidateGeneratorQuery): ScribeMetadata
def from(query: AdsCandidateGeneratorQuery): ScribeMetadata
class RelatedTweetScribeMetadata(
requestUUID: Long,
internalId: InternalId,
clientContext: ClientContext,
product: Product)
object RelatedTweetScribeMetadata
def from(query: RelatedTweetCandidateGeneratorQuery): RelatedTweetScribeMetadata
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\TopLevelDdgMetricsMetadata.scala
class TopLevelDdgMetricsMetadata(
userId: Option[Long],
product: Product,
clientApplicationId: Option[Long],
countryCode: Option[String])
object TopLevelDdgMetricsMetadata
def from(request: CrMixerTweetRequest): TopLevelDdgMetricsMetadata
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\UtegTweetScribeLogger.scala
class UtegTweetScribeLogger @Inject() (
decider: CrMixerDecider,
statsReceiver: StatsReceiver,
@Named(ModuleNames.UtegTweetsLogger) utegTweetScribeLogger: Logger)
def scribeInitialCandidates(
query: UtegTweetCandidateGeneratorQuery,
getResultFn: => Future[Seq[TweetWithScoreAndSocialProof]]
): Future[Seq[TweetWithScoreAndSocialProof]]
def scribeGetUtegTweetRecommendations(
request: UtegTweetRequest,
startTime: Long,
scribeMetadata: ScribeMetadata,
getResultFn: => Future[UtegTweetResponse]
): Future[UtegTweetResponse]
def convertTopLevelAPIResult(
request: UtegTweetRequest,
response: UtegTweetResponse,
startTime: Long
): UtegTweetResult
def buildScribeMessage(
utegTweetResult: UtegTweetResult,
scribeMetadata: ScribeMetadata,
latencyMs: Long,
traceId: Long
): GetUtegTweetsScribe
def scribeResult(
scribeMsg: GetUtegTweetsScribe
): Unit
def convertFetchCandidatesResult(
candidates: Seq[TweetWithScoreAndSocialProof],
requestUserId: UserId
): UtegTweetResult
def scribeResultsAndPerformanceMetrics[T](
scribeMetadata: ScribeMetadata,
getResultFn: => Future[T],
convertToResultFn: (T, UserId) => UtegTweetResult
): Future[T]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\Candidate.scala
trait Candidate
def hashCode: Int = tweetId.toInt
}
case class TweetWithCandidateGenerationInfo(
tweetId: TweetId,
candidateGenerationInfo: CandidateGenerationInfo)
extends Candidate
def getSimilarityScore: Double =
candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0)
}
case class InitialCandidate(
tweetId: TweetId,
tweetInfo: TweetInfo,
candidateGenerationInfo: CandidateGenerationInfo)
extends Candidate
def getSimilarityScore: Double =
candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0)
/**
* The same candidate can be generated by multiple algorithms.
* During blending, candidate deduping happens. In order to retain the candidateGenerationInfo
* from different algorithms, we attach them to a list of potentialReasons.
*/
def toBlendedCandidate(
potentialReasons: Seq[CandidateGenerationInfo],
): BlendedCandidate
def toRankedCandidate(): RankedCandidate
class InitialAdsCandidate(
tweetId: TweetId,
lineItemInfo: Seq[LineItemInfo],
candidateGenerationInfo: CandidateGenerationInfo)
extends Candidate
def getSimilarityScore: Double =
candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0)
/**
* The same candidate can be generated by multiple algorithms.
* During blending, candidate deduping happens. In order to retain the candidateGenerationInfo
* from different algorithms, we attach them to a list of potentialReasons.
*/
def toBlendedAdsCandidate(
potentialReasons: Seq[CandidateGenerationInfo],
): BlendedAdsCandidate
def toRankedAdsCandidate(): RankedAdsCandidate
class BlendedCandidate(
tweetId: TweetId,
tweetInfo: TweetInfo,
reasonChosen: CandidateGenerationInfo,
potentialReasons: Seq[CandidateGenerationInfo])
extends Candidate
def getSimilarityScore: Double =
reasonChosen.similarityEngineInfo.score.getOrElse(0.0)
assert(potentialReasons.contains(reasonChosen))
def toRankedCandidate(predictionScore: Double): RankedCandidate
class BlendedAdsCandidate(
tweetId: TweetId,
lineItemInfo: Seq[LineItemInfo],
reasonChosen: CandidateGenerationInfo,
potentialReasons: Seq[CandidateGenerationInfo])
extends Candidate
def getSimilarityScore: Double =
reasonChosen.similarityEngineInfo.score.getOrElse(0.0)
assert(potentialReasons.contains(reasonChosen))
def toRankedAdsCandidate(predictionScore: Double): RankedAdsCandidate
class RankedCandidate(
tweetId: TweetId,
tweetInfo: TweetInfo,
predictionScore: Double,
reasonChosen: CandidateGenerationInfo,
potentialReasons: Seq[CandidateGenerationInfo])
extends Candidate
def getSimilarityScore: Double =
reasonChosen.similarityEngineInfo.score.getOrElse(0.0)
assert(potentialReasons.contains(reasonChosen))
}
case class RankedAdsCandidate(
tweetId: TweetId,
lineItemInfo: Seq[LineItemInfo],
predictionScore: Double,
reasonChosen: CandidateGenerationInfo,
potentialReasons: Seq[CandidateGenerationInfo])
extends Candidate
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\CandidateGeneratorQuery.scala
trait CandidateGeneratorQuery
trait HasUserId
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\HealthThreshold.scala
object HealthThreshold
object Enum extends Enumeration
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\ModelConfig.scala
class for all Model Based Candidate Sources.
*
* The Model Name Guideline. Please your modelId as "Algorithm_Product_Date"
* If your model is used for multiple product surfaces, name it as all
* Don't name your algorithm as MBCG. All the algorithms here are MBCG =.=
*
* Don't forgot to add your new models into allHnswANNSimilarityEngineModelIds list.
*/
object ModelConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\ModuleNames.scala
object ModuleNames
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\ActivePromotedTweetStoreModule.scala
object ActivePromotedTweetStoreModule extends TwitterModule
class ActivePromotedTweetStore(
activePromotedTweetMHStore: ReadableStore[String, DataRecord],
statsReceiver: StatsReceiver)
extends ReadableStore[TweetId, Seq[LineItemInfo]]
def get(tweetId: TweetId): Future[Option[Seq[LineItemInfo]]]
def providesActivePromotedTweetStore(
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams,
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
crMixerStatsReceiver: StatsReceiver
): ReadableStore[TweetId, Seq[LineItemInfo]]
def statsReceiver: StatsReceiver =
crMixerStatsReceiver.scope("active_promoted_tweets_mh")
}
val mhStore: ReadableStore[String, DataRecord] =
ManhattanRO
.getReadableStoreWithMtls[String, DataRecord](
mhConfig,
manhattanKVClientMtlsParams
)(
implicitly[Injection[String, Array[Byte]]],
CompactThriftCodec[DataRecord]
)
val underlyingStore =
ActivePromotedTweetStore(mhStore, crMixerStatsReceiver.scope("ActivePromotedTweetStore"))
val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient(
backingStore = underlyingStore,
cacheClient = crMixerUnifiedCacheClient,
ttl = 60.minutes,
asyncUpdate = false
)(
valueInjection = LZ4Injection.compose(SeqObjectInjection[LineItemInfo]()),
statsReceiver = crMixerStatsReceiver.scope("memCachedActivePromotedTweetStore"),
keyToString
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\BlueVerifiedAnnotationStoreModule.scala
object BlueVerifiedAnnotationStoreModule extends TwitterModule
def providesBlueVerifiedAnnotationStore(
statsReceiver: StatsReceiver,
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams,
): ReadableStore[String, BlueVerifiedAnnotationsV2]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\CertoStratoStoreModule.scala
object CertoStratoStoreModule extends TwitterModule
def providesCertoStratoStore(
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
stratoClient: Client,
statsReceiver: StatsReceiver
): ReadableStore[TopicId, Seq[TweetWithScores]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\ConsumersBasedUserAdGraphStoreModule.scala
object ConsumersBasedUserAdGraphStoreModule extends TwitterModule
def providesConsumerBasedUserAdGraphStore(
userAdGraphService: UserAdGraph.MethodPerEndpoint
): ReadableStore[ConsumersBasedRelatedAdRequest, RelatedAdResponse]
def get(
k: ConsumersBasedRelatedAdRequest
): Future[Option[RelatedAdResponse]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\ConsumersBasedUserTweetGraphStoreModule.scala
object ConsumersBasedUserTweetGraphStoreModule extends TwitterModule
def providesConsumerBasedUserTweetGraphStore(
userTweetGraphService: UserTweetGraph.MethodPerEndpoint
): ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse]
def get(
k: ConsumersBasedRelatedTweetRequest
): Future[Option[RelatedTweetResponse]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\ConsumersBasedUserVideoGraphStoreModule.scala
object ConsumersBasedUserVideoGraphStoreModule extends TwitterModule
def providesConsumerBasedUserVideoGraphStore(
userVideoGraphService: UserVideoGraph.MethodPerEndpoint
): ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse]
def get(
k: ConsumersBasedRelatedTweetRequest
): Future[Option[RelatedTweetResponse]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\CrMixerParamConfigModule.scala
object CrMixerParamConfigModule extends TwitterModule
def provideConfig(): Config
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\DiffusionStoreModule.scala
object DiffusionStoreModule extends TwitterModule
def retweetBasedDiffusionRecsMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[Long, TweetsWithScore]
def buildTweetRecsStore(
serviceIdentifier: ServiceIdentifier,
manhattanROConfig: ManhattanROConfig
): ReadableStore[Long, TweetsWithScore]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\EarlybirdRecencyBasedCandidateStoreModule.scala
object EarlybirdRecencyBasedCandidateStoreModule extends TwitterModule
def providesEarlybirdRecencyBasedWithoutRetweetsRepliesCandidateStore(
statsReceiver: StatsReceiver,
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint,
@Named(ModuleNames.EarlybirdTweetsCache) earlybirdRecencyBasedTweetsCache: MemcachedClient,
timeoutConfig: TimeoutConfig
): ReadableStore[UserId, Seq[TweetId]]
def get(userId: UserId): Future[Option[Seq[TweetId]]]
def providesEarlybirdRecencyBasedWithRetweetsRepliesCandidateStore(
statsReceiver: StatsReceiver,
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint,
@Named(ModuleNames.EarlybirdTweetsCache) earlybirdRecencyBasedTweetsCache: MemcachedClient,
timeoutConfig: TimeoutConfig
): ReadableStore[UserId, Seq[TweetId]]
def get(userId: UserId): Future[Option[Seq[TweetId]]]
def buildEarlybirdRequest(
seedUserId: UserId,
filterOutRetweetsAndReplies: Boolean,
maxNumTweetsPerSeedUser: Int,
processingTimeout: Duration
): EarlybirdRequest =
EarlybirdRequest(
searchQuery = getThriftSearchQuery(
seedUserId = seedUserId,
filterOutRetweetsAndReplies = filterOutRetweetsAndReplies,
maxNumTweetsPerSeedUser = maxNumTweetsPerSeedUser,
processingTimeout = processingTimeout
),
clientId = Some(EarlybirdClientId),
timeoutMs = processingTimeout.inMilliseconds.intValue(),
getOlderResults = Some(false),
adjustedProtectedRequestParams = None,
adjustedFullArchiveRequestParams = None,
getProtectedTweetsOnly = Some(false),
skipVeryRecentTweets = true,
)
private def getThriftSearchQuery(
seedUserId: UserId,
filterOutRetweetsAndReplies: Boolean,
maxNumTweetsPerSeedUser: Int,
processingTimeout: Duration
): ThriftSearchQuery = ThriftSearchQuery(
serializedQuery = GetEarlybirdQuery(
None,
None,
Set.empty,
filterOutRetweetsAndReplies
).map(_.serialize),
fromUserIDFilter64 = Some(Seq(seedUserId)),
numResults = maxNumTweetsPerSeedUser,
rankingMode = ThriftSearchRankingMode.Recency,
collectorParams = Some(
CollectorParams(
// numResultsToReturn defines how many results each EB shard will return to search root
numResultsToReturn = maxNumTweetsPerSeedUser,
// terminationParams.maxHitsToProcess is used for early terminating per shard results fetching.
terminationParams =
GetCollectorTerminationParams(maxNumTweetsPerSeedUser, processingTimeout)
)),
facetFieldNames = Some(FacetsToFetch),
resultMetadataOptions = Some(MetadataOptions),
searchStatusIds = None
)
private def getEarlybirdSearchResult(
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint,
request: EarlybirdRequest,
statsReceiver: StatsReceiver
): Future[Option[Seq[TweetId]]] = earlybirdSearchClient
.search(request)
.map
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\EmbeddingStoreModule.scala
object EmbeddingStoreModule extends TwitterModule
def twHINEmbeddingRegularUpdateMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[InternalId, api.Embedding]
def consumerBasedTwHINEmbeddingRegularUpdateMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[InternalId, api.Embedding]
def twoTowerFavConsumerEmbeddingMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[InternalId, api.Embedding]
def debuggerDemoUserEmbeddingStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[InternalId, api.Embedding]
def debuggerDemoTweetEmbeddingStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[InternalId, api.Embedding]
def buildUserEmbeddingStore(
serviceIdentifier: ServiceIdentifier,
manhattanROConfig: ManhattanROConfig
): ReadableStore[InternalId, api.Embedding]
def buildTweetEmbeddingStore(
serviceIdentifier: ServiceIdentifier,
manhattanROConfig: ManhattanROConfig
): ReadableStore[InternalId, api.Embedding]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\FrsStoreModule.scala
object FrsStoreModule extends TwitterModule
def providesFrsStore(
frsClient: FollowRecommendationsThriftService.MethodPerEndpoint,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): ReadableStore[FrsStore.Query, Seq[FrsQueryResult]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\MHMtlsParamsModule.scala
object MHMtlsParamsModule extends TwitterModule
def providesManhattanMtlsParams(
serviceIdentifier: ServiceIdentifier
): ManhattanKVClientMtlsParams
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\OfflineCandidateStoreModule.scala
object OfflineCandidateStoreModule extends TwitterModule
def offlineTweet2020CandidateMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def offlineTweet2020Hl0El15CandidateMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def offlineTweet2020Hl2El15CandidateMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def offlineTweet2020Hl2El50CandidateMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def offlineTweet2020Hl8El50CandidateMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def offlineTweetMTSCandidateMhStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def offlineFavDecayedSumCandidateStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def offlineFtrAt5Pop1000RankDecay11CandidateStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def offlineFtrAt5Pop10000RankDecay11CandidateStore(
serviceIdentifier: ServiceIdentifier
): ReadableStore[UserId, CandidateTweetsList]
def buildOfflineCandidateStore(
serviceIdentifier: ServiceIdentifier,
datasetName: String
): ReadableStore[UserId, CandidateTweetsList]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\RealGraphOonStoreModule.scala
object RealGraphOonStoreModule extends TwitterModule
def providesRealGraphOonStore(
stratoClient: StratoClient,
statsReceiver: StatsReceiver
): ReadableStore[UserId, CandidateSeq]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\RealGraphStoreMhModule.scala
object RealGraphStoreMhModule extends TwitterModule
def providesRealGraphStoreMh(
decider: CrMixerDecider,
statsReceiver: StatsReceiver,
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams,
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
): ReadableStore[UserId, CandidateSeq]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\RepresentationManagerModule.scala
object RepresentationManagerModule extends TwitterModule
def providesRepresentationManagerTweetStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[TweetId, SimClustersEmbedding]
def providesRepresentationManagerUserFavBasedProducerEmbeddingStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[UserId, SimClustersEmbedding]
def providesRepresentationManagerUserLogFavConsumerEmbeddingStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[UserId, SimClustersEmbedding]
def providesRepresentationManagerUserFollowInterestedInEmbeddingStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[UserId, SimClustersEmbedding]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\RepresentationScorerModule.scala
object RepresentationScorerModule extends TwitterModule
def providesRepresentationScorerStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[(UserId, TweetId), Double]
def representationScorerStoreKeyMapping(t1: TweetId, t2: TweetId): ListScoreId
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\SampleSimilarityEngineModule.scala
object SimpleSimilarityEngineModule extends TwitterModule
def providesSimpleSimilarityEngine(
timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver
): StandardSimilarityEngine[UserId, (TweetId, Double)]
object LookupSimilarityEngineModule extends TwitterModule
def providesLookupSimilarityEngine(
timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver
): LookupSimilarityEngine[UserId, (TweetId, Double)]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\SimClustersANNServiceNameToClientMapper.scala
object SimClustersANNServiceNameToClientMapper extends TwitterModule
def providesSimClustersANNServiceNameToClientMapping(
@Named(ModuleNames.ProdSimClustersANNServiceClientName) simClustersANNServiceProd: SimClustersANNService.MethodPerEndpoint,
@Named(ModuleNames.ExperimentalSimClustersANNServiceClientName) simClustersANNServiceExperimental: SimClustersANNService.MethodPerEndpoint,
@Named(ModuleNames.SimClustersANNServiceClientName1) simClustersANNService1: SimClustersANNService.MethodPerEndpoint,
@Named(ModuleNames.SimClustersANNServiceClientName2) simClustersANNService2: SimClustersANNService.MethodPerEndpoint,
@Named(ModuleNames.SimClustersANNServiceClientName3) simClustersANNService3: SimClustersANNService.MethodPerEndpoint,
@Named(ModuleNames.SimClustersANNServiceClientName5) simClustersANNService5: SimClustersANNService.MethodPerEndpoint,
@Named(ModuleNames.SimClustersANNServiceClientName4) simClustersANNService4: SimClustersANNService.MethodPerEndpoint
): Map[String, SimClustersANNService.MethodPerEndpoint]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\SkitStratoStoreModule.scala
object SkitStratoStoreModule extends TwitterModule
def providesSkitStratoStore(
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
stratoClient: Client,
statsReceiver: StatsReceiver
): ReadableStore[TopicTweetPartitionFlatKey, Seq[TopicTweet]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\StrongTiePredictionStoreModule.scala
object StrongTiePredictionStoreModule extends TwitterModule
def providesStrongTiePredictionStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[UserId, STPResult]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TripCandidateStoreModule.scala
object TripCandidateStoreModule extends TwitterModule
def providesSimClustersTripCandidateStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient
): ReadableStore[TripDomain, Seq[TripTweet]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TweetInfoStoreModule.scala
object TweetInfoStoreModule extends TwitterModule
def modules: Seq[Module] = Seq(UnifiedCacheClient)
@Provides
@Singleton
def providesTweetInfoStore(
statsReceiver: StatsReceiver,
serviceIdentifier: ServiceIdentifier,
stratoClient: StratoClient,
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams,
tweetyPieService: TweetService.MethodPerEndpoint,
userTweetGraphPlusService: UserTweetGraphPlus.MethodPerEndpoint,
@Named(ModuleNames.BlueVerifiedAnnotationStore) blueVerifiedAnnotationStore: ReadableStore[
String,
BlueVerifiedAnnotationsV2
],
decider: CrMixerDecider
): ReadableStore[TweetId, TweetInfo]
def get(
k: TweetId
): Future[Option[TweetEngagementScores]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TweetRecentEngagedUserStoreModule.scala
object TweetRecentEngagedUserStoreModule extends TwitterModule
def providesTweetRecentEngagedUserStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[TweetId, TweetRecentEngagedUsers]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TweetRecommendationResultsStoreModule.scala
object TweetRecommendationResultsStoreModule extends TwitterModule
def providesTweetRecommendationResultsStore(
@Named(ModuleNames.TweetRecommendationResultsCache) tweetRecommendationResultsCacheClient: MemcachedClient,
statsReceiver: StatsReceiver
): ReadableWritableStore[UserId, CrMixerTweetResponse]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TwhinCollabFilterStratoStoreModule.scala
object TwhinCollabFilterStratoStoreModule extends TwitterModule
def providesTwhinCollabFilterStratoStoreForFollow(
stratoClient: StratoClient
): ReadableStore[Long, Seq[TweetId]]
def providesTwhinCollabFilterStratoStoreForEngagement(
stratoClient: StratoClient
): ReadableStore[Long, Seq[TweetId]]
def providesTwhinMultiClusterStratoStoreForFollow(
stratoClient: StratoClient
): ReadableStore[Long, Seq[TweetId]]
def providesTwhinMultiClusterStratoStoreForEngagement(
stratoClient: StratoClient
): ReadableStore[Long, Seq[TweetId]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TwiceClustersMembersStoreModule.scala
object TwiceClustersMembersStoreModule extends TwitterModule
def providesTweetRecentEngagedUserStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[UserId, OrderedClustersAndMembers]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\UnifiedCacheClient.scala
object UnifiedCacheClient extends TwitterModule
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\UserSignalServiceColumnModule.scala
object UserSignalServiceColumnModule extends TwitterModule
def providesUserSignalServiceStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[BatchSignalRequest, BatchSignalResponse]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\UserSignalServiceStoreModule.scala
object UserSignalServiceStoreModule extends TwitterModule
def providesUserSignalServiceStore(
statsReceiver: StatsReceiver,
stratoClient: StratoClient,
): ReadableStore[Query, Seq[(SignalType, Seq[UssSignal])]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\UserStateStoreModule.scala
object UserStateStoreModule extends TwitterModule
class UserStateStore(
userStateStore: ReadableStore[UserId, CondensedUserState],
timeout: Duration,
statsReceiver: StatsReceiver)
extends ReadableStore[UserId, UserState]
def get(userId: UserId): Future[Option[UserState]]
def providesUserStateStore(
crMixerDecider: CrMixerDecider,
statsReceiver: StatsReceiver,
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams,
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
timeoutConfig: TimeoutConfig
): ReadableStore[UserId, UserState]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\ABDeciderModule.scala
object ABDeciderModule extends TwitterModule
def provideABDecider(
@Flag("abdecider.path") abDeciderYmlPath: String,
@Named(ModuleNames.AbDeciderLogger) scribeLogger: Logger
): LoggingABDecider
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\CrMixerFlagModule.scala
object CrMixerFlagName
object CrMixerFlagModule extends TwitterModule
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\CrMixerLoggingABDeciderModule.scala
object CrMixerLoggingABDeciderModule extends TwitterModule
def provideABDecider(
loggingABDecider: LoggingABDecider,
statsReceiver: StatsReceiver
): CrMixerLoggingABDecider
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\FeatureContextBuilderModule.scala
object FeatureContextBuilderModule extends TwitterModule
def providesFeatureContextBuilder(featureSwitches: FeatureSwitches): FeatureContextBuilder
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\FeatureSwitchesModule.scala
object FeatureSwitchesModule extends TwitterModule
def providesFeatureSwitches(
@Flag("featureswitches.path") featureSwitchDirectory: String,
@Flag("use_config_repo_mirror.bool") useConfigRepoMirrorFlag: Boolean,
abDecider: CrMixerLoggingABDecider,
statsReceiver: StatsReceiver
): FeatureSwitches
def getConfigRepoAbsPath(
useConfigRepoMirrorFlag: Boolean
): String
def shouldFastRefresh(
useConfigRepoMirrorFlag: Boolean
): Boolean
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\KafkaProducerModule.scala
object KafkaProducerModule extends TwitterModule
def provideTweetRecsLoggerFactory(
serviceIdentifier: ServiceIdentifier,
): KafkaProducerBase[String, GetTweetsRecommendationsScribe]
object KafkaProducerFactory
def getKafkaProducer(
environment: String
): KafkaProducerBase[String, GetTweetsRecommendationsScribe]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\LoggerFactoryModule.scala
object LoggerFactoryModule extends TwitterModule
def provideAbDeciderLogger(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Logger
def provideTopLevelApiDdgMetricsLogger(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Logger
def provideTweetRecsLogger(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Logger
def provideVITTweetRecsLogger(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Logger
def provideRelatedTweetsLogger(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Logger
def provideUtegTweetsLogger(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Logger
def provideAdsRecommendationsLogger(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Logger
def buildLoggerFactory(
category: ScribeCategory,
environment: String,
statsReceiver: StatsReceiver
): TwitterLoggerFactory
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\MemoizingStatsReceiverModule.scala
object MemoizingStatsReceiverModule extends TwitterModule
def configure(): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\TimeoutConfigModule.scala
object TimeoutConfigModule extends TwitterModule
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\grpc_client\NaviGRPCClientModule.scala
object NaviGRPCClientModule extends TwitterModule
def providesHomeNaviGRPCClient(
serviceIdentifier: ServiceIdentifier,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): ManagedChannel
def providesAdsFavedNaviGRPCClient(
serviceIdentifier: ServiceIdentifier,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): ManagedChannel
def providesAdsMonetizableNaviGRPCClient(
serviceIdentifier: ServiceIdentifier,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): ManagedChannel
def buildClient(
serviceIdentifier: ServiceIdentifier,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
dest: String,
label: String
): ManagedChannel
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\CertoTopicTweetSimilarityEngineModule.scala
object CertoTopicTweetSimilarityEngineModule extends TwitterModule
def providesCertoTopicTweetSimilarityEngine(
@Named(ModuleNames.CertoStratoStoreName) certoStratoStore: ReadableStore[
TopicId,
Seq[TweetWithScores]
],
timeoutConfig: TimeoutConfig,
decider: CrMixerDecider,
statsReceiver: StatsReceiver
): StandardSimilarityEngine[
EngineQuery[Query],
TopicTweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumerBasedWalsSimilarityEngineModule.scala
object ConsumerBasedWalsSimilarityEngineModule extends TwitterModule
def providesConsumerBasedWalsSimilarityEngine(
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
@Named(ModuleNames.HomeNaviGRPCClient) homeNaviGRPCClient: ManagedChannel,
@Named(ModuleNames.AdsFavedNaviGRPCClient) adsFavedNaviGRPCClient: ManagedChannel,
@Named(ModuleNames.AdsMonetizableNaviGRPCClient) adsMonetizableNaviGRPCClient: ManagedChannel,
): StandardSimilarityEngine[
ConsumerBasedWalsSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumerEmbeddingBasedTripSimilarityEngineModule.scala
object ConsumerEmbeddingBasedTripSimilarityEngineModule extends TwitterModule
def providesConsumerEmbeddingBasedTripSimilarityEngineModule(
@Named(ModuleNames.RmsUserLogFavInterestedInEmbeddingStore)
userLogFavInterestedInEmbeddingStore: ReadableStore[UserId, SimClustersEmbedding],
@Named(ModuleNames.RmsUserFollowInterestedInEmbeddingStore)
userFollowInterestedInEmbeddingStore: ReadableStore[UserId, SimClustersEmbedding],
@Named(ModuleNames.TripCandidateStore)
tripCandidateStore: ReadableStore[TripDomain, Seq[TripTweet]],
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): StandardSimilarityEngine[TripEngineQuery, TripTweetWithScore]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumerEmbeddingBasedTwHINSimilarityEngineModule.scala
object ConsumerEmbeddingBasedTwHINSimilarityEngineModule extends TwitterModule
def providesConsumerEmbeddingBasedTwHINANNSimilarityEngine(
// MH stores
@Named(EmbeddingStoreModule.ConsumerBasedTwHINEmbeddingRegularUpdateMhStoreName)
consumerBasedTwHINEmbeddingRegularUpdateMhStore: ReadableStore[InternalId, api.Embedding],
@Named(EmbeddingStoreModule.DebuggerDemoUserEmbeddingMhStoreName)
debuggerDemoUserEmbeddingMhStore: ReadableStore[InternalId, api.Embedding],
@Named(AnnQueryServiceClientModule.TwHINRegularUpdateAnnServiceClientName)
twHINRegularUpdateAnnService: AnnQueryService.MethodPerEndpoint,
@Named(AnnQueryServiceClientModule.DebuggerDemoAnnServiceClientName)
debuggerDemoAnnService: AnnQueryService.MethodPerEndpoint,
// Other configs
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver
): HnswANNSimilarityEngine
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.scala
object ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule extends TwitterModule
def providesConsumerEmbeddingBasedTwoTowerANNSimilarityEngine(
@Named(EmbeddingStoreModule.TwoTowerFavConsumerEmbeddingMhStoreName)
twoTowerFavConsumerEmbeddingMhStore: ReadableStore[InternalId, api.Embedding],
@Named(AnnQueryServiceClientModule.TwoTowerFavAnnServiceClientName)
twoTowerFavAnnService: AnnQueryService.MethodPerEndpoint,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver
): HnswANNSimilarityEngine
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumersBasedUserAdGraphSimilarityEngineModule.scala
object ConsumersBasedUserAdGraphSimilarityEngineModule extends TwitterModule
def providesConsumersBasedUserAdGraphSimilarityEngine(
@Named(ModuleNames.ConsumerBasedUserAdGraphStore)
consumersBasedUserAdGraphStore: ReadableStore[
ConsumersBasedRelatedAdRequest,
RelatedAdResponse
],
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
ConsumersBasedUserAdGraphSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumersBasedUserVideoGraphSimilarityEngineModule.scala
object ConsumersBasedUserVideoGraphSimilarityEngineModule extends TwitterModule
def providesConsumersBasedUserVideoGraphSimilarityEngine(
@Named(ModuleNames.ConsumerBasedUserVideoGraphStore)
consumersBasedUserVideoGraphStore: ReadableStore[
ConsumersBasedRelatedTweetRequest,
RelatedTweetResponse
],
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
ConsumersBasedUserVideoGraphSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\DiffusionBasedSimilarityEngineModule.scala
object DiffusionBasedSimilarityEngineModule extends TwitterModule
def providesDiffusionBasedSimilarityEngineModule(
@Named(ModuleNames.RetweetBasedDiffusionRecsMhStore)
retweetBasedDiffusionRecsMhStore: ReadableStore[Long, TweetsWithScore],
timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver
): LookupSimilarityEngine[Query, TweetWithScore]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\EarlybirdSimilarityEngineModule.scala
object EarlybirdSimilarityEngineModule extends TwitterModule
def providesRecencyBasedEarlybirdSimilarityEngine(
earlybirdRecencyBasedSimilarityEngine: EarlybirdRecencyBasedSimilarityEngine,
timeoutConfig: TimeoutConfig,
decider: CrMixerDecider,
statsReceiver: StatsReceiver
): EarlybirdSimilarityEngine[
EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery,
EarlybirdRecencyBasedSimilarityEngine
]
def providesModelBasedEarlybirdSimilarityEngine(
earlybirdModelBasedSimilarityEngine: EarlybirdModelBasedSimilarityEngine,
timeoutConfig: TimeoutConfig,
decider: CrMixerDecider,
statsReceiver: StatsReceiver
): EarlybirdSimilarityEngine[
EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery,
EarlybirdModelBasedSimilarityEngine
]
def providesTensorflowBasedEarlybirdSimilarityEngine(
earlybirdTensorflowBasedSimilarityEngine: EarlybirdTensorflowBasedSimilarityEngine,
timeoutConfig: TimeoutConfig,
decider: CrMixerDecider,
statsReceiver: StatsReceiver
): EarlybirdSimilarityEngine[
EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery,
EarlybirdTensorflowBasedSimilarityEngine
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ProducerBasedUnifiedSimilarityEngineModule.scala
object ProducerBasedUnifiedSimilarityEngineModule extends TwitterModule
def providesProducerBasedUnifiedSimilarityEngine(
@Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine)
producerBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[
ProducerBasedUserTweetGraphSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.SimClustersANNSimilarityEngine)
simClustersANNSimilarityEngine: StandardSimilarityEngine[
SimClustersANNSimilarityEngine.Query,
TweetWithScore
],
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): StandardSimilarityEngine[
ProducerBasedUnifiedSimilarityEngine.Query,
TweetWithCandidateGenerationInfo
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ProducerBasedUserAdGraphSimilarityEngineModule.scala
object ProducerBasedUserAdGraphSimilarityEngineModule extends TwitterModule
def providesProducerBasedUserAdGraphSimilarityEngine(
userAdGraphService: UserAdGraph.MethodPerEndpoint,
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
ProducerBasedUserAdGraphSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ProducerBasedUserTweetGraphSimilarityEngineModule.scala
object ProducerBasedUserTweetGraphSimilarityEngineModule extends TwitterModule
def providesProducerBasedUserTweetGraphSimilarityEngine(
userTweetGraphService: UserTweetGraph.MethodPerEndpoint,
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
ProducerBasedUserTweetGraphSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\SimClustersANNSimilarityEngineModule.scala
object SimClustersANNSimilarityEngineModule extends TwitterModule
def providesProdSimClustersANNSimilarityEngine(
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint],
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver
): StandardSimilarityEngine[Query, TweetWithScore]
def buildWrapperStore(
memCachedStore: ReadableStore[Query, Seq[TweetWithScore]],
underlyingStore: ReadableStore[Query, Seq[TweetWithScore]],
wrapperStats: StatsReceiver
): ReadableStore[Query, Seq[TweetWithScore]]
def multiGet[K1 <: Query](
queries: Set[K1]
): Map[K1, Future[Option[Seq[TweetWithScore]]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\SkitTopicTweetSimilarityEngineModule.scala
object SkitTopicTweetSimilarityEngineModule extends TwitterModule
def providesSkitHighPrecisionTopicTweetSimilarityEngine(
@Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[
TopicTweetPartitionFlatKey,
Seq[TopicTweet]
],
timeoutConfig: TimeoutConfig,
decider: CrMixerDecider,
statsReceiver: StatsReceiver
): StandardSimilarityEngine[
EngineQuery[Query],
TopicTweetWithScore
]
def providesSkitTfgTopicTweetSimilarityEngine(
@Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[
TopicTweetPartitionFlatKey,
Seq[TopicTweet]
],
timeoutConfig: TimeoutConfig,
decider: CrMixerDecider,
statsReceiver: StatsReceiver
): StandardSimilarityEngine[
EngineQuery[Query],
TopicTweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedQigSimilarityEngineModule.scala
object TweetBasedQigSimilarityEngineModule extends TwitterModule
def providesTweetBasedQigSimilarTweetsCandidateSource(
qigRanker: QigRanker.MethodPerEndpoint,
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
TweetBasedQigSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedTwHINSimlarityEngineModule.scala
object TweetBasedTwHINSimlarityEngineModule extends TwitterModule
def providesTweetBasedTwHINANNSimilarityEngine(
// MH stores
@Named(EmbeddingStoreModule.TwHINEmbeddingRegularUpdateMhStoreName)
twHINEmbeddingRegularUpdateMhStore: ReadableStore[InternalId, api.Embedding],
@Named(EmbeddingStoreModule.DebuggerDemoTweetEmbeddingMhStoreName)
debuggerDemoTweetEmbeddingMhStore: ReadableStore[InternalId, api.Embedding],
// ANN clients
@Named(AnnQueryServiceClientModule.TwHINRegularUpdateAnnServiceClientName)
twHINRegularUpdateAnnService: AnnQueryService.MethodPerEndpoint,
@Named(AnnQueryServiceClientModule.DebuggerDemoAnnServiceClientName)
debuggerDemoAnnService: AnnQueryService.MethodPerEndpoint,
// Other configs
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver
): HnswANNSimilarityEngine
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedUnifiedSimilarityEngineModule.scala
object TweetBasedUnifiedSimilarityEngineModule extends TwitterModule
def providesTweetBasedUnifiedSimilarityEngine(
@Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) tweetBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[
TweetBasedUserTweetGraphSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) tweetBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[
TweetBasedUserVideoGraphSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine)
tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine,
@Named(ModuleNames.TweetBasedQigSimilarityEngine) tweetBasedQigSimilarityEngine: StandardSimilarityEngine[
TweetBasedQigSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.SimClustersANNSimilarityEngine)
simClustersANNSimilarityEngine: StandardSimilarityEngine[
SimClustersANNSimilarityEngine.Query,
TweetWithScore
],
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): StandardSimilarityEngine[
TweetBasedUnifiedSimilarityEngine.Query,
TweetWithCandidateGenerationInfo
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedUserAdGraphSimilarityEngineModule.scala
object TweetBasedUserAdGraphSimilarityEngineModule extends TwitterModule
def providesTweetBasedUserAdGraphSimilarityEngine(
userAdGraphService: UserAdGraph.MethodPerEndpoint,
tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers],
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
TweetBasedUserAdGraphSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedUserTweetGraphSimilarityEngineModule.scala
object TweetBasedUserTweetGraphSimilarityEngineModule extends TwitterModule
def providesTweetBasedUserTweetGraphSimilarityEngine(
userTweetGraphService: UserTweetGraph.MethodPerEndpoint,
tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers],
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
TweetBasedUserTweetGraphSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedUserVideoGraphSimilarityEngineModule.scala
object TweetBasedUserVideoGraphSimilarityEngineModule extends TwitterModule
def providesTweetBasedUserVideoGraphSimilarityEngine(
userVideoGraphService: UserVideoGraph.MethodPerEndpoint,
tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers],
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
TweetBasedUserVideoGraphSimilarityEngine.Query,
TweetWithScore
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TwhinCollabFilterLookupSimilarityEngineModule.scala
object TwhinCollabFilterLookupSimilarityEngineModule extends TwitterModule
def providesTwhinCollabFilterLookupSimilarityEngineModule(
@Named(ModuleNames.TwhinCollabFilterStratoStoreForFollow)
twhinCollabFilterStratoStoreForFollow: ReadableStore[Long, Seq[TweetId]],
@Named(ModuleNames.TwhinCollabFilterStratoStoreForEngagement)
twhinCollabFilterStratoStoreForEngagement: ReadableStore[Long, Seq[TweetId]],
@Named(ModuleNames.TwhinMultiClusterStratoStoreForFollow)
twhinMultiClusterStratoStoreForFollow: ReadableStore[Long, Seq[TweetId]],
@Named(ModuleNames.TwhinMultiClusterStratoStoreForEngagement)
twhinMultiClusterStratoStoreForEngagement: ReadableStore[Long, Seq[TweetId]],
timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver
): LookupSimilarityEngine[Query, TweetWithScore]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\UserTweetEntityGraphSimilarityEngineModule.scala
object UserTweetEntityGraphSimilarityEngineModule extends TwitterModule
def providesUserTweetEntityGraphSimilarityEngine(
userTweetEntityGraphService: UserTweetEntityGraph.MethodPerEndpoint,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
decider: CrMixerDecider
): StandardSimilarityEngine[
UserTweetEntityGraphSimilarityEngine.Query,
TweetWithScoreAndSocialProof
]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\AnnQueryServiceClientModule.scala
object AnnQueryServiceClientModule extends TwitterModule
def debuggerDemoAnnServiceClient(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
statsReceiver: StatsReceiver,
timeoutConfig: TimeoutConfig,
): AnnQueryService.MethodPerEndpoint
def twhinUuaAnnServiceClient(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
statsReceiver: StatsReceiver,
timeoutConfig: TimeoutConfig,
): AnnQueryService.MethodPerEndpoint
def twHINRegularUpdateAnnServiceClient(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
statsReceiver: StatsReceiver,
timeoutConfig: TimeoutConfig,
): AnnQueryService.MethodPerEndpoint
def twoTowerFavAnnServiceClient(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
statsReceiver: StatsReceiver,
timeoutConfig: TimeoutConfig,
): AnnQueryService.MethodPerEndpoint
def buildClient(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
dest: String,
label: String
): AnnQueryService.MethodPerEndpoint
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\EarlybirdSearchClientModule.scala
object EarlybirdSearchClientModule
extends ThriftMethodBuilderClientModule[
EarlybirdService.ServicePerEndpoint,
EarlybirdService.MethodPerEndpoint
]
with MtlsClient
def label: String = "earlybird"
override def dest: String = "/s/earlybird-root-superroot/root-superroot"
private val requestTimeoutFlag: Flag[Duration] =
flag[Duration](EarlybirdClientTimeoutFlagName, "Earlybird client timeout")
override protected def requestTimeout: Duration = requestTimeoutFlag()
override def retryBudget: RetryBudget = RetryBudget.Empty
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\FrsClientModule.scala
object FrsClientModule
extends ThriftMethodBuilderClientModule[
FollowRecommendationsThriftService.ServicePerEndpoint,
FollowRecommendationsThriftService.MethodPerEndpoint
]
with MtlsClient
def label: String = "follow-recommendations-service"
override def dest: String = "/s/follow-recommendations/follow-recos-service"
private val frsSignalFetchTimeout: Flag[Duration] =
flag[Duration](FrsClientTimeoutFlagName, "FRS signal fetch client timeout")
override def requestTimeout: Duration = frsSignalFetchTimeout()
override def retryBudget: RetryBudget = RetryBudget.Empty
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\HydraPartitionClientModule.scala
object HydraPartitionClientModule
extends ThriftMethodBuilderClientModule[
ht.HydraPartition.ServicePerEndpoint,
ht.HydraPartition.MethodPerEndpoint
]
with MtlsClient
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\HydraRootClientModule.scala
object HydraRootClientModule
extends ThriftMethodBuilderClientModule[
ht.HydraRoot.ServicePerEndpoint,
ht.HydraRoot.MethodPerEndpoint
]
with MtlsClient
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\QigServiceClientModule.scala
object QigServiceClientModule
extends ThriftMethodBuilderClientModule[
QigRanker.ServicePerEndpoint,
QigRanker.MethodPerEndpoint
]
with MtlsClient
def requestTimeout: Duration = qigRankerClientTimeout()
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client =
super
.configureThriftMuxClient(injector, client)
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt"))
.withResponseClassifier
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\SimClustersAnnServiceClientModule.scala
object SimClustersAnnServiceClientModule extends TwitterModule
def providesProdSimClustersANNServiceClient(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): t.SimClustersANNService.MethodPerEndpoint
def providesExperimentalSimClustersANNServiceClient(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): t.SimClustersANNService.MethodPerEndpoint
def providesSimClustersANNServiceClient1(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): t.SimClustersANNService.MethodPerEndpoint
def providesSimClustersANNServiceClient2(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): t.SimClustersANNService.MethodPerEndpoint
def providesSimClustersANNServiceClient3(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): t.SimClustersANNService.MethodPerEndpoint
def providesSimClustersANNServiceClient5(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): t.SimClustersANNService.MethodPerEndpoint
def providesSimClustersANNServiceClient4(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
): t.SimClustersANNService.MethodPerEndpoint
def buildClient(
serviceIdentifier: ServiceIdentifier,
clientId: ClientId,
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver,
dest: String,
label: String
): t.SimClustersANNService.MethodPerEndpoint
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\TweetyPieClientModule.scala
object TweetyPieClientModule
extends ThriftMethodBuilderClientModule[
TweetService.ServicePerEndpoint,
TweetService.MethodPerEndpoint
]
with MtlsClient
def requestTimeout: Duration = tweetypieClientTimeout()
override def retryBudget: RetryBudget = RetryBudget.Empty
// We bump the success rate from the default of 0.8 to 0.9 since we're dropping the
// consecutive failures part of the default policy.
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client =
super
.configureThriftMuxClient(injector, client)
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt"))
.withSessionQualifier
.successRateFailureAccrual(successRate = 0.9, window = 30.seconds)
.withResponseClassifier
def providesTweetyPie(
tweetyPieService: TweetService.MethodPerEndpoint
): STweetyPie
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserAdGraphClientModule.scala
object UserAdGraphClientModule
extends ThriftMethodBuilderClientModule[
UserAdGraph.ServicePerEndpoint,
UserAdGraph.MethodPerEndpoint
]
with MtlsClient
def requestTimeout: Duration = userAdGraphClientTimeout()
override def retryBudget: RetryBudget = RetryBudget.Empty
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client =
super
.configureThriftMuxClient(injector, client)
.withMutualTls(injector.instance[ServiceIdentifier])
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt"))
.withResponseClassifier
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserTweetEntityGraphClientModule.scala
object UserTweetEntityGraphClientModule
extends ThriftMethodBuilderClientModule[
UserTweetEntityGraph.ServicePerEndpoint,
UserTweetEntityGraph.MethodPerEndpoint
]
with MtlsClient
def requestTimeout: Duration = userTweetEntityGraphClientTimeout()
override def retryBudget: RetryBudget = RetryBudget.Empty
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client =
super
.configureThriftMuxClient(injector, client)
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt"))
.withResponseClassifier
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserTweetGraphClientModule.scala
object UserTweetGraphClientModule
extends ThriftMethodBuilderClientModule[
UserTweetGraph.ServicePerEndpoint,
UserTweetGraph.MethodPerEndpoint
]
with MtlsClient
def requestTimeout: Duration = userTweetGraphClientTimeout()
override def retryBudget: RetryBudget = RetryBudget.Empty
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client =
super
.configureThriftMuxClient(injector, client)
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt"))
.withResponseClassifier
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserTweetGraphPlusClientModule.scala
object UserTweetGraphPlusClientModule
extends ThriftMethodBuilderClientModule[
UserTweetGraphPlus.ServicePerEndpoint,
UserTweetGraphPlus.MethodPerEndpoint
]
with MtlsClient
def requestTimeout: Duration = userTweetGraphPlusClientTimeout()
override def retryBudget: RetryBudget = RetryBudget.Empty
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client =
super
.configureThriftMuxClient(injector, client)
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt"))
.withResponseClassifier
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserVideoGraphClientModule.scala
object UserVideoGraphClientModule
extends ThriftMethodBuilderClientModule[
UserVideoGraph.ServicePerEndpoint,
UserVideoGraph.MethodPerEndpoint
]
with MtlsClient
def requestTimeout: Duration = userVideoGraphClientTimeout()
override def retryBudget: RetryBudget = RetryBudget.Empty
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client =
super
.configureThriftMuxClient(injector, client)
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt"))
.withResponseClassifier
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\AdsParams.scala
object AdsParams
object AdsCandidateGenerationMaxCandidatesNumParam
extends FSBoundedParam[Int](
name = "ads_candidate_generation_max_candidates_num",
default = 400,
min = 0,
max = 2000
)
object EnableScoreBoost
extends FSParam[Boolean](
name = "ads_candidate_generation_enable_score_boost",
default = false
)
object AdsCandidateGenerationScoreBoostFactor
extends FSBoundedParam[Double](
name = "ads_candidate_generation_score_boost_factor",
default = 10000.0,
min = 1.0,
max = 100000.0
)
object EnableScribe
extends FSParam[Boolean](
name = "ads_candidate_generation_enable_scribe",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
AdsCandidateGenerationMaxCandidatesNumParam,
EnableScoreBoost,
AdsCandidateGenerationScoreBoostFactor
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\BlenderParams.scala
object BlenderParams
object BlendingAlgorithmEnum extends Enumeration
object ContentBasedSortingAlgorithmEnum extends Enumeration
object BlendingAlgorithmParam
extends FSEnumParam[BlendingAlgorithmEnum.type](
name = "blending_algorithm_id",
default = BlendingAlgorithmEnum.RoundRobin,
enum = BlendingAlgorithmEnum
)
object RankingInterleaveWeightShrinkageParam
extends FSBoundedParam[Double](
name = "blending_enable_ml_ranking_interleave_weights_shrinkage",
default = 1.0,
min = 0.0,
max = 1.0
)
object RankingInterleaveMaxWeightAdjustments
extends FSBoundedParam[Int](
name = "blending_interleave_max_weighted_adjustments",
default = 3000,
min = 0,
max = 9999
)
object SignalTypeSortingAlgorithmParam
extends FSEnumParam[ContentBasedSortingAlgorithmEnum.type](
name = "blending_algorithm_inner_signal_sorting_id",
default = ContentBasedSortingAlgorithmEnum.SourceSignalRecency,
enum = ContentBasedSortingAlgorithmEnum
)
object ContentBlenderTypeSortingAlgorithmParam
extends FSEnumParam[ContentBasedSortingAlgorithmEnum.type](
name = "blending_algorithm_content_blender_sorting_id",
default = ContentBasedSortingAlgorithmEnum.FavoriteCount,
enum = ContentBasedSortingAlgorithmEnum
)
//UserAffinities Algo Param: whether to distributed the source type weights
object EnableDistributedSourceTypeWeightsParam
extends FSParam[Boolean](
name = "blending_algorithm_enable_distributed_source_type_weights",
default = false
)
object BlendGroupingMethodEnum extends Enumeration
object BlendGroupingMethodParam
extends FSEnumParam[BlendGroupingMethodEnum.type](
name = "blending_grouping_method_id",
default = BlendGroupingMethodEnum.SourceKeyDefault,
enum = BlendGroupingMethodEnum
)
object RecencyBasedRandomSamplingHalfLifeInDays
extends FSBoundedParam[Int](
name = "blending_interleave_random_sampling_recency_based_half_life_in_days",
default = 7,
min = 1,
max = 28
)
object RecencyBasedRandomSamplingDefaultWeight
extends FSBoundedParam[Double](
name = "blending_interleave_random_sampling_recency_based_default_weight",
default = 1.0,
min = 0.1,
max = 2.0
)
object SourceTypeBackFillEnableVideoBackFill
extends FSParam[Boolean](
name = "blending_enable_video_backfill",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
BlendingAlgorithmParam,
RankingInterleaveWeightShrinkageParam,
RankingInterleaveMaxWeightAdjustments,
EnableDistributedSourceTypeWeightsParam,
BlendGroupingMethodParam,
RecencyBasedRandomSamplingHalfLifeInDays,
RecencyBasedRandomSamplingDefaultWeight,
SourceTypeBackFillEnableVideoBackFill,
SignalTypeSortingAlgorithmParam,
ContentBlenderTypeSortingAlgorithmParam,
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\BypassInterleaveAndRankParams.scala
object BypassInterleaveAndRankParams
object EnableTwhinCollabFilterBypassParam
extends FSParam[Boolean](
name = "bypass_interleave_and_rank_twhin_collab_filter",
default = false
)
object EnableTwoTowerBypassParam
extends FSParam[Boolean](
name = "bypass_interleave_and_rank_two_tower",
default = false
)
object EnableConsumerBasedTwhinBypassParam
extends FSParam[Boolean](
name = "bypass_interleave_and_rank_consumer_based_twhin",
default = false
)
object EnableConsumerBasedWalsBypassParam
extends FSParam[Boolean](
name = "bypass_interleave_and_rank_consumer_based_wals",
default = false
)
object TwhinCollabFilterBypassPercentageParam
extends FSBoundedParam[Double](
name = "bypass_interleave_and_rank_twhin_collab_filter_percentage",
default = 0.0,
min = 0.0,
max = 1.0
)
object TwoTowerBypassPercentageParam
extends FSBoundedParam[Double](
name = "bypass_interleave_and_rank_two_tower_percentage",
default = 0.0,
min = 0.0,
max = 1.0
)
object ConsumerBasedTwhinBypassPercentageParam
extends FSBoundedParam[Double](
name = "bypass_interleave_and_rank_consumer_based_twhin_percentage",
default = 0.0,
min = 0.0,
max = 1.0
)
object ConsumerBasedWalsBypassPercentageParam
extends FSBoundedParam[Double](
name = "bypass_interleave_and_rank_consumer_based_wals_percentage",
default = 0.0,
min = 0.0,
max = 1.0
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableTwhinCollabFilterBypassParam,
EnableTwoTowerBypassParam,
EnableConsumerBasedTwhinBypassParam,
EnableConsumerBasedWalsBypassParam,
TwhinCollabFilterBypassPercentageParam,
TwoTowerBypassPercentageParam,
ConsumerBasedTwhinBypassPercentageParam,
ConsumerBasedWalsBypassPercentageParam,
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerBasedWalsParams.scala
object ConsumerBasedWalsParams
object EnableSourceParam
extends FSParam[Boolean](
name = "consumer_based_wals_enable_source",
default = false
)
object ModelNameParam
extends FSParam[String](
name = "consumer_based_wals_model_name",
default = "model_0"
)
object WilyNsNameParam
extends FSParam[String](
name = "consumer_based_wals_wily_ns_name",
default = ""
)
object ModelInputNameParam
extends FSParam[String](
name = "consumer_based_wals_model_input_name",
default = "examples"
)
object ModelOutputNameParam
extends FSParam[String](
name = "consumer_based_wals_model_output_name",
default = "all_tweet_ids"
)
object ModelSignatureNameParam
extends FSParam[String](
name = "consumer_based_wals_model_signature_name",
default = "serving_default"
)
object MaxTweetSignalAgeHoursParam
extends FSBoundedParam[Duration](
name = "consumer_based_wals_max_tweet_signal_age_hours",
default = 72.hours,
min = 1.hours,
max = 720.hours
)
with HasDurationConversion
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerEmbeddingBasedCandidateGenerationParams.scala
object ConsumerEmbeddingBasedCandidateGenerationParams
object EnableTwHINParam
extends FSParam[Boolean](
name = "consumer_embedding_based_candidate_generation_enable_twhin",
default = false
)
object EnableTwoTowerParam
extends FSParam[Boolean](
name = "consumer_embedding_based_candidate_generation_enable_two_tower",
default = false
)
object EnableLogFavBasedSimClustersTripParam
extends FSParam[Boolean](
name = "consumer_embedding_based_candidate_generation_enable_logfav_based_simclusters_trip",
default = false
)
object EnableFollowBasedSimClustersTripParam
extends FSParam[Boolean](
name = "consumer_embedding_based_candidate_generation_enable_follow_based_simclusters_trip",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableTwHINParam,
EnableTwoTowerParam,
EnableFollowBasedSimClustersTripParam,
EnableLogFavBasedSimClustersTripParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerEmbeddingBasedTripParams.scala
object ConsumerEmbeddingBasedTripParams
object SourceIdParam
extends FSParam[String](
name = "consumer_embedding_based_trip_source_id",
default = "EXPLR_TOPK_VID_48H_V3")
object MaxNumCandidatesParam
extends FSBoundedParam[Int](
name = "consumer_embedding_based_trip_max_num_candidates",
default = 80,
min = 0,
max = 200
)
val AllParams: Seq[Param[_] with FSName] = Seq(
SourceIdParam,
MaxNumCandidatesParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerEmbeddingBasedTwHINParams.scala
object ConsumerEmbeddingBasedTwHINParams
object ModelIdParam
extends FSParam[String](
name = "consumer_embedding_based_twhin_model_id",
default = ModelConfig.ConsumerBasedTwHINRegularUpdateAll20221024,
) // Note: this default value does not match with ModelIds yet. This FS is a placeholder
val AllParams: Seq[Param[_] with FSName] = Seq(
ModelIdParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerEmbeddingBasedTwoTowerParams.scala
object ConsumerEmbeddingBasedTwoTowerParams
object ModelIdParam
extends FSParam[String](
name = "consumer_embedding_based_two_tower_model_id",
default = TwoTowerFavALL20220808,
) // Note: this default value does not match with ModelIds yet. This FS is a placeholder
val AllParams: Seq[Param[_] with FSName] = Seq(
ModelIdParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumersBasedUserAdGraphParams.scala
object ConsumersBasedUserAdGraphParams
object EnableSourceParam
extends FSParam[Boolean](
name = "consumers_based_user_ad_graph_enable_source",
default = false
)
// UTG-Lookalike
object MinCoOccurrenceParam
extends FSBoundedParam[Int](
name = "consumers_based_user_ad_graph_min_co_occurrence",
default = 2,
min = 0,
max = 500
)
object MinScoreParam
extends FSBoundedParam[Double](
name = "consumers_based_user_ad_graph_min_score",
default = 0.0,
min = 0.0,
max = 10.0
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam,
MinCoOccurrenceParam,
MinScoreParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumersBasedUserTweetGraphParams.scala
object ConsumersBasedUserTweetGraphParams
object EnableSourceParam
extends FSParam[Boolean](
name = "consumers_based_user_tweet_graph_enable_source",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam,
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumersBasedUserVideoGraphParams.scala
object ConsumersBasedUserVideoGraphParams
object EnableSourceParam
extends FSParam[Boolean](
name = "consumers_based_user_video_graph_enable_source",
default = false
)
// UTG-RealGraphIN
object RealGraphInMinCoOccurrenceParam
extends FSBoundedParam[Int](
name = "consumers_based_user_video_graph_real_graph_in_min_co_occurrence",
default = 3,
min = 0,
max = 500
)
object RealGraphInMinScoreParam
extends FSBoundedParam[Double](
name = "consumers_based_user_video_graph_real_graph_in_min_score",
default = 2.0,
min = 0.0,
max = 10.0
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam,
RealGraphInMinCoOccurrenceParam,
RealGraphInMinScoreParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CrMixerParamConfig.scala
object CrMixerParamConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CustomizedRetrievalBasedCandidateGenerationParams.scala
object CustomizedRetrievalBasedCandidateGenerationParams
object EnableOfflineInterestedInParam
extends FSParam[Boolean](
name = "customized_retrieval_based_candidate_generation_enable_offline_interestedin",
default = false
)
// Offline SimClusters FTR-based InterestedIn
object EnableOfflineFTRInterestedInParam
extends FSParam[Boolean](
name = "customized_retrieval_based_candidate_generation_enable_ftr_offline_interestedin",
default = false
)
// TwHin Collab Filter Cluster params
object EnableTwhinCollabFilterClusterParam
extends FSParam[Boolean](
name = "customized_retrieval_based_candidate_generation_enable_twhin_collab_filter_cluster",
default = false
)
// TwHin Multi Cluster params
object EnableTwhinMultiClusterParam
extends FSParam[Boolean](
name = "customized_retrieval_based_candidate_generation_enable_twhin_multi_cluster",
default = false
)
object EnableRetweetBasedDiffusionParam
extends FSParam[Boolean](
name = "customized_retrieval_based_candidate_generation_enable_retweet_based_diffusion",
default = false
)
object CustomizedRetrievalBasedRetweetDiffusionSource
extends FSParam[String](
name =
"customized_retrieval_based_candidate_generation_offline_retweet_based_diffusion_model_id",
default = ModelConfig.RetweetBasedDiffusion
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableOfflineInterestedInParam,
EnableOfflineFTRInterestedInParam,
EnableTwhinCollabFilterClusterParam,
EnableTwhinMultiClusterParam,
EnableRetweetBasedDiffusionParam,
CustomizedRetrievalBasedRetweetDiffusionSource
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CustomizedRetrievalBasedFTROfflineInterestedInParams.scala
object CustomizedRetrievalBasedFTROfflineInterestedInParams
object CustomizedRetrievalBasedFTROfflineInterestedInSource
extends FSParam[String](
name = "customized_retrieval_based_ftr_offline_interestedin_model_id",
default = ModelConfig.OfflineFavDecayedSum
)
val AllParams: Seq[Param[_] with FSName] = Seq(
CustomizedRetrievalBasedFTROfflineInterestedInSource)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CustomizedRetrievalBasedOfflineInterestedInParams.scala
object CustomizedRetrievalBasedOfflineInterestedInParams
object CustomizedRetrievalBasedOfflineInterestedInSource
extends FSParam[String](
name = "customized_retrieval_based_offline_interestedin_model_id",
default = ModelConfig.OfflineInterestedInFromKnownFor2020
)
val AllParams: Seq[Param[_] with FSName] = Seq(CustomizedRetrievalBasedOfflineInterestedInSource)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CustomizedRetrievalBasedTwhinParams.scala
object CustomizedRetrievalBasedTwhinParams
object CustomizedRetrievalBasedTwhinCollabFilterFollowSource
extends FSParam[String](
name = "customized_retrieval_based_offline_twhin_collab_filter_follow_model_id",
default = ModelConfig.TwhinCollabFilterForFollow
)
object CustomizedRetrievalBasedTwhinCollabFilterEngagementSource
extends FSParam[String](
name = "customized_retrieval_based_offline_twhin_collab_filter_engagement_model_id",
default = ModelConfig.TwhinCollabFilterForEngagement
)
object CustomizedRetrievalBasedTwhinMultiClusterFollowSource
extends FSParam[String](
name = "customized_retrieval_based_offline_twhin_multi_cluster_follow_model_id",
default = ModelConfig.TwhinMultiClusterForFollow
)
object CustomizedRetrievalBasedTwhinMultiClusterEngagementSource
extends FSParam[String](
name = "customized_retrieval_based_offline_twhin_multi_cluster_engagement_model_id",
default = ModelConfig.TwhinMultiClusterForEngagement
)
val AllParams: Seq[Param[_] with FSName] =
Seq(
CustomizedRetrievalBasedTwhinCollabFilterFollowSource,
CustomizedRetrievalBasedTwhinCollabFilterEngagementSource,
CustomizedRetrievalBasedTwhinMultiClusterFollowSource,
CustomizedRetrievalBasedTwhinMultiClusterEngagementSource,
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\EarlybirdFrsBasedCandidateGenerationParams.scala
object EarlybirdFrsBasedCandidateGenerationParams
object CandidateGenerationEarlybirdSimilarityEngineType extends Enumeration
class SimilarityEngineType(rankingMode: EarlybirdSimilarityEngineType)
extends super.Val
import scala.language.implicitConversions
implicit def valueToEarlybirdRankingMode(x: Value): SimilarityEngineType =
x.asInstanceOf[SimilarityEngineType]
val EarlybirdRankingMode_RecencyBased: SimilarityEngineType = SimilarityEngineType(
EarlybirdSimilarityEngineType_RecencyBased)
val EarlybirdRankingMode_ModelBased: SimilarityEngineType = SimilarityEngineType(
EarlybirdSimilarityEngineType_ModelBased)
val EarlybirdRankingMode_TensorflowBased: SimilarityEngineType = SimilarityEngineType(
EarlybirdSimilarityEngineType_TensorflowBased)
}
object FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam
extends FSEnumParam[CandidateGenerationEarlybirdSimilarityEngineType.type](
name = "frs_based_candidate_generation_earlybird_ranking_mode_id",
default =
CandidateGenerationEarlybirdSimilarityEngineType.EarlybirdRankingMode_RecencyBased,
enum = CandidateGenerationEarlybirdSimilarityEngineType
)
object FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser
extends FSBoundedParam[Int](
name = "frs_based_candidate_generation_earlybird_max_tweets_per_user",
default = 100,
min = 0,
/**
* Note max should be equal to EarlybirdRecencyBasedCandidateStoreModule.DefaultMaxNumTweetPerUser.
* Which is the size of the memcached result list.
*/
max = 100
)
object FrsBasedCandidateGenerationEarlybirdMaxTweetAge
extends FSBoundedParam[Duration](
name = "frs_based_candidate_generation_earlybird_max_tweet_age_hours",
default = 24.hours,
min = 12.hours,
/**
* Note max could be related to EarlybirdRecencyBasedCandidateStoreModule.DefaultMaxNumTweetPerUser.
* Which is the size of the memcached result list for recency based earlybird candidate source.
* E.g. if max = 720.hours, we may want to increase the DefaultMaxNumTweetPerUser.
*/
max = 96.hours
)
with HasDurationConversion
object FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies
extends FSParam[Boolean](
name = "frs_based_candidate_generation_earlybird_filter_out_retweets_and_replies",
default = true
)
val AllParams: Seq[Param[_] with FSName] = Seq(
FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam,
FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser,
FrsBasedCandidateGenerationEarlybirdMaxTweetAge,
FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies,
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\FrsParams.scala
object FrsParams
object EnableSourceParam
extends FSParam[Boolean](
name = "signal_frs_enable_source",
default = false
)
object EnableSourceGraphParam
extends FSParam[Boolean](
name = "graph_frs_enable_source",
default = false
)
object MinScoreParam
extends FSBoundedParam[Double](
name = "signal_frs_min_score",
default = 0.4,
min = 0.0,
max = 1.0
)
object MaxConsumerSeedsNumParam
extends FSBoundedParam[Int](
name = "graph_frs_max_user_seeds_num",
default = 200,
min = 0,
max = 1000
)
/**
* These params below are only used for FrsTweetCandidateGenerator and shouldn't be used in other endpoints
* * FrsBasedCandidateGenerationMaxSeedsNumParam
* * FrsCandidateGenerationDisplayLocationParam
* * FrsCandidateGenerationDisplayLocation
* * FrsBasedCandidateGenerationMaxCandidatesNumParam
*/
object FrsBasedCandidateGenerationEnableVisibilityFilteringParam
extends FSParam[Boolean](
name = "frs_based_candidate_generation_enable_vf",
default = true
)
object FrsBasedCandidateGenerationMaxSeedsNumParam
extends FSBoundedParam[Int](
name = "frs_based_candidate_generation_max_seeds_num",
default = 100,
min = 0,
max = 800
)
object FrsBasedCandidateGenerationDisplayLocation extends Enumeration
class FrsDisplayLocationValue(displayLocation: DisplayLocation) extends super.Val
import scala.language.implicitConversions
implicit def valueToDisplayLocationValue(x: Value): FrsDisplayLocationValue =
x.asInstanceOf[FrsDisplayLocationValue]
val DisplayLocation_ContentRecommender: FrsDisplayLocationValue = FrsDisplayLocationValue(
DisplayLocation.ContentRecommender)
val DisplayLocation_Home: FrsDisplayLocationValue = FrsDisplayLocationValue(
DisplayLocation.HomeTimelineTweetRecs)
val DisplayLocation_Notifications: FrsDisplayLocationValue = FrsDisplayLocationValue(
DisplayLocation.TweetNotificationRecs)
}
object FrsBasedCandidateGenerationDisplayLocationParam
extends FSEnumParam[FrsBasedCandidateGenerationDisplayLocation.type](
name = "frs_based_candidate_generation_display_location_id",
default = FrsBasedCandidateGenerationDisplayLocation.DisplayLocation_Home,
enum = FrsBasedCandidateGenerationDisplayLocation
)
object FrsBasedCandidateGenerationMaxCandidatesNumParam
extends FSBoundedParam[Int](
name = "frs_based_candidate_generation_max_candidates_num",
default = 100,
min = 0,
max = 2000
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam,
EnableSourceGraphParam,
MinScoreParam,
MaxConsumerSeedsNumParam,
FrsBasedCandidateGenerationMaxSeedsNumParam,
FrsBasedCandidateGenerationDisplayLocationParam,
FrsBasedCandidateGenerationMaxCandidatesNumParam,
FrsBasedCandidateGenerationEnableVisibilityFilteringParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\GlobalParams.scala
object GlobalParams
object MaxCandidatesPerRequestParam
extends FSBoundedParam[Int](
name = "twistly_core_max_candidates_per_request",
default = 100,
min = 0,
max = 9000
)
object ModelVersionParam
extends FSEnumParam[ModelVersions.Enum.type](
name = "twistly_core_simclusters_model_version_id",
default = ModelVersions.Enum.Model20M145K2020,
enum = ModelVersions.Enum
)
object UnifiedMaxSourceKeyNum
extends FSBoundedParam[Int](
name = "twistly_core_unified_max_sourcekey_num",
default = 15,
min = 0,
max = 100
)
object MaxCandidateNumPerSourceKeyParam
extends FSBoundedParam[Int](
name = "twistly_core_candidate_per_sourcekey_max_num",
default = 200,
min = 0,
max = 1000
)
// 1 hours to 30 days
object MaxTweetAgeHoursParam
extends FSBoundedParam[Duration](
name = "twistly_core_max_tweet_age_hours",
default = 720.hours,
min = 1.hours,
max = 720.hours
)
with HasDurationConversion
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\GoodProfileClickParams.scala
object GoodProfileClickParams
object ClickMinDwellTimeParam extends Enumeration
class SignalTypeValue(signalType: SignalType) extends super.Val
import scala.language.implicitConversions
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue =
x.asInstanceOf[SignalTypeValue]
val TotalDwellTime10s = SignalTypeValue(SignalType.GoodProfileClick)
val TotalDwellTime20s = SignalTypeValue(SignalType.GoodProfileClick20s)
val TotalDwellTime30s = SignalTypeValue(SignalType.GoodProfileClick30s)
}
object EnableSourceParam
extends FSParam[Boolean](
name = "signal_good_profile_clicks_enable_source",
default = false
)
object ClickMinDwellTimeType
extends FSEnumParam[ClickMinDwellTimeParam.type](
name = "signal_good_profile_clicks_min_dwelltime_type_id",
default = ClickMinDwellTimeParam.TotalDwellTime10s,
enum = ClickMinDwellTimeParam
)
val AllParams: Seq[Param[_] with FSName] =
Seq(EnableSourceParam, ClickMinDwellTimeType)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\GoodTweetClickParams.scala
object GoodTweetClickParams
object ClickMinDwellTimeParam extends Enumeration
class SignalTypeValue(signalType: SignalType) extends super.Val
import scala.language.implicitConversions
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue =
x.asInstanceOf[SignalTypeValue]
val TotalDwellTime2s = SignalTypeValue(SignalType.GoodTweetClick)
val TotalDwellTime5s = SignalTypeValue(SignalType.GoodTweetClick5s)
val TotalDwellTime10s = SignalTypeValue(SignalType.GoodTweetClick10s)
val TotalDwellTime30s = SignalTypeValue(SignalType.GoodTweetClick30s)
}
object EnableSourceParam
extends FSParam[Boolean](
name = "signal_good_tweet_clicks_enable_source",
default = false
)
object ClickMinDwellTimeType
extends FSEnumParam[ClickMinDwellTimeParam.type](
name = "signal_good_tweet_clicks_min_dwelltime_type_id",
default = ClickMinDwellTimeParam.TotalDwellTime2s,
enum = ClickMinDwellTimeParam
)
object MaxSignalNumParam
extends FSBoundedParam[Int](
name = "signal_good_tweet_clicks_max_signal_num",
default = 15,
min = 0,
max = 15
)
val AllParams: Seq[Param[_] with FSName] =
Seq(EnableSourceParam, ClickMinDwellTimeType, MaxSignalNumParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\InterestedInParams.scala
object InterestedInParams
object SourceEmbedding extends Enumeration
class EmbeddingType(embeddingType: SimClustersEmbeddingType) extends super.Val
import scala.language.implicitConversions
implicit def valueToEmbeddingtype(x: Value): EmbeddingType = x.asInstanceOf[EmbeddingType]
val UserInterestedIn: Value = EmbeddingType(SimClustersEmbeddingType.FilteredUserInterestedIn)
val UnfilteredUserInterestedIn: Value = EmbeddingType(
SimClustersEmbeddingType.UnfilteredUserInterestedIn)
val FromProducerEmbedding: Value = EmbeddingType(
SimClustersEmbeddingType.FilteredUserInterestedInFromPE)
val LogFavBasedUserInterestedInFromAPE: Value = EmbeddingType(
SimClustersEmbeddingType.LogFavBasedUserInterestedInFromAPE)
val FollowBasedUserInterestedInFromAPE: Value = EmbeddingType(
SimClustersEmbeddingType.FollowBasedUserInterestedInFromAPE)
val UserNextInterestedIn: Value = EmbeddingType(SimClustersEmbeddingType.UserNextInterestedIn)
// AddressBook based InterestedIn
val LogFavBasedUserInterestedAverageAddressBookFromIIAPE: Value = EmbeddingType(
SimClustersEmbeddingType.LogFavBasedUserInterestedAverageAddressBookFromIIAPE)
val LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType(
SimClustersEmbeddingType.LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE)
val LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType(
SimClustersEmbeddingType.LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE)
val LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType(
SimClustersEmbeddingType.LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE)
val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType(
SimClustersEmbeddingType.LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE)
val LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType(
SimClustersEmbeddingType.LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE)
}
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_interestedin_enable_source",
default = true
)
object InterestedInEmbeddingIdParam
extends FSEnumParam[SourceEmbedding.type](
name = "twistly_interestedin_embedding_id",
default = SourceEmbedding.UnfilteredUserInterestedIn,
enum = SourceEmbedding
)
object MinScoreParam
extends FSBoundedParam[Double](
name = "twistly_interestedin_min_score",
default = 0.072,
min = 0.0,
max = 1.0
)
object EnableSourceSequentialModelParam
extends FSParam[Boolean](
name = "twistly_interestedin_sequential_model_enable_source",
default = false
)
object NextInterestedInEmbeddingIdParam
extends FSEnumParam[SourceEmbedding.type](
name = "twistly_interestedin_sequential_model_embedding_id",
default = SourceEmbedding.UserNextInterestedIn,
enum = SourceEmbedding
)
object MinScoreSequentialModelParam
extends FSBoundedParam[Double](
name = "twistly_interestedin_sequential_model_min_score",
default = 0.0,
min = 0.0,
max = 1.0
)
object EnableSourceAddressBookParam
extends FSParam[Boolean](
name = "twistly_interestedin_addressbook_enable_source",
default = false
)
object AddressBookInterestedInEmbeddingIdParam
extends FSEnumParam[SourceEmbedding.type](
name = "twistly_interestedin_addressbook_embedding_id",
default = SourceEmbedding.LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE,
enum = SourceEmbedding
)
object MinScoreAddressBookParam
extends FSBoundedParam[Double](
name = "twistly_interestedin_addressbook_min_score",
default = 0.0,
min = 0.0,
max = 1.0
)
// Prod SimClusters ANN param
// This is used to enable/disable querying of production SANN service. Useful when experimenting
// with replacements to it.
object EnableProdSimClustersANNParam
extends FSParam[Boolean](
name = "twistly_interestedin_enable_prod_simclusters_ann",
default = true
)
// Experimental SimClusters ANN params
object EnableExperimentalSimClustersANNParam
extends FSParam[Boolean](
name = "twistly_interestedin_enable_experimental_simclusters_ann",
default = false
)
// SimClusters ANN 1 cluster params
object EnableSimClustersANN1Param
extends FSParam[Boolean](
name = "twistly_interestedin_enable_simclusters_ann_1",
default = false
)
// SimClusters ANN 2 cluster params
object EnableSimClustersANN2Param
extends FSParam[Boolean](
name = "twistly_interestedin_enable_simclusters_ann_2",
default = false
)
// SimClusters ANN 3 cluster params
object EnableSimClustersANN3Param
extends FSParam[Boolean](
name = "twistly_interestedin_enable_simclusters_ann_3",
default = false
)
// SimClusters ANN 5 cluster params
object EnableSimClustersANN5Param
extends FSParam[Boolean](
name = "twistly_interestedin_enable_simclusters_ann_5",
default = false
)
// SimClusters ANN 4 cluster params
object EnableSimClustersANN4Param
extends FSParam[Boolean](
name = "twistly_interestedin_enable_simclusters_ann_4",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam,
EnableSourceSequentialModelParam,
EnableSourceAddressBookParam,
EnableProdSimClustersANNParam,
EnableExperimentalSimClustersANNParam,
EnableSimClustersANN1Param,
EnableSimClustersANN2Param,
EnableSimClustersANN3Param,
EnableSimClustersANN5Param,
EnableSimClustersANN4Param,
MinScoreParam,
MinScoreSequentialModelParam,
MinScoreAddressBookParam,
InterestedInEmbeddingIdParam,
NextInterestedInEmbeddingIdParam,
AddressBookInterestedInEmbeddingIdParam,
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ProducerBasedCandidateGenerationParams.scala
object ProducerBasedCandidateGenerationParams
object EnableSourceParam
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_source",
default = false
)
object UtgCombinationMethodParam
extends FSEnumParam[UnifiedSETweetCombinationMethod.type](
name = "producer_based_candidate_generation_utg_combination_method_id",
default = UnifiedSETweetCombinationMethod.Frontload,
enum = UnifiedSETweetCombinationMethod
)
// UTG params
object EnableUTGParam
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_utg",
default = false
)
object EnableUAGParam
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_uag",
default = false
)
// SimClusters params
object EnableSimClustersANNParam
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_simclusters",
default = true
)
// Filter params
object SimClustersMinScoreParam
extends FSBoundedParam[Double](
name = "producer_based_candidate_generation_filter_simclusters_min_score",
default = 0.7,
min = 0.0,
max = 1.0
)
// Experimental SimClusters ANN params
object EnableExperimentalSimClustersANNParam
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_experimental_simclusters_ann",
default = false
)
// SimClusters ANN cluster 1 params
object EnableSimClustersANN1Param
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_simclusters_ann_1",
default = false
)
// SimClusters ANN cluster 2 params
object EnableSimClustersANN2Param
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_simclusters_ann_2",
default = false
)
// SimClusters ANN cluster 3 params
object EnableSimClustersANN3Param
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_simclusters_ann_3",
default = false
)
// SimClusters ANN cluster 5 params
object EnableSimClustersANN5Param
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_simclusters_ann_5",
default = false
)
object EnableSimClustersANN4Param
extends FSParam[Boolean](
name = "producer_based_candidate_generation_enable_simclusters_ann_4",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam,
EnableUAGParam,
EnableUTGParam,
EnableSimClustersANNParam,
EnableSimClustersANN1Param,
EnableSimClustersANN2Param,
EnableSimClustersANN3Param,
EnableSimClustersANN5Param,
EnableSimClustersANN4Param,
EnableExperimentalSimClustersANNParam,
SimClustersMinScoreParam,
UtgCombinationMethodParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ProducerBasedUserAdGraphParams.scala
object ProducerBasedUserAdGraphParams
object MinCoOccurrenceParam
extends FSBoundedParam[Int](
name = "producer_based_user_ad_graph_min_co_occurrence",
default = 2,
min = 0,
max = 500
)
object MinScoreParam
extends FSBoundedParam[Double](
name = "producer_based_user_ad_graph_min_score",
default = 3.0,
min = 0.0,
max = 10.0
)
object MaxNumFollowersParam
extends FSBoundedParam[Int](
name = "producer_based_user_ad_graph_max_num_followers",
default = 500,
min = 100,
max = 1000
)
val AllParams: Seq[Param[_] with FSName] =
Seq(MinCoOccurrenceParam, MaxNumFollowersParam, MinScoreParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ProducerBasedUserTweetGraphParams.scala
object ProducerBasedUserTweetGraphParams
object MinCoOccurrenceParam
extends FSBoundedParam[Int](
name = "producer_based_user_tweet_graph_min_co_occurrence",
default = 4,
min = 0,
max = 500
)
object MinScoreParam
extends FSBoundedParam[Double](
name = "producer_based_user_tweet_graph_min_score",
default = 3.0,
min = 0.0,
max = 10.0
)
object MaxNumFollowersParam
extends FSBoundedParam[Int](
name = "producer_based_user_tweet_graph_max_num_followers",
default = 500,
min = 100,
max = 1000
)
val AllParams: Seq[Param[_] with FSName] =
Seq(MinCoOccurrenceParam, MaxNumFollowersParam, MinScoreParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RankerParams.scala
object RankerParams
object MaxCandidatesToRank
extends FSBoundedParam[Int](
name = "twistly_core_max_candidates_to_rank",
default = 2000,
min = 0,
max = 9999
)
object EnableBlueVerifiedTopK
extends FSParam[Boolean](
name = "twistly_core_blue_verified_top_k",
default = true
)
val AllParams: Seq[Param[_] with FSName] = Seq(
MaxCandidatesToRank,
EnableBlueVerifiedTopK
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RealGraphInParams.scala
object RealGraphInParams
object EnableSourceGraphParam
extends FSParam[Boolean](
name = "graph_realgraphin_enable_source",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceGraphParam,
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RealGraphOonParams.scala
object RealGraphOonParams
object EnableSourceParam
extends FSParam[Boolean](
name = "signal_realgraphoon_enable_source",
default = false
)
object EnableSourceGraphParam
extends FSParam[Boolean](
name = "graph_realgraphoon_enable_source",
default = false
)
object MaxConsumerSeedsNumParam
extends FSBoundedParam[Int](
name = "graph_realgraphoon_max_user_seeds_num",
default = 200,
min = 0,
max = 1000
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam,
EnableSourceGraphParam,
MaxConsumerSeedsNumParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentFollowsParams.scala
object RecentFollowsParams
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_recentfollows_enable_source",
default = true
)
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentNegativeSignalParams.scala
object RecentNegativeSignalParams
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_recentnegativesignals_enable_source",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentNotificationsParams.scala
object RecentNotificationsParams
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_recentnotifications_enable_source",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentOriginalTweetsParams.scala
object RecentOriginalTweetsParams
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_recentoriginaltweets_enable_source",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentReplyTweetsParams.scala
object RecentReplyTweetsParams
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_recentreplytweets_enable_source",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentRetweetsParams.scala
object RecentRetweetsParams
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_recentretweets_enable_source",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentTweetFavoritesParams.scala
object RecentTweetFavoritesParams
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_recenttweetfavorites_enable_source",
default = true
)
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedTweetGlobalParams.scala
object RelatedTweetGlobalParams
object MaxCandidatesPerRequestParam
extends FSBoundedParam[Int](
name = "related_tweet_core_max_candidates_per_request",
default = 100,
min = 0,
max = 500
)
val AllParams: Seq[Param[_] with FSName] = Seq(MaxCandidatesPerRequestParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedTweetProducerBasedParams.scala
object RelatedTweetProducerBasedParams
object EnableUTGParam
extends FSParam[Boolean](
name = "related_tweet_producer_based_enable_utg",
default = false
)
// SimClusters params
object EnableSimClustersANNParam
extends FSParam[Boolean](
name = "related_tweet_producer_based_enable_simclusters",
default = true
)
// Filter params
object SimClustersMinScoreParam
extends FSBoundedParam[Double](
name = "related_tweet_producer_based_filter_simclusters_min_score",
default = 0.0,
min = 0.0,
max = 1.0
)
// Experimental SimClusters ANN params
object EnableExperimentalSimClustersANNParam
extends FSParam[Boolean](
name = "related_tweet_producer_based_enable_experimental_simclusters_ann",
default = false
)
// SimClusters ANN cluster 1 params
object EnableSimClustersANN1Param
extends FSParam[Boolean](
name = "related_tweet_producer_based_enable_simclusters_ann_1",
default = false
)
// SimClusters ANN cluster 2 params
object EnableSimClustersANN2Param
extends FSParam[Boolean](
name = "related_tweet_producer_based_enable_simclusters_ann_2",
default = false
)
// SimClusters ANN cluster 3 params
object EnableSimClustersANN3Param
extends FSParam[Boolean](
name = "related_tweet_producer_based_enable_simclusters_ann_3",
default = false
)
// SimClusters ANN cluster 3 params
object EnableSimClustersANN5Param
extends FSParam[Boolean](
name = "related_tweet_producer_based_enable_simclusters_ann_5",
default = false
)
// SimClusters ANN cluster 4 params
object EnableSimClustersANN4Param
extends FSParam[Boolean](
name = "related_tweet_producer_based_enable_simclusters_ann_4",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableUTGParam,
EnableSimClustersANNParam,
EnableSimClustersANN1Param,
EnableSimClustersANN2Param,
EnableSimClustersANN3Param,
EnableSimClustersANN5Param,
EnableSimClustersANN4Param,
EnableExperimentalSimClustersANNParam,
SimClustersMinScoreParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedTweetTweetBasedParams.scala
object RelatedTweetTweetBasedParams
object EnableUTGParam
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_utg",
default = false
)
// UVG params
object EnableUVGParam
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_uvg",
default = false
)
// UAG params
object EnableUAGParam
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_uag",
default = false
)
// SimClusters params
object EnableSimClustersANNParam
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_simclusters",
default = true
)
// Experimental SimClusters ANN params
object EnableExperimentalSimClustersANNParam
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_experimental_simclusters_ann",
default = false
)
// SimClusters ANN cluster 1 params
object EnableSimClustersANN1Param
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_simclusters_ann_1",
default = false
)
// SimClusters ANN cluster 2 params
object EnableSimClustersANN2Param
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_simclusters_ann_2",
default = false
)
// SimClusters ANN cluster 3 params
object EnableSimClustersANN3Param
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_simclusters_ann_3",
default = false
)
// SimClusters ANN cluster 5 params
object EnableSimClustersANN5Param
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_simclusters_ann_5",
default = false
)
object EnableSimClustersANN4Param
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_simclusters_ann_4",
default = false
)
// TwHIN params
object EnableTwHINParam
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_twhin",
default = false
)
// QIG params
object EnableQigSimilarTweetsParam
extends FSParam[Boolean](
name = "related_tweet_tweet_based_enable_qig_similar_tweets",
default = false
)
// Filter params
object SimClustersMinScoreParam
extends FSBoundedParam[Double](
name = "related_tweet_tweet_based_filter_simclusters_min_score",
default = 0.3,
min = 0.0,
max = 1.0
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableTwHINParam,
EnableQigSimilarTweetsParam,
EnableUTGParam,
EnableUVGParam,
EnableSimClustersANNParam,
EnableSimClustersANN2Param,
EnableSimClustersANN3Param,
EnableSimClustersANN5Param,
EnableSimClustersANN4Param,
EnableExperimentalSimClustersANNParam,
SimClustersMinScoreParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedVideoTweetGlobalParams.scala
object RelatedVideoTweetGlobalParams
object MaxCandidatesPerRequestParam
extends FSBoundedParam[Int](
name = "related_video_tweet_core_max_candidates_per_request",
default = 100,
min = 0,
max = 500
)
val AllParams: Seq[Param[_] with FSName] = Seq(MaxCandidatesPerRequestParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedVideoTweetTweetBasedParams.scala
object RelatedVideoTweetTweetBasedParams
object EnableUTGParam
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_utg",
default = false
)
// SimClusters params
object EnableSimClustersANNParam
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_simclusters",
default = true
)
// Experimental SimClusters ANN params
object EnableExperimentalSimClustersANNParam
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_experimental_simclusters_ann",
default = false
)
// SimClusters ANN cluster 1 params
object EnableSimClustersANN1Param
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_simclusters_ann_1",
default = false
)
// SimClusters ANN cluster 2 params
object EnableSimClustersANN2Param
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_simclusters_ann_2",
default = false
)
// SimClusters ANN cluster 3 params
object EnableSimClustersANN3Param
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_simclusters_ann_3",
default = false
)
// SimClusters ANN cluster 5 params
object EnableSimClustersANN5Param
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_simclusters_ann_5",
default = false
)
// SimClusters ANN cluster 4 params
object EnableSimClustersANN4Param
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_simclusters_ann_4",
default = false
)
// TwHIN params
object EnableTwHINParam
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_twhin",
default = false
)
// QIG params
object EnableQigSimilarTweetsParam
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_qig_similar_tweets",
default = false
)
// Filter params
object SimClustersMinScoreParam
extends FSBoundedParam[Double](
name = "related_video_tweet_tweet_based_filter_simclusters_min_score",
default = 0.3,
min = 0.0,
max = 1.0
)
object EnableUVGParam
extends FSParam[Boolean](
name = "related_video_tweet_tweet_based_enable_uvg",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableTwHINParam,
EnableQigSimilarTweetsParam,
EnableUTGParam,
EnableUVGParam,
EnableSimClustersANNParam,
EnableSimClustersANN2Param,
EnableSimClustersANN3Param,
EnableSimClustersANN5Param,
EnableSimClustersANN4Param,
EnableExperimentalSimClustersANNParam,
SimClustersMinScoreParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RepeatedProfileVisitsParams.scala
object RepeatedProfileVisitsParams
object ProfileMinVisitParam extends Enumeration
class SignalTypeValue(signalType: SignalType) extends super.Val
import scala.language.implicitConversions
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue =
x.asInstanceOf[SignalTypeValue]
val TotalVisitsInPast180Days = SignalTypeValue(SignalType.RepeatedProfileVisit180dMinVisit6V1)
val TotalVisitsInPast90Days = SignalTypeValue(SignalType.RepeatedProfileVisit90dMinVisit6V1)
val TotalVisitsInPast14Days = SignalTypeValue(SignalType.RepeatedProfileVisit14dMinVisit2V1)
val TotalVisitsInPast180DaysNoNegative = SignalTypeValue(
SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative)
val TotalVisitsInPast90DaysNoNegative = SignalTypeValue(
SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative)
val TotalVisitsInPast14DaysNoNegative = SignalTypeValue(
SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative)
}
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_repeatedprofilevisits_enable_source",
default = true
)
object MinScoreParam
extends FSBoundedParam[Double](
name = "twistly_repeatedprofilevisits_min_score",
default = 0.5,
min = 0.0,
max = 1.0
)
object ProfileMinVisitType
extends FSEnumParam[ProfileMinVisitParam.type](
name = "twistly_repeatedprofilevisits_min_visit_type_id",
default = ProfileMinVisitParam.TotalVisitsInPast14Days,
enum = ProfileMinVisitParam
)
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam, ProfileMinVisitType)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\SimClustersANNParams.scala
object SimClustersANNParams
object SimClustersANNConfigId
extends FSParam[String](
name = "similarity_simclusters_ann_simclusters_ann_config_id",
default = "Default"
)
object SimClustersANN1ConfigId
extends FSParam[String](
name = "similarity_simclusters_ann_simclusters_ann_1_config_id",
default = "20220810"
)
object SimClustersANN2ConfigId
extends FSParam[String](
name = "similarity_simclusters_ann_simclusters_ann_2_config_id",
default = "20220818"
)
object SimClustersANN3ConfigId
extends FSParam[String](
name = "similarity_simclusters_ann_simclusters_ann_3_config_id",
default = "20220819"
)
object SimClustersANN5ConfigId
extends FSParam[String](
name = "similarity_simclusters_ann_simclusters_ann_5_config_id",
default = "20221221"
)
object SimClustersANN4ConfigId
extends FSParam[String](
name = "similarity_simclusters_ann_simclusters_ann_4_config_id",
default = "20221220"
)
object ExperimentalSimClustersANNConfigId
extends FSParam[String](
name = "similarity_simclusters_ann_experimental_simclusters_ann_config_id",
default = "20220801"
)
val AllParams: Seq[Param[_] with FSName] = Seq(
SimClustersANNConfigId,
SimClustersANN1ConfigId,
SimClustersANN2ConfigId,
SimClustersANN3ConfigId,
SimClustersANN5ConfigId,
ExperimentalSimClustersANNConfigId
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TopicTweetParams.scala
object TopicTweetParams
object MaxTweetAge
extends FSBoundedParam[Duration](
name = "topic_tweet_candidate_generation_max_tweet_age_hours",
default = 24.hours,
min = 12.hours,
max = 48.hours
)
with HasDurationConversion
object MaxTopicTweetCandidatesParam
extends FSBoundedParam[Int](
name = "topic_tweet_max_candidates_num",
default = 200,
min = 0,
max = 1000
)
object MaxSkitTfgCandidatesParam
extends FSBoundedParam[Int](
name = "topic_tweet_skit_tfg_max_candidates_num",
default = 100,
min = 0,
max = 1000
)
object MaxSkitHighPrecisionCandidatesParam
extends FSBoundedParam[Int](
name = "topic_tweet_skit_high_precision_max_candidates_num",
default = 100,
min = 0,
max = 1000
)
object MaxCertoCandidatesParam
extends FSBoundedParam[Int](
name = "topic_tweet_certo_max_candidates_num",
default = 100,
min = 0,
max = 1000
)
// The min prod score for Certo L2-normalized cosine candidates
object CertoScoreThresholdParam
extends FSBoundedParam[Double](
name = "topic_tweet_certo_score_threshold",
default = 0.015,
min = 0,
max = 1
)
object SemanticCoreVersionIdParam
extends FSParam[Long](
name = "semantic_core_version_id",
default = 1380520918896713735L
)
val AllParams: Seq[Param[_] with FSName] = Seq(
CertoScoreThresholdParam,
MaxTopicTweetCandidatesParam,
MaxTweetAge,
MaxCertoCandidatesParam,
MaxSkitTfgCandidatesParam,
MaxSkitHighPrecisionCandidatesParam,
SemanticCoreVersionIdParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedCandidateGenerationParams.scala
object TweetBasedCandidateGenerationParams
object EnableSourceParam
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_source",
default = false
)
// UTG params
object EnableUTGParam
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_utg",
default = true
)
// SimClusters params
object EnableSimClustersANNParam
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_simclusters",
default = true
)
// Experimental SimClusters ANN params
object EnableExperimentalSimClustersANNParam
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_experimental_simclusters_ann",
default = false
)
// SimClusters ANN cluster 1 params
object EnableSimClustersANN1Param
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_simclusters_ann_1",
default = false
)
// SimClusters ANN cluster 2 params
object EnableSimClustersANN2Param
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_simclusters_ann_2",
default = false
)
// SimClusters ANN cluster 3 params
object EnableSimClustersANN3Param
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_simclusters_ann_3",
default = false
)
// SimClusters ANN cluster 3 params
object EnableSimClustersANN5Param
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_simclusters_ann_5",
default = false
)
// SimClusters ANN cluster 4 params
object EnableSimClustersANN4Param
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_simclusters_ann_4",
default = false
)
// TwHIN params
object EnableTwHINParam
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_twhin",
default = false
)
// QIG params
object EnableQigSimilarTweetsParam
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_qig_similar_tweets",
default = false
)
object QigMaxNumSimilarTweetsParam
extends FSBoundedParam[Int](
name = "tweet_based_candidate_generation_qig_max_num_similar_tweets",
default = 100,
min = 10,
max = 100
)
// UVG params
object EnableUVGParam
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_uvg",
default = false
)
// UAG params
object EnableUAGParam
extends FSParam[Boolean](
name = "tweet_based_candidate_generation_enable_uag",
default = false
)
// Filter params
object SimClustersMinScoreParam
extends FSBoundedParam[Double](
name = "tweet_based_candidate_generation_filter_simclusters_min_score",
default = 0.5,
min = 0.0,
max = 1.0
)
// for learning DDG that has a higher threshold for video based SANN
object SimClustersVideoBasedMinScoreParam
extends FSBoundedParam[Double](
name = "tweet_based_candidate_generation_filter_simclusters_video_based_min_score",
default = 0.5,
min = 0.0,
max = 1.0
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableSourceParam,
EnableTwHINParam,
EnableQigSimilarTweetsParam,
EnableUTGParam,
EnableUVGParam,
EnableUAGParam,
EnableSimClustersANNParam,
EnableSimClustersANN1Param,
EnableSimClustersANN2Param,
EnableSimClustersANN3Param,
EnableSimClustersANN5Param,
EnableSimClustersANN4Param,
EnableExperimentalSimClustersANNParam,
SimClustersMinScoreParam,
SimClustersVideoBasedMinScoreParam,
QigMaxNumSimilarTweetsParam,
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedTwHINParams.scala
object TweetBasedTwHINParams
object ModelIdParam
extends FSParam[String](
name = "tweet_based_twhin_model_id",
default = ModelConfig.TweetBasedTwHINRegularUpdateAll20221024,
)
val AllParams: Seq[Param[_] with FSName] = Seq(ModelIdParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedUserAdGraphParams.scala
object TweetBasedUserAdGraphParams
object MinCoOccurrenceParam
extends FSBoundedParam[Int](
name = "tweet_based_user_ad_graph_min_co_occurrence",
default = 1,
min = 0,
max = 500
)
object ConsumersBasedMinScoreParam
extends FSBoundedParam[Double](
name = "tweet_based_user_ad_graph_consumers_based_min_score",
default = 0.0,
min = 0.0,
max = 10.0
)
object MaxConsumerSeedsNumParam
extends FSBoundedParam[Int](
name = "tweet_based_user_ad_graph_max_user_seeds_num",
default = 100,
min = 0,
max = 300
)
val AllParams: Seq[Param[_] with FSName] = Seq(
MinCoOccurrenceParam,
MaxConsumerSeedsNumParam,
ConsumersBasedMinScoreParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedUserTweetGraphParams.scala
object TweetBasedUserTweetGraphParams
object MinCoOccurrenceParam
extends FSBoundedParam[Int](
name = "tweet_based_user_tweet_graph_min_co_occurrence",
default = 3,
min = 0,
max = 500
)
object TweetBasedMinScoreParam
extends FSBoundedParam[Double](
name = "tweet_based_user_tweet_graph_tweet_based_min_score",
default = 0.5,
min = 0.0,
max = 10.0
)
object ConsumersBasedMinScoreParam
extends FSBoundedParam[Double](
name = "tweet_based_user_tweet_graph_consumers_based_min_score",
default = 4.0,
min = 0.0,
max = 10.0
)
object MaxConsumerSeedsNumParam
extends FSBoundedParam[Int](
name = "tweet_based_user_tweet_graph_max_user_seeds_num",
default = 100,
min = 0,
max = 300
)
object EnableCoverageExpansionOldTweetParam
extends FSParam[Boolean](
name = "tweet_based_user_tweet_graph_enable_coverage_expansion_old_tweet",
default = false
)
object EnableCoverageExpansionAllTweetParam
extends FSParam[Boolean](
name = "tweet_based_user_tweet_graph_enable_coverage_expansion_all_tweet",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableCoverageExpansionAllTweetParam,
EnableCoverageExpansionOldTweetParam,
MinCoOccurrenceParam,
MaxConsumerSeedsNumParam,
TweetBasedMinScoreParam,
ConsumersBasedMinScoreParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedUserVideoGraphParams.scala
object TweetBasedUserVideoGraphParams
object MinCoOccurrenceParam
extends FSBoundedParam[Int](
name = "tweet_based_user_video_graph_min_co_occurrence",
default = 5,
min = 0,
max = 500
)
object TweetBasedMinScoreParam
extends FSBoundedParam[Double](
name = "tweet_based_user_video_graph_tweet_based_min_score",
default = 0.0,
min = 0.0,
max = 100.0
)
object ConsumersBasedMinScoreParam
extends FSBoundedParam[Double](
name = "tweet_based_user_video_graph_consumers_based_min_score",
default = 4.0,
min = 0.0,
max = 10.0
)
object MaxConsumerSeedsNumParam
extends FSBoundedParam[Int](
name = "tweet_based_user_video_graph_max_user_seeds_num",
default = 200,
min = 0,
max = 500
)
object EnableCoverageExpansionOldTweetParam
extends FSParam[Boolean](
name = "tweet_based_user_video_graph_enable_coverage_expansion_old_tweet",
default = false
)
object EnableCoverageExpansionAllTweetParam
extends FSParam[Boolean](
name = "tweet_based_user_video_graph_enable_coverage_expansion_all_tweet",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
MinCoOccurrenceParam,
MaxConsumerSeedsNumParam,
TweetBasedMinScoreParam,
EnableCoverageExpansionOldTweetParam,
EnableCoverageExpansionAllTweetParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetSharesParams.scala
object TweetSharesParams
object EnableSourceParam
extends FSParam[Boolean](
name = "twistly_tweetshares_enable_source",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\UnifiedSETweetCombinationMethod.scala
object UnifiedSETweetCombinationMethod extends Enumeration
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\UnifiedUSSSignalParams.scala
object UnifiedUSSSignalParams
object TweetAggregationTypeParam extends Enumeration
class SignalTypeValue(signalType: SignalType) extends super.Val
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue =
x.asInstanceOf[SignalTypeValue]
val UniformAggregation = SignalTypeValue(SignalType.TweetBasedUnifiedUniformSignal)
val EngagementAggregation = SignalTypeValue(
SignalType.TweetBasedUnifiedEngagementWeightedSignal)
}
object ProducerAggregationTypeParam extends Enumeration
class SignalTypeValue(signalType: SignalType) extends super.Val
import scala.language.implicitConversions
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue =
x.asInstanceOf[SignalTypeValue]
val UniformAggregation = SignalTypeValue(SignalType.ProducerBasedUnifiedUniformSignal)
val EngagementAggregation = SignalTypeValue(
SignalType.ProducerBasedUnifiedEngagementWeightedSignal)
}
object ReplaceIndividualUSSSourcesParam
extends FSParam[Boolean](
name = "twistly_agg_replace_enable_source",
default = false
)
object EnableTweetAggSourceParam
extends FSParam[Boolean](
name = "twistly_agg_tweet_agg_enable_source",
default = false
)
object TweetAggTypeParam
extends FSEnumParam[TweetAggregationTypeParam.type](
name = "twistly_agg_tweet_agg_type_id",
default = TweetAggregationTypeParam.EngagementAggregation,
enum = TweetAggregationTypeParam
)
object UnifiedTweetSourceNumberParam
extends FSBoundedParam[Int](
name = "twistly_agg_tweet_agg_source_number",
default = 0,
min = 0,
max = 100,
)
object EnableProducerAggSourceParam
extends FSParam[Boolean](
name = "twistly_agg_producer_agg_enable_source",
default = false
)
object ProducerAggTypeParam
extends FSEnumParam[ProducerAggregationTypeParam.type](
name = "twistly_agg_producer_agg_type_id",
default = ProducerAggregationTypeParam.EngagementAggregation,
enum = ProducerAggregationTypeParam
)
object UnifiedProducerSourceNumberParam
extends FSBoundedParam[Int](
name = "twistly_agg_producer_agg_source_number",
default = 0,
min = 0,
max = 100,
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableTweetAggSourceParam,
EnableProducerAggSourceParam,
TweetAggTypeParam,
ProducerAggTypeParam,
UnifiedTweetSourceNumberParam,
UnifiedProducerSourceNumberParam,
ReplaceIndividualUSSSourcesParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\UtegTweetGlobalParams.scala
object UtegTweetGlobalParams
object MaxUtegCandidatesToRequestParam
extends FSBoundedParam[Int](
name = "max_uteg_candidates_to_request",
default = 800,
min = 10,
max = 200
)
object CandidateRefreshSinceTimeOffsetHoursParam
extends FSBoundedParam[Duration](
name = "candidate_refresh_since_time_offset_hours",
default = 48.hours,
min = 1.hours,
max = 96.hours
)
with HasDurationConversion
object EnableTLRHealthFilterParam
extends FSParam[Boolean](
name = "enable_uteg_tlr_health_filter",
default = true
)
object EnableRepliesToNonFollowedUsersFilterParam
extends FSParam[Boolean](
name = "enable_uteg_replies_to_non_followed_users_filter",
default = false
)
object EnableRetweetFilterParam
extends FSParam[Boolean](
name = "enable_uteg_retweet_filter",
default = true
)
object EnableInNetworkFilterParam
extends FSParam[Boolean](
name = "enable_uteg_in_network_filter",
default = true
)
val AllParams: Seq[Param[_] with FSName] =
Seq(
MaxUtegCandidatesToRequestParam,
CandidateRefreshSinceTimeOffsetHoursParam,
EnableTLRHealthFilterParam,
EnableRepliesToNonFollowedUsersFilterParam,
EnableRetweetFilterParam,
EnableInNetworkFilterParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\VideoTweetFilterParams.scala
object VideoTweetFilterParams
object EnableVideoTweetFilterParam
extends FSParam[Boolean](
name = "video_tweet_filter_enable_filter",
default = false
)
val AllParams: Seq[Param[_] with FSName] = Seq(
EnableVideoTweetFilterParam
)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\VideoViewTweetsParams.scala
object VideoViewTweetsParams
object EnableSourceParam
extends FSParam[Boolean](
name = "signal_videoviewtweets_enable_source",
default = false
)
object EnableSourceImpressionParam
extends FSParam[Boolean](
name = "signal_videoviewtweets_enableimpression_source",
default = false
)
object VideoViewTweetType extends Enumeration
class SignalTypeValue(signalType: SignalType) extends super.Val
import scala.language.implicitConversions
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue =
x.asInstanceOf[SignalTypeValue]
val VideoTweetQualityView: SignalTypeValue = SignalTypeValue(SignalType.VideoView90dQualityV1)
val VideoTweetPlayback50: SignalTypeValue = SignalTypeValue(SignalType.VideoView90dPlayback50V1)
}
object VideoViewTweetTypeParam
extends FSEnumParam[VideoViewTweetType.type](
name = "signal_videoviewtweets_videoviewtype_id",
default = VideoViewTweetType.VideoTweetQualityView,
enum = VideoViewTweetType
)
val AllParams: Seq[Param[_] with FSName] =
Seq(EnableSourceParam, EnableSourceImpressionParam, VideoViewTweetTypeParam)
lazy val config: BaseConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\decider\CrMixerDecider.scala
class CrMixerDecider @Inject() (decider: Decider)
def isAvailable(feature: String, recipient: Option[Recipient]): Boolean
def isAvailable(feature: String, useRandomRecipient: Boolean = true): Boolean
def isAvailableForId(
id: Long,
deciderConstants: String
): Boolean
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\decider\DeciderKey.scala
object DeciderConstants
object DeciderKey extends DeciderKeyEnum
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\decider\EndpointLoadShedder.scala
class EndpointLoadShedder @Inject() (
decider: Decider,
statsReceiver: StatsReceiver)
def apply[T](endpointName: String, product: String)(serve: => Future[T]): Future[T]
object EndpointLoadShedder
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\ranker\DefaultRanker.scala
class DefaultRanker()
def rank(
candidates: Seq[BlendedCandidate],
): Future[Seq[RankedCandidate]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\ranker\SwitchRanker.scala
class SwitchRanker @Inject() (
defaultRanker: DefaultRanker,
globalStats: StatsReceiver)
def rank(
query: CrCandidateGeneratorQuery,
candidates: Seq[BlendedCandidate],
): Future[Seq[RankedCandidate]]
object SwitchRanker
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\scribe\ScribeCategory.scala
object ScribeCategories
class ScribeCategory(
loggerFactoryNode: String,
scribeCategory: String)
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\service\CrMixerAlertNotificationConfig.scala
object CrMixerAlertNotificationConfig
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\CertoTopicTweetSimilarityEngine.scala
class CertoTopicTweetSimilarityEngine @Inject() (
@Named(ModuleNames.CertoStratoStoreName) certoStratoStore: ReadableStore[
TopicId,
Seq[TweetWithScores]
],
statsReceiver: StatsReceiver)
extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]]
def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]]
object CertoTopicTweetSimilarityEngine
class Query(
topicId: TopicId,
maxCandidates: Int,
certoScoreTheshold: Double)
def fromParams(
topicId: TopicId,
isVideoOnly: Boolean,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumerBasedWalsSimilarityEngine.scala
object maintain a set of stats that are specific to the Wals Engine.
case class WalsStats(scope: String, scopedStats: StatsReceiver)
def onFailure(t: Throwable, startTimeMs: Long)
def onSuccess(startTimeMs: Long)
object and can be expensive
// in performance critical paths.
object WalsStatsMap
def get(scope: String, scopedStats: StatsReceiver): WalsStats
class ConsumerBasedWalsSimilarityEngine(
homeNaviGRPCClient: ManagedChannel,
adsFavedNaviGRPCClient: ManagedChannel,
adsMonetizableNaviGRPCClient: ManagedChannel,
statsReceiver: StatsReceiver)
extends ReadableStore[
Query,
Seq[TweetWithScore]
]
def get(
query: ConsumerBasedWalsSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
def getFeaturesForRecommendations(query: ConsumerBasedWalsSimilarityEngine.Query): Example
def getModelInput(query: ConsumerBasedWalsSimilarityEngine.Query): PredictRequest
def getModelOutput(query: Query, response: PredictResponse): Seq[TweetWithScore]
object ConsumerBasedWalsSimilarityEngine
class Query(
sourceIds: Seq[SourceInfo],
modelName: String,
modelInputName: String,
modelOutputName: String,
modelSignatureName: String,
wilyNsName: String,
)
def fromParams(
sourceIds: Seq[SourceInfo],
params: configapi.Params,
): EngineQuery[Query]
def toSimilarityEngineInfo(
score: Double
): SimilarityEngineInfo
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumerEmbeddingBasedTripSimilarityEngine.scala
class TripEngineQuery(
modelId: String,
sourceId: InternalId,
tripSourceId: String,
maxResult: Int,
params: Params)
case class ConsumerEmbeddingBasedTripSimilarityEngine(
embeddingStoreLookUpMap: Map[String, ReadableStore[UserId, SimClustersEmbedding]],
tripCandidateSource: ReadableStore[TripDomain, Seq[TripTweet]],
statsReceiver: StatsReceiver,
) extends ReadableStore[TripEngineQuery, Seq[TripTweetWithScore]]
def fetchTopClusters(query: TripEngineQuery): Future[Option[Seq[ClusterId]]]
def fetchCandidates(
topClusters: Seq[ClusterId],
tripSourceId: String
): Future[Seq[Seq[TripTweetWithScore]]]
def get(engineQuery: TripEngineQuery): Future[Option[Seq[TripTweetWithScore]]]
object ConsumerEmbeddingBasedTripSimilarityEngine
def fromParams(
modelId: String,
sourceId: InternalId,
params: configapi.Params
): TripEngineQuery
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumerEmbeddingBasedTwHINSimilarityEngine.scala
object ConsumerEmbeddingBasedTwHINSimilarityEngine
def fromParams(
sourceId: InternalId,
params: configapi.Params,
): HnswANNEngineQuery
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumerEmbeddingBasedTwoTowerSimilarityEngine.scala
object ConsumerEmbeddingBasedTwoTowerSimilarityEngine
def fromParams(
sourceId: InternalId,
params: configapi.Params,
): HnswANNEngineQuery
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumersBasedUserAdGraphSimilarityEngine.scala
class ConsumersBasedUserAdGraphSimilarityEngine(
consumersBasedUserAdGraphStore: ReadableStore[
ConsumersBasedRelatedAdRequest,
RelatedAdResponse
],
statsReceiver: StatsReceiver)
extends ReadableStore[
ConsumersBasedUserAdGraphSimilarityEngine.Query,
Seq[TweetWithScore]
]
def get(
query: ConsumersBasedUserAdGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object ConsumersBasedUserAdGraphSimilarityEngine
class Query(
seedWithScores: Map[UserId, Double],
maxResults: Int,
minCooccurrence: Int,
minScore: Double,
maxTweetAgeInHours: Int)
def toSimilarityEngineInfo(
score: Double
): SimilarityEngineInfo
def fromParams(
seedWithScores: Map[UserId, Double],
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumersBasedUserVideoGraphSimilarityEngine.scala
class ConsumersBasedUserVideoGraphSimilarityEngine(
consumersBasedUserVideoGraphStore: ReadableStore[
ConsumersBasedRelatedTweetRequest,
RelatedTweetResponse
],
statsReceiver: StatsReceiver)
extends ReadableStore[
ConsumersBasedUserVideoGraphSimilarityEngine.Query,
Seq[TweetWithScore]
]
def get(
query: ConsumersBasedUserVideoGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object ConsumersBasedUserVideoGraphSimilarityEngine
class Query(
seedWithScores: Map[UserId, Double],
maxResults: Int,
minCooccurrence: Int,
minScore: Double,
maxTweetAgeInHours: Int)
def toSimilarityEngineInfo(
score: Double
): SimilarityEngineInfo
def fromParamsForRealGraphIn(
seedWithScores: Map[UserId, Double],
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\DiffusionBasedSimilarityEngine.scala
class DiffusionBasedSimilarityEngine(
retweetBasedDiffusionRecsMhStore: ReadableStore[Long, TweetsWithScore],
statsReceiver: StatsReceiver)
extends ReadableStore[
DiffusionBasedSimilarityEngine.Query,
Seq[TweetWithScore]
]
def get(
query: DiffusionBasedSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object DiffusionBasedSimilarityEngine
class Query(
sourceId: InternalId,
)
def toSimilarityEngineInfo(
query: LookupEngineQuery[Query],
score: Double
): SimilarityEngineInfo
def fromParams(
sourceId: InternalId,
modelId: String,
params: configapi.Params,
): LookupEngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdModelBasedSimilarityEngine.scala
class EarlybirdModelBasedSimilarityEngine @Inject() (
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint,
timeoutConfig: TimeoutConfig,
stats: StatsReceiver)
extends EarlybirdSimilarityEngineBase[EarlybirdModelBasedSearchQuery]
def getEarlybirdRequest(
query: EarlybirdModelBasedSearchQuery
): Option[EarlybirdRequest] =
if (query.seedUserIds.nonEmpty)
Some(
EarlybirdRequest(
searchQuery = getThriftSearchQuery(query),
clientId = Some(EarlybirdClientId),
timeoutMs = timeoutConfig.earlybirdServerTimeout.inMilliseconds.intValue(),
clientRequestID = Some(s"$
object EarlybirdModelBasedSimilarityEngine
class EarlybirdModelBasedSearchQuery(
seedUserIds: Seq[UserId],
maxNumTweets: Int,
oldestTweetTimestampInSec: Option[UserId],
frsUserToScoresForScoreAdjustment: Option[Map[UserId, Double]])
extends EarlybirdSearchQuery
/**
* Used by Push Service
*/
val RealGraphScoringModel = "frigate_unified_engagement_rg"
val MaxHitsToProcess = 1000
val MaxConsecutiveSameUser = 1
private def getModelBasedRankingParams(
authorSpecificScoreAdjustments: Map[Long, Double]
): ThriftRankingParams = ThriftRankingParams(
`type` = Some(ThriftScoringFunctionType.ModelBased),
selectedModels = Some(Map(RealGraphScoringModel -> 1.0)),
applyBoosts = false,
authorSpecificScoreAdjustments = Some(authorSpecificScoreAdjustments)
)
private def getRelevanceOptions(
authorSpecificScoreAdjustments: Map[Long, Double],
): ThriftSearchRelevanceOptions
def getThriftSearchQuery(query: EarlybirdModelBasedSearchQuery): ThriftSearchQuery =
ThriftSearchQuery(
serializedQuery = Some(f"(* [since_time $
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdRecencyBasedSimilarityEngine.scala
class EarlybirdRecencyBasedSimilarityEngine @Inject() (
@Named(ModuleNames.EarlybirdRecencyBasedWithoutRetweetsRepliesTweetsCache)
earlybirdRecencyBasedWithoutRetweetsRepliesTweetsCacheStore: ReadableStore[
UserId,
Seq[TweetId]
],
@Named(ModuleNames.EarlybirdRecencyBasedWithRetweetsRepliesTweetsCache)
earlybirdRecencyBasedWithRetweetsRepliesTweetsCacheStore: ReadableStore[
UserId,
Seq[TweetId]
],
timeoutConfig: TimeoutConfig,
stats: StatsReceiver)
extends ReadableStore[EarlybirdRecencyBasedSearchQuery, Seq[TweetWithAuthor]]
def get(
query: EarlybirdRecencyBasedSearchQuery
): Future[Option[Seq[TweetWithAuthor]]]
object EarlybirdRecencyBasedSimilarityEngine
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdSimilarityEngine.scala
class EarlybirdSimilarityEngine[
Query,
EarlybirdSimilarityEngineStore <: ReadableStore[Query, Seq[TweetWithAuthor]]
](
implementingStore: EarlybirdSimilarityEngineStore,
override val identifier: SimilarityEngineType,
globalStats: StatsReceiver,
engineConfig: SimilarityEngineConfig,
) extends SimilarityEngine[EngineQuery[Query], TweetWithAuthor]
def getScopedStats: StatsReceiver = scopedStats
def getCandidates(query: EngineQuery[Query]): Future[Option[Seq[TweetWithAuthor]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdSimilarityEngineBase.scala
trait is a base trait for Earlybird similarity engines. All Earlybird similarity
* engines extend from it and override the construction method for EarlybirdRequest
*/
trait EarlybirdSimilarityEngineBase[EarlybirdSearchQuery]
extends ReadableStore[EarlybirdSearchQuery, Seq[TweetWithAuthor]]
def earlybirdSearchClient: EarlybirdService.MethodPerEndpoint
def statsReceiver: StatsReceiver
def getEarlybirdRequest(query: EarlybirdSearchQuery): Option[EarlybirdRequest]
override def get(query: EarlybirdSearchQuery): Future[Option[Seq[TweetWithAuthor]]]
object EarlybirdSimilarityEngineBase
trait EarlybirdSearchQuery
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdSimilarityEngineRouter.scala
class EarlybirdSimilarityEngineRouter @Inject() (
earlybirdRecencyBasedSimilarityEngine: EarlybirdSimilarityEngine[
EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery,
EarlybirdRecencyBasedSimilarityEngine
],
earlybirdModelBasedSimilarityEngine: EarlybirdSimilarityEngine[
EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery,
EarlybirdModelBasedSimilarityEngine
],
earlybirdTensorflowBasedSimilarityEngine: EarlybirdSimilarityEngine[
EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery,
EarlybirdTensorflowBasedSimilarityEngine
],
timeoutConfig: TimeoutConfig,
statsReceiver: StatsReceiver)
extends ReadableStore[EarlybirdSimilarityEngineRouter.Query, Seq[TweetWithAuthor]]
def get(
k: EarlybirdSimilarityEngineRouter.Query
): Future[Option[Seq[TweetWithAuthor]]]
object EarlybirdSimilarityEngineRouter
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdTensorflowBasedSimilarityEngine.scala
class EarlybirdTensorflowBasedSimilarityEngine @Inject() (
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint,
timeoutConfig: TimeoutConfig,
stats: StatsReceiver)
extends EarlybirdSimilarityEngineBase[EarlybirdTensorflowBasedSearchQuery]
def getEarlybirdRequest(
query: EarlybirdTensorflowBasedSearchQuery
): Option[EarlybirdRequest]
object EarlybirdTensorflowBasedSimilarityEngine
class EarlybirdTensorflowBasedSearchQuery(
searcherUserId: Option[UserId],
seedUserIds: Seq[UserId],
maxNumTweets: Int,
beforeTweetIdExclusive: Option[TweetId],
afterTweetIdExclusive: Option[TweetId],
filterOutRetweetsAndReplies: Boolean,
useTensorflowRanking: Boolean,
excludedTweetIds: Set[TweetId],
maxNumHitsPerShard: Int)
extends EarlybirdSearchQuery
private def getThriftSearchQuery(
query: EarlybirdTensorflowBasedSearchQuery,
processingTimeout: Duration
): ThriftSearchQuery =
ThriftSearchQuery(
serializedQuery = GetEarlybirdQuery(
query.beforeTweetIdExclusive,
query.afterTweetIdExclusive,
query.excludedTweetIds,
query.filterOutRetweetsAndReplies).map(_.serialize),
fromUserIDFilter64 = Some(query.seedUserIds),
numResults = query.maxNumTweets,
// Whether to collect conversation IDs. Remove it for now.
// collectConversationId = Gate.True(), // true for Home
rankingMode = ThriftSearchRankingMode.Relevance,
relevanceOptions = Some(getRelevanceOptions(query.useTensorflowRanking)),
collectorParams = Some(
CollectorParams(
// numResultsToReturn defines how many results each EB shard will return to search root
numResultsToReturn = 1000,
// terminationParams.maxHitsToProcess is used for early terminating per shard results fetching.
terminationParams =
GetCollectorTerminationParams(query.maxNumHitsPerShard, processingTimeout)
)),
facetFieldNames = Some(FacetsToFetch),
resultMetadataOptions = Some(MetadataOptions),
searcherId = query.searcherUserId,
searchStatusIds = None,
namedDisjunctionMap = GetNamedDisjunctions(query.excludedTweetIds)
)
// The specific values of recap relevance/reranking options correspond to
// experiment: enable_recap_reranking_2988,timeline_internal_disable_recap_filter
// bucket : enable_rerank,disable_filter
private def getRelevanceOptions(useTensorflowRanking: Boolean): ThriftSearchRelevanceOptions
def getTensorflowBasedRankingParams: ThriftRankingParams
def getLinearRankingParams: ThriftRankingParams
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\FilterUtil.scala
object FilterUtil
def tweetAgeFilter(
candidates: Seq[TweetWithScore],
maxTweetAgeHours: Duration
): Seq[TweetWithScore]
def tweetSourceAgeFilter(
candidates: Seq[SourceInfo],
maxTweetSignalAgeHoursParam: Duration
): Seq[SourceInfo]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\HnswANNSimilarityEngine.scala
class HnswANNEngineQuery(
modelId: String,
sourceId: InternalId,
params: Params,
)
class HnswANNSimilarityEngine(
embeddingStoreLookUpMap: Map[String, ReadableStore[InternalId, ThriftEmbedding]],
annServiceLookUpMap: Map[String, AnnQueryService.MethodPerEndpoint],
globalStats: StatsReceiver,
override val identifier: SimilarityEngineType,
engineConfig: SimilarityEngineConfig,
memCacheConfigOpt: Option[MemCacheConfig[HnswANNEngineQuery]] = None)
extends SimilarityEngine[HnswANNEngineQuery, TweetWithScore]
def getScopedStats: StatsReceiver = scopedStats
private def fetchEmbedding(
query: HnswANNEngineQuery,
): Future[Option[ThriftEmbedding]]
def fetchCandidates(
query: HnswANNEngineQuery,
embedding: ThriftEmbedding
): Future[Seq[TweetWithScore]]
def toScore(distance: Distance): Double
def getEmbeddingAndCandidates(
query: HnswANNEngineQuery
): Future[Option[Seq[TweetWithScore]]]
def toSimilarityEngineInfo(
query: HnswANNEngineQuery,
score: Double
): SimilarityEngineInfo
def getCandidates(
engineQuery: HnswANNEngineQuery
): Future[Option[Seq[TweetWithScore]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\LookupSimilarityEngine.scala
class LookupEngineQuery[Query](
storeQuery: Query, // the actual Query type of the underlying store
lookupKey: String,
params: Params,
)
/**
* This Engine provides a map interface for looking up different model implementations.
* It provides modelId level monitoring for free.
*
* Example use cases include OfflineSimClusters lookup
*
*
* @param versionedStoreMap A mapping from a modelId to a corresponding implementation
* @param memCacheConfigOpt If specified, it will wrap the underlying store with a MemCache layer
* You should only enable this for cacheable queries, e.x. TweetIds.
* consumer based UserIds are generally not possible to cache.
*/
class LookupSimilarityEngine[Query, Candidate <: Serializable](
versionedStoreMap: Map[String, ReadableStore[Query, Seq[Candidate]]], // key = modelId
override val identifier: SimilarityEngineType,
globalStats: StatsReceiver,
engineConfig: SimilarityEngineConfig,
memCacheConfigOpt: Option[MemCacheConfig[Query]] = None)
extends SimilarityEngine[LookupEngineQuery[Query], Candidate]
def getCandidates(
engineQuery: LookupEngineQuery[Query]
): Future[Option[Seq[Candidate]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ModelBasedANNStore.scala
class ModelBasedANNStore(
embeddingStoreLookUpMap: Map[String, ReadableStore[InternalId, ThriftEmbedding]],
annServiceLookUpMap: Map[String, AnnQueryService.MethodPerEndpoint],
globalStats: StatsReceiver)
extends ReadableStore[
ModelBasedANNStore.Query,
Seq[TweetWithScore]
]
def get(query: Query): Future[Option[Seq[TweetWithScore]]]
def fetchEmbedding(query: Query): Future[Option[ThriftEmbedding]]
def fetchCandidates(
query: Query,
embedding: ThriftEmbedding
): Future[Option[NearestNeighborResult]]
object ModelBasedANNStore
class Query(
sourceId: InternalId,
modelId: String,
similarityEngineType: SimilarityEngineType,
ef: Int = 800)
def toScore(distance: Distance): Double
def toSimilarityEngineInfo(query: Query, score: Double): SimilarityEngineInfo
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ProducerBasedUnifiedSimilarityEngine.scala
class ProducerBasedUnifiedSimilarityEngine(
@Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine)
producerBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[
ProducerBasedUserTweetGraphSimilarityEngine.Query,
TweetWithScore
],
simClustersANNSimilarityEngine: StandardSimilarityEngine[
SimClustersANNSimilarityEngine.Query,
TweetWithScore
],
statsReceiver: StatsReceiver)
extends ReadableStore[ProducerBasedUnifiedSimilarityEngine.Query, Seq[
TweetWithCandidateGenerationInfo
]]
def get(
query: Query
): Future[Option[Seq[TweetWithCandidateGenerationInfo]]]
def simClustersCandidateMinScoreFilter(
simClustersAnnCandidates: Seq[TweetWithScore],
simClustersMinScore: Double,
simClustersANNConfigId: String
): Seq[TweetWithScore]
def userTweetGraphFilter(
userTweetGraphCandidates: Seq[TweetWithScore]
): Seq[TweetWithScore]
object ProducerBasedUnifiedSimilarityEngine
def getProducerBasedUnifiedCGInfo(
sourceInfoOpt: Option[SourceInfo],
unifiedScore: Double,
contributingSimilarityEngines: Seq[SimilarityEngineInfo]
): CandidateGenerationInfo
class Query(
sourceInfo: SourceInfo,
maxCandidateNumPerSourceKey: Int,
maxTweetAgeHours: Duration,
// SimClusters
enableSimClustersANN: Boolean,
simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableExperimentalSimClustersANN: Boolean,
experimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN1: Boolean,
simClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN2: Boolean,
simClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN4: Boolean,
simClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN3: Boolean,
simClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN5: Boolean,
simClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
simClustersMinScore: Double,
// UTG
enableUtg: Boolean,
utgCombinationMethod: UnifiedSETweetCombinationMethod.Value,
utgQuery: EngineQuery[ProducerBasedUserTweetGraphSimilarityEngine.Query])
def fromParams(
sourceInfo: SourceInfo,
params: configapi.Params,
): EngineQuery[Query]
def fromParamsForRelatedTweet(
internalId: InternalId,
params: configapi.Params
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ProducerBasedUserAdGraphSimilarityEngine.scala
class ProducerBasedUserAdGraphSimilarityEngine(
userAdGraphService: UserAdGraph.MethodPerEndpoint,
statsReceiver: StatsReceiver)
extends ReadableStore[ProducerBasedUserAdGraphSimilarityEngine.Query, Seq[
TweetWithScore
]]
def get(
query: ProducerBasedUserAdGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object ProducerBasedUserAdGraphSimilarityEngine
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo
class Query(
sourceId: InternalId,
maxResults: Int,
minCooccurrence: Int, // require at least
def fromParams(
sourceId: InternalId,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ProducerBasedUserTweetGraphSimilarityEngine.scala
class ProducerBasedUserTweetGraphSimilarityEngine(
userTweetGraphService: UserTweetGraph.MethodPerEndpoint,
statsReceiver: StatsReceiver)
extends ReadableStore[ProducerBasedUserTweetGraphSimilarityEngine.Query, Seq[
TweetWithScore
]]
def get(
query: ProducerBasedUserTweetGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object ProducerBasedUserTweetGraphSimilarityEngine
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo
class Query(
sourceId: InternalId,
maxResults: Int,
minCooccurrence: Int, // require at least
def fromParams(
sourceId: InternalId,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SimClustersANNSimilarityEngine.scala
class SimClustersANNSimilarityEngine(
simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint],
statsReceiver: StatsReceiver)
extends ReadableStore[
SimClustersANNSimilarityEngine.Query,
Seq[TweetWithScore]
]
def getSimClustersANNService(
query: SimClustersANNQuery
): Option[SimClustersANNService.MethodPerEndpoint]
def get(
query: SimClustersANNSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object SimClustersANNSimilarityEngine
class Query(
simClustersANNQuery: SimClustersANNQuery,
simClustersANNConfigId: String)
def toSimilarityEngineInfo(
query: EngineQuery[Query],
score: Double
): SimilarityEngineInfo
def fromParams(
internalId: InternalId,
embeddingType: EmbeddingType,
modelVersion: ModelVersion,
simClustersANNConfigId: String,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SimilarityEngine.scala
class local this directory only
*
*/
trait SimilarityEngine[Query, Candidate]
def identifier: SimilarityEngineType
def getCandidates(query: Query): Future[Option[Seq[Candidate]]]
}
object SimilarityEngine extends Logging
class SimilarityEngineConfig(
timeout: Duration,
gatingConfig: GatingConfig)
/**
* Controls for whether or not this Engine is enabled.
* In our previous design, we were expecting a Sim Engine will only take one set of Params,
* and that’s why we decided to have GatingConfig and the EnableFeatureSwitch in the trait.
* However, we now have two candidate generation pipelines: Tweet Rec, Related Tweets
* and they are now having their own set of Params, but EnableFeatureSwitch can only put in 1 fixed value.
* We need some further refactor work to make it more flexible.
*
* @param deciderConfig Gate the Engine by a decider. If specified,
* @param enableFeatureSwitch. DO NOT USE IT FOR NOW. It needs some refactorting. Please set it to None (SD-20268)
*/
case class GatingConfig(
deciderConfig: Option[DeciderConfig],
enableFeatureSwitch: Option[
FSParam[Boolean]
]) // Do NOT use the enableFeatureSwitch. It needs some refactoring.
case class DeciderConfig(
decider: CrMixerDecider,
deciderString: String)
case class MemCacheConfig[K](
cacheClient: Client,
ttl: Duration,
asyncUpdate: Boolean = false,
keyToString: K => String)
private[similarity_engine] def isEnabled(
params: Params,
gatingConfig: GatingConfig
): Boolean
object needs to be cacheable,
* i.e. it cannot be a runtime objects or complex objects, for example, configapi.Params
*
* @param underlyingStore un-cached store implementation
* @param keyPrefix a prefix differentiates 2 stores if they share the same key space.
* e.x. 2 implementations of ReadableStore[UserId, Seq[Candidiate] ]
* can use prefix "store_v1", "store_v2"
* @return A ReadableStore with a MemCache wrapper
*/
private[similarity_engine] def addMemCache[Query, Candidate <: Serializable](
underlyingStore: ReadableStore[Query, Seq[Candidate]],
memCacheConfig: MemCacheConfig[Query],
keyPrefix: Option[String] = None,
statsReceiver: StatsReceiver
): ReadableStore[Query, Seq[Candidate]]
def getFromFn[Query, Candidate](
fn: Query => Future[Option[Seq[Candidate]]],
storeQuery: Query,
engineConfig: SimilarityEngineConfig,
params: Params,
scopedStats: StatsReceiver
): Future[Option[Seq[Candidate]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SimilaritySourceOrderingUtil.scala
object SimilaritySourceOrderingUtil
def keepGivenOrder(
candidates: Seq[Seq[TweetWithCandidateGenerationInfo]],
): Seq[TweetWithCandidateGenerationInfo]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SkitHighPrecisionTopicTweetSimilarityEngine.scala
class SkitHighPrecisionTopicTweetSimilarityEngine @Inject() (
@Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[
TopicTweetPartitionFlatKey,
Seq[TopicTweet]
],
statsReceiver: StatsReceiver)
extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]]
def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]]
def fetch(query: EngineQuery[Query]): Future[Seq[SkitTopicTweet]]
def getTweetsForKeys(
keys: Seq[TopicTweetPartitionFlatKey],
sourceTopic: TopicId
): Future[Seq[SkitTopicTweet]]
object SkitHighPrecisionTopicTweetSimilarityEngine
def fromParams(
topicId: TopicId,
isVideoOnly: Boolean,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SkitTopicTweetSimilarityEngine.scala
class SkitTopicTweetSimilarityEngine @Inject() (
@Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[
TopicTweetPartitionFlatKey,
Seq[TopicTweet]
],
statsReceiver: StatsReceiver)
extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]]
def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]]
def fetch(query: EngineQuery[Query]): Future[Seq[SkitTopicTweet]]
def getTweetsForKeys(
keys: Seq[TopicTweetPartitionFlatKey],
sourceTopic: TopicId
): Future[Seq[SkitTopicTweet]]
object SkitTopicTweetSimilarityEngine
class Query(
topicId: TopicId,
maxCandidates: Int,
maxTweetAge: Duration,
semanticCoreVersionId: Long)
case class SkitTopicTweet(
sourceTopic: TopicId,
tweetId: TweetId,
favCount: Long,
cosineSimilarityScore: Double)
def fromParams(
topicId: TopicId,
isVideoOnly: Boolean,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\StandardSimilarityEngine.scala
class EngineQuery[Query](
storeQuery: Query,
params: Params,
)
/**
* A straight forward SimilarityEngine implementation that wraps a ReadableStore
*
* @param implementingStore Provides the candidate retrieval's implementations
* @param memCacheConfig If specified, it will wrap the underlying store with a MemCache layer
* You should only enable this for cacheable queries, e.x. TweetIds.
* consumer based UserIds are generally not possible to cache.
* @tparam Query ReadableStore's input type
* @tparam Candidate ReadableStore's return type is Seq[[[Candidate]]]
*/
class StandardSimilarityEngine[Query, Candidate <: Serializable](
implementingStore: ReadableStore[Query, Seq[Candidate]],
override val identifier: SimilarityEngineType,
globalStats: StatsReceiver,
engineConfig: SimilarityEngineConfig,
memCacheConfig: Option[MemCacheConfig[Query]] = None)
extends SimilarityEngine[EngineQuery[Query], Candidate]
def getScopedStats: StatsReceiver = scopedStats
// Add memcache wrapper, if specified
private val store
def getCandidates(
engineQuery: EngineQuery[Query]
): Future[Option[Seq[Candidate]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedQigSimilarityEngine.scala
class TweetBasedQigSimilarityEngine(
qigRanker: QigRanker.MethodPerEndpoint,
statsReceiver: StatsReceiver)
extends ReadableStore[
TweetBasedQigSimilarityEngine.Query,
Seq[TweetWithScore]
]
def get(
query: TweetBasedQigSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
def getQigSimilarTweetsRequest(
tweetId: Long
): QigRankerRequest
def getCandidatesFromQigResponse(
qigSimilarTweetsResponse: QigRankerResponse
): Option[Seq[TweetWithScore]]
object TweetBasedQigSimilarityEngine
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo
class Query(sourceId: InternalId)
def fromParams(
sourceId: InternalId,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedUnifiedSimilarityEngine.scala
class TweetBasedUnifiedSimilarityEngine(
@Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine)
tweetBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[
TweetBasedUserTweetGraphSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine)
tweetBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[
TweetBasedUserVideoGraphSimilarityEngine.Query,
TweetWithScore
],
simClustersANNSimilarityEngine: StandardSimilarityEngine[
SimClustersANNSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.TweetBasedQigSimilarityEngine)
tweetBasedQigSimilarTweetsSimilarityEngine: StandardSimilarityEngine[
TweetBasedQigSimilarityEngine.Query,
TweetWithScore
],
@Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine)
tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine,
statsReceiver: StatsReceiver)
extends ReadableStore[
TweetBasedUnifiedSimilarityEngine.Query,
Seq[TweetWithCandidateGenerationInfo]
]
def get(
query: Query
): Future[Option[Seq[TweetWithCandidateGenerationInfo]]]
def simClustersCandidateMinScoreFilter(
simClustersAnnCandidates: Seq[TweetWithScore],
simClustersMinScore: Double,
simClustersANNConfigId: String
): Seq[TweetWithScore]
def tweetAgeFilter(
candidates: Seq[TweetWithScore],
maxTweetAgeHours: Duration
): Seq[TweetWithScore]
def twhinFilter(
twhinCandidates: Seq[TweetWithScore],
twhinMaxTweetAgeHours: Duration,
simEngineStats: StatsReceiver
): Seq[TweetWithScore]
def userTweetGraphFilter(
userTweetGraphCandidates: Seq[TweetWithScore]
): Seq[TweetWithScore]
def userVideoGraphFilter(
userVideoGraphCandidates: Seq[TweetWithScore]
): Seq[TweetWithScore]
def qigSimilarTweetsFilter(
qigSimilarTweetsCandidates: Seq[TweetWithScore],
qigMaxTweetAgeHours: Duration,
qigMaxNumSimilarTweets: Int
): Seq[TweetWithScore]
def getTweetBasedUnifiedCGInfo(
sourceInfoOpt: Option[SourceInfo],
unifiedScore: Double,
contributingSimilarityEngines: Seq[SimilarityEngineInfo]
): CandidateGenerationInfo
object TweetBasedUnifiedSimilarityEngine
class Query(
sourceInfo: SourceInfo,
maxCandidateNumPerSourceKey: Int,
enableSimClustersANN: Boolean,
simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableExperimentalSimClustersANN: Boolean,
experimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN1: Boolean,
simClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN2: Boolean,
simClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN3: Boolean,
simClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN5: Boolean,
simClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
enableSimClustersANN4: Boolean,
simClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query],
simClustersMinScore: Double,
simClustersVideoBasedMinScore: Double,
twhinModelId: String,
enableTwHIN: Boolean,
twhinMaxTweetAgeHours: Duration,
qigMaxTweetAgeHours: Duration,
qigMaxNumSimilarTweets: Int,
enableUtg: Boolean,
utgQuery: EngineQuery[TweetBasedUserTweetGraphSimilarityEngine.Query],
enableUvg: Boolean,
uvgQuery: EngineQuery[TweetBasedUserVideoGraphSimilarityEngine.Query],
enableQig: Boolean,
qigQuery: EngineQuery[TweetBasedQigSimilarityEngine.Query],
params: configapi.Params)
def fromParams(
sourceInfo: SourceInfo,
params: configapi.Params,
): EngineQuery[Query]
def fromParamsForRelatedTweet(
internalId: InternalId,
params: configapi.Params,
): EngineQuery[Query]
def fromParamsForRelatedVideoTweet(
internalId: InternalId,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedUserAdGraphSimilarityEngine.scala
class TweetBasedUserAdGraphSimilarityEngine(
userAdGraphService: UserAdGraph.MethodPerEndpoint,
tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers],
statsReceiver: StatsReceiver)
extends ReadableStore[
TweetBasedUserAdGraphSimilarityEngine.Query,
Seq[TweetWithScore]
]
def get(
query: TweetBasedUserAdGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
def getCandidates(
tweetId: TweetId,
query: TweetBasedUserAdGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object TweetBasedUserAdGraphSimilarityEngine
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo
def toTweetWithScore(
relatedAdResponseFut: Future[Option[RelatedAdResponse]]
): Future[Option[Seq[TweetWithScore]]]
class Query(
sourceId: InternalId,
maxResults: Int,
minCooccurrence: Int,
consumersBasedMinScore: Double,
maxTweetAgeInHours: Int,
maxConsumerSeedsNum: Int,
)
def fromParams(
sourceId: InternalId,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedUserTweetGraphSimilarityEngine.scala
class TweetBasedUserTweetGraphSimilarityEngine(
userTweetGraphService: UserTweetGraph.MethodPerEndpoint,
tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers],
statsReceiver: StatsReceiver)
extends ReadableStore[
TweetBasedUserTweetGraphSimilarityEngine.Query,
Seq[TweetWithScore]
]
def get(
query: TweetBasedUserTweetGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
def getCandidates(
tweetId: TweetId,
query: TweetBasedUserTweetGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
def getCoverageExpansionCandidates(
tweetId: TweetId,
query: TweetBasedUserTweetGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object TweetBasedUserTweetGraphSimilarityEngine
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo
def toTweetWithScore(
relatedTweetResponseFut: Future[Option[RelatedTweetResponse]]
): Future[Option[Seq[TweetWithScore]]]
def isOldTweet(tweetId: TweetId): Boolean
class Query(
sourceId: InternalId,
maxResults: Int,
minCooccurrence: Int,
tweetBasedMinScore: Double,
consumersBasedMinScore: Double,
maxTweetAgeInHours: Int,
maxConsumerSeedsNum: Int,
enableCoverageExpansionOldTweet: Boolean,
enableCoverageExpansionAllTweet: Boolean,
)
def fromParams(
sourceId: InternalId,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedUserVideoGraphSimilarityEngine.scala
class TweetBasedUserVideoGraphSimilarityEngine(
userVideoGraphService: UserVideoGraph.MethodPerEndpoint,
tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers],
statsReceiver: StatsReceiver)
extends ReadableStore[
TweetBasedUserVideoGraphSimilarityEngine.Query,
Seq[TweetWithScore]
]
def get(
query: TweetBasedUserVideoGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
def getCandidates(
tweetId: TweetId,
query: TweetBasedUserVideoGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
def getCoverageExpansionCandidates(
tweetId: TweetId,
query: TweetBasedUserVideoGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object TweetBasedUserVideoGraphSimilarityEngine
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo
def toTweetWithScore(
relatedTweetResponseFut: Future[Option[RelatedTweetResponse]]
): Future[Option[Seq[TweetWithScore]]]
def isOldTweet(tweetId: TweetId): Boolean
class Query(
sourceId: InternalId,
maxResults: Int,
minCooccurrence: Int,
tweetBasedMinScore: Double,
consumersBasedMinScore: Double,
maxTweetAgeInHours: Int,
maxConsumerSeedsNum: Int,
enableCoverageExpansionOldTweet: Boolean,
enableCoverageExpansionAllTweet: Boolean)
def fromParams(
sourceId: InternalId,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TwhinCollabFilterSimilarityEngine.scala
class TwhinCollabFilterSimilarityEngine(
twhinCandidatesStratoStore: ReadableStore[Long, Seq[TweetId]],
statsReceiver: StatsReceiver)
extends ReadableStore[
TwhinCollabFilterSimilarityEngine.Query,
Seq[TweetWithScore]
]
def get(
query: TwhinCollabFilterSimilarityEngine.Query
): Future[Option[Seq[TweetWithScore]]]
object TwhinCollabFilterSimilarityEngine
class TwhinCollabFilterView(clusterVersion: String)
case class Query(
sourceId: InternalId,
)
def toSimilarityEngineInfo(
query: LookupEngineQuery[Query],
score: Double
): SimilarityEngineInfo
def fromParams(
sourceId: InternalId,
modelId: String,
params: configapi.Params,
): LookupEngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\UserTweetEntityGraphSimilarityEngine.scala
class UserTweetEntityGraphSimilarityEngine(
userTweetEntityGraph: UserTweetEntityGraph.MethodPerEndpoint,
statsReceiver: StatsReceiver)
extends ReadableStore[
UserTweetEntityGraphSimilarityEngine.Query,
Seq[TweetWithScoreAndSocialProof]
]
def get(
query: UserTweetEntityGraphSimilarityEngine.Query
): Future[Option[Seq[TweetWithScoreAndSocialProof]]]
object UserTweetEntityGraphSimilarityEngine
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo
class Query(
userId: UserId,
seedsWithWeights: Map[UserId, Double],
excludedTweetIds: Option[Seq[Long]] = None,
maxUtegCandidates: Int,
maxTweetAge: Duration,
socialProofTypes: Option[Seq[SocialProofType]])
def fromParams(
userId: UserId,
seedsWithWeights: Map[UserId, Double],
excludedTweetIds: Option[Seq[TweetId]] = None,
params: configapi.Params,
): EngineQuery[Query]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\FrsSourceGraphFetcher.scala
class FrsSourceGraphFetcher @Inject() (
@Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]],
override val timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver)
extends SourceGraphFetcher
def isEnabled(query: FetcherQuery): Boolean
def fetchAndProcess(
query: FetcherQuery,
): Future[Option[GraphSourceInfo]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\FrsSourceSignalFetcher.scala
class FrsSourceSignalFetcher @Inject() (
@Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]],
override val timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver)
extends SourceSignalFetcher
def isEnabled(query: FetcherQuery): Boolean
def fetchAndProcess(query: FetcherQuery): Future[Option[Seq[SourceInfo]]]
def convertSourceInfo(
sourceType: SourceType,
signals: Seq[SignalConvertType]
): Seq[SourceInfo]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\FrsStore.scala
class FrsStore(
frsClient: FollowRecommendationsThriftService.MethodPerEndpoint,
statsReceiver: StatsReceiver,
decider: CrMixerDecider)
extends ReadableStore[Query, Seq[FrsQueryResult]]
def get(
query: Query
): Future[Option[Seq[FrsQueryResult]]]
def buildFollowRecommendationRequest(
query: Query
): RecommendationRequest
object FrsStore
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\RealGraphInSourceGraphFetcher.scala
class RealGraphInSourceGraphFetcher @Inject() (
@Named(ModuleNames.RealGraphInStore) realGraphStoreMh: ReadableStore[UserId, CandidateSeq],
override val timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver)
extends SourceGraphFetcher
def isEnabled(query: FetcherQuery): Boolean
def fetchAndProcess(
query: FetcherQuery,
): Future[Option[GraphSourceInfo]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\RealGraphOonSourceGraphFetcher.scala
class RealGraphOonSourceGraphFetcher @Inject() (
@Named(ModuleNames.RealGraphOonStore) realGraphOonStore: ReadableStore[UserId, CandidateSeq],
override val timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver)
extends SourceGraphFetcher
def isEnabled(query: FetcherQuery): Boolean
def fetchAndProcess(
query: FetcherQuery,
): Future[Option[GraphSourceInfo]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\SourceFetcher.scala
trait which, given a [[FetcherQuery]], returns [[ResultType]]
* The main purposes of a SourceFetcher is to provide a consistent interface for source fetch
* logic, and provides default functions, including:
* - Identification
* - Observability
* - Timeout settings
* - Exception Handling
*/
trait SourceFetcher[ResultType] extends ReadableStore[FetcherQuery, ResultType] with Logging
def identifier: String = this.getClass.getSimpleName
protected def stats: StatsReceiver
protected def timeoutConfig: TimeoutConfig
/***
* Use FeatureSwitch to decide if a specific source is enabled.
*/
def isEnabled(query: FetcherQuery): Boolean
/***
* This function fetches the raw sources and process them.
* Custom stats tracking can be added depending on the type of ResultType
*/
def fetchAndProcess(
query: FetcherQuery,
): Future[Option[ResultType]]
/***
* Side-effect function to track stats for signal fetching and processing.
*/
def trackStats(
query: FetcherQuery
)(
func: => Future[Option[ResultType]]
): Future[Option[ResultType]]
/***
* This function is called by the top level class to fetch sources. It executes the pipeline to
* fetch raw data, process and transform the sources. Exceptions, Stats, and timeout control are
* handled here.
*/
override def get(
query: FetcherQuery
): Future[Option[ResultType]]
object SourceFetcher
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\SourceGraphFetcher.scala
trait that extends from `SourceFetcher`
* and is specialized in tackling User Graph (eg., RealGraphOon, FRS) fetch.
*
* The [[ResultType]] of a SourceGraphFetcher is a `GraphSourceInfo` which contains a userSeedSet.
* When we pass in userId, the underlying store returns one GraphSourceInfo.
*/
trait SourceGraphFetcher extends SourceFetcher[GraphSourceInfo]
def graphSourceType: SourceType
/***
* RawDataType contains a consumers seed UserId and a score (weight)
*/
protected type RawDataType = (UserId, Double)
def trackStats(
query: FetcherQuery
)(
func: => Future[Option[GraphSourceInfo]]
): Future[Option[GraphSourceInfo]]
def trackPerItemStats(
query: FetcherQuery
)(
func: => Future[Option[Seq[RawDataType]]]
): Future[Option[Seq[RawDataType]]]
def convertGraphSourceInfo(
userWithScores: Seq[RawDataType]
): GraphSourceInfo
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\SourceInfoRouter.scala
class SourceInfoRouter @Inject() (
ussSourceSignalFetcher: UssSourceSignalFetcher,
frsSourceSignalFetcher: FrsSourceSignalFetcher,
frsSourceGraphFetcher: FrsSourceGraphFetcher,
realGraphOonSourceGraphFetcher: RealGraphOonSourceGraphFetcher,
realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher,
)
def get(
userId: UserId,
product: TProduct,
userState: UserState,
params: configapi.Params
): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])]
def getSourceSignals(
fetcherQuery: FetcherQuery
): Future[Set[SourceInfo]]
def getSourceGraphs(
fetcherQuery: FetcherQuery
): Future[Map[String, Option[GraphSourceInfo]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\SourceSignalFetcher.scala
trait that extends from `SourceFetcher`
* and is specialized in tackling Signals (eg., USS, FRS) fetch.
* Currently, we define Signals as (but not limited to) a set of past engagements that
* the user makes, such as RecentFav, RecentFollow, etc.
*
* The [[ResultType]] of a SourceSignalFetcher is `Seq[SourceInfo]`. When we pass in userId,
* the underlying store returns a list of signals.
*/
trait SourceSignalFetcher extends SourceFetcher[Seq[SourceInfo]]
def trackStats(
query: FetcherQuery
)(
func: => Future[Option[Seq[SourceInfo]]]
): Future[Option[Seq[SourceInfo]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\UssSourceSignalFetcher.scala
class UssSourceSignalFetcher @Inject() (
@Named(ModuleNames.UssStore) ussStore: ReadableStore[UssStore.Query, Seq[
(SignalType, Seq[UssSignal])
]],
override val timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver)
extends SourceSignalFetcher
def isEnabled(query: FetcherQuery): Boolean = true
override def fetchAndProcess(
query: FetcherQuery,
): Future[Option[Seq[SourceInfo]]]
def convertSourceInfo(
sourceType: SourceType,
signals: Seq[SignalConvertType]
): Seq[SourceInfo]
def trackUssSignalStatsPerSignalType(
query: FetcherQuery,
signalType: SignalType,
ussSignals: Seq[UssSignal]
): Unit
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\UssStore.scala
class UssStore(
stratoStore: ReadableStore[BatchSignalRequest, BatchSignalResponse],
statsReceiver: StatsReceiver)
extends ReadableStore[Query, Seq[(SignalType, Seq[UssSignal])]]
def get(query: Query): Future[Option[Seq[(SignalType, Seq[UssSignal])]]]
def buildUserSignalServiceRequests(
param: Params,
): Seq[SignalRequest]
object UssStore
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\CandidateGenerationKeyUtil.scala
object CandidateGenerationKeyUtil
def toThrift(
candidateGenerationInfo: CandidateGenerationInfo,
requestUserId: UserId
): CandidateGenerationKey
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\CountWeightedInterleaveUtil.scala
object CountWeightedInterleaveUtil
class GroupingKey(
sourceInfoOpt: Option[SourceInfo],
similarityEngineTypeOpt: Option[SimilarityEngineType],
modelIdOpt: Option[String],
authorIdOpt: Option[Long],
groupIdOpt: Option[Int])
/**
* Converts candidates to grouping key based upon the feature that we interleave with.
*/
def toGroupingKey[CandidateType <: Candidate](
candidate: CandidateType,
interleaveFeature: Option[BlendGroupingMethodEnum.Value],
groupId: Option[Int],
): GroupingKey
def calculateWeightsKeyByFeature[CandidateType <: Candidate](
candidateSeqKeyByFeature: Map[GroupingKey, Seq[CandidateType]],
rankerWeightShrinkage: Double
): Map[GroupingKey, Double]
def buildRankedCandidatesWithWeightKeyByFeature(
rankedCandidateSeq: Seq[RankedCandidate],
rankerWeightShrinkage: Double,
interleaveFeature: BlendGroupingMethodEnum.Value
): Seq[(Seq[RankedCandidate], Double)]
def buildInitialCandidatesWithWeightKeyByFeature(
initialCandidatesSeqSeq: Seq[Seq[InitialCandidate]],
rankerWeightShrinkage: Double,
): Seq[(Seq[InitialCandidate], Double)]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\EarlybirdSearchUtil.scala
object EarlybirdSearchUtil
def GetCollectorTerminationParams(
maxNumHitsPerShard: Int,
processingTimeout: Duration
): Option[CollectorTerminationParams]
def GetEarlybirdQuery(
beforeTweetIdExclusive: Option[TweetId],
afterTweetIdExclusive: Option[TweetId],
excludedTweetIds: Set[TweetId],
filterOutRetweetsAndReplies: Boolean
): Option[EbQuery] =
CreateConjunction(
Seq(
CreateRangeQuery(beforeTweetIdExclusive, afterTweetIdExclusive),
CreateExcludedTweetIdsQuery(excludedTweetIds),
CreateTweetTypesFilters(filterOutRetweetsAndReplies)
).flatten)
def CreateRangeQuery(
beforeTweetIdExclusive: Option[TweetId],
afterTweetIdExclusive: Option[TweetId]
): Option[EbQuery]
def CreateTweetTypesFilters(filterOutRetweetsAndReplies: Boolean): Option[EbQuery]
def CreateConjunction(clauses: Seq[EbQuery]): Option[EbQuery]
def CreateExcludedTweetIdsQuery(tweetIds: Set[TweetId]): Option[EbQuery]
def GetNamedDisjunctions(excludedTweetIds: Set[TweetId]): Option[Map[String, Seq[Long]]] =
if (excludedTweetIds.nonEmpty)
createNamedDisjunctionsExcludedTweetIds(excludedTweetIds)
else None
val EXCLUDE_TWEET_IDS = "exclude_tweet_ids"
private def createNamedDisjunctionsExcludedTweetIds(
tweetIds: Set[TweetId]
): Option[Map[String, Seq[Long]]]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\InterleaveUtil.scala
object InterleaveUtil
def interleave[CandidateType <: Candidate](
candidates: Seq[Seq[CandidateType]]
): Seq[CandidateType]
def weightedInterleave[CandidateType <: Candidate](
candidatesAndWeight: Seq[(Seq[CandidateType], Double)],
maxWeightAdjustments: Int = 0
): Seq[CandidateType]
def buildCandidatesKeyByCGInfo(
candidates: Seq[RankedCandidate],
): Seq[Seq[RankedCandidate]]
class GroupingKey(
sourceInfoOpt: Option[SourceInfo],
similarityEngineType: SimilarityEngineType,
modelId: Option[String])
object GroupingKey
def toGroupingKey(candidateGenerationInfo: CandidateGenerationInfo): GroupingKey
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\MetricTagUtil.scala
object MetricTagUtil
def buildMetricTags(candidate: RankedCandidate): Seq[MetricTag]
def toMetricTagFromSource(sourceType: SourceType): Option[MetricTag]
def toMetricTagFromSimilarityEngine(
seInfo: SimilarityEngineInfo,
cseInfo: Seq[SimilarityEngineInfo]
): Seq[Option[MetricTag]]
def toMetricTagFromModelId(
modelId: String
): Option[MetricTag]
def toMetricTagFromSourceAndSimilarityEngine(
sourceInfo: SourceInfo,
seInfo: SimilarityEngineInfo
): Option[MetricTag]
def isFromInterestedIn(candidate: RankedCandidate): Set[MetricTag]
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\SignalTimestampStatsUtil.scala
class SignalTimestampStatsUtil @Inject() (statsReceiver: StatsReceiver)
def statsSignalTimestamp(
tweets: Seq[TweetRecommendation],
): Seq[TweetRecommendation]
object SignalTimestampStatsUtil
def buildLatestSourceSignalTimestamp(candidate: RankedCandidate): Option[Long]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\CandidateSourceRegistry.scala
trait CandidateSourceRegistry[Target, Candidate]
def sources: Set[CandidateSource[Target, Candidate]]
final lazy val candidateSources: Map[
CandidateSourceIdentifier,
CandidateSource[Target, Candidate]
]
def select(
identifiers: Set[CandidateSourceIdentifier]
): Set[CandidateSource[Target, Candidate]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\EnrichedCandidateSource.scala
class EnrichedCandidateSource[Target, Candidate](original: CandidateSource[Target, Candidate])
def gate(predicate: Predicate[Target]): CandidateSource[Target, Candidate]
def observe(statsReceiver: StatsReceiver): CandidateSource[Target, Candidate]
def apply(target: Target): Stitch[Seq[Candidate]]
def stitchMapKey[Target2](
targetMapper: Target2 => Stitch[Option[Target]]
): CandidateSource[Target2, Candidate]
def stitchMapKeys[Target2](
targetMapper: Target2 => Stitch[Seq[Target]]
): CandidateSource[Target2, Candidate]
def apply(target: Target2): Stitch[Seq[Candidate]]
def mapKeys[Target2](
targetMapper: Target2 => Seq[Target]
): CandidateSource[Target2, Candidate]
def mapValues[Candidate2](
candidateMapper: Candidate => Stitch[Option[Candidate2]]
): CandidateSource[Target, Candidate2]
def apply(target: Target): Stitch[Seq[Candidate2]]
def mapValue[Candidate2](
candidateMapper: Candidate => Candidate2
): CandidateSource[Target, Candidate2]
def within(
candidateTimeout: Duration,
statsReceiver: StatsReceiver
): CandidateSource[Target, Candidate]
def apply(target: Target): Stitch[Seq[Candidate]]
def failOpenWithin(
candidateTimeout: Duration,
statsReceiver: StatsReceiver
): CandidateSource[Target, Candidate]
def apply(target: Target): Stitch[Seq[Candidate]]
object EnrichedCandidateSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\ParamPredicate.scala
class ParamPredicate[Request <: HasParams](param: Param[Boolean]) extends Predicate[Request]
def apply(request: Request): Stitch[PredicateResult]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\Predicate.scala
trait Predicate[-Q]
def apply(item: Q): Stitch[PredicateResult]
def arrow: Arrow[Q, PredicateResult] = Arrow.apply(apply)
def map[K](mapper: K => Q): Predicate[K] = Predicate(arrow.contramap(mapper))
/**
* check the predicate results for a batch of items for convenience.
*
* mark it as final to avoid potential abuse usage
*/
final def batch(items: Seq[Q]): Stitch[Seq[PredicateResult]]
def apply[Q1, Q2](item1: Q1, item2: Q2)(implicit ev: ((Q1, Q2)) => Q): Stitch[PredicateResult]
object that represents the logical AND of both predicates
*/
def andThen[Q1 <: Q](p: Predicate[Q1]): Predicate[Q1]
object that represents the logical OR of both predicates.
* if both are invalid, the reason would be the set of all invalid reasons.
*/
def or[Q1 <: Q](p: Predicate[Q1]): Predicate[Q1]
def gate[Q1 <: Q](gatingPredicate: Predicate[Q1]): Predicate[Q1]
def observe(statsReceiver: StatsReceiver): Predicate[Q] = Predicate(
StatsUtil.profilePredicateResult(this.arrow, statsReceiver))
def convertToFailOpenWithResultType(resultType: PredicateResult): Predicate[Q]
class TruePredicate[Q] extends Predicate[Q]
def apply(item: Q): Stitch[PredicateResult] = Predicate.AlwaysTrueStitch
}
class FalsePredicate[Q](reason: FilterReason) extends Predicate[Q]
def apply(item: Q): Stitch[PredicateResult] = InvalidResult
}
object Predicate
def apply[Q](func: Q => Stitch[PredicateResult]): Predicate[Q] = new Predicate[Q]
def apply(item: Q): Stitch[PredicateResult] = func(item)
override val arrow: Arrow[Q, PredicateResult] = Arrow(func)
}
def apply[Q](outerArrow: Arrow[Q, PredicateResult]): Predicate[Q] = new Predicate[Q]
def apply(item: Q): Stitch[PredicateResult] = arrow(item)
override val arrow: Arrow[Q, PredicateResult] = outerArrow
}
/**
* Given some items, this function
* 1. chunks them up in groups
* 2. lazily applies a predicate on each group
* 3. filters based on the predicate
* 4. takes first numToTake items.
*
* If numToTake is satisfied, then any later predicates are not called.
*
* @param items items of type Q
* @param predicate predicate that determines whether an item is acceptable
* @param batchSize batch size to call the predicate with
* @param numToTake max number of items to return
* @param stats stats receiver
* @tparam Q type of item
*
* @return a future of K items
*/
def batchFilterTake[Q](
items: Seq[Q],
predicate: Predicate[Q],
batchSize: Int,
numToTake: Int,
stats: StatsReceiver
): Stitch[Seq[Q]]
def take(
input: Iterator[Stitch[Seq[Q]]],
prev: Seq[Q],
takeSize: Int,
numOfBatch: Int
): Stitch[(Seq[Q], Int)]
def filter[Q](items: Seq[Q], predicate: Predicate[Q]): Stitch[Seq[Q]]
def filter[T, Q](target: T, items: Seq[Q], predicate: Predicate[(T, Q)]): Stitch[Seq[Q]]
object that is the logical "and" of the input predicates
*/
def andConcurrently[Q](predicates: Seq[Predicate[Q]]): Predicate[Q]
abstract class GatedPredicateBase[Q](
underlyingPredicate: Predicate[Q],
stats: StatsReceiver = NullStatsReceiver)
extends Predicate[Q]
def gate(item: Q): Boolean
val underlyingPredicateTotal = stats.counter("underlying_total")
val underlyingPredicateValid = stats.counter("underlying_valid")
val underlyingPredicateInvalid = stats.counter("underlying_invalid")
val notGatedCounter = stats.counter("not_gated")
val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid)
override def apply(item: Q): Stitch[PredicateResult]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\PredicateResult.scala
trait PredicateResult
def value: Boolean
}
object PredicateResult
object Valid extends PredicateResult
class Invalid(reasons: Set[FilterReason] = Set.empty[FilterReason]) extends PredicateResult
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\Ranker.scala
trait Ranker[Target, Candidate] extends Transform[Target, Candidate]
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]]
override def transform(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]]
def observe(statsReceiver: StatsReceiver): Ranker[Target, Candidate]
def rank(target: Target, items: Seq[Candidate]): Stitch[Seq[Candidate]]
def reverse: Ranker[Target, Candidate] = new Ranker[Target, Candidate]
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] =
ranker.rank(target, candidates).map(_.reverse)
}
def andThen(other: Ranker[Target, Candidate]): Ranker[Target, Candidate]
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]]
def within(timeout: Duration, statsReceiver: StatsReceiver): Ranker[Target, Candidate]
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]]
object Ranker
def chain[Target, Candidate](
transformer: Transform[Target, Candidate],
ranker: Ranker[Target, Candidate]
): Ranker[Target, Candidate]
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]]
class IdentityRanker[Target, Candidate] extends Ranker[Target, Candidate]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\RecommendationFlow.scala
class RecommendationResultsConfig(desiredCandidateCount: Int, batchForCandidatesCheck: Int)
trait BaseRecommendationFlow[Target, Candidate <: UniversalNoun[Long]]
def process(
pipelineRequest: Target
): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]]
def mapKey[Target2](fn: Target2 => Target): BaseRecommendationFlow[Target2, Candidate]
def process(
pipelineRequest: Target2
): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] =
original.process(fn(pipelineRequest))
}
}
}
/**
* Defines a typical recommendation flow to fetch, filter, rank and transform candidates.
*
* 1. targetEligibility: determine the eligibility of target request
* 2. candidateSources: fetch candidates from candidate sources based on target type
* 3. preRankerCandidateFilter: light filtering of candidates
* 4. ranker: ranking of candidates (could be composed of multiple stages, light ranking, heavy ranking and etc)
* 5. postRankerTransform: deduping, grouping, rule based promotion / demotions and etc
* 6. validateCandidates: heavy filters to determine the eligibility of the candidates.
* will only be applied to candidates that we expect to return.
* 7. transformResults: transform the individual candidates into desired format (e.g. hydrate social proof)
*
* Note that the actual implementations may not need to implement all the steps if not needed
* (could just leave to IdentityRanker if ranking is not needed).
*
* Theoretically, the actual implementation could override the above flow to add
* more steps (e.g. add a transform step before ranking).
* But it is recommended to add the additional steps into this base flow if the step proves
* to have significant justification, or merge it into an existing step if it is a minor change.
*
* @tparam Target type of target request
* @tparam Candidate type of candidate to return
*/
trait RecommendationFlow[Target, Candidate <: UniversalNoun[Long]]
extends BaseRecommendationFlow[Target, Candidate]
with SideEffectsUtil[Target, Candidate]
def updateTarget(target: Target): Stitch[Target] = Stitch.value(target)
/**
* check if the target is eligible for the flow
*/
protected def targetEligibility: Predicate[Target]
/**
* define the candidate sources that should be used for the given target
*/
protected def candidateSources(target: Target): Seq[CandidateSource[Target, Candidate]]
/**
* filter invalid candidates before the ranking phase.
*/
protected def preRankerCandidateFilter: Predicate[(Target, Candidate)]
/**
* rank the candidates
*/
protected def selectRanker(target: Target): Ranker[Target, Candidate]
/**
* transform the candidates after ranking (e.g. dedupping, grouping and etc)
*/
protected def postRankerTransform: Transform[Target, Candidate]
/**
* filter invalid candidates before returning the results.
*
* Some heavy filters e.g. SGS filter could be applied in this step
*/
protected def validateCandidates: Predicate[(Target, Candidate)]
/**
* transform the candidates into results and return
*/
protected def transformResults: Transform[Target, Candidate]
/**
* configuration for recommendation results
*/
protected def resultsConfig(target: Target): RecommendationResultsConfig
/**
* track the quality factor the recommendation pipeline
*/
protected def qualityFactorObserver: Option[QualityFactorObserver] = None
def statsReceiver: StatsReceiver
/**
* high level monitoring for the whole flow
* (make sure to add monitoring for each individual component by yourself)
*
* additional candidates: count, stats, non_empty_count
* target eligibility: latency, success, failures, request, count, valid_count, invalid_count, invalid_reasons
* candidate generation: latency, success, failures, request, count, non_empty_count, results_stat
* pre ranker filter: latency, success, failures, request, count, non_empty_count, results_stat
* ranker: latency, success, failures, request, count, non_empty_count, results_stat
* post ranker: latency, success, failures, request, count, non_empty_count, results_stat
* filter and take: latency, success, failures, request, count, non_empty_count, results_stat, batch count
* transform results: latency, success, failures, request, count, non_empty_count, results_stat
*/
import RecommendationFlow._
lazy val additionalCandidatesStats = statsReceiver.scope(AdditionalCandidatesStats)
lazy val targetEligibilityStats = statsReceiver.scope(TargetEligibilityStats)
lazy val candidateGenerationStats = statsReceiver.scope(CandidateGenerationStats)
lazy val preRankerFilterStats = statsReceiver.scope(PreRankerFilterStats)
lazy val rankerStats = statsReceiver.scope(RankerStats)
lazy val postRankerTransformStats = statsReceiver.scope(PostRankerTransformStats)
lazy val filterAndTakeStats = statsReceiver.scope(FilterAndTakeStats)
lazy val transformResultsStats = statsReceiver.scope(TransformResultsStats)
lazy val overallStats = statsReceiver.scope(OverallStats)
import StatsUtil._
override def process(
pipelineRequest: Target
): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]]
def processValidTarget(
target: Target,
additionalCandidates: Seq[Candidate]
): Stitch[Seq[Candidate]]
def take(
target: Target,
candidates: Seq[Candidate],
config: RecommendationResultsConfig
): Stitch[Seq[Candidate]]
object RecommendationFlow
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\SideEffectsUtil.scala
trait SideEffectsUtil[Target, Candidate]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\StatsUtil.scala
object StatsUtil
def profileStitch[T](stitch: Stitch[T], stat: StatsReceiver): Stitch[T]
def profileArrow[T, U](arrow: Arrow[T, U], stat: StatsReceiver): Arrow[T, U]
def profileResults[T](results: T, stat: StatsReceiver, size: T => Int): T
def profileSeqResults[T](results: Seq[T], stat: StatsReceiver): Seq[T]
def profileStitchResults[T](stitch: Stitch[T], stat: StatsReceiver, size: T => Int): Stitch[T]
def profileArrowResults[T, U](
arrow: Arrow[T, U],
stat: StatsReceiver,
size: U => Int
): Arrow[T, U]
def profileStitchSeqResults[T](stitch: Stitch[Seq[T]], stat: StatsReceiver): Stitch[Seq[T]]
def profileStitchOptionalResults[T](
stitch: Stitch[Option[T]],
stat: StatsReceiver
): Stitch[Option[T]]
def profileStitchMapResults[K, V](
stitch: Stitch[Map[K, V]],
stat: StatsReceiver
): Stitch[Map[K, V]]
def getCleanClassName(obj: Object): String =
obj.getClass.getSimpleName.stripSuffix("$")
/**
* Helper function for timing a stitch and count a list of PredicateResult
*/
def profilePredicateResults(
predicateResult: Stitch[Seq[PredicateResult]],
statsReceiver: StatsReceiver
): Stitch[Seq[PredicateResult]]
def profilePredicateResult(
predicateResult: Stitch[PredicateResult],
statsReceiver: StatsReceiver
): Stitch[PredicateResult]
def profilePredicateResults[Q](
predicateResult: Arrow[Q, Seq[PredicateResult]],
statsReceiver: StatsReceiver
): Arrow[Q, Seq[PredicateResult]]
def profilePredicateResult[Q](
predicateResult: Arrow[Q, PredicateResult],
statsReceiver: StatsReceiver
): Arrow[Q, PredicateResult]
def profileStitchSeqResults[T](
stats: StatsReceiver
)(
block: => Stitch[Seq[T]]
): Stitch[Seq[T]]
def profileStitch[A](stat: Stat, unit: TimeUnit)(f: => Stitch[A]): Stitch[A]
def observeStitchQualityFactor[T](
stitch: Stitch[T],
qualityFactorObserverOption: Option[QualityFactorObserver],
statsReceiver: StatsReceiver
): Stitch[T]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\Transform.scala
trait Transform[-T, C]
def transformItem(target: T, item: C): Stitch[C]
def transform(target: T, items: Seq[C]): Stitch[Seq[C]]
def mapTarget[T2](mapper: T2 => T): Transform[T2, C]
def transformItem(target: T2, item: C): Stitch[C]
def transform(target: T2, items: Seq[C]): Stitch[Seq[C]]
def andThen[T1 <: T](other: Transform[T1, C]): Transform[T1, C]
def transformItem(target: T1, item: C): Stitch[C] =
original.transformItem(target, item).flatMap(other.transformItem(target, _))
override def transform(target: T1, items: Seq[C]): Stitch[Seq[C]] =
original.transform(target, items).flatMap(other.transform(target, _))
}
}
def observe(statsReceiver: StatsReceiver): Transform[T, C]
def transform(target: T, items: Seq[C]): Stitch[Seq[C]]
def transformItem(target: T, item: C): Stitch[C]
trait GatedTransform[T <: HasParams, C] extends Transform[T, C]
def gated(param: Param[Boolean]): Transform[T, C]
object Transform
class IdentityTransform[T, C] extends Transform[T, C]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\AddressBookParams.scala
object AddressBookParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\ForwardEmailBookSource.scala
class ForwardEmailBookSource @Inject() (
forwardEmailBookClientColumn: ForwardEmailBookClientColumn,
addressBookClient: AddressbookClient,
statsReceiver: StatsReceiver = NullStatsReceiver)
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
def apply(
target: HasParams with HasClientContext
): Stitch[Seq[CandidateUser]]
object ForwardEmailBookSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\ForwardPhoneBookSource.scala
class ForwardPhoneBookSource @Inject() (
forwardPhoneContactsClientColumn: ForwardPhoneContactsClientColumn,
addressBookClient: AddressbookClient,
statsReceiver: StatsReceiver = NullStatsReceiver)
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]]
object ForwardPhoneBookSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\ReverseEmailBookSource.scala
class ReverseEmailBookSource @Inject() (
reverseEmailContactsClientColumn: ReverseEmailContactsClientColumn,
essClient: EmailStorageServiceClient,
addressBookClient: AddressbookClient,
statsReceiver: StatsReceiver = NullStatsReceiver)
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]]
object ReverseEmailBookSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\ReversePhoneBookSource.scala
class ReversePhoneBookSource @Inject() (
reversePhoneContactsClientColumn: ReversePhoneContactsClientColumn,
pssClient: PhoneStorageServiceClient,
addressBookClient: AddressbookClient,
statsReceiver: StatsReceiver = NullStatsReceiver)
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]]
object ReversePhoneBookSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\CachedCandidateSource.scala
class CachedCandidateSource[K <: Object, V <: Object](
candidateSource: CandidateSource[K, V],
maxCacheSize: Int,
cacheTTL: Duration,
statsReceiver: StatsReceiver,
override val identifier: CandidateSourceIdentifier)
extends CandidateSource[K, V]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\ExperimentalCandidateSource.scala
class ExperimentalCandidateSource[T <: HasParams, V](
baseSource: CandidateSource[T, V],
darkreadAlgorithmParam: Param[Boolean],
keepCandidatesParam: Param[Boolean],
resultCountThresholdParam: Param[Int],
baseStatsReceiver: StatsReceiver)
extends CandidateSource[T, V]
def apply(request: T): Stitch[Seq[V]]
def fetchFromCandidateSourceAndProcessResults(request: T): Stitch[Seq[V]]
def processResults(results: Seq[V], keepResults: Boolean): Seq[V]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\RealGraphExpansionRepository.scala
class InterestExpansionCandidate(
userID: Long,
score: Double,
features: RawYMBIICandidateFeatures)
abstract class RealGraphExpansionRepository[Request](
realgraphExpansionStore: Fetcher[
Long,
Unit,
CandidatesFollowedV1
],
override val identifier: CandidateSourceIdentifier,
statsReceiver: StatsReceiver = NullStatsReceiver,
maxUnderlyingCandidatesToQuery: Int = 50,
maxCandidatesToReturn: Int = 40,
overrideUnderlyingTimeout: Option[Duration] = None,
appendSocialProof: Boolean = false)
extends CandidateSource[
Request,
CandidateUser
]
def apply(
request: Request,
): Stitch[Seq[CandidateUser]]
def rerankCandidateExpansions(
underlyingCandidatesMap: Map[Long, Double],
expansionCandidateMap: Map[Long, Option[CandidatesFollowedV1]]
): Seq[InterestExpansionCandidate]
object RealGraphExpansionRepository
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SimilarUserExpanderParams.scala
object SimilarUserExpanderParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SimilarUserExpanderRepository.scala
class SecondDegreeCandidate(userId: Long, score: Double, socialProof: Option[Seq[Long]])
abstract class SimilarUserExpanderRepository[-Request <: HasParams](
override val identifier: CandidateSourceIdentifier,
similarToCandidatesFetcher: Fetcher[
Long,
Unit,
Candidates
],
expansionInputSizeParam: FSBoundedParam[Int] = DefaultExpansionInputCount,
candidatesReturnedSizeParam: FSBoundedParam[Int] = DefaultFinalCandidatesReturnedCount,
enableImplicitEngagedExpansion: FSParam[Boolean] = DefaultEnableImplicitEngagedExpansion,
thresholdToAvoidExpansion: Int = 30,
maxExpansionPerCandidate: Option[Int] = None,
includingOriginalCandidates: Boolean = false,
scorer: (Double, Double) => Double = SimilarUserExpanderRepository.DefaultScorer,
aggregator: (Seq[Double]) => Double = ScoreAggregator.Max,
candidateBuilder: (Long, CandidateSourceIdentifier, Double, CandidateUser) => CandidateUser =
DefaultCandidateBuilder)
extends TwoHopExpansionCandidateSource[
Request,
CandidateUser,
SecondDegreeCandidate,
CandidateUser
]
def firstDegreeNodes(request: Request): Stitch[Seq[CandidateUser]]
def secondaryDegreeNodes(
request: Request,
firstDegreeCandidate: CandidateUser
): Stitch[Seq[SecondDegreeCandidate]]
def aggregateAndScore(
req: Request,
firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SecondDegreeCandidate]]
): Stitch[Seq[CandidateUser]]
def aggregateCandidateSourceDetails(
candidates: Seq[CandidateUser]
): Option[UserCandidateSourceDetails]
def aggregateAccountSocialProof(candidates: Seq[CandidateUser]): Option[Reason]
def getCandidatesAfterImplicitEngagementFiltering(
params: Params,
firstDegreeCandidatesStitch: Stitch[Seq[CandidateUser]]
): Stitch[Seq[CandidateUser]]
object SimilarUserExpanderRepository
object ScoreAggregator
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SocialProofEnforcedCandidateSource.scala
abstract class SocialProofEnforcedCandidateSource(
candidateSource: CandidateSource[HasClientContext with HasParams, CandidateUser],
modifySocialProof: ModifySocialProof,
minNumSocialProofsRequired: Int,
override val identifier: CandidateSourceIdentifier,
baseStatsReceiver: StatsReceiver)
extends CandidateSource[HasClientContext with HasParams, CandidateUser]
def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SocialProofEnforcedCandidateSourceFSConfig.scala
class SocialProofEnforcedCandidateSourceFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SocialProofEnforcedCandidateSourceParams.scala
object SocialProofEnforcedCandidateSourceParams
object MustCallSgs
extends FSParam[Boolean]("social_proof_enforced_candidate_source_must_call_sgs", true)
case object CallSgsCachedColumn
extends FSParam[Boolean](
"social_proof_enforced_candidate_source_call_sgs_cached_column",
false)
case object QueryIntersectionIdsNum
extends FSBoundedParam[Int](
name = "social_proof_enforced_candidate_source_query_intersection_ids_num",
default = 3,
min = 0,
max = Integer.MAX_VALUE)
case object MaxNumCandidatesToAnnotate
extends FSBoundedParam[Int](
name = "social_proof_enforced_candidate_source_max_num_candidates_to_annotate",
default = 50,
min = 0,
max = Integer.MAX_VALUE)
case object GfsIntersectionIdsNum
extends FSBoundedParam[Int](
name = "social_proof_enforced_candidate_source_gfs_intersection_ids_num",
default = 3,
min = 0,
max = Integer.MAX_VALUE)
case object SgsIntersectionIdsNum
extends FSBoundedParam[Int](
name = "social_proof_enforced_candidate_source_sgs_intersection_ids_num",
default = 10,
min = 0,
max = Integer.MAX_VALUE)
case object GfsLagDurationInDays
extends FSBoundedParam[Duration](
name = "social_proof_enforced_candidate_source_gfs_lag_duration_in_days",
default = 14.days,
min = 1.days,
max = 60.days)
with HasDurationConversion
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\StratoFetcherSource.scala
abstract class StratoFetcherSource[K, U, V](
fetcher: Fetcher[K, U, V],
view: U,
override val identifier: CandidateSourceIdentifier)
extends CandidateSource[K, CandidateUser]
def map(user: K, v: V): Seq[CandidateUser]
override def apply(target: K): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\TweetAuthorsCandidateSource.scala
trait for tweet authors based algorithms, e.g. topical tweet authors, twistly, ...
*
* @tparam Target target type
* @tparam Candidate output candidate types
*/
trait TweetAuthorsCandidateSource[-Target, +Candidate] extends CandidateSource[Target, Candidate]
def getTweetCandidates(target: Target): Stitch[Seq[TweetCandidate]]
/**
* fetch authorId
*/
def getTweetAuthorId(tweetCandidate: TweetCandidate): Stitch[Option[Long]]
/**
* wrap candidate ID and TweetAuthorProof in Candidate
*/
def toCandidate(authorId: Long, tweetIds: Seq[Long], score: Option[Double]): Candidate
/**
* aggregate scores, default to the first score
*/
def aggregator(scores: Seq[Double]): Double =
scores.headOption.getOrElse(TweetAuthorsCandidateSource.DefaultScore)
/**
* aggregation method for a group of tweet candidates
*/
def aggregateAndScore(
target: Target,
tweetCandidates: Seq[TweetCandidate]
): Seq[Candidate]
/**
* generate a list of candidates for the target
*/
def build(
target: Target
): Stitch[Seq[Candidate]]
def apply(target: Target): Stitch[Seq[Candidate]] =
build(target)
}
object TweetAuthorsCandidateSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\TwoHopExpansionCandidateSource.scala
trait for two-hop expansion based algorithms, e.g. online_stp, phonebook_prediction,
* recent following sims, recent engagement sims, ...
*
* @tparam Target target type
* @tparam FirstDegree type of first degree nodes
* @tparam SecondaryDegree type of secondary degree nodes
* @tparam Candidate output candidate types
*/
trait TwoHopExpansionCandidateSource[-Target, FirstDegree, SecondaryDegree, +Candidate]
extends CandidateSource[Target, Candidate]
def firstDegreeNodes(req: Target): Stitch[Seq[FirstDegree]]
/**
* fetch secondary degree nodes given request and first degree nodes
*/
def secondaryDegreeNodes(req: Target, node: FirstDegree): Stitch[Seq[SecondaryDegree]]
/**
* aggregate and score the candidates to generate final results
*/
def aggregateAndScore(
req: Target,
firstDegreeToSecondDegreeNodesMap: Map[FirstDegree, Seq[SecondaryDegree]]
): Stitch[Seq[Candidate]]
/**
* Generate a list of candidates for the target
*/
def apply(target: Target): Stitch[Seq[Candidate]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\crowd_search_accounts\CrowdSearchAccountsFSConfig.scala
class CrowdSearchAccountsFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\crowd_search_accounts\CrowdSearchAccountsParams.scala
object CrowdSearchAccountsParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\crowd_search_accounts\CrowdSearchAccountsSource.scala
object AccountsFilteringAndRankingLogicId extends Enumeration
object CrowdSearchAccountsSource
class CrowdSearchAccountsSource @Inject() (
crowdSearchAccountsClientColumn: CrowdSearchAccountsClientColumn,
statsReceiver: StatsReceiver,
) extends CandidateSource[CrowdSearchAccountsSource.Target, CandidateUser]
with Logging
def apply(
target: CrowdSearchAccountsSource.Target
): Stitch[Seq[CandidateUser]]
def transformCrowdSearchAccountsToCandidateSource(
crowdSearchAccounts: Seq[Option[CrowdSearchAccounts]]
): Seq[CandidateUser]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\BasePopGeoHashSource.scala
class BasePopGeohashSource @Inject() (
popGeoSource: CandidateSource[String, CandidateUser],
statsReceiver: StatsReceiver)
extends CandidateSource[
HasParams with HasClientContext with HasGeohashAndCountryCode,
CandidateUser
]
with BasePopGeohashSourceConfig
def apply(
target: HasParams with HasClientContext with HasGeohashAndCountryCode
): Stitch[Seq[CandidateUser]]
trait BasePopGeohashSourceConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopCountryBackFillSource.scala
class PopCountryBackFillSource @Inject() (popGeoSource: PopGeoSource)
extends CandidateSource[HasClientContext with HasParams, CandidateUser]
def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]]
object PopCountryBackFillSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopCountrySource.scala
class PopCountrySource @Inject() (
popGeoSource: PopGeoSource,
statsReceiver: StatsReceiver)
extends CandidateSource[
HasClientContext with HasParams with HasUserState with HasGeohashAndCountryCode,
CandidateUser
]
def apply(
target: HasClientContext with HasParams with HasUserState with HasGeohashAndCountryCode
): Stitch[Seq[CandidateUser]]
object PopCountrySource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeohashSource.scala
class PopGeohashSource @Inject() (
popGeoSource: PopGeoSource,
statsReceiver: StatsReceiver)
extends BasePopGeohashSource(
popGeoSource = popGeoSource,
statsReceiver = statsReceiver.scope("PopGeohashSource"),
)
def candidateSourceEnabled(target: Target): Boolean = true
override val identifier: CandidateSourceIdentifier = PopGeohashSource.Identifier
override def minGeohashLength(target: Target): Int
def maxResults(target: Target): Int
def maxGeohashLength(target: Target): Int
def returnResultFromAllPrecision(target: Target): Boolean
object PopGeohashSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoQualityFollowSource.scala
class PopGeohashQualityFollowSource @Inject() (
popGeoSource: PopGeoQualityFollowSource,
statsReceiver: StatsReceiver)
extends BasePopGeohashSource(
popGeoSource = popGeoSource,
statsReceiver = statsReceiver.scope("PopGeohashQualityFollowSource"),
)
def maxResults(target: Target): Int
def minGeohashLength(target: Target): Int
def maxGeohashLength(target: Target): Int
def returnResultFromAllPrecision(target: Target): Boolean
def candidateSourceEnabled(target: Target): Boolean
object PopGeohashQualityFollowSource
object PopGeoQualityFollowSource
class PopGeoQualityFollowSource @Inject() (
popGeoQualityFollowClientColumn: UniquePopQualityFollowUsersInPlaceClientColumn,
statsReceiver: StatsReceiver,
) extends CandidateSource[String, CandidateUser]
def apply(target: String): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoQualityFollowSourceFSConfig.scala
class PopGeoQualityFollowSourceFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoQualityFollowSourceParams.scala
object PopGeoQualityFollowSourceParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoSource.scala
class BasePopGeoSource @Inject() (
@Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER) fetcher: Fetcher[
String,
Unit,
PopUsersInPlace
]) extends StratoFetcherWithUnitViewSource[String, PopUsersInPlace](
fetcher,
BasePopGeoSource.Identifier)
def map(target: String, candidates: PopUsersInPlace): Seq[CandidateUser] =
BasePopGeoSource.map(target, candidates)
}
object BasePopGeoSource
def map(target: String, candidates: PopUsersInPlace): Seq[CandidateUser] =
candidates.popUsers.sortBy(-_.score).take(BasePopGeoSource.MaxResults).view.map
class PopGeoSource @Inject() (basePopGeoSource: BasePopGeoSource, statsReceiver: StatsReceiver)
extends CachedCandidateSource[String, CandidateUser](
basePopGeoSource,
PopGeoSource.MaxCacheSize,
PopGeoSource.CacheTTL,
statsReceiver,
PopGeoSource.Identifier)
object PopGeoSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoSourceFSConfig.scala
class PopGeoSourceFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoSourceParams.scala
object PopGeoSourceParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\ppmi_locale_follow\PPMILocaleFollowSource.scala
class PPMILocaleFollowSource @Inject() (
userPreferredLanguagesOnUserClientColumn: UserPreferredLanguagesOnUserClientColumn,
localeFollowPpmiClientColumn: LocaleFollowPpmiClientColumn,
statsReceiver: StatsReceiver)
extends CandidateSource[HasClientContext with HasParams, CandidateUser]
def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]]
def getPPMILocaleFollowCandidates(
locales: Seq[String]
): Stitch[Seq[CandidateUser]]
def getPreferredLocales(userId: Long, countryCode: String): Stitch[Seq[String]]
object PPMILocaleFollowSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\ppmi_locale_follow\PPMILocaleFollowSourceFSConfig.scala
class PPMILocaleFollowSourceFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\ppmi_locale_follow\PPMILocaleFollowSourceParams.scala
class PPMILocaleFollowSourceParams
object PPMILocaleFollowSourceParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\promoted_accounts\PromotedAccountsCandidateSource.scala
class PromotedCandidateUser(
id: Long,
position: Int,
adImpression: adthrift.AdImpression,
followProof: FollowProof,
primaryCandidateSource: Option[CandidateSourceIdentifier])
@Singleton
class PromotedAccountsCandidateSource @Inject() (
adserverClient: AdserverClient,
sgsClient: SocialGraphClient,
statsReceiver: StatsReceiver)
extends CandidateSource[AdRequest, PromotedCandidateUser]
with Logging
def apply(request: AdRequest): Stitch[Seq[PromotedCandidateUser]]
def shouldShowSocialContext(imp: adthrift.AdImpression): Boolean =
imp.experimentValues.exists
def getInsertionPositionDefaultValue(isTest: Boolean): Int
def profileNumResults(resultsSize: Int, statName: String): Unit
object PromotedAccountsCandidateSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\real_graph\RealGraphOonFSConfig.scala
class RealGraphOonFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\real_graph\RealGraphOonParams.scala
object RealGraphOonParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\real_graph\RealGraphOonV2Source.scala
class RealGraphOonV2Source @Inject() (
realGraphClientColumn: UserRealgraphOonV2ClientColumn)
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]]
def parseStratoResults(
request: HasParams with HasClientContext,
candidateSeqThrift: CandidateSeq
): Seq[CandidateUser]
object RealGraphOonV2Source
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\real_graph\RealGraphSource.scala
class RealGraphSource @Inject() (
realGraph: RealTimeRealGraphClient)
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]]
object RealGraphSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RecentEngagementDirectFollowSource.scala
class RecentEngagementDirectFollowSource @Inject() (
realTimeRealGraphClient: RealTimeRealGraphClient)
extends CandidateSource[Long, CandidateUser]
def apply(targetUserId: Long): Stitch[Seq[CandidateUser]]
object RecentEngagementDirectFollowSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RecentEngagementNonDirectFollowSource.scala
class RecentEngagementNonDirectFollowSource @Inject() (
realTimeRealGraphClient: RealTimeRealGraphClient)
extends CandidateSource[Long, CandidateUser]
def apply(targetUserId: Long): Stitch[Seq[CandidateUser]]
object RecentEngagementNonDirectFollowSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RepeatedProfileVisitsFSConfig.scala
class RepeatedProfileVisitsFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RepeatedProfileVisitsParams.scala
object RepeatedProfileVisitsParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RepeatedProfileVisitsSource.scala
class RepeatedProfileVisitsSource @Inject() (
repeatedProfileVisitsAggregateClientColumn: RepeatedProfileVisitsAggregateClientColumn,
realTimeRealGraphClient: RealTimeRealGraphClient,
statsReceiver: StatsReceiver)
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
with Logging
def applyWithOfflineDataset(targetUserId: Long): Stitch[Map[Long, Int]]
def applyWithOnlineData(targetUserId: Long): Stitch[Map[Long, Int]]
def getRepeatedVisitedAccounts(params: Params, targetUserId: Long): Stitch[Map[Long, Int]]
def getRecommendations(params: Params, userId: Long): Stitch[Seq[CandidateUser]]
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]]
object RepeatedProfileVisitsSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\salsa\RecentEngagementDirectFollowSalsaExpansionSource.scala
class RecentEngagementDirectFollowSalsaExpansionSource @Inject() (
realTimeRealGraphClient: RealTimeRealGraphClient,
salsaExpander: SalsaExpander)
extends SalsaExpansionBasedCandidateSource[Long](salsaExpander)
def firstDegreeNodes(target: Long): Stitch[Seq[Long]] = realTimeRealGraphClient
.getUsersRecentlyEngagedWith(
target,
RealTimeRealGraphClient.EngagementScoreMap,
includeDirectFollowCandidates = true,
includeNonDirectFollowCandidates = false
).map
def maxResults(target: Long): Int =
RecentEngagementDirectFollowSalsaExpansionSource.OutputSize
}
object RecentEngagementDirectFollowSalsaExpansionSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\salsa\SalsaExpander.scala
class SalsaExpandedCandidate(
candidateId: Long,
numberOfConnections: Int,
totalScore: Double,
connectingUsers: Seq[Long])
def toCandidateUser: CandidateUser =
CandidateUser(
id = candidateId,
score = Some(totalScore),
reason = Some(Reason(
Some(AccountProof(followProof = Some(FollowProof(connectingUsers, connectingUsers.size))))))
)
}
case class SimilarUserCandidate(candidateId: Long, score: Double, similarToCandidate: Long)
/**
* Salsa expander uses pre-computed lists of candidates for each input user id and returns the highest scored candidates in the pre-computed lists as the expansion for the corresponding input id.
*/
@Singleton
class SalsaExpander @Inject() (
statsReceiver: StatsReceiver,
firstDegreeClient: SalsaFirstDegreeOnUserClientColumn,
secondDegreeClient: SalsaSecondDegreeOnUserClientColumn,
)
def similarUsers(
input: Seq[Long],
neighbors: Seq[Option[Seq[Candidate]]]
): Seq[SalsaExpandedCandidate]
def apply(
firstDegreeInput: Seq[Long],
secondDegreeInput: Seq[Long],
maxNumOfCandidatesToReturn: Int
): Stitch[Seq[CandidateUser]]
object SalsaExpander
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\salsa\SalsaExpansionBasedCandidateSource.scala
abstract class SalsaExpansionBasedCandidateSource[Target](salsaExpander: SalsaExpander)
extends CandidateSource[Target, CandidateUser]
class
def firstDegreeNodes(target: Target): Stitch[Seq[Long]] = Stitch.value(Seq())
def secondDegreeNodes(target: Target): Stitch[Seq[Long]] = Stitch.value(Seq())
// max number output results
def maxResults(target: Target): Int
override def apply(target: Target): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\CacheBasedSimsStore.scala
class CacheBasedSimsStore(
id: CandidateSourceIdentifier,
fetcher: Fetcher[Long, Unit, Candidates],
maxCacheSize: Int,
cacheTtl: Duration,
statsReceiver: StatsReceiver)
extends CandidateSource[HasParams with HasSimilarToContext, CandidateUser]
def getUsersFromSimsSource(userId: JLong): Stitch[Option[Candidates]]
def apply(request: HasParams with HasSimilarToContext): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\DBV2SimsRefreshStore.scala
class DBV2SimsRefreshStore @Inject() (
newSimsRefreshOnUserClientColumn: NewSimsRefreshOnUserClientColumn)
extends StratoBasedSimsCandidateSourceWithUnitView(
fetcher = newSimsRefreshOnUserClientColumn.fetcher,
identifier = DBV2SimsRefreshStore.Identifier)
@Singleton
class CachedDBV2SimsRefreshStore @Inject() (
newSimsRefreshOnUserClientColumn: NewSimsRefreshOnUserClientColumn,
statsReceiver: StatsReceiver)
extends CacheBasedSimsStore(
id = DBV2SimsRefreshStore.Identifier,
fetcher = newSimsRefreshOnUserClientColumn.fetcher,
maxCacheSize = DBV2SimsRefreshStore.MaxCacheSize,
cacheTtl = DBV2SimsRefreshStore.CacheTTL,
statsReceiver = statsReceiver.scope("CachedDBV2SimsRefreshStore", "cache")
)
object DBV2SimsRefreshStore
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\DBV2SimsStore.scala
class DBV2SimsStore @Inject() (
@Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates])
extends StratoBasedSimsCandidateSourceWithUnitView(
fetcher,
identifier = DBV2SimsStore.Identifier)
@Singleton
class CachedDBV2SimsStore @Inject() (
@Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates],
statsReceiver: StatsReceiver)
extends CacheBasedSimsStore(
id = DBV2SimsStore.Identifier,
fetcher = fetcher,
maxCacheSize = DBV2SimsStore.MaxCacheSize,
cacheTtl = DBV2SimsStore.CacheTTL,
statsReceiver = statsReceiver.scope("CachedDBV2SimsStore", "cache")
)
object DBV2SimsStore
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\Follow2vecNearestNeighborsStore.scala
class LinearRegressionFollow2vecNearestNeighborsStore @Inject() (
linearRegressionFollow2vecNearestNeighborsClientColumn: LinearRegressionFollow2vecNearestNeighborsClientColumn)
extends StratoBasedSimsCandidateSource[NearestNeighborParamsType](
Follow2vecNearestNeighborsStore.convertFetcher(
linearRegressionFollow2vecNearestNeighborsClientColumn.fetcher),
view = Follow2vecNearestNeighborsStore.defaultSearchParams,
identifier = Follow2vecNearestNeighborsStore.IdentifierF2vLinearRegression
)
object Follow2vecNearestNeighborsStore
def convertFetcher(
fetcher: Fetcher[NearestNeighborKeyType, NearestNeighborParamsType, NearestNeighborValueType]
): Fetcher[Long, NearestNeighborParamsType, Candidates]
def toCandidates(
results: Option[NearestNeighborValueType]
): Option[Candidates]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SimsExperimentalStore.scala
class SimsExperimentalStore @Inject() (
simsExperimentalOnUserClientColumn: SimilarUsersBySimsExperimentalOnUserClientColumn)
extends StratoBasedSimsCandidateSourceWithUnitView(
fetcher = simsExperimentalOnUserClientColumn.fetcher,
identifier = SimsExperimentalStore.Identifier
)
@Singleton
class CachedSimsExperimentalStore @Inject() (
simsExperimentalOnUserClientColumn: SimilarUsersBySimsExperimentalOnUserClientColumn,
statsReceiver: StatsReceiver)
extends CacheBasedSimsStore(
id = SimsExperimentalStore.Identifier,
fetcher = simsExperimentalOnUserClientColumn.fetcher,
maxCacheSize = SimsExperimentalStore.MaxCacheSize,
cacheTtl = SimsExperimentalStore.CacheTTL,
statsReceiver = statsReceiver.scope("CachedSimsExperimentalStore", "cache")
)
object SimsExperimentalStore
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SimsSourceFSConfig.scala
class SimsSourceFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SimsSourceParams.scala
object SimsSourceParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SimsStore.scala
class SimsStore @Inject() (
@Named(GuiceNamedConstants.SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates])
extends StratoBasedSimsCandidateSourceWithUnitView(fetcher, identifier = SimsStore.Identifier)
@Singleton
class CachedSimsStore @Inject() (
@Named(GuiceNamedConstants.SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates],
statsReceiver: StatsReceiver)
extends CacheBasedSimsStore(
id = SimsStore.Identifier,
fetcher = fetcher,
maxCacheSize = SimsStore.MaxCacheSize,
cacheTtl = SimsStore.CacheTTL,
statsReceiver = statsReceiver.scope("CachedSimsStore", "cache")
)
object SimsStore
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\StratoBasedSimsCandidateSource.scala
abstract class StratoBasedSimsCandidateSource[U](
fetcher: Fetcher[Long, U, Candidates],
view: U,
override val identifier: CandidateSourceIdentifier)
extends StratoFetcherSource[Long, U, Candidates](fetcher, view, identifier)
def map(target: Long, candidates: Candidates): Seq[CandidateUser] =
StratoBasedSimsCandidateSource.map(target, candidates)
}
object StratoBasedSimsCandidateSource
def map(target: Long, candidates: Candidates): Seq[CandidateUser]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SwitchingSimsSource.scala
class SwitchingSimsSource @Inject() (
cachedDBV2SimsStore: CachedDBV2SimsStore,
cachedDBV2SimsRefreshStore: CachedDBV2SimsRefreshStore,
cachedSimsExperimentalStore: CachedSimsExperimentalStore,
cachedSimsStore: CachedSimsStore,
statsReceiver: StatsReceiver = NullStatsReceiver)
extends CandidateSource[HasParams with HasSimilarToContext, CandidateUser]
def apply(request: HasParams with HasSimilarToContext): Stitch[Seq[CandidateUser]]
object SwitchingSimsSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\DBV2SimsExpansionParams.scala
object DBV2SimsExpansionParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentEngagementSimilarUsersFSConfig.scala
class RecentEngagementSimilarUsersFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentEngagementSimilarUsersParams.scala
object RecentEngagementSimilarUsersParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentEngagementSimilarUsersSource.scala
class RecentEngagementSimilarUsersSource @Inject() (
realTimeRealGraphClient: RealTimeRealGraphClient,
switchingSimsSource: SwitchingSimsSource,
statsReceiver: StatsReceiver)
extends SimsExpansionBasedCandidateSource[HasClientContext with HasParams](
switchingSimsSource)
def maxSecondaryDegreeNodes(req: HasClientContext with HasParams): Int = Int.MaxValue
override def maxResults(req: HasClientContext with HasParams): Int =
RecentEngagementSimilarUsersSource.MaxResults
override val identifier: CandidateSourceIdentifier = RecentEngagementSimilarUsersSource.Identifier
private val stats = statsReceiver.scope(identifier.name)
private val calibratedScoreCounter = stats.counter("calibrated_scores_counter")
override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double
def calibrateDivisor(req: HasClientContext with HasParams): Double
def calibrateScore(
candidateScore: Double,
req: HasClientContext with HasParams
): Double
def firstDegreeNodes(
target: HasClientContext with HasParams
): Stitch[Seq[CandidateUser]]
def aggregateAndScore(
request: HasClientContext with HasParams,
firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SimilarUser]]
): Stitch[Seq[CandidateUser]]
object RecentEngagementSimilarUsersSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentFollowingSimilarUsersParams.scala
object RecentFollowingSimilarUsersParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentFollowingSimilarUsersSource.scala
object RecentFollowingSimilarUsersSource
class RecentFollowingSimilarUsersSource @Inject() (
socialGraph: SocialGraphClient,
switchingSimsSource: SwitchingSimsSource,
statsReceiver: StatsReceiver)
extends SimsExpansionBasedCandidateSource[
HasParams with HasRecentFollowedUserIds with HasClientContext
](switchingSimsSource)
def firstDegreeNodes(
request: HasParams with HasRecentFollowedUserIds with HasClientContext
): Stitch[Seq[CandidateUser]]
def maxSecondaryDegreeNodes(
req: HasParams with HasRecentFollowedUserIds with HasClientContext
): Int
def maxResults(
req: HasParams with HasRecentFollowedUserIds with HasClientContext
): Int
def scoreCandidate(sourceScore: Double, similarToScore: Double): Double
def calibrateDivisor(
req: HasParams with HasRecentFollowedUserIds with HasClientContext
): Double
def calibrateScore(
candidateScore: Double,
req: HasParams with HasRecentFollowedUserIds with HasClientContext
): Double
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentStrongEngagementDirectFollowSimilarUsersSource.scala
class RecentStrongEngagementDirectFollowSimilarUsersSource @Inject() (
realTimeRealGraphClient: RealTimeRealGraphClient,
switchingSimsSource: SwitchingSimsSource)
extends SimsExpansionBasedCandidateSource[HasClientContext with HasParams](
switchingSimsSource)
def firstDegreeNodes(
request: HasClientContext with HasParams
): Stitch[Seq[CandidateUser]] = request.getOptionalUserId
.map
def maxSecondaryDegreeNodes(request: HasClientContext with HasParams): Int = Int.MaxValue
override def maxResults(request: HasClientContext with HasParams): Int =
RecentStrongEngagementDirectFollowSimilarUsersSource.MaxResults
override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double
def calibrateDivisor(req: HasClientContext with HasParams): Double = 1.0d
}
object RecentStrongEngagementDirectFollowSimilarUsersSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\SimsExpansionBasedCandidateSource.scala
class SimilarUser(candidateId: Long, similarTo: Long, score: Double)
abstract class SimsExpansionBasedCandidateSource[-Target <: HasParams](
switchingSimsSource: SwitchingSimsSource)
extends TwoHopExpansionCandidateSource[Target, CandidateUser, SimilarUser, CandidateUser]
def maxSecondaryDegreeNodes(req: Target): Int
// max number output results
def maxResults(req: Target): Int
// scorer to score candidate based on first and second degree node scores
def scoreCandidate(source: Double, similarToScore: Double): Double
def calibrateDivisor(req: Target): Double
def calibrateScore(candidateScore: Double, req: Target): Double
def secondaryDegreeNodes(req: Target, node: CandidateUser): Stitch[Seq[SimilarUser]]
def aggregateAndScore(
request: Target,
firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SimilarUser]]
): Stitch[Seq[CandidateUser]]
object SimsExpansionBasedCandidateSource
object ScoreAggregator
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\SimsExpansionFSConfig.scala
class SimsExpansionFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\SimsExpansionSourceParams.scala
object SimsExpansionSourceParams
object Aggregator
extends FSEnumParam[SimsExpansionSourceAggregatorId.type](
name = "sims_expansion_aggregator_id",
default = SimsExpansionSourceAggregatorId.Sum,
enum = SimsExpansionSourceAggregatorId)
}
object SimsExpansionSourceAggregatorId extends Enumeration
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\socialgraph\RecentFollowingRecentFollowingExpansionSource.scala
class RecentFollowingRecentFollowingExpansionSource @Inject() (
socialGraphClient: SocialGraphClient,
statsReceiver: StatsReceiver)
extends TwoHopExpansionCandidateSource[
HasParams with HasRecentFollowedUserIds,
Long,
Long,
CandidateUser
]
with Logging
def firstDegreeNodes(
target: HasParams with HasRecentFollowedUserIds
): Stitch[Seq[Long]] = Stitch.value(
target.recentFollowedUserIds
.getOrElse(Nil).take(
RecentFollowingRecentFollowingExpansionSource.NumFirstDegreeNodesToRetrieve)
)
override def secondaryDegreeNodes(
target: HasParams with HasRecentFollowedUserIds,
node: Long
): Stitch[Seq[Long]] = socialGraphClient
.getRecentEdgesCached(
RecentEdgesQuery(
node,
Seq(RelationshipType.Following),
Some(RecentFollowingRecentFollowingExpansionSource.NumSecondDegreeNodesToRetrieve)),
useCachedStratoColumn =
target.params(RecentFollowingRecentFollowingExpansionSourceParams.CallSgsCachedColumn)
).map(
_.take(RecentFollowingRecentFollowingExpansionSource.NumSecondDegreeNodesToRetrieve)).rescue
def aggregateAndScore(
target: HasParams with HasRecentFollowedUserIds,
firstDegreeToSecondDegreeNodesMap: Map[Long, Seq[Long]]
): Stitch[Seq[CandidateUser]]
object RecentFollowingRecentFollowingExpansionSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\socialgraph\RecentFollowingRecentFollowingExpansionSourceFSConfig.scala
class RecentFollowingRecentFollowingExpansionSourceFSConfig @Inject() ()
extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\socialgraph\RecentFollowingRecentFollowingExpansionSourceParams.scala
object RecentFollowingRecentFollowingExpansionSourceParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\BaseOnlineSTPSource.scala
abstract class BaseOnlineSTPSource(
stpGraphBuilder: STPGraphBuilder,
baseStatsReceiver: StatsReceiver)
extends CandidateSource[
HasClientContext with HasParams with HasRecentFollowedUserIds,
CandidateUser
]
with Logging
def getCandidates(
records: Seq[STPRecord],
request: HasClientContext with HasParams with HasRecentFollowedUserIds
): Stitch[Seq[CandidateUser]]
override def apply(
request: HasClientContext with HasParams with HasRecentFollowedUserIds
): Stitch[Seq[CandidateUser]] =
request.getOptionalUserId
.map
object BaseOnlineSTPSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\Dbv2StpScorer.scala
class Dbv2StpScorer @Inject() (
@Named(GuiceNamedConstants.STP_DBV2_SCORER) tfPredictionEngine: TensorflowPredictionEngine)
def getScoredResponse(record: STPRecord): Stitch[Option[Double]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\EpStpScorer.scala
class ScoredResponse(score: Double, featuresBreakdown: Option[String] = None)
/**
* STP ML ranker trained using prehistoric ML framework
*/
@Singleton
class EpStpScorer @Inject() (epScorer: EPScorer)
def getScore(responses: List[EPScoringResponse]): Option[ScoredResponse] =
responses.headOption
.flatMap
def getScoredResponse(
record: Record,
details: Boolean = false
): Stitch[Option[ScoredResponse]]
object EpStpScorer
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\MutualFollowStrongTiePredictionSource.scala
class MutualFollowStrongTiePredictionSource @Inject() (
sgsClient: SocialGraphClient,
strongTiePredictionFeaturesOnUserClientColumn: StrongTiePredictionFeaturesOnUserClientColumn)
extends CandidateSource[HasClientContext with HasRecentFollowedUserIds, CandidateUser]
def apply(
target: HasClientContext with HasRecentFollowedUserIds
): Stitch[Seq[CandidateUser]]
object MutualFollowStrongTiePredictionSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineMutualFollowExpansionSource.scala
class OfflineMutualFollowExpansionSource @Inject() (
column: MutualFollowExpansionClientColumn)
extends OfflineStrongTiePredictionBaseSource(column.fetcher)
object OfflineMutualFollowExpansionSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStpSourceFsConfig.scala
class OfflineStpSourceFsConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStpSourceParams.scala
object OfflineStpSourceParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStpSourceWithDensePmiMatrix.scala
class OfflineStpSourceWithDensePmiMatrix @Inject() (
stpColumn: PpmiDenseMatrixCandidatesClientColumn)
extends OfflineStrongTiePredictionBaseSource(stpColumn.fetcher)
object OfflineStpSourceWithDensePmiMatrix
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStpSourceWithLegacyPmiMatrix.scala
class OfflineStpSourceWithLegacyPmiMatrix @Inject() (
stpColumn: StrongTiePredictionClientColumn)
extends OfflineStrongTiePredictionBaseSource(stpColumn.fetcher)
object OfflineStpSourceWithLegacyPmiMatrix
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStrongTiePredictionBaseSource.scala
class that all variants of our offline stp dataset can extend. Assumes the same STPResult
* value in the key and converts the result into the necessary internal model.
*/
abstract class OfflineStrongTiePredictionBaseSource(
fetcher: Fetcher[Long, Unit, STPResult])
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
def fetch(
target: Long,
): Stitch[Seq[CandidateUser]]
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]]
object OfflineStrongTiePredictionBaseSource
def map(target: Long, candidates: STPResult): Seq[CandidateUser]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStrongTiePredictionSource.scala
object OfflineStpScore extends Feature[UserCandidate, Option[Double]]
/**
* Main source for strong-tie-prediction candidates generated offline.
*/
@Singleton
class OfflineStrongTiePredictionSource @Inject() (
offlineStpSourceWithLegacyPmiMatrix: OfflineStpSourceWithLegacyPmiMatrix,
offlineStpSourceWithDensePmiMatrix: OfflineStpSourceWithDensePmiMatrix)
extends CandidateSource[HasParams with HasClientContext, CandidateUser]
with Logging
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]]
object OfflineStrongTiePredictionSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceFSConfig.scala
class OnlineSTPSourceFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceParams.scala
object OnlineSTPSourceParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceScorer.scala
class OnlineSTPSourceScorer @Inject() (
onlineSTPSourceWithEPScorer: OnlineSTPSourceWithEPScorer)
extends CandidateSource[
HasClientContext with HasParams with HasRecentFollowedUserIds,
CandidateUser
]
def apply(
request: HasClientContext with HasParams with HasRecentFollowedUserIds
): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceWithDeepbirdV2Scorer.scala
class OnlineSTPSourceWithDeepbirdV2Scorer @Inject() (
dbv2StpScorer: Dbv2StpScorer,
stpGraphBuilder: STPGraphBuilder,
baseStatReceiver: StatsReceiver)
extends BaseOnlineSTPSource(stpGraphBuilder, baseStatReceiver)
def getCandidates(
records: Seq[STPRecord],
request: HasClientContext with HasParams with HasRecentFollowedUserIds,
): Stitch[Seq[CandidateUser]]
object OnlineSTPSourceWithDeepbirdV2Scorer
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceWithEPScorer.scala
class OnlineSTPSourceWithEPScorer @Inject() (
epStpScorer: EpStpScorer,
stpGraphBuilder: STPGraphBuilder,
baseStatReceiver: StatsReceiver)
extends BaseOnlineSTPSource(stpGraphBuilder, baseStatReceiver)
with Logging
def getCandidates(
records: Seq[STPRecord],
request: HasClientContext with HasParams with HasRecentFollowedUserIds,
): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\SocialProofEnforcedOfflineStrongTiePredictionSource.scala
class SocialProofEnforcedOfflineStrongTiePredictionSource @Inject() (
offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource,
modifySocialProof: ModifySocialProof,
statsReceiver: StatsReceiver)
extends SocialProofEnforcedCandidateSource(
offlineStrongTiePredictionSource,
modifySocialProof,
SocialProofEnforcedOfflineStrongTiePredictionSource.MinNumSocialProofsRequired,
SocialProofEnforcedOfflineStrongTiePredictionSource.Identifier,
statsReceiver)
object SocialProofEnforcedOfflineStrongTiePredictionSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\STPFirstDegreeFetcher.scala
class STPFirstDegreeFetcher @Inject() (
realTimeGraphClient: RealTimeRealGraphClient,
reversePhoneBookSource: ReversePhoneBookSource,
reverseEmailBookSource: ReverseEmailBookSource,
forwardEmailBookSource: ForwardEmailBookSource,
forwardPhoneBookSource: ForwardPhoneBookSource,
mutualFollowStrongTiePredictionSource: MutualFollowStrongTiePredictionSource,
lowTweepCredFollowStore: LowTweepCredFollowStore,
timer: Timer,
statsReceiver: StatsReceiver)
extends Logging
def getPotentialFirstEdgesFromFetcher(
userId: Long,
target: HasClientContext with HasParams with HasRecentFollowedUserIds,
algorithm: Algorithm,
weight: Double
): Stitch[Seq[PotentialFirstDegreeEdge]]
def getFirstDegreeEdges(
target: HasClientContext with HasParams with HasRecentFollowedUserIds
): Stitch[Seq[FirstDegreeEdge]]
object STPFirstDegreeFetcher
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\STPGraphBuilder.scala
class STPGraphBuilder @Inject() (
stpFirstDegreeFetcher: STPFirstDegreeFetcher,
stpSecondDegreeFetcher: STPSecondDegreeFetcher,
statsReceiver: StatsReceiver)
def apply(
target: HasClientContext with HasParams with HasRecentFollowedUserIds
): Stitch[STPGraph] = stpFirstDegreeFetcher
.getFirstDegreeEdges(target).flatMap
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\STPSecondDegreeFetcher.scala
class STPSecondDegreeFetcher @Inject() (
strongTiePredictionFeaturesOnUserClientColumn: StrongTiePredictionFeaturesOnUserClientColumn)
def scoreSecondDegreeEdge(edge: SecondDegreeEdge): (Int, Int, Int)
def bool2int(b: Boolean): Int = if (b) 1 else 0
(
-edge.edgeInfo.numMutualFollowPath,
-edge.edgeInfo.numLowTweepcredFollowPath,
-(bool2int(edge.edgeInfo.forwardEmailPath) + bool2int(edge.edgeInfo.reverseEmailPath) +
bool2int(edge.edgeInfo.forwardPhonePath) + bool2int(edge.edgeInfo.reversePhonePath))
)
}
// Use each first-degree edge(w/ candidateId) to expand and find mutual follows.
// Then, with the mutual follows, group-by candidateId and join edge information
// to create secondDegree edges.
def getSecondDegreeEdges(
target: HasClientContext with HasParams,
firstDegreeEdges: Seq[FirstDegreeEdge]
): Stitch[Seq[SecondDegreeEdge]]
object STPSecondDegreeFetcher
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\top_organic_follows_accounts\TopOrganicFollowsAccountsFSConfig.scala
class TopOrganicFollowsAccountsFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\top_organic_follows_accounts\TopOrganicFollowsAccountsParams.scala
object TopOrganicFollowsAccountsParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\top_organic_follows_accounts\TopOrganicFollowsAccountsSource.scala
object AccountsFilteringAndRankingLogicId extends Enumeration
object TopOrganicFollowsAccountsSource
class TopOrganicFollowsAccountsSource @Inject() (
organicFollowsAccountsClientColumn: OrganicFollowsAccountsClientColumn,
statsReceiver: StatsReceiver,
) extends CandidateSource[TopOrganicFollowsAccountsSource.Target, CandidateUser]
with Logging
def apply(
target: TopOrganicFollowsAccountsSource.Target
): Stitch[Seq[CandidateUser]]
def transformOrganicFollowAccountssToCandidateSource(
organicFollowsAccounts: Seq[Option[OrganicFollowsAccounts]]
): Seq[CandidateUser]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\triangular_loops\TriangularLoopsFSConfig.scala
class TriangularLoopsFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\triangular_loops\TriangularLoopsParams.scala
object TriangularLoopsParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\triangular_loops\TriangularLoopsSource.scala
class TriangularLoopsSource @Inject() (
triangularLoopsV2Column: TriangularLoopsV2OnUserClientColumn)
extends CandidateSource[
HasParams with HasClientContext with HasRecentFollowedByUserIds,
CandidateUser
]
def apply(
target: HasParams with HasClientContext with HasRecentFollowedByUserIds
): Stitch[Seq[CandidateUser]]
def filterOutCandidatesNotFollowingTargetUser(
candidatesStitch: Stitch[Seq[CandidateUser]],
recentFollowings: Option[Seq[Long]]
): Stitch[Seq[CandidateUser]]
object TriangularLoopsSource
def mapCandidatesToCandidateUsers(candidates: Candidates): Seq[CandidateUser]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\two_hop_random_walk\TwoHopRandomWalkSource.scala
class TwoHopRandomWalkSource @Inject() (
@Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER) fetcher: Fetcher[
Long,
Unit,
TCandidateSeq
]) extends StratoFetcherWithUnitViewSource[Long, TCandidateSeq](
fetcher,
TwoHopRandomWalkSource.Identifier)
def map(targetUserId: Long, tCandidateSeq: TCandidateSeq): Seq[CandidateUser] =
TwoHopRandomWalkSource.map(targetUserId, tCandidateSeq)
}
object TwoHopRandomWalkSource
def map(targetUserId: Long, tCandidateSeq: TCandidateSeq): Seq[CandidateUser]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\user_user_graph\UserUserGraphCandidateSource.scala
class UserUserGraphCandidateSource @Inject() (
@Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER)
fetcher: Fetcher[RecommendUserRequest, Unit, RecommendUserResponse],
statsReceiver: StatsReceiver)
extends CandidateSource[
UserUserGraphCandidateSource.Target,
CandidateUser
]
def apply(
target: UserUserGraphCandidateSource.Target
): Stitch[Seq[CandidateUser]]
def buildRecommendUserRequest(
target: UserUserGraphCandidateSource.Target
): Option[RecommendUserRequest]
def convertToCandidateUsers(
recommendedUser: RecommendedUser
): CandidateUser
object UserUserGraphCandidateSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\user_user_graph\UserUserGraphFSConfig.scala
class UserUserGraphFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\user_user_graph\UserUserGraphParams.scala
object UserUserGraphParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\AddressbookClient.scala
class AddressbookClient @Inject() (
addressbookService: Addressbook2.MethodPerEndpoint,
statsReceiver: StatsReceiver = NullStatsReceiver)
def getResponseFromService(
identifiers: Seq[RecordIdentifier],
batchSize: Int,
edgeType: EdgeType,
maxFetches: Int,
queryOption: Option[QueryOption]
): Stitch[Seq[AddressBookGetResponse]]
def getContactsResponseFromService(
identifiers: Seq[RecordIdentifier],
batchSize: Int,
edgeType: EdgeType,
maxFetches: Int,
queryOption: Option[QueryOption]
): Stitch[Seq[AddressBookGetResponse]]
def getUsers(
userId: Long,
identifiers: Seq[RecordIdentifier],
batchSize: Int,
edgeType: EdgeType,
fetcherOption: Option[Fetcher[Long, Unit, tUserContacts]] = None,
maxFetches: Int = 1,
queryOption: Option[QueryOption] = None,
): Stitch[Seq[Long]]
def getHashedContacts(
normalizeFn: String => String,
extractField: String,
)(
userId: Long,
identifiers: Seq[RecordIdentifier],
batchSize: Int,
edgeType: EdgeType,
fetcherOption: Option[Fetcher[String, Unit, STPResultFeature]] = None,
maxFetches: Int = 1,
queryOption: Option[QueryOption] = None,
): Stitch[Seq[String]]
def getEmailContacts = getHashedContacts(normalizeEmail, "emails") _
def getPhoneContacts = getHashedContacts(normalizePhoneNumber, "phoneNumbers") _
private def getUsersFromManhattan(
userId: Long,
fetcher: Fetcher[Long, Unit, tUserContacts],
): Stitch[Seq[Long]] = fetcher
.fetch(userId)
.map(_.v.map(_.destinationIds).toSeq.flatten.distinct)
private def getContactsFromManhattan(
userId: Long,
fetcher: Fetcher[String, Unit, STPResultFeature],
): Stitch[Seq[String]] = fetcher
.fetch(userId.toString)
.map(_.v.map(_.strongTieUserFeature.map(_.destId)).toSeq.flatten.distinct)
}
object AddressbookClient
def createQueryOption(edgeType: EdgeType, isPhone: Boolean): Option[QueryOption] =
(edgeType, isPhone) match
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\AddressbookModule.scala
object AddressbookModule extends BaseClientModule[Addressbook2.MethodPerEndpoint] with MtlsClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\models\Contact.scala
class Contact(
id: Long,
emails: Option[Set[String]],
phoneNumbers: Option[Set[String]],
firstName: Option[String],
lastName: Option[String],
name: Option[String],
appId: Option[Long],
appIds: Option[Set[Long]],
importedTimestamp: Option[Time])
object Contact
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\models\EdgeType.scala
trait EdgeType
def toThrift: t.EdgeType
}
object EdgeType
object Forward extends EdgeType
object Reverse extends EdgeType
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\models\QueryOption.scala
class QueryOption(
onlyDiscoverableInExpansion: Boolean,
onlyConfirmedInExpansion: Boolean,
onlyDiscoverableInResult: Boolean,
onlyConfirmedInResult: Boolean,
fetchGlobalApiNamespace: Boolean,
isDebugRequest: Boolean,
resolveEmails: Boolean,
resolvePhoneNumbers: Boolean)
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\models\RecordIdentifier.scala
class RecordIdentifier(
userId: Option[Long],
email: Option[String],
phoneNumber: Option[String])
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\adserver\AdRequest.scala
class AdRequest(
clientContext: ClientContext,
displayLocation: DisplayLocation,
isTest: Option[Boolean],
profileUserId: Option[Long])
def toThrift: t.AdRequestParams
object AdRequest
class MissingAdDisplayLocation(displayLocation: DisplayLocation)
extends Exception(
s"Display Location $
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\adserver\AdserverClient.scala
class AdserverClient @Inject() (adserverService: NewAdServer.MethodPerEndpoint)
def getAdImpressions(adRequest: AdRequest): Stitch[Seq[t.AdImpression]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\adserver\AdserverModule.scala
object AdserverModule extends BaseClientModule[NewAdServer.MethodPerEndpoint] with MtlsClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\cache\MemcacheClient.scala
object MemcacheClient
def apply[V](
client: Client,
dest: String,
valueBijection: Bijection[Buf, V],
ttl: Duration,
statsReceiver: StatsReceiver
): MemcacheClient[V]
class MemcacheClient[V](
client: Client,
dest: String,
valueBijection: Bijection[Buf, V],
ttl: Duration,
statsReceiver: StatsReceiver)
def readThrough(
key: String,
underlyingCall: () => Stitch[V]
): Stitch[V]
def getIfPresent(key: String): Future[Option[V]]
def put(key: String, value: V): Future[Unit]
def hashString(input: String): String
def profileStitch[T](stitch: Stitch[T], stat: StatsReceiver): Stitch[T]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\cache\MemcacheModule.scala
object MemcacheModule extends TwitterModule
def provideMemcacheClient(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver,
): Client
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\cache\ThriftBijection.scala
abstract class ThriftBijection[T <: ThriftStruct] extends Bijection[Buf, T]
def apply(b: Buf): T
def invert(a: T): Buf
abstract class ThriftOptionBijection[T <: ThriftStruct] extends Bijection[Buf, Option[T]]
def apply(b: Buf): Option[T]
def invert(a: Option[T]): Buf
class ThriftEnumBijection[T <: ThriftEnum](constructor: Int => T) extends Bijection[Buf, T]
def apply(b: Buf): T
def invert(a: T): Buf
class ThriftEnumOptionBijection[T <: ThriftEnum](constructor: Int => T) extends Bijection[Buf, Option[T]]
def apply(b: Buf): Option[T]
def invert(a: Option[T]): Buf
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\common\BaseClientModule.scala
abstract class BaseClientModule[T: ClassTag] extends ThriftClientModule[T]
def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\deepbirdv2\DeepBirdV2PredictionServiceClientModule.scala
object DeepBirdV2PredictionServiceClientModule extends TwitterModule
def getDeepbirdPredictionServiceClient(
clientId: ClientId,
label: String,
dest: String,
statsReceiver: StatsReceiver,
serviceIdentifier: ServiceIdentifier
): DeepbirdPredictionService.ServiceToClient
def providesWtfProdDeepbirdV2PredictionService(
clientId: ClientId,
statsReceiver: StatsReceiver,
serviceIdentifier: ServiceIdentifier
): DeepbirdPredictionService.ServiceToClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\dismiss_store\DismissStore.scala
class DismissStore @Inject() (
@Named(GuiceNamedConstants.DISMISS_STORE_SCANNER)
scanner: Scanner[(Long, Slice[
(Long, Long)
]), Unit, (Long, (Long, Long)), WhoToFollowDismissEventDetails],
stats: StatsReceiver)
extends Logging
def get(
userId: Long,
negStartTimeMs: Long,
maxCandidatesToFetchOption: Option[Int]
): Stitch[Seq[Long]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\email_storage_service\EmailStorageServiceClient.scala
class EmailStorageServiceClient @Inject() (
val emailStorageService: EmailStorageService.MethodPerEndpoint)
def getVerifiedEmail(
userId: Long,
purposeOfProcessing: PurposeOfProcessing
): Stitch[Option[String]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\email_storage_service\EmailStorageServiceModule.scala
object EmailStorageServiceModule
extends BaseClientModule[EmailStorageService.MethodPerEndpoint]
with MtlsClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\geoduck\LocationServiceClient.scala
class LocationServiceClient @Inject() (locationService: LocationService.MethodPerEndpoint)
def getGeohashAndCountryCode(userId: Long): Stitch[GeohashAndCountryCode]
def getGeohashFromTransactionLocation(
transactionLocation: TransactionLocation
): Option[String]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\geoduck\LocationServiceModule.scala
object LocationServiceModule
extends BaseClientModule[LocationService.MethodPerEndpoint]
with MtlsClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\geoduck\ReverseGeocodeClient.scala
class ReverseGeocodeClient @Inject() (rgcService: ReverseGeocoder.MethodPerEndpoint)
def getGeohashAndCountryCode(ipAddress: String): Stitch[GeohashAndCountryCode]
def getGeohashAndCountryCodeFromLocation(location: Location): GeohashAndCountryCode
object ReverseGeocodeClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\geoduck\UserLocationFetcher.scala
class UserLocationFetcher @Inject() (
locationServiceClient: LocationServiceClient,
reverseGeocodeClient: ReverseGeocodeClient,
statsReceiver: StatsReceiver)
def getGeohashAndCountryCode(
userId: Option[Long],
ipAddress: Option[String]
): Stitch[Option[GeohashAndCountryCode]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\gizmoduck\GizmoduckClient.scala
class GizmoduckClient @Inject() (gizmoduckStitchClient: Gizmoduck, statsReceiver: StatsReceiver)
def isProtected(userId: Long): Stitch[Boolean]
def getUserName(userId: Long, forUserId: Long): Stitch[Option[String]]
object GizmoduckClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\gizmoduck\GizmoduckModule.scala
object GizmoduckModule extends BaseClientModule[UserService.MethodPerEndpoint] with MtlsClient
def provideExtraGizmoduckQueryFields: Set[QueryFields] = Set.empty
@Provides
@Singleton
def providesStitchClient(futureIface: UserService.MethodPerEndpoint): Gizmoduck
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\graph_feature_service\GraphFeatureServiceClient.scala
class GraphFeatureServiceClient @Inject() (
graphFeatureService: GraphFeatureService.MethodPerEndpoint)
def getIntersections(
userId: Long,
candidateIds: Seq[Long],
numIntersectionIds: Int
): Stitch[Map[Long, FollowProof]]
object GraphFeatureServiceClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\graph_feature_service\GraphFeatureStoreModule.scala
object GraphFeatureStoreModule
extends BaseClientModule[GraphFeatureService.MethodPerEndpoint]
with MtlsClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\impression_store\ImpressionStoreModule.scala
object ImpressionStoreModule extends TwitterModule
def providesImpressionStore(stratoClient: Client): WtfImpressionStore
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\impression_store\WtfImpressionStore.scala
class WtfImpressionStore @Inject() (
scanner: Scanner[
((Long, TDisplayLocation), Slice[Long]),
Unit,
((Long, TDisplayLocation), Long),
(Long, Int)
]) extends Logging
def get(userId: Long, dl: DisplayLocation): Stitch[Seq[WtfImpression]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\interests_service\InterestServiceClient.scala
class InterestServiceClient @Inject() (
stratoClient: Client,
statsReceiver: StatsReceiver = NullStatsReceiver)
extends Logging
def fetchUttInterestIds(
userId: Long
): Stitch[Seq[Long]]
def extractUttInterest(
interestRelationShip: InterestRelationship
): Option[Long]
def fetchCustomInterests(
userId: Long
): Stitch[Seq[String]]
def extractCustomInterest(
interestRelationShip: InterestRelationship
): Option[String]
def fetchInterestRelationships(
userId: Long
): Stitch[Option[Seq[InterestRelationship]]]
def getInterestRelationship(
interestData: UserInterestData
): Seq[InterestRelationship]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\phone_storage_service\PhoneStorageServiceClient.scala
class PhoneStorageServiceClient @Inject() (
val phoneStorageService: PhoneStorageService.MethodPerEndpoint)
def getPhoneNumbers(
userId: Long,
purposeOfProcessing: PurposeOfProcessing,
forceCarrierLookup: Option[Boolean] = None
): Stitch[Seq[String]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\phone_storage_service\PhoneStorageServiceModule.scala
object PhoneStorageServiceModule
extends BaseClientModule[PhoneStorageService.MethodPerEndpoint]
with MtlsClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\real_time_real_graph\Engagement.scala
trait EngagementType
// We do not include SoftFollow since it's deprecated
object EngagementType
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\real_time_real_graph\EngagementScorer.scala
object EngagementScorer
def apply(
engagements: Map[Long, Seq[Engagement]],
engagementScoreMap: Map[EngagementType, Double],
minScore: Double = 0.0
): Seq[(Long, Double, Seq[EngagementType])]
def score(
engagement: Engagement,
now: Time,
engagementScoreMap: Map[EngagementType, Double]
): Double
def getEngagementProof(
engagements: Seq[Engagement],
engagementScoreMap: Map[EngagementType, Double]
): Seq[EngagementType]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\real_time_real_graph\RealTimeRealGraphClient.scala
class RealTimeRealGraphClient @Inject() (
timelinesUserVertexOnUserClientColumn: TimelinesUserVertexOnUserClientColumn,
realGraphScoresMhOnUserClientColumn: RealGraphScoresMhOnUserClientColumn)
def mapUserVertexToEngagementAndFilter(userVertex: UserVertex): Map[Long, Seq[Engagement]]
def getRecentProfileViewEngagements(userId: Long): Stitch[Map[Long, Seq[Engagement]]]
def getUsersRecentlyEngagedWith(
userId: Long,
engagementScoreMap: Map[EngagementType, Double],
includeDirectFollowCandidates: Boolean,
includeNonDirectFollowCandidates: Boolean
): Stitch[Seq[CandidateUser]]
def getRealGraphWeights(userId: Long): Stitch[Map[Long, Double]] =
realGraphScoresMhOnUserClientColumn.fetcher
.fetch(userId)
.map(
_.v
.map(_.candidates.map(candidate => (candidate.userId, candidate.score)).toMap)
.getOrElse(Map.empty[Long, Double]))
}
object RealTimeRealGraphClient
def toEngagement(interaction: Interaction): Option[Engagement]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\socialgraph\SocialGraphClient.scala
class RecentEdgesQuery(
userId: Long,
relations: Seq[RelationshipType],
// prefer to default value to better utilize the caching function of stitch
count: Option[Int] = Some(SocialGraphClient.MaxQuerySize),
performUnion: Boolean = true,
recentEdgesWindowOpt: Option[Duration] = None,
targets: Option[Seq[Long]] = None)
case class EdgeRequestQuery(
userId: Long,
relation: RelationshipType,
count: Option[Int] = Some(SocialGraphClient.MaxQuerySize),
performUnion: Boolean = true,
recentEdgesWindowOpt: Option[Duration] = None,
targets: Option[Seq[Long]] = None)
@Singleton
class SocialGraphClient @Inject() (
socialGraph: SocialGraph,
idsClientColumn: IdsClientColumn,
statsReceiver: StatsReceiver = NullStatsReceiver)
extends Logging
def getRecentEdgesCached(
rq: RecentEdgesQuery,
useCachedStratoColumn: Boolean = true
): Stitch[Seq[Long]]
def getRecentEdgesCachedInternal(rq: RecentEdgesQuery): Stitch[Seq[Long]]
def getRecentEdgesFromCachedColumn(rq: RecentEdgesQuery): Stitch[Seq[Long]]
def getRecentEdges(rq: RecentEdgesQuery): Stitch[Seq[Long]]
def getRecentEdgesWithTime(rq: EdgeRequestQuery): Stitch[Seq[UserIdWithTimestamp]]
def getEdgeCursor(window: Duration): ByteBuffer
def getIntersections(
userId: Long,
candidateIds: Seq[Long],
numIntersectionIds: Int
): Stitch[Map[Long, FollowProof]]
def getIntersectionsFromCachedColumn(
userId: Long,
candidateIds: Seq[Long],
numIntersectionIds: Int
): Stitch[Map[Long, FollowProof]]
def getInvalidRelationshipUserIds(
userId: Long,
maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship
): Stitch[Seq[Long]]
def getInvalidRelationshipUserIdsFromCachedColumn(
userId: Long,
maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship
): Stitch[Seq[Long]]
def getRecentFollowedUserIds(userId: Long): Stitch[Seq[Long]]
def getRecentFollowedUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]]
def getRecentFollowedUserIdsWithTime(userId: Long): Stitch[Seq[UserIdWithTimestamp]]
def getRecentFollowedByUserIds(userId: Long): Stitch[Seq[Long]]
def getRecentFollowedByUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]]
def getRecentFollowedUserIdsWithTimeWindow(
userId: Long,
timeWindow: Duration
): Stitch[Seq[Long]]
object SocialGraphClient
def enablePostRankerSgsPredicate(numInvalidRelationshipUsers: Int): Boolean
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\socialgraph\SocialGraphModule.scala
object SocialGraphModule
extends BaseClientModule[SocialGraphService.MethodPerEndpoint]
with MtlsClient
def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client =
client.withSessionQualifier.noFailFast
@Provides
@Singleton
def providesStitchClient(futureIface: SocialGraphService.MethodPerEndpoint): SocialGraph
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\strato\StratoClientModule.scala
object StratoClientModule extends TwitterModule
def stratoClient(serviceIdentifier: ServiceIdentifier): Client
def cosineFollowFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] =
stratoClient.fetcher[Long, Unit, HermitCandidates](CosineFollowPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.COSINE_LIST_FETCHER)
def cosineListFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] =
stratoClient.fetcher[Long, Unit, HermitCandidates](CosineListPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER)
def curatedBlacklistedAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] =
stratoClient.fetcher[String, Unit, Seq[Long]](CuratedFilteredAccountsPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.CURATED_CANDIDATES_FETCHER)
def curatedCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] =
stratoClient.fetcher[String, Unit, Seq[Long]](CuratedCandidatesPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER)
def popUsersInPlaceFetcher(stratoClient: Client): Fetcher[String, Unit, PopUsersInPlace] =
stratoClient.fetcher[String, Unit, PopUsersInPlace](PopUsersInPlacePath)
@Provides
@Singleton
@Named(GuiceNamedConstants.RELATABLE_ACCOUNTS_FETCHER)
def relatableAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, RelatableAccounts] =
stratoClient.fetcher[String, Unit, RelatableAccounts](RelatableAccountsPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.PROFILE_SIDEBAR_BLACKLIST_SCANNER)
def profileSidebarBlacklistScanner(
stratoClient: Client
): Scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit] =
stratoClient.scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit](ProfileSidebarBlacklistPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.REAL_TIME_INTERACTIONS_FETCHER)
def realTimeInteractionsFetcher(
stratoClient: Client
): Fetcher[(Long, Long), Unit, Seq[Interaction]] =
stratoClient.fetcher[(Long, Long), Unit, Seq[Interaction]](RealTimeInteractionsPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.SIMS_FETCHER)
def simsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] =
stratoClient.fetcher[Long, Unit, HermitCandidates](SimsPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.DBV2_SIMS_FETCHER)
def dbv2SimsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] =
stratoClient.fetcher[Long, Unit, HermitCandidates](DBV2SimsPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.TRIANGULAR_LOOPS_FETCHER)
def triangularLoopsFetcher(stratoClient: Client): Fetcher[Long, Unit, TriangularLoopCandidates] =
stratoClient.fetcher[Long, Unit, TriangularLoopCandidates](TriangularLoopsPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER)
def twoHopRandomWalkFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateSeq] =
stratoClient.fetcher[Long, Unit, CandidateSeq](TwoHopRandomWalkPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER)
def userRecommendabilityFetcher(
stratoClient: Client
): Fetcher[Long, Unit, UserRecommendabilityFeatures] =
stratoClient.fetcher[Long, Unit, UserRecommendabilityFeatures](UserRecommendabilityPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.USER_STATE_FETCHER)
def userStateFetcher(stratoClient: Client): Fetcher[Long, Unit, CondensedUserState] =
stratoClient.fetcher[Long, Unit, CondensedUserState](UserStatePath)
@Provides
@Singleton
@Named(GuiceNamedConstants.UTT_ACCOUNT_RECOMMENDATIONS_FETCHER)
def uttAccountRecommendationsFetcher(
stratoClient: Client
): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] =
stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations](
UTTAccountRecommendationsPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.UTT_SEED_ACCOUNTS_FETCHER)
def uttSeedAccountRecommendationsFetcher(
stratoClient: Client
): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] =
stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations](
UttSeedAccountsRecommendationPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.ELECTION_CANDIDATES_FETCHER)
def electionCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] =
stratoClient.fetcher[String, Unit, Seq[Long]](ElectionCandidatesPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER)
def userUserGraphFetcher(
stratoClient: Client
): Fetcher[RecommendUserRequest, Unit, RecommendUserResponse] =
stratoClient.fetcher[RecommendUserRequest, Unit, RecommendUserResponse](UserUserGraphPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.POST_NUX_WTF_FEATURES_FETCHER)
def wtfPostNuxFeaturesFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateFeatures]
def extendedNetworkFetcher(
stratoClient: Client
): Fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal]
def dismissStoreScanner(
stratoClient: Client
): Scanner[
(Long, Slice[(Long, Long)]),
Unit,
(Long, (Long, Long)),
WhoToFollowDismissEventDetails
] =
stratoClient.scanner[
(Long, Slice[(Long, Long)]), // PKEY: userId, LKEY: (-ts, candidateId)
Unit,
(Long, (Long, Long)),
WhoToFollowDismissEventDetails
](WtfDissmissEventsPath)
@Provides
@Singleton
@Named(GuiceNamedConstants.LABELED_NOTIFICATION_FETCHER)
def labeledNotificationFetcher(
stratoClient: Client
): Fetcher[Long, Unit, LatestEvents]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\user_state\UserStateClient.scala
class UserStateClient @Inject() (
@Named(GuiceNamedConstants.USER_STATE_FETCHER) userStateFetcher: Fetcher[
Long,
Unit,
CondensedUserState
],
client: Client,
statsReceiver: StatsReceiver,
decider: Decider = Decider.False)
def getUserState(userId: Long): Stitch[Option[UserState]]
def fetchUserState(userId: JLong): Stitch[Option[UserState]]
object UserStateClient
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\constants\CandidateAlgorithmTypeConstants.scala
object CandidateAlgorithmTypeConstants
def getAlgorithmTypes(algoId: String): Set[String]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\constants\GuiceNamedConstants.scala
object GuiceNamedConstants
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\constants\ServiceConstants.scala
object ServiceConstants
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\features\LocationFeature.scala
object LocationFeature
extends FeatureWithDefaultOnFailure[PipelineQuery, Option[GeohashAndCountryCode]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\features\TrackingTokenFeature.scala
object TrackingTokenFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Int]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\features\UserStateFeature.scala
object UserStateFeature extends Feature[PipelineQuery, Option[UserState]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\adapters\CandidateAlgorithmAdapter.scala
object CandidateAlgorithmAdapter
extends IRecordOneToOneAdapter[Option[UserCandidateSourceDetails]]
def remapCandidateSource(a: Algorithm): Algorithm = a match
def adaptToDataRecord(
userCandidateSourceDetailsOpt: Option[UserCandidateSourceDetails]
): DataRecord
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\adapters\ClientContextAdapter.scala
object ClientContextAdapter extends IRecordOneToOneAdapter[(ClientContext, DisplayLocation)]
def adaptToDataRecord(target: (ClientContext, DisplayLocation)): DataRecord
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\adapters\PostNuxAlgorithmAdapter.scala
object PostNuxAlgorithmIdAdapter extends PostNuxAlgorithmAdapter
object PostNuxAlgorithmTypeAdapter extends PostNuxAlgorithmAdapter
trait PostNuxAlgorithmAdapter extends IRecordOneToOneAdapter[DataRecord]
class TransformedAlgorithmFeatures(
ratioLog: Continuous)
def getFeatures: Seq[Continuous] = Seq(ratioLog)
}
private def applyFeatureStorePrefix(feature: Continuous) = new Continuous(
s"$FeatureStorePrefix$
def addTransformedFeaturesToDataRecordFunc(
originalDr: DataRecord,
numImpressions: Double,
): (DataRecord, Continuous) => DataRecord
def adaptToDataRecord(record: DataRecord): DataRecord
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\adapters\PreFetchedFeatureAdapter.scala
object PreFetchedFeatureAdapter
extends IRecordOneToOneAdapter[
(HasPreFetchedFeature, CandidateUser)
]
def adaptToDataRecord(
record: (HasPreFetchedFeature, CandidateUser)
): DataRecord
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\common\FeatureSource.scala
trait FeatureSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\common\FeatureSourceId.scala
trait FeatureSourceId
object FeatureSourceId
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\common\HasPreFetchedFeature.scala
trait HasPreFetchedFeature extends HasMutualFollowedUserIds with HasWtfImpressions
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\CandidateAlgorithmSource.scala
class CandidateAlgorithmSource @Inject() (stats: StatsReceiver) extends FeatureSource
def hydrateFeatures(
t: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation, // we don't use the target here
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\ClientContextSource.scala
class ClientContextSource() extends FeatureSource
def hydrateFeatures(
t: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation,
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureHydrationSourcesFeatureSwitchKeys.scala
object FeatureHydrationSourcesFeatureSwitchKeys
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureHydrationSourcesFSConfig.scala
class FeatureHydrationSourcesFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreFeatures.scala
object FeatureStoreFeatures
object FeatureStoreRawFeatures
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreGizmoduckSource.scala
class FeatureStoreGizmoduckSource @Inject() (
serviceIdentifier: ServiceIdentifier,
stats: StatsReceiver)
extends FeatureSource
def id: FeatureSourceId = FeatureSourceId.FeatureStoreGizmoduckSourceId
override def featureContext: FeatureContext = getFeatureContext
val clientConfig: ClientConfig[HasParams] = ClientConfig(
dynamicHydrationConfig = dynamicHydrationConfig,
featureStoreParamsConfig =
FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty),
/**
* The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout`
* used below takes effect.
*/
timeoutProvider = Function.const(800.millis),
serviceIdentifier = serviceIdentifier
)
private val datasetsToCache = Set(
UsersourceEntityDataset
).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]]
private val datasetValuesCache: DatasetValuesCache =
DatasetValuesCache(
Caffeine
.newBuilder()
.expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS)
.maximumSize(DefaultCacheMaxKeys)
.build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]]
.asMap,
datasetsToCache,
DatasetCacheScope
)
private val dynamicFeatureStoreClient = DynamicFeatureStoreClient(
clientConfig,
backupSourceStats,
Set(datasetValuesCache)
)
private val adapter: IRecordOneToOneAdapter[PredictionRecord] =
PredictionRecordAdapter.oneToOne(
BoundFeatureSet(allFeatures),
OnlineFeatureGenerationStats(backupSourceStats)
)
override def hydrateFeatures(
target: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation,
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
object FeatureStoreGizmoduckSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreParameters.scala
object FeatureStoreParameters
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStorePostNuxAlgorithmSource.scala
class FeatureStorePostNuxAlgorithmSource @Inject() (
serviceIdentifier: ServiceIdentifier,
stats: StatsReceiver)
extends FeatureSource
def id: FeatureSourceId = FeatureSourceId.FeatureStorePostNuxAlgorithmSourceId
override def featureContext: FeatureContext = getFeatureContext
private val dataRecordMerger = new DataRecordMerger
val clientConfig: ClientConfig[HasParams] = ClientConfig(
dynamicHydrationConfig = dynamicHydrationConfig,
featureStoreParamsConfig =
FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty),
/**
* The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout`
* used below takes effect.
*/
timeoutProvider = Function.const(800.millis),
serviceIdentifier = serviceIdentifier
)
private val datasetsToCache = Set(
PostNuxAlgorithmIdAggregateDataset,
PostNuxAlgorithmTypeAggregateDataset,
).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]]
private val datasetValuesCache: DatasetValuesCache =
DatasetValuesCache(
Caffeine
.newBuilder()
.expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS)
.maximumSize(DefaultCacheMaxKeys)
.build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]]
.asMap,
datasetsToCache,
DatasetCacheScope
)
private val dynamicFeatureStoreClient = DynamicFeatureStoreClient(
clientConfig,
backupSourceStats,
Set(datasetValuesCache)
)
private val adapterToDataRecord: IRecordOneToOneAdapter[PredictionRecord] =
PredictionRecordAdapter.oneToOne(
BoundFeatureSet(allFeatures),
OnlineFeatureGenerationStats(backupSourceStats)
)
// These two calculate the rate for each feature by dividing it by the number of impressions, then
// apply a log transformation.
private val transformAdapters = Seq(PostNuxAlgorithmIdAdapter, PostNuxAlgorithmTypeAdapter)
override def hydrateFeatures(
target: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation,
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
object FeatureStorePostNuxAlgorithmSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreSource.scala
class FeatureStoreSource @Inject() (
serviceIdentifier: ServiceIdentifier,
stats: StatsReceiver)
extends FeatureSource
def hydrateFeatures(
target: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation,
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
object FeatureStoreSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreSourceParams.scala
object FeatureStoreSourceParams
object EnableTopicAggregateFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTopicAggregateFeatures,
default = true
)
case object EnableAlgorithmAggregateFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAlgorithmAggregateFeatures,
default = false
)
case object EnableAuthorTopicAggregateFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAuthorTopicAggregateFeatures,
default = true
)
case object EnableUserTopicFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserTopicFeatures,
default = false
)
case object EnableTargetUserFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserFeatures,
default = true
)
case object EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature
extends FSParam[Boolean](
name =
FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature,
default = true
)
case object EnableTargetUserResurrectionFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserResurrectionFeatures,
default = true
)
case object EnableTargetUserWtfImpressionFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserWtfImpressionFeatures,
default = true
)
case object EnableCandidateUserFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserFeatures,
default = true
)
case object EnableCandidateUserAuthorRealTimeAggregateFeatures
extends FSParam[Boolean](
name =
FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserAuthorRealTimeAggregateFeatures,
default = true
)
case object EnableCandidateUserResurrectionFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserResurrectionFeatures,
default = true
)
case object EnableCandidateUserTimelinesAuthorAggregateFeatures
extends FSParam[Boolean](
name =
FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserTimelinesAuthorAggregateFeatures,
default = true
)
case object EnableUserCandidateEdgeFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateEdgeFeatures,
default = true
)
case object EnableUserCandidateWtfImpressionCandidateFeatures
extends FSParam[Boolean](
name =
FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateWtfImpressionCandidateFeatures,
default = true
)
case object EnableUserWtfAlgEdgeFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserWtfAlgEdgeFeatures,
default = false
)
case object EnableSimilarToUserFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableSimilarToUserFeatures,
default = true
)
case object EnableCandidatePrecomputedNotificationFeatures
extends FSParam[Boolean](
name =
FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidatePrecomputedNotificationFeatures,
default = false
)
case object EnableCandidateClientFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateClientFeatures,
default = false
)
case object EnableUserClientFeatures
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserClientFeatures,
default = false
)
case object EnableSeparateClientForTimelinesAuthors
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForTimelinesAuthor,
default = false
)
case object EnableSeparateClientForMetricCenterUserCounting
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientMetricCenterUserCounting,
default = false
)
case object EnableSeparateClientForNotifications
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForNotifications,
default = false
)
case object EnableSeparateClientForGizmoduck
extends FSParam[Boolean](
name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForGizmoduck,
default = false
)
case object GlobalFetchTimeout
extends FSBoundedParam[Duration](
name = FeatureHydrationSourcesFeatureSwitchKeys.FeatureHydrationTimeout,
default = 240.millisecond,
min = 100.millisecond,
max = 400.millisecond)
with HasDurationConversion
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreTimelinesAuthorSource.scala
class FeatureStoreTimelinesAuthorSource @Inject() (
serviceIdentifier: ServiceIdentifier,
stats: StatsReceiver)
extends FeatureSource
def id: FeatureSourceId = FeatureSourceId.FeatureStoreTimelinesAuthorSourceId
override def featureContext: FeatureContext = getFeatureContext
val clientConfig: ClientConfig[HasParams] = ClientConfig(
dynamicHydrationConfig = dynamicHydrationConfig,
featureStoreParamsConfig =
FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty),
/**
* The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout`
* used below takes effect.
*/
timeoutProvider = Function.const(800.millis),
serviceIdentifier = serviceIdentifier
)
private val datasetsToCache = Set(
AuthorFeaturesEntityDataset
).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]]
private val datasetValuesCache: DatasetValuesCache =
DatasetValuesCache(
Caffeine
.newBuilder()
.expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS)
.maximumSize(DefaultCacheMaxKeys)
.build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]]
.asMap,
datasetsToCache,
DatasetCacheScope
)
private val dynamicFeatureStoreClient = DynamicFeatureStoreClient(
clientConfig,
backupSourceStats,
Set(datasetValuesCache)
)
private val adapter: IRecordOneToOneAdapter[PredictionRecord] =
PredictionRecordAdapter.oneToOne(
BoundFeatureSet(allFeatures),
OnlineFeatureGenerationStats(backupSourceStats)
)
override def hydrateFeatures(
target: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation,
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
object FeatureStoreTimelinesAuthorSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreUserMetricCountsSource.scala
class FeatureStoreUserMetricCountsSource @Inject() (
serviceIdentifier: ServiceIdentifier,
stats: StatsReceiver)
extends FeatureSource
def id: FeatureSourceId = FeatureSourceId.FeatureStoreUserMetricCountsSourceId
override def featureContext: FeatureContext = getFeatureContext
val clientConfig: ClientConfig[HasParams] = ClientConfig(
dynamicHydrationConfig = dynamicHydrationConfig,
featureStoreParamsConfig =
FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty),
/**
* The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout`
* used below takes effect.
*/
timeoutProvider = Function.const(800.millis),
serviceIdentifier = serviceIdentifier
)
private val datasetsToCache = Set(
MetricCenterUserCountingFeaturesDataset
).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]]
private val datasetValuesCache: DatasetValuesCache =
DatasetValuesCache(
Caffeine
.newBuilder()
.expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS)
.maximumSize(DefaultCacheMaxKeys)
.build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]]
.asMap,
datasetsToCache,
DatasetCacheScope
)
private val dynamicFeatureStoreClient = DynamicFeatureStoreClient(
clientConfig,
backupSourceStats,
Set(datasetValuesCache)
)
private val adapter: IRecordOneToOneAdapter[PredictionRecord] =
PredictionRecordAdapter.oneToOne(
BoundFeatureSet(allFeatures),
OnlineFeatureGenerationStats(backupSourceStats)
)
override def hydrateFeatures(
target: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation,
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
object FeatureStoreUserMetricCountsSource
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\HydrationSourcesModule.scala
object HydrationSourcesModule extends TwitterModule
def providesAthenaManhattanClient(
serviceIdentifier: ServiceIdentifier
): ManhattanKVEndpoint
def timelinesAuthorStitchCache(
manhattanReadOnlyEndpoint: ManhattanKVEndpoint,
timelinesAuthorFeaturesColumn: TimelinesAuthorFeaturesOnUserClientColumn,
stats: StatsReceiver
): StitchCache[JLong, Option[AuthorFeatures]]
def metricCenterUserCountingStitchCache(
mcUserCountingFeaturesColumn: McUserCountingOnUserClientColumn,
stats: StatsReceiver
): StitchCache[JLong, Option[MCUserCountingFeatures]]
def clearUnsedFieldsForAuthorFeature(entry: AuthorFeatures): AuthorFeatures
def randomizedTTL(ttl: Long): Long
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\PreFetchedFeatureSource.scala
class PreFetchedFeatureSource @Inject() () extends FeatureSource
def id: FeatureSourceId = FeatureSourceId.PreFetchedFeatureSourceId
override def featureContext: FeatureContext = PreFetchedFeatureAdapter.getFeatureContext
override def hydrateFeatures(
target: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation,
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\UserScoringFeatureSource.scala
class UserScoringFeatureSource @Inject() (
featureStoreSource: FeatureStoreSource,
featureStoreGizmoduckSource: FeatureStoreGizmoduckSource,
featureStorePostNuxAlgorithmSource: FeatureStorePostNuxAlgorithmSource,
featureStoreTimelinesAuthorSource: FeatureStoreTimelinesAuthorSource,
featureStoreUserMetricCountsSource: FeatureStoreUserMetricCountsSource,
clientContextSource: ClientContextSource,
candidateAlgorithmSource: CandidateAlgorithmSource,
preFetchedFeatureSource: PreFetchedFeatureSource)
extends FeatureSource
def hydrateFeatures(
target: HasClientContext
with HasPreFetchedFeature
with HasParams
with HasSimilarToContext
with HasDisplayLocation,
candidates: Seq[CandidateUser]
): Stitch[Map[CandidateUser, DataRecord]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\Utils.scala
object Utils
def adaptAdditionalFeaturesToDataRecord(
record: DataRecord,
adapterStats: StatsReceiver,
featureAdapters: Seq[IRecordOneToOneAdapter[DataRecord]]
): DataRecord
def randomizedTTL(ttl: Long): Long
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\AddressBookMetadata.scala
class AddressBookMetadata(
inForwardPhoneBook: Boolean,
inReversePhoneBook: Boolean,
inForwardEmailBook: Boolean,
inReverseEmailBook: Boolean)
object AddressBookMetadata
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\AlgorithmType.scala
object AlgorithmType extends Enumeration
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\CandidateUser.scala
trait FollowableEntity extends UniversalNoun[Long]
trait Recommendation
extends FollowableEntity
with HasReason
with HasAdMetadata
with HasTrackingToken
def toThrift: t.Recommendation
def toOfflineThrift: offline.OfflineRecommendation
}
case class CandidateUser(
override val id: Long,
override val score: Option[Double] = None,
override val reason: Option[Reason] = None,
override val userCandidateSourceDetails: Option[UserCandidateSourceDetails] = None,
override val adMetadata: Option[AdMetadata] = None,
override val trackingToken: Option[TrackingToken] = None,
override val dataRecord: Option[RichDataRecord] = None,
override val scores: Option[Scores] = None,
override val infoPerRankingStage: Option[scala.collection.Map[String, RankingInfo]] = None,
override val params: Params = Params.Invalid,
override val engagements: Seq[EngagementType] = Nil,
override val recommendationFlowIdentifier: Option[String] = None)
extends Recommendation
with HasUserCandidateSourceDetails
with HasDataRecord
with HasScores
with HasParams
with HasEngagements
with HasRecommendationFlowIdentifier
with HasInfoPerRankingStage
def setFollowProof(followProofOpt: Option[FollowProof]): CandidateUser
def addScore(score: Score): CandidateUser
object CandidateUser
def fromUserRecommendation(candidate: t.UserRecommendation): CandidateUser
def fromThriftScoreMap(
thriftMapOpt: Option[scala.collection.Map[String, Double]]
): Map[CandidateSourceIdentifier, Option[Double]]
def fromThriftRankMap(
thriftMapOpt: Option[scala.collection.Map[String, Int]]
): Map[CandidateSourceIdentifier, Int]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\ClientContextConverter.scala
object ClientContextConverter
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\DisplayLocation.scala
trait DisplayLocation
def toThrift: TDisplayLocation
def toOfflineThrift: TOfflineDisplayLocation
def toFsName: String
// corresponding display location in adserver if available
// make sure to be consistent with the definition here
def toAdDisplayLocation: Option[AdDisplayLocation] = None
}
/**
* Make sure you add the new DL to the following files and redeploy our attribution jobs
* - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift
* - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift
* - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala
*/
object DisplayLocation
object ProfileSidebar extends DisplayLocation
object HomeTimeline extends DisplayLocation
object ReactiveFollow extends DisplayLocation
object ExploreTab extends DisplayLocation
object MagicRecs extends DisplayLocation
object AbUploadInjection extends DisplayLocation
object RuxLandingPage extends DisplayLocation
object ProfileBonusFollow extends DisplayLocation
object ElectionExploreWtf extends DisplayLocation
object ClusterFollow extends DisplayLocation
object HtlBonusFollow extends DisplayLocation
object TopicLandingPageHeader extends DisplayLocation
object NewUserSarusBackfill extends DisplayLocation
object NuxPymk extends DisplayLocation
object NuxInterests extends DisplayLocation
object NuxTopicBonusFollow extends DisplayLocation
object Sidebar extends DisplayLocation
object CampaignForm extends DisplayLocation
object ProfileTopFollowers extends DisplayLocation
object ProfileTopFollowing extends DisplayLocation
object RuxPymk extends DisplayLocation
object IndiaCovid19CuratedAccountsWtf extends DisplayLocation
object PeoplePlusPlus extends DisplayLocation
object TweetNotificationRecs extends DisplayLocation
object ProfileDeviceFollow extends DisplayLocation
object RecosBackfill extends DisplayLocation
object HtlSpaceHosts extends DisplayLocation
object PostNuxFollowTask extends DisplayLocation
object TopicLandingPage extends DisplayLocation
object UserTypeaheadPrefetch extends DisplayLocation
object HomeTimelineRelatableAccounts extends DisplayLocation
object NuxGeoCategory extends DisplayLocation
object NuxInterestsCategory extends DisplayLocation
object TopArticles extends DisplayLocation
object NuxPymkCategory extends DisplayLocation
object HomeTimelineTweetRecs extends DisplayLocation
object HtlBulkFriendFollows extends DisplayLocation
object NuxAutoFollow extends DisplayLocation
object SearchBonusFollow extends DisplayLocation
object ContentRecommender extends DisplayLocation
object HomeTimelineReverseChron extends DisplayLocation
def fromThrift(displayLocation: TDisplayLocation): DisplayLocation = displayLocation match
def fromOfflineThrift(displayLocation: TOfflineDisplayLocation): DisplayLocation =
displayLocation match
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\EngagementType.scala
trait EngagementType
def toThrift: TEngagementType
def toOfflineThrift: OfflineEngagementType
}
object EngagementType
object Click extends EngagementType
object Like extends EngagementType
object Mention extends EngagementType
object Retweet extends EngagementType
object ProfileView extends EngagementType
def fromThrift(engagementType: TEngagementType): EngagementType = engagementType match
def fromOfflineThrift(engagementType: OfflineEngagementType): EngagementType =
engagementType match
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\FilterReason.scala
trait FilterReason
def reason: String
}
object FilterReason
object NoReason extends FilterReason
class ParamReason(paramName: String) extends FilterReason
object ExcludedId extends FilterReason
object ProfileSidebarBlacklist extends FilterReason
object CuratedAccountsCompetitorList extends FilterReason
class InvalidRelationshipTypes(relationshipTypes: String) extends FilterReason
object ProfileId extends FilterReason
object DismissedId extends FilterReason
object OptedOutId extends FilterReason
object NoUser extends FilterReason
object AddressBookUndiscoverable extends FilterReason
object PhoneBookUndiscoverable extends FilterReason
object Deactivated extends FilterReason
object Suspended extends FilterReason
object Restricted extends FilterReason
object NsfwUser extends FilterReason
object NsfwAdmin extends FilterReason
object HssSignal extends FilterReason
object IsProtected extends FilterReason
class CountryTakedown(countryCode: String) extends FilterReason
object Blink extends FilterReason
object AlreadyFollowed extends FilterReason
object InvalidRelationship extends FilterReason
object NotFollowingTargetUser extends FilterReason
object CandidateSideHoldback extends FilterReason
object Inactive extends FilterReason
object MissingRecommendabilityData extends FilterReason
object HighTweetVelocity extends FilterReason
object AlreadyRecommended extends FilterReason
object MinStateNotMet extends FilterReason
object FailOpen extends FilterReason
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\FlowContext.scala
class FlowContext(steps: Seq[RecommendationStep])
def toThrift: t.FlowContext = t.FlowContext(steps = steps.map(_.toThrift))
def toOfflineThrift: offline.OfflineFlowContext =
offline.OfflineFlowContext(steps = steps.map(_.toOfflineThrift))
}
object FlowContext
def fromThrift(flowContext: t.FlowContext): FlowContext
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\FlowRecommendation.scala
class FlowRecommendation(userId: Long)
def toThrift: t.FlowRecommendation =
t.FlowRecommendation(userId = userId)
def toOfflineThrift: offline.OfflineFlowRecommendation =
offline.OfflineFlowRecommendation(userId = userId)
}
object FlowRecommendation
def fromThrift(flowRecommendation: t.FlowRecommendation): FlowRecommendation
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasAdMetadata.scala
class AdMetadata(
insertPosition: Int,
// use original ad impression info to avoid losing data in domain model translations
adImpression: t.AdImpression)
trait HasAdMetadata
def adMetadata: Option[AdMetadata]
def adImpression: Option[t.AdImpression]
def insertPosition: Option[Int]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasByfSeedUserIds.scala
trait HasByfSeedUserIds
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasDataRecord.scala
class RichDataRecord(
dataRecord: Option[DataRecord] = None,
debugDataRecord: Option[DebugDataRecord] = None,
)
trait HasDataRecord extends Logging
def dataRecord: Option[RichDataRecord]
def toDebugDataRecord(dr: DataRecord, featureContext: FeatureContext): DebugDataRecord
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasDebugOptions.scala
class DebugOptions(
randomizationSeed: Option[Long] = None,
fetchDebugInfo: Boolean = false,
doNotLog: Boolean = false)
object DebugOptions
def fromDebugParamsThrift(debugParams: DebugParams): DebugOptions
trait HasDebugOptions
def debugOptions: Option[DebugOptions]
def getRandomizationSeed: Option[Long] = debugOptions.flatMap(_.randomizationSeed)
def fetchDebugInfo: Option[Boolean] = debugOptions.map(_.fetchDebugInfo)
}
trait HasFrsDebugOptions
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasDismissedUserIds.scala
trait HasDismissedUserIds
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasDisplayLocation.scala
trait HasDisplayLocation
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasEngagements.scala
trait HasEngagements
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasExcludedUserIds.scala
trait HasExcludedUserIds
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasGeohashAndCountryCode.scala
trait HasGeohashAndCountryCode
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasInfoPerRankingStage.scala
trait HasInfoPerRankingStage
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasInterestIds.scala
trait HasCustomInterests
def customInterests: Option[Seq[String]]
}
trait HasUttInterests
def uttInterestIds: Option[Seq[Long]]
}
trait HasInterestIds extends HasCustomInterests with HasUttInterests
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasInvalidRelationshipUserIds.scala
trait HasInvalidRelationshipUserIds
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasIsSoftUser.scala
trait HasIsSoftUser
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasMutualFollowedUserIds.scala
trait HasMutualFollowedUserIds extends HasRecentFollowedUserIds with HasRecentFollowedByUserIds
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasPreviousRecommendationsContext.scala
trait HasPreviousRecommendationsContext
def previouslyRecommendedUserIDs: Set[Long]
def previouslyFollowedUserIds: Set[Long]
def skippedFollows: Set[Long]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasProfileId.scala
trait HasProfileId
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasQualityFactor.scala
trait HasQualityFactor
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecentFollowedByUserIds.scala
trait HasRecentFollowedByUserIds
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecentFollowedUserIds.scala
trait HasRecentFollowedUserIds
def recentFollowedUserIds: Option[Seq[Long]]
// user ids that are recently followed by the target user in set data-structure
lazy val recentFollowedUserIdsSet: Option[Set[Long]] = recentFollowedUserIds match
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecentFollowedUserIdsWithTime.scala
trait HasRecentFollowedUserIdsWithTime
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecentlyEngagedUserIds.scala
trait HasRecentlyEngagedUserIds
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecommendationFlowIdentifier.scala
trait HasRecommendationFlowIdentifier
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasScores.scala
trait HasScores
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasSimilarToContext.scala
trait HasSimilarToContext
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasTopicId.scala
trait HasTopicId
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasUserCandidateSourceDetails.scala
trait HasUserCandidateSourceDetails
def userCandidateSourceDetails: Option[UserCandidateSourceDetails]
def getAlgorithm: Algorithm
def getAllAlgorithms: Seq[Algorithm]
def getAddressBookMetadata: Option[AddressBookMetadata]
def getCandidateSources: Map[CandidateSourceIdentifier, Option[Double]]
def getCandidateRanks: Map[CandidateSourceIdentifier, Int]
def getCandidateFeatures: Map[CandidateSourceIdentifier, Seq[Feature]]
def getPrimaryCandidateSource: Option[CandidateSourceIdentifier]
def withCandidateSource(source: CandidateSourceIdentifier): CandidateUser
def withCandidateSourceAndScore(
source: CandidateSourceIdentifier,
score: Option[Double]
): CandidateUser
def withCandidateSourceAndFeatures(
source: CandidateSourceIdentifier,
features: Seq[Feature]
): CandidateUser
def withCandidateSourceScoreAndFeatures(
source: CandidateSourceIdentifier,
score: Option[Double],
features: Seq[Feature]
): CandidateUser
def addCandidateSourceScoresMap(
scoreMap: Map[CandidateSourceIdentifier, Option[Double]]
): CandidateUser
def addCandidateSourceRanksMap(
rankMap: Map[CandidateSourceIdentifier, Int]
): CandidateUser
def addInfoPerRankingStage(
rankingStage: String,
scores: Option[Scores],
rank: Int
): CandidateUser
def addAddressBookMetadataIfAvailable(
candidateSources: Seq[CandidateSourceIdentifier]
): CandidateUser
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasUserState.scala
trait HasUserState
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasWtfImpressions.scala
trait HasWtfImpressions
def wtfImpressions: Option[Seq[WtfImpression]]
lazy val numWtfImpressions: Int = wtfImpressions.map(_.size).getOrElse(0)
lazy val candidateImpressions: Map[Long, WtfImpression] = wtfImpressions
.map
def getCandidateImpressionCounts(id: Long): Option[Int] =
candidateImpressions.get(id).map(_.counts)
def getCandidateLatestTime(id: Long): Option[Time]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\OptimusRequest.scala
trait to group together all traits needed for optimus ranking
*/
trait OptimusRequest
extends HasParams
with HasClientContext
with HasDisplayLocation
with HasInterestIds
with HasDebugOptions
with HasPreviousRecommendationsContext
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\Product.scala
object Product
object MagicRecs extends ProductMixerProduct
object PlaceholderProductMixerProduct extends ProductMixerProduct
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\RankingInfo.scala
class RankingInfo(
scores: Option[Scores],
rank: Option[Int])
def toThrift: t.RankingInfo
def toOfflineThrift: offline.RankingInfo
object RankingInfo
def fromThrift(rankingInfo: t.RankingInfo): RankingInfo
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\Reason.scala
class FollowProof(followedBy: Seq[Long], numIds: Int)
def toThrift: t.FollowProof
def toOfflineThrift: offline.FollowProof = offline.FollowProof(followedBy, numIds)
}
object FollowProof
def fromThrift(proof: t.FollowProof): FollowProof
class SimilarToProof(similarTo: Seq[Long])
def toThrift: t.SimilarToProof
def toOfflineThrift: offline.SimilarToProof = offline.SimilarToProof(similarTo)
}
object SimilarToProof
def fromThrift(proof: t.SimilarToProof): SimilarToProof
class PopularInGeoProof(location: String)
def toThrift: t.PopularInGeoProof
def toOfflineThrift: offline.PopularInGeoProof = offline.PopularInGeoProof(location)
}
object PopularInGeoProof
def fromThrift(proof: t.PopularInGeoProof): PopularInGeoProof
class TttInterestProof(interestId: Long, interestDisplayName: String)
def toThrift: t.TttInterestProof
def toOfflineThrift: offline.TttInterestProof =
offline.TttInterestProof(interestId, interestDisplayName)
}
object TttInterestProof
def fromThrift(proof: t.TttInterestProof): TttInterestProof
class TopicProof(topicId: Long)
def toThrift: t.TopicProof
def toOfflineThrift: offline.TopicProof =
offline.TopicProof(topicId)
}
object TopicProof
def fromThrift(proof: t.TopicProof): TopicProof
class CustomInterest(query: String)
def toThrift: t.CustomInterestProof
def toOfflineThrift: offline.CustomInterestProof =
offline.CustomInterestProof(query)
}
object CustomInterest
def fromThrift(proof: t.CustomInterestProof): CustomInterest
class TweetsAuthorProof(tweetIds: Seq[Long])
def toThrift: t.TweetsAuthorProof
def toOfflineThrift: offline.TweetsAuthorProof =
offline.TweetsAuthorProof(tweetIds)
}
object TweetsAuthorProof
def fromThrift(proof: t.TweetsAuthorProof): TweetsAuthorProof
class DeviceFollowProof(isDeviceFollow: Boolean)
def toThrift: t.DeviceFollowProof
def toOfflineThrift: offline.DeviceFollowProof =
offline.DeviceFollowProof(isDeviceFollow)
}
object DeviceFollowProof
def fromThrift(proof: t.DeviceFollowProof): DeviceFollowProof
class AccountProof(
followProof: Option[FollowProof] = None,
similarToProof: Option[SimilarToProof] = None,
popularInGeoProof: Option[PopularInGeoProof] = None,
tttInterestProof: Option[TttInterestProof] = None,
topicProof: Option[TopicProof] = None,
customInterestProof: Option[CustomInterest] = None,
tweetsAuthorProof: Option[TweetsAuthorProof] = None,
deviceFollowProof: Option[DeviceFollowProof] = None)
def toThrift: t.AccountProof
def toOfflineThrift: offline.AccountProof
object AccountProof
def fromThrift(proof: t.AccountProof): AccountProof
class Reason(accountProof: Option[AccountProof])
def toThrift: t.Reason
def toOfflineThrift: offline.Reason
object Reason
def fromThrift(reason: t.Reason): Reason
trait HasReason
def reason: Option[Reason]
// helper methods below
def followedBy: Option[Seq[Long]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\RecentlyEngagedUserId.scala
class RecentlyEngagedUserId(id: Long, engagementType: EngagementType)
def toThrift: t.RecentlyEngagedUserId =
t.RecentlyEngagedUserId(id = id, engagementType = engagementType.toThrift)
def toOfflineThrift: offline.RecentlyEngagedUserId =
offline.RecentlyEngagedUserId(id = id, engagementType = engagementType.toOfflineThrift)
}
object RecentlyEngagedUserId
def fromThrift(recentlyEngagedUserId: t.RecentlyEngagedUserId): RecentlyEngagedUserId
def fromOfflineThrift(
recentlyEngagedUserId: offline.RecentlyEngagedUserId
): RecentlyEngagedUserId
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\RecommendationStep.scala
class RecommendationStep(
recommendations: Seq[FlowRecommendation],
followedUserIds: Set[Long])
def toThrift: t.RecommendationStep = t.RecommendationStep(
recommendations = recommendations.map(_.toThrift),
followedUserIds = followedUserIds
)
def toOfflineThrift: offline.OfflineRecommendationStep =
offline.OfflineRecommendationStep(
recommendations = recommendations.map(_.toOfflineThrift),
followedUserIds = followedUserIds)
}
object RecommendationStep
def fromThrift(recommendationStep: t.RecommendationStep): RecommendationStep
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\SafetyLevel.scala
trait SafetyLevel
def toThrift: ThriftSafetyLevel
}
object SafetyLevel
object Recommendations extends SafetyLevel
object TopicsLandingPageTopicRecommendations extends SafetyLevel
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\Score.scala
trait so it is possible to add more information for different score types.
*/
sealed trait ScoreType
def getName: String
}
/**
* Existing Score Types
*/
object ScoreType
object HeuristicBasedScore extends ScoreType
def getName: String = "HeuristicBasedScore"
}
/**
* probability of follow after the candidate is recommended to the user
*/
case object PFollowGivenReco extends ScoreType
def getName: String = "PFollowGivenReco"
}
/**
* probability of engage after the user follows the candidate
*/
case object PEngagementGivenFollow extends ScoreType
def getName: String = "PEngagementGivenFollow"
}
/**
* probability of engage per tweet impression
*/
case object PEngagementPerImpression extends ScoreType
def getName: String = "PEngagementPerImpression"
}
/**
* probability of engage per tweet impression
*/
case object PEngagementGivenReco extends ScoreType
def getName: String = "PEngagementGivenReco"
}
def fromScoreTypeString(scoreTypeName: String): ScoreType = scoreTypeName match
class Score(
value: Double,
rankerId: Option[RankerId] = None,
scoreType: Option[ScoreType] = None)
def toThrift: t.Score = t.Score(
value = value,
rankerId = rankerId.map(_.toString),
scoreType = scoreType.map(_.getName)
)
def toOfflineThrift: offline.Score =
offline.Score(
value = value,
rankerId = rankerId.map(_.toString),
scoreType = scoreType.map(_.getName)
)
}
object Score
def optimusScore(score: Double, scoreType: ScoreType): Score
def predictionScore(score: Double, rankerId: RankerId): Score
def fromThrift(thriftScore: t.Score): Score =
Score(
value = thriftScore.value,
rankerId = thriftScore.rankerId.flatMap(RankerId.getRankerByName),
scoreType = thriftScore.scoreType.map(ScoreType.fromScoreTypeString)
)
}
/**
* a list of scores
*/
final case class Scores(
scores: Seq[Score],
selectedRankerId: Option[RankerId] = None,
isInProducerScoringExperiment: Boolean = false)
def toThrift: t.Scores =
t.Scores(
scores = scores.map(_.toThrift),
selectedRankerId = selectedRankerId.map(_.toString),
isInProducerScoringExperiment = isInProducerScoringExperiment
)
def toOfflineThrift: offline.Scores =
offline.Scores(
scores = scores.map(_.toOfflineThrift),
selectedRankerId = selectedRankerId.map(_.toString),
isInProducerScoringExperiment = isInProducerScoringExperiment
)
}
object Scores
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\Session.scala
object Session
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\SignalData.scala
trait SignalData
class RecentFollowsSignal(
override val userId: Long,
override val signalType: SignalType,
followedUserId: Long,
timestamp: Long)
extends SignalData
object RecentFollowsSignal
def fromUssSignal(targetUserId: Long, signal: Signal): RecentFollowsSignal
def getRecentFollowedUserIds(
signalDataMap: Option[Map[SignalType, Seq[SignalData]]]
): Option[Seq[Long]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\TrackingToken.scala
class TrackingToken(
sessionId: Long,
displayLocation: Option[DisplayLocation],
controllerData: Option[ControllerData],
algorithmId: Option[Int])
def toThrift: t.TrackingToken
def toOfflineThrift: offline.TrackingToken
object TrackingToken
def serialize(trackingToken: TrackingToken): String
def deserialize(trackingTokenStr: String): TrackingToken
def fromThrift(token: t.TrackingToken): TrackingToken
trait HasTrackingToken
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\UserCandidateSourceDetails.scala
class UserCandidateSourceDetails(
primaryCandidateSource: Option[CandidateSourceIdentifier],
candidateSourceScores: Map[CandidateSourceIdentifier, Option[Double]] = Map.empty,
candidateSourceRanks: Map[CandidateSourceIdentifier, Int] = Map.empty,
addressBookMetadata: Option[AddressBookMetadata] = None,
candidateSourceFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] = Map.empty,
)
def toThrift: t.CandidateSourceDetails
def toOfflineThrift: offline.CandidateSourceDetails
object UserCandidateSourceDetails
def fromThrift(details: t.CandidateSourceDetails): UserCandidateSourceDetails
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\CandidateParamPredicate.scala
class CandidateParamPredicate[A <: HasParams](
param: Param[Boolean],
reason: FilterReason)
extends Predicate[A]
def apply(candidate: A): Stitch[PredicateResult]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\CandidateSourceParamPredicate.scala
class CandidateSourceParamPredicate(
val param: Param[Boolean],
val reason: FilterReason,
candidateSources: Set[CandidateSourceIdentifier])
extends Predicate[CandidateUser]
def apply(candidate: CandidateUser): Stitch[PredicateResult]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\CuratedCompetitorListPredicate.scala
class CuratedCompetitorListPredicate @Inject() (
statsReceiver: StatsReceiver,
@Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) competitorAccountFetcher: Fetcher[
String,
Unit,
Seq[Long]
]) extends Predicate[CandidateUser]
def query(prefix: String): Stitch[Set[Long]] =
competitorAccountFetcher.fetch(prefix).map(_.v.getOrElse(Nil).toSet)
/**
* Caveat here is that though the similarToUserIds allows for a Seq[Long], in practice we would
* only return 1 userId. Multiple userId's would result in filtering candidates associated with
* a different similarToUserId. For example:
* - similarToUser1 -> candidate1, candidate2
* - similarToUser2 -> candidate3
* and in the competitorList store we have:
* - similarToUser1 -> candidate3
* we'll be filtering candidate3 on account of similarToUser1, even though it was generated
* with similarToUser2. This might still be desirable at a product level (since we don't want
* to show these accounts anyway), but might not achieve what you intend to code-wise.
*/
override def apply(candidate: CandidateUser): Stitch[PredicateResult]
object CuratedCompetitorListPredicate
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\ExcludedUserIdPredicate.scala
object ExcludedUserIdPredicate extends Predicate[(HasExcludedUserIds, CandidateUser)]
def apply(pair: (HasExcludedUserIds, CandidateUser)): Stitch[PredicateResult]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\InactivePredicate.scala
class InactivePredicate @Inject() (
statsReceiver: StatsReceiver,
@Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) userRecommendabilityFetcher: Fetcher[
Long,
Unit,
UserRecommendabilityFeatures
]) extends Predicate[(HasParams with HasClientContext with HasUserState, CandidateUser)]
def queryUserRecommendable(userId: Long): Stitch[Option[UserRecommendabilityFeatures]] =
userRecommendabilityFetcher.fetch(userId).map(_.v)
private val userRecommendableCache =
StitchCache[JLong, Option[UserRecommendabilityFeatures]](
maxCacheSize = 100000,
ttl = 12.hours,
statsReceiver = cacheStats.scope("UserRecommendable"),
underlyingCall = (userId: JLong) => queryUserRecommendable(userId)
)
override def apply(
targetAndCandidate: (HasParams with HasClientContext with HasUserState, CandidateUser)
): Stitch[PredicateResult]
def disableInactivityPredicate(
target: HasParams,
consumerState: Option[UserState],
candidateState: Option[UserState]
): Boolean
object InactivePredicate
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\InactivePredicateParams.scala
object InactivePredicateParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\PreviouslyRecommendedUserIdsPredicate.scala
class PreviouslyRecommendedUserIdsPredicate
extends Predicate[(HasPreviousRecommendationsContext, CandidateUser)]
def apply(
pair: (HasPreviousRecommendationsContext, CandidateUser)
): Stitch[PredicateResult]
object PreviouslyRecommendedUserIdsPredicate
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\dismiss\DismissedCandidatePredicate.scala
class DismissedCandidatePredicate extends Predicate[(HasDismissedUserIds, CandidateUser)]
def apply(pair: (HasDismissedUserIds, CandidateUser)): Stitch[PredicateResult]
object DismissedCandidatePredicate
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\dismiss\DismissedCandidatePredicateParams.scala
object DismissedCandidatePredicateParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\gizmoduck\GizmoduckPredicate.scala
class GizmoduckPredicate @Inject() (
gizmoduck: Gizmoduck,
client: Client,
statsReceiver: StatsReceiver,
decider: Decider = Decider.False)
extends Predicate[(HasClientContext with HasParams, CandidateUser)]
with Logging
def apply(
pair: (HasClientContext with HasParams, CandidateUser)
): Stitch[PredicateResult]
def getGizmoduckPredicateResult(
request: HasClientContext with HasParams,
candidate: CandidateUser
): Stitch[PredicateResult]
def logPredicateResultEquality(
request: HasClientContext with HasParams,
candidate: CandidateUser,
predicateResult: PredicateResult
): Unit
def getPredicateResult(
request: HasClientContext with HasParams,
candidate: CandidateUser,
userResult: UserResult,
): PredicateResult
def getByUserId(userId: JLong): Stitch[UserResult]
object GizmoduckPredicate
def getAbPbReason(
user: User,
abMetadataOpt: Option[AddressBookMetadata]
): Set[FilterReason]
def getSafetyReasons(user: User): Set[FilterReason]
def getCountryTakedownReasons(
user: User,
countryCodeOpt: Option[String]
): Set[FilterReason]
def getBlinkReasons(user: User): Set[FilterReason]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\gizmoduck\GizmoduckPredicateCache.scala
object GizmoduckPredicateCache
class TimeTicker extends Ticker
def read(): Long = Time.now.inNanoseconds
}
def apply[K, V](
maxCacheSize: Int,
ttl: Duration,
statsReceiver: StatsReceiver
): Cache[K, V]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\gizmoduck\GizmoduckPredicateFSConfig.scala
class GizmoduckPredicateFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\gizmoduck\GizmoduckPredicateParams.scala
object GizmoduckPredicateParams
object GizmoduckGetTimeout
extends FSBoundedParam[Duration](
name = "gizmoduck_predicate_timeout_in_millis",
default = 200.millisecond,
min = 1.millisecond,
max = 500.millisecond)
with HasDurationConversion
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\health\HssPredicate.scala
class HssPredicate @Inject() (
healthSignalsOnUserClientColumn: HealthSignalsOnUserClientColumn,
statsReceiver: StatsReceiver)
extends Predicate[(HasClientContext with HasParams, CandidateUser)]
with Logging
def apply(
pair: (HasClientContext with HasParams, CandidateUser)
): Stitch[PredicateResult]
def getHssPredicateResult(
request: HasClientContext with HasParams,
candidate: CandidateUser
): Stitch[PredicateResult]
def userHealthSignalValueToDoubleOpt(signalValue: Option[SignalValue]): Option[Double]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\health\HssPredicateFSConfig.scala
class HssPredicateFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\health\HssPredicateParams.scala
object HssPredicateParams
object HssCseScoreThreshold
extends FSBoundedParam[Double](
"hss_predicate_cse_score_threshold",
default = 0.992d,
min = 0.0d,
max = 1.0d)
object HssNsfwScoreThreshold
extends FSBoundedParam[Double](
"hss_predicate_nsfw_score_threshold",
default = 1.5d,
min = -100.0d,
max = 100.0d)
object HssApiTimeout
extends FSBoundedParam[Duration](
name = "hss_predicate_timeout_in_millis",
default = 200.millisecond,
min = 1.millisecond,
max = 500.millisecond)
with HasDurationConversion
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\InvalidRelationshipPredicate.scala
class InvalidRelationshipPredicate
extends Predicate[(HasInvalidRelationshipUserIds, CandidateUser)]
def apply(
pair: (HasInvalidRelationshipUserIds, CandidateUser)
): Stitch[PredicateResult]
object InvalidRelationshipPredicate
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\RecentFollowingPredicate.scala
class RecentFollowingPredicate extends Predicate[(HasRecentFollowedUserIds, CandidateUser)]
def apply(pair: (HasRecentFollowedUserIds, CandidateUser)): Stitch[PredicateResult]
object RecentFollowingPredicate
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\SgsPredicateFSConfig.scala
class SgsPredicateFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\SgsPredicateParams.scala
object SgsPredicateParams
object SgsRelationshipsPredicateTimeout
extends FSBoundedParam[Duration](
name = "sgs_predicate_relationships_timeout_in_millis",
default = 300.millisecond,
min = 1.millisecond,
max = 1000.millisecond)
with HasDurationConversion
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\SgsRelationshipsByUserIdPredicate.scala
class SgsRelationshipsByUserIdPredicate(
socialGraph: SocialGraph,
relationshipMappings: Seq[RelationshipMapping],
statsReceiver: StatsReceiver)
extends Predicate[(Option[Long], CandidateUser)]
with Logging
def apply(
pair: (Option[Long], CandidateUser)
): Stitch[PredicateResult]
object SgsRelationshipsByUserIdPredicate
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\SgsRelationshipsPredicate.scala
class RelationshipMapping(
relationshipType: RelationshipType,
includeBasedOnRelationship: Boolean)
class SgsRelationshipsPredicate(
socialGraph: SocialGraph,
relationshipMappings: Seq[RelationshipMapping],
statsReceiver: StatsReceiver = NullStatsReceiver)
extends Predicate[(HasClientContext with HasParams, CandidateUser)]
with Logging
def apply(
pair: (HasClientContext with HasParams, CandidateUser)
): Stitch[PredicateResult]
object SgsRelationshipsPredicate
def extractUserId(target: HasClientContext with HasParams): Option[Long] = target match
class InvalidTargetCandidateRelationshipTypesPredicate @Inject() (
socialGraph: SocialGraph)
extends SgsRelationshipsPredicate(
socialGraph,
InvalidRelationshipTypesPredicate.InvalidRelationshipTypes)
class NoteworthyAccountsSgsPredicate @Inject() (
socialGraph: SocialGraph)
extends SgsRelationshipsPredicate(
socialGraph,
InvalidRelationshipTypesPredicate.NoteworthyAccountsInvalidRelationshipTypes)
object InvalidRelationshipTypesPredicate
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\user_activity\UserActivityPredicate.scala
abstract case class UserStateActivityPredicate(
userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn,
validCandidateStates: Set[UserState],
client: Client,
statsReceiver: StatsReceiver,
decider: Decider = Decider.False)
extends Predicate[(HasParams with HasClientContext, CandidateUser)]
def apply(
targetAndCandidate: (HasParams with HasClientContext, CandidateUser)
): Stitch[PredicateResult]
def queryUserRecommendable(
userId: Long
): Stitch[Option[UserState]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\user_activity\UserActivityPredicateParams.scala
object UserActivityPredicateParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\common\AdhocScoreModificationType.scala
object AdhocScoreModificationType extends Enumeration
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\common\DedupCandidates.scala
object DedupCandidates
def apply[C <: UniversalNoun[Long]](input: Seq[C]): Seq[C]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\common\RankerId.scala
object RankerId extends Enumeration
def getRankerByName(name: String): Option[RankerId] =
RankerId.values.toSeq.find(_.equals(Value(name)))
}
/**
* ML model based heavy ranker ids.
*/
object ModelBasedHeavyRankerId
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\fatigue_ranker\ImpressionBasedFatigueRanker.scala
class ImpressionBasedFatigueRanker[
Target <: HasClientContext with HasDisplayLocation with HasParams with HasWtfImpressions
](
fatigueFactor: Int,
statsReceiver: StatsReceiver)
extends Ranker[Target, CandidateUser]
def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]]
def trackTimeSinceOldestImpression(impressions: Seq[WtfImpression]): Unit
def rankCandidates(
target: Target,
candidates: Seq[CandidateUser]
): Seq[CandidateUser]
object ImpressionBasedFatigueRanker
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\fatigue_ranker\ImpressionBasedFatigueRankerFSConfig.scala
class ImpressionBasedFatigueRankerFSConfig @Inject() extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\fatigue_ranker\ImpressionBasedFatigueRankerParams.scala
object ImpressionBasedFatigueRankerParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\first_n_ranker\FirstNRanker.scala
class is meant to filter candidates between stages of our ranker by taking the first N
* candidates, merging any candidate source information for candidates with multiple entries.
* To allow us to chain this truncation operation any number of times sequentially within the main
* ranking builder, we abstract the truncation as a separate Ranker
*/
@Singleton
class FirstNRanker[Target <: HasClientContext with HasParams with HasQualityFactor] @Inject() (
stats: StatsReceiver)
extends Ranker[Target, CandidateUser]
class CandidateSourceScore(
candidateId: Long,
sourceId: CandidateSourceIdentifier,
score: Option[Double])
/**
* Adds the rank of each candidate based on the primary candidate source's score.
* In the event where the provided ordering of candidates do not align with the score,
* we will respect the score, since the ordering might have been mixed up due to other previous
* steps like the shuffleFn in the `WeightedCandidateSourceRanker`.
* @param candidates ordered list of candidates
* @return same ordered list of candidates, but with the rank information appended
*/
def addRank(candidates: Seq[CandidateUser]): Seq[CandidateUser]
def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]]
def merge(candidates: Seq[CandidateUser]): CandidateUser
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\first_n_ranker\FirstNRankerFeatureSwitchKeys.scala
object FirstNRankerFeatureSwitchKeys
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\first_n_ranker\FirstNRankerFSConfig.scala
class FirstNRankerFSConfig @Inject() extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\first_n_ranker\FirstNRankerParams.scala
object FirstNRankerParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\interleave_ranker\InterleaveRanker.scala
class InterleaveRanker[Target <: HasParams] @Inject() (
statsReceiver: StatsReceiver)
extends Ranker[Target, CandidateUser]
def rank(
target: Target,
candidates: Seq[CandidateUser]
): Stitch[Seq[CandidateUser]]
def rankCandidates(
target: Target,
candidates: Seq[CandidateUser]
): Seq[CandidateUser]
def interleaveCandidates(
candidates: Seq[CandidateUser],
rankerIds: Seq[RankerId.RankerId]
): Seq[CandidateUser]
def resolveConflicts(
candidatesWithRank: Seq[(CandidateUser, Int)]
): Seq[(CandidateUser, Int)]
def getCandidateScoreByRankerId(
candidate: CandidateUser,
rankerIdOpt: Option[RankerId.RankerId]
): Option[Double]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\interleave_ranker\InterleaveRankerFSConfig.scala
class InterleaveRankerFSConfig @Inject() extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\interleave_ranker\InterleaveRankerParams.scala
object InterleaveRankerParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\ranking\HydrateFeaturesTransform.scala
class HydrateFeaturesTransform[
Target <: HasClientContext with HasParams with HasDebugOptions with HasPreFetchedFeature with HasSimilarToContext with HasDisplayLocation] @Inject() (
userScoringFeatureSource: UserScoringFeatureSource,
stats: StatsReceiver)
extends GatedTransform[Target, CandidateUser]
with Logging
def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\ranking\MlRanker.scala
class has a rank function that will perform 4 steps:
* - choose which scorer to use for each candidate
* - score candidates given their respective features
* - add scoring information to the candidate
* - sort candidates by their respective scores
* The feature source and scorer will depend on the request's params
*/
@Singleton
class MlRanker[
Target <: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions] @Inject() (
scorerFactory: ScorerFactory,
statsReceiver: StatsReceiver)
extends Ranker[Target, CandidateUser]
with Logging
def rank(
target: Target,
candidates: Seq[CandidateUser]
): Stitch[Seq[CandidateUser]]
def chooseRankerByCandidate(
candidates: Seq[CandidateUser],
requestRankerId: RankerId
): Map[CandidateUser, RankerId]
def score(
candidates: Seq[CandidateUser],
rankerIds: Map[CandidateUser, RankerId],
requestRankerId: RankerId
): Stitch[Seq[CandidateUser]]
object
case (candidate, scores) =>
val selectedRankerId = rankerIds(candidate)
val useRequestRanker =
candidate.params == Params.Invalid ||
candidate.params == Params.Empty ||
candidate.params(MlRankerParams.CandidateScorerIdParam) == RankerId.None
candidate.copy(
score = scores.scores.find(_.rankerId.contains(requestRankerId)).map(_.value),
scores = if (scores.scores.nonEmpty)
def addMlBaseScoresForAdhocScorers(
candidates: Seq[CandidateUser],
requestRankerId: RankerId,
adhocScorers: Seq[Scorer]
): Seq[CandidateUser]
def score(
dataRecords: Seq[DataRecord],
scorers: Seq[Scorer]
): Stitch[Seq[Scores]]
def sort(
target: Target,
candidates: Seq[CandidateUser]
): Seq[CandidateUser]
def scribeCandidates(
target: Target,
candidates: Seq[CandidateUser]
): Seq[CandidateUser]
object MlRanker
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\ranking\MlRankerFSConfig.scala
class MlRankerFSConfig @Inject() extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\ranking\MlRankerParams.scala
object MlRankerParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\AdhocScorer.scala
trait AdhocScorer extends Scorer
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\DeepbirdScorer.scala
trait that implements the scoring given a deepbirdClient
* To test out a new model, create a scorer extending this trait, override the modelName and inject the scorer
*/
trait DeepbirdScorer extends Scorer
def modelName: String
def predictionFeature: Feature.Continuous
// Set a default batchSize of 100 when making model prediction calls to the Deepbird V2 prediction server
def batchSize: Int = 100
def deepbirdClient: DeepbirdPredictionService.ServiceToClient
def baseStats: StatsReceiver
def modelSelector: ModelSelector = new ModelSelector().setId(modelName)
def stats: StatsReceiver = baseStats.scope(this.getClass.getSimpleName).scope(modelName)
private def requestCount = stats.counter("requests")
private def emptyRequestCount = stats.counter("empty_requests")
private def successCount = stats.counter("success")
private def failureCount = stats.counter("failures")
private def inputRecordsStat = stats.stat("input_records")
private def outputRecordsStat = stats.stat("output_records")
// Counters for tracking batch-prediction statistics when making DBv2 prediction calls
//
// numBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers
private def numBatchRequests = stats.counter("batches")
// numEmptyBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers
// that had an empty input DataRecord
private def numEmptyBatchRequests = stats.counter("empty_batches")
// numTimedOutBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers
// that had timed-out
private def numTimedOutBatchRequests = stats.counter("timeout_batches")
private def batchPredictionLatency = stats.stat("batch_prediction_latency")
private def predictionLatency = stats.stat("prediction_latency")
private def numEmptyModelPredictions = stats.counter("empty_model_predictions")
private def numNonEmptyModelPredictions = stats.counter("non_empty_model_predictions")
private val DefaultPredictionScore = 0.0
/**
* NOTE: For instances of [[DeepbirdScorer]] this function SHOULD NOT be used.
* Please use [[score(records: Seq[DataRecord])]] instead.
*/
@Deprecated
def score(
target: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions,
candidates: Seq[CandidateUser]
): Seq[Option[Score]] =
throw new UnsupportedOperationException(
"For instances of DeepbirdScorer this operation is not defined. Please use " +
"`def score(records: Seq[DataRecord]): Stitch[Seq[Score]]` " +
"instead.")
override def score(records: Seq[DataRecord]): Stitch[Seq[Score]]
def batchPredict(
dataRecords: Seq[DataRecord],
batchSize: Int
): Future[Seq[Option[Double]]]
def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\PostnuxDeepbirdProdScorer.scala
trait when adding new fields to DeepbirdV2 scorers which
trait DeepbirdProdScorer extends DeepbirdScorer
trait PostNuxV1DeepbirdProdScorer extends DeepbirdProdScorer
class PostnuxDeepbirdProdScorer @Inject() (
@Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT)
override val deepbirdClient: DeepbirdPredictionService.ServiceToClient,
override val baseStats: StatsReceiver)
extends PostNuxV1DeepbirdProdScorer
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\RandomScorer.scala
class RandomScorer @Inject() (
@Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT)
override val deepbirdClient: DeepbirdPredictionService.ServiceToClient,
override val baseStats: StatsReceiver)
extends DeepbirdScorer
def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\Scorer.scala
trait Scorer
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\ScorerFactory.scala
class ScorerFactory @Inject() (
postnuxProdScorer: PostnuxDeepbirdProdScorer,
randomScorer: RandomScorer,
stats: StatsReceiver)
def getScorers(
rankerIds: Seq[RankerId]
): Seq[Scorer]
def getScorerById(scorerId: RankerId): Scorer = scorerId match
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\utils\Utils.scala
object Utils
def addRankingInfo(candidates: Seq[CandidateUser], rankingStage: String): Seq[CandidateUser]
def getCandidateScoreByRankerId(candidate: CandidateUser, rankerId: RankerId): Option[Score] =
candidate.scores.flatMap
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\CandidateShuffle.scala
trait CandidateShuffler[T]
def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T]
}
class NoShuffle[T]() extends CandidateShuffler[T]
def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = input
}
class RandomShuffler[T]() extends CandidateShuffler[T]
def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T]
trait RankWeightedRandomShuffler[T] extends CandidateShuffler[T]
def rankToWeight(rank: Int): Double
def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T]
class ExponentialShuffler[T]() extends RankWeightedRandomShuffler[T]
def rankToWeight(rank: Int): Double
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightedCandidateSourceBaseRanker.scala
class WeightedCandidateSourceBaseRanker[A, Rec](
candidateSourceWeights: Map[A, Double],
weightMethod: WeightMethod = WeightedRandomSampling,
randomSeed: Option[Long])
def stream(
candidateSources: Set[A],
candidateSourceWeights: Map[A, Double],
candidates: Map[A, Iterator[Rec]],
weightMethod: WeightMethod = WeightedRandomSampling,
random: Option[Random] = None
): Stream[Rec]
def apply(a: A): Double = candidateSourceWeights.getOrElse(a, 0)
}
/**
* Generates a stream of candidates.
*
* @param candidateSourceIter an iterator over candidate sources returned by the sampling procedure
* @return stream of candidates
*/
def next(candidateSourceIter: Iterator[A]): Stream[Rec]
def apply(input: Map[A, TraversableOnce[Rec]]): Stream[Rec]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightedCandidateSourceRanker.scala
class WeightedCandidateSourceRanker[Target <: HasParams](
basedRanker: WeightedCandidateSourceBaseRanker[
CandidateSourceIdentifier,
CandidateUser
],
shuffleFn: Seq[CandidateUser] => Seq[CandidateUser],
dedup: Boolean)
extends Ranker[Target, CandidateUser]
def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]]
def group(
candidates: Seq[CandidateUser]
): Map[CandidateSourceIdentifier, Seq[CandidateUser]]
def rankCandidates(
input: Map[CandidateSourceIdentifier, Seq[CandidateUser]]
): Seq[CandidateUser]
object WeightedCandidateSourceRanker
def build[Target <: HasParams](
candidateSourceWeight: Map[CandidateSourceIdentifier, Double],
shuffleFn: Seq[CandidateUser] => Seq[CandidateUser] = identity,
dedup: Boolean = false,
randomSeed: Option[Long] = None
): WeightedCandidateSourceRanker[Target]
object WeightedCandidateSourceRankerWithoutRandomSampling
def build[Target <: HasParams](
candidateSourceWeight: Map[CandidateSourceIdentifier, Double]
): WeightedCandidateSourceRanker[Target]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightedCandidateSourceRankerFSConfig.scala
class WeightedCandidateSourceRankerFSConfig @Inject() extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightedCandidateSourceRankerParams.scala
object WeightedCandidateSourceRankerParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightMethod.scala
object WeightMethod extends Enumeration
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\stores\LowTweepCredFollowStore.scala
class LowTweepCredFollowStore @Inject() (tweepCredOnUserClientColumn: TweepCredOnUserClientColumn)
def getLowTweepCredUsers(target: HasRecentFollowedUserIds): Stitch[Seq[CandidateUser]]
object LowTweepCredFollowStore
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\dedup\DedupTransform.scala
class DedupTransform[Request, Candidate <: UniversalNoun[Long]]()
extends Transform[Request, Candidate]
def transform(target: Request, candidates: Seq[Candidate]): Stitch[Seq[Candidate]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\modify_social_proof\ModifySocialProofTransform.scala
object ModifySocialProof
def addIntersectionIdsToCandidate(
candidate: CandidateUser,
followProof: FollowProof,
stats: StatsReceiver
): CandidateUser
def addCandidatesWithSocialContextCountStat(
statsReceiver: StatsReceiver,
count: Int
): Unit
class makes a request to gfs/sgs for hydrating additional social proof on each of the
* provided candidates.
*/
@Singleton
class ModifySocialProof @Inject() (
gfsClient: GraphFeatureServiceClient,
socialGraphClient: SocialGraphClient,
statsReceiver: StatsReceiver,
decider: Decider = Decider.True)
extends Logging
def hydrateSocialProof(
userId: Long,
candidates: Seq[CandidateUser],
intersectionIdsNum: Option[Int] = None,
mustCallSgs: Boolean = false,
callSgsCachedColumn: Boolean = false,
gfsLagDuration: Duration = GfsLagDuration,
gfsIntersectionIds: Int = GfsIntersectionIds,
sgsIntersectionIds: Int = SgsIntersectionIds,
): Stitch[Seq[CandidateUser]]
class ModifySocialProofTransform @Inject() (modifySocialProof: ModifySocialProof)
extends GatedTransform[HasClientContext with HasParams, CandidateUser]
with Logging
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\modify_social_proof\RemoveAccountProofTransform.scala
class RemoveAccountProofTransform @Inject() (statsReceiver: StatsReceiver)
extends GatedTransform[HasClientContext with HasParams, CandidateUser]
def transform(
target: HasClientContext with HasParams,
items: Seq[CandidateUser]
): Stitch[Seq[CandidateUser]] =
Stitch.value(items.map
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\ranker_id\RandomRankerIdTransform.scala
class appends each candidate's rankerIds with the RandomRankerId.
* This is primarily for determining if a candidate was generated via random shuffling.
*/
@Singleton
class RandomRankerIdTransform @Inject() () extends GatedTransform[HasParams, CandidateUser]
def transform(
target: HasParams,
candidates: Seq[CandidateUser]
): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\recommendation_flow_identifier\AddRecommendationFlowIdentifierTransform.scala
class AddRecommendationFlowIdentifierTransform @Inject()
extends Transform[HasRecommendationFlowIdentifier, CandidateUser]
def transform(
target: HasRecommendationFlowIdentifier,
items: Seq[CandidateUser]
): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\tracking_token\TrackingTokenTransform.scala
class TrackingTokenTransform @Inject() (baseStatsReceiver: StatsReceiver)
extends Transform[HasDisplayLocation with HasClientContext, CandidateUser]
with Logging
def profileResults(
target: HasDisplayLocation with HasClientContext,
candidates: Seq[CandidateUser]
)
def transform(
target: HasDisplayLocation with HasClientContext,
candidates: Seq[CandidateUser]
): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\weighted_sampling\SamplingTransform.scala
class SamplingTransform @Inject() ()
extends GatedTransform[HasClientContext with HasParams with HasDebugOptions, CandidateUser]
def transform(
target: HasClientContext with HasParams with HasDebugOptions,
candidates: Seq[CandidateUser]
): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\weighted_sampling\SamplingTransformFSConfig.scala
class SamplingTransformFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\weighted_sampling\SamplingTransformParams.scala
object SamplingTransformParams
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\CollectionUtil.scala
object CollectionUtil
def transposeLazy[A](seq: Seq[Seq[A]]): Stream[Seq[A]] =
seq.filter(_.nonEmpty) match
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\DisplayLocationProductConverterUtil.scala
object DisplayLocationProductConverterUtil
def productToDisplayLocation(product: Product): DisplayLocation
def displayLocationToProduct(displayLocation: DisplayLocation): Product
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\MergeUtil.scala
object MergeUtil
def weightedRoundRobin[T](
items: Seq[T]
)(
implicit weighted: Weighted[T]
): Stream[T]
def next(): Stream[T]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\RandomUtil.scala
object RandomUtil
def weightedRandomSamplingWithReplacement[T](
items: Seq[T],
random: Option[Random] = None
)(
implicit weighted: Weighted[T]
): Stream[T]
def next(): Stream[T]
def weightedRandomShuffle[T](
items: Seq[T],
random: Option[Random] = None
)(
implicit weighted: Weighted[T]
): Stream[T]
def next(it: Seq[T]): Stream[T]
def weightedRandomShuffleByRank[T](
items: Seq[T],
rankToWeight: Int => Double,
random: Option[Random] = None
): Stream[T]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\RescueWithStatsUtils.scala
object RescueWithStatsUtils
def rescueWithStats[T](
s: Stitch[Seq[T]],
stats: StatsReceiver,
source: String
): Stitch[Seq[T]]
def rescueOptionalWithStats[T](
s: Stitch[Option[T]],
stats: StatsReceiver,
source: String
): Stitch[Option[T]]
def rescueWithStatsWithin[T](
s: Stitch[Seq[T]],
stats: StatsReceiver,
source: String,
timeout: Duration
): Stitch[Seq[T]]
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\UserSignupUtil.scala
object UserSignupUtil
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\Weighted.scala
class for any Recommendation type that has a weight
*
*/
trait Weighted[-Rec]
def apply(rec: Rec): Double
}
object Weighted
object WeightedTuple extends Weighted[(_, Double)]
def apply(rec: (_, Double)): Double = rec._2
}
def fromFunction[Rec](f: Rec => Double): Weighted[Rec]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\FollowRecommendationsServiceThriftServer.scala
object FollowRecommendationsServiceThriftServerMain extends FollowRecommendationsServiceThriftServer
class FollowRecommendationsServiceThriftServer
extends ThriftServer
with Mtls
with HttpServer
with HttpMtls
def configureThrift(router: ThriftRouter): Unit
def configureThriftServer(server: ThriftMux.Server): ThriftMux.Server
def configureHttp(router: HttpRouter): Unit = router.add(
ProductMixerController[FollowRecommendationsThriftService.MethodPerEndpoint](
this.injector,
FollowRecommendationsThriftService.ExecutePipeline))
override def warmup(): Unit
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\Action.scala
class Action(text: String, actionURL: String)
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\FeedbackAction.scala
trait FeedbackAction
def toThrift: t.FeedbackAction
}
case class DismissUserId() extends FeedbackAction
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\Footer.scala
class Footer(action: Option[Action])
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\Header.scala
class Header(title: Title)
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\SocialProof.scala
trait SocialProof
case class GeoContextProof(popularInCountryText: ExternalString) extends SocialProof
case class FollowedByUsersProof(text1: ExternalString, text2: ExternalString, textN: ExternalString)
extends SocialProof
sealed trait SocialText
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\Title.scala
class Title(text: String)
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\WTFPresentation.scala
trait WTFPresentation
def toThrift: t.WTFPresentation
}
case class UserList(
userBioEnabled: Boolean,
userBioTruncated: Boolean,
userBioMaxLines: Option[Long],
feedbackAction: Option[FeedbackAction])
extends WTFPresentation
def toThrift: t.WTFPresentation
object UserList
def fromUserListOptions(
userListOptions: UserListOptions
): UserList
class Carousel(
feedbackAction: Option[FeedbackAction])
extends WTFPresentation
def toThrift: t.WTFPresentation
object Carousel
def fromCarouselOptions(
carouselOptions: CarouselOptions
): Carousel
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\blenders\PromotedAccountsBlender.scala
class PromotedAccountsBlender @Inject() (statsReceiver: StatsReceiver)
extends Transform[Int, Recommendation]
with Logging
def transform(
maxResults: Int,
items: Seq[Recommendation]
): Stitch[Seq[Recommendation]]
def mergePromotedAccounts(
organicUsers: Seq[Recommendation],
promotedUsers: Seq[Recommendation]
): Seq[Recommendation]
def mergeAccountWithIndex(
organicUsers: Seq[Recommendation],
promotedUsers: Seq[Recommendation],
index: Int
): Stream[Recommendation]
def blendPromotedAccount(
organic: Seq[Recommendation],
promoted: Seq[Recommendation],
maxResults: Int
): Seq[Recommendation]
def isBlendPromotedNeeded(
organicSize: Int,
promotedSize: Int,
maxResults: Int
): Boolean
object PromotedAccountsBlender
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\ConfigBuilder.scala
class ConfigBuilder @Inject() (
deciderConfigs: DeciderConfigs,
featureSwitchConfigs: FeatureSwitchConfigs)
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\DeciderConfigs.scala
class DeciderConfigs @Inject() (deciderGateBuilder: DeciderGateBuilder)
object DeciderConfigs
object UserOrGuestOrRequest extends RecipientBuilder
def apply(requestContext: BaseRequestContext): Option[Recipient] = requestContext match
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\FeatureSwitchConfigs.scala
class FeatureSwitchConfigs @Inject() (
globalFeatureSwitchConfig: GlobalFeatureSwitchConfig,
featureHydrationSourcesFSConfig: FeatureHydrationSourcesFSConfig,
weightedCandidateSourceRankerFSConfig: WeightedCandidateSourceRankerFSConfig,
// Flow related config
contentRecommenderFlowFSConfig: ContentRecommenderFlowFSConfig,
postNuxMlFlowFSConfig: PostNuxMlFlowFSConfig,
// Candidate source related config
crowdSearchAccountsFSConfig: CrowdSearchAccountsFSConfig,
offlineStpSourceFsConfig: OfflineStpSourceFsConfig,
onlineSTPSourceFSConfig: OnlineSTPSourceFSConfig,
popGeoSourceFSConfig: PopGeoSourceFSConfig,
popGeoQualityFollowFSConfig: PopGeoQualityFollowSourceFSConfig,
realGraphOonFSConfig: RealGraphOonFSConfig,
repeatedProfileVisitsFSConfig: RepeatedProfileVisitsFSConfig,
recentEngagementSimilarUsersFSConfig: RecentEngagementSimilarUsersFSConfig,
recentFollowingRecentFollowingExpansionSourceFSConfig: RecentFollowingRecentFollowingExpansionSourceFSConfig,
simsExpansionFSConfig: SimsExpansionFSConfig,
simsSourceFSConfig: SimsSourceFSConfig,
socialProofEnforcedCandidateSourceFSConfig: SocialProofEnforcedCandidateSourceFSConfig,
triangularLoopsFSConfig: TriangularLoopsFSConfig,
userUserGraphFSConfig: UserUserGraphFSConfig,
// Predicate related configs
gizmoduckPredicateFSConfig: GizmoduckPredicateFSConfig,
hssPredicateFSConfig: HssPredicateFSConfig,
sgsPredicateFSConfig: SgsPredicateFSConfig,
ppmiLocaleSourceFSConfig: PPMILocaleFollowSourceFSConfig,
topOrganicFollowsAccountsFSConfig: TopOrganicFollowsAccountsFSConfig,
statsReceiver: StatsReceiver)
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\GlobalFeatureSwitchConfig.scala
class GlobalFeatureSwitchConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\ParamsFactory.scala
class ParamsFactory @Inject() (
config: Config,
requestContextFactory: RequestContextFactory,
statsReceiver: StatsReceiver)
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\RequestContextFactory.scala
class RequestContextFactory @Inject() (featureSwitches: FeatureSwitches, decider: Decider)
def apply(
clientContext: ClientContext,
displayLocation: DisplayLocation,
featureOverrides: Map[String, FeatureValue]
): RequestContext
def getFeatureContext(
clientContext: ClientContext,
displayLocation: DisplayLocation,
featureOverrides: Map[String, FeatureValue]
): FeatureContext
def getFeatureSwitchRecipient(
clientContext: ClientContext
): FeatureSwitchRecipient
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\candidates\CandidateUserContextFactory.scala
class CandidateUserContextFactory @Inject() (
@Named(PRODUCER_SIDE_FEATURE_SWITCHES) featureSwitches: FeatureSwitches,
decider: Decider)
def apply(
candidateUser: CandidateUser,
displayLocation: DisplayLocation
): CandidateUserContext
def getFeatureContext(
candidateUser: CandidateUser,
displayLocation: DisplayLocation
): FeatureContext
def getFeatureSwitchRecipient(
candidateUser: CandidateUser
): FeatureSwitchRecipient
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\candidates\CandidateUserParamsFactory.scala
class CandidateUserParamsFactory[T <: HasParams with HasDisplayLocation] @Inject() (
config: Config,
candidateContextFactory: CandidateUserContextFactory,
statsReceiver: StatsReceiver)
def apply(candidateContext: CandidateUser, request: T): CandidateUser
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\candidates\HydrateCandidateParamsTransform.scala
class HydrateCandidateParamsTransform[Target <: HasParams with HasDisplayLocation] @Inject() (
candidateParamsFactory: CandidateUserParamsFactory[Target])
extends Transform[Target, CandidateUser]
with Logging
def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\common\FeatureSwitchConfig.scala
trait FeatureSwitchConfig
def booleanFSParams: Seq[Param[Boolean] with FSName] = Nil
def intFSParams: Seq[FSBoundedParam[Int]] = Nil
def longFSParams: Seq[FSBoundedParam[Long]] = Nil
def doubleFSParams: Seq[FSBoundedParam[Double]] = Nil
def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Nil
def optionalDoubleFSParams: Seq[
(BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName)
] = Nil
def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = Nil
/**
* Apply overrides in list when the given FS Key is enabled.
* This override type does NOT work with experiments. Params here will be evaluated for every
* request IMMEDIATELY, not upon param.apply. If you would like to use an experiment pls use
* the primitive type or ENUM overrides.
*/
def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = Map.empty
}
object FeatureSwitchConfig
def merge(configs: Seq[FeatureSwitchConfig]): FeatureSwitchConfig = new FeatureSwitchConfig
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\deciders\DeciderKey.scala
object DeciderKey extends DeciderKeyEnum
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\deciders\DeciderParams.scala
object DeciderParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\params\GlobalParams.scala
object GlobalParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\CandidateUserDebugParamsBuilder.scala
class CandidateUserDebugParamsBuilder @Inject() (paramsFactory: ParamsFactory)
def fromThrift(req: t.ScoringUserRequest): CandidateUserDebugParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\RecommendationRequestBuilder.scala
class RecommendationRequestBuilder @Inject() (
requestBuilderUserFetcher: RequestBuilderUserFetcher,
statsReceiver: StatsReceiver)
def fromThrift(tRequest: t.RecommendationRequest): Stitch[RecommendationRequest]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\RequestBuilderUserFetcher.scala
class RequestBuilderUserFetcher @Inject() (
gizmoduck: Gizmoduck,
statsReceiver: StatsReceiver,
decider: Decider)
def fetchUser(userIdOpt: Option[Long]): Stitch[Option[User]]
def enableDecider(userId: Long): Boolean
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\ScoringUserRequestBuilder.scala
class ScoringUserRequestBuilder @Inject() (
requestBuilderUserFetcher: RequestBuilderUserFetcher,
candidateUserDebugParamsBuilder: CandidateUserDebugParamsBuilder,
statsReceiver: StatsReceiver)
def fromThrift(req: t.ScoringUserRequest): Stitch[ScoringUserRequest]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\ThriftController.scala
class ThriftController @Inject() (
userScoringService: UserScoringService,
recommendationRequestBuilder: RecommendationRequestBuilder,
scoringUserRequestBuilder: ScoringUserRequestBuilder,
productPipelineSelector: ProductPipelineSelector,
paramsFactory: ParamsFactory)
extends Controller(FollowRecommendationsThriftService)
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\ads\PromotedAccountsFlow.scala
class PromotedAccountsFlow @Inject() (
promotedAccountsCandidateSource: PromotedAccountsCandidateSource,
trackingTokenTransform: TrackingTokenTransform,
baseStatsReceiver: StatsReceiver,
@Flag("fetch_prod_promoted_accounts") fetchProductionPromotedAccounts: Boolean)
extends RecommendationFlow[PromotedAccountsFlowRequest, CandidateUser]
def targetEligibility: Predicate[PromotedAccountsFlowRequest] =
new ParamPredicate[PromotedAccountsFlowRequest](
PromotedAccountsFlowParams.TargetEligibility
)
protected override def candidateSources(
target: PromotedAccountsFlowRequest
): Seq[CandidateSource[PromotedAccountsFlowRequest, CandidateUser]]
def preRankerCandidateFilter: Predicate[
(PromotedAccountsFlowRequest, CandidateUser)
]
def selectRanker(
target: PromotedAccountsFlowRequest
): Ranker[PromotedAccountsFlowRequest, CandidateUser]
def postRankerTransform: Transform[
PromotedAccountsFlowRequest,
CandidateUser
]
def validateCandidates: Predicate[
(PromotedAccountsFlowRequest, CandidateUser)
]
def transformResults: Transform[PromotedAccountsFlowRequest, CandidateUser]
def resultsConfig(
target: PromotedAccountsFlowRequest
): RecommendationResultsConfig
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\ads\PromotedAccountsFlowParams.scala
abstract class PromotedAccountsFlowParams[A](default: A) extends Param[A](default)
object PromotedAccountsFlowParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\ads\PromotedAccountsFlowRequest.scala
class PromotedAccountsFlowRequest(
override val clientContext: ClientContext,
override val params: Params,
displayLocation: DisplayLocation,
profileId: Option[Long],
// note we also add userId and profileId to excludeUserIds
excludeIds: Seq[Long])
extends HasParams
with HasClientContext
with HasExcludedUserIds
with HasDisplayLocation
def toAdsRequest(fetchProductionPromotedAccounts: Boolean): AdRequest
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\ads\PromotedAccountsUtil.scala
object PromotedAccountsUtil
def toCandidateUser(promotedCandidateUser: PromotedCandidateUser): CandidateUser
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlow.scala
class ContentRecommenderFlow @Inject() (
contentRecommenderFlowCandidateSourceRegistry: ContentRecommenderFlowCandidateSourceRegistry,
recentFollowingPredicate: RecentFollowingPredicate,
gizmoduckPredicate: GizmoduckPredicate,
inactivePredicate: InactivePredicate,
sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate,
invalidRelationshipPredicate: InvalidRelationshipPredicate,
trackingTokenTransform: TrackingTokenTransform,
baseStatsReceiver: StatsReceiver)
extends RecommendationFlow[ContentRecommenderRequest, CandidateUser]
with RecommendationFlowBaseSideEffectsUtil[ContentRecommenderRequest, CandidateUser]
with CandidateSourceHoldbackUtil
def targetEligibility: Predicate[ContentRecommenderRequest] =
new ParamPredicate[ContentRecommenderRequest](
ContentRecommenderParams.TargetEligibility
)
protected override def candidateSources(
target: ContentRecommenderRequest
): Seq[CandidateSource[ContentRecommenderRequest, CandidateUser]]
object recentFollowingGatedPredicate
extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)](
recentFollowingPredicate,
recentFollowingPredicateStats
)
def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean =
item._1.params(ContentRecommenderParams.EnableRecentFollowingPredicate)
}
object invalidRelationshipGatedPredicate
extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)](
invalidRelationshipPredicate,
invalidRelationshipPredicateStats
)
def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean =
item._1.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate)
}
ExcludedUserIdPredicate
.observe(preRankerFilterStats.scope("exclude_user_id_predicate"))
.andThen(recentFollowingGatedPredicate.observe(recentFollowingPredicateStats))
.andThen(invalidRelationshipGatedPredicate.observe(invalidRelationshipPredicateStats))
}
/**
* rank the candidates
*/
protected override def selectRanker(
target: ContentRecommenderRequest
): Ranker[ContentRecommenderRequest, CandidateUser]
def postRankerTransform: Transform[
ContentRecommenderRequest,
CandidateUser
]
def validateCandidates: Predicate[
(ContentRecommenderRequest, CandidateUser)
]
def transformResults: Transform[ContentRecommenderRequest, CandidateUser]
def resultsConfig(
target: ContentRecommenderRequest
): RecommendationResultsConfig
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowCandidateSourceRegistry.scala
class ContentRecommenderFlowCandidateSourceRegistry @Inject() (
// social based
forwardPhoneBookSource: ForwardPhoneBookSource,
forwardEmailBookSource: ForwardEmailBookSource,
reversePhoneBookSource: ReversePhoneBookSource,
reverseEmailBookSource: ReverseEmailBookSource,
offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource,
triangularLoopsSource: TriangularLoopsSource,
userUserGraphCandidateSource: UserUserGraphCandidateSource,
realGraphOonSource: RealGraphOonV2Source,
recentFollowingRecentFollowingExpansionSource: RecentFollowingRecentFollowingExpansionSource,
// activity based
recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource,
recentEngagementSimilarUsersSource: RecentEngagementSimilarUsersSource,
repeatedProfileVisitsSource: RepeatedProfileVisitsSource,
// geo based
popCountrySource: PopCountrySource,
popGeohashSource: PopGeohashSource,
popCountryBackFillSource: PopCountryBackFillSource,
crowdSearchAccountsSource: CrowdSearchAccountsSource,
topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource,
ppmiLocaleFollowSource: PPMILocaleFollowSource,
baseStatsReceiver: StatsReceiver)
extends CandidateSourceRegistry[ContentRecommenderRequest, CandidateUser]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowCandidateSourceWeights.scala
object ContentRecommenderFlowCandidateSourceWeights
def getWeights(
params: Params
): Map[CandidateSourceIdentifier, Double]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowCandidateSourceWeightsParams.scala
object ContentRecommenderFlowCandidateSourceWeightsParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowFeatureSwitchKeys.scala
object ContentRecommenderFlowFeatureSwitchKeys
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowFSConfig.scala
class ContentRecommenderFlowFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderParams.scala
abstract class ContentRecommenderParams[A](default: A) extends Param[A](default)
object ContentRecommenderParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderRequest.scala
class ContentRecommenderRequest(
override val params: Params,
override val clientContext: ClientContext,
inputExcludeUserIds: Seq[Long],
override val recentFollowedUserIds: Option[Seq[Long]],
override val recentFollowedByUserIds: Option[Seq[Long]],
override val invalidRelationshipUserIds: Option[Set[Long]],
override val displayLocation: DisplayLocation,
maxResults: Option[Int] = None,
override val debugOptions: Option[DebugOptions] = None,
override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None,
override val userState: Option[UserState] = None)
extends HasParams
with HasClientContext
with HasDisplayLocation
with HasDebugOptions
with HasRecentFollowedUserIds
with HasRecentFollowedByUserIds
with HasInvalidRelationshipUserIds
with HasExcludedUserIds
with HasUserState
with HasGeohashAndCountryCode
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderRequestBuilder.scala
class ContentRecommenderRequestBuilder @Inject() (
socialGraph: SocialGraphClient,
userLocationFetcher: UserLocationFetcher,
userStateClient: UserStateClient,
statsReceiver: StatsReceiver)
def build(req: ProductRequest): Stitch[ContentRecommenderRequest]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlCandidateSourceRegistry.scala
class PostNuxMlCandidateSourceRegistry @Inject() (
crowdSearchAccountsCandidateSource: CrowdSearchAccountsSource,
topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource,
linearRegressionfollow2vecNearestNeighborsStore: LinearRegressionFollow2vecNearestNeighborsStore,
forwardEmailBookSource: ForwardEmailBookSource,
forwardPhoneBookSource: ForwardPhoneBookSource,
offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource,
onlineSTPSource: OnlineSTPSourceScorer,
popCountrySource: PopCountrySource,
popCountryBackFillSource: PopCountryBackFillSource,
popGeohashSource: PopGeohashSource,
recentEngagementDirectFollowSimilarUsersSource: RecentEngagementSimilarUsersSource,
recentEngagementNonDirectFollowSource: RecentEngagementNonDirectFollowSource,
recentEngagementDirectFollowSalsaExpansionSource: RecentEngagementDirectFollowSalsaExpansionSource,
recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource,
realGraphOonV2Source: RealGraphOonV2Source,
repeatedProfileVisitSource: RepeatedProfileVisitsSource,
reverseEmailBookSource: ReverseEmailBookSource,
reversePhoneBookSource: ReversePhoneBookSource,
triangularLoopsSource: TriangularLoopsSource,
userUserGraphCandidateSource: UserUserGraphCandidateSource,
ppmiLocaleFollowSource: PPMILocaleFollowSource,
popGeohashQualityFollowSource: PopGeohashQualityFollowSource,
baseStatsReceiver: StatsReceiver,
) extends CandidateSourceRegistry[PostNuxMlRequest, CandidateUser]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlCandidateSourceWeightParams.scala
abstract class PostNuxMlCandidateSourceWeightParams[A](default: A) extends Param[A](default)
object PostNuxMlCandidateSourceWeightParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlCombinedRankerBuilder.scala
class PostNuxMlCombinedRankerBuilder[
T <: HasParams with HasSimilarToContext with HasClientContext with HasExcludedUserIds with HasDisplayLocation with HasDebugOptions with HasPreFetchedFeature with HasDismissedUserIds with HasQualityFactor] @Inject() (
firstNRanker: FirstNRanker[T],
hydrateFeaturesTransform: HydrateFeaturesTransform[T],
hydrateCandidateParamsTransform: HydrateCandidateParamsTransform[T],
mlRanker: MlRanker[T],
statsReceiver: StatsReceiver)
def build(
request: T,
candidateSourceWeights: Map[CandidateSourceIdentifier, Double]
): Ranker[T, CandidateUser]
def buildMainRanker(
request: T,
isMainRankerPostNuxProd: Boolean,
displayLocationStats: StatsReceiver
): Ranker[T, CandidateUser]
def buildInterleaveRanker(
request: T,
isMainRankerPostNuxProd: Boolean,
interleaveRankerStats: StatsReceiver
): Ranker[T, CandidateUser]
def buildFatigueRanker(
request: T,
fatigueRankerStats: StatsReceiver
): Ranker[T, CandidateUser]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlow.scala
class PostNuxMlFlow @Inject() (
postNuxMlCandidateSourceRegistry: PostNuxMlCandidateSourceRegistry,
postNuxMlCombinedRankerBuilder: PostNuxMlCombinedRankerBuilder[PostNuxMlRequest],
curatedCompetitorListPredicate: CuratedCompetitorListPredicate,
gizmoduckPredicate: GizmoduckPredicate,
sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate,
hssPredicate: HssPredicate,
invalidRelationshipPredicate: InvalidRelationshipPredicate,
recentFollowingPredicate: RecentFollowingPredicate,
nonNearZeroUserActivityPredicate: NonNearZeroUserActivityPredicate,
inactivePredicate: InactivePredicate,
dismissedCandidatePredicate: DismissedCandidatePredicate,
previouslyRecommendedUserIdsPredicate: PreviouslyRecommendedUserIdsPredicate,
modifySocialProofTransform: ModifySocialProofTransform,
removeAccountProofTransform: RemoveAccountProofTransform,
trackingTokenTransform: TrackingTokenTransform,
randomRankerIdTransform: RandomRankerIdTransform,
candidateParamsFactory: CandidateUserParamsFactory[PostNuxMlRequest],
samplingTransform: SamplingTransform,
frsLogger: FrsLogger,
baseStatsReceiver: StatsReceiver)
extends RecommendationFlow[PostNuxMlRequest, CandidateUser]
with RecommendationFlowBaseSideEffectsUtil[PostNuxMlRequest, CandidateUser]
with CandidateSourceHoldbackUtil
def updateTarget(request: PostNuxMlRequest): Stitch[PostNuxMlRequest]
def getCandidateSourceIdentifiers(
params: Params
): Set[CandidateSourceIdentifier]
def candidateSources(
request: PostNuxMlRequest
): Seq[CandidateSource[PostNuxMlRequest, CandidateUser]]
object excludeNearZeroUserPredicate
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)](
nonNearZeroUserActivityPredicate,
stats.scope("exclude_near_zero_predicate")
)
def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean =
item._1.params(PostNuxMlParams.ExcludeNearZeroCandidates)
}
object invalidRelationshipGatedPredicate
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)](
invalidRelationshipPredicate,
stats.scope("invalid_relationship_predicate")
)
def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean =
item._1.params(PostNuxMlParams.EnableInvalidRelationshipPredicate)
}
ExcludedUserIdPredicate
.observe(stats.scope("exclude_user_id_predicate"))
.andThen(
recentFollowingPredicate.observe(stats.scope("recent_following_predicate"))
)
.andThen(
dismissedCandidatePredicate.observe(stats.scope("dismissed_candidate_predicate"))
)
.andThen(
previouslyRecommendedUserIdsPredicate.observe(
stats.scope("previously_recommended_user_ids_predicate"))
)
.andThen(
invalidRelationshipGatedPredicate.observe(stats.scope("invalid_relationship_predicate"))
)
.andThen(
excludeNearZeroUserPredicate.observe(stats.scope("exclude_near_zero_user_state"))
)
.observe(stats.scope("overall_pre_ranker_candidate_filter"))
}
override protected def selectRanker(
request: PostNuxMlRequest
): Ranker[PostNuxMlRequest, CandidateUser]
object sgsGatedPredicate
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)](
sgsPredicate.observe(sgsPredicateStats),
sgsPredicateStats
)
def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean =
item._1.params(PostNuxMlParams.EnableSGSPredicate) ||
SocialGraphClient.enablePostRankerSgsPredicate(
item._1.invalidRelationshipUserIds.getOrElse(Set.empty).size)
}
val hssPredicateStats = stats.scope("hss_predicate")
object hssGatedPredicate
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)](
hssPredicate.observe(hssPredicateStats),
hssPredicateStats
)
def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean =
item._1.params(PostNuxMlParams.EnableHssPredicate)
}
Predicate
.andConcurrently[(PostNuxMlRequest, CandidateUser)](
Seq(
competitorPredicate.observe(stats.scope("curated_competitor_predicate")),
gizmoduckPredicate.observe(stats.scope("gizmoduck_predicate")),
sgsGatedPredicate,
hssGatedPredicate,
inactivePredicate.observe(stats.scope("inactive_predicate")),
)
)
// to avoid dilutions, we need to apply the receiver holdback predicates at the very last step
.andThen(pymkProducerHoldbackPredicate.observe(stats.scope("pymk_receiver_side_holdback")))
.andThen(producerHoldbackPredicate.observe(stats.scope("receiver_side_holdback")))
.observe(stats.scope("overall_validate_candidates"))
}
override protected val transformResults: Transform[PostNuxMlRequest, CandidateUser]
def resultsConfig(request: PostNuxMlRequest): RecommendationResultsConfig
def applySideEffects(
target: PostNuxMlRequest,
candidateSources: Seq[CandidateSource[PostNuxMlRequest, CandidateUser]],
candidatesFromCandidateSources: Seq[CandidateUser],
mergedCandidates: Seq[CandidateUser],
filteredCandidates: Seq[CandidateUser],
rankedCandidates: Seq[CandidateUser],
transformedCandidates: Seq[CandidateUser],
truncatedCandidates: Seq[CandidateUser],
results: Seq[CandidateUser]
): Stitch[Unit]
object PostNuxMlFlow
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlowCandidateSourceWeights.scala
object PostNuxMlFlowCandidateSourceWeights
def getWeights(params: Params): Map[CandidateSourceIdentifier, Double]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala
object PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlowFeatureSwitchKeys.scala
object PostNuxMlFlowFeatureSwitchKeys
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlowFSConfig.scala
class PostNuxMlFlowFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlParams.scala
abstract class PostNuxMlParams[A](default: A) extends Param[A](default)
object PostNuxMlParams
object FetchCandidateSourceBudget extends PostNuxMlParams[Duration](90.millisecond)
// WTF Impression Store has very high tail latency (p9990 or p9999), but p99 latency is pretty good (~100ms)
// set the time budget for this step to be 200ms to make the performance of service more predictable
case object FatigueRankerBudget extends PostNuxMlParams[Duration](200.millisecond)
case object MlRankerBudget
extends FSBoundedParam[Duration](
name = PostNuxMlFlowFeatureSwitchKeys.MLRankerBudget,
default = 400.millisecond,
min = 100.millisecond,
max = 800.millisecond)
with HasDurationConversion
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlRequest.scala
class PostNuxMlRequest(
override val params: Params,
override val clientContext: ClientContext,
override val similarToUserIds: Seq[Long],
inputExcludeUserIds: Seq[Long],
override val recentFollowedUserIds: Option[Seq[Long]],
override val invalidRelationshipUserIds: Option[Set[Long]],
override val recentFollowedByUserIds: Option[Seq[Long]],
override val dismissedUserIds: Option[Seq[Long]],
override val displayLocation: DisplayLocation,
maxResults: Option[Int] = None,
override val debugOptions: Option[DebugOptions] = None,
override val wtfImpressions: Option[Seq[WtfImpression]],
override val uttInterestIds: Option[Seq[Long]] = None,
override val customInterests: Option[Seq[String]] = None,
override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None,
inputPreviouslyRecommendedUserIds: Option[Set[Long]] = None,
inputPreviouslyFollowedUserIds: Option[Set[Long]] = None,
override val isSoftUser: Boolean = false,
override val userState: Option[UserState] = None,
override val qualityFactor: Option[Double] = None)
extends HasParams
with HasSimilarToContext
with HasClientContext
with HasExcludedUserIds
with HasDisplayLocation
with HasDebugOptions
with HasGeohashAndCountryCode
with HasPreFetchedFeature
with HasDismissedUserIds
with HasInterestIds
with HasPreviousRecommendationsContext
with HasIsSoftUser
with HasUserState
with HasInvalidRelationshipUserIds
with HasQualityFactor
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlRequestBuilder.scala
class PostNuxMlRequestBuilder @Inject() (
socialGraph: SocialGraphClient,
wtfImpressionStore: WtfImpressionStore,
dismissStore: DismissStore,
userLocationFetcher: UserLocationFetcher,
interestServiceClient: InterestServiceClient,
userStateClient: UserStateClient,
statsReceiver: StatsReceiver)
extends Logging
def build(
req: ProductRequest,
previouslyRecommendedUserIds: Option[Set[Long]] = None,
previouslyFollowedUserIds: Option[Set[Long]] = None
): Stitch[PostNuxMlRequest]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlRequestBuilderParams.scala
object PostNuxMlRequestBuilderParams
object TopicIdFetchBudget
extends FSBoundedParam[Duration](
name = "post_nux_ml_request_builder_topic_id_fetch_budget_millis",
default = 200.millisecond,
min = 80.millisecond,
max = 400.millisecond)
with HasDurationConversion
object DismissedIdScanBudget
extends FSBoundedParam[Duration](
name = "post_nux_ml_request_builder_dismissed_id_scan_budget_millis",
default = 200.millisecond,
min = 80.millisecond,
max = 400.millisecond)
with HasDurationConversion
object WTFImpressionsScanBudget
extends FSBoundedParam[Duration](
name = "post_nux_ml_request_builder_wtf_impressions_scan_budget_millis",
default = 200.millisecond,
min = 80.millisecond,
max = 400.millisecond)
with HasDurationConversion
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\logging\FrsLogger.scala
class we use to log data into:
* 1) logs.follow_recommendations_logs
*
* This logger logs data for 2 endpoints: getRecommendations, scoreUserCandidates
* All data scribed via this logger have to be converted into the same thrift type: RecommendationLog
*
* 2) logs.frs_recommendation_flow_logs
*
* This logger logs recommendation flow data for getRecommendations requests
* All data scribed via this logger have to be converted into the same thrift type: FrsRecommendationFlowLog
*/
@Singleton
class FrsLogger @Inject() (
@Named(GuiceNamedConstants.REQUEST_LOGGER) loggerFactory: LoggerFactory,
@Named(GuiceNamedConstants.FLOW_LOGGER) flowLoggerFactory: LoggerFactory,
stats: StatsReceiver,
@Flag("log_results") serviceShouldLogResults: Boolean)
extends ScribeSerialization
def logRecommendationResult(
request: RecommendationRequest,
response: RecommendationResponse
): Unit
def logScoringResult(request: ScoringUserRequest, response: ScoringUserResponse): Unit
def logRecommendationFlowData[Target <: HasClientContext with HasIsSoftUser with HasParams](
request: Target,
flowData: RecommendationFlowData[Target]
): Unit
def shouldLog(debugParamsOpt: Option[DebugParams]): Boolean =
debugParamsOpt match
object FrsLogger
def mkProvider(clientContext: ClientContext) = new ClientDataProvider
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\CandidateSourceType.scala
object CandidateSourceType extends Enumeration
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\DebugParams.scala
class DebugParams(
featureOverrides: Option[Map[String, ConfigApiFeatureValue]],
debugOptions: Option[DebugOptions])
object DebugParams
def fromThrift(thrift: t.DebugParams): DebugParams = DebugParams(
featureOverrides = thrift.featureOverrides.map
def toOfflineThrift(model: DebugParams): offline.OfflineDebugParams =
offline.OfflineDebugParams(randomizationSeed = model.debugOptions.flatMap(_.randomizationSeed))
}
trait HasFrsDebugParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\DisplayContext.scala
trait DisplayContext
def toOfflineThrift: offline.OfflineDisplayContext
}
object DisplayContext
class Profile(profileId: Long) extends DisplayContext
class Search(searchQuery: String) extends DisplayContext
class Rux(focalAuthorId: Long) extends DisplayContext
class Topic(topicId: Long) extends DisplayContext
class ReactiveFollow(followedUserIds: Seq[Long]) extends DisplayContext
class NuxInterests(flowContext: Option[FlowContext], uttInterestIds: Option[Seq[Long]])
extends DisplayContext
class PostNuxFollowTask(flowContext: Option[FlowContext]) extends DisplayContext
class AdCampaignTarget(similarToUserIds: Seq[Long]) extends DisplayContext
class ConnectTab(
byfSeedUserIds: Seq[Long],
similarToUserIds: Seq[Long],
engagedUserIds: Seq[RecentlyEngagedUserId])
extends DisplayContext
class SimilarToUser(similarToUserId: Long) extends DisplayContext
def fromThrift(tDisplayContext: t.DisplayContext): DisplayContext = tDisplayContext match
def getDisplayContextAs[T <: DisplayContext: ClassTag](displayContext: DisplayContext): T =
displayContext match
class UnknownDisplayContextException(name: String)
extends Exception(s"Unknown DisplayContext in Thrift: $
class UnexpectedDisplayContextTypeException(displayContext: DisplayContext, expectedType: String)
extends Exception(s"DisplayContext $
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\FeatureValue.scala
object FeatureValue
def fromThrift(thriftFeatureValue: t.FeatureValue): FeatureValue = thriftFeatureValue match
class UnknownFeatureValueException(fieldName: String)
extends Exception(s"Unknown FeatureValue name in thrift: $
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\RecommendationFlowData.scala
class RecommendationFlowData[Target <: HasClientContext](
request: Target,
recommendationFlowIdentifier: RecommendationPipelineIdentifier,
candidateSources: Seq[CandidateSource[Target, CandidateUser]],
candidatesFromCandidateSources: Seq[CandidateUser],
mergedCandidates: Seq[CandidateUser],
filteredCandidates: Seq[CandidateUser],
rankedCandidates: Seq[CandidateUser],
transformedCandidates: Seq[CandidateUser],
truncatedCandidates: Seq[CandidateUser],
results: Seq[CandidateUser])
extends HasMarshalling
object RecommendationFlowData
def userToOfflineRecommendationFlowUserMetadata[Target <: HasClientContext](
request: Target
): Option[offline.OfflineRecommendationFlowUserMetadata]
def userToOfflineRecommendationFlowSignals[Target <: HasClientContext](
request: Target
): Option[offline.OfflineRecommendationFlowSignals]
def candidatesToOfflineRecommendationFlowCandidateSourceCandidates[Target <: HasClientContext](
candidateSources: Seq[CandidateSource[Target, CandidateUser]],
candidates: Seq[CandidateUser],
): Seq[offline.OfflineRecommendationFlowCandidateSourceCandidates]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\RecommendationRequest.scala
class RecommendationRequest(
clientContext: ClientContext,
displayLocation: DisplayLocation,
displayContext: Option[DisplayContext],
maxResults: Option[Int],
cursor: Option[String],
excludedIds: Option[Seq[Long]],
fetchPromotedContent: Option[Boolean],
debugParams: Option[DebugParams] = None,
userLocationState: Option[String] = None,
isSoftUser: Boolean = false)
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\RecommendationResponse.scala
class RecommendationResponse(recommendations: Seq[Recommendation]) extends HasMarshalling
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\Request.scala
class Request(
override val maxResults: Option[Int],
override val debugParams: Option[request.DebugParams],
override val productContext: Option[ProductContext],
override val product: request.Product,
override val clientContext: ClientContext,
override val serializedRequestCursor: Option[String],
override val frsDebugParams: Option[DebugParams],
displayLocation: DisplayLocation,
excludedIds: Option[Seq[Long]],
fetchPromotedContent: Option[Boolean],
userLocationState: Option[String] = None)
extends ProductMixerRequest
with HasFrsDebugParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\ScoringUserRequest.scala
class ScoringUserRequest(
override val clientContext: ClientContext,
override val displayLocation: DisplayLocation,
override val params: Params,
override val debugOptions: Option[DebugOptions] = None,
override val recentFollowedUserIds: Option[Seq[Long]],
override val recentFollowedByUserIds: Option[Seq[Long]],
override val wtfImpressions: Option[Seq[WtfImpression]],
override val similarToUserIds: Seq[Long],
candidates: Seq[CandidateUser],
debugParams: Option[DebugParams] = None,
isSoftUser: Boolean = false)
extends HasClientContext
with HasDisplayLocation
with HasParams
with HasDebugOptions
with HasPreFetchedFeature
with HasSimilarToContext
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\ScoringUserResponse.scala
class ScoringUserResponse(candidates: Seq[CandidateUser])
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\failures\TimeoutPipelineFailure.scala
object TimeoutPipelineFailure
def apply(candidateSourceName: String): PipelineFailure
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ABDeciderModule.scala
object ABDeciderModule extends TwitterModule
def provideABDecider(
stats: StatsReceiver,
@Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) factory: LoggerFactory
): LoggingABDecider
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ConfigApiModule.scala
object ConfigApiModule extends TwitterModule
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\DiffyModule.scala
object DiffyModule extends TwitterModule
def provideDarkTrafficService(
serviceIdentifier: ServiceIdentifier
): FollowRecommendationsThriftService.ReqRepServicePerEndpoint
def provideDarkTrafficFilter(
@DarkTrafficService darkService: FollowRecommendationsThriftService.ReqRepServicePerEndpoint,
deciderGateBuilder: DeciderGateBuilder,
statsReceiver: StatsReceiver,
@Flag("environment") env: String
): DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\FeatureSwitchesModule.scala
object FeaturesSwitchesModule extends TwitterModule
def providesFeatureSwitches(
abDecider: LoggingABDecider,
statsReceiver: StatsReceiver
): FeatureSwitches
def providesProducerFeatureSwitches(
abDecider: LoggingABDecider,
statsReceiver: StatsReceiver
): FeatureSwitches
object ProducerFeatureFilter extends FeatureFilter
def filter(feature: Feature): Option[Feature]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\FlagsModule.scala
object FlagsModule extends TwitterModule
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ProductRegistryModule.scala
object ProductRegistryModule extends TwitterModule
def configure(): Unit
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ScorerModule.scala
object ScorerModule extends TwitterModule
def fileFromResource(resource: String): File
def provideEpScorer: EPScorer
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ScribeModule.scala
object ScribeModule extends TwitterModule
def provideClientEventsLoggerFactory(stats: StatsReceiver): LoggerFactory
def provideFollowRecommendationsLoggerFactory(stats: StatsReceiver): LoggerFactory
def provideFrsRecommendationFlowLoggerFactory(stats: StatsReceiver): LoggerFactory
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\TimerModule.scala
object TimerModule extends TwitterModule
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\ProdProductRegistry.scala
class ProdProductRegistry @Inject() (
exploreTabProduct: ExploreTabProduct,
homeTimelineProduct: HomeTimelineProduct,
homeTimelineTweetRecsProduct: HomeTimelineTweetRecsProduct,
sidebarProduct: SidebarProduct,
) extends ProductRegistry
def getProductByDisplayLocation(displayLocation: DisplayLocation): common.Product
class MissingProductException(displayLocation: DisplayLocation)
extends Exception(s"No Product found for $
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\common\Exceptions.scala
abstract class ProductException(message: String) extends Exception(message)
class MissingFieldException(productRequest: ProductRequest, fieldName: String)
extends ProductException(
s"Missing $
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\common\Product.scala
trait Product
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\common\ProductRegistry.scala
trait ProductRegistry
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\explore_tab\ExploreTabProduct.scala
class ExploreTabProduct @Inject() (
postNuxMlFlow: PostNuxMlFlow,
postNuxMlRequestBuilder: PostNuxMlRequestBuilder)
extends Product
def selectWorkflows(
request: ProductRequest
): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]]
def resultsTransformer(
request: ProductRequest
): Stitch[Transform[ProductRequest, Recommendation]] =
Stitch.value(new IdentityTransform[ProductRequest, Recommendation])
override def enabled(request: ProductRequest): Stitch[Boolean]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\explore_tab\configapi\ExploreTabFSConfig.scala
class ExploreTabFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\explore_tab\configapi\ExploreTabParams.scala
object ExploreTabParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\HomeTimelineProduct.scala
class HomeTimelineProduct @Inject() (
postNuxMlFlow: PostNuxMlFlow,
postNuxMlRequestBuilder: PostNuxMlRequestBuilder,
promotedAccountsFlow: PromotedAccountsFlow,
promotedAccountsBlender: PromotedAccountsBlender,
productScope: ProductScope,
injector: Injector,
) extends Product
def selectWorkflows(
request: ProductRequest
): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]]
def resultsTransformer(
request: ProductRequest
): Stitch[Transform[ProductRequest, Recommendation]] = Stitch.value(identityTransform)
override def enabled(request: ProductRequest): Stitch[Boolean] =
Stitch.value(request.params(EnableProduct))
override def layout: Option[Layout]
def productMixerProduct: Option[request.Product] = Some(HTLProductMixer)
private[home_timeline] def mkPromotedAccountsRequest(
req: ProductRequest
): PromotedAccountsFlowRequest
def getMaxResults(req: ProductRequest): Int
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\HomeTimelineStrings.scala
class HomeTimelineStrings @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry])
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\HTLProductMixer.scala
object HTLProductMixer extends Product
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\configapi\HomeTimelineFSConfig.scala
class HomeTimelineFSConfig @Inject() () extends FeatureSwitchConfig
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\configapi\HomeTimelineParams.scala
object HomeTimelineParams
object EnableProduct extends Param[Boolean](false)
object DefaultMaxResults extends Param[Int](20)
object EnableWritingServingHistory
extends FSParam[Boolean]("home_timeline_enable_writing_serving_history", false)
object DurationGuardrailToForceSuggest
extends FSBoundedParam[Duration](
name = "home_timeline_duration_guardrail_to_force_suggest_in_hours",
default = 0.hours,
min = 0.hours,
max = 1000.hours)
with HasDurationConversion
object SuggestBasedFatigueDuration
extends FSBoundedParam[Duration](
name = "home_timeline_suggest_based_fatigue_duration_in_hours",
default = 0.hours,
min = 0.hours,
max = 1000.hours)
with HasDurationConversion
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline_tweet_recs\HomeTimelineTweetRecsProduct.scala
class HomeTimelineTweetRecsProduct @Inject() (
contentRecommenderFlow: ContentRecommenderFlow,
contentRecommenderRequestBuilder: ContentRecommenderRequestBuilder)
extends Product
def selectWorkflows(
request: ProductRequest
): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline_tweet_recs\configapi\HomeTimelineTweetRecsParams.scala
object HomeTimelineTweetRecsParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\sidebar\SidebarProduct.scala
class SidebarProduct @Inject() (
postNuxMlFlow: PostNuxMlFlow,
postNuxMlRequestBuilder: PostNuxMlRequestBuilder,
promotedAccountsFlow: PromotedAccountsFlow,
promotedAccountsBlender: PromotedAccountsBlender)
extends Product
def selectWorkflows(
request: ProductRequest
): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]]
def mkPromotedAccountsRequest(
req: ProductRequest
): PromotedAccountsFlowRequest
def getMaxResults(req: ProductRequest): Int
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\sidebar\configapi\SidebarParams.scala
object SidebarParams
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\FollowRecommendationsServiceWarmupHandler.scala
class FollowRecommendationsServiceWarmupHandler @Inject() (warmup: ThriftWarmup)
extends Handler
with Logging
def handle(): Unit
def warmupQuery(userId: Long, displayLocation: DisplayLocation): RecommendationRequest
def assertWarmupResponse(result: Try[Response[GetRecommendations.SuccessType]]): Unit
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\ProductMixerRecommendationService.scala
class ProductMixerRecommendationService @Inject() (
productPipelineRegistry: ProductPipelineRegistry,
resultLogger: FrsLogger,
baseStats: StatsReceiver)
def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse]
def convertToProductMixerRequest(frsRequest: RecommendationRequest): Request
def convertToProductMixerDebugParams(
frsDebugParams: Option[FrsDebugParams]
): Option[ProductMixerDebugParams]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\ProductPipelineSelector.scala
class ProductPipelineSelector @Inject() (
recommendationsService: RecommendationsService,
productMixerRecommendationService: ProductMixerRecommendationService,
productPipelineSelectorConfig: ProductPipelineSelectorConfig,
baseStats: StatsReceiver)
def selectPipeline(
request: RecommendationRequest,
params: Params
): Stitch[RecommendationResponse]
def readFromProductMixerPipeline(
request: RecommendationRequest,
params: Params
): Stitch[RecommendationResponse]
def readFromOldFrsPipeline(
request: RecommendationRequest,
params: Params
): Stitch[RecommendationResponse]
def darkReadAndReturnResult(
request: RecommendationRequest,
params: Params
): Stitch[RecommendationResponse]
def compare(
request: RecommendationRequest,
frsOldPipelineResponse: RecommendationResponse,
frsProductMixerResponse: RecommendationResponse
): Unit
def compareTopNResults(
n: Int,
frsOldPipelineResponse: RecommendationResponse,
frsProductMixerResponse: RecommendationResponse,
compareStats: StatsReceiver
): Unit
def compareUser(
oldFrsUser: CandidateUser,
productMixerUser: CandidateUser,
stats: StatsReceiver
): Unit
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\ProductPipelineSelectorConfig.scala
class ProductPipelineSelectorConfig
def getDarkReadAndExpParams(
displayLocation: DisplayLocation
): Option[DarkReadAndExpParams]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\ProductRecommenderService.scala
class ProductRecommenderService @Inject() (
productRegistry: ProductRegistry,
statsReceiver: StatsReceiver)
def getRecommendations(
request: RecommendationRequest,
params: Params
): Stitch[Seq[Recommendation]]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\RecommendationsService.scala
class RecommendationsService @Inject() (
productRecommenderService: ProductRecommenderService,
resultLogger: FrsLogger)
def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\UserScoringService.scala
class UserScoringService @Inject() (
socialGraph: SocialGraphClient,
wtfImpressionStore: WtfImpressionStore,
hydrateFeaturesTransform: HydrateFeaturesTransform[ScoringUserRequest],
mlRanker: MlRanker[ScoringUserRequest],
resultLogger: FrsLogger,
stats: StatsReceiver)
def get(request: ScoringUserRequest): Stitch[ScoringUserResponse]
def hydrate(request: ScoringUserRequest): Stitch[ScoringUserRequest]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\exceptions\UnknownExceptionMapper.scala
class UnknownLoggingExceptionMapper extends ExceptionMapper[Exception, Throwable] with Logging
def handleException(throwable: Exception): Future[Throwable]
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\utils\CandidateSourceHoldbackUtil.scala
trait CandidateSourceHoldbackUtil
def filterCandidateSources[T <: HasParams](
request: T,
sources: Seq[CandidateSource[T, CandidateUser]]
): Seq[CandidateSource[T, CandidateUser]]
object CandidateSourceHoldbackUtil
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\utils\RecommendationFlowBaseSideEffectsUtil.scala
trait RecommendationFlowBaseSideEffectsUtil[Target <: HasClientContext, Candidate <: CandidateUser]
extends SideEffectsUtil[Target, Candidate]
def applySideEffects(
target: Target,
candidateSources: Seq[CandidateSource[Target, Candidate]],
candidatesFromCandidateSources: Seq[Candidate],
mergedCandidates: Seq[Candidate],
filteredCandidates: Seq[Candidate],
rankedCandidates: Seq[Candidate],
transformedCandidates: Seq[Candidate],
truncatedCandidates: Seq[Candidate],
results: Seq[Candidate]
): Stitch[Unit]
def applySideEffectsCandidateSourceCandidates(
target: Target,
candidateSources: Seq[CandidateSource[Target, Candidate]],
candidatesFromCandidateSources: Seq[Candidate]
): Stitch[Unit]
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\common\Configs.scala
object Configs
def getTimedHdfsShardPath(shardId: Int, path: String, time: Time): String
def getHdfsPath(path: String, overrideBaseHdfsPath: Option[String] = None): String
def hash(kArr: Array[Byte], seed: Int): Int
def hashLong(l: Long, seed: Int): Int
def shardForUser(userId: Long): Int
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\Main.scala
object Main extends ServerMain
class ServerMain extends ThriftServer with Mtls
def configureThrift(router: ThriftRouter): Unit
def warmup(): Unit
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\controllers\ServerController.scala
class ServerController @Inject() (
serverGetIntersectionHandler: ServerGetIntersectionHandler
)(
implicit statsReceiver: StatsReceiver)
extends Controller(thriftscala.Server)
def getPresetIntersection: Service[
GetPresetIntersection.Args,
GfsIntersectionResponse
]
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\handlers\ServerGetIntersectionHandler.scala
class ServerGetIntersectionHandler @Inject() (
@Named("ReadThroughGetIntersectionStore")
readThroughStore: ReadableStore[GetIntersectionQuery, CachedIntersectionResult],
@Named("BypassCacheGetIntersectionStore")
readOnlyStore: ReadableStore[GetIntersectionQuery, CachedIntersectionResult]
)(
implicit statsReceiver: StatsReceiver)
extends RequestHandler[GetIntersectionRequest, GfsIntersectionResponse]
def apply(request: GetIntersectionRequest): Future[GfsIntersectionResponse]
object ServerGetIntersectionHandler
class GetIntersectionRequest(
userId: Long,
candidateUserIds: Seq[Long],
featureTypes: Seq[FeatureType],
presetFeatureTypes: PresetFeatureTypes,
intersectionIdLimit: Option[Int],
cacheable: Boolean)
object GetIntersectionRequest
def fromGfsIntersectionRequest(
request: GfsIntersectionRequest,
cacheable: Boolean
): GetIntersectionRequest
def fromGfsPresetIntersectionRequest(
request: GfsPresetIntersectionRequest,
cacheable: Boolean
): GetIntersectionRequest
def isZero(opt: Option[Int]): Boolean
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\handlers\ServerWarmupHandler.scala
class ServerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler
def getRandomRequest: GfsIntersectionRequest
def handle(): Unit
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\modules\GetIntersectionStoreModule.scala
object GetIntersectionStoreModule extends TwitterModule
def provideReadThroughGetIntersectionStore(
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients,
serviceIdentifier: ServiceIdentifier,
@Flag(ServerFlagNames.MemCacheClientName) memCacheName: String,
@Flag(ServerFlagNames.MemCachePath) memCachePath: String
)(
implicit statsReceiver: StatsReceiver
): ReadableStore[GetIntersectionQuery, CachedIntersectionResult]
def provideReadOnlyGetIntersectionStore(
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients,
)(
implicit statsReceiver: StatsReceiver
): ReadableStore[GetIntersectionQuery, CachedIntersectionResult]
def buildMemcacheStore(
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients,
memCacheName: String,
memCachePath: String,
serviceIdentifier: ServiceIdentifier,
)(
implicit statsReceiver: StatsReceiver
): ReadableStore[GetIntersectionQuery, CachedIntersectionResult]
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\modules\GraphFeatureServiceWorkerClientsModule.scala
class GraphFeatureServiceWorkerClients(
workers: Seq[thriftscala.Worker.MethodPerEndpoint])
object GraphFeatureServiceWorkerClientsModule extends TwitterModule
def provideGraphFeatureServiceWorkerClient(
@Flag(ServerFlagNames.NumWorkers) numWorkers: Int,
@Flag(ServerFlagNames.ServiceRole) serviceRole: String,
@Flag(ServerFlagNames.ServiceEnv) serviceEnv: String,
serviceIdentifier: ServiceIdentifier
): GraphFeatureServiceWorkerClients
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\modules\LZ4Injection.scala
object LZ4Injection extends Injection[Array[Byte], Array[Byte]]
def apply(a: Array[Byte]): Array[Byte] = LZ4Injection.fastCompressor.compress(a)
override def invert(b: Array[Byte]): Try[Array[Byte]] = Try
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\modules\ServerFlagModule.scala
object ServerFlagNames
object ServerFlagsModule extends TwitterModule
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\stores\FeatureTypesEncoder.scala
object FeatureTypesEncoder
def apply(featureTypes: Seq[FeatureType]): String
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\stores\GetIntersectionStore.scala
class GetIntersectionStore(
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients,
statsReceiver: StatsReceiver)
extends ReadableStore[GetIntersectionQuery, CachedIntersectionResult]
with Logging
def multiGet[K1 <: GetIntersectionQuery](
ks: Set[K1]
): Map[K1, Future[Option[CachedIntersectionResult]]]
def gfsIntersectionResponseAggregator(
responseList: Seq[WorkerIntersectionResponse],
features: Seq[FeatureType],
candidates: Seq[Long],
intersectionIdLimit: Int
): Map[Long, CachedIntersectionResult]
object GetIntersectionStore
class GetIntersectionQuery(
userId: Long,
candidateId: Long,
featureTypes: Seq[FeatureType],
presetFeatureTypes: PresetFeatureTypes,
featureTypesString: String,
calculatedFeatureTypes: Seq[FeatureType],
intersectionIdLimit: Int)
private[graph_feature_service] object GetIntersectionQuery
def buildQueries(request: GetIntersectionRequest): Set[GetIntersectionQuery]
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\util\FeatureTypesCalculator.scala
object FeatureTypesCalculator
def getFeatureTypes(
presetFeatureTypes: PresetFeatureTypes,
featureTypes: Seq[FeatureType]
): Seq[FeatureType]
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\util\IntersectionValueCalculator.scala
object IntersectionValueCalculator
def computeArraySize(x: ByteBuffer): Int
def apply(x: ByteBuffer, y: ByteBuffer, intersectionIdLimit: Int): WorkerIntersectionValue
def computeIntersectionUsingBinarySearchOnLargerByteBuffer(
smallArray: ByteBuffer,
largeArray: ByteBuffer
): Int
def computeIntersectionWithIds(
smallArray: ByteBuffer,
largeArray: ByteBuffer,
intersectionLimit: Int
): (Int, Seq[Long])
def binarySearch(arr: ByteBuffer, value: Long): Int
def apply(x: Array[Long], y: Array[Long], featureType: FeatureType): IntersectionValue
def computeIntersectionUsingListMerging[T](
x: Array[T],
y: Array[T]
)(
implicit ordering: Ordering[T]
): Int
def computeIntersectionUsingBinarySearchOnLargerArray[T](
smallArray: Array[T],
largeArray: Array[T]
)(
implicit ordering: Ordering[T]
): Int
def binarySearch[T](
arr: Array[T],
value: T
)(
implicit ordering: Ordering[T]
): Int
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\Main.scala
object Main extends WorkerMain
class WorkerMain extends ThriftServer with Mtls
def configureThrift(router: ThriftRouter): Unit
def warmup(): Unit
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\controllers\WorkerController.scala
class WorkerController @Inject() (
workerGetIntersectionHandler: WorkerGetIntersectionHandler
)(
implicit statsReceiver: StatsReceiver)
extends Controller(thriftscala.Worker)
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\handlers\WorkerGetIntersectionHandler.scala
class WorkerGetIntersectionHandler @Inject() (
graphContainer: GraphContainer,
statsReceiver: StatsReceiver)
extends RequestHandler[WorkerIntersectionRequest, WorkerIntersectionResponse]
def apply(request: WorkerIntersectionRequest): Future[WorkerIntersectionResponse]
object WorkerGetIntersectionHandler
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\handlers\WorkerWarmupHandler.scala
class WorkerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging
def handle(): Unit
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\modules\GraphContainerProviderModule.scala
object GraphContainerProviderModule extends TwitterModule
def provideAutoUpdatingGraphs(
@Flag(WorkerFlagNames.HdfsCluster) hdfsCluster: String,
@Flag(WorkerFlagNames.HdfsClusterUrl) hdfsClusterUrl: String,
@Flag(WorkerFlagNames.ShardId) shardId: Int
)(
implicit statsReceiver: StatsReceiver,
timer: Timer
): GraphContainer
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\modules\WorkerFlagModule.scala
object WorkerFlagNames
object WorkerFlagModule extends TwitterModule
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\util\AutoUpdatingGraph.scala
class AutoUpdatingGraph(
dataPath: String,
hdfsCluster: String,
hdfsClusterUrl: String,
shard: Int,
minimumSizeForCompleteGraph: Long,
updateIntervalMin: Duration = 1.hour,
updateIntervalMax: Duration = 12.hours,
deleteInterval: Duration = 2.seconds,
sharedSemaphore: Option[AsyncSemaphore] = None
)(
implicit statsReceiver: StatsReceiver,
timer: Timer)
extends AutoUpdatingReadOnlyGraph[Long, ByteBuffer](
hdfsCluster,
hdfsClusterUrl,
shard,
minimumSizeForCompleteGraph,
updateIntervalMin,
updateIntervalMax,
deleteInterval,
sharedSemaphore
)
with ConstDBImporter[Long, ByteBuffer]
def numGraphShards: Int = Configs.NumGraphShards
override def basePath: String = dataPath
override val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint
override val valueInj: Injection[ByteBuffer, ByteBuffer] = Injection.identity
override def get(targetId: Long): Future[Option[ByteBuffer]] =
super
.get(targetId)
.map
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\util\GfsQuery.scala
trait GfsQuery
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\util\GraphContainer.scala
class GraphContainer(
graphs: Map[GraphKey, AutoUpdatingGraph])
def warmup: Future[Unit]
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\util\GraphKey.scala
trait GraphKey
def edgeType: EdgeType
}
sealed trait PartialValueGraph extends GraphKey
/**
* Follow Graphs
*/
object FollowingPartialValueGraph extends PartialValueGraph
def edgeType: EdgeType = Following
}
object FollowedByPartialValueGraph extends PartialValueGraph
def edgeType: EdgeType = FollowedBy
}
/**
* Mutual Follow Graphs
*/
object MutualFollowPartialValueGraph extends PartialValueGraph
.\graph-feature-service\src\main\scalding\com\twitter\graph_feature_service\scalding\GraphFeatureServiceAppBase.scala
trait GraphFeatureServiceBaseJob
def runOnDateRange(
enableValueGraphs: Option[Boolean] = None,
enableKeyGraphs: Option[Boolean] = None
)(
implicit dateRange: DateRange,
timeZone: TimeZone,
uniqueID: UniqueID
): Execution[Unit]
/**
* Print customized counters in the log
*/
def printerCounters[T](execution: Execution[T]): Execution[Unit]
trait GraphFeatureServiceAdhocBaseApp extends TwitterExecutionApp with GraphFeatureServiceBaseJob
def job: Execution[Unit] = Execution.withId
trait GraphFeatureServiceScheduledBaseApp
extends TwitterScheduledExecutionApp
with GraphFeatureServiceBaseJob
def firstTime: RichDate // for example: RichDate("2018-02-21")
def batchIncrement: Duration = Days(1)
override def scheduledJob: Execution[Unit] = Execution.withId
.\graph-feature-service\src\main\scalding\com\twitter\graph_feature_service\scalding\GraphFeatureServiceApps.scala
object GraphFeatureServiceAdhocApp
extends GraphFeatureServiceMainJob
with GraphFeatureServiceAdhocBaseApp
object GraphFeatureServiceScheduledApp
extends GraphFeatureServiceMainJob
with GraphFeatureServiceScheduledBaseApp
def firstTime: RichDate = RichDate("2018-05-18")
override def runOnDateRange(
enableValueGraphs: Option[Boolean],
enableKeyGraphs: Option[Boolean]
)(
implicit dateRange: DateRange,
timeZone: TimeZone,
uniqueID: UniqueID
): Execution[Unit]
.\graph-feature-service\src\main\scalding\com\twitter\graph_feature_service\scalding\GraphFeatureServiceMainJob.scala
trait GraphFeatureServiceMainJob extends GraphFeatureServiceBaseJob
def getShardIdForUser(userId: Long): Int = shardForUser(userId)
protected implicit val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint
protected implicit val valueInj: Injection[Long, ByteBuffer] = Injections.long2ByteBuffer
protected val bufferSize: Int = 1 << 26
protected val maxNumKeys: Int = 1 << 24
protected val numReducers: Int = NumGraphShards
protected val outputStreamBufferSize: Int = 1 << 26
protected final val shardingByKey
def writeGraphToDB(
graph: TypedPipe[(Long, Long)],
shardingFunction: (Long, Long) => Int,
path: String
)(
implicit dateRange: DateRange
): Execution[TypedPipe[(Int, Unit)]]
def extractFeature(
featureList: Seq[TEdgeFeature],
featureName: FeatureName
): Option[Float]
def getSubGraph(
input: TypedPipe[(Long, Long, EdgeFeature)],
edgeFilter: EdgeFeature => Boolean,
counter: Stat
): TypedPipe[(Long, Long)]
def getMauIds()(implicit dateRange: DateRange, uniqueID: UniqueID): TypedPipe[Long]
def getRealGraphWithMAUOnly(
implicit dateRange: DateRange,
timeZone: TimeZone,
uniqueID: UniqueID
): TypedPipe[(Long, Long, EdgeFeature)]
def getTopKFollowGraph(
implicit dateRange: DateRange,
timeZone: TimeZone,
uniqueID: UniqueID
): TypedPipe[(Long, Long)]
def runOnDateRange(
enableValueGraphs: Option[Boolean],
enableKeyGraphs: Option[Boolean]
)(
implicit dateRange: DateRange,
timeZone: TimeZone,
uniqueID: UniqueID
): Execution[Unit]
.\graph-feature-service\src\main\scalding\com\twitter\graph_feature_service\scalding\adhoc\RandomRequestGenerationApp.scala
object RandomRequestGenerationJob
def run(
dataSetPath: String,
outPutPath: String,
numOfPairsToTake: Int
)(
implicit dateRange: DateRange,
uniqueID: UniqueID
): Execution[Unit]
object RandomRequestGenerationApp extends TwitterExecutionApp
def job: Execution[Unit] = Execution.withId
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\HomeMixerHttpServerWarmupHandler.scala
class HomeMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging
def handle(): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\HomeMixerServer.scala
object HomeMixerServerMain extends HomeMixerServer
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls
def configureThrift(router: ThriftRouter): Unit
def configureHttp(router: HttpRouter): Unit =
router.add(
ProductMixerController[st.HomeMixer.MethodPerEndpoint](
this.injector,
st.HomeMixer.ExecutePipeline))
override protected def warmup(): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\HomeMixerThriftServerWarmupHandler.scala
class HomeMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup)
extends Handler
with Logging
def handle(): Unit
def warmupQuery(userId: Long): st.HomeMixerRequest
def assertWarmupResponse(
result: Try[Response[st.HomeMixer.GetUrtResponse.SuccessType]]
): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\ConversationServiceCandidatePipelineConfig.scala
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator,
override val gates: Seq[BaseGate[Query]],
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
extends DependentCandidatePipelineConfig[
Query,
ConversationServiceCandidateSourceRequest,
TweetWithConversationMetadata,
TweetCandidate
]
def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
RetweetDeduplicationFilter,
FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature),
PredicateFeatureFilter.fromPredicate(
FilterIdentifier(QuotedTweetDroppedFilterId),
shouldKeepCandidate
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\ConversationServiceCandidatePipelineConfigBuilder.scala
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator)
def build(
gates: Seq[BaseGate[Query]] = Seq.empty,
decorator: Option[CandidateDecorator[Query, TweetCandidate]] = None
): ConversationServiceCandidatePipelineConfig[Query]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\ConversationServiceResponseFeatureTransformer.scala
object ConversationServiceResponseFeatureTransformer
extends CandidateFeatureTransformer[TweetWithConversationMetadata]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\EditedTweetsCandidatePipelineConfig.scala
class EditedTweetsCandidatePipelineConfig @Inject() (
staleTweetsCacheCandidateSource: StaleTweetsCacheCandidateSource,
namesFeatureHydrator: NamesFeatureHydrator,
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder)
extends DependentCandidatePipelineConfig[
PipelineQuery,
Seq[Long],
Long,
TweetCandidate
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\NewTweetsPillCandidatePipelineConfig.scala
class NewTweetsPillCandidatePipelineConfig[Query <: PipelineQuery with HasDeviceContext] @Inject() (
) extends DependentCandidatePipelineConfig[
Query,
Unit,
ShowAlertCandidate,
ShowAlertCandidate
]
def apply(
query: Query,
candidate: ShowAlertCandidate,
features: FeatureMap
): Option[Duration]
object NewTweetsPillCandidatePipelineConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\TimelineServiceResponseFeatureTransformer.scala
object TimelineServiceResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\controller\HomeThriftController.scala
class HomeThriftController @Inject() (
homeRequestUnmarshaller: HomeMixerRequestUnmarshaller,
urtService: UrtService,
scoredTweetsService: ScoredTweetsService,
paramsBuilder: ParamsBuilder)
extends Controller(t.HomeMixer)
with DebugTwitterContext
def buildParams(request: HomeMixerRequest): Params
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\candidate_source\EarlybirdCandidateSource.scala
object EarlybirdResponseTruncatedFeature
extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Boolean]
object EarlybirdBottomTweetFeature
extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Option[Long]]
class EarlybirdCandidateSource @Inject() (
earlybird: t.EarlybirdService.MethodPerEndpoint)
extends CandidateSourceWithExtractedFeatures[t.EarlybirdRequest, t.ThriftSearchResult]
def apply(
request: t.EarlybirdRequest
): Stitch[CandidatesWithSourceFeatures[t.ThriftSearchResult]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\candidate_source\SimilarityBasedUsersCandidateSource.scala
class SimilarityBasedUsersCandidateSource @Inject() (
similarUsersBySimsOnUserClientColumn: SimilarUsersBySimsOnUserClientColumn)
extends CandidateSource[Seq[Long], t.Candidate]
def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\candidate_source\StaleTweetsCacheCandidateSource.scala
class StaleTweetsCacheCandidateSource @Inject() (
@Named(StaleTweetsCache) staleTweetsCache: MemcachedClient)
extends CandidateSource[Seq[Long], Long]
def apply(request: Seq[Long]): Stitch[Seq[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\AuthorChildFeedbackActionBuilder.scala
class AuthorChildFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings)
def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\BlockUserChildFeedbackActionBuilder.scala
class BlockUserChildFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings)
def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\DontLikeFeedbackActionBuilder.scala
class DontLikeFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings,
authorChildFeedbackActionBuilder: AuthorChildFeedbackActionBuilder,
retweeterChildFeedbackActionBuilder: RetweeterChildFeedbackActionBuilder,
notRelevantChildFeedbackActionBuilder: NotRelevantChildFeedbackActionBuilder,
unfollowUserChildFeedbackActionBuilder: UnfollowUserChildFeedbackActionBuilder,
muteUserChildFeedbackActionBuilder: MuteUserChildFeedbackActionBuilder,
blockUserChildFeedbackActionBuilder: BlockUserChildFeedbackActionBuilder,
reportTweetChildFeedbackActionBuilder: ReportTweetChildFeedbackActionBuilder)
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\EngagerSocialContextBuilder.scala
class SocialContextIdAndScreenName(
socialContextId: Long,
screenName: String)
object EngagerSocialContextBuilder
class EngagerSocialContextBuilder(
contextType: GeneralContextType,
stringCenter: StringCenter,
oneUserString: ExternalString,
twoUsersString: ExternalString,
moreUsersString: ExternalString,
timelineTitle: ExternalString)
def apply(
socialContextIds: Seq[Long],
query: PipelineQuery,
candidateFeatures: FeatureMap
): Option[SocialContext]
def mkOneUserSocialContext(socialContextString: String, userId: Long): GeneralContext
def mkManyUserSocialContext(
socialContextString: String,
viewerId: Long,
socialContextIds: Seq[Long]
): GeneralContext
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\ExtendedReplySocialContextBuilder.scala
class ExtendedReplySocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\FeedbackUtil.scala
object FeedbackUtil
def buildUserSeeFewerChildFeedbackAction(
userId: Long,
namesByUserId: Map[Long, String],
promptExternalString: ExternalString,
confirmationExternalString: ExternalString,
engagementType: t.FeedbackEngagementType,
stringCenter: StringCenter,
injectionType: Option[st.SuggestType]
): Option[ChildFeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\FollowedBySocialContextBuilder.scala
class FollowedBySocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeAdsClientEventDetailsBuilder.scala
class HomeAdsClientEventDetailsBuilder(injectionType: Option[String])
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidate: UniversalNoun[Any],
candidateFeatures: FeatureMap
): Option[ClientEventDetails]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeClientEventDetailsBuilder.scala
object HomeClientEventDetailsBuilder
def getRequestJoinId(): Option[Long] =
RequestJoinKeyContext.current.flatMap(_.requestJoinId)
}
case class HomeClientEventDetailsBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
) extends BaseClientEventDetailsBuilder[Query, Candidate]
with TweetTypeGenerator[FeatureMap]
def apply(
query: Query,
candidate: Candidate,
candidateFeatures: FeatureMap
): Option[ClientEventDetails]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeConversationServiceCandidateDecorator.scala
object HomeConversationServiceCandidateDecorator
def apply(
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder
): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeFeedbackActionInfoBuilder.scala
class HomeFeedbackActionInfoBuilder @Inject() (
notInterestedTopicFeedbackActionBuilder: NotInterestedTopicFeedbackActionBuilder,
dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder)
extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackActionInfo]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeQueryTypePredicates.scala
object HomeQueryTypePredicates
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeTimelinesScoreInfoBuilder.scala
object HomeTimelinesScoreInfoBuilder
extends BaseTimelinesScoreInfoBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[TimelinesScoreInfo]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeTweetSocialContextBuilder.scala
class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder: LikedBySocialContextBuilder,
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
topicSocialContextBuilder: TopicSocialContextBuilder,
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
features: FeatureMap
): Option[SocialContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeTweetTypePredicates.scala
object HomeTweetTypePredicates
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\LikedBySocialContextBuilder.scala
class LikedBySocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\ListConversationServiceCandidateDecorator.scala
object ListConversationServiceCandidateDecorator
def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\MuteUserChildFeedbackActionBuilder.scala
class MuteUserChildFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings)
def apply(
candidateFeatures: FeatureMap
): Option[ChildFeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\NotInterestedTopicFeedbackActionBuilder.scala
class NotInterestedTopicFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings)
def apply(
candidateFeatures: FeatureMap
): Option[FeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\NotRelevantChildFeedbackActionBuilder.scala
class NotRelevantChildFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings)
def apply(
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[ChildFeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\ReceivedReplySocialContextBuilder.scala
class ReceivedReplySocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\ReportTweetChildFeedbackActionBuilder.scala
class ReportTweetChildFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings)
def apply(
candidate: TweetCandidate
): Option[ChildFeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\RetweeterChildFeedbackActionBuilder.scala
class RetweeterChildFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings)
def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\TopicSocialContextBuilder.scala
class TopicSocialContextBuilder @Inject() ()
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\UnfollowUserChildFeedbackActionBuilder.scala
class UnfollowUserChildFeedbackActionBuilder @Inject() (
@ProductScoped stringCenter: StringCenter,
externalStrings: HomeMixerExternalStrings)
def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\YouMightLikeSocialContextBuilder.scala
class YouMightLikeSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\builder\HomeClientEventInfoBuilder.scala
class HomeClientEventInfoBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
detailsBuilder: Option[BaseClientEventDetailsBuilder[Query, Candidate]] = None)
extends BaseClientEventInfoBuilder[Query, Candidate]
def apply(
query: Query,
candidate: Candidate,
candidateFeatures: FeatureMap,
element: Option[String]
): Option[ClientEventInfo]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\builder\HomeConversationModuleMetadataBuilder.scala
class HomeConversationModuleMetadataBuilder[
-Query <: PipelineQuery,
-Candidate <: BaseTweetCandidate
]() extends BaseModuleMetadataBuilder[Query, Candidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\builder\ListClientEventDetailsBuilder.scala
object ListClientEventDetailsBuilder
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidate: UniversalNoun[Any],
candidateFeatures: FeatureMap
): Option[ClientEventDetails]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\urt\builder\AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala
class AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder[Query <: PipelineQuery](
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends UrtInstructionBuilder[Query, AddEntriesTimelineInstruction]
def build(
query: Query,
entries: Seq[TimelineEntry]
): Seq[AddEntriesTimelineInstruction]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\urt\builder\HomeWhoToFollowFeedbackActionInfoBuilder.scala
object HomeWhoToFollowFeedbackActionInfoBuilder
class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\AncestorFeatureHydrator.scala
class AncestorFeatureHydrator @Inject() (
conversationServiceClient: tcs.ConversationService.MethodPerEndpoint)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
def getTruncatedRootTweet(
ancestors: ta.TweetAncestors,
): Option[ta.TweetAncestor]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\AuthorFeatureHydrator.scala
object AuthorFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class AuthorFeatureHydrator @Inject() (
@Named(AuthorFeatureRepository) client: KeyValueRepository[Seq[Long], Long, af.AuthorFeatures],
override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord]
def extractKeys(
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\DismissInfoQueryFeatureHydrator.scala
object DismissInfoQueryFeatureHydrator
class DismissInfoQueryFeatureHydrator @Inject() (
dismissInfoClient: InjectionHistoryClient)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap] =
Stitch.callFuture
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\EarlybirdFeatureHydrator.scala
object EarlybirdDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class EarlybirdFeatureHydrator @Inject() (
@Named(EarlybirdRepository) client: KeyValueRepository[
(Seq[Long], Long),
Long,
eb.ThriftSearchResult
],
override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def handleResponse(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]],
results: KeyValueResult[Long, eb.ThriftSearchResult]
): Seq[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\FeedbackHistoryQueryFeatureHydrator.scala
class FeedbackHistoryQueryFeatureHydrator @Inject() (
feedbackHistoryClient: FeedbackHistoryManhattanClient)
extends QueryFeatureHydrator[PipelineQuery]
with Conditionally[PipelineQuery]
def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableFeedbackFatigueParam)
override def hydrate(
query: PipelineQuery
): Stitch[FeatureMap] =
Stitch
.callFuture(feedbackHistoryClient.get(query.getRequiredUserId))
.map
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\FocalTweetFeatureHydrator.scala
class FocalTweetFeatureHydrator @Inject() ()
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\FollowedTopicsQueryFeatureHydrator.scala
class FollowedTopicsQueryFeatureHydrator @Inject() (
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\GizmoduckAuthorSafetyFeatureHydrator.scala
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery]
def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\GizmoduckUserQueryFeatureHydrator.scala
class GizmoduckUserQueryFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\GraphTwoHopFeatureHydrator.scala
object GraphTwoHopFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class GraphTwoHopFeatureHydrator @Inject() (
@Named(GraphTwoHopRepository) client: KeyValueRepository[(Seq[Long], Long), Long, Seq[
gfs.IntersectionValue
]],
override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\ImpressionBloomFilterQueryFeatureHydrator.scala
class ImpressionBloomFilterQueryFeatureHydrator[
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
bloomFilter: ImpressionBloomFilter)
extends QueryFeatureHydrator[Query]
def hydrate(query: Query): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\LastNonPollingTimeQueryFeatureHydrator.scala
class LastNonPollingTimeQueryFeatureHydrator @Inject() (
userSessionStore: ReadWriteUserSessionStore)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\ListMembersQueryFeatureHydrator.scala
object ListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]]
class ListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph)
extends QueryFeatureHydrator[PipelineQuery with HasListId]
def hydrate(query: PipelineQuery with HasListId): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\MetricCenterUserCountingFeatureHydrator.scala
object MetricCenterUserCountingFeature
extends Feature[TweetCandidate, Option[rf.MCUserCountingFeatures]]
@Singleton
class MetricCenterUserCountingFeatureHydrator @Inject() (
@Named(MetricCenterUserCountingFeatureRepository) client: KeyValueRepository[Seq[
Long
], Long, rf.MCUserCountingFeatures],
override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def extractKeys(
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\NamesFeatureHydrator.scala
class ProfileNames(screenName: String, realName: String)
@Singleton
class NamesFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery]
def onlyIf(query: PipelineQuery): Boolean = query.product match
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\PersistenceStoreQueryFeatureHydrator.scala
class PersistenceStoreQueryFeatureHydrator @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\PerspectiveFilteredSocialContextFeatureHydrator.scala
class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService: TimelineService)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealGraphInNetworkScoresQueryFeatureHydrator.scala
class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
@Named(RealGraphInNetworkScores) store: ReadableStore[Long, Seq[wtf.Candidate]])
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealGraphQueryFeatureHydrator.scala
object RealGraphFeatures extends Feature[PipelineQuery, Option[Map[UserId, RealGraphEdgeFeatures]]]
@Singleton
class RealGraphQueryFeatureHydrator @Inject() (
@Named(RealGraphFeatureRepository) repository: Repository[Long, Option[uss.UserSession]])
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealGraphViewerAuthorFeatureHydrator.scala
object RealGraphViewerAuthorDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
object RealGraphViewerAuthorsDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class RealGraphViewerAuthorFeatureHydrator @Inject() ()
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
def getRealGraphViewerAuthorFeatures(
viewerId: Long,
authorId: Long,
realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures]
): rg.UserRealGraphFeatures
object RealGraphViewerAuthorFeatureHydrator
def getCombinedRealGraphFeatures(
userIds: Seq[Long],
realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures]
): rg.RealGraphFeatures
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealGraphViewerRelatedUsersFeatureHydrator.scala
object RealGraphViewerRelatedUsersDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class RealGraphViewerRelatedUsersFeatureHydrator @Inject() ()
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
def getRelatedUserIds(features: FeatureMap): Seq[Long]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealTimeInteractionGraphEdgeFeatureHydrator.scala
object RealTimeInteractionGraphEdgeFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() ()
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala
object RealTimeInteractionGraphUserVertexQueryFeature
extends Feature[PipelineQuery, Option[ig.UserVertex]]
@Singleton
class RealTimeInteractionGraphUserVertexQueryFeatureHydrator @Inject() (
@Named(RealTimeInteractionGraphUserVertexCache) client: ReadCache[Long, ig.UserVertex],
override val statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery]
with ObservedKeyValueResultHandler
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\ReplyFeatureHydrator.scala
object InReplyToTweetHydratedEarlybirdFeature
extends Feature[TweetCandidate, Option[ThriftTweetFeatures]]
/**
* The purpose of this hydrator is to
* 1) hydrate simple features into replies and their ancestor tweets
* 2) keep both the normal replies and ancestor source candidates, but hydrate into the candidates
* features useful for predicting the quality of the replies and source ancestor tweets.
*/
@Singleton
class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def hydratedReplyCandidate(
replyCandidate: CandidateWithFeatures[TweetCandidate],
inReplyToTweetCandidate: CandidateWithFeatures[TweetCandidate]
): (
CandidateWithFeatures[TweetCandidate],
Option[ConversationFeatures],
Option[ThriftTweetFeatures]
)
def hydrateAncestorTweetCandidate(
ancestorTweetCandidate: CandidateWithFeatures[TweetCandidate],
descendantReplies: Seq[CandidateWithFeatures[TweetCandidate]],
updatedReplyConversationFeatures: Option[ConversationFeatures]
): (CandidateWithFeatures[TweetCandidate], Option[ConversationFeatures])
def originalTweetAgeFromSnowflake(
candidate: CandidateWithFeatures[TweetCandidate]
): Option[Duration]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RequestQueryFeatureHydrator.scala
class RequestQueryFeatureHydrator[
Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext] @Inject() (
) extends QueryFeatureHydrator[Query]
def getLanguageISOFormatByCode(languageCode: String): String =
ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguageUtil.getThriftLanguageOf(languageCode))
private def getRequestJoinId(servedRequestId: Long): Option[Long] =
Some(RequestJoinKeyContext.current.flatMap(_.requestJoinId).getOrElse(servedRequestId))
private def hasDarkRequest: Option[Boolean] = ForwardAnnotation.current
.getOrElse(Seq[BinaryAnnotation]())
.find(_.key == DarkRequestAnnotation)
.map(_.value.asInstanceOf[Boolean])
override def hydrate(query: Query): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RetweetSourceTweetFeatureHydrator.scala
object SourceTweetEarlybirdFeature extends Feature[TweetCandidate, Option[ThriftTweetFeatures]]
/**
* Feature Hydrator that bulk hydrates source tweets' features to retweet candidates
*/
object RetweetSourceTweetFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\SGSFollowedUsersQueryFeatureHydrator.scala
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
socialGraphStitchClient: SocialGraphStitchClient)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\SGSValidSocialContextFeatureHydrator.scala
class SGSValidSocialContextFeatureHydrator @Inject() (
socialGraph: SocialGraph)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def getValidUserIds(
viewerId: Long,
socialProofUserIds: Seq[Long]
): Stitch[Seq[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\SimClustersEngagementSimilarityFeatureHydrator.scala
object SimClustersEngagementSimilarityFeature
extends DataRecordInAFeature[PipelineQuery]
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient,
statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery]
def onlyIf(query: PipelineQuery): Boolean
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\SocialGraphServiceFeatureHydrator.scala
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def createIdsRequest(
userId: Long,
relationshipTypes: Set[sg.RelationshipType],
targetIds: Option[Seq[Long]] = None
): sg.IdsRequest = sg.IdsRequest(
relationshipTypes.map
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TimeFeaturesHydrator.scala
object TimeFeaturesDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
def setTimeFeatures(
richDataRecord: RichDataRecord,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
query: PipelineQuery,
): Unit
def getTimeFeatures(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
): Option[TimeFeatures]
def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long]
def addLastEngagementTimeFeatures(
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
timeFeatures: TimeFeatures,
timeSinceSourceTweetCreation: Long
): Option[TimeFeatures]
def addNonPollingTimeFeatures(
timeFeatures: TimeFeatures,
requestTimestampMs: Long,
creationTimeMs: Long,
nonPollingTimestampsMs: Option[Seq[Long]]
): Option[TimeFeatures]
def getTimeSinceLastEngagementHrs(
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
timeSinceTweetCreation: Long
): Option[Double]
def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TimelineServiceTweetsQueryFeatureHydrator.scala
object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class TimelineServiceTweetsQueryFeatureHydrator @Inject() (
timelineService: TimelineService,
deviceContextMarshaller: DeviceContextMarshaller)
extends QueryFeatureHydrator[PipelineQuery with HasDeviceContext]
def hydrate(query: PipelineQuery with HasDeviceContext): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TSPInferredTopicFeatureHydrator.scala
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
object TSPInferredTopicDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TSPInferredTopicFeatureHydrator @Inject() (
topicSocialProofClientColumn: TopicSocialProofClientColumn,
statsReceiver: StatsReceiver,
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def getSocialProof(
topicWithScores: Seq[tsp.TopicWithScore]
): (Option[Long], Option[TopicContextFunctionalityType])
def convertTopicWithScores(
topicWithScores: Seq[tsp.TopicWithScore],
): Map[Long, Double]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetImpressionsQueryFeatureHydrator.scala
class TweetImpressionsQueryFeatureHydrator[
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
extends QueryFeatureHydrator[Query]
def hydrate(query: Query): Stitch[FeatureMap]
def updateTweetImpressions(
tweetImpressionsFromStore: Seq[t.TweetImpressionsEntry],
seenIdsFromClient: Seq[Long],
currentTime: Long = Time.now.inMilliseconds,
tweetImpressionTTL: Duration = TweetImpressionTTL,
tweetImpressionCap: Int = TweetImpressionCap,
): Seq[t.TweetImpressionsEntry]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetMetaDataFeatureHydrator.scala
object TweetMetaDataDataRecord
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
object TweetMetaDataFeatureHydrator
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def features: Set[Feature[_, _]] = Set(TweetMetaDataDataRecord)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
def setFeatures(
richDataRecord: RichDataRecord,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetypieContentFeatureHydrator.scala
object TweetypieContentDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TweetypieContentFeatureHydrator @Inject() (
@Named(TweetypieContentRepository) client: KeyValueRepository[Seq[Long], Long, tp.Tweet],
override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def parTransformer(
result: KeyValueResult[Long, tp.Tweet],
candidate: CandidateWithFeatures[TweetCandidate]
): Try[(Seq[Long], DataRecord)]
def postTransformer(
result: Try[Option[tp.Tweet]]
): Try[(Seq[Long], DataRecord)]
def getCandidateOriginalTweetId(
candidate: CandidateWithFeatures[TweetCandidate]
): Long
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetypieFeatureHydrator.scala
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetypieStaticEntitiesFeatureHydrator.scala
class TweetypieStaticEntitiesFeatureHydrator @Inject() (
tweetypieStitchClient: TweetypieStitchClient,
@Named(TweetypieStaticEntitiesCache) cacheClient: TtlCache[Long, tp.Tweet])
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def createFeatureMap(tweet: tp.Tweet): FeatureMap
def readFromTweetypie(
query: PipelineQuery,
candidate: CandidateWithFeatures[TweetCandidate]
): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TwhinAuthorFollow20220101FeatureHydrator.scala
object TwhinAuthorFollow20220101Feature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TwhinAuthorFollow20220101FeatureHydrator @Inject() (
@Named(TwhinAuthorFollow20200101FeatureRepository)
client: KeyValueRepository[Seq[Long], Long, ml.Embedding],
override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def postTransformer(embedding: Try[Option[ml.Embedding]]): Try[DataRecord]
def extractKeys(
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TwhinUserEngagementQueryFeatureHydrator.scala
object TwhinUserEngagementFeature
extends DataRecordInAFeature[PipelineQuery]
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TwhinUserEngagementQueryFeatureHydrator @Inject() (
@Named(TwhinUserEngagementFeatureRepository)
client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor],
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TwhinUserFollowQueryFeatureHydrator.scala
object TwhinUserFollowFeature
extends DataRecordInAFeature[PipelineQuery]
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TwhinUserFollowQueryFeatureHydrator @Inject() (
@Named(TwhinUserFollowFeatureRepository)
client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor],
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\UserFollowedTopicIdsFeatureHydrator.scala
object UserFollowedTopicIdsFeature extends Feature[TweetCandidate, Seq[Long]]
@Singleton
class UserFollowedTopicIdsFeatureHydrator @Inject() (
@Named(UserFollowedTopicIdsRepository)
client: KeyValueRepository[Seq[Long], Long, Seq[Long]],
override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def postTransformer(input: Try[Option[Seq[Long]]]): Try[Seq[Long]]
def extractKeys(
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\UserLanguagesFeatureHydrator.scala
object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguage]]
@Singleton
case class UserLanguagesFeatureHydrator @Inject() (
@Named(UserLanguagesStore) store: ReadableStore[Long, Seq[scc.ThriftLanguage]])
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\UserStateQueryFeatureHydrator.scala
class UserStateQueryFeatureHydrator @Inject() (
userSessionStore: ReadOnlyUserSessionStore)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\UtegFeatureHydrator.scala
class UtegFeatureHydrator @Inject() (
@Named(UtegSocialProofRepository) client: KeyValueRepository[
(Seq[Long], (Long, Map[Long, Double])),
Long,
uteg.TweetRecommendation
]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery]
def onlyIf(query: PipelineQuery): Boolean = query.features
.exists(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double]).nonEmpty)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def handleResponse(
candidates: Seq[CandidateWithFeatures[TweetCandidate]],
results: KeyValueResult[Long, uteg.TweetRecommendation],
): Seq[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\author_features\AuthorFeaturesAdapter.scala
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]]
def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set.empty
private val compactDataRecordConverter = new CompactDataRecordConverter()
private val drMerger = new DataRecordMerger()
override def setFeatures(
authorFeaturesOpt: Option[af.AuthorFeatures],
richDataRecord: RichDataRecord
): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\content\ContentFeatureAdapter.scala
object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]]
def getTweetLengthType(tweetLength: Int): Long
def setFeatures(
contentFeatures: Option[ContentFeatures],
richDataRecord: RichDataRecord
): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\earlybird\EarlybirdAdapter.scala
object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]]
def setFeatures(
ebFeatures: Option[sc.ThriftTweetFeatures],
richDataRecord: RichDataRecord
): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\inferred_topic\InferredTopicAdapter.scala
object InferredTopicAdapter extends TimelinesMutatingAdapterBase[Map[Long, Double]]
def setFeatures(
inferredTopicFeatures: Map[Long, Double],
richDataRecord: RichDataRecord
): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\non_ml_features\NonMLCandidateFeaturesAdapter.scala
class NonMLCandidateFeatures(
tweetId: Long,
sourceTweetId: Option[Long],
originalAuthorId: Option[Long],
)
/**
* define non ml features adapter to create a data record which includes many non ml features
* e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline.
*/
object NonMLCandidateFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCandidateFeatures]
def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set.empty
override def setFeatures(
nonMLCandidateFeatures: NonMLCandidateFeatures,
richDataRecord: RichDataRecord
): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\non_ml_features\NonMLCommonFeaturesAdapter.scala
class NonMLCommonFeatures(
userId: Long,
predictionRequestId: Option[Long],
servedTimestamp: Long,
)
/**
* define non ml features adapter to create a data record which includes many non ml features
* e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline.
*/
object NonMLCommonFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCommonFeatures]
def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set(
SharedFeatures.USER_ID,
TimelinesSharedFeatures.PREDICTION_REQUEST_ID,
TimelinesSharedFeatures.SERVED_TIMESTAMP,
)
override def setFeatures(
nonMLCommonFeatures: NonMLCommonFeatures,
richDataRecord: RichDataRecord
): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\offline_aggregates\PassThroughAdapter.scala
object PassThroughAdapter extends IRecordOneToOneAdapter[Seq[DataRecord]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\offline_aggregates\SparseAggregatesToDenseAdapter.scala
class SparseAggregatesToDenseAdapter(policy: CombineCountsPolicy)
extends TimelinesIRecordAdapter[Seq[DataRecord]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\twhin_embeddings\TwhinEmbeddingsAdapter.scala
trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.Embedding]]
def twhinEmbeddingsFeature: Feature.Tensor
override def getFeatureContext: FeatureContext = new FeatureContext(
twhinEmbeddingsFeature
)
override def setFeatures(
embedding: Option[ml.Embedding],
richDataRecord: RichDataRecord
): Unit
object TwhinEmbeddingsFeatures
object TwhinAuthorFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter
object TwhinUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter
object TwhinUserFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\AggregateFeatureInfo.scala
class deriving aggregate feature info from the given configuration parameters.
class AggregateFeatureInfo(
val aggregateGroups: Set[AggregateGroup],
val aggregateType: AggregateType)
object AggregateFeatureInfo
def pickFeature(aggregateType: AggregateType): BaseAggregateRootFeature
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\AggregateFeaturesToDecodeWithMetadata.scala
class AggregateFeaturesToDecodeWithMetadata(
metadataOpt: Option[DenseFeatureMetadata],
aggregates: UserAggregateInteractions)
def toDataRecord(dr: DenseCompactDataRecord): DataRecord =
VersionedAggregateFeaturesDecoder.fromJDenseCompact(
metadataOpt,
dr.versionId,
NullStatsReceiver,
s"V$
def userAggregatesOpt: Option[DenseCompactDataRecord]
def userAuthorAggregates = extract(_.user_author_aggregates)
def userEngagerAggregates = extract(_.user_engager_aggregates)
def userMentionAggregates = extract(_.user_mention_aggregates)
def userOriginalAuthorAggregates = extract(_.user_original_author_aggregates)
def userRequestDowAggregates = extract(_.user_request_dow_aggregates)
def userRequestHourAggregates = extract(_.user_request_hour_aggregates)
def rectweetUserSimclustersTweetAggregates = extract(_.rectweet_user_simclusters_tweet_aggregates)
def userTwitterListAggregates = extract(_.user_list_aggregates)
def userTopicAggregates = extract(_.user_topic_aggregates)
def userInferredTopicAggregates = extract(_.user_inferred_topic_aggregates)
def userMediaUnderstandingAnnotationAggregates = extract(
_.user_media_understanding_annotation_aggregates)
private def extract[T](
v17Fn: V17UserAggregateInteractions => JMap[JLong, DenseCompactDataRecord]
): JMap[JLong, DenseCompactDataRecord]
object AggregateFeaturesToDecodeWithMetadata
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\BaseAggregateQueryFeatureHydrator.scala
abstract class BaseAggregateQueryFeatureHydrator(
featureRepository: Repository[Long, Option[UserSession]],
metadataRepository: Repository[Int, Option[DenseFeatureMetadata]],
feature: Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]])
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
def decodeUserSession(
session: UserSession
): Option[Future[AggregateFeaturesToDecodeWithMetadata]]
def getAggregateFeaturesWithMetadata(
versionId: Int,
userAggregateInteractions: UserAggregateInteractions,
): Future[AggregateFeaturesToDecodeWithMetadata]
trait BaseAggregateRootFeature
extends Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\BaseEdgeAggregateFeatureHydrator.scala
abstract case class BaseEdgeAggregateFeature(
aggregateGroups: Set[AggregateGroup],
aggregateType: AggregateType,
extractMapFn: AggregateFeaturesToDecodeWithMetadata => JMap[JLong, DenseCompactDataRecord],
adapter: IRecordOneToOneAdapter[Seq[DataRecord]],
getSecondaryKeysFn: CandidateWithFeatures[TweetCandidate] => Seq[Long])
extends DataRecordInAFeature[PipelineQuery]
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord]
def defaultValue: DataRecord = new DataRecord
private val rootFeatureInfo = new AggregateFeatureInfo(aggregateGroups, aggregateType)
val featureContext: FeatureContext = rootFeatureInfo.featureContext
val rootFeature: BaseAggregateRootFeature = rootFeatureInfo.feature
}
trait BaseEdgeAggregateFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def aggregateFeatures: Set[BaseEdgeAggregateFeature]
override def features = aggregateFeatures.asInstanceOf[Set[Feature[_, _]]]
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def hydrateAggregateFeature(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]],
feature: BaseEdgeAggregateFeature
): Seq[DataRecord]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\EdgeAggregateFeatures.scala
object EdgeAggregateFeatures
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\PartAAggregateQueryFeatureHydrator.scala
object PartAAggregateRootFeature extends BaseAggregateRootFeature
class PartAAggregateQueryFeatureHydrator @Inject() (
@Named(TimelineAggregatePartARepository)
repository: Repository[Long, Option[UserSession]],
@Named(TimelineAggregateMetadataRepository)
metadataRepository: Repository[Int, Option[DenseFeatureMetadata]])
extends BaseAggregateQueryFeatureHydrator(
repository,
metadataRepository,
PartAAggregateRootFeature
)
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\PartBAggregateQueryFeatureHydrator.scala
object PartBAggregateRootFeature extends BaseAggregateRootFeature
object UserAggregateFeature
extends DataRecordInAFeature[PipelineQuery]
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class PartBAggregateQueryFeatureHydrator @Inject() (
@Named(TimelineAggregatePartBRepository)
repository: Repository[Long, Option[UserSession]],
@Named(TimelineAggregateMetadataRepository)
metadataRepository: Repository[Int, Option[DenseFeatureMetadata]])
extends BaseAggregateQueryFeatureHydrator(
repository,
metadataRepository,
PartBAggregateRootFeature
)
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
def mergeDataRecordOpts(dataRecordOpts: Option[DataRecord]*): DataRecord =
dataRecordOpts.flatten.foldLeft(new DataRecord)
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\Phase1EdgeAggregateFeatureHydrator.scala
class Phase1EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\Phase2EdgeAggregateFeatureHydrator.scala
class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\Utils.scala
object Utils
def selectAndTransform(
idsToSelect: Seq[Long],
transform: DenseCompactDataRecord => DataRecord,
map: java.util.Map[java.lang.Long, DenseCompactDataRecord],
): Map[Long, DataRecord]
def filterDataRecord(dr: DataRecord, featureContext: FeatureContext): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala
trait BaseRealTimeAggregateBulkCandidateFeatureHydrator[K]
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with BaseRealtimeAggregateHydrator[K]
def features: Set[Feature[_, _]] = Set(outputFeature)
override lazy val statScope: String = identifier.toString
def keysFromQueryAndCandidates(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[K]]
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\BaseRealtimeAggregateHydrator.scala
trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler
def postTransformer(dataRecord: Try[Option[DataRecord]]): Try[DataRecord]
def fetchAndConstructDataRecords(possiblyKeys: Seq[Option[K]]): Stitch[Seq[Try[DataRecord]]]
object BaseRealtimeAggregateHydrator
def applyDecay(
dataRecord: DataRecord,
featureContext: FeatureContext,
decay: TimeDecay
): DataRecord
def time: Long = Time.now.inMillis
val richFullDr = new SRichDataRecord(dataRecord, featureContext)
val richNewDr = new SRichDataRecord(new DataRecord, featureContext)
val featureIterator = featureContext.iterator()
featureIterator.forEachRemaining
def applyRename(
dataRecord: DataRecord,
featureContext: FeatureContext,
renamedFeatureContext: FeatureContext,
featureRenamingMap: Map[MLApiFeature[_], MLApiFeature[_]]
): DataRecord
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\BaseRealTimeAggregateQueryFeatureHydrator.scala
trait BaseRealTimeAggregateQueryFeatureHydrator[K]
extends QueryFeatureHydrator[PipelineQuery]
with BaseRealtimeAggregateHydrator[K]
def features: Set[Feature[_, _]] = Set(outputFeature)
override lazy val statScope: String = identifier.toString
def keysFromQueryAndCandidates(
query: PipelineQuery
): Option[K]
override def hydrate(
query: PipelineQuery
): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala
object EngagementsReceivedByAuthorRealTimeAggregateFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator @Inject() (
@Named(EngagementsReceivedByAuthorCache) override val client: ReadCache[Long, DataRecord],
override val statsReceiver: StatsReceiver)
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\RealTimeAggregateTimeDecay.scala
class RealTimeAggregateTimeDecay(
featureIdToHalfLife: Map[Long, Duration],
timestampFeatureId: Long = TIMESTAMP.getFeatureId)
def apply(record: DataRecord, timeNow: Long): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala
object TopicCountryEngagementRealTimeAggregateFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TopicCountryEngagementRealTimeAggregateFeatureHydrator @Inject() (
@Named(TopicCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord],
override val statsReceiver: StatsReceiver)
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)]
def keysFromQueryAndCandidates(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[(Long, String)]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TopicEngagementRealTimeAggregateFeatureHydrator.scala
object TopicEngagementRealTimeAggregateFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TopicEngagementRealTimeAggregateFeatureHydrator @Inject() (
@Named(TopicEngagementCache) override val client: ReadCache[Long, DataRecord],
override val statsReceiver: StatsReceiver)
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long]
def keysFromQueryAndCandidates(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala
object TweetCountryEngagementRealTimeAggregateFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() (
@Named(TweetCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord],
override val statsReceiver: StatsReceiver)
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)]
def keysFromQueryAndCandidates(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[(Long, String)]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TweetEngagementRealTimeAggregateFeatureHydrator.scala
object TweetEngagementRealTimeAggregateFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() (
@Named(TweetEngagementCache) override val client: ReadCache[Long, DataRecord],
override val statsReceiver: StatsReceiver)
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long]
def keysFromQueryAndCandidates(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TwitterListEngagementRealTimeAggregateFeatureHydrator.scala
object TwitterListEngagementRealTimeAggregateFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TwitterListEngagementRealTimeAggregateFeatureHydrator @Inject() (
@Named(TwitterListEngagementCache) override val client: ReadCache[Long, DataRecord],
override val statsReceiver: StatsReceiver)
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long]
def keysFromQueryAndCandidates(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala
object UserAuthorEngagementRealTimeAggregateFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class UserAuthorEngagementRealTimeAggregateFeatureHydrator @Inject() (
@Named(UserAuthorEngagementCache) override val client: ReadCache[(Long, Long), DataRecord],
override val statsReceiver: StatsReceiver)
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long)]
def keysFromQueryAndCandidates(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[(Long, Long)]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\UserEngagementRealTimeAggregatesFeatureHydrator.scala
object UserEngagementRealTimeAggregateFeature
extends DataRecordInAFeature[PipelineQuery]
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class UserEngagementRealTimeAggregatesFeatureHydrator @Inject() (
@Named(UserEngagementCache) override val client: ReadCache[Long, DataRecord],
override val statsReceiver: StatsReceiver)
extends BaseRealTimeAggregateQueryFeatureHydrator[Long]
def keysFromQueryAndCandidates(query: PipelineQuery): Option[Long]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\DropMaxCandidatesFilter.scala
class DropMaxCandidatesFilter[Candidate <: UniversalNoun[Any]](
maxCandidatesParam: FSBoundedParam[Int])
extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\FeedbackFatigueFilter.scala
object FeedbackFatigueFilter
extends Filter[PipelineQuery, TweetCandidate]
with Filter.Conditionally[PipelineQuery, TweetCandidate]
def onlyIf(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Boolean =
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
private val DurationForFiltering = 14.days
override def apply(
query: pipeline.PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
def getUserIds(
feedbackEntries: Seq[FeedbackEntry],
): Set[Long] =
feedbackEntries.collect
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\InvalidConversationModuleFilter.scala
object InvalidConversationModuleFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\KeepBestOutOfNetworkTweetPerAuthorFilter.scala
object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\OutOfNetworkCompetitorFilter.scala
object OutOfNetworkCompetitorFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
def isOutOfNetworkTweetFromCompetitor(
candidate: CandidateWithFeatures[TweetCandidate],
competitorAuthors: Set[Long]
): Boolean
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\OutOfNetworkCompetitorURLFilter.scala
object OutOfNetworkCompetitorURLFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
def hasOutOfNetworkUrlFromCompetitor(
candidate: CandidateWithFeatures[TweetCandidate],
competitorUrls: Set[String]
): Boolean
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PredicateFeatureFilter.scala
trait ShouldKeepCandidate
def apply(features: FeatureMap): Boolean
}
object PredicateFeatureFilter
def fromPredicate[Candidate <: UniversalNoun[Any]](
identifier: FilterIdentifier,
shouldKeepCandidate: ShouldKeepCandidate
): Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PredicateGatedFilter.scala
trait FilterPredicate[-Query <: PipelineQuery]
def apply(query: Query): Boolean
}
/**
* A [[Filter]] with [[Conditionally]] based on a [[FilterPredicate]]
*
* @param predicate the predicate to turn this filter on and off
* @param filter the underlying filter to run when `predicate` is true
* @tparam Query The domain model for the query or request
* @tparam Candidate The type of the candidates
*/
case class PredicateGatedFilter[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
predicate: FilterPredicate[Query],
filter: Filter[Query, Candidate])
extends Filter[Query, Candidate]
with Filter.Conditionally[Query, Candidate]
def onlyIf(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): Boolean =
Conditionally.and(Filter.Input(query, candidates), filter, predicate(query))
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]] = filter.apply(query, candidates)
}
object PredicateGatedFilter
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PreviouslySeenTweetsFilter.scala
object PreviouslySeenTweetsFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PreviouslyServedAncestorsFilter.scala
object PreviouslyServedAncestorsFilter
extends Filter[PipelineQuery, TweetCandidate]
with TimelinePersistenceUtils
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PreviouslyServedTweetsFilter.scala
object PreviouslyServedTweetsFilter
extends Filter[PipelineQuery, TweetCandidate]
with Filter.Conditionally[PipelineQuery, TweetCandidate]
def onlyIf(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Boolean
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\RejectTweetFromViewerFilter.scala
object RejectTweetFromViewerFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\RetweetDeduplicationFilter.scala
object RetweetDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\RetweetSourceTweetRemovingFilter.scala
object RetweetSourceTweetRemovingFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\SocialContextFilter.scala
object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\DismissFatigueGate.scala
object DismissFatigueGate
class DismissFatigueGate(
suggestType: SuggestType,
dismissInfoFeature: Feature[PipelineQuery, Map[SuggestType, Option[DismissInfo]]],
baseDismissDuration: Duration = DismissFatigueGate.DefaultBaseDismissDuration,
) extends Gate[PipelineQuery]
def shouldContinue(query: PipelineQuery): Stitch[Boolean]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\ExcludeSoftUserGate.scala
object ExcludeSoftUserGate extends Gate[PipelineQuery]
def shouldContinue(query: PipelineQuery): Stitch[Boolean]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\MinCachedTweetsGate.scala
class MinCachedTweetsGate(
candidatePipelineIdentifier: CandidatePipelineIdentifier,
minCachedTweetsParam: Param[Int])
extends Gate[PipelineQuery]
def shouldContinue(query: PipelineQuery): Stitch[Boolean]
object MinCachedTweetsGate
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\NonEmptySeqFeatureGate.scala
class NonEmptySeqFeatureGate[T: TypeTag](
feature: Feature[PipelineQuery, Seq[T]])
extends Gate[PipelineQuery]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\RequestContextGate.scala
class RequestContextGate(requestContexts: Seq[RequestContext.Value])
extends Gate[PipelineQuery with HasDeviceContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\RequestContextNotGate.scala
class RequestContextNotGate(requestContexts: Seq[RequestContext.Value])
extends Gate[PipelineQuery with HasDeviceContext]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\SupportedLanguagesGate.scala
object SupportedLanguagesGate extends Gate[PipelineQuery]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\TimelinesPersistenceStoreLastInjectionGate.scala
class TimelinesPersistenceStoreLastInjectionGate(
minInjectionIntervalParam: Param[Duration],
persistenceEntriesFeature: Feature[PipelineQuery, Seq[TimelineResponseV3]],
entityIdType: EntityIdType.Value)
extends Gate[PipelineQuery]
with TimelinePersistenceUtils
def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
Stitch(
query.queryTime.since(getLastInjectionTime(query)) > query.params(minInjectionIntervalParam))
private def getLastInjectionTime(query: PipelineQuery) = query.features
.flatMap
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\ViewerIsListOwnerGate.scala
class ViewerIsListOwnerGate @Inject() (socialGraph: SocialGraph)
extends Gate[PipelineQuery with HasListId]
def shouldContinue(query: PipelineQuery with HasListId): Stitch[Boolean]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\query_transformer\EditedTweetsCandidatePipelineQueryTransformer.scala
object EditedTweetsCandidatePipelineQueryTransformer
extends CandidatePipelineQueryTransformer[PipelineQuery, Seq[Long]]
def transform(query: PipelineQuery): Seq[Long]
def getApplicableCandidates(query: PipelineQuery): Seq[PersistenceStoreEntry]
class PersistenceStoreEntry(
entryWithItemIds: EntryWithItemIds,
servedTime: Time,
clientPlatform: ClientPlatform,
requestType: RequestType)
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\scorer\FeedbackFatigueScorer.scala
object FeedbackFatigueScorer
extends Scorer[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery]
def features: Set[Feature[_, _]] = Set(ScoreFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
private val DurationForFiltering = 14.days
private val DurationForDiscounting = 140.days
private val ScoreMultiplierLowerBound = 0.2
private val ScoreMultiplierUpperBound = 1.0
private val ScoreMultiplierIncrementsCount = 4
private val ScoreMultiplierIncrement =
(ScoreMultiplierUpperBound - ScoreMultiplierLowerBound) / ScoreMultiplierIncrementsCount
private val ScoreMultiplierIncrementDurationInDays =
DurationForDiscounting.inDays / ScoreMultiplierIncrementsCount.toDouble
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def getUserDiscounts(
queryTime: Time,
feedbackEntries: Seq[FeedbackEntry],
): Map[Long, Double]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\scorer\OONTweetScalingScorer.scala
object OONTweetScalingScorer extends Scorer[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\scorer\VerifiedAuthorScalingScorer.scala
object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def getUpdatedScore(
score: Option[Double],
candidate: CandidateWithFeatures[TweetCandidate],
query: PipelineQuery
): Option[Double]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\selector\DebunchCandidates.scala
trait MustDebunch
def apply(candidate: CandidateWithDetails): Boolean
}
object DebunchCandidates
class DebunchCandidates(
override val pipelineScope: CandidateScope,
mustDebunch: MustDebunch,
maxBunchSize: Int)
extends Selector[PipelineQuery]
def apply(
query: PipelineQuery,
remainingCandidates: Seq[CandidateWithDetails],
result: Seq[CandidateWithDetails]
): SelectorResult
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\selector\UpdateConversationModuleId.scala
class UpdateConversationModuleId(
override val pipelineScope: CandidateScope)
extends Selector[PipelineQuery]
def apply(
query: PipelineQuery,
remainingCandidates: Seq[CandidateWithDetails],
result: Seq[CandidateWithDetails]
): SelectorResult
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\selector\UpdateHomeClientEventDetails.scala
class UpdateHomeClientEventDetails(candidatePipelines: Set[CandidatePipelineIdentifier])
extends Selector[PipelineQuery]
def apply(
query: PipelineQuery,
remainingCandidates: Seq[CandidateWithDetails],
result: Seq[CandidateWithDetails]
): SelectorResult
def updateItemPresentation(
query: PipelineQuery,
item: ItemCandidateWithDetails,
resultCandidateFeatures: FeatureMap,
resultFeatures: FeatureMap,
): ItemCandidateWithDetails
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\selector\UpdateNewTweetsPillDecoration.scala
object UpdateNewTweetsPillDecoration
class UpdateNewTweetsPillDecoration[Query <: PipelineQuery with HasDeviceContext](
override val pipelineScope: CandidateScope,
stringCenter: StringCenter,
seeNewTweetsString: ExternalString,
tweetedString: ExternalString)
extends Selector[Query]
def apply(
query: Query,
remainingCandidates: Seq[CandidateWithDetails],
result: Seq[CandidateWithDetails]
): SelectorResult
def useAvatars(query: Query, userIds: Seq[Long]): Boolean
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ClientEventsBuilder.scala
trait ClientEventsBuilder
def section(query: PipelineQuery): Option[String]
def count(
candidates: Seq[CandidateWithDetails],
predicate: FeatureMap => Boolean = _ => true,
queryFeatures: FeatureMap = FeatureMap.empty
): Option[Long] = Some(candidates.view.count(item => predicate(item.features ++ queryFeatures)))
protected def sum(
candidates: Seq[CandidateWithDetails],
predicate: FeatureMap => Option[Int],
queryFeatures: FeatureMap = FeatureMap.empty
): Option[Long] =
Some(candidates.view.flatMap(item => predicate(item.features ++ queryFeatures)).sum)
}
private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder
def build(
query: PipelineQuery,
injectedTweets: Seq[ItemCandidateWithDetails],
promotedTweets: Seq[ItemCandidateWithDetails],
whoToFollowUsers: Seq[ItemCandidateWithDetails]
): Seq[ClientEvent]
object EmptyTimelineEventsBuilder extends ClientEventsBuilder
def build(
query: PipelineQuery,
injectedTweets: Seq[ItemCandidateWithDetails]
): Seq[ClientEvent]
object QueryEventsBuilder extends ClientEventsBuilder
def build(
query: PipelineQuery,
injectedTweets: Seq[ItemCandidateWithDetails]
): Seq[ClientEvent]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\HomeScribeClientEventSideEffect.scala
class HomeScribeClientEventSideEffect(
override val logPipelinePublisher: EventPublisher[LogEvent],
injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
adsCandidatePipelineIdentifier: CandidatePipelineIdentifier,
whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
) extends ScribeClientEventSideEffect[PipelineQuery, Timeline]
def buildClientEvents(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Seq[ScribeClientEventSideEffect.ClientEvent]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\HomeScribeServedEntriesSideEffect.scala
class HomeScribeServedEntriesSideEffect @Inject() (
scribeEventPublisher: EventPublisher[thrift.Timeline])
extends PipelineResultSideEffect[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
def apply(
inputs: PipelineResultSideEffect.Inputs[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
): Stitch[Unit]
def buildTimeline(
inputs: PipelineResultSideEffect.Inputs[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
): thrift.Timeline
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\PublishClientSentImpressionsEventBusSideEffect.scala
object PublishClientSentImpressionsEventBusSideEffect
class PublishClientSentImpressionsEventBusSideEffect @Inject() (
eventBusPublisher: EventBusPublisher[PublishedImpressionList])
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling]
with PipelineResultSideEffect.Conditionally[
PipelineQuery with HasSeenTweetIds,
HasMarshalling
]
def onlyIf(
query: PipelineQuery with HasSeenTweetIds,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: HasMarshalling
): Boolean = query.seenTweetIds.exists(_.nonEmpty)
def buildEvents(
query: PipelineQuery with HasSeenTweetIds,
currentTime: Long
): Option[Seq[Impression]]
def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling]
): Stitch[Unit]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\PublishClientSentImpressionsManhattanSideEffect.scala
class PublishClientSentImpressionsManhattanSideEffect @Inject() (
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling]
with PipelineResultSideEffect.Conditionally[
PipelineQuery with HasSeenTweetIds,
HasMarshalling
]
def onlyIf(
query: PipelineQuery with HasSeenTweetIds,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: HasMarshalling
): Boolean = query.seenTweetIds.exists(_.nonEmpty)
def buildEvents(query: PipelineQuery): Option[(Long, t.TweetImpressionsEntries)]
def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling]
): Stitch[Unit]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateFeatureKeysKafkaSideEffect.scala
class ServedCandidateFeatureKeysKafkaSideEffect(
topic: String,
sourceIdentifiers: Set[identifier.CandidatePipelineIdentifier])
extends KafkaPublishingSideEffect[
sc.CandidateFeatureKey,
pldr.PolyDataRecord,
PipelineQuery,
Timeline
]
with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline]
def onlyIf(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam)
override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls"
override val keySerde: Serializer[sc.CandidateFeatureKey] =
CandidateFeatureKeySerde().serializer()
override val valueSerde: Serializer[pldr.PolyDataRecord] =
TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer
override val clientId: String = "home_mixer_served_candidate_feature_keys_producer"
override def buildRecords(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Seq[ProducerRecord[sc.CandidateFeatureKey, pldr.PolyDataRecord]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala
class ServedCandidateFeatureKeysKafkaSideEffectBuilder @Inject() (
injectedServiceIdentifier: ServiceIdentifier)
def build(
sourceIdentifiers: Set[CandidatePipelineIdentifier]
): ServedCandidateFeatureKeysKafkaSideEffect
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateKafkaSideEffect.scala
object ServedCandidateKafkaSideEffect
def extractCandidates(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
sourceIdentifiers: Set[CandidatePipelineIdentifier]
): Seq[ItemCandidateWithDetails]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateKeysKafkaSideEffect.scala
class ServedCandidateKeysKafkaSideEffect(
topic: String,
sourceIdentifiers: Set[CandidatePipelineIdentifier])
extends KafkaPublishingSideEffect[
sc.ServedCandidateKey,
pldr.PolyDataRecord,
PipelineQuery,
Timeline
]
with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline]
def onlyIf(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam)
override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls"
override val keySerde: Serializer[sc.ServedCandidateKey] = ServedCandidateKeySerde.serializer()
override val valueSerde: Serializer[pldr.PolyDataRecord] =
TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer
override val clientId: String = "home_mixer_served_candidate_keys_producer"
override def buildRecords(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Seq[ProducerRecord[sc.ServedCandidateKey, pldr.PolyDataRecord]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateKeysKafkaSideEffectBuilder.scala
class ServedCandidateKeysKafkaSideEffectBuilder @Inject() (
injectedServiceIdentifier: ServiceIdentifier)
def build(
sourceIdentifiers: Set[CandidatePipelineIdentifier]
): ServedCandidateKeysKafkaSideEffect
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedStatsSideEffect.scala
class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver)
extends PipelineResultSideEffect[PipelineQuery, Timeline]
def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
): Stitch[Unit]
def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit
def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit
def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\TruncateTimelinesPersistenceStoreSideEffect.scala
class TruncateTimelinesPersistenceStoreSideEffect @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
extends PipelineResultSideEffect[PipelineQuery, Timeline]
def getResponsesToDelete(query: PipelineQuery): Seq[TimelineResponseV3]
def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
): Stitch[Unit]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\UpdateImpressionBloomFilterSideEffect.scala
class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter)
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline]
with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline]
def onlyIf(
query: PipelineQuery with HasSeenTweetIds,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = query.seenTweetIds.exists(_.nonEmpty)
def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq]
def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline]
): Stitch[Unit]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\UpdateLastNonPollingTimeSideEffect.scala
class UpdateLastNonPollingTimeSideEffect[
Query <: PipelineQuery with HasDeviceContext,
ResponseType <: HasMarshalling] @Inject() (
override val userSessionStore: ReadWriteUserSessionStore)
extends UserSessionStoreUpdateSideEffect[
WriteRequest,
Query,
ResponseType
]
def buildWriteRequest(query: Query): Option[WriteRequest]
def makeLastNonPollingTimestamps(query: Query): NonPollingTimestamps
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\UpdateTimelinesPersistenceStoreSideEffect.scala
object UpdateTimelinesPersistenceStoreSideEffect
class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
extends PipelineResultSideEffect[PipelineQuery, Timeline]
def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
): Stitch[Unit]
def buildTweetEntryWithItemIds(
candidate: ItemCandidateWithDetails,
sortIndex: Long
): EntryWithItemIds
def requestTypeFromQuery(query: PipelineQuery): persistence.RequestType
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\DeviceContextUnmarshaller.scala
class DeviceContextUnmarshaller @Inject() ()
def apply(deviceContext: t.DeviceContext): DeviceContext
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\HomeMixerDebugParamsUnmarshaller.scala
class HomeMixerDebugParamsUnmarshaller @Inject() (
featureValueUnmarshaller: FeatureValueUnmarshaller)
def apply(debugParams: t.DebugParams): DebugParams
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\HomeMixerProductContextUnmarshaller.scala
class HomeMixerProductContextUnmarshaller @Inject() (
deviceContextUnmarshaller: DeviceContextUnmarshaller)
def apply(productContext: t.ProductContext): ProductContext = productContext match
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\HomeMixerProductUnmarshaller.scala
class HomeMixerProductUnmarshaller @Inject() ()
def apply(product: t.Product): Product = product match
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\HomeMixerRequestUnmarshaller.scala
class HomeMixerRequestUnmarshaller @Inject() (
clientContextUnmarshaller: ClientContextUnmarshaller,
homeProductUnmarshaller: HomeMixerProductUnmarshaller,
homeProductContextUnmarshaller: HomeMixerProductContextUnmarshaller,
homeDebugParamsUnmarshaller: HomeMixerDebugParamsUnmarshaller)
def apply(homeRequest: t.HomeMixerRequest): HomeMixerRequest
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\ChronologicalCursorMarshaller.scala
object ChronologicalCursorMarshaller
def apply(cursor: UrtOrderedCursor): Option[t.ChronologicalCursor]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\ChronologicalCursorUnmarshaller.scala
object ChronologicalCursorUnmarshaller
def apply(requestCursor: t.RequestCursor): Option[UrtOrderedCursor]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\DeviceContextMarshaller.scala
class DeviceContextMarshaller @Inject() ()
def apply(deviceContext: DeviceContext, clientContext: ClientContext): t.DeviceContext
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\RecommendedUsersCursorUnmarshaller.scala
object RecommendedUsersCursorUnmarshaller
def apply(requestCursor: t.RequestCursor): Option[UrtUnorderedExcludeIdsCursor]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\TimelineServiceCursorMarshaller.scala
object TimelineServiceCursorMarshaller
def apply(cursor: UrtOrderedCursor): Option[t.Cursor2]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\TopicContextFunctionalityTypeUnmarshaller.scala
object TopicContextFunctionalityTypeUnmarshaller
def apply(
topicContextFunctionalityType: urt.TopicContextFunctionalityType
): TopicContextFunctionalityType = topicContextFunctionalityType match
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timeline_logging\ConversationEntryMarshaller.scala
object ConversationEntryMarshaller
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timeline_logging\PromotedTweetEntryMarshaller.scala
object PromotedTweetEntryMarshaller
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetEntry
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timeline_logging\TweetEntryMarshaller.scala
object TweetEntryMarshaller
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetEntry
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timeline_logging\WhoToFollowEntryMarshaller.scala
object WhoToFollowEntryMarshaller
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\ClearCacheIncludeInstruction.scala
class ClearCacheIncludeInstruction(
enableParam: FSParam[Boolean],
minEntriesParam: FSBoundedParam[Int])
extends IncludeInstruction[PipelineQuery with HasDeviceContext]
def apply(
query: PipelineQuery with HasDeviceContext,
entries: Seq[TimelineEntry]
): Boolean
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\ContentFeatures.scala
object ContentFeatures
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\GapIncludeInstruction.scala
object GapIncludeInstruction
extends IncludeInstruction[PipelineQuery with HasPipelineCursor[UrtOrderedCursor]]
def apply(
query: PipelineQuery with HasPipelineCursor[UrtOrderedCursor],
entries: Seq[TimelineEntry]
): Boolean
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\HomeAdsQuery.scala
trait HomeAdsQuery
extends AdsQuery
with PipelineQuery
with HasDeviceContext
with HasPipelineCursor[UrtOrderedCursor]
def requestTriggerType: Option[RequestTriggerType]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\HomeFeatures.scala
object HomeFeatures
object AncestorsFeature extends Feature[TweetCandidate, Seq[ta.TweetAncestor]]
object AudioSpaceMetaDataFeature extends Feature[TweetCandidate, Option[AudioSpaceMetaData]]
object TwitterListIdFeature extends Feature[TweetCandidate, Option[Long]]
/**
* For Retweets, this should refer to the retweeting user. Use [[SourceUserIdFeature]] if you want to know
* who created the Tweet that was retweeted.
*/
object AuthorIdFeature
extends Feature[TweetCandidate, Option[Long]]
with LongDiscreteDataRecordCompatible
object AuthorIsEligibleForConnectBoostFeature extends Feature[TweetCandidate, Boolean]
object AuthorIsBlueVerifiedFeature extends Feature[TweetCandidate, Boolean]
object AuthoredByContextualUserFeature extends Feature[TweetCandidate, Boolean]
object CachedCandidatePipelineIdentifierFeature extends Feature[TweetCandidate, Option[String]]
object CandidateSourceIdFeature
extends Feature[TweetCandidate, Option[cts.CandidateTweetSourceId]]
object ConversationFeature extends Feature[TweetCandidate, Option[ConversationFeatures]]
/**
* This field should be set to the focal Tweet's tweetId for all tweets which are expected to
* be rendered in the same convo module. For non-convo module Tweets, this will be
* set to None. Note this is different from how TweetyPie defines ConversationId which is defined
* on all Tweets and points to the root tweet. This feature is used for grouping convo modules together.
*/
object ConversationModuleFocalTweetIdFeature extends Feature[TweetCandidate, Option[Long]]
/**
* This field should always be set to the root Tweet in a conversation for all Tweets. For replies, this will
* point back to the root Tweet. For non-replies, this will be the candidate's Tweet id. This is consistent with
* the TweetyPie definition of ConversationModuleId.
*/
object ConversationModuleIdFeature extends Feature[TweetCandidate, Option[Long]]
object DirectedAtUserIdFeature extends Feature[TweetCandidate, Option[Long]]
object EarlybirdFeature extends Feature[TweetCandidate, Option[sc.ThriftTweetFeatures]]
object EarlybirdScoreFeature extends Feature[TweetCandidate, Option[Double]]
object EntityTokenFeature extends Feature[TweetCandidate, Option[String]]
object ExclusiveConversationAuthorIdFeature extends Feature[TweetCandidate, Option[Long]]
object FavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]]
object FeedbackHistoryFeature extends Feature[TweetCandidate, Seq[FeedbackEntry]]
object RetweetedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]]
object RepliedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]]
object FollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]]
object TopicIdSocialContextFeature extends Feature[TweetCandidate, Option[Long]]
object TopicContextFunctionalityTypeFeature
extends Feature[TweetCandidate, Option[TopicContextFunctionalityType]]
object FromInNetworkSourceFeature extends Feature[TweetCandidate, Boolean]
object FullScoringSucceededFeature extends Feature[TweetCandidate, Boolean]
object HasDisplayedTextFeature extends Feature[TweetCandidate, Boolean]
object InReplyToTweetIdFeature extends Feature[TweetCandidate, Option[Long]]
object InReplyToUserIdFeature extends Feature[TweetCandidate, Option[Long]]
object IsAncestorCandidateFeature extends Feature[TweetCandidate, Boolean]
object IsExtendedReplyFeature
extends DataRecordFeature[TweetCandidate, Boolean]
with BoolDataRecordCompatible
object IsRandomTweetFeature
extends Feature[TweetCandidate, Boolean]
with BoolDataRecordCompatible
object IsReadFromCacheFeature extends Feature[TweetCandidate, Boolean]
object IsRetweetFeature extends Feature[TweetCandidate, Boolean]
object IsRetweetedReplyFeature extends Feature[TweetCandidate, Boolean]
object LastScoredTimestampMsFeature extends Feature[TweetCandidate, Option[Long]]
object NonSelfFavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]]
object NumImagesFeature extends Feature[TweetCandidate, Option[Int]]
object OriginalTweetCreationTimeFromSnowflakeFeature extends Feature[TweetCandidate, Option[Time]]
object PositionFeature extends Feature[TweetCandidate, Option[Int]]
// Internal id generated per prediction service request
object PredictionRequestIdFeature extends Feature[TweetCandidate, Option[Long]]
object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]]
object QuotedUserIdFeature extends Feature[TweetCandidate, Option[Long]]
object ScoreFeature extends Feature[TweetCandidate, Option[Double]]
object SemanticCoreIdFeature extends Feature[TweetCandidate, Option[Long]]
// Key for kafka logging
object ServedIdFeature extends Feature[TweetCandidate, Option[Long]]
object SimclustersTweetTopKClustersWithScoresFeature
extends Feature[TweetCandidate, Map[String, Double]]
object SocialContextFeature extends Feature[TweetCandidate, Option[tst.SocialContext]]
object SourceTweetIdFeature
extends Feature[TweetCandidate, Option[Long]]
with LongDiscreteDataRecordCompatible
object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]]
object StreamToKafkaFeature extends Feature[TweetCandidate, Boolean]
object SuggestTypeFeature extends Feature[TweetCandidate, Option[st.SuggestType]]
object TSPMetricTagFeature extends Feature[TweetCandidate, Set[tsp.MetricTag]]
object TweetLanguageFeature extends Feature[TweetCandidate, Option[String]]
object TweetUrlsFeature extends Feature[TweetCandidate, Seq[String]]
object VideoDurationMsFeature extends Feature[TweetCandidate, Option[Int]]
object ViewerIdFeature
extends Feature[TweetCandidate, Long]
with LongDiscreteDataRecordCompatible
def featureName: String = SharedFeatures.USER_ID.getFeatureName
override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId)
}
object WeightedModelScoreFeature extends Feature[TweetCandidate, Option[Double]]
object MentionUserIdFeature extends Feature[TweetCandidate, Seq[Long]]
object MentionScreenNameFeature extends Feature[TweetCandidate, Seq[String]]
object SemanticAnnotationFeature extends Feature[TweetCandidate, Seq[esb.TweetEntityAnnotation]]
object HasImageFeature extends Feature[TweetCandidate, Boolean]
object HasVideoFeature extends Feature[TweetCandidate, Boolean]
// Tweetypie VF Features
object IsHydratedFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Boolean]
object IsNsfwFeature extends Feature[TweetCandidate, Boolean]
object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean]
// Raw Tweet Text from Tweetypie
object TweetTextFeature extends Feature[TweetCandidate, Option[String]]
// SGS Features
/**
* By convention, this is set to true for retweets of non-followed authors
* E.g. where somebody the viewer follows retweets a Tweet from somebody the viewer doesn't follow
*/
object InNetworkFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Boolean]
object AccountAgeFeature extends Feature[PipelineQuery, Option[Time]]
object ClientIdFeature
extends Feature[PipelineQuery, Option[Long]]
with LongDiscreteDataRecordCompatible
def featureName: String = SharedFeatures.CLIENT_ID.getFeatureName
override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.ClientType)
}
object CachedScoredTweetsFeature extends Feature[PipelineQuery, Seq[hmt.CachedScoredTweet]]
object DeviceLanguageFeature extends Feature[PipelineQuery, Option[String]]
object DismissInfoFeature
extends FeatureWithDefaultOnFailure[PipelineQuery, Map[st.SuggestType, Option[DismissInfo]]]
def defaultValue: Map[st.SuggestType, Option[DismissInfo]] = Map.empty
}
object FollowingLastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]]
object GetInitialFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible
def featureName: String = RequestContextFeatures.IS_GET_INITIAL.getFeatureName
override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty
}
object GetMiddleFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible
def featureName: String = RequestContextFeatures.IS_GET_MIDDLE.getFeatureName
override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty
}
object GetNewerFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible
def featureName: String = RequestContextFeatures.IS_GET_NEWER.getFeatureName
override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty
}
object GetOlderFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible
def featureName: String = RequestContextFeatures.IS_GET_OLDER.getFeatureName
override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty
}
object GuestIdFeature
extends Feature[PipelineQuery, Option[Long]]
with LongDiscreteDataRecordCompatible
def featureName: String = SharedFeatures.GUEST_ID.getFeatureName
override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.GuestId)
}
object HasDarkRequestFeature extends Feature[TweetCandidate, Option[Boolean]]
object ImpressionBloomFilterFeature
extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq]
def defaultValue: blm.ImpressionBloomFilterSeq =
blm.ImpressionBloomFilterSeq(Seq.empty)
}
object IsForegroundRequestFeature extends Feature[PipelineQuery, Boolean]
object IsLaunchRequestFeature extends Feature[PipelineQuery, Boolean]
object LastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]]
object NonPollingTimesFeature extends Feature[PipelineQuery, Seq[Long]]
object PersistenceEntriesFeature extends Feature[PipelineQuery, Seq[TimelineResponseV3]]
object PollingFeature extends Feature[PipelineQuery, Boolean]
object PullToRefreshFeature extends Feature[PipelineQuery, Boolean]
// Scores from Real Graph representing the relationship between the viewer and another user
object RealGraphInNetworkScoresFeature extends Feature[PipelineQuery, Map[UserId, Double]]
object RequestJoinIdFeature extends Feature[TweetCandidate, Option[Long]]
// Internal id generated per request, mainly to deduplicate re-served cached tweets in logging
object ServedRequestIdFeature extends Feature[PipelineQuery, Option[Long]]
object ServedTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]]
object TweetImpressionsFeature extends Feature[PipelineQuery, Seq[imp.TweetImpressionsEntry]]
object UserFollowedTopicsCountFeature extends Feature[PipelineQuery, Option[Int]]
object UserFollowingCountFeature extends Feature[PipelineQuery, Option[Int]]
object UserScreenNameFeature extends Feature[PipelineQuery, Option[String]]
object UserStateFeature extends Feature[PipelineQuery, Option[um.UserState]]
object UserTypeFeature extends Feature[PipelineQuery, Option[gt.UserType]]
object WhoToFollowExcludedUserIdsFeature
extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\request\DeviceContext.scala
class DeviceContext(
isPolling: Option[Boolean],
requestContext: Option[String],
latestControlAvailable: Option[Boolean],
autoplayEnabled: Option[Boolean])
def toTimelineServiceDeviceContext(clientContext: ClientContext): tls.DeviceContext =
tls.DeviceContext(
countryCode = clientContext.countryCode,
languageCode = clientContext.languageCode,
clientAppId = clientContext.appId,
ipAddress = clientContext.ipAddress,
guestId = clientContext.guestId,
sessionId = None,
timezone = None,
userAgent = clientContext.userAgent,
deviceId = clientContext.deviceId,
isPolling = isPolling,
requestProvenance = requestContext,
referrer = None,
tfeAuthHeader = None,
mobileDeviceId = clientContext.mobileDeviceId,
isSessionStart = None,
displaySize = None,
isURTRequest = Some(true),
latestControlAvailable = latestControlAvailable,
guestIdMarketing = clientContext.guestIdMarketing,
isInternalOrTwoffice = clientContext.isTwoffice,
browserNotificationPermission = None,
guestIdAds = clientContext.guestIdAds,
)
}
object DeviceContext
object RequestContext extends Enumeration
trait HasDeviceContext
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\request\HasListId.scala
trait HasListId
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\request\HasSeenTweetIds.scala
trait HasSeenTweetIds
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\request\HomeMixerProduct.scala
object FollowingProduct extends Product
object ForYouProduct extends Product
object ScoredTweetsProduct extends Product
object ListTweetsProduct extends Product
object ListRecommendedUsersProduct extends Product
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\AdvertiserBrandSafetySettingsStoreModule.scala
object AdvertiserBrandSafetySettingsStoreModule extends TwitterModule
def providesAdvertiserBrandSafetySettingsStore(
injectedServiceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): ReadableStore[Long, ads.AdvertiserBrandSafetySettings]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ClientSentImpressionsPublisherModule.scala
object ClientSentImpressionsPublisherModule extends TwitterModule with ConfigUtils
def providesClientSentImpressionsPublisher(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): EventBusPublisher[PublishedImpressionList]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ConversationServiceModule.scala
object ConversationServiceModule
extends ThriftMethodBuilderClientModule[
ConversationService.ServicePerEndpoint,
ConversationService.MethodPerEndpoint
]
with MtlsClient
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\FeedbackHistoryClientModule.scala
object FeedbackHistoryClientModule extends TwitterModule
def providesFeedbackHistoryClient(
serviceId: ServiceIdentifier,
statsReceiver: StatsReceiver
)
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\HomeAdsCandidateSourceModule.scala
object HomeAdsCandidateSourceModule
extends ThriftMethodBuilderClientModule[
NewAdServer.ServicePerEndpoint,
NewAdServer.MethodPerEndpoint
]
with MtlsClient
def configureMethodBuilder(
injector: Injector,
methodBuilder: MethodBuilder
): MethodBuilder
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\HomeMixerFlagsModule.scala
object HomeMixerFlagsModule extends TwitterModule
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\HomeMixerResourcesModule.scala
object HomeMixerResourcesModule extends TwitterModule
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\HomeNaviModelClientModule.scala
object HomeNaviModelClientModule extends TwitterModule
def providesPredictionGRPCService(
serviceIdentifier: ServiceIdentifier,
): PredictionGRPCService
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ImpressionBloomFilterModule.scala
object ImpressionBloomFilterModule extends TwitterModule
def providesImpressionBloomFilter(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): ImpressionBloomFilter
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\InjectionHistoryClientModule.scala
object InjectionHistoryClientModule extends TwitterModule
def providesInjectionHistoryClient(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
)
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ManhattanClientsModule.scala
object ManhattanClientsModule extends TwitterModule with ConfigUtils
def providesRealGraphManhattanEndpoint(
serviceIdentifier: ServiceIdentifier
): ManhattanKVEndpoint
def providesUserMetadataManhattanEndpoint(
serviceIdentifier: ServiceIdentifier
): ManhattanKVEndpoint
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ManhattanFeatureRepositoryModule.scala
object ManhattanFeatureRepositoryModule extends TwitterModule
def to(userId: Long): Try[ByteBuffer]
def from(b: ByteBuffer): Try[Long] = ???
}
val FloatTensorTransformer = new Transformer[ByteBuffer, FloatTensor]
def to(input: ByteBuffer): Try[FloatTensor]
def from(b: FloatTensor): Try[ByteBuffer] = ???
}
// manhattan clients
@Provides
@Singleton
@Named(ManhattanApolloClient)
def providesManhattanApolloClient(
serviceIdentifier: ServiceIdentifier
): mh.ManhattanCoordinator.MethodPerEndpoint
def providesManhattanAthenaClient(
serviceIdentifier: ServiceIdentifier
): mh.ManhattanCoordinator.MethodPerEndpoint
def providesManhattanOmegaClient(
serviceIdentifier: ServiceIdentifier
): mh.ManhattanCoordinator.MethodPerEndpoint
def providesManhattanStarbuckClient(
serviceIdentifier: ServiceIdentifier
): mh.ManhattanCoordinator.MethodPerEndpoint
def providesMetricCenterUserCountingFeatureRepository(
@Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint
): KeyValueRepository[Seq[Long], Long, rf.MCUserCountingFeatures]
def providesTimelineAggregateMetadataRepository(
@Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint
): Repository[Int, Option[DenseFeatureMetadata]]
def to(buffer: ByteBuffer): Try[DenseFeatureMetadata] = Try
def from(metadata: DenseFeatureMetadata): Try[ByteBuffer] = ???
}
val inProcessCache: Cache[Int, Cached[DenseFeatureMetadata]] = InProcessLruCacheFactory(
ttl = Duration.fromMinutes(20),
lruSize = 30
).apply(serializer = Transformer(_ => ???, _ => ???)) // Serialization is not necessary here.
val keyValueRepository = new ManhattanKeyValueRepository(
client = client,
keyTransformer = keyTransformer,
valueTransformer = valueTransformer,
appId = "timelines_dense_aggregates_encoding_metadata", // Expected QPS is negligible.
dataset = "user_session_dense_feature_metadata",
timeoutInMillis = 100
)
KeyValueRepository
.singular(
new CachingKeyValueRepository[Seq[Int], Int, DenseFeatureMetadata](
keyValueRepository,
new NonLockingCache(inProcessCache),
keysAsQuery[Int]
)
)
}
@Provides
@Singleton
@Named(RealGraphFeatureRepository)
def providesRealGraphFeatureRepository(
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint
): Repository[Long, Option[UserSession]]
def providesAuthorFeatureRepository(
@Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint,
@Named(HomeAuthorFeaturesCacheClient) cacheClient: Memcache
): KeyValueRepository[Seq[Long], Long, af.AuthorFeatures]
def providesTwhinAuthorFollow20200101FeatureRepository(
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint,
@Named(TwhinAuthorFollow20200101FeatureCacheClient) cacheClient: Memcache
): KeyValueRepository[Seq[Long], Long, ml.Embedding]
def providesTwhinUserFollowFeatureRepository(
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint
): KeyValueRepository[Seq[Long], Long, FloatTensor]
def providesTimelineAggregatePartARepository(
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint,
): Repository[Long, Option[uss.UserSession]] =
timelineAggregateRepository(
mhClient = client,
mhDataset = "timelines_aggregates_v2_features_by_user_part_a_apollo",
mhAppId = "timelines_aggregates_v2_features_by_user_part_a_apollo"
)
@Provides
@Singleton
@Named(TimelineAggregatePartBRepository)
def providesTimelineAggregatePartBRepository(
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint,
): Repository[Long, Option[uss.UserSession]] =
timelineAggregateRepository(
mhClient = client,
mhDataset = "timelines_aggregates_v2_features_by_user_part_b_apollo",
mhAppId = "timelines_aggregates_v2_features_by_user_part_b_apollo"
)
@Provides
@Singleton
@Named(TwhinUserEngagementFeatureRepository)
def providesTwhinUserEngagementFeatureRepository(
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint
): KeyValueRepository[Seq[Long], Long, FloatTensor]
def buildMemCachedRepository[K, V](
keyValueRepository: KeyValueRepository[Seq[K], K, V],
cacheClient: Memcache,
cachePrefix: String,
ttl: Duration,
valueInjection: Injection[V, Array[Byte]]
): CachingKeyValueRepository[Seq[K], K, V]
def buildInProcessCachedRepository[K, V](
keyValueRepository: KeyValueRepository[Seq[K], K, V],
ttl: Duration,
size: Int,
valueInjection: Injection[V, Array[Byte]]
): CachingKeyValueRepository[Seq[K], K, V]
def batchedManhattanKeyValueRepository[K, V](
client: ManhattanCoordinator.MethodPerEndpoint,
keyTransformer: Transformer[K, ByteBuffer],
valueTransformer: Transformer[ByteBuffer, V],
appId: String,
dataset: String,
timeoutInMillis: Int,
chunkSize: Int = DEFAULT_RPC_CHUNK_SIZE
): KeyValueRepository[Seq[K], K, V] =
KeyValueRepository.chunked(
new ManhattanKeyValueRepository(
client = client,
keyTransformer = keyTransformer,
valueTransformer = valueTransformer,
appId = appId,
dataset = dataset,
timeoutInMillis = timeoutInMillis
),
chunker = ChunkingStrategy.equalSize(chunkSize)
)
private def transportFromByteBuffer(buffer: ByteBuffer): TTransport =
new TMemoryInputTransport(
buffer.array(),
buffer.arrayOffset() + buffer.position(),
buffer.remaining())
private def timelineAggregateRepository(
mhClient: mh.ManhattanCoordinator.MethodPerEndpoint,
mhDataset: String,
mhAppId: String
): Repository[Long, Option[uss.UserSession]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ManhattanTweetImpressionStoreModule.scala
object ManhattanTweetImpressionStoreModule extends TwitterModule
def providesManhattanTweetImpressionStoreClient(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): ManhattanTweetImpressionStoreClient
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\MemcachedFeatureRepositoryModule.scala
object MemcachedFeatureRepositoryModule extends TwitterModule
def providesTimelinesRealTimeAggregateClient(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Memcache
def providesHomeAuthorFeaturesCacheClient(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Memcache
def providesTwhinAuthorFollow20200101FeatureCacheClient(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Memcache
def providesRealTimeInteractionGraphUserVertexClient(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Memcache
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\OptimizedStratoClientModule.scala
object OptimizedStratoClientModule extends TwitterModule
def mkRetryPolicy(tries: Int): RetryPolicy[Try[Nothing]] =
RetryPolicy.tries(tries, DefaultRetryPartialFunction)
@Singleton
@Provides
@Named(BatchedStratoClientWithModerateTimeout)
def providesStratoClient(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): Client
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\PeopleDiscoveryServiceModule.scala
object PeopleDiscoveryServiceModule
extends ThriftMethodBuilderClientModule[
ThriftPeopleDiscoveryService.ServicePerEndpoint,
ThriftPeopleDiscoveryService.MethodPerEndpoint
]
with MtlsClient
def configureMethodBuilder(
injector: Injector,
methodBuilder: MethodBuilder
): MethodBuilder
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\PipelineFailureExceptionMapper.scala
class PipelineFailureExceptionMapper
extends ExceptionMapper[PipelineFailure, ThriftException]
with Logging
def handleException(throwable: PipelineFailure): Future[ThriftException]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\RealGraphInNetworkScoresModule.scala
object RealGraphInNetworkScoresModule extends TwitterModule
def providesRealGraphInNetworkScoresFeaturesStore(
@Named(RealGraphManhattanEndpoint) realGraphInNetworkScoresManhattanKVEndpoint: ManhattanKVEndpoint
): ReadableStore[ViewerId, Seq[Candidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\RealtimeAggregateFeatureRepositoryModule.scala
object RealtimeAggregateFeatureRepositoryModule
extends TwitterModule
with RealtimeAggregateHelpers
def providesUserTopicEngagementForNewUserCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[(Long, Long), ml.DataRecord]
def providesTwitterListEngagementCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[Long, ml.DataRecord]
def providesTopicEngagementCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[Long, ml.DataRecord]
def providesUserAuthorEngagementCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[(Long, Long), ml.DataRecord]
def providesUserEngagementCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[Long, ml.DataRecord]
def providesTweetCountryEngagementCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[(Long, String), ml.DataRecord]
def providesTweetEngagementCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[Long, ml.DataRecord]
def providesEngagementsReceivedByAuthorCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[Long, ml.DataRecord]
def providesTopicCountryEngagementCache(
@Named(TimelinesRealTimeAggregateClient) client: Memcache
): ReadCache[(Long, String), ml.DataRecord]
def providesRealTimeInteractionGraphUserVertexCache(
@Named(RealTimeInteractionGraphUserVertexClient) client: Memcache
): ReadCache[Long, ig.UserVertex]
trait RealtimeAggregateHelpers
def customKeyBuilder[K](prefix: String, f: K => Array[Byte]): K => String
def g(arr: Array[Byte]) = ???
MemcacheHelper.keyEncoder(prefix)(Injection.build(f)(g))
}
private val keyEncoder: AggregationKey => String
def keyTransformD1(f1: Feature.Discrete)(key: Long): String
def keyTransformD2(
f1: Feature.Discrete,
f2: Feature.Discrete
)(
keys: (Long, Long)
): String
def keyTransformD1T1(
f1: Feature.Discrete,
f2: Feature.Text
)(
keys: (Long, String)
): String
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ScoredTweetsMemcacheModule.scala
object ScoredTweetsMemcacheModule extends TwitterModule
def providesScoredTweetsCache(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): TtlCache[UserId, t.CachedScoredTweets]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ScribeEventPublisherModule.scala
object ScribeEventPublisherModule extends TwitterModule
def providesClientEventsScribeEventPublisher(
@Flag(ScribeClientEventsFlag) sendToScribe: Boolean
): EventPublisher[ca.LogEvent]
def providesCommonFeaturesScribeEventPublisher(
@Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean
): EventPublisher[pldr.PolyDataRecord]
def providesCandidateFeaturesScribeEventPublisher(
@Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean
): EventPublisher[pldr.PolyDataRecord]
def providesMinimumFeaturesScribeEventPublisher(
@Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean
): EventPublisher[pldr.PolyDataRecord]
def providesServedEntriesScribeEventPublisher(
@Flag(ScribeServedEntriesFlag) sendToScribe: Boolean
): EventPublisher[tl.Timeline]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\SimClustersRecentEngagementsClientModule.scala
object SimClustersRecentEngagementsClientModule extends TwitterModule
def providesSimilarityClient(
@Named(BatchedStratoClientWithModerateTimeout)
stratoClient: Client,
statsReceiver: StatsReceiver
): SimClustersRecentEngagementSimilarityClient
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\StaleTweetsCacheModule.scala
object StaleTweetsCacheModule extends TwitterModule
def providesCache(
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): MemcachedClient
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ThriftFeatureRepositoryModule.scala
object ThriftFeatureRepositoryModule extends TwitterModule
def providesInterestsThriftServiceClient(
clientId: ClientId,
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): int.InterestsThriftService.MethodPerEndpoint
def providesUserFollowedTopicIdsRepository(
@Named(InterestsThriftServiceClient) client: int.InterestsThriftService.MethodPerEndpoint
): KeyValueRepository[Seq[Long], Long, Seq[Long]]
def lookup(userId: Long): Future[Seq[Long]]
def providesUtegSocialProofRepository(
clientId: ClientId,
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): KeyValueRepository[UtegQuery, Long, uteg.TweetRecommendation]
def lookup(
tweetIds: Seq[Long],
view: (Long, Map[Long, Double])
): Future[Seq[Option[uteg.TweetRecommendation]]]
def providesTweetypieContentRepository(
clientId: ClientId,
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): KeyValueRepository[Seq[Long], Long, tp.Tweet]
def lookup(tweetIds: Seq[Long]): Future[Seq[Option[tp.Tweet]]]
def providesGraphTwoHopRepository(
clientId: ClientId,
serviceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): KeyValueRepository[(Seq[Long], Long), Long, Seq[gfs.IntersectionValue]]
def lookup(
userIds: Seq[Long],
viewerId: Long
): Future[Seq[Option[Seq[gfs.IntersectionValue]]]]
def providesEarlybirdSearchRepository(
client: eb.EarlybirdService.MethodPerEndpoint,
clientId: ClientId
): KeyValueRepository[EarlybirdQuery, Long, eb.ThriftSearchResult]
def lookup(
tweetIds: Seq[Long],
viewerId: Long
): Future[Seq[Option[eb.ThriftSearchResult]]]
def toRepository[K, V](
hydrate: K => Future[V]
): KeyValueRepository[Seq[K], K, V]
def asRepository(keys: Seq[K]): Future[KeyValueResult[K, V]]
def toRepositoryBatch[K, V](
hydrate: Seq[K] => Future[Seq[Option[V]]],
chunkSize: Int = DefaultRPCChunkSize
): KeyValueRepository[Seq[K], K, V]
def repository(keys: Seq[K]): Future[KeyValueResult[K, V]] =
batchRepositoryProcess(keys, hydrate(keys))
KeyValueRepository.chunked(repository, ChunkingStrategy.equalSize(chunkSize))
}
protected def toRepositoryBatchWithView[K, T, V](
hydrate: (Seq[K], T) => Future[Seq[Option[V]]],
chunkSize: Int = DefaultRPCChunkSize
): KeyValueRepository[(Seq[K], T), K, V]
def repository(input: (Seq[K], T)): Future[KeyValueResult[K, V]]
def batchRepositoryProcess[K, V](
keys: Seq[K],
f: Future[Seq[Option[V]]]
): Future[KeyValueResult[K, V]]
object CustomChunkingStrategy
def equalSizeWithView[K, T](maxSize: Int): ((Seq[K], T)) => Seq[(Seq[K], T)]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\TimelinesPersistenceStoreClientModule.scala
object TimelinesPersistenceStoreClientModule extends TwitterModule
def providesTimelinesPersistenceStoreClient(
injectedServiceIdentifier: ServiceIdentifier,
statsReceiver: StatsReceiver
): TimelineResponseBatchesClient[TimelineResponseV3]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\TweetyPieClientModule.scala
object TweetyPieClientModule
extends ThriftMethodBuilderClientModule[
TweetService.ServicePerEndpoint,
TweetService.MethodPerEndpoint
]
with MtlsClient
def providesTweetypieStitchClient(tweetService: TweetService.MethodPerEndpoint): TweetyPie =
new TweetyPie(tweetService)
/**
* TweetyPie client id must be in the form of
def clientId(injector: Injector): ClientId
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\TweetypieStaticEntitiesCacheClientModule.scala
object TweetypieStaticEntitiesCacheClientModule extends TwitterModule
def providesTweetypieStaticEntitiesCache(
statsReceiver: StatsReceiver,
serviceIdentifier: ServiceIdentifier
): TtlCache[Long, tp.Tweet]
def mkCache(
finagleMemcache: FinagleMemcache,
statsReceiver: StatsReceiver
): TtlCache[Long, tp.Tweet]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\UserMetadataStoreModule.scala
object UserMetadataStoreModule extends TwitterModule
def providesUserLanguagesFeaturesStore(
@Named(UserMetadataManhattanEndpoint) UserMetadataManhattanKVEndpoint: ManhattanKVEndpoint,
statsReceiver: StatsReceiver
): ReadableStore[Long, Seq[scc.ThriftLanguage]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\GlobalParamConfigModule.scala
object GlobalParamConfigModule extends TwitterModule
def configure(): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\HomeGlobalParamConfig.scala
class HomeGlobalParamConfig @Inject() () extends GlobalParamConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\HomeGlobalParams.scala
object HomeGlobalParams
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\HomeMixerFlagName.scala
object HomeMixerFlagName
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\HomeMixerInjectionNames.scala
object HomeMixerInjectionNames
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\decider\DeciderKey.scala
object DeciderKey extends DeciderKeyEnum
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\HomeMixerProductModule.scala
object HomeMixerProductModule extends TwitterModule
def configure(): Unit
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\HomeProductPipelineRegistryConfig.scala
class HomeProductPipelineRegistryConfig @Inject() (
injector: Injector,
productScope: ProductScope)
extends ProductPipelineRegistryConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingAdsCandidatePipelineBuilder.scala
class FollowingAdsCandidatePipelineBuilder @Inject() (
adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder,
adsCandidateSource: AdsProdThriftCandidateSource,
advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[
FollowingQuery,
AdsCandidate
])
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingEarlybirdCandidatePipelineConfig.scala
class FollowingEarlybirdCandidatePipelineConfig @Inject() (
earlybirdCandidateSource: EarlybirdCandidateSource,
followingEarlybirdQueryTransformer: FollowingEarlybirdQueryTransformer)
extends CandidatePipelineConfig[
FollowingQuery,
t.EarlybirdRequest,
t.ThriftSearchResult,
TweetCandidate
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingEarlybirdQueryTransformer.scala
class FollowingEarlybirdQueryTransformer @Inject() (clientId: ClientId)
extends CandidatePipelineQueryTransformer[FollowingQuery, t.EarlybirdRequest]
def transform(query: FollowingQuery): t.EarlybirdRequest
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingEarlybirdResponseFeatureTransformer.scala
object FollowingEarlybirdResponseFeatureTransformer
extends CandidateFeatureTransformer[t.ThriftSearchResult]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingMixerPipelineConfig.scala
class FollowingMixerPipelineConfig @Inject() (
followingEarlybirdCandidatePipelineConfig: FollowingEarlybirdCandidatePipelineConfig,
conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[
FollowingQuery
],
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder,
followingAdsCandidatePipelineBuilder: FollowingAdsCandidatePipelineBuilder,
followingWhoToFollowArmCandidatePipelineConfigBuilder: FollowingWhoToFollowArmCandidatePipelineConfigBuilder,
flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder,
editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig,
newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery],
dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator,
gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator,
persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator,
realGraphInNetworkSourceQueryHydrator: RealGraphInNetworkScoresQueryFeatureHydrator,
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery],
sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator,
tweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[FollowingQuery],
lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator,
adsInjector: AdsInjector,
updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline],
publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect,
publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect,
updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect,
truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect,
homeTimelineServedEntriesSideEffect: HomeScribeServedEntriesSideEffect,
clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent],
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter],
urtTransportMarshaller: UrtTransportMarshaller)
extends MixerPipelineConfig[FollowingQuery, Timeline, urt.TimelineResponse]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingProductPipelineConfig.scala
class FollowingProductPipelineConfig @Inject() (
followingMixerPipelineConfig: FollowingMixerPipelineConfig,
followingParamConfig: FollowingParamConfig)
extends ProductPipelineConfig[HomeMixerRequest, FollowingQuery, urt.TimelineResponse]
def pipelineQueryTransformer(
request: HomeMixerRequest,
params: Params
): FollowingQuery
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingWhoToFollowArmCandidatePipelineConfigBuilder.scala
class FollowingWhoToFollowArmCandidatePipelineConfigBuilder @Inject() (
whoToFollowArmDependentCandidatePipelineConfigBuilder: WhoToFollowArmDependentCandidatePipelineConfigBuilder,
homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder)
def build(
requiredNonEmptyPipelines: CandidateScope
): WhoToFollowArmDependentCandidatePipelineConfig[FollowingQuery]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\model\FollowingQuery.scala
class FollowingQuery(
override val params: Params,
override val clientContext: ClientContext,
override val pipelineCursor: Option[UrtOrderedCursor],
override val requestedMaxResults: Option[Int],
override val debugOptions: Option[DebugOptions],
override val features: Option[FeatureMap],
override val deviceContext: Option[DeviceContext],
override val seenTweetIds: Option[Seq[Long]],
override val dspClientContext: Option[dsp.DspClientContext])
extends PipelineQuery
with HasPipelineCursor[UrtOrderedCursor]
with HasDeviceContext
with HasSeenTweetIds
with HasFlipInjectionParams
with HomeAdsQuery
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\model\HomeMixerExternalStrings.scala
class HomeMixerExternalStrings @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry])
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\param\FollowingParam.scala
object FollowingParam
object ServerMaxResultsParam
extends FSBoundedParam[Int](
name = "following_server_max_results",
default = 100,
min = 1,
max = 500
)
object EnableWhoToFollowCandidatePipelineParam
extends FSParam[Boolean](
name = "following_enable_who_to_follow",
default = true
)
object EnableAdsCandidatePipelineParam
extends FSParam[Boolean](
name = "following_enable_ads",
default = true
)
object EnableFlipInjectionModuleCandidatePipelineParam
extends FSParam[Boolean](
name = "following_enable_flip_inline_injection_module",
default = true
)
object FlipInlineInjectionModulePosition
extends FSBoundedParam[Int](
name = "following_flip_inline_injection_module_position",
default = 0,
min = 0,
max = 1000
)
object WhoToFollowPositionParam
extends FSBoundedParam[Int](
name = "following_who_to_follow_position",
default = 5,
min = 0,
max = 99
)
object WhoToFollowMinInjectionIntervalParam
extends FSBoundedParam[Duration](
"following_who_to_follow_min_injection_interval_in_minutes",
default = 1800.minutes,
min = 0.minutes,
max = 6000.minutes)
with HasDurationConversion
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\param\FollowingParamConfig.scala
class FollowingParamConfig @Inject() () extends ProductParamConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouAdsCandidatePipelineBuilder.scala
class ForYouAdsCandidatePipelineBuilder @Inject() (
adsCandidatePipelineConfigBuilder: AdsCandidatePipelineConfigBuilder,
adsCandidateSource: AdsProdThriftCandidateSource,
advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[
ForYouQuery,
AdsCandidate
])
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouConversationServiceCandidatePipelineConfig.scala
class ForYouConversationServiceCandidatePipelineConfig @Inject() (
forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig,
forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig,
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator,
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder)
extends DependentCandidatePipelineConfig[
ForYouQuery,
ConversationServiceCandidateSourceRequest,
TweetWithConversationMetadata,
TweetCandidate
]
def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq(
PreviouslyServedTweetsFilter,
PreviouslySeenTweetsFilter,
RetweetDeduplicationFilter,
FeatureFilter.fromFeature(FilterIdentifier("TweetypieHydrated"), IsHydratedFeature),
PredicateFeatureFilter.fromPredicate(
FilterIdentifier("QuotedTweetDropped"),
shouldKeepCandidate
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouProductPipelineConfig.scala
class ForYouProductPipelineConfig @Inject() (
forYouTimelineScorerMixerPipelineConfig: ForYouTimelineScorerMixerPipelineConfig,
forYouScoredTweetsMixerPipelineConfig: ForYouScoredTweetsMixerPipelineConfig,
forYouParamConfig: ForYouParamConfig)
extends ProductPipelineConfig[HomeMixerRequest, ForYouQuery, urt.TimelineResponse]
def pipelineQueryTransformer(
request: HomeMixerRequest,
params: Params
): ForYouQuery
def pipelineSelector(
query: ForYouQuery
): ComponentIdentifier
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouScoredTweetsCandidatePipelineConfig.scala
class ForYouScoredTweetsCandidatePipelineConfig @Inject() (
scoredTweetsProductCandidateSource: ScoredTweetsProductCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator,
sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator,
perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator,
focalTweetFeatureHydrator: FocalTweetFeatureHydrator,
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder,
homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder)
extends CandidatePipelineConfig[
ForYouQuery,
ForYouQuery,
ScoredTweetWithConversationMetadata,
TweetCandidate
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouScoredTweetsMixerPipelineConfig.scala
class ForYouScoredTweetsMixerPipelineConfig @Inject() (
forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig,
forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig,
forYouAdsCandidatePipelineBuilder: ForYouAdsCandidatePipelineBuilder,
forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder,
flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder,
editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig,
newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery],
dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator,
gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator,
persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator,
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery],
feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator,
timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator,
adsInjector: AdsInjector,
servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder,
servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder,
updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect,
truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect,
homeScribeServedEntriesSideEffect: HomeScribeServedEntriesSideEffect,
servedStatsSideEffect: ServedStatsSideEffect,
clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent],
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter],
urtTransportMarshaller: UrtTransportMarshaller)
extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouScoredTweetsResponseFeatureTransformer.scala
object ForYouScoredTweetsResponseFeatureTransformer
extends CandidateFeatureTransformer[ScoredTweetWithConversationMetadata]
def transform(input: ScoredTweetWithConversationMetadata): FeatureMap =
FeatureMapBuilder()
.add(AncestorsFeature, input.ancestors.getOrElse(Seq.empty))
.add(AuthorIdFeature, Some(input.authorId))
.add(ConversationModuleIdFeature, input.conversationId)
.add(ConversationModuleFocalTweetIdFeature, input.conversationFocalTweetId)
.add(DirectedAtUserIdFeature, input.directedAtUserId)
.add(FavoritedByUserIdsFeature, input.favoritedByUserIds.getOrElse(Seq.empty))
.add(FollowedByUserIdsFeature, input.followedByUserIds.getOrElse(Seq.empty))
.add(InNetworkFeature, input.inNetwork.getOrElse(false))
.add(InReplyToTweetIdFeature, input.inReplyToTweetId)
.add(InReplyToUserIdFeature, input.inReplyToUserId)
.add(IsReadFromCacheFeature, input.isReadFromCache.getOrElse(false))
.add(IsRetweetFeature, input.sourceTweetId.isDefined)
.add(QuotedTweetIdFeature, input.quotedTweetId)
.add(QuotedUserIdFeature, input.quotedUserId)
.add(ScoreFeature, input.score)
.add(SourceTweetIdFeature, input.sourceTweetId)
.add(SourceUserIdFeature, input.sourceUserId)
.add(StreamToKafkaFeature, input.streamToKafka.getOrElse(false))
.add(SuggestTypeFeature, input.suggestType)
.add(
TopicContextFunctionalityTypeFeature,
input.topicFunctionalityType.collect
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouTimelineScorerCandidatePipelineConfig.scala
class ForYouTimelineScorerCandidatePipelineConfig @Inject() (
timelineScorerCandidateSource: TimelineScorerCandidateSource,
deviceContextMarshaller: DeviceContextMarshaller,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
sgsFeatureHydrator: SocialGraphServiceFeatureHydrator,
sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator,
perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator,
focalTweetFeatureHydrator: FocalTweetFeatureHydrator,
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder,
homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder)
extends CandidatePipelineConfig[
ForYouQuery,
t.ScoredTweetsRequest,
ScoredTweetCandidateWithFocalTweet,
TweetCandidate
]
def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq(
RetweetDeduplicationFilter,
FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature),
PredicateFeatureFilter.fromPredicate(
FilterIdentifier(QuotedTweetDroppedFilterId),
shouldKeepCandidate
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouTimelineScorerMixerPipelineConfig.scala
class ForYouTimelineScorerMixerPipelineConfig @Inject() (
forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig,
forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig,
forYouAdsCandidatePipelineBuilder: ForYouAdsCandidatePipelineBuilder,
forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder,
flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder,
editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig,
newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery],
dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator,
gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator,
manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ForYouQuery],
memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator,
persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator,
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery],
timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator,
lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator,
feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator,
adsInjector: AdsInjector,
servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder,
servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder,
updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ForYouQuery, Timeline],
publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect,
publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect,
updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect,
truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect,
homeScribeServedEntriesSideEffect: HomeScribeServedEntriesSideEffect,
servedStatsSideEffect: ServedStatsSideEffect,
clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent],
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter],
urtTransportMarshaller: UrtTransportMarshaller)
extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse]
def fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq(
requestQueryFeatureHydrator,
persistenceStoreQueryFeatureHydrator,
timelineServiceTweetsQueryFeatureHydrator,
feedbackHistoryQueryFeatureHydrator,
AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator),
AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator),
AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator),
AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator),
AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator)
)
private val forYouAdsCandidatePipelineConfig = forYouAdsCandidatePipelineBuilder.build()
private val forYouWhoToFollowCandidatePipelineConfig =
forYouWhoToFollowCandidatePipelineConfigBuilder.build()
private val flipPromptCandidatePipelineConfig =
flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery](
supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam)
)
override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq(
forYouTimelineScorerCandidatePipelineConfig,
forYouAdsCandidatePipelineConfig,
forYouWhoToFollowCandidatePipelineConfig,
flipPromptCandidatePipelineConfig
)
override val dependentCandidatePipelines: Seq[
DependentCandidatePipelineConfig[ForYouQuery, _, _, _]
] = Seq(
forYouConversationServiceCandidatePipelineConfig,
editedTweetsCandidatePipelineConfig,
newTweetsPillCandidatePipelineConfig
)
override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map(
forYouTimelineScorerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always,
forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always,
forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always,
flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always,
editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always,
newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always,
)
override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq(
UpdateSortCandidates(
ordering = CandidatesUtil.reverseChronTweetsOrdering,
candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier
),
UpdateSortCandidates(
ordering = CandidatesUtil.scoreOrdering,
candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier
),
UpdateSortModuleItemCandidates(
candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier,
ordering = CandidatesUtil.conversationModuleTweetsOrdering
),
DebunchCandidates(
pipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier),
mustDebunch
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouTimelineScorerResponseFeatureTransformer.scala
object ForYouTimelineScorerResponseFeatureTransformer
extends CandidateFeatureTransformer[ScoredTweetCandidateWithFocalTweet]
def getLanguageISOFormatByValue(languageCodeValue: Int): String =
ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguage.findByValue(languageCodeValue))
override def transform(
candidateWithFocalTweet: ScoredTweetCandidateWithFocalTweet
): FeatureMap
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouWhoToFollowCandidatePipelineConfigBuilder.scala
class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() (
whoToFollowCandidatePipelineConfigBuilder: WhoToFollowCandidatePipelineConfigBuilder,
homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder)
def build(): WhoToFollowCandidatePipelineConfig[ForYouQuery]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\candidate_source\ScoredTweetsProductCandidateSource.scala
class ScoredTweetWithConversationMetadata(
tweetId: Long,
authorId: Long,
score: Option[Double] = None,
suggestType: Option[st.SuggestType] = None,
sourceTweetId: Option[Long] = None,
sourceUserId: Option[Long] = None,
quotedTweetId: Option[Long] = None,
quotedUserId: Option[Long] = None,
inReplyToTweetId: Option[Long] = None,
inReplyToUserId: Option[Long] = None,
directedAtUserId: Option[Long] = None,
inNetwork: Option[Boolean] = None,
favoritedByUserIds: Option[Seq[Long]] = None,
followedByUserIds: Option[Seq[Long]] = None,
ancestors: Option[Seq[ta.TweetAncestor]] = None,
topicId: Option[Long] = None,
topicFunctionalityType: Option[tl.TopicContextFunctionalityType] = None,
conversationId: Option[Long] = None,
conversationFocalTweetId: Option[Long] = None,
isReadFromCache: Option[Boolean] = None,
streamToKafka: Option[Boolean] = None)
@Singleton
class ScoredTweetsProductCandidateSource @Inject() (
override val productPipelineRegistry: Provider[ProductPipelineRegistry],
override val paramsBuilder: Provider[ParamsBuilder])
extends ProductPipelineCandidateSource[
ForYouQuery,
HomeMixerRequest,
t.ScoredTweetsResponse,
ScoredTweetWithConversationMetadata
]
def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest
def productPipelineResultTransformer(
productPipelineResult: t.ScoredTweetsResponse
): Seq[ScoredTweetWithConversationMetadata]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\model\ForYouQuery.scala
class ForYouQuery(
override val params: Params,
override val clientContext: ClientContext,
override val pipelineCursor: Option[UrtOrderedCursor],
override val requestedMaxResults: Option[Int],
override val debugOptions: Option[DebugOptions],
override val features: Option[FeatureMap],
override val deviceContext: Option[DeviceContext],
override val seenTweetIds: Option[Seq[Long]],
override val dspClientContext: Option[dsp.DspClientContext])
extends PipelineQuery
with HasPipelineCursor[UrtOrderedCursor]
with HasDeviceContext
with HasSeenTweetIds
with HasFlipInjectionParams
with HomeAdsQuery
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\param\ForYouParam.scala
object ForYouParam
object EnableTimelineScorerCandidatePipelineParam
extends FSParam[Boolean](
name = "for_you_enable_timeline_scorer_candidate_pipeline",
default = true
)
object EnableScoredTweetsCandidatePipelineParam
extends BooleanDeciderParam(DeciderKey.EnableForYouScoredTweetsCandidatePipeline)
object EnableWhoToFollowCandidatePipelineParam
extends FSParam[Boolean](
name = "for_you_enable_who_to_follow",
default = true
)
object EnableScoredTweetsMixerPipelineParam
extends FSParam[Boolean](
name = "for_you_enable_scored_tweets_mixer_pipeline",
default = true
)
object ServerMaxResultsParam
extends FSBoundedParam[Int](
name = "for_you_server_max_results",
default = 35,
min = 1,
max = 500
)
object TimelineServiceMaxResultsParam
extends FSBoundedParam[Int](
name = "for_you_timeline_service_max_results",
default = 800,
min = 1,
max = 800
)
object AdsNumOrganicItemsParam
extends FSBoundedParam[Int](
name = "for_you_ads_num_organic_items",
default = 35,
min = 1,
max = 100
)
object WhoToFollowPositionParam
extends FSBoundedParam[Int](
name = "for_you_who_to_follow_position",
default = 5,
min = 0,
max = 99
)
object WhoToFollowMinInjectionIntervalParam
extends FSBoundedParam[Duration](
"for_you_who_to_follow_min_injection_interval_in_minutes",
default = 1800.minutes,
min = 0.minutes,
max = 6000.minutes)
with HasDurationConversion
object WhoToFollowDisplayTypeIdParam
extends FSEnumParam[WhoToFollowModuleDisplayType.type](
name = "for_you_enable_who_to_follow_display_type_id",
default = WhoToFollowModuleDisplayType.Vertical,
enum = WhoToFollowModuleDisplayType
)
object EnableFlipInjectionModuleCandidatePipelineParam
extends FSParam[Boolean](
name = "for_you_enable_flip_inline_injection_module",
default = true
)
object FlipInlineInjectionModulePosition
extends FSBoundedParam[Int](
name = "for_you_flip_inline_injection_module_position",
default = 0,
min = 0,
max = 1000
)
object ClearCacheOnPtr
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\param\ForYouParamConfig.scala
class ForYouParamConfig @Inject() () extends ProductParamConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\ListMemberBasedUsersCandidatePipelineConfig.scala
class ListMemberBasedUsersCandidatePipelineConfig @Inject() (
similarityBasedUsersCandidateSource: SimilarityBasedUsersCandidateSource,
gizmoduckUserFeatureHydrator: GizmoduckUserFeatureHydrator,
isListMemberFeatureHydrator: IsListMemberFeatureHydrator)
extends CandidatePipelineConfig[
ListRecommendedUsersQuery,
Seq[Long],
t.Candidate,
UserCandidate
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\ListMemberBasedUsersResponseFeatureTransfromer.scala
object ListMemberBasedUsersResponseFeatureTransfromer
extends CandidateFeatureTransformer[t.Candidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\ListRecommendedUsersMixerPipelineConfig.scala
class ListRecommendedUsersMixerPipelineConfig @Inject() (
listMemberBasedUsersCandidatePipelineConfig: ListMemberBasedUsersCandidatePipelineConfig,
viewerIsListOwnerGate: ViewerIsListOwnerGate,
listMembersQueryFeatureHydrator: ListMembersQueryFeatureHydrator,
urtTransportMarshaller: UrtTransportMarshaller)
extends MixerPipelineConfig[ListRecommendedUsersQuery, Timeline, urt.TimelineResponse]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\ListRecommendedUsersProductPipelineConfig.scala
class ListRecommendedUsersProductPipelineConfig @Inject() (
listRecommendedUsersMixerPipelineConfig: ListRecommendedUsersMixerPipelineConfig,
listRecommendedUsersParamConfig: ListRecommendedUsersParamConfig)
extends ProductPipelineConfig[
HomeMixerRequest,
ListRecommendedUsersQuery,
urt.TimelineResponse
]
def pipelineQueryTransformer(
request: HomeMixerRequest,
params: Params
): ListRecommendedUsersQuery
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\feature_hydrator\GizmoduckUserFeatureHydrator.scala
class GizmoduckUserFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends BulkCandidateFeatureHydrator[PipelineQuery, UserCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[UserCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\feature_hydrator\IsListMemberFeatureHydrator.scala
class IsListMemberFeatureHydrator @Inject() (socialGraph: SocialGraph)
extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate]
def apply(
query: PipelineQuery with HasListId,
candidates: Seq[CandidateWithFeatures[UserCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\filter\DropMaxCandidatesByScoreFilter.scala
object DropMaxCandidatesByScoreFilter extends Filter[PipelineQuery, UserCandidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[UserCandidate]]
): Stitch[FilterResult[UserCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\filter\PreviouslyServedUsersFilter.scala
object PreviouslyServedUsersFilter extends Filter[ListRecommendedUsersQuery, UserCandidate]
def apply(
query: ListRecommendedUsersQuery,
candidates: Seq[CandidateWithFeatures[UserCandidate]]
): Stitch[FilterResult[UserCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\model\ListFeatures.scala
object ListFeatures
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\model\ListRecommendedUsersQuery.scala
class ListRecommendedUsersQuery(
override val listId: Long,
override val params: Params,
override val clientContext: ClientContext,
override val pipelineCursor: Option[UrtUnorderedExcludeIdsCursor],
override val requestedMaxResults: Option[Int],
override val debugOptions: Option[DebugOptions],
override val features: Option[FeatureMap],
selectedUserIds: Option[Seq[Long]],
excludedUserIds: Option[Seq[Long]])
extends PipelineQuery
with HasPipelineCursor[UrtUnorderedExcludeIdsCursor]
with HasListId
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\param\ListRecommendedUsersParam.scala
object ListRecommendedUsersParam
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\param\ListRecommendedUsersParamConfig.scala
class ListRecommendedUsersParamConfig @Inject() () extends ProductParamConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\ListTweetsAdsCandidatePipelineBuilder.scala
class ListTweetsAdsCandidatePipelineBuilder @Inject() (
adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder,
adsCandidateSource: AdsProdThriftCandidateSource,
advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[
ListTweetsQuery,
AdsCandidate
])
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\ListTweetsMixerPipelineConfig.scala
class ListTweetsMixerPipelineConfig @Inject() (
listTweetsTimelineServiceCandidatePipelineConfig: ListTweetsTimelineServiceCandidatePipelineConfig,
conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[
ListTweetsQuery
],
listTweetsAdsCandidatePipelineBuilder: ListTweetsAdsCandidatePipelineBuilder,
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ListTweetsQuery],
adsInjector: AdsInjector,
clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent],
urtTransportMarshaller: UrtTransportMarshaller)
extends MixerPipelineConfig[ListTweetsQuery, Timeline, urt.TimelineResponse]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\ListTweetsProductPipelineConfig.scala
class ListTweetsProductPipelineConfig @Inject() (
listTweetsMixerPipelineConfig: ListTweetsMixerPipelineConfig,
listTweetsParamConfig: ListTweetsParamConfig)
extends ProductPipelineConfig[HomeMixerRequest, ListTweetsQuery, urt.TimelineResponse]
def pipelineQueryTransformer(
request: HomeMixerRequest,
params: Params
): ListTweetsQuery
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\ListTweetsTimelineServiceCandidatePipelineConfig.scala
class ListTweetsTimelineServiceCandidatePipelineConfig @Inject() (
timelineServiceTweetCandidateSource: TimelineServiceTweetCandidateSource)
extends CandidatePipelineConfig[ListTweetsQuery, t.TimelineQuery, t.Tweet, TweetCandidate]
def candidateSource: BaseCandidateSource[t.TimelineQuery, t.Tweet] =
timelineServiceTweetCandidateSource
override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\model\ListTweetsQuery.scala
class ListTweetsQuery(
override val params: Params,
override val clientContext: ClientContext,
override val pipelineCursor: Option[UrtOrderedCursor],
override val requestedMaxResults: Option[Int],
override val debugOptions: Option[DebugOptions],
override val features: Option[FeatureMap],
override val listId: Long,
override val deviceContext: Option[DeviceContext],
override val dspClientContext: Option[dsp.DspClientContext])
extends PipelineQuery
with HasPipelineCursor[UrtOrderedCursor]
with HasListId
with HomeAdsQuery
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\param\ListTweetsParam.scala
object ListTweetsParam
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\param\ListTweetsParamConfig.scala
class ListTweetsParamConfig @Inject() () extends ProductParamConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\ScoredTweetsProductPipelineConfig.scala
class ScoredTweetsProductPipelineConfig @Inject() (
scoredTweetsRecommendationPipelineConfig: ScoredTweetsRecommendationPipelineConfig,
scoredTweetsParamConfig: ScoredTweetsParamConfig)
extends ProductPipelineConfig[HomeMixerRequest, ScoredTweetsQuery, t.ScoredTweets]
def pipelineQueryTransformer(
request: HomeMixerRequest,
params: Params
): ScoredTweetsQuery
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\ScoredTweetsRecommendationPipelineConfig.scala
class ScoredTweetsRecommendationPipelineConfig @Inject() (
scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig,
scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig,
scoredTweetsCrMixerCandidatePipelineConfig: ScoredTweetsCrMixerCandidatePipelineConfig,
scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig,
cachedScoredTweetsCandidatePipelineConfig: CachedScoredTweetsCandidatePipelineConfig,
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ScoredTweetsQuery],
lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator,
realTimeInteractionGraphUserVertexQueryFeatureHydrator: RealTimeInteractionGraphUserVertexQueryFeatureHydrator,
userStateQueryFeatureHydrator: UserStateQueryFeatureHydrator,
userEngagementRealTimeAggregatesFeatureHydrator: UserEngagementRealTimeAggregatesFeatureHydrator,
twhinUserEngagementQueryFeatureHydrator: TwhinUserEngagementQueryFeatureHydrator,
twhinUserFollowQueryFeatureHydrator: TwhinUserFollowQueryFeatureHydrator,
cachedScoredTweetsQueryFeatureHydrator: CachedScoredTweetsQueryFeatureHydrator,
scoredTweetsScoringPipelineConfig: ScoredTweetsScoringPipelineConfig,
scoredTweetsWeightedScoresSumScoringPipelineConfig: ScoredTweetsWeightedScoresSumScoringPipelineConfig,
manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[
ScoredTweetsQuery
],
memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator,
publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect,
publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect,
realGraphInNetworkScoresQueryFeatureHydrator: RealGraphInNetworkScoresQueryFeatureHydrator,
realGraphQueryFeatureHydrator: RealGraphQueryFeatureHydrator,
userLanguagesFeatureHydrator: UserLanguagesFeatureHydrator,
partAAggregateQueryFeatureHydrator: PartAAggregateQueryFeatureHydrator,
partBAggregateQueryFeatureHydrator: PartBAggregateQueryFeatureHydrator,
cachedScoredTweetsSideEffect: CachedScoredTweetsSideEffect,
scribeServedCommonFeaturesAndCandidateFeaturesSideEffect: ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect,
updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[
ScoredTweetsQuery,
ScoredTweetsResponse
],
@Flag(TargetFetchLatency) targetFetchLatency: Duration,
@Flag(TargetScoringLatency) targetScoringLatency: Duration)
extends RecommendationPipelineConfig[
ScoredTweetsQuery,
TweetCandidate,
ScoredTweetsResponse,
t.ScoredTweetsResponse
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\CachedScoredTweetsCandidatePipelineConfig.scala
class CachedScoredTweetsCandidatePipelineConfig @Inject() (
cachedScoredTweetsCandidateSource: CachedScoredTweetsCandidateSource)
extends CandidatePipelineConfig[
ScoredTweetsQuery,
ScoredTweetsQuery,
hmt.CachedScoredTweet,
TweetCandidate
]
object CachedScoredTweetsCandidatePipelineConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\ScoredTweetsCrMixerCandidatePipelineConfig.scala
class ScoredTweetsCrMixerCandidatePipelineConfig @Inject() (
crMixerTweetRecommendationsCandidateSource: CrMixerTweetRecommendationsCandidateSource,
tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator)
extends CandidatePipelineConfig[
ScoredTweetsQuery,
t.CrMixerTweetRequest,
t.TweetRecommendation,
TweetCandidate
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\ScoredTweetsFrsCandidatePipelineConfig.scala
class ScoredTweetsFrsCandidatePipelineConfig @Inject() (
timelineRankerRecapCandidateSource: TimelineRankerRecapCandidateSource,
frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator)
extends CandidatePipelineConfig[
ScoredTweetsQuery,
tlr.RecapQuery,
tlr.CandidateTweet,
TweetCandidate
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\ScoredTweetsInNetworkCandidatePipelineConfig.scala
class ScoredTweetsInNetworkCandidatePipelineConfig @Inject() (
timelineRankerInNetworkCandidateSource: TimelineRankerInNetworkCandidateSource,
replyFeatureHydrator: ReplyFeatureHydrator)
extends CandidatePipelineConfig[
ScoredTweetsQuery,
t.RecapQuery,
t.CandidateTweet,
TweetCandidate
]
def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq(
RetweetSourceTweetRemovingFilter
)
override val postFilterFeatureHydration: Seq[
BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _]
] = Seq(replyFeatureHydrator)
override val featuresFromCandidateSourceTransformers: Seq[
CandidateFeatureTransformer[t.CandidateTweet]
] = Seq(ScoredTweetsInNetworkResponseFeatureTransformer)
override val resultTransformer: CandidatePipelineResultsTransformer[
t.CandidateTweet,
TweetCandidate
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\ScoredTweetsUtegCandidatePipelineConfig.scala
class ScoredTweetsUtegCandidatePipelineConfig @Inject() (
timelineRankerUtegCandidateSource: TimelineRankerUtegCandidateSource)
extends CandidatePipelineConfig[
ScoredTweetsQuery,
t.UtegLikedByTweetsQuery,
t.CandidateTweet,
TweetCandidate
]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_source\CachedScoredTweetsCandidateSource.scala
class CachedScoredTweetsCandidateSource @Inject() ()
extends CandidateSource[PipelineQuery, hmt.CachedScoredTweet]
def apply(request: PipelineQuery): Stitch[Seq[hmt.CachedScoredTweet]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\feature_hydrator\CachedScoredTweetsQueryFeatureHydrator.scala
class CachedScoredTweetsQueryFeatureHydrator @Inject() (
scoredTweetsCache: TtlCache[UserId, hmt.CachedScoredTweets])
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\marshaller\ScoredTweetsResponseDomainMarshaller.scala
object ScoredTweetsResponseDomainMarshaller
extends DomainMarshaller[ScoredTweetsQuery, ScoredTweetsResponse]
def apply(
query: ScoredTweetsQuery,
selections: Seq[CandidateWithDetails]
): ScoredTweetsResponse = ScoredTweetsResponse(
scoredTweets = selections.collect
def mkScoredTweet(tweetId: Long, features: FeatureMap): ScoredTweet
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\marshaller\ScoredTweetsResponseTransportMarshaller.scala
object ScoredTweetsResponseTransportMarshaller
extends TransportMarshaller[ScoredTweetsResponse, t.ScoredTweetsResponse]
def apply(input: ScoredTweetsResponse): t.ScoredTweetsResponse
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\model\ScoredTweetsQuery.scala
class ScoredTweetsQuery(
override val params: Params,
override val clientContext: ClientContext,
override val pipelineCursor: Option[UrtOrderedCursor],
override val requestedMaxResults: Option[Int],
override val debugOptions: Option[DebugOptions],
override val features: Option[FeatureMap],
override val deviceContext: Option[DeviceContext],
override val seenTweetIds: Option[Seq[Long]],
override val qualityFactorStatus: Option[QualityFactorStatus])
extends PipelineQuery
with HasPipelineCursor[UrtOrderedCursor]
with HasDeviceContext
with HasSeenTweetIds
with HasQualityFactorStatus
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\param\ScoredTweetsParam.scala
object ScoredTweetsParam
object CrMixerSource
object EnableCandidatePipelineParam
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsCrMixerCandidatePipeline)
}
object FrsTweetSource
object EnableCandidatePipelineParam
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsFrsCandidatePipeline)
}
object InNetworkSource
object EnableCandidatePipelineParam
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsInNetworkCandidatePipeline)
}
object QualityFactor
object MaxTweetsToScoreParam
extends FSBoundedParam[Int](
name = "scored_tweets_quality_factor_max_tweets_to_score",
default = 1100,
min = 0,
max = 10000
)
object CrMixerMaxTweetsToScoreParam
extends FSBoundedParam[Int](
name = "scored_tweets_quality_factor_cr_mixer_max_tweets_to_score",
default = 500,
min = 0,
max = 10000
)
}
object ServerMaxResultsParam
extends FSBoundedParam[Int](
name = "scored_tweets_server_max_results",
default = 120,
min = 1,
max = 500
)
object UtegSource
object EnableCandidatePipelineParam
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsUtegCandidatePipeline)
}
object CachedScoredTweets
object TTLParam
extends FSBoundedParam[Duration](
name = "scored_tweets_cached_scored_tweets_ttl_minutes",
default = 3.minutes,
min = 0.minute,
max = 60.minutes
)
with HasDurationConversion
object MinCachedTweetsParam
extends FSBoundedParam[Int](
name = "scored_tweets_cached_scored_tweets_min_cached_tweets",
default = 30,
min = 0,
max = 1000
)
}
object Scoring
object HomeModelParam
extends FSParam[String](name = "scored_tweets_home_model", default = "Home")
object ModelWeights
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\param\ScoredTweetsParamConfig.scala
class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_feature_hydrator\FrsSeedUsersQueryFeatureHydrator.scala
object FrsSeedUserIdsFeature extends Feature[TweetCandidate, Option[Seq[Long]]]
object FrsUserToFollowedByUserIdsFeature extends Feature[TweetCandidate, Map[Long, Seq[Long]]]
@Singleton
case class FrsSeedUsersQueryFeatureHydrator @Inject() (
userFollowRecommendationsCandidateSource: UserFollowRecommendationsCandidateSource)
extends QueryFeatureHydrator[ScoredTweetsQuery]
def features: Set[Feature[_, _]] = Set(
FrsSeedUserIdsFeature,
FrsUserToFollowedByUserIdsFeature
)
override def hydrate(query: ScoredTweetsQuery): Stitch[FeatureMap]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_transformer\TimelineRankerFrsQueryTransformer.scala
object TimelineRankerFrsQueryTransformer
class TimelineRankerFrsQueryTransformer[
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext
](
override val candidatePipelineIdentifier: CandidatePipelineIdentifier,
override val maxTweetsToFetch: Int = MaxTweetsToFetch,
override val sinceDuration: Duration = SinceDuration)
extends CandidatePipelineQueryTransformer[Query, t.RecapQuery]
with TimelineRankerQueryTransformer[Query]
def seedAuthorIds(query: Query): Option[Seq[Long]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_transformer\TimelineRankerInNetworkQueryTransformer.scala
object TimelineRankerInNetworkQueryTransformer
class TimelineRankerInNetworkQueryTransformer[
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext
](
override val candidatePipelineIdentifier: CandidatePipelineIdentifier,
override val maxTweetsToFetch: Int = MaxTweetsToFetch,
override val sinceDuration: Duration = SinceDuration)
extends CandidatePipelineQueryTransformer[Query, t.RecapQuery]
with TimelineRankerQueryTransformer[Query]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_transformer\TimelineRankerQueryTransformer.scala
object TimelineRankerQueryTransformer
trait TimelineRankerQueryTransformer[
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext]
def maxTweetsToFetch: Int
def sinceDuration: Duration
def options: TweetKindOption.ValueSet = TweetKindOption.Default
def candidateTweetSourceId: CandidateTweetSourceId.Value
def skipVeryRecentTweets: Boolean
def utegLikedByTweetsOptions(query: Query): Option[tlr.UtegLikedByTweetsOptions] = None
def seedAuthorIds(query: Query): Option[Seq[Long]] = None
def candidatePipelineIdentifier: CandidatePipelineIdentifier
def earlybirdModels: Seq[EarlybirdScoringModelConfig] =
EarlybirdScoringModels.fromEnum(EarlybirdScoringModelsId.UnifiedEngagementProd)
def tensorflowModel: Option[String] = None
def buildTimelineRankerQuery(query: Query): tlr.RecapQuery
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_transformer\TimelineRankerUtegQueryTransformer.scala
object TimelineRankerUtegQueryTransformer
def utegEarlybirdModels =
EarlybirdScoringModels.fromEnum(EarlybirdScoringModelsId.UnifiedEngagementRectweet)
}
case class TimelineRankerUtegQueryTransformer[
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext
](
override val candidatePipelineIdentifier: CandidatePipelineIdentifier,
override val maxTweetsToFetch: Int = MaxTweetsToFetch,
override val sinceDuration: Duration = SinceDuration)
extends CandidatePipelineQueryTransformer[Query, t.UtegLikedByTweetsQuery]
with TimelineRankerQueryTransformer[Query]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\CachedScoredTweetsResponseFeatureTransformer.scala
object CachedScoredTweetsResponseFeatureTransformer
extends CandidateFeatureTransformer[hmt.CachedScoredTweet]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\ScoredTweetsCrMixerResponseFeatureTransformer.scala
object ScoredTweetsCrMixerResponseFeatureTransformer
extends CandidateFeatureTransformer[crm.TweetRecommendation]
def transform(candidate: crm.TweetRecommendation): FeatureMap
def CrMixerMetricTagToTspMetricTag(
crMixerMetricTag: crm.MetricTag
): Option[tsp.MetricTag] = crMixerMetricTag match
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\ScoredTweetsFrsResponseFeatureTransformer.scala
object ScoredTweetsFrsResponseFeatureTransformer
extends CandidateFeatureTransformer[tlr.CandidateTweet]
def transform(candidate: tlr.CandidateTweet): FeatureMap
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\ScoredTweetsInNetworkResponseFeatureTransformer.scala
object ScoredTweetsInNetworkResponseFeatureTransformer
extends CandidateFeatureTransformer[tlr.CandidateTweet]
def transform(candidate: tlr.CandidateTweet): FeatureMap
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\ScoredTweetsUtegResponseFeatureTransformer.scala
object ScoredTweetsUtegResponseFeatureTransformer
extends CandidateFeatureTransformer[tlr.CandidateTweet]
def transform(candidate: tlr.CandidateTweet): FeatureMap
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\TimelineRankerResponseTransformer.scala
object TimelineRankerResponseTransformer
def transform(candidate: tlr.CandidateTweet): FeatureMap
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scorer\DiversityDiscountProvider.scala
trait DiversityDiscountProvider
def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long]
/**
* Compute the discounted score for the position
* @param score the previous score for the candidate
* @param position zero-based position for the candidate for the given entity
* @return the discounted score for the candidate
*/
def discount(score: Double, position: Int): Double
}
object AuthorDiversityDiscountProvider extends DiversityDiscountProvider
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scorer\DiversityScorer.scala
class DiversityScorer(diversityDiscountProvider: DiversityDiscountProvider)
extends Scorer[ScoredTweetsQuery, TweetCandidate]
def apply(
query: ScoredTweetsQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scorer\HomeNaviModelDataRecordScorer.scala
object CommonFeaturesDataRecordFeature
extends DataRecordInAFeature[PipelineQuery]
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
object CandidateFeaturesDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord]
def defaultValue: DataRecord = new DataRecord()
}
case class HomeNaviModelDataRecordScorer[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any],
CandidateFeatures <: BaseDataRecordFeature[Candidate, _],
ResultFeatures <: BaseDataRecordFeature[Candidate, _]
](
override val identifier: ScorerIdentifier,
modelClient: PredictionServiceGRPCClient,
candidateFeatures: FeaturesScope[CandidateFeatures],
resultFeatures: Set[ResultFeatures],
statsReceiver: StatsReceiver)
extends Scorer[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[Seq[FeatureMap]]
object HomeNaviModelDataRecordScorer
trait PredictedScoreFeature
extends DataRecordOptionalFeature[TweetCandidate, Double]
with DoubleDataRecordCompatible
def statName: String
def modelWeightParam: FSBoundedParam[Double]
}
object PredictedFavoriteScoreFeature extends PredictedScoreFeature
object PredictedReplyScoreFeature extends PredictedScoreFeature
object PredictedRetweetScoreFeature extends PredictedScoreFeature
object PredictedReplyEngagedByAuthorScoreFeature extends PredictedScoreFeature
object PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PredictedScoreFeature
object PredictedGoodClickConvoDescUamGt2ScoreFeature extends PredictedScoreFeature
object PredictedNegativeFeedbackV2ScoreFeature extends PredictedScoreFeature
object PredictedGoodProfileClickScoreFeature extends PredictedScoreFeature
object PredictedReportedScoreFeature extends PredictedScoreFeature
object PredictedVideoPlayback50ScoreFeature extends PredictedScoreFeature
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scorer\WeightedScoresSumScorer.scala
class WeightedScoresSumScorer @Inject() (statsReceiver: StatsReceiver)
extends Scorer[ScoredTweetsQuery, TweetCandidate]
def apply(
query: ScoredTweetsQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]]
def weightedModelScore(
query: PipelineQuery,
features: FeatureMap
): Double
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsDiversityScoringPipelineConfig.scala
object ScoredTweetsDiversityScoringPipelineConfig
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsRescoreOONScoringPipelineConfig.scala
object ScoredTweetsRescoreOONScoringPipelineConfig
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig.scala
object ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsScoringPipelineConfig.scala
class ScoredTweetsScoringPipelineConfig @Inject() (
scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig,
scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig,
scoredTweetsCrMixerCandidatePipelineConfig: ScoredTweetsCrMixerCandidatePipelineConfig,
scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig,
predictionGRPCService: PredictionGRPCService,
ancestorFeatureHydrator: AncestorFeatureHydrator,
authorFeatureHydrator: AuthorFeatureHydrator,
earlybirdFeatureHydrator: EarlybirdFeatureHydrator,
metricCenterUserCountingFeatureHydrator: MetricCenterUserCountingFeatureHydrator,
tweetypieContentFeatureHydrator: TweetypieContentFeatureHydrator,
gizmoduckAuthorSafetyFeatureHydrator: GizmoduckAuthorSafetyFeatureHydrator,
graphTwoHopFeatureHydrator: GraphTwoHopFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
twhinAuthorFollow20220101FeatureHydrator: TwhinAuthorFollow20220101FeatureHydrator,
userFollowedTopicIdsFeatureHydrator: UserFollowedTopicIdsFeatureHydrator,
utegFeatureHydrator: UtegFeatureHydrator,
realGraphViewerAuthorFeatureHydrator: RealGraphViewerAuthorFeatureHydrator,
realGraphViewerRelatedUsersFeatureHydrator: RealGraphViewerRelatedUsersFeatureHydrator,
realTimeInteractionGraphEdgeFeatureHydrator: RealTimeInteractionGraphEdgeFeatureHydrator,
engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator: EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator,
topicCountryEngagementRealTimeAggregateFeatureHydrator: TopicCountryEngagementRealTimeAggregateFeatureHydrator,
topicEngagementRealTimeAggregateFeatureHydrator: TopicEngagementRealTimeAggregateFeatureHydrator,
tspInferredTopicFeatureHydrator: TSPInferredTopicFeatureHydrator,
tweetCountryEngagementRealTimeAggregateFeatureHydrator: TweetCountryEngagementRealTimeAggregateFeatureHydrator,
tweetEngagementRealTimeAggregateFeatureHydrator: TweetEngagementRealTimeAggregateFeatureHydrator,
twitterListEngagementRealTimeAggregateFeatureHydrator: TwitterListEngagementRealTimeAggregateFeatureHydrator,
userAuthorEngagementRealTimeAggregateFeatureHydrator: UserAuthorEngagementRealTimeAggregateFeatureHydrator,
simClustersEngagementSimilarityFeatureHydrator: SimClustersEngagementSimilarityFeatureHydrator,
phase1EdgeAggregateFeatureHydrator: Phase1EdgeAggregateFeatureHydrator,
phase2EdgeAggregateFeatureHydrator: Phase2EdgeAggregateFeatureHydrator,
statsReceiver: StatsReceiver)
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsWeightedScoresSumScoringPipelineConfig.scala
class ScoredTweetsWeightedScoresSumScoringPipelineConfig @Inject() (
weightedScoresSumScorer: WeightedScoresSumScorer)
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\side_effect\CachedScoredTweetsSideEffect.scala
class CachedScoredTweetsSideEffect @Inject() (
scoredTweetsCache: TtlCache[Long, hmt.CachedScoredTweets])
extends PipelineResultSideEffect[PipelineQuery, ScoredTweetsResponse]
def buildCachedScoredTweets(
candidates: Seq[CandidateWithDetails]
): hmt.CachedScoredTweets
def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, ScoredTweetsResponse]
): Stitch[Unit]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\side_effect\ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala
class ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect @Inject() (
@Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String,
@Named(CommonFeaturesScribeEventPublisher) commonFeaturesScribeEventPublisher: EventPublisher[
pldr.PolyDataRecord
],
@Named(CandidateFeaturesScribeEventPublisher) candidateFeaturesScribeEventPublisher: EventPublisher[
pldr.PolyDataRecord
],
@Named(MinimumFeaturesScribeEventPublisher) minimumFeaturesScribeEventPublisher: EventPublisher[
pldr.PolyDataRecord
],
statsReceiver: StatsReceiver,
) extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse]
with Logging
def apply(
inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse]
): Stitch[Unit]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\service\HomeMixerAccessPolicy.scala
object HomeMixerAccessPolicy
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\service\HomeMixerAlertConfig.scala
object HomeMixerAlertConfig
object BusinessHours
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\service\ScoredTweetsService.scala
class ScoredTweetsService @Inject() (productPipelineRegistry: ProductPipelineRegistry)
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\store\RealGraphInNetworkScoresStore.scala
object ManhattanRealGraphKVDescriptor
class RealGraphInNetworkScoresStore(manhattanKVEndpoint: ManhattanKVEndpoint)
extends ReadableStore[Long, Seq[wtf.Candidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\store\UserLanguagesStore.scala
object ManhattanUserLanguagesKVDescriptor
class UserLanguagesStore(
manhattanKVEndpoint: ManhattanKVEndpoint,
statsReceiver: StatsReceiver)
extends ReadableStore[Long, Seq[scc.ThriftLanguage]]
def get(viewerId: Long): Future[Option[Seq[scc.ThriftLanguage]]] =
Stitch
.run(
manhattanKVEndpoint.get(key = userLanguagesDatasetKey.withPkey(viewerId), valueDescriptor))
.map
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\CachedScoredTweetsHelper.scala
object CachedScoredTweetsHelper
def tweetImpressionsAndCachedScoredTweets(
features: FeatureMap,
candidatePipelineIdentifier: CandidatePipelineIdentifier
): Seq[Long]
def tweetImpressionsAndCachedScoredTweetsInRange(
features: FeatureMap,
candidatePipelineIdentifier: CandidatePipelineIdentifier,
maxNumImpressions: Int,
sinceTime: Time,
untilTime: Time
): Seq[Long] =
tweetImpressionsAndCachedScoredTweets(features, candidatePipelineIdentifier)
.filter
def unseenCachedScoredTweets(
features: FeatureMap
): Seq[hmt.CachedScoredTweet]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\CandidatesUtil.scala
object CandidatesUtil
def getItemCandidates(candidates: Seq[CandidateWithDetails]): Seq[ItemCandidateWithDetails]
def getItemCandidatesWithOnlyModuleLast(
candidates: Seq[CandidateWithDetails]
): Seq[ItemCandidateWithDetails]
def containsType[CandidateType <: UniversalNoun[_]](
candidates: Seq[CandidateWithDetails]
)(
implicit tag: ClassTag[CandidateType]
): Boolean = candidates.exists
def getOriginalAuthorId(candidateFeatures: FeatureMap): Option[Long] =
if (candidateFeatures.getOrElse(IsRetweetFeature, false))
candidateFeatures.getOrElse(SourceUserIdFeature, None)
else candidateFeatures.getOrElse(AuthorIdFeature, None)
def getEngagerUserIds(
candidateFeatures: FeatureMap
): Seq[Long]
def getMediaUnderstandingAnnotationIds(
candidateFeatures: FeatureMap
): Seq[Long]
def getTweetIdAndSourceId(candidate: CandidateWithFeatures[TweetCandidate]): Seq[Long] =
Seq(candidate.candidate.id) ++ candidate.features.getOrElse(SourceTweetIdFeature, None)
def isAuthoredByViewer(query: PipelineQuery, candidateFeatures: FeatureMap): Boolean =
candidateFeatures.getOrElse(AuthorIdFeature, None).contains(query.getRequiredUserId) ||
(candidateFeatures.getOrElse(IsRetweetFeature, false) &&
candidateFeatures.getOrElse(SourceUserIdFeature, None).contains(query.getRequiredUserId))
val reverseChronTweetsOrdering: Ordering[CandidateWithDetails] =
Ordering.by[CandidateWithDetails, Long]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\InjectionTransformer.scala
object InjectionTransformerImplicits
class ByteArrayInjectionToByteBufferTransformer[A](baInj: Injection[A, Array[Byte]])
def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj)
def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj)
}
implicit class BufInjectionToByteBufferTransformer[A](bufInj: Injection[A, Buf])
def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj)
def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj)
}
implicit class ByteBufferInjectionToByteBufferTransformer[A](bbInj: Injection[A, ByteBuffer])
def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj)
def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj)
}
}
class InjectionTransformer[A, B](inj: Injection[A, B]) extends Transformer[A, B]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\LanguageUtil.scala
object LanguageUtil
def computeLanguages(
userLanguages: smg.UserLanguages,
minProducedLanguageRatio: Double = DafaultMinProducedLanguageRatio,
minConsumedLanguageConfidence: Double = DefaultMinConsumedLanguageConfidence
): Seq[scc.ThriftLanguage]
def computeLanguageConfidenceMap(
userLanguages: smg.UserLanguages,
minProducedLanguageRatio: Double,
minConsumedLanguageConfidence: Double
): Map[scc.ThriftLanguage, Double]
def getLanguageMap(
scoredLanguages: Seq[smg.ScoredString]
): Map[scc.ThriftLanguage, Double]
def getThriftLanguage(languageName: String): Option[scc.ThriftLanguage]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\MissingKeyException.scala
object MissingKeyException extends Exception("Missing key")
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\ObservedKeyValueResultHandler.scala
trait ObservedKeyValueResultHandler
def observedGet[K, V](
key: Option[K],
keyValueResult: KeyValueResult[K, V],
): Try[Option[V]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\ReplyRetweetUtil.scala
object ReplyRetweetUtil
def isEligibleReply(candidate: CandidateWithFeatures[TweetCandidate]): Boolean
def replyToAncestorTweetCandidatesMap(
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Map[Long, Seq[CandidateWithFeatures[TweetCandidate]]]
def ancestorTweetIdToDescendantRepliesMap(
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Map[Long, Seq[CandidateWithFeatures[TweetCandidate]]]
def replyTweetIdToInReplyToTweetMap(
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Map[Long, CandidateWithFeatures[TweetCandidate]]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\TensorFlowUtil.scala
object TensorFlowUtil
def skipEmbeddingBBHeader(bb: ByteBuffer): ByteBuffer
def byteBufferToFloatIterator(
bb: ByteBuffer
): Iterator[Float]
def embeddingByteBufferToFloatTensor(
bb: ByteBuffer
): FloatTensor
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\TweetImpressionsHelper.scala
object TweetImpressionsHelper
def tweetImpressions(features: FeatureMap): Set[Long]
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\earlybird\EarlybirdRequestUtil.scala
object EarlybirdRequestUtil
def getTweetsEBFeaturesRequest(
userId: Option[Long],
tweetIds: Option[Seq[Long]],
clientId: Option[String],
getTweetsFromArchiveIndex: Boolean = false,
getOnlyProtectedTweets: Boolean = false,
): eb.EarlybirdRequest
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\earlybird\EarlybirdResponseUtil.scala
object EarlybirdResponseUtil
def getTweetCountByAuthorId(
searchResults: Seq[eb.ThriftSearchResult]
): Map[Long, Int]
def getLanguage(uiLanguageCode: Option[String]): Option[scc.ThriftLanguage]
def getMentions(result: eb.ThriftSearchResult): Seq[String]
def getHashtags(result: eb.ThriftSearchResult): Seq[String]
def getFacets(
facetLabels: Seq[eb.ThriftFacetLabel],
facetName: String,
charsToRemove: Set[Char]
): Seq[String]
def isUserMentioned(
screenName: Option[String],
mentions: Seq[String]
): Boolean
def isUsersMainLanguage(
tweetLanguage: scc.ThriftLanguage,
userLanguages: Seq[scc.ThriftLanguage]
): Boolean
def isUsersLanguage(
tweetLanguage: scc.ThriftLanguage,
userLanguages: Seq[scc.ThriftLanguage]
): Boolean
def isUILanguage(
tweetLanguage: scc.ThriftLanguage,
uiLanguage: Option[scc.ThriftLanguage]
): Boolean
def getBooleanOptFeature(
featureName: EarlybirdFieldConstant,
resultMapOpt: Option[scala.collection.Map[Int, Boolean]],
defaultValue: Boolean = false,
): Option[Boolean]
def getDoubleAsIntOptFeature(
featureName: EarlybirdFieldConstant,
resultMapOpt: Option[scala.collection.Map[Int, Double]]
): Option[Int]
def getIntOptFeature(
featureName: EarlybirdFieldConstant,
resultMapOpt: Option[scala.collection.Map[Int, Int]]
): Option[Int]
def getOONTweetThriftFeaturesByTweetId(
searcherUserId: Long,
screenName: Option[String],
userLanguages: Seq[scc.ThriftLanguage],
uiLanguageCode: Option[String] = None,
searchResults: Seq[eb.ThriftSearchResult],
): Map[Long, sc.ThriftTweetFeatures]
def getOONThriftTweetFeaturesFromSearchResult(
searcherUserId: Long,
screenName: Option[String],
userLanguages: Seq[scc.ThriftLanguage],
uiLanguage: Option[scc.ThriftLanguage],
tweetCountByAuthorId: Map[Long, Int],
searchResult: eb.ThriftSearchResult
): sc.ThriftTweetFeatures
def applyUserIndependentFeatures(
result: eb.ThriftSearchResult
)(
thriftTweetFeatures: sc.ThriftTweetFeatures
): sc.ThriftTweetFeatures
def applyOONUserDependentFeatures(
searcherUserId: Long,
screenName: Option[String],
userLanguages: Seq[scc.ThriftLanguage],
uiLanguage: Option[scc.ThriftLanguage],
tweetCountByAuthorId: Map[Long, Int],
result: eb.ThriftSearchResult
)(
thriftTweetFeatures: sc.ThriftTweetFeatures
): sc.ThriftTweetFeatures
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\earlybird\RelevanceSearchUtil.scala
object RelevanceSearchUtil
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\tweetypie\RequestFields.scala
object RequestFields
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\tweetypie\content\FeatureExtractionHelper.scala
object FeatureExtractionHelper
def extractFeatures(
tweet: tp.Tweet
): ContentFeatures
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\tweetypie\content\TweetMediaFeaturesExtractor.scala
object TweetMediaFeaturesExtractor
def hasImage(tweet: tp.Tweet): Boolean = hasMediaByCategory(tweet, ImageCategories)
def hasVideo(tweet: tp.Tweet): Boolean = hasMediaByCategory(tweet, VideoCategories)
private def hasMediaByCategory(tweet: tp.Tweet, categories: Set[Int]): Boolean
def addMediaFeaturesFromTweet(
inputFeatures: ContentFeatures,
tweet: tp.Tweet,
): ContentFeatures
def getSizeFeatures(mediaEntities: Seq[tp.MediaEntity]): Seq[MediaSizeFeatures]
def getPlaybackFeatures(mediaEntities: Seq[tp.MediaEntity]): PlaybackFeatures
def getMediaTagScreenNames(tagMap: Map[Long, Seq[tp.MediaTag]]): Seq[String] =
tagMap.values
.flatMap(seqMediaTag => seqMediaTag.flatMap(_.screenName))
.toSeq
// Areas of the faces identified in the media entities
private def getFaceMapAreas(mediaEntities: Seq[tp.MediaEntity]): Seq[Int]
def getSortedColorPalette(
mediaEntities: Seq[tp.MediaEntity]
): Seq[mi.ColorPaletteItem]
def getStickerFeatures(mediaEntities: Seq[tp.MediaEntity]): Seq[Long]
def getMediaOriginProviders(mediaEntities: Seq[tp.MediaEntity]): Seq[String] =
for
def getIsManaged(mediaEntities: Seq[tp.MediaEntity]): Boolean
def getIs360(mediaEntities: Seq[tp.MediaEntity]): Boolean
def getViewCount(mediaEntities: Seq[tp.MediaEntity]): Option[Long]
def getUserDefinedProductMetadataFeatures(
mediaEntities: Seq[tp.MediaEntity]
): Seq[UserDefinedProductMetadataFeatures] =
for
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\tweetypie\content\TweetTextFeaturesExtractor.scala
object TweetTextFeaturesExtractor
def addTextFeaturesFromTweet(
inputFeatures: ContentFeatures,
tweet: tp.Tweet
): ContentFeatures
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\account_recommendations_mixer\AccountRecommendationsMixerCandidateSource.scala
object WhoToFollowModuleHeaderFeature extends Feature[UserCandidate, t.Header]
object WhoToFollowModuleFooterFeature extends Feature[UserCandidate, Option[t.Footer]]
object WhoToFollowModuleDisplayOptionsFeature
extends Feature[UserCandidate, Option[t.DisplayOptions]]
@Singleton
class AccountRecommendationsMixerCandidateSource @Inject() (
accountRecommendationsMixer: t.AccountRecommendationsMixer.MethodPerEndpoint)
extends CandidateSourceWithExtractedFeatures[
t.AccountRecommendationsMixerRequest,
t.RecommendedUser
]
def apply(
request: t.AccountRecommendationsMixerRequest
): Stitch[CandidatesWithSourceFeatures[t.RecommendedUser]]
def responseToCandidatesWithSourceFeatures(
userRecommendations: Seq[t.RecommendedUser],
header: t.Header,
footer: Option[t.Footer],
displayOptions: Option[t.DisplayOptions],
): CandidatesWithSourceFeatures[t.RecommendedUser]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\ads\AdsProdStratoCandidateSource.scala
class AdsProdStratoCandidateSource @Inject() (adsClient: MakeAdRequestClientColumn)
extends StratoKeyFetcherSource[
AdRequestParams,
AdRequestResponse,
AdImpression
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\ads\AdsProdThriftCandidateSource.scala
class AdsProdThriftCandidateSource @Inject() (
adServerClient: NewAdServer.MethodPerEndpoint)
extends CandidateSource[AdRequestParams, AdImpression]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\ads\AdsStagingCandidateSource.scala
class AdsStagingCandidateSource @Inject() (adsClient: MakeAdRequestStagingClientColumn)
extends StratoKeyFetcherSource[
AdRequestParams,
AdRequestResponse,
AdImpression
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\ann\AnnCandidateSource.scala
class AnnCandidateSource[T1, T2, P <: RuntimeParams, D <: Distance[D]](
val annQueryableById: QueryableById[T1, T2, P, D],
val batchSize: Int,
val timeoutPerRequest: Duration,
override val identifier: CandidateSourceIdentifier)
extends CandidateSource[AnnIdQuery[T1, P], NeighborWithDistanceWithSeed[T1, T2, D]]
def apply(
request: AnnIdQuery[T1, P]
): Stitch[Seq[NeighborWithDistanceWithSeed[T1, T2, D]]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\audiospace\CreatedSpacesCandidateSource.scala
class CreatedSpacesCandidateSource @Inject() (
column: CreatedSpacesSliceOnUserClientColumn)
extends StratoKeyViewFetcherWithSourceFeaturesSource[
Long,
CreatedSpacesView,
SpaceSlice,
String
]
def stratoResultTransformer(
stratoKey: Long,
stratoResult: SpaceSlice
): Seq[String] =
stratoResult.items
override protected def extractFeaturesFromStratoResult(
stratoKey: Long,
stratoResult: SpaceSlice
): FeatureMap
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\business_profiles\TeamMembersCandidateSource.scala
class TeamMembersCandidateSource @Inject() (
column: BusinessProfileTeamMembersOnUserClientColumn)
extends StratoKeyViewFetcherWithSourceFeaturesSource[
Long,
TeamMembersView,
TeamMembersSlice,
Long
]
def stratoResultTransformer(
stratoKey: Long,
stratoResult: TeamMembersSlice
): Seq[Long] =
stratoResult.members
override protected def extractFeaturesFromStratoResult(
stratoKey: Long,
stratoResult: TeamMembersSlice
): FeatureMap
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\cr_mixer\CrMixerFrsBasedTweetRecommendationsCandidateSource.scala
class CrMixerFrsBasedTweetRecommendationsCandidateSource @Inject() (
crMixerClient: t.CrMixer.MethodPerEndpoint)
extends CandidateSource[t.FrsTweetRequest, t.FrsTweet]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\cr_mixer\CrMixerTweetRecommendationsCandidateSource.scala
class CrMixerTweetRecommendationsCandidateSource @Inject() (
crMixerClient: t.CrMixer.MethodPerEndpoint)
extends CandidateSource[t.CrMixerTweetRequest, t.TweetRecommendation]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\earlybird\EarlybirdTweetCandidateSource.scala
class EarlybirdTweetCandidateSource @Inject() (
earlybirdService: t.EarlybirdService.MethodPerEndpoint)
extends CandidateSource[t.EarlybirdRequest, t.ThriftSearchResult]
with Logging
def apply(request: t.EarlybirdRequest): Stitch[Seq[t.ThriftSearchResult]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\explore_ranker\ExploreRankerCandidateSource.scala
class ExploreRankerCandidateSource @Inject() (
exploreRankerService: t.ExploreRanker.MethodPerEndpoint)
extends CandidateSource[t.ExploreRankerRequest, t.ImmersiveRecsResult]
def apply(
request: t.ExploreRankerRequest
): Stitch[Seq[t.ImmersiveRecsResult]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\flexible_injection_pipeline\PromptCandidateSource.scala
class PromptCandidateSource @Inject() (taskService: servicethrift.TaskService.MethodPerEndpoint)
extends CandidateSource[servicethrift.GetInjectionsRequest, IntermediatePrompt]
with Logging
def apply(
request: servicethrift.GetInjectionsRequest
): Stitch[Seq[IntermediatePrompt]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\hermit\UsersSimilarToMeCandidateSource.scala
class UsersSimilarToMeCandidateSource @Inject() (
column: HermitRecommendUsersClientColumn)
extends StratoKeyViewFetcherSource[
Long,
RecommendationRequest,
RecommendationResponse,
RelatedUser
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\interest_discovery\RelatedTopicsCandidateSource.scala
class RelatedTopicsCandidateSource @Inject() (
interestDiscoveryService: t.InterestsDiscoveryService.MethodPerEndpoint)
extends CandidateSource[t.RelatedTopicsRequest, t.RelatedTopic]
with Logging
def apply(
request: t.RelatedTopicsRequest
): Stitch[Seq[t.RelatedTopic]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\lists\OrganicPopGeoListsCandidateSource.scala
class OrganicPopGeoListsCandidateSource @Inject() (
organicPopgeoListsClientColumn: OrganicPopgeoListsClientColumn)
extends StratoKeyFetcherSource[
OrganicPopgeoListsClientColumn.Key,
OrganicPopgeoListsClientColumn.Value,
TwitterListCandidate
]
def stratoResultTransformer(
stratoResult: OrganicPopgeoListsClientColumn.Value
): Seq[TwitterListCandidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\people_discovery\PeopleDiscoveryCandidateSource.scala
object WhoToFollowModuleHeaderFeature extends Feature[UserCandidate, t.Header]
object WhoToFollowModuleDisplayOptionsFeature
extends Feature[UserCandidate, Option[t.DisplayOptions]]
object WhoToFollowModuleShowMoreFeature extends Feature[UserCandidate, Option[t.ShowMore]]
@Singleton
class PeopleDiscoveryCandidateSource @Inject() (
peopleDiscoveryService: t.ThriftPeopleDiscoveryService.MethodPerEndpoint)
extends CandidateSourceWithExtractedFeatures[t.GetModuleRequest, t.RecommendedUser]
with Logging
def apply(
request: t.GetModuleRequest
): Stitch[CandidatesWithSourceFeatures[t.RecommendedUser]]
def layoutToCandidatesWithSourceFeatures(
userRecommendations: Seq[t.RecommendedUser],
header: t.Header,
displayOptions: Option[t.DisplayOptions],
showMore: Option[t.ShowMore],
): CandidatesWithSourceFeatures[t.RecommendedUser]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\recommendations\UserFollowRecommendationsCandidateSource.scala
class UserFollowRecommendationsCandidateSource @Inject() (
getRecommendationsClientColumn: GetRecommendationsClientColumn)
extends StratoKeyViewFetcherSource[
fr.RecommendationRequest,
Unit,
fr.RecommendationResponse,
fr.UserRecommendation
]
def stratoResultTransformer(
stratoKey: fr.RecommendationRequest,
stratoResult: fr.RecommendationResponse
): Seq[fr.UserRecommendation]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\social_graph\SocialgraphCandidateSource.scala
trait SocialgraphResponse
case class SocialgraphResult(id: Long) extends SocialgraphResponse
case class SocialgraphCursor(cursor: Long, cursorType: CursorType) extends SocialgraphResponse
@Singleton
class SocialgraphCandidateSource @Inject() (
override val fetcher: Fetcher[thriftscala.IdsRequest, Option[
thriftscala.RequestContext
], thriftscala.IdsResult])
extends StratoKeyViewFetcherSource[
thriftscala.IdsRequest,
Option[thriftscala.RequestContext],
thriftscala.IdsResult,
SocialgraphResponse
]
def stratoResultTransformer(
stratoKey: IdsRequest,
stratoResult: IdsResult
): Seq[SocialgraphResponse]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\social_graph\SocialgraphCursorConstants.scala
object SocialgraphCursorConstants
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timelines_impression_store\TimelinesImpressionStoreCandidateSourceV2.scala
class TimelinesImpressionStoreCandidateSourceV2 @Inject() (
client: TweetImpressionStoreManhattanV2OnUserClientColumn)
extends StratoKeyFetcherSource[
Long,
t.TweetImpressionsEntries,
t.TweetImpressionsEntry
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_ranker\TimelineRankerInNetworkCandidateSource.scala
object TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature
extends Feature[PipelineQuery, Map[Long, t.CandidateTweet]]
@Singleton
class TimelineRankerInNetworkCandidateSource @Inject() (
timelineRankerClient: t.TimelineRanker.MethodPerEndpoint)
extends CandidateSourceWithExtractedFeatures[t.RecapQuery, t.CandidateTweet]
def apply(
request: t.RecapQuery
): Stitch[CandidatesWithSourceFeatures[t.CandidateTweet]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_ranker\TimelineRankerRecapCandidateSource.scala
class TimelineRankerRecapCandidateSource @Inject() (
timelineRankerClient: t.TimelineRanker.MethodPerEndpoint)
extends CandidateSource[t.RecapQuery, t.CandidateTweet]
def apply(
request: t.RecapQuery
): Stitch[Seq[t.CandidateTweet]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_ranker\TimelineRankerUtegCandidateSource.scala
object TimelineRankerUtegSourceTweetsFeature
extends Feature[PipelineQuery, Seq[t.CandidateTweet]]
@Singleton
class TimelineRankerUtegCandidateSource @Inject() (
timelineRankerClient: t.TimelineRanker.MethodPerEndpoint)
extends CandidateSourceWithExtractedFeatures[t.UtegLikedByTweetsQuery, t.CandidateTweet]
def apply(
request: t.UtegLikedByTweetsQuery
): Stitch[CandidatesWithSourceFeatures[t.CandidateTweet]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_scorer\TimelineScorerCandidateSource.scala
class ScoredTweetCandidateWithFocalTweet(
candidate: v1.ScoredTweetCandidate,
focalTweetIdOpt: Option[Long])
case object TimelineScorerCandidateSourceSucceededFeature extends Feature[PipelineQuery, Boolean]
@Singleton
class TimelineScorerCandidateSource @Inject() (
timelineScorerClient: t.TimelineScorer.MethodPerEndpoint)
extends CandidateSourceWithExtractedFeatures[
t.ScoredTweetsRequest,
ScoredTweetCandidateWithFocalTweet
]
def apply(
request: t.ScoredTweetsRequest
): Stitch[CandidatesWithSourceFeatures[ScoredTweetCandidateWithFocalTweet]]
def isEligibleReply(candidate: ct.ScoredTweetCandidateAliases.V1Alias): Boolean
def originalTweetId(candidate: ct.ScoredTweetCandidateAliases.V1Alias): Long
def toScoredTweetCandidateFromAncestor(
ancestor: Ancestor,
inReplyToTweetId: Option[Long],
conversationId: Option[Long],
ancestors: Option[Seq[Ancestor]],
candidateTweetSourceId: Option[CandidateTweetSourceId]
): ct.ScoredTweetCandidateAliases.V1Alias
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_service\TimelineServiceTweetCandidateSource.scala
object TimelineServiceResponseWasTruncatedFeature
extends FeatureWithDefaultOnFailure[PipelineQuery, Boolean]
class TimelineServiceTweetCandidateSource @Inject() (
timelineService: TimelineService)
extends CandidateSourceWithExtractedFeatures[t.TimelineQuery, t.Tweet]
def apply(request: t.TimelineQuery): Stitch[CandidatesWithSourceFeatures[t.Tweet]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\topics\FollowedTopicsCandidateSource.scala
class FollowedTopicsCandidateSource @Inject() (
column: FollowedTopicsGetterClientColumn)
extends StratoKeyViewFetcherSeqSource[
Long,
Unit,
Long
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\tweetconvosvc\ConversationServiceCandidateSource.scala
class ConversationServiceCandidateSourceRequest(
tweetsWithConversationMetadata: Seq[TweetWithConversationMetadata])
case class TweetWithConversationMetadata(
tweetId: Long,
userId: Option[Long],
sourceTweetId: Option[Long],
sourceUserId: Option[Long],
inReplyToTweetId: Option[Long],
conversationId: Option[Long],
ancestors: Seq[ta.TweetAncestor])
/**
* Candidate source that fetches ancestors of input candidates from Tweetconvosvc and
* returns a flattened list of input and ancestor candidates.
*/
@Singleton
class ConversationServiceCandidateSource @Inject() (
conversationServiceClient: tcs.ConversationService.MethodPerEndpoint)
extends CandidateSourceWithExtractedFeatures[
ConversationServiceCandidateSourceRequest,
TweetWithConversationMetadata
]
def apply(
request: ConversationServiceCandidateSourceRequest
): Stitch[CandidatesWithSourceFeatures[TweetWithConversationMetadata]]
def getTweetsInThread(
focalTweet: TweetWithConversationMetadata,
ancestors: ta.TweetAncestors
): Seq[TweetWithConversationMetadata]
def getTruncatedRootTweet(
ancestors: ta.TweetAncestors,
focalTweetId: Long
): Option[TweetWithConversationMetadata]
def dedupeCandidates(
inputTweetsWithConversationMetadata: Seq[TweetWithConversationMetadata],
ancestors: Seq[TweetWithConversationMetadata]
): Seq[TweetWithConversationMetadata]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\tweetconvosvc\ConversationServiceResponseFeatureTransformer.scala
object AuthorIdFeature extends Feature[TweetCandidate, Option[Long]]
object AncestorIdsFeature extends Feature[TweetCandidate, Seq[Long]]
object ConversationModuleFocalTweetIdFeature extends Feature[TweetCandidate, Option[Long]]
object InReplyToFeature extends Feature[TweetCandidate, Option[Long]]
object IsRetweetFeature extends Feature[TweetCandidate, Boolean]
object SourceTweetIdFeature extends Feature[TweetCandidate, Option[Long]]
object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]]
object SuggestTypeFeature extends Feature[TweetCandidate, Option[SuggestType]]
object ConversationServiceResponseFeatureTransformer
extends CandidateFeatureTransformer[TweetWithConversationMetadata]
def transform(candidate: TweetWithConversationMetadata): FeatureMap
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\tweetconvosvc\DropMaxConversationModuleItemCandidates.scala
class DropMaxConversationModuleItemCandidates[-Query <: PipelineQuery](
override val pipelineScope: CandidateScope,
includeRootTweet: Boolean)
extends Selector[Query]
def apply(
query: Query,
remainingCandidates: Seq[CandidateWithDetails],
result: Seq[CandidateWithDetails]
): SelectorResult
def updateConversationModule(
module: ModuleCandidateWithDetails,
includeRootTweet: Boolean
): ModuleCandidateWithDetails
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\slice\SliceItemCandidateDecorator.scala
class SliceItemCandidateDecorator[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
cursorBuilder: CandidateSliceItemBuilder[Query, CursorCandidate, CursorItem],
override val identifier: DecoratorIdentifier = DecoratorIdentifier("SliceItemCandidate"))
extends CandidateDecorator[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[Seq[Decoration]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\slice\builder\CursorCandidateSliceItemBuilder.scala
class CursorCandidateSliceItemBuilder()
extends CandidateSliceItemBuilder[PipelineQuery, CursorCandidate, CursorItem]
def apply(
query: PipelineQuery,
candidate: CursorCandidate,
featureMap: FeatureMap
): CursorItem =
candidate.cursorType match
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\UrtConversationItemCandidateDecorator.scala
class UrtConversationItemCandidateDecorator[
Query <: PipelineQuery,
Candidate <: BaseTweetCandidate
](
tweetCandidateUrtItemBuilder: TweetCandidateUrtItemBuilder[Query, Candidate],
override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtConversationItem"))
extends CandidateDecorator[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[Seq[Decoration]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\UrtItemCandidateDecorator.scala
class UrtItemCandidateDecorator[
Query <: PipelineQuery,
BuilderInput <: UniversalNoun[Any],
BuilderOutput <: TimelineItem
](
builder: CandidateUrtEntryBuilder[Query, BuilderInput, BuilderOutput],
override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtItemCandidate"))
extends CandidateDecorator[Query, BuilderInput]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[BuilderInput]]
): Stitch[Seq[Decoration]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\UrtItemInModuleDecorator.scala
class UrtItemInModuleDecorator[
Query <: PipelineQuery,
BuilderInput <: UniversalNoun[Any],
BuilderOutput <: TimelineItem
](
urtItemCandidateDecorator: CandidateDecorator[Query, BuilderInput],
moduleBuilder: BaseTimelineModuleBuilder[Query, BuilderInput],
override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtItemInModule"))
extends CandidateDecorator[Query, BuilderInput]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[BuilderInput]]
): Stitch[Seq[Decoration]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\UrtMultipleModulesDecorator.scala
trait GroupByKey[-Query <: PipelineQuery, -BuilderInput <: UniversalNoun[Any], Key]
def apply(query: Query, candidate: BuilderInput, candidateFeatures: FeatureMap): Option[Key]
}
/**
* Similar to [[UrtItemInModuleDecorator]] except that this decorator can assign items to different
* modules based on the provided [[GroupByKey]].
*
* @param urtItemCandidateDecorator decorates individual item candidates
* @param moduleBuilder builds a module from a particular candidate group
* @param groupByKey assigns each candidate a module group. Returning [[None]] will result in the
* candidate not being assigned to a module
*/
case class UrtMultipleModulesDecorator[
-Query <: PipelineQuery,
-BuilderInput <: UniversalNoun[Any],
GroupKey
](
urtItemCandidateDecorator: CandidateDecorator[Query, BuilderInput],
moduleBuilder: BaseTimelineModuleBuilder[Query, BuilderInput],
groupByKey: GroupByKey[Query, BuilderInput, GroupKey],
override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtMultipleModules"))
extends CandidateDecorator[Query, BuilderInput]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[BuilderInput]]
): Stitch[Seq[Decoration]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\contextual_ref\ContextualTweetRefBuilder.scala
class ContextualTweetRefBuilder[-Candidate <: BaseTweetCandidate](
tweetHydrationContext: TweetHydrationContext)
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\conversations\ConversationModuleMetadataBuilder.scala
class ConversationModuleMetadataBuilder[
Query <: PipelineQuery,
Candidate <: BaseTweetCandidate
](
ancestorIdsFeature: Feature[_, Seq[Long]],
allIdsOrdering: Ordering[Long])
extends BaseModuleMetadataBuilder[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): ModuleMetadata
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\FlipPromptCandidateUrtItemBuilder.scala
object FlipPromptCandidateUrtItemBuilder
class FlipPromptCandidateUrtItemBuilder[-Query <: PipelineQuery]()
extends CandidateUrtEntryBuilder[Query, BasePromptCandidate[Any], TimelineItem]
def apply(
query: Query,
promptCandidate: BasePromptCandidate[Any],
candidateFeatures: FeatureMap
): TimelineItem
def getInlinePromptMessageContent(
candidate: onboardingthrift.InlinePrompt
): MessageContent
def getFullCoverContent(
candidate: onboardingthrift.FullCover
): FullCoverContent =
FullCoverContent(
displayType = CoverFullCoverDisplayType,
primaryText = convertRichText(candidate.primaryText),
primaryCoverCta = convertCoverCta(candidate.primaryButtonAction),
secondaryCoverCta = candidate.secondaryButtonAction.map(convertCoverCta),
secondaryText = candidate.secondaryText.map(convertRichText),
imageVariant = candidate.image.map(img => convertImageVariant(img.image)),
details = candidate.detailText.map(convertRichText),
dismissInfo = candidate.dismissInfo.map(convertDismissInfo),
imageDisplayType = candidate.image.map(img => convertImageDisplayType(img.imageDisplayType)),
impressionCallbacks = candidate.impressionCallbacks.map(_.map(convertCallback).toList)
)
private def getHalfCoverContent(
candidate: onboardingthrift.HalfCover
): HalfCoverContent =
HalfCoverContent(
displayType =
candidate.displayType.map(convertHalfCoverDisplayType).getOrElse(CoverHalfCoverDisplayType),
primaryText = convertRichText(candidate.primaryText),
primaryCoverCta = convertCoverCta(candidate.primaryButtonAction),
secondaryCoverCta = candidate.secondaryButtonAction.map(convertCoverCta),
secondaryText = candidate.secondaryText.map(convertRichText),
coverImage = candidate.image.map(convertCoverImage),
dismissible = candidate.dismissible,
dismissInfo = candidate.dismissInfo.map(convertDismissInfo),
impressionCallbacks = candidate.impressionCallbacks.map(_.map(convertCallback).toList)
)
private def buildClientEventInfo(
injection: Injection
): Option[ClientEventInfo]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\FlipPromptModuleGrouping.scala
object FlipPromptModuleGrouping extends GroupByKey[PipelineQuery, UniversalNoun[Any], Int]
def apply(
query: PipelineQuery,
candidate: UniversalNoun[Any],
candidateFeatures: FeatureMap
): Option[Int]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\FlipPromptUrtModuleBuilder.scala
class FlipPromptUrtModuleBuilder[-Query <: PipelineQuery](
moduleIdGeneration: ModuleIdGeneration = AutomaticUniqueModuleId())
extends BaseTimelineModuleBuilder[Query, BasePromptCandidate[Any]]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[BasePromptCandidate[Any]]]
): TimelineModule
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\OnboardingInjectionConversions.scala
class to convert onboarding thrift to product-mixer models
*/
object OnboardingInjectionConversions
def convertFeedbackInfo(
feedbackInfo: onboardingthrift.FeedbackInfo
): FeedbackActionInfo
def convertClientEventInfo(input: onboardingthrift.ClientEventInfo): ClientEventInfo =
ClientEventInfo(
component = input.component,
element = input.element,
details = None,
action = input.action,
entityToken = None)
def convertCallback(callback: onboardingthrift.Callback): Callback =
Callback(callback.endpoint)
def convertImage(image: onboardingthrift.Image): MessageImage =
MessageImage(
Set(convertImageVariant(image.image)),
backgroundColor =
None
)
def convertCoverImage(image: onboardingthrift.Image): CoverImage =
CoverImage(
convertImageVariant(image.image),
imageDisplayType = convertImageDisplayType(image.imageDisplayType),
imageAnimationType = image.imageAnimationType.map(convertImageAnimationType),
)
def convertImageDisplayType(
imageDisplayType: onboardingthrift.ImageDisplayType
): ImageDisplayType =
imageDisplayType match
def convertImageAnimationType(
imageAnimationType: onboardingthrift.ImageAnimationType
): ImageAnimationType =
imageAnimationType match
def convertImageVariant(imageVariant: onboardingthrift.ImageVariant): ImageVariant =
ImageVariant(
url = imageVariant.url,
width = imageVariant.width,
height = imageVariant.height,
palette = None)
def convertButtonAction(
buttonAction: onboardingthrift.ButtonAction
): MessageTextAction =
MessageTextAction(
buttonAction.text,
MessageAction(
dismissOnClick = buttonAction.dismissOnClick.getOrElse(true),
url = getActionUrl(buttonAction),
clientEventInfo = Some(convertClientEventInfo(buttonAction.clientEventInfo)),
onClickCallbacks = buttonAction.callbacks.map(_.map(convertCallback).toList)
)
)
private def getActionUrl(buttonAction: onboardingthrift.ButtonAction) =
buttonAction.buttonBehavior match
def convertRichText(
richText: com.twitter.onboarding.injections.thriftscala.RichText
): RichText
def convertAlignment(alignment: onboardingthrift.RichTextAlignment): RichTextAlignment =
alignment match
def convertRef(ref: onboardingthrift.ReferenceObject): ReferenceObject =
ref match
def convertFormat(format: onboardingthrift.RichTextFormat): RichTextFormat =
format match
def convertSocialContext(socialContext: onboardingthrift.RichText): SocialContext =
GeneralContext(
contextType = FollowGeneralContextType,
text = socialContext.text,
url = None,
contextImageUrls = None,
landingUrl = None)
def convertUserFacePile(
userFacepile: onboardingthrift.PromptUserFacepile
): UserFacepile =
UserFacepile(
userIds = userFacepile.userIds.toList,
featuredUserIds = userFacepile.featuredUserIds.toList,
action = userFacepile.action.map(convertButtonAction),
actionType = userFacepile.actionType.map(convertUserFacePileActionType),
displaysFeaturingText = userFacepile.displaysFeaturingText,
displayType = Some(LargeUserFacepileDisplayType)
)
private def convertUserFacePileActionType(
actionType: onboardingthrift.FacepileActionType
): MessageActionType =
actionType match
def convertHalfCoverDisplayType(
displayType: onboardingthrift.HalfCoverDisplayType
): HalfCoverDisplayType =
displayType match
def convertDismissInfo(dismissInfo: onboardingthrift.DismissInfo): DismissInfo =
DismissInfo(dismissInfo.callbacks.map(_.map(convertCallback)))
def convertCoverCta(
buttonAction: onboardingthrift.ButtonAction
): CoverCta =
CoverCta(
buttonAction.text,
ctaBehavior = convertCoverCtaBehavior(buttonAction.buttonBehavior),
callbacks = buttonAction.callbacks.map(_.map(convertCallback).toList),
clientEventInfo = Some(convertClientEventInfo(buttonAction.clientEventInfo)),
icon = buttonAction.icon.map(covertHorizonIcon),
buttonStyle = buttonAction.buttonStyle.map(covertButtonStyle)
)
private def convertCoverCtaBehavior(
behavior: onboardingthrift.ButtonBehavior
): CoverCtaBehavior =
behavior match
def covertButtonStyle(bStyle: onboardingthrift.CtaButtonStyle): ButtonStyle =
bStyle match
def covertHorizonIcon(icon: onboardingthrift.HorizonIcon): HorizonIcon =
icon match
def convertUrl(url: onboardingthrift.Url): Url
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\RelevancePromptConversions.scala
class to convert Relevance Prompt related onboarding thrift to product-mixer models
*/
object RelevancePromptConversions
def convertContent(
candidate: onboardingthrift.RelevancePrompt
): RelevancePromptContent =
RelevancePromptContent(
displayType = convertDisplayType(candidate.displayType),
title = candidate.title.text,
confirmation = buildConfirmation(candidate),
isRelevantText = candidate.isRelevantButton.text,
notRelevantText = candidate.notRelevantButton.text,
isRelevantCallback = convertCallbacks(candidate.isRelevantButton.callbacks),
notRelevantCallback = convertCallbacks(candidate.notRelevantButton.callbacks),
isRelevantFollowUp = None,
notRelevantFollowUp = None
)
// Based on com.twitter.timelinemixer.injection.model.candidate.OnboardingRelevancePromptDisplayType#fromThrift
def convertDisplayType(
displayType: onboardingthrift.RelevancePromptDisplayType
): RelevancePromptDisplayType =
displayType match
def buildConfirmation(candidate: onboardingthrift.RelevancePrompt): String
def buttonToDismissFeedbackText(button: onboardingthrift.ButtonAction): Option[String]
def convertCallbacks(onboardingCallbacks: Option[Seq[onboardingthrift.Callback]]): Callback
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\TilesCarouselConversions.scala
object TilesCarouselConversions
def convertTile(tile: onboardingthrift.Tile, id: Long): TileItem
def convertTileBadge(badge: onboardingthrift.Badge): Badge
def convertModuleHeader(header: onboardingthrift.TilesCarouselHeader): ModuleHeader
def convertRosettaColor(color: onboardingthrift.RosettaColor): RosettaColor =
color match
class UnknownThriftEnumException(enumName: String)
extends Exception(s"Unknown Thrift Enum Found: $
class UnsupportedTileCarouselConversionException(UnsupportedTileType: String)
extends Exception(s"Unsupported Tile Type Found: $
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\icon\HorizonIconBuilder.scala
class HorizonIconBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
icon: HorizonIcon)
extends BaseHorizonIconBuilder[Query, Candidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\ad\AdsCandidateUrtItemBuilder.scala
class AdsCandidateUrtItemBuilder[Query <: PipelineQuery](
tweetClientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, AdsTweetCandidate]] = None,
contextualTweetRefBuilder: Option[ContextualTweetRefBuilder[AdsTweetCandidate]] = None,
tweetDisplayType: TweetDisplayType = Tweet)
extends CandidateUrtEntryBuilder[Query, AdsCandidate, TimelineItem]
def apply(
pipelineQuery: Query,
candidate: AdsCandidate,
candidateFeatures: FeatureMap
): TimelineItem
def promotedMetadata(impression: adserver.AdImpression)
def convertDisclosureType(
disclosureType: adserver.DisclosureType
): DisclosureType = disclosureType match
def convertClickTrackingInfo(
clickTracking: adserver.ClickTrackingInfo
): ClickTrackingInfo = ClickTrackingInfo(
urlParams = clickTracking.urlParams.getOrElse(Map.empty),
urlOverride = clickTracking.urlOverride,
urlOverrideType = clickTracking.urlOverrideType.map
def prerollMetadata(adImpression: adserver.AdImpression): Option[PrerollMetadata]
def adMetadataContainer(
adImpression: adserver.AdImpression
): Option[AdMetadataContainer]
def convertSponsorshipType(
sponsorshipType: ads.SponsorshipType
): SponsorshipType = sponsorshipType match
def convertDisclaimerType(
disclaimerType: ads.DisclaimerType
): DisclaimerType = disclaimerType match
def convertDynamicPrerollType(
dynamicPrerollType: ads.DynamicPrerollType
): DynamicPrerollType =
dynamicPrerollType match
def convertMediaInfo(mediaInfo: ads.MediaInfo): MediaInfo
def convertVideoVariants(videoVariants: Seq[ads.VideoVariant]): Seq[VideoVariant]
def convertCallToAction(callToAction: ads.CallToAction): CallToAction
def convertPreroll(
preroll: ads.Preroll
): Preroll
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\DurationParamBuilder.scala
class DurationParamBuilder(
durationParam: Param[Duration])
extends BaseDurationBuilder[PipelineQuery]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\ShowAlertCandidateUrtItemBuilder.scala
object ShowAlertCandidateUrtItemBuilder
class ShowAlertCandidateUrtItemBuilder[-Query <: PipelineQuery](
alertType: ShowAlertType,
displayLocationBuilder: BaseShowAlertDisplayLocationBuilder[Query],
colorConfigBuilder: BaseShowAlertColorConfigurationBuilder[Query],
triggerDelayBuilder: Option[BaseDurationBuilder[Query]] = None,
displayDurationBuilder: Option[BaseDurationBuilder[Query]] = None,
clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, ShowAlertCandidate]] = None,
collapseDelayBuilder: Option[BaseDurationBuilder[Query]] = None,
userIdsBuilder: Option[BaseShowAlertUserIdsBuilder[Query]] = None,
richTextBuilder: Option[BaseRichTextBuilder[Query, ShowAlertCandidate]] = None,
iconDisplayInfoBuilder: Option[BaseShowAlertIconDisplayInfoBuilder[Query]] = None,
navigationMetadataBuilder: Option[BaseShowAlertNavigationMetadataBuilder[Query]] = None)
extends CandidateUrtEntryBuilder[
Query,
ShowAlertCandidate,
ShowAlert
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\StaticShowAlertColorConfigurationBuilder.scala
class StaticShowAlertColorConfigurationBuilder[-Query <: PipelineQuery](
configuration: ShowAlertColorConfiguration)
extends BaseShowAlertColorConfigurationBuilder[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\StaticShowAlertDisplayLocationBuilder.scala
class StaticShowAlertDisplayLocationBuilder[-Query <: PipelineQuery](
location: ShowAlertDisplayLocation)
extends BaseShowAlertDisplayLocationBuilder[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\StaticShowAlertIconDisplayInfoBuilder.scala
class StaticShowAlertIconDisplayInfoBuilder[-Query <: PipelineQuery](
iconDisplayInfo: ShowAlertIconDisplayInfo)
extends BaseShowAlertIconDisplayInfoBuilder[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\article\ArticleCandidateUrtItemBuilder.scala
object ArticleCandidateUrtItemBuilder
class ArticleCandidateUrtItemBuilder[
-Query <: PipelineQuery,
Candidate <: BaseArticleCandidate
](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate],
articleSeedType: ArticleSeedType = FollowingListSeed,
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, Candidate]
] = None,
displayType: Option[ArticleDisplayType] = None,
socialContextBuilder: Option[BaseSocialContextBuilder[Query, Candidate]] = None,
) extends CandidateUrtEntryBuilder[Query, Candidate, ArticleItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\audio_space\AudioSpaceCandidateUrtItemBuilder.scala
object AudioSpaceCandidateUrtItemBuilder
class AudioSpaceCandidateUrtItemBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UniversalNoun[Any]],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, UniversalNoun[Any]]
] = None)
extends CandidateUrtEntryBuilder[Query, AudioSpaceCandidate, AudioSpaceItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\card\CardCandidateUtrItemBuilder.scala
object CardCandidateUtrItemBuilder
class CardCandidateUtrItemBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CardCandidate],
cardUrlBuilder: BaseStr[Query, CardCandidate],
textBuilder: Option[BaseStr[Query, CardCandidate]],
subtextBuilder: Option[BaseStr[Query, CardCandidate]],
urlBuilder: Option[BaseUrlBuilder[Query, CardCandidate]],
cardDisplayType: Option[CardDisplayType],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, CardCandidate],
] = None)
extends CandidateUrtEntryBuilder[Query, CardCandidate, CardItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\commerce\CommerceProductCandidateUrtItemBuilder.scala
object CommerceProductCandidateUrtItemBuilder
class CommerceProductCandidateUrtItemBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CommerceProductCandidate],
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, CommerceProductCandidate]])
extends CandidateUrtEntryBuilder[
Query,
CommerceProductCandidate,
CommerceProductItem
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\commerce\CommerceProductGroupCandidateUrtItemBuilder.scala
object CommerceProductGroupCandidateUrtItemBuilder
class CommerceProductGroupCandidateUrtItemBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CommerceProductGroupCandidate],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, CommerceProductGroupCandidate]
]) extends CandidateUrtEntryBuilder[
Query,
CommerceProductGroupCandidate,
CommerceProductGroupItem
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\event_summary\EventCandidateUrtItemBuilder.scala
object EventCandidateUrtItemBuilder
class EventCandidateUrtItemBuilder[Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UnifiedEventCandidate],
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, UnifiedEventCandidate]] =
None)
extends CandidateUrtEntryBuilder[Query, UnifiedEventCandidate, TimelineItem]
def apply(
query: Query,
candidate: UnifiedEventCandidate,
candidateFeatures: FeatureMap
): TimelineItem
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\generic_summary\GenericSummaryActionBuilder.scala
object GenericSummaryActionBuilder
class GenericSummaryActionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
urlBuilder: BaseUrlBuilder[Query, Candidate],
clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, Candidate]] = None)
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\generic_summary\GenericSummaryCandidateUrtItemBuilder.scala
object GenericSummaryCandidateUrtItemBuilder
class GenericSummaryCandidateUrtItemBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, GenericSummaryCandidate],
headlineRichTextBuilder: BaseRichTextBuilder[Query, GenericSummaryCandidate],
displayType: GenericSummaryItemDisplayType,
genericSummaryContextCandidateUrtItemBuilder: Option[
GenericSummaryContextBuilder[Query, GenericSummaryCandidate]
] = None,
genericSummaryActionCandidateUrtItemBuilder: Option[
GenericSummaryActionBuilder[Query, GenericSummaryCandidate]
] = None,
timestamp: Option[Time] = None,
userAttributionIds: Option[Seq[Long]] = None,
media: Option[Media] = None,
promotedMetadata: Option[PromotedMetadata] = None,
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, GenericSummaryCandidate]] =
None)
extends CandidateUrtEntryBuilder[Query, GenericSummaryCandidate, GenericSummaryItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\generic_summary\GenericSummaryContextBuilder.scala
class GenericSummaryContextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
richTextBuilder: BaseRichTextBuilder[Query, Candidate],
icon: Option[HorizonIcon] = None)
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\icon_label\IconLabelCandidateUrtItemBuilder.scala
object IconLabelCandidateUrtItemBuilder
class IconLabelCandidateUrtItemBuilder[-Query <: PipelineQuery, Candidate <: LabelCandidate](
richTextBuilder: BaseRichTextBuilder[Query, Candidate],
icon: Option[HorizonIcon] = None,
entities: Option[List[RichTextEntity]] = None,
clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, Candidate]] = None,
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, Candidate]] = None)
extends CandidateUrtEntryBuilder[Query, Candidate, IconLabelItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\message\CompactPromptCandidateUrtItemStringCenterBuilder.scala
object CompactPromptCandidateUrtItemStringCenterBuilder
class CompactPromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CompactPromptCandidate],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, CompactPromptCandidate]
] = None,
headerTextBuilder: BaseStr[Query, CompactPromptCandidate],
bodyTextBuilder: Option[BaseStr[Query, CompactPromptCandidate]] = None,
headerRichTextBuilder: Option[BaseRichTextBuilder[Query, CompactPromptCandidate]] = None,
bodyRichTextBuilder: Option[BaseRichTextBuilder[Query, CompactPromptCandidate]] = None)
extends CandidateUrtEntryBuilder[Query, CompactPromptCandidate, MessagePromptItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\message\InlinePromptCandidateUrtItemStringCenterBuilder.scala
object InlinePromptCandidateUrtItemStringCenterBuilder
class InlinePromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, InlinePromptCandidate],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, InlinePromptCandidate]
] = None,
headerTextBuilder: BaseStr[Query, InlinePromptCandidate],
bodyTextBuilder: Option[BaseStr[Query, InlinePromptCandidate]] = None,
headerRichTextBuilder: Option[BaseRichTextBuilder[Query, InlinePromptCandidate]] = None,
bodyRichTextBuilder: Option[BaseRichTextBuilder[Query, InlinePromptCandidate]] = None,
primaryMessageTextActionBuilder: Option[
MessageTextActionBuilder[Query, InlinePromptCandidate]
] = None,
secondaryMessageTextActionBuilder: Option[
MessageTextActionBuilder[Query, InlinePromptCandidate]
] = None,
socialContextBuilder: Option[BaseSocialContextBuilder[Query, InlinePromptCandidate]] = None,
userFacePileBuilder: Option[
UserFacePileBuilder
] = None)
extends CandidateUrtEntryBuilder[Query, InlinePromptCandidate, MessagePromptItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\message\MessageTextActionBuilder.scala
object MessageTextActionBuilder
class MessageTextActionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
textBuilder: BaseStr[Query, Candidate],
dismissOnClick: Boolean,
url: Option[String] = None,
clientEventInfo: Option[ClientEventInfo] = None,
onClickCallbacks: Option[List[Callback]] = None)
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\message\UserFacePileBuilder.scala
class UserFacePileBuilder(
userIds: Seq[Long],
featuredUserIds: Seq[Long],
action: Option[MessageTextAction],
actionType: Option[MessageActionType],
displaysFeaturingText: Option[Boolean],
displayType: Option[UserFacepileDisplayType])
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\moment\MomentAnnotationCandidateUrtItemBuilder.scala
object MomentAnnotationCandidateUrtItemBuilder
class MomentAnnotationCandidateUrtItemBuilder[Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, MomentAnnotationCandidate],
annotationTextRichTextBuilder: BaseRichTextBuilder[Query, MomentAnnotationCandidate],
annotationHeaderRichTextBuilder: BaseRichTextBuilder[Query, MomentAnnotationCandidate],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, MomentAnnotationCandidate]
] = None,
) extends CandidateUrtEntryBuilder[Query, MomentAnnotationCandidate, MomentAnnotationItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\relevance_prompt\RelevancePromptCandidateUrtItemStringCenterBuilder.scala
object RelevancePromptCandidateUrtItemStringCenterBuilder
class RelevancePromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, RelevancePromptCandidate],
titleTextBuilder: BaseStr[Query, RelevancePromptCandidate],
confirmationTextBuilder: BaseStr[Query, RelevancePromptCandidate],
isRelevantTextBuilder: BaseStr[Query, RelevancePromptCandidate],
notRelevantTextBuilder: BaseStr[Query, RelevancePromptCandidate],
displayType: RelevancePromptDisplayType,
isRelevantCallback: Callback,
notRelevantCallback: Callback,
isRelevantFollowUp: Option[RelevancePromptFollowUpFeedbackType] = None,
notRelevantFollowUp: Option[RelevancePromptFollowUpFeedbackType] = None,
impressionCallbacks: Option[List[Callback]] = None)
extends CandidateUrtEntryBuilder[Query, RelevancePromptCandidate, PromptItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\suggestion\SpellingSuggestionCandidateUrtItemBuilder.scala
object SpellingSuggestionCandidateUrtItemBuilder
class SpellingSuggestionCandidateUrtItemBuilder[Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, SpellingSuggestionCandidate],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, SpellingSuggestionCandidate]
] = None,
) extends CandidateUrtEntryBuilder[Query, SpellingSuggestionCandidate, SpellingItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\tile\TileCandidateUrtItemBuilder.scala
object TileCandidateUrtItemBuilder
class TileCandidateUrtItemBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, PromptCarouselTileCandidate],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, PromptCarouselTileCandidate]
] = None)
extends CandidateUrtEntryBuilder[Query, PromptCarouselTileCandidate, TileItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\ParamTopicDisplayTypeBuilder.scala
object TopicCandidateDisplayType extends Enumeration
class ParamTopicDisplayTypeBuilder(
displayTypeParam: FSEnumParam[TopicCandidateDisplayType.type])
extends BaseTopicDisplayTypeBuilder[PipelineQuery, TopicCandidate]
def apply(
query: PipelineQuery,
candidate: TopicCandidate,
candidateFeatures: FeatureMap
): Option[TopicDisplayType]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\ParamTopicFunctionalityTypeBuilder.scala
object TopicFunctionalityTypeParamValue extends Enumeration
class ParamTopicFunctionalityTypeBuilder(
functionalityTypeParam: FSEnumParam[TopicFunctionalityTypeParamValue.type])
extends BaseTopicFunctionalityTypeBuilder[PipelineQuery, TopicCandidate]
def apply(
query: PipelineQuery,
candidate: TopicCandidate,
candidateFeatures: FeatureMap
): Option[TopicFunctionalityType]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\StaticTopicDisplayTypeBuilder.scala
class StaticTopicDisplayTypeBuilder(
displayType: TopicDisplayType)
extends BaseTopicDisplayTypeBuilder[PipelineQuery, BaseTopicCandidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\StaticTopicFunctionalityTypeBuilder.scala
class StaticTopicFunctionalityTypeBuilder(
functionalityType: TopicFunctionalityType)
extends BaseTopicFunctionalityTypeBuilder[PipelineQuery, BaseTopicCandidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\TopicCandidateUrtItemBuilder.scala
object TopicCandidateUrtItemBuilder
class TopicCandidateUrtItemBuilder[-Query <: PipelineQuery, Candidate <: BaseTopicCandidate](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate],
topicFunctionalityTypeBuilder: Option[BaseTopicFunctionalityTypeBuilder[Query, Candidate]] = None,
topicDisplayTypeBuilder: Option[BaseTopicDisplayTypeBuilder[Query, Candidate]] = None,
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, Candidate]
] = None)
extends CandidateUrtEntryBuilder[Query, Candidate, TopicItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\VerticalGridTopicCandidateUrtItemBuilder.scala
class VerticalGridTopicCandidateUrtItemBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, TopicCandidate],
verticalGridItemTopicFunctionalityType: VerticalGridItemTopicFunctionalityType,
verticalGridItemTileStyle: VerticalGridItemTileStyle,
urlBuilder: Option[BaseUrlBuilder[Query, TopicCandidate]] = None,
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, TopicCandidate]
] = None)
extends CandidateUrtEntryBuilder[Query, TopicCandidate, VerticalGridItem]
def apply(
query: Query,
topicCandidate: TopicCandidate,
candidateFeatures: FeatureMap
): VerticalGridItem
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\trend\TrendCandidateUrtItemBuilder.scala
object TrendCandidateUrtItemBuilder
class TrendCandidateUrtItemBuilder[Query <: PipelineQuery](
trendMetaDescriptionBuilder: TrendMetaDescriptionBuilder[Query, UnifiedTrendCandidate],
promotedMetadataBuilder: BasePromotedMetadataBuilder[Query, UnifiedTrendCandidate],
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UnifiedTrendCandidate],
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, UnifiedTrendCandidate]] =
None)
extends CandidateUrtEntryBuilder[Query, UnifiedTrendCandidate, TimelineItem]
def apply(
query: Query,
candidate: UnifiedTrendCandidate,
candidateFeatures: FeatureMap
): TimelineItem
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\trend\TrendMetaDescriptionBuilder.scala
class TrendMetaDescriptionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
promotedByMetaDescriptionStr: Str[PipelineQuery, UniversalNoun[Any]],
tweetCountMetaDescriptionStr: Str[PipelineQuery, UniversalNoun[Any]],
compactingNumberLocalizer: CompactingNumberLocalizer)
def apply(
query: Query,
candidate: Candidate,
candidateFeatures: FeatureMap
): Option[String]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\trend\TrendPromotedMetadataBuilder.scala
object TrendPromotedMetadataBuilder
extends BasePromotedMetadataBuilder[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidate: UniversalNoun[Any],
candidateFeatures: FeatureMap
): Option[PromotedMetadata]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\tweet\TweetCandidateUrtItemBuilder.scala
object TweetCandidateUrtItemBuilder
class TweetCandidateUrtItemBuilder[Query <: PipelineQuery, Candidate <: BaseTweetCandidate](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate],
displayType: TweetDisplayType = Tweet,
entryIdToReplaceBuilder: Option[BaseEntryIdToReplaceBuilder[Query, Candidate]] = None,
socialContextBuilder: Option[BaseSocialContextBuilder[Query, Candidate]] = None,
highlightsBuilder: Option[BaseTweetHighlightsBuilder[Query, Candidate]] = None,
innerTombstoneInfo: Option[TombstoneInfo] = None,
timelinesScoreInfoBuilder: Option[BaseTimelinesScoreInfoBuilder[Query, Candidate]] = None,
hasModeratedReplies: Option[Boolean] = None,
forwardPivot: Option[ForwardPivot] = None,
innerForwardPivot: Option[ForwardPivot] = None,
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, Candidate]] = None,
promotedMetadata: Option[PromotedMetadata] = None,
conversationAnnotation: Option[ConversationAnnotation] = None,
contextualTweetRefBuilder: Option[ContextualTweetRefBuilder[Candidate]] = None,
prerollMetadata: Option[PrerollMetadata] = None,
replyBadge: Option[Badge] = None,
destinationBuilder: Option[BaseUrlBuilder[Query, Candidate]] = None)
extends CandidateUrtEntryBuilder[Query, Candidate, TweetItem]
def apply(
pipelineQuery: Query,
tweetCandidate: Candidate,
candidateFeatures: FeatureMap
): TweetItem
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\twitter_list\TwitterListCandidateUrtItemBuilder.scala
object TwitterListCandidateUrtItemBuilder
class TwitterListCandidateUrtItemBuilder[-Query <: PipelineQuery](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, TwitterListCandidate],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, TwitterListCandidate]
] = None,
displayType: Option[TwitterListDisplayType] = None)
extends CandidateUrtEntryBuilder[Query, TwitterListCandidate, TwitterListItem]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\unified_trend_event\UnifiedTrendEventCandidateUrtItemBuilder.scala
class UnifiedTrendEventCandidateUrtItemBuilder[Query <: PipelineQuery](
eventCandidateUrtItemBuilder: EventCandidateUrtItemBuilder[Query],
trendCandidateUrtItemBuilder: TrendCandidateUrtItemBuilder[Query])
extends CandidateUrtEntryBuilder[Query, UnifiedTrendEventCandidate[Any], TimelineItem]
def apply(
query: Query,
candidate: UnifiedTrendEventCandidate[Any],
candidateFeatures: FeatureMap
): TimelineItem
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\user\UserCandidateUrtItemBuilder.scala
object UserCandidateUrtItemBuilder
class UserCandidateUrtItemBuilder[Query <: PipelineQuery, UserCandidate <: BaseUserCandidate](
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UserCandidate],
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, UserCandidate]
] = None,
displayType: UserDisplayType = User,
promotedMetadataBuilder: Option[BasePromotedMetadataBuilder[Query, UserCandidate]] = None,
socialContextBuilder: Option[BaseSocialContextBuilder[Query, UserCandidate]] = None,
reactiveTriggersBuilder: Option[BaseUserReactiveTriggersBuilder[Query, UserCandidate]] = None,
enableReactiveBlending: Option[Boolean] = None)
extends CandidateUrtEntryBuilder[Query, UserCandidate, UserItem]
def apply(
query: Query,
userCandidate: UserCandidate,
candidateFeatures: FeatureMap
): UserItem
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\ClientEventInfoBuilder.scala
class ClientEventInfoBuilder[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
component: String,
detailsBuilder: Option[BaseClientEventDetailsBuilder[Query, Candidate]] = None)
extends BaseClientEventInfoBuilder[Query, Candidate]
def apply(
query: Query,
candidate: Candidate,
candidateFeatures: FeatureMap,
element: Option[String]
): Option[ClientEventInfo] =
Some(
ClientEventInfo(
component = Some(component),
element = element,
details = detailsBuilder.flatMap(_.apply(query, candidate, candidateFeatures)),
action = None,
entityToken = None)
)
}
/**
* In rare cases you might not want to send client event info. For
* example, this might be set already on the client for some legacy
* timelines.
*/
object EmptyClientEventInfoBuilder
extends BaseClientEventInfoBuilder[PipelineQuery, UniversalNoun[Any]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\ConversationTweetClientEventDetailsBuilder.scala
object ConversationTweetClientEventDetailsBuilder
class ConversationTweetClientEventDetailsBuilder[-Query <: PipelineQuery](
injectionType: Option[String])
extends BaseClientEventDetailsBuilder[Query, BaseTweetCandidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\StaticUrlBuilder.scala
class StaticUrlBuilder(url: String, urlType: UrlType)
extends BaseUrlBuilder[PipelineQuery, UniversalNoun[Any]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\TopicClientEventDetailsBuilder.scala
object TopicClientEventDetailsBuilder
class TopicClientEventDetailsBuilder[-Query <: PipelineQuery]()
extends BaseClientEventDetailsBuilder[Query, BaseTopicCandidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\TopicNotInterestedFeedbackActionInfoBuilder.scala
class TopicNotInterestedFeedbackActionInfoBuilder[-Query <: PipelineQuery]()
extends BaseFeedbackActionInfoBuilder[Query, BaseTopicCandidate]
def apply(
query: Query,
topicCandidate: BaseTopicCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackActionInfo]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\TopicsToFollowModuleMetadataBuilder.scala
object TopicsToFollowModuleMetadataBuilder
def getCarouselRowCount(topicsCount: Int, maxCarouselRows: Int): Int =
Math.min(maxCarouselRows, (topicsCount / TopicsPerRow) + 1)
}
case class TopicsToFollowModuleMetadataBuilder(maxCarouselRowsParam: Param[Int])
extends BaseModuleMetadataBuilder[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[UniversalNoun[Any]]]
): ModuleMetadata
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\TopicTweetClientEventDetailsBuilder.scala
object TopicTweetClientEventDetailsBuilder
class TopicTweetClientEventDetailsBuilder[-Query <: PipelineQuery]()
extends BaseClientEventDetailsBuilder[Query, TweetCandidate]
def apply(
query: Query,
topicTweetCandidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[ClientEventDetails] =
Some(
ClientEventDetails(
conversationDetails = None,
timelinesDetails = Some(
TimelinesDetails(
injectionType = None,
controllerData = buildControllerData(getTopicId(query)),
sourceData = None)),
articleDetails = None,
liveEventDetails = None,
commerceDetails = None
))
private def getTopicId(query: Query): Option[Long]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\WhoToFollowFeedbackActionInfoBuilder.scala
class WhoToFollowFeedbackActionInfoBuilder[
-Query <: PipelineQuery,
-Candidate <: UniversalNoun[Any]
](
externalStringRegistry: ExternalStringRegistry,
stringCenter: StringCenter,
encodedFeedbackRequest: Option[String])
extends BaseFeedbackActionInfoBuilder[Query, Candidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\operation\CursorCandidateUrtOperationBuilder.scala
class CursorCandidateUrtOperationBuilder[-Query <: PipelineQuery](
cursorType: CursorType,
displayTreatment: Option[CursorDisplayTreatment] = None,
idToReplace: Option[Long] = None)
extends CandidateUrtEntryBuilder[Query, CursorCandidate, CursorOperation]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\promoted\FeaturePromotedMetadataBuilder.scala
class FeaturePromotedMetadataBuilder(adImpressionFeature: Feature[_, Option[ad.AdImpression]])
extends BasePromotedMetadataBuilder[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidate: UniversalNoun[Any],
candidateFeatures: FeatureMap
): Option[PromotedMetadata]
def convertAdMetadataContainer(
serializedAdMetadataContainer: ac.SerializedThrift
): Option[AdMetadataContainer] =
AdMetadataContainerSerializer.deserialize(serializedAdMetadataContainer).map
def convertDisclosureType(disclosureType: ad.DisclosureType): DisclosureType =
disclosureType match
def convertSponsorshipType(sponsorshipType: ads.SponsorshipType): SponsorshipType =
sponsorshipType match
def convertDisclaimerType(disclaimerType: ads.DisclaimerType): DisclaimerType =
disclaimerType match
def convertSkAdNetworkDataList(
skAdNetworkDataList: Seq[ads.SkAdNetworkData]
): Seq[SkAdNetworkData] = skAdNetworkDataList.map
def convertClickTrackingInfo(clickTracking: ad.ClickTrackingInfo): ClickTrackingInfo =
ClickTrackingInfo(
urlParams = clickTracking.urlParams.getOrElse(Map.empty),
urlOverride = clickTracking.urlOverride,
urlOverrideType = clickTracking.urlOverrideType.map
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\RichTextBuilder.scala
class RichTextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
textBuilder: BaseStr[Query, Candidate],
linkMap: Map[String, String],
rtl: Option[Boolean],
alignment: Option[RichTextAlignment],
linkTypeMap: Map[String, UrlType] = Map.empty)
extends BaseRichTextBuilder[Query, Candidate]
def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): RichText
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\RichTextMarkupUtil.scala
object out of
* a string with inline XML markup.
*
* This allows us to use a string like "Our system <a href="#promix">Product Mixer</a> is the <b>best</b>". Using
* inline markup like this is advantageous since the string can go through translation/localization and the
* translators will move the tags around in each language as appropriate.
*
* This class is derived from the OCF (onboarding/serve)'s RichTextUtil, but they diverge because:
* - We generate ProMix URT structures, not OCF URT structures
* - The OCF supports some internal OCF tags, like <data>
* - The OCF has additional legacy support and processing that we don't need
*/
object RichTextMarkupUtil
def richTextFromMarkup(
text: String,
linkMap: Map[String, String],
rtl: Option[Boolean] = None,
alignment: Option[RichTextAlignment] = None,
linkTypeMap: Map[String, UrlType] = Map.empty
): RichText
def adjustEntities(
entities: scala.collection.mutable.ArrayBuffer[RichTextEntity],
start: Int,
length: Int
): Unit
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\RichTextReferenceObjectBuilder.scala
trait RichTextReferenceObjectBuilder
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\RichTextRtlOptionBuilder.scala
trait RichTextRtlOptionBuilder[-Query <: PipelineQuery]
def apply(query: Query): Option[Boolean]
}
case class StaticRichTextRtlOptionBuilder[-Query <: PipelineQuery](rtlOption: Option[Boolean])
extends RichTextRtlOptionBuilder[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\StaticRichTextBuilder.scala
class StaticRichTextBuilder(richText: RichText)
extends BaseRichTextBuilder[PipelineQuery, UniversalNoun[Any]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextEntityProcessor.scala
object TwitterTextEntityProcessor
object DefaultReferenceObjectBuilder extends RichTextReferenceObjectBuilder
def apply(twitterEntity: Extractor.Entity): Option[ReferenceObject]
class TwitterTextEntityProcessor(
twitterTextReferenceObjectBuilder: RichTextReferenceObjectBuilder = DefaultReferenceObjectBuilder)
extends TwitterTextRendererProcessor
def process(
twitterTextRenderer: TwitterTextRenderer
): TwitterTextRenderer
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextFormatProcessor.scala
object TwitterTextFormatProcessor
class TwitterTextFormatProcessor(
formats: Set[RichTextFormat] = Set(Plain, Strong),
) extends TwitterTextRendererProcessor
def renderText(text: String): RichText
def process(richTextBuilder: TwitterTextRenderer): TwitterTextRenderer
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextRenderer.scala
object TwitterTextRenderer
def apply(
text: String,
rtl: Option[Boolean] = None,
alignment: Option[RichTextAlignment] = None
): TwitterTextRenderer
def fromRichText(richText: RichText): TwitterTextRenderer
def buildRichTextEntity(
text: String,
entity: TwitterTextRendererEntity[_]
): RichTextEntity
class TwitterTextRenderer(
rtl: Option[Boolean],
alignment: Option[RichTextAlignment],
)
def append(
string: String,
format: Option[RichTextFormat] = None,
refObject: Option[ReferenceObject] = None
): TwitterTextRenderer
def build: RichText
def transform(twitterTextProcessor: TwitterTextRendererProcessor): TwitterTextRenderer
def entities: Seq[TwitterTextRendererEntity[_]]
def mergeFormat(start: Int, end: Int, format: RichTextFormat): TwitterTextRenderer
def remove(start: Int, end: Int): TwitterTextRenderer = replace(start, end, "")
/**
* Replaces text, formatting, and refObject information in the given range.
* @param start Start index to apply formatting (inclusive)
* @param end End index to apply formatting (exclusive)
* @param string The new text to insert
* @param format The [[RichTextFormat]] assigned to the new text
* @param refObject The [[ReferenceObject]] assigned to the new text
* @return this
*/
def replace(
start: Int,
end: Int,
string: String,
format: Option[RichTextFormat] = None,
refObject: Option[ReferenceObject] = None
): TwitterTextRenderer
def setFormat(start: Int, end: Int, format: RichTextFormat): TwitterTextRenderer
def removeAndOffsetFormats(start: Int, end: Int, newSize: Int): Int
def validateRange(start: Int, end: Int): Unit
def setRefObject(start: Int, end: Int, refObject: ReferenceObject): TwitterTextRenderer
def removeAndOffsetRefObjects(start: Int, end: Int, newSize: Int): Int
def text: String
def buildEntities(
formats: List[TwitterTextRendererEntity[RichTextFormat]],
refs: List[TwitterTextRendererEntity[ReferenceObject]],
acc: List[TwitterTextRendererEntity[_]] = List()
): Seq[TwitterTextRendererEntity[_]]
class TwitterTextRendererEntity[+T] private[richtext] (
startIndex: Int,
endIndex: Int,
value: T)
def nonEmpty: Boolean = !isEmpty
def isEmpty: Boolean = startIndex == endIndex
private[richtext] def enclosedIn(start: Int, end: Int): Boolean
def encloses(start: Int, end: Int): Boolean
def endsBetween(start: Int, end: Int): Boolean
def offset(num: Int): TwitterTextRendererEntity[T]
def startsBetween(start: Int, end: Int): Boolean
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextRendererProcessor.scala
trait TwitterTextRendererProcessor
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextRichTextBuilder.scala
class TwitterTextRichTextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
stringBuilder: BaseStr[Query, Candidate],
alignment: Option[RichTextAlignment] = None,
formats: Set[RichTextFormat] = Set(Plain, Strong),
twitterTextRtlOptionBuilder: RichTextRtlOptionBuilder[Query] =
StaticRichTextRtlOptionBuilder[Query](None),
twitterTextReferenceObjectBuilder: RichTextReferenceObjectBuilder = DefaultReferenceObjectBuilder)
extends BaseRichTextBuilder[Query, Candidate]
def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): RichText
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\social_context\FeatureSocialContextBuilder.scala
class FeatureSocialContextBuilder(
socialContextFeature: Feature[_, Option[t.SocialContext]])
extends BaseSocialContextBuilder[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidate: UniversalNoun[Any],
candidateFeatures: FeatureMap
): Option[SocialContext]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\social_context\GeneralModuleSocialContextBuilder.scala
class works the same as [[GeneralSocialContextBuilder]] but passes a list of candidates
* into [[BaseModuleStr]] when rendering the string.
*/
case class GeneralModuleSocialContextBuilder[
-Query <: PipelineQuery,
-Candidate <: UniversalNoun[Any]
](
textBuilder: BaseModuleStr[Query, Candidate],
contextType: GeneralContextType,
url: Option[String] = None,
contextImageUrls: Option[List[String]] = None,
landingUrl: Option[Url] = None)
extends BaseModuleSocialContextBuilder[Query, Candidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\social_context\GeneralSocialContextBuilder.scala
class GeneralSocialContextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
textBuilder: BaseStr[Query, Candidate],
contextType: GeneralContextType,
url: Option[String] = None,
contextImageUrls: Option[List[String]] = None,
landingUrl: Option[Url] = None)
extends BaseSocialContextBuilder[Query, Candidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\social_context\WhoToFollowSocialContextBuilder.scala
class WhoToFollowSocialContextBuilder(
socialTextFeature: Feature[_, Option[String]],
contextTypeFeature: Feature[_, Option[h.ContextType]])
extends BaseSocialContextBuilder[PipelineQuery, UserCandidate]
def apply(
query: PipelineQuery,
candidate: UserCandidate,
candidateFeatures: FeatureMap
): Option[GeneralContext]
def convertContextType(contextType: Option[h.ContextType]): Option[GeneralContextType] =
contextType match
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\stringcenter\ModuleStr.scala
class works the same as [[Str]] but passes in a list of candidates to the
* [[BaseModuleStringCenterPlaceholderBuilder]] when building the placeholders.
*/
case class ModuleStr[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
text: ExternalString,
stringCenter: StringCenter,
stringCenterPlaceholderBuilder: Option[
BaseModuleStringCenterPlaceholderBuilder[Query, Candidate]
] = None)
extends BaseModuleStr[Query, Candidate]
def apply(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): String
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\stringcenter\Str.scala
class StrStatic(
text: String)
extends BaseStr[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidate: UniversalNoun[Any],
candidateFeatures: FeatureMap
): String = text
}
case class Str[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
text: ExternalString,
stringCenter: StringCenter,
stringCenterPlaceholderBuilder: Option[BaseStringCenterPlaceholderBuilder[Query, Candidate]] =
None)
extends BaseStr[Query, Candidate]
def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): String
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\FeatureModuleDisplayTypeBuilder.scala
class FeatureModuleDisplayTypeBuilder(
displayTypeFeature: Feature[_, Option[ModuleDisplayType]],
defaultDisplayType: ModuleDisplayType = VerticalConversation)
extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleDynamicShowMoreBehaviorRevealByCountBuilder.scala
class ModuleDynamicShowMoreBehaviorRevealByCountBuilder(
initialItemsCount: Int,
showMoreItemsCount: Int)
extends BaseModuleShowMoreBehaviorBuilder[PipelineQuery, UniversalNoun[Any]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleFooterBuilder.scala
class ModuleFooterBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
textBuilder: BaseStr[Query, Candidate],
urlBuilder: Option[BaseUrlBuilder[Query, Candidate]])
extends BaseModuleFooterBuilder[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Option[ModuleFooter]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleHeaderBuilder.scala
class ModuleHeaderBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
textBuilder: BaseStr[Query, Candidate],
isSticky: Option[Boolean] = None,
moduleHeaderIconBuilder: Option[BaseHorizonIconBuilder[Query, Candidate]] = None,
customIcon: Option[ImageVariant] = None,
moduleSocialContextBuilder: Option[BaseModuleSocialContextBuilder[Query, Candidate]] = None,
moduleHeaderDisplayTypeBuilder: BaseModuleHeaderDisplayTypeBuilder[Query, Candidate] =
ModuleHeaderDisplayTypeBuilder(Classic))
extends BaseModuleHeaderBuilder[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Option[ModuleHeader]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleHeaderDisplayTypeBuilder.scala
class ModuleHeaderDisplayTypeBuilder[
-Query <: PipelineQuery,
-Candidate <: UniversalNoun[Any]
](
moduleHeaderDisplayType: ModuleHeaderDisplayType = Classic)
extends BaseModuleHeaderDisplayTypeBuilder[Query, Candidate]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleIdGeneration.scala
trait is used for Module ID generation. Clients are safe to ignore this code unless they
* have a specific use case that requires hard-coded, specific, module ids. In that scenario,
* they can use the [[ManualModuleId]] case class.
*/
sealed trait ModuleIdGeneration
object ModuleIdGeneration
def apply(moduleId: Long): ModuleIdGeneration = moduleId match
class AutomaticUniqueModuleId private (moduleId: Long = 0L) extends ModuleIdGeneration
def withOffset(offset: Long): AutomaticUniqueModuleId = copy(
AutomaticUniqueModuleId.idRange.min + offset)
}
object AutomaticUniqueModuleId
def apply(): AutomaticUniqueModuleId = AutomaticUniqueModuleId(idRange.min)
def isAutomaticUniqueModuleId(moduleId: Long): Boolean = idRange.contains(moduleId)
}
/**
* ManualModuleId should normally not be required, but is helpful if the
* entryId of the module must be controlled. A scenario where this may be
* required is if a single candidate source returns multiple modules, and
* each module has the same presentation (e.g. Header, Footer). By setting
* different IDs, we signal to the platform that each module should be separate
* by using a different manual Id.
*/
case class ManualModuleId(override val moduleId: Long) extends ModuleIdGeneration
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleShowMoreBehaviorRevealByCountBuilder.scala
class ModuleShowMoreBehaviorRevealByCountBuilder(
initialItemsCountParam: Param[Int],
showMoreItemsCountParam: Param[Int])
extends BaseModuleShowMoreBehaviorBuilder[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidate: Seq[CandidateWithFeatures[UniversalNoun[Any]]]
): ModuleShowMoreBehavior
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ParamGatedModuleFooterBuilder.scala
class ParamGatedModuleFooterBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
enableParam: Param[Boolean],
enabledBuilder: BaseModuleFooterBuilder[Query, Candidate],
defaultBuilder: Option[BaseModuleFooterBuilder[Query, Candidate]] = None)
extends BaseModuleFooterBuilder[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Option[ModuleFooter]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ParamGatedModuleHeaderBuilder.scala
class ParamGatedModuleHeaderBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
enableParam: Param[Boolean],
enabledBuilder: BaseModuleHeaderBuilder[Query, Candidate],
defaultBuilder: Option[BaseModuleHeaderBuilder[Query, Candidate]] = None)
extends BaseModuleHeaderBuilder[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Option[ModuleHeader]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ParamWhoToFollowModuleDisplayTypeBuilder.scala
object WhoToFollowModuleDisplayType extends Enumeration
class ParamWhoToFollowModuleDisplayTypeBuilder(
displayTypeParam: Param[WhoToFollowModuleDisplayType.Value] =
StaticParam(WhoToFollowModuleDisplayType.Vertical))
extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[UniversalNoun[Any]]]
): ModuleDisplayType
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\StaticModuleDisplayTypeBuilder.scala
class StaticModuleDisplayTypeBuilder(displayType: ModuleDisplayType)
extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\TimelineModuleBuilder.scala
class TimelineModuleBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]](
entryNamespace: EntryNamespace,
displayTypeBuilder: BaseModuleDisplayTypeBuilder[Query, Candidate],
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate],
moduleIdGeneration: ModuleIdGeneration = AutomaticUniqueModuleId(),
feedbackActionInfoBuilder: Option[
BaseFeedbackActionInfoBuilder[Query, Candidate]
] = None,
headerBuilder: Option[BaseModuleHeaderBuilder[Query, Candidate]] = None,
footerBuilder: Option[BaseModuleFooterBuilder[Query, Candidate]] = None,
metadataBuilder: Option[BaseModuleMetadataBuilder[Query, Candidate]] = None,
showMoreBehaviorBuilder: Option[BaseModuleShowMoreBehaviorBuilder[Query, Candidate]] = None)
extends BaseTimelineModuleBuilder[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): TimelineModule
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\MetricDefinitions.scala
object MetricDefinition
class for all metric definitions
*/
sealed trait MetricDefinition
def toCsvField: Seq[String]
val metricDefinitionType: String
}
/**
* Pattern Metric Definition
* @param pattern the regex pattern for this metric
*/
case class NamedPatternMetricDefinition(
pattern: Seq[String])
extends MetricDefinition
def toCsvField: Seq[String] = pattern
override val metricDefinitionType: String = "NAMED_PATTERN"
}
/**
* Strainer Metric Definition
* @param strainerExpression a filter on top of client events
*/
case class StrainerMetricDefinition(
strainerExpression: String)
extends MetricDefinition
def toCsvField: Seq[String]
class LambdaMetricDefinition(
lambdaExpression: String)
extends MetricDefinition
def toCsvField: Seq[String]
class BucketRatioMetricDefinition(
numerator: String,
denominator: String)
extends MetricDefinition
def toCsvField: Seq[String]
object Metric
def fromLine(line: String): Metric
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\MetricGroup.scala
class MetricGroup(
id: Option[Long],
name: String,
description: String,
metrics: ListSet[Metric])
def toCsv: String
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\MetricTemplateCLIRunner.scala
class MetricTemplateCLIConfig(
// default values required for OptionParser
templateFileName: String = null,
outputFileName: String = null,
metricGroupName: String = null,
metricGroupDesc: String = null,
metricGroupId: Option[Long] = None,
absolutePath: Option[String] = None)
trait MetricTemplateCLIRunner
def templateDir: String
def placeholders: PlaceholdersMap
private val ProgramName = "Metric Template CLI"
private val VersionNumber = "1.0"
private def mkPath(fileName: String, absolutePath: Option[String]): String
def main(args: Array[String]): Unit
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\MetricTemplates.scala
class MatchedPlaceholder(outerKey: String, innerKey: Option[String] = None)
object MetricTemplates
def interpolate(inputTemplate: String, placeholders: PlaceholdersMap): Seq[String]
def getMatchedPlaceholders(inputTemplate: String): Seq[MatchedPlaceholder]
def unwrap(str: String): String =
str.stripPrefix("$
def wrap(str: String): String =
"\\$\\
def getPlaceholderKeyValues(
groupedPlaceholders: Map[String, Seq[MatchedPlaceholder]],
placeholders: PlaceholdersMap
): Seq[(String, Seq[Named])]
def crossProduct[T](seqOfSeqOfItems: Seq[Seq[T]]): Seq[List[T]]
def generateTemplateKey(matched: MatchedPlaceholder): String
def getFieldValue[T: ClassTag](
mirror: Mirror,
cls: T,
accessors: Map[String, MethodSymbol],
fieldName: String
): String
def caseAccessors[T: ClassTag](mirror: Mirror, cls: T): Map[String, MethodSymbol]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\PlaceholderConfig.scala
trait for all placeholder values
sealed trait Named
def name: String
}
case class Const(override val name: String) extends Named
// contains only client event patterns
case class CEPattern(
override val name: String,
client: String = "",
page: String = "",
section: String = "",
component: String = "",
element: String = "",
action: String = "",
strainer: String = "")
extends Named
def toString: String
class Topic(
override val name: String,
topicId: String = "")
extends Named
object PlaceholderConfig
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1QueryUserIdFeature.scala
object FeatureStoreV1QueryUserIdFeature
def apply[Query <: PipelineQuery, Value](
feature: FSv1Feature[UserId, Value],
legacyName: Option[String] = None,
defaultValue: Option[Value] = None,
enabledParam: Option[FSParam[Boolean]] = None
): FeatureStoreV1Feature[Query, Query, _ <: EntityId, Value]
with FeatureStoreV1QueryFeature[Query, _ <: EntityId, Value] =
FeatureStoreV1QueryFeature(feature, QueryUserIdEntity, legacyName, defaultValue, enabledParam)
}
object FeatureStoreV1QueryUserIdAggregateFeature
def apply[Query <: PipelineQuery](
featureGroup: TimelinesAggregationFrameworkFeatureGroup[UserId],
enabledParam: Option[FSParam[Boolean]] = None,
keepLegacyNames: Boolean = false,
featureNameTransform: Option[FeatureRenameTransform] = None
): FeatureStoreV1QueryFeatureGroup[Query, _ <: EntityId] =
FeatureStoreV1QueryFeatureGroup(
featureGroup,
QueryUserIdEntity,
enabledParam,
keepLegacyNames,
featureNameTransform)((implicitly[ClassTag[UserId]]))
}
object QueryUserIdEntity extends FeatureStoreV1QueryEntity[PipelineQuery, UserId]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.scala
object FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature
def apply[Query <: PipelineQuery, Value](
feature: FSv1Feature[EdgeEntityId[UserId, UserId], Value],
legacyName: Option[String] = None,
defaultValue: Option[Value] = None,
enabledParam: Option[FSParam[Boolean]] = None
): FeatureStoreV1CandidateFeature[Query, TweetCandidate, _ <: EntityId, Value] =
FeatureStoreV1CandidateFeature(
feature,
QueryUserIdTweetCandidateAuthorIdEntity,
legacyName,
defaultValue,
enabledParam)
}
object FeatureStoreV1QueryUserIdTweetCandidateAuthorIdAggregateFeature
def apply[Query <: PipelineQuery](
featureGroup: TimelinesAggregationFrameworkFeatureGroup[EdgeEntityId[UserId, UserId]],
enabledParam: Option[FSParam[Boolean]] = None,
keepLegacyNames: Boolean = false,
featureNameTransform: Option[FeatureRenameTransform] = None
): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] =
FeatureStoreV1CandidateFeatureGroup(
featureGroup,
QueryUserIdTweetCandidateAuthorIdEntity,
enabledParam,
keepLegacyNames,
featureNameTransform
)(implicitly[ClassTag[EdgeEntityId[UserId, UserId]]])
}
object QueryUserIdTweetCandidateAuthorIdEntity
extends FeatureStoreV1CandidateEntity[
PipelineQuery,
TweetCandidate,
EdgeEntityId[UserId, UserId]
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.scala
object FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature
def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate, Value](
feature: FSv1Feature[EdgeEntityId[UserId, TweetId], Value],
legacyName: Option[String] = None,
defaultValue: Option[Value] = None,
enabledParam: Option[FSParam[Boolean]] = None
): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] =
FeatureStoreV1CandidateFeature(
feature,
QueryUserIdTweetCandidateTweetIdEntity,
legacyName,
defaultValue,
enabledParam)
}
object FeatureStoreV1QueryUserIdTweetCandidateTweetIdAggregateFeature
def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate](
featureGroup: TimelinesAggregationFrameworkFeatureGroup[EdgeEntityId[UserId, TweetId]],
enabledParam: Option[FSParam[Boolean]] = None,
keepLegacyNames: Boolean = false,
featureNameTransform: Option[FeatureRenameTransform] = None
): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] =
FeatureStoreV1CandidateFeatureGroup(
featureGroup,
QueryUserIdTweetCandidateTweetIdEntity,
enabledParam,
keepLegacyNames,
featureNameTransform
)(implicitly[ClassTag[EdgeEntityId[UserId, TweetId]]])
}
object QueryUserIdTweetCandidateTweetIdEntity
extends FeatureStoreV1CandidateEntity[
PipelineQuery,
BaseTweetCandidate,
EdgeEntityId[UserId, TweetId]
]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1TweetCandidateAuthorIdFeature.scala
object FeatureStoreV1TweetCandidateAuthorIdFeature
def apply[Query <: PipelineQuery, Value](
feature: FSv1Feature[UserId, Value],
legacyName: Option[String] = None,
defaultValue: Option[Value] = None,
enabledParam: Option[FSParam[Boolean]] = None
): FeatureStoreV1CandidateFeature[Query, TweetCandidate, _ <: EntityId, Value] =
FeatureStoreV1CandidateFeature(
feature,
TweetCandidateAuthorIdEntity,
legacyName,
defaultValue,
enabledParam)
}
object FeatureStoreV1TweetCandidateAuthorIdAggregateFeature
def apply[Query <: PipelineQuery](
featureGroup: TimelinesAggregationFrameworkFeatureGroup[UserId],
enabledParam: Option[FSParam[Boolean]] = None,
keepLegacyNames: Boolean = false,
featureNameTransform: Option[FeatureRenameTransform] = None
): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] =
FeatureStoreV1CandidateFeatureGroup(
featureGroup,
TweetCandidateAuthorIdEntity,
enabledParam,
keepLegacyNames,
featureNameTransform
)(implicitly[ClassTag[UserId]])
}
object TweetCandidateAuthorIdEntity
extends FeatureStoreV1CandidateEntity[PipelineQuery, TweetCandidate, UserId]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1TweetCandidateTweetIdFeature.scala
object FeatureStoreV1TweetCandidateTweetIdFeature
def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate, Value](
feature: FSv1Feature[TweetId, Value],
legacyName: Option[String] = None,
defaultValue: Option[Value] = None,
enabledParam: Option[FSParam[Boolean]] = None
): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] =
FeatureStoreV1CandidateFeature(
feature,
TweetCandidateTweetIdEntity,
legacyName,
defaultValue,
enabledParam)
}
object FeatureStoreV1TweetCandidateTweetIdAggregateFeature
def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate](
featureGroup: TimelinesAggregationFrameworkFeatureGroup[TweetId],
enabledParam: Option[FSParam[Boolean]] = None,
keepLegacyNames: Boolean = false,
featureNameTransform: Option[FeatureRenameTransform] = None
): FeatureStoreV1CandidateFeatureGroup[Query, Candidate, _ <: EntityId] =
FeatureStoreV1CandidateFeatureGroup(
featureGroup,
TweetCandidateTweetIdEntity,
enabledParam,
keepLegacyNames,
featureNameTransform
)
}
object TweetCandidateTweetIdEntity
extends FeatureStoreV1CandidateEntity[PipelineQuery, BaseTweetCandidate, TweetId]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1UserCandidateUserIdFeature.scala
object FeatureStoreV1UserCandidateUserIdFeature
def apply[Query <: PipelineQuery, Candidate <: BaseUserCandidate, Value](
feature: FSv1Feature[UserId, Value],
legacyName: Option[String] = None,
defaultValue: Option[Value] = None,
enabledParam: Option[FSParam[Boolean]] = None
): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] =
FeatureStoreV1CandidateFeature(
feature,
UserCandidateUserIdEntity,
legacyName,
defaultValue,
enabledParam)
}
object UserCandidateUserIdEntity
extends FeatureStoreV1CandidateEntity[PipelineQuery, BaseUserCandidate, UserId]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\ads\AdvertiserBrandSafetySettingsFeatureHydrator.scala
object AdvertiserBrandSafetySettingsFeature
extends FeatureWithDefaultOnFailure[AdsCandidate, Option[ad.AdvertiserBrandSafetySettings]]
class AdvertiserBrandSafetySettingsFeatureHydrator[
Query <: PipelineQuery with AdsQuery,
Candidate <: AdsCandidate] @Inject() (
advertiserBrandSafetySettingsStore: ReadableStore[Long, ad.AdvertiserBrandSafetySettings])
extends CandidateFeatureHydrator[Query, Candidate]
def apply(
query: Query,
candidate: Candidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\decay\DecayCandidateFeatureHydrator.scala
object DecayScore extends Feature[UniversalNoun[Long], Double]
/**
* Hydrates snowflake ID candidates with a decay score:
*
* It is using exponential decay formula to calculate the score
* exp(k * age)
* where k = ln(0.5) / half-life
*
* Here is an example for half-life = 1 day
* For the brand new tweet it will be exp((ln(0.5)/1)*0) = 1
* For the tweet which was created 1 day ago it will be exp((ln(0.5)/1)*1) = 0.5
* For the tweet which was created 10 day ago it will be exp((ln(0.5)/1)*10) = 0.00097
*
* Reference: https://www.cuemath.com/exponential-decay-formula/
*
* @note This penalizes but does not filter out the candidate, so "stale" candidates can still appear.
*/
case class DecayCandidateFeatureHydrator[Candidate <: UniversalNoun[Long]](
halfLife: Param[Duration] = StaticParam[Duration](2.days),
resultFeature: Feature[UniversalNoun[Long], Double] = DecayScore)
extends CandidateFeatureHydrator[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidate: Candidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\param_gated\ParamGatedBulkCandidateFeatureHydrator.scala
class ParamGatedBulkCandidateFeatureHydrator[
-Query <: PipelineQuery,
Result <: UniversalNoun[Any]
](
enabledParam: Param[Boolean],
bulkCandidateFeatureHydrator: BulkCandidateFeatureHydrator[Query, Result])
extends BulkCandidateFeatureHydrator[Query, Result]
with Conditionally[Query]
def onlyIf(query: Query): Boolean =
Conditionally.and(query, bulkCandidateFeatureHydrator, query.params(enabledParam))
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Result]]
): Stitch[Seq[FeatureMap]] = bulkCandidateFeatureHydrator(query, candidates)
}
object ParamGatedBulkCandidateFeatureHydrator
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\param_gated\ParamGatedCandidateFeatureHydrator.scala
class ParamGatedCandidateFeatureHydrator[
-Query <: PipelineQuery,
-Result <: UniversalNoun[Any]
](
enabledParam: Param[Boolean],
candidateFeatureHydrator: CandidateFeatureHydrator[Query, Result])
extends CandidateFeatureHydrator[Query, Result]
with Conditionally[Query]
def onlyIf(query: Query): Boolean =
Conditionally.and(query, candidateFeatureHydrator, query.params(enabledParam))
override def apply(
query: Query,
candidate: Result,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = candidateFeatureHydrator.apply(query, candidate, existingFeatures)
}
object ParamGatedCandidateFeatureHydrator
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\param_gated\featurestorev1\ParamGatedFeatureStoreV1CandidateFeatureHydrator.scala
class ParamGatedFeatureStoreV1CandidateFeatureHydrator[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any]
](
enabledParam: Param[Boolean],
candidateFeatureHydrator: FeatureStoreV1CandidateFeatureHydrator[Query, Candidate])
extends FeatureStoreV1CandidateFeatureHydrator[Query, Candidate]
with Conditionally[Query]
def onlyIf(query: Query): Boolean =
Conditionally.and(query, candidateFeatureHydrator, query.params(enabledParam))
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[Seq[FeatureMap]] = candidateFeatureHydrator(query, candidates)
}
object ParamGatedFeatureStoreV1CandidateFeatureHydrator
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\qualityfactor_gated\QualityFactorGatedCandidateFeatureHydrator.scala
object QualityFactorGatedCandidateFeatureHydrator
class QualityFactorGatedCandidateFeatureHydrator[
-Query <: PipelineQuery with HasQualityFactorStatus,
Result <: UniversalNoun[Any]
](
pipelineIdentifier: ComponentIdentifier,
qualityFactorInclusiveThreshold: Param[Double],
candidateFeatureHydrator: CandidateFeatureHydrator[Query, Result])
extends CandidateFeatureHydrator[Query, Result]
with Conditionally[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\tweet_is_nsfw\TweetIsNsfwCandidateFeatureHydrator.scala
object IsNsfw extends FeatureWithDefaultOnFailure[TweetCandidate, Option[Boolean]]
def apply(
hasNsfwHighPrecisionLabel: Option[Boolean],
isNsfwUser: Option[Boolean],
isNsfwAdmin: Option[Boolean]
): Boolean
class TweetIsNsfwCandidateFeatureHydrator(
tweetypieStitchClient: TweetypieStitchClient,
tweetVisibilityPolicy: t.TweetVisibilityPolicy)
extends BulkCandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate]
with Logging
def features: Set[Feature[_, _]] = Set(IsNsfw)
private val NsfwLabelFields: Set[t.TweetInclude] = Set[t.TweetInclude](
// Tweet fields containing NSFW related attributes, in addition to what exists in coreData.
t.TweetInclude.TweetFieldId(t.Tweet.NsfwHighPrecisionLabelField.id),
t.TweetInclude.TweetFieldId(t.Tweet.CoreDataField.id)
)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[BaseTweetCandidate]]
): Stitch[Seq[FeatureMap]]
class IsNsfwFeatureHydrationFailure(message: String)
extends Exception(s"IsNsfwFeatureHydrationFailure($
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\tweet_tlx\TweetTLXScoreCandidateFeatureHydrator.scala
class TweetTLXScoreCandidateFeatureHydrator @Inject() (
column: TimelineScorerTweetScoresV1ClientColumn)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\tweet_tweetypie\TweetTweetypieCandidateFeatureHydrator.scala
object IsCommunityTweetFeature extends Feature[TweetCandidate, Boolean]
// Tweetypie VF Features
object HasTakedownFeature extends Feature[TweetCandidate, Boolean]
object HasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean]
object IsHydratedFeature extends Feature[TweetCandidate, Boolean]
object IsNarrowcastFeature extends Feature[TweetCandidate, Boolean]
object IsNsfwAdminFeature extends Feature[TweetCandidate, Boolean]
object IsNsfwFeature extends Feature[TweetCandidate, Boolean]
object IsNsfwUserFeature extends Feature[TweetCandidate, Boolean]
object IsNullcastFeature extends Feature[TweetCandidate, Boolean]
object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean]
object QuotedTweetHasTakedownFeature extends Feature[TweetCandidate, Boolean]
object QuotedTweetHasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean]
object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]]
object SourceTweetHasTakedownFeature extends Feature[TweetCandidate, Boolean]
object SourceTweetHasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean]
object TakedownCountryCodesFeature extends Feature[TweetCandidate, Set[String]]
object IsReplyFeature extends Feature[TweetCandidate, Boolean]
object InReplyToFeature extends Feature[TweetCandidate, Option[Long]]
object IsRetweetFeature extends Feature[TweetCandidate, Boolean]
object TweetTweetypieCandidateFeatureHydrator
class TweetTweetypieCandidateFeatureHydrator(
tweetypieStitchClient: TweetypieStitchClient,
safetyLevelPredicate: PipelineQuery => SafetyLevel)
extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate]
def apply(
query: PipelineQuery,
candidate: BaseTweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\tweet_visibility_reason\TweetVisibilityReasonBulkCandidateFeatureHydrator.scala
object VisibilityReason
extends FeatureWithDefaultOnFailure[TweetCandidate, Option[SPAM.FilteredReason]]
class TweetVisibilityReasonBulkCandidateFeatureHydrator @Inject() (
tweetypieStitchClient: TweetypieStitchClient,
safetyLevel: SPAM.SafetyLevel)
extends BulkCandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate]
with Logging
def features: Set[Feature[_, _]] = Set(VisibilityReason)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[BaseTweetCandidate]]
): Stitch[Seq[FeatureMap]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\async\AsyncQueryFeatureHydrator.scala
class AsyncQueryFeatureHydrator[-Query <: PipelineQuery] private[async] (
override val hydrateBefore: PipelineStepIdentifier,
queryFeatureHydrator: QueryFeatureHydrator[Query])
extends QueryFeatureHydrator[Query]
with AsyncHydrator
def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query)
}
/**
* A [[FeatureStoreV1QueryFeatureHydrator]] with [[AsyncHydrator]] that hydrated asynchronously for features
* to be before the step identified in [[hydrateBefore]]. We need a standalone class for feature store,
* different from the above as FStore hydrators are exempt from validations at run time.
*
* @param hydrateBefore the [[PipelineStepIdentifier]] step to make sure this feature is hydrated before.
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run asynchronously
* @tparam Query The domain model for the query or request
*/
case class AsyncFeatureStoreV1QueryFeatureHydrator[Query <: PipelineQuery] private[async] (
override val hydrateBefore: PipelineStepIdentifier,
featureStoreV1QueryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query])
extends FeatureStoreV1QueryFeatureHydrator[
Query
]
with AsyncHydrator
object AsyncQueryFeatureHydrator
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\cr_ml_ranker\CrMlRankerCommonQueryFeatureHydrator.scala
object CrMlRankerCommonFeatures extends Feature[PipelineQuery, t.CommonFeatures]
object CrMlRankerRankingConfig extends Feature[PipelineQuery, t.RankingConfig]
private[cr_ml_ranker] class CrMlRankerCommonQueryFeatureHydrator(
crMlRanker: t.CrMLRanker.MethodPerEndpoint,
rankingConfigSelector: RankingConfigBuilder)
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\cr_ml_ranker\CrMlRankerCommonQueryFeatureHydratorBuilder.scala
class CrMlRankerCommonQueryFeatureHydratorBuilder @Inject() (
crMlRanker: t.CrMLRanker.MethodPerEndpoint)
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\cr_ml_ranker\RankingConfigBuilder.scala
trait RankingConfigBuilder
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\impressed_tweets\ImpressedTweetsQueryFeatureHydrator.scala
object ImpressedTweets extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]]
class ImpressedTweetsQueryFeatureHydrator @Inject() (
tweetImpressionStore: ReadableStore[Long, ImpressionList])
extends QueryFeatureHydrator[PipelineQuery]
def hydrate(query: PipelineQuery): Stitch[FeatureMap]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\logged_in_only\LoggedInOnlyQueryFeatureHydrator.scala
class LoggedInOnlyQueryFeatureHydrator[-Query <: PipelineQuery, Result <: UniversalNoun[Any]](
queryFeatureHydrator: QueryFeatureHydrator[Query])
extends QueryFeatureHydrator[Query]
with Conditionally[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\param_gated\AsyncParamGatedQueryFeatureHydrator.scala
class AsyncParamGatedQueryFeatureHydrator[
-Query <: PipelineQuery,
Result <: UniversalNoun[Any]
](
enabledParam: Param[Boolean],
override val hydrateBefore: PipelineStepIdentifier,
queryFeatureHydrator: QueryFeatureHydrator[Query])
extends QueryFeatureHydrator[Query]
with Conditionally[Query]
with AsyncHydrator
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\param_gated\ParamGatedQueryFeatureHydrator.scala
class ParamGatedQueryFeatureHydrator[-Query <: PipelineQuery, Result <: UniversalNoun[Any]](
enabledParam: Param[Boolean],
queryFeatureHydrator: QueryFeatureHydrator[Query])
extends QueryFeatureHydrator[Query]
with Conditionally[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\param_gated\featurestorev1\AsyncParamGatedFeatureStoreV1QueryFeatureHydrator.scala
class AsyncParamGatedFeatureStoreV1QueryFeatureHydrator[
Query <: PipelineQuery,
Result <: UniversalNoun[Any]
](
enabledParam: Param[Boolean],
override val hydrateBefore: PipelineStepIdentifier,
queryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query])
extends FeatureStoreV1QueryFeatureHydrator[Query]
with Conditionally[Query]
with AsyncHydrator
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\param_gated\featurestorev1\ParamGatedFeatureStoreV1QueryFeatureHydrator.scala
class ParamGatedFeatureStoreV1QueryFeatureHydrator[
Query <: PipelineQuery,
Result <: UniversalNoun[Any]
](
enabledParam: Param[Boolean],
queryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query])
extends FeatureStoreV1QueryFeatureHydrator[Query]
with Conditionally[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\qualityfactor_gated\QualityFactorGatedQueryFeatureHydrator.scala
object QualityFactorGatedQueryFeatureHydrator
class QualityFactorGatedQueryFeatureHydrator[
-Query <: PipelineQuery with HasQualityFactorStatus,
Result <: UniversalNoun[Any]
](
pipelineIdentifier: ComponentIdentifier,
qualityFactorInclusiveThreshold: Param[Double],
queryFeatureHydrator: QueryFeatureHydrator[Query])
extends QueryFeatureHydrator[Query]
with Conditionally[Query]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\AdaptiveLongIntBloomFilterDedupFilter.scala
trait GetAdaptiveLongIntBloomFilter[Query <: PipelineQuery]
def apply(query: Query): Option[AdaptiveLongIntBloomFilter]
}
case class AdaptiveLongIntBloomFilterDedupFilter[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Long]
](
getBloomFilter: GetAdaptiveLongIntBloomFilter[Query])
extends Filter[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\ExcludedIdsFilter.scala
class ExcludedIdsFilter[
Query <: PipelineQuery with HasExcludedIds,
Candidate <: UniversalNoun[Long]
]() extends Filter[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\FeatureFilter.scala
object FeatureFilter
def fromFeature[Candidate <: UniversalNoun[Any]](
feature: Feature[Candidate, Boolean]
): Filter[PipelineQuery, Candidate] =
FeatureFilter.fromFeature(FilterIdentifier(feature.toString), feature)
/**
* Builds a Filter that keeps candidates when the provided Boolean Feature is present and True.
* If the Feature is missing or False, the candidate is removed.
*
*
def fromFeature[Candidate <: UniversalNoun[Any]](
identifier: FilterIdentifier,
feature: Feature[Candidate, Boolean]
): Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\FeatureValueConditionalFilter.scala
trait ShouldApplyFilter[FeatureValue]
def apply(feature: FeatureValue): Boolean
}
/**
* A filter that applies the [[filter]] for candidates for which [[shouldApplyFilter]] is true, and keeps the others
* @param feature feature to determine whether to apply underyling filter
* @param shouldApplyFilter function to determine whether to apply filter
* @param filter the actual filter to apply if shouldApplyFilter is True
* @tparam Query The domain model for the query or request
* @tparam Candidate The type of the candidates
* @tparam FeatureValueType
*/
case class FeatureValueConditionalFilter[
-Query <: PipelineQuery,
Candidate <: UniversalNoun[Any],
FeatureValueType
](
feature: Feature[Candidate, FeatureValueType],
shouldApplyFilter: ShouldApplyFilter[FeatureValueType],
filter: Filter[Query, Candidate])
extends Filter[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
object FeatureConditionalFilter
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\HasAuthorIdFeatureFilter.scala
class HasAuthorIdFeatureFilter[Candidate <: TweetCandidate]()
extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\ParamGatedFilter.scala
class ParamGatedFilter[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
enabledParam: Param[Boolean],
filter: Filter[Query, Candidate])
extends Filter[Query, Candidate]
with Filter.Conditionally[Query, Candidate]
def onlyIf(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): Boolean =
Conditionally.and(Filter.Input(query, candidates), filter, query.params(enabledParam))
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]] = filter.apply(query, candidates)
}
object ParamGatedFilter
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\PredicateFilter.scala
trait ShouldKeepCandidate[Candidate]
def apply(candidate: Candidate): Boolean
}
object PredicateFilter
def fromPredicate[Candidate <: UniversalNoun[Any]](
identifier: FilterIdentifier,
shouldKeepCandidate: ShouldKeepCandidate[Candidate]
): Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\SnowflakeIdAgeFilter.scala
class SnowflakeIdAgeFilter[Candidate <: UniversalNoun[Long]](
maxAgeParam: Param[Duration])
extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetAuthorCountryFilter.scala
class TweetAuthorCountryFilter[Candidate <: BaseTweetCandidate](
countryCodeFeature: Feature[Candidate, Option[String]])
extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetAuthorIsSelfFilter.scala
class TweetAuthorIsSelfFilter[Candidate <: BaseTweetCandidate]()
extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetIsNotReplyFilter.scala
class TweetIsNotReplyFilter[Candidate <: BaseTweetCandidate]()
extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetLanguageFilter.scala
class TweetLanguageFilter[Candidate <: BaseTweetCandidate](
languageCodeFeature: Feature[Candidate, Option[String]])
extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetVisibilityFilter.scala
object TweetVisibilityFilter
class TweetVisibilityFilter[Candidate <: BaseTweetCandidate](
tweetypieStitchClient: TweetypieStitchClient,
tweetVisibilityPolicy: TP.TweetVisibilityPolicy,
safetyLevel: SafetyLevel,
tweetIncludes: Set[TP.TweetInclude.TweetFieldId] = DefaultTweetIncludes)
extends Filter[PipelineQuery, Candidate]
with Logging
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\UrtUnorderedExcludeIdsCursorFilter.scala
class UrtUnorderedExcludeIdsCursorFilter[
Candidate <: UniversalNoun[Long],
Query <: PipelineQuery with HasPipelineCursor[UrtUnorderedExcludeIdsCursor]
]() extends Filter[Query, Candidate]
def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\list_visibility\ListVisibilityFilter.scala
class ListVisibilityFilter[Candidate <: UniversalNoun[Long]](
listsColumn: CoreOnListClientColumn)
extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\tweet_impression\TweetImpressionFilter.scala
class TweetImpressionFilter[Candidate <: BaseTweetCandidate](
) extends Filter[PipelineQuery, Candidate]
def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\DefinedCountryCodeGate.scala
object DefinedCountryCodeGate extends Gate[PipelineQuery]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\FeatureGate.scala
trait ShouldContinue[Value]
def apply(featureValue: Value): Boolean
/** If the [[Feature]] is a failure, use this value */
def onFailedFeature(t: Throwable): GateResult = GateResult.Stop
/**
* If the [[Feature]], or [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]],
* is missing use this value
*/
def onMissingFeature: GateResult = GateResult.Stop
}
object FeatureGate
def fromFeature(
feature: Feature[_, Boolean]
): FeatureGate[Boolean] =
FeatureGate.fromFeature(GateIdentifier(feature.toString), feature)
def fromNegatedFeature(
feature: Feature[_, Boolean]
): FeatureGate[Boolean] =
FeatureGate.fromNegatedFeature(GateIdentifier(feature.toString), feature)
def fromFeature(
gateIdentifier: GateIdentifier,
feature: Feature[_, Boolean]
): FeatureGate[Boolean] =
FeatureGate[Boolean](gateIdentifier, feature, identity)
def fromNegatedFeature(
gateIdentifier: GateIdentifier,
feature: Feature[_, Boolean]
): FeatureGate[Boolean] =
FeatureGate[Boolean](gateIdentifier, feature, !identity(_))
}
/**
* A [[Gate]] that is actuated based upon the value of the provided feature
*/
case class FeatureGate[Value](
gateIdentifier: GateIdentifier,
feature: Feature[_, Value],
continue: ShouldContinue[Value])
extends Gate[PipelineQuery]
def shouldContinue(query: PipelineQuery): Stitch[Boolean]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\FirstPageGate.scala
object FirstPageGate extends Gate[PipelineQuery with HasPipelineCursor[_]]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\NoCandidatesGate.scala
class NoCandidatesGate(scope: CandidateScope) extends QueryAndCandidateGate[PipelineQuery]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\NonEmptyAdsQueryStringGate.scala
object NonEmptyAdsQueryStringGate extends Gate[PipelineQuery with AdsQuery]
def shouldContinue(query: PipelineQuery with AdsQuery): Stitch[Boolean]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\NonEmptyCandidatesGate.scala
class NonEmptyCandidatesGate(scope: CandidateScope)
extends QueryAndCandidateGate[PipelineQuery]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\QualityFactorGate.scala
class QualityFactorGate(pipelineIdentifier: ComponentIdentifier, threshold: Double)
extends Gate[PipelineQuery with HasQualityFactorStatus]
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\any_candidates_without_feature\AnyCandidatesWithoutFeatureGate.scala
class AnyCandidatesWithoutFeatureGate(
override val identifier: GateIdentifier,
scope: CandidateScope,
missingFeature: Feature[_, _])
extends QueryAndCandidateGate[PipelineQuery]
def shouldContinue(
query: PipelineQuery,
candidates: Seq[CandidateWithDetails]
): Stitch[Boolean] =
Stitch.value(scope.partition(candidates).candidatesInScope.exists
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\ArticleCandidate.scala
trait BaseArticleCandidate extends UniversalNoun[Int]
/**
* Canonical ArticleCandidate model. Always prefer this version over all other variants.
*
* @note Any additional fields should be added as a [[com.twitter.product_mixer.core.feature.Feature]]
* on the candidate's [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]]. If the
* features come from the candidate source itself (as opposed to hydrated via a
* [[com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator]]),
* then [[com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig.featuresFromCandidateSourceTransformers]]
* can be used to extract features from the candidate source response.
*
* @note This class should always remain `final`. If for any reason the `final` modifier is removed,
* the equals() implementation must be updated in order to handle class inheritor equality
* (see note on the equals method below)
*/
final class ArticleCandidate private (
override val id: Int)
extends BaseArticleCandidate
def canEqual(that: Any): Boolean = that.isInstanceOf[ArticleCandidate]
/**
* High performance implementation of equals method that leverages:
* - Referential equality short circuit
* - Cached hashcode equality short circuit
* - Field values are only checked if the hashCodes are equal to handle the unlikely case
* of a hashCode collision
* - Removal of check for `that` being an equals-compatible descendant since this class is final
*
* @note `candidate.canEqual(this)` is not necessary because this class is final
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala,
* Chapter 28]] for discussion and design.
*/
override def equals(that: Any): Boolean =
that match
object construction. This prevents the
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field
* object mutability and hashCode implementations.
*
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode
* are immutable. This includes:
* - Inability to mutate the object reference on for an existing instantiated candidate
* (i.e. each field is a val)
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* data structure), assuming stable hashCode implementations for these objects
*
* @note In order for the hashCode to be consistent with object equality, `##` must be used for
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`.
*/
override val hashCode: Int = id.##
}
object ArticleCandidate
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\AudioSpaceCandidate.scala
class should always remain `final`. If for any reason the `final` modifier is removed,
* the equals() implementation must be updated in order to handle class inheritor equality
* (see note on the equals method below)
*/
final class AudioSpaceCandidate private (
override val id: String)
extends UniversalNoun[String]
def canEqual(that: Any): Boolean = that.isInstanceOf[AudioSpaceCandidate]
/**
* High performance implementation of equals method that leverages:
* - Referential equality short circuit
* - Cached hashcode equality short circuit
* - Field values are only checked if the hashCodes are equal to handle the unlikely case
* of a hashCode collision
* - Removal of check for `that` being an equals-compatible descendant since this class is final
*
* @note `candidate.canEqual(this)` is not necessary because this class is final
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala,
* Chapter 28]] for discussion and design.
*/
override def equals(that: Any): Boolean =
that match
object construction. This prevents the
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field
* object mutability and hashCode implementations.
*
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode
* are immutable. This includes:
* - Inability to mutate the object reference on for an existing instantiated candidate
* (i.e. each field is a val)
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* data structure), assuming stable hashCode implementations for these objects
*
* @note In order for the hashCode to be consistent with object equality, `##` must be used for
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`.
*/
override val hashCode: Int = id.##
}
object AudioSpaceCandidate
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\CardCandidate.scala
class should always remain `final`. If for any reason the `final` modifier is removed,
* the equals() implementation must be updated in order to handle class inheritor equality
* (see note on the equals method below)
*/
final class CardCandidate private (
override val id: String)
extends UniversalNoun[String]
def canEqual(that: Any): Boolean = that.isInstanceOf[CardCandidate]
/**
* High performance implementation of equals method that leverages:
* - Referential equality short circuit
* - Cached hashcode equality short circuit
* - Field values are only checked if the hashCodes are equal to handle the unlikely case
* of a hashCode collision
* - Removal of check for `that` being an equals-compatible descendant since this class is final
*
* @note `candidate.canEqual(this)` is not necessary because this class is final
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala,
* Chapter 28]] for discussion and design.
*/
override def equals(that: Any): Boolean =
that match
object construction. This prevents the
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field
* object mutability and hashCode implementations.
*
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode
* are immutable. This includes:
* - Inability to mutate the object reference on for an existing instantiated candidate
* (i.e. each field is a val)
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* data structure), assuming stable hashCode implementations for these objects
*
* @note In order for the hashCode to be consistent with object equality, `##` must be used for
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`.
*/
override val hashCode: Int = id.##
}
object CardCandidate
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\CommerceItemCandidate.scala
class should always remain `final`. If for any reason the `final` modifier is removed,
* the equals() implementation must be updated in order to handle class inheritor equality
* (see note on the equals method below)
*/
final class CommerceProductCandidate private (
override val id: Long)
extends UniversalNoun[Long]
def canEqual(that: Any): Boolean = that.isInstanceOf[CommerceProductCandidate]
/**
* High performance implementation of equals method that leverages:
* - Referential equality short circuit
* - Cached hashcode equality short circuit
* - Field values are only checked if the hashCodes are equal to handle the unlikely case
* of a hashCode collision
* - Removal of check for `that` being an equals-compatible descendant since this class is final
*
* @note `candidate.canEqual(this)` is not necessary because this class is final
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala,
* Chapter 28]] for discussion and design.
*/
override def equals(that: Any): Boolean =
that match
object construction. This prevents the
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field
* object mutability and hashCode implementations.
*
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode
* are immutable. This includes:
* - Inability to mutate the object reference on for an existing instantiated candidate
* (i.e. each field is a val)
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* data structure), assuming stable hashCode implementations for these objects
*
* @note In order for the hashCode to be consistent with object equality, `##` must be used for
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`.
*/
override val hashCode: Int = id.##
}
object CommerceProductCandidate
def apply(id: Long): CommerceProductCandidate = new CommerceProductCandidate(id)
}
/**
* Canonical CommerceProductGroupCandidate model which encapsulates information about a Single
* Product Type and its corresponding versions. Always prefer this version over all other variants.
* For example:
* iPhone 14
* - 128 GB, White
* - 128 GB, Blue
* - 1TB, Grey
* When a user clicks on a Product Group, they will be shown information about all of the possible
* versions of the top level product.
*
* @note Both CommerceProduct (above) and CommerceProductGroups can be shown in the same
* TimelineModule (i.e Carousel)
*
* @note Any additional fields should be added as a [[com.twitter.product_mixer.core.feature.Feature]]
* on the candidate's [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]]. If the
* features come from the candidate source itself (as opposed to hydrated via a
* [[com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator]]),
* then [[com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig.featuresFromCandidateSourceTransformers]]
* can be used to extract features from the candidate source response.
*
* @note This class should always remain `final`. If for any reason the `final` modifier is removed,
* the equals() implementation must be updated in order to handle class inheritor equality
* (see note on the equals method below)
*/
final class CommerceProductGroupCandidate private (
override val id: Long)
extends UniversalNoun[Long]
def canEqual(that: Any): Boolean = that.isInstanceOf[CommerceProductGroupCandidate]
/**
* High performance implementation of equals method that leverages:
* - Referential equality short circuit
* - Cached hashcode equality short circuit
* - Field values are only checked if the hashCodes are equal to handle the unlikely case
* of a hashCode collision
* - Removal of check for `that` being an equals-compatible descendant since this class is final
*
* @note `candidate.canEqual(this)` is not necessary because this class is final
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala,
* Chapter 28]] for discussion and design.
*/
override def equals(that: Any): Boolean =
that match
object construction. This prevents the
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field
* object mutability and hashCode implementations.
*
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode
* are immutable. This includes:
* - Inability to mutate the object reference on for an existing instantiated candidate
* (i.e. each field is a val)
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* - Inability to mutate the field object instance itself (i.e. each field is an immutable
* data structure), assuming stable hashCode implementations for these objects
* @note In order for the hashCode to be consistent with object equality, `##` must be used for
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`.
*/
override val hashCode: Int = id.##
}
object CommerceProductGroupCandidate
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\CursorCandidate.scala
trait CursorType
case object PreviousCursor extends CursorType
case object NextCursor extends CursorType
/**
* Canonical CursorCandidate model. Always prefer this version over all other variants.
*
* @note Any additional fields should be added as a [[com.twitter.product_mixer.core.feature.Feature]]
* on the candidate's [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]]. If the
* features come from the candidate source itself (as opposed to hydrated via a
* [[com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator]]),
* then [[com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig.featuresFromCandidateSourceTransformers]]
* can be used to extract features from the candidate source response.
*
* @note This class should always remain `final`. If for any reason the `final` modifier is removed,
* the equals() implementation must be updated in order to handle cl
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment