Created
April 2, 2023 13:21
-
-
Save B-R-P/9f1086d6f756c043bcf0f56f59040169 to your computer and use it in GitHub Desktop.
scala declarations in Twitter Algorithm
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
.\ann\src\main\scala\com\twitter\ann\annoy\AnnoyCommon.scala | |
object AnnoyCommon | |
def apply(scalaParams: AnnoyRuntimeParams): ServiceRuntimeParams | |
def invert(thriftParams: ServiceRuntimeParams): Try[AnnoyRuntimeParams] = | |
thriftParams match | |
class AnnoyRuntimeParams( | |
/* Number of vectors to evaluate while searching. A larger value will give more accurate results, but will take longer time to return. | |
* Default value would be numberOfTrees*numberOfNeigboursRequested | |
*/ | |
nodesToExplore: Option[Int]) | |
extends RuntimeParams | |
.\ann\src\main\scala\com\twitter\ann\annoy\RawAnnoyIndexBuilder.scala | |
object RawAnnoyIndexBuilder | |
def apply[D <: Distance[D]]( | |
dimension: Int, | |
numOfTrees: Int, | |
metric: Metric[D], | |
futurePool: FuturePool | |
): RawAppendable[AnnoyRuntimeParams, D] with Serialization | |
def annoyMetric(metric: Metric[_]): AnnoyLib.Metric | |
class RawAnnoyIndexBuilder[D <: Distance[D]]( | |
dimension: Int, | |
numOfTrees: Int, | |
metric: Metric[D], | |
indexBuilder: AnnoyLib.Builder, | |
futurePool: FuturePool) | |
extends RawAppendable[AnnoyRuntimeParams, D] | |
with Serialization | |
def append(embedding: EmbeddingVector): Future[Long] = | |
semaphore.acquireAndRun( | |
def toQueryable: Queryable[Long, AnnoyRuntimeParams, D] | |
def toDirectory(directory: ResourceId): Unit | |
def toDirectory(directory: AbstractFile): Unit | |
def toDirectory(directory: IndexOutputFile): Unit | |
def saveIndex(indexFile: IndexOutputFile): Unit | |
def saveMetadata(metadataFile: IndexOutputFile): Unit | |
.\ann\src\main\scala\com\twitter\ann\annoy\RawAnnoyQueryIndex.scala | |
object RawAnnoyQueryIndex | |
def apply[D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
futurePool: FuturePool, | |
directory: AbstractFile | |
): Queryable[Long, AnnoyRuntimeParams, D] | |
def annoyMetric(metric: Metric[_]): IndexType | |
def loadIndex( | |
indexFile: AbstractFile, | |
dimension: Int, | |
indexType: IndexType | |
): ANNIndex | |
class RawAnnoyQueryIndex[D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
numOfTrees: Int, | |
index: ANNIndex, | |
futurePool: FuturePool) | |
extends Queryable[Long, AnnoyRuntimeParams, D] | |
with AutoCloseable | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: AnnoyRuntimeParams | |
): Future[List[Long]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: AnnoyRuntimeParams | |
): Future[List[NeighborWithDistance[Long, D]]] | |
def neighboursToRequest( | |
numOfNeighbours: Int, | |
annoyParams: AnnoyRuntimeParams | |
): Int | |
.\ann\src\main\scala\com\twitter\ann\annoy\TypedAnnoyIndex.scala | |
object TypedAnnoyIndex | |
def indexBuilder[T, D <: Distance[D]]( | |
dimension: Int, | |
numOfTrees: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
futurePool: FuturePool | |
): Appendable[T, AnnoyRuntimeParams, D] with Serialization | |
def loadQueryableIndex[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
futurePool: FuturePool, | |
directory: AbstractFile | |
): Queryable[T, AnnoyRuntimeParams, D] | |
.\ann\src\main\scala\com\twitter\ann\annoy\TypedAnnoyIndexBuilderWithFile.scala | |
object TypedAnnoyIndexBuilderWithFile | |
def apply[T, D <: Distance[D]]( | |
dimension: Int, | |
numOfTrees: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
futurePool: FuturePool | |
): Appendable[T, AnnoyRuntimeParams, D] with Serialization | |
class TypedAnnoyIndexBuilderWithFile[T, D <: Distance[D]]( | |
indexBuilder: RawAppendable[AnnoyRuntimeParams, D] with Serialization, | |
store: WritableIndexIdFileStore[T]) | |
extends Appendable[T, AnnoyRuntimeParams, D] | |
with Serialization | |
def append(entity: EntityEmbedding[T]): Future[Unit] | |
def toDirectory(directory: ResourceId): Unit | |
def toDirectory(directory: AbstractFile): Unit | |
def toDirectory(directory: IndexOutputFile): Unit | |
def toQueryable: Queryable[T, AnnoyRuntimeParams, D] | |
.\ann\src\main\scala\com\twitter\ann\annoy\TypedAnnoyQueryIndexWithFile.scala | |
object TypedAnnoyQueryIndexWithFile | |
def apply[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
futurePool: FuturePool, | |
directory: AbstractFile | |
): Queryable[T, AnnoyRuntimeParams, D] | |
class TypedAnnoyQueryIndexWithFile[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
futurePool: FuturePool, | |
injection: Injection[T, Array[Byte]]) | |
extends QueryableDeserialization[ | |
T, | |
AnnoyRuntimeParams, | |
D, | |
Queryable[T, AnnoyRuntimeParams, D] | |
] | |
def fromDirectory(directory: AbstractFile): Queryable[T, AnnoyRuntimeParams, D] | |
.\ann\src\main\scala\com\twitter\ann\brute_force\BruteForceDeserialization.scala | |
class BruteForceDeserialization[T, D <: Distance[D]] @VisibleForTesting private[brute_force] ( | |
metric: Metric[D], | |
embeddingInjection: PersistedEmbeddingInjection[T], | |
futurePool: FuturePool, | |
thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding], | |
factory: (Metric[D], FuturePool, Iterator[EntityEmbedding[T]]) => BruteForceIndex[T, D]) | |
extends QueryableDeserialization[T, BruteForceRuntimeParams.type, D, BruteForceIndex[T, D]] | |
def this( | |
metric: Metric[D], | |
embeddingInjection: PersistedEmbeddingInjection[T], | |
futurePool: FuturePool, | |
thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding] | |
) | |
def fromDirectory( | |
serializationDirectory: AbstractFile | |
): BruteForceIndex[T, D] | |
.\ann\src\main\scala\com\twitter\ann\brute_force\BruteForceIndex.scala | |
object BruteForceRuntimeParams extends RuntimeParams | |
object BruteForceIndex | |
def apply[T, D <: Distance[D]]( | |
metric: Metric[D], | |
futurePool: FuturePool, | |
initialEmbeddings: Iterator[EntityEmbedding[T]] = Iterator() | |
): BruteForceIndex[T, D] | |
class BruteForceIndex[T, D <: Distance[D]] private ( | |
metric: Metric[D], | |
futurePool: FuturePool, | |
// visible for serialization | |
private[brute_force] val linkedQueue: ConcurrentLinkedQueue[EntityEmbedding[T]]) | |
extends Appendable[T, BruteForceRuntimeParams.type, D] | |
with Queryable[T, BruteForceRuntimeParams.type, D] | |
def append(embedding: EntityEmbedding[T]): Future[Unit] | |
def toQueryable: Queryable[T, BruteForceRuntimeParams.type, D] = this | |
override def query( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: BruteForceRuntimeParams.type | |
): Future[List[T]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: BruteForceRuntimeParams.type | |
): Future[List[NeighborWithDistance[T, D]]] | |
object SerializableBruteForceIndex | |
def apply[T, D <: Distance[D]]( | |
metric: Metric[D], | |
futurePool: FuturePool, | |
embeddingInjection: PersistedEmbeddingInjection[T], | |
thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding] | |
): SerializableBruteForceIndex[T, D] | |
class that wrapps a BruteForceIndex and provides a method for serialization. | |
* | |
* @param bruteForceIndex all queries and updates are sent to this index. | |
* @param embeddingInjection injection that can convert embeddings to thrift embeddings. | |
* @param thriftIteratorIO class that provides a way to write PersistedEmbeddings to disk | |
*/ | |
class SerializableBruteForceIndex[T, D <: Distance[D]]( | |
bruteForceIndex: BruteForceIndex[T, D], | |
embeddingInjection: PersistedEmbeddingInjection[T], | |
thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding]) | |
extends Appendable[T, BruteForceRuntimeParams.type, D] | |
with Queryable[T, BruteForceRuntimeParams.type, D] | |
with Serialization | |
def append(entity: EntityEmbedding[T]): Future[Unit] = | |
bruteForceIndex.append(entity) | |
override def toQueryable: Queryable[T, BruteForceRuntimeParams.type, D] = this | |
override def query( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: BruteForceRuntimeParams.type | |
): Future[List[T]] = | |
bruteForceIndex.query(embedding, numOfNeighbours, runtimeParams) | |
override def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: BruteForceRuntimeParams.type | |
): Future[List[NeighborWithDistance[T, D]]] = | |
bruteForceIndex.queryWithDistance(embedding, numOfNeighbours, runtimeParams) | |
override def toDirectory(serializationDirectory: ResourceId): Unit | |
def toDirectory(serializationDirectory: AbstractFile): Unit | |
def toDirectory(serializationDirectory: IndexOutputFile): Unit | |
.\ann\src\main\scala\com\twitter\ann\common\AnnInjections.scala | |
object AnnInjections | |
.\ann\src\main\scala\com\twitter\ann\common\Api.scala | |
object EmbeddingType | |
class EntityEmbedding[T](id: T, embedding: EmbeddingVector) | |
// Query interface for ANN | |
trait Queryable[T, P <: RuntimeParams, D <: Distance[D]] | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[T]] | |
/** | |
* ANN query for ids with distance. | |
* @param embedding: Embedding/Vector to be queried with. | |
* @param numOfNeighbors: Number of neighbours to be queried for. | |
* @param runtimeParams: Runtime params associated with index to control accuracy/latency etc. | |
* @return List of approximate nearest neighbour ids with distance from the query embedding. | |
*/ | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[NeighborWithDistance[T, D]]] | |
} | |
// Query interface for ANN over indexes that are grouped | |
trait QueryableGrouped[T, P <: RuntimeParams, D <: Distance[D]] extends Queryable[T, P, D] | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P, | |
key: Option[String] | |
): Future[List[T]] | |
/** | |
* ANN query for ids with distance. | |
* @param embedding: Embedding/Vector to be queried with. | |
* @param numOfNeighbors: Number of neighbours to be queried for. | |
* @param runtimeParams: Runtime params associated with index to control accuracy/latency etc. | |
* @param key: Optional key to lookup specific ANN index and perform query there | |
* @return List of approximate nearest neighbour ids with distance from the query embedding. | |
*/ | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P, | |
key: Option[String] | |
): Future[List[NeighborWithDistance[T, D]]] | |
} | |
/** | |
* Runtime params associated with index to control accuracy/latency etc while querying. | |
*/ | |
trait RuntimeParams | |
class NeighborWithDistance[T, D <: Distance[D]](neighbor: T, distance: D) | |
/** | |
* ANN query result with seed entity for which this neighbor was provided. | |
* @param seed: Seed Id for which ann query was called | |
* @param neighbor : Id of the neighbours | |
*/ | |
case class NeighborWithSeed[T1, T2](seed: T1, neighbor: T2) | |
/** | |
* ANN query result with distance with seed entity for which this neighbor was provided. | |
* @param seed: Seed Id for which ann query was called | |
* @param neighbor : Id of the neighbours | |
* @param distance: Distance of neighbour from query ex: D: CosineDistance, L2Distance, InnerProductDistance | |
*/ | |
case class NeighborWithDistanceWithSeed[T1, T2, D <: Distance[D]]( | |
seed: T1, | |
neighbor: T2, | |
distance: D) | |
trait RawAppendable[P <: RuntimeParams, D <: Distance[D]] | |
def append(embedding: EmbeddingVector): Future[Long] | |
/** | |
* Convert an Appendable to Queryable interface to query an index. | |
*/ | |
def toQueryable: Queryable[Long, P, D] | |
} | |
// Index building interface for ANN. | |
trait Appendable[T, P <: RuntimeParams, D <: Distance[D]] | |
def append(entity: EntityEmbedding[T]): Future[Unit] | |
/** | |
* Convert an Appendable to Queryable interface to query an index. | |
*/ | |
def toQueryable: Queryable[T, P, D] | |
} | |
// Updatable index interface for ANN. | |
trait Updatable[T] | |
.\ann\src\main\scala\com\twitter\ann\common\EmbeddingProducer.scala | |
trait EmbeddingProducer[T] | |
.\ann\src\main\scala\com\twitter\ann\common\IndexOutputFile.scala | |
class creates a wrapper around GCS filesystem and HDFS filesystem for the index | |
* generation job. It implements the basic methods required by the index generation job and hides | |
* the logic around handling HDFS vs GCS. | |
*/ | |
class IndexOutputFile(val abstractFile: AbstractFile, val resourceId: ResourceId) | |
def this(resourceId: ResourceId) | |
def this(abstractFile: AbstractFile) | |
def isAbstractFile(): Boolean | |
def createSuccessFile(): Unit | |
def isDirectory(): Boolean | |
def getPath(): String | |
def createFile(fileName: String): IndexOutputFile | |
def createDirectory(directoryName: String): IndexOutputFile | |
def getChild(fileName: String, isDirectory: Boolean = false): IndexOutputFile | |
def getOutputStream(): OutputStream | |
def getInputStream(): InputStream | |
def copyFrom(srcIn: InputStream): Unit | |
def writeIndexMetadata(annIndexMetadata: AnnIndexMetadata): Unit | |
def loadIndexMetadata(): AnnIndexMetadata | |
.\ann\src\main\scala\com\twitter\ann\common\IndexTransformer.scala | |
object IndexTransformer | |
def transformQueryable[T, P <: RuntimeParams, D <: Distance[D]]( | |
index: Queryable[Long, P, D], | |
store: ReadableStore[Long, T] | |
): Queryable[T, P, D] | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[T]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[NeighborWithDistance[T, D]]] | |
def transformAppendable[T, P <: RuntimeParams, D <: Distance[D]]( | |
index: RawAppendable[P, D], | |
store: Store[Long, T] | |
): Appendable[T, P, D] | |
def append(entity: EntityEmbedding[T]): Future[Unit] | |
def toQueryable: Queryable[T, P, D] | |
def transform1[ | |
Index <: RawAppendable[P, D] with Queryable[Long, P, D], | |
T, | |
P <: RuntimeParams, | |
D <: Distance[D] | |
]( | |
index: Index, | |
store: Store[Long, T] | |
): Queryable[T, P, D] with Appendable[T, P, D] | |
.\ann\src\main\scala\com\twitter\ann\common\MemoizedInEpochs.scala | |
class MemoizedInEpochs[K, V](f: K => Try[V]) extends Logging | |
def epoch(keys: Seq[K]): Seq[V] | |
.\ann\src\main\scala\com\twitter\ann\common\Metric.scala | |
trait Distance[D] extends Any with Ordered[D] | |
def distance: Float | |
} | |
case class L2Distance(distance: Float) extends AnyVal with Distance[L2Distance] | |
def compare(that: L2Distance): Int = | |
Ordering.Float.compare(this.distance, that.distance) | |
} | |
case class CosineDistance(distance: Float) extends AnyVal with Distance[CosineDistance] | |
def compare(that: CosineDistance): Int = | |
Ordering.Float.compare(this.distance, that.distance) | |
} | |
case class InnerProductDistance(distance: Float) | |
extends AnyVal | |
with Distance[InnerProductDistance] | |
def compare(that: InnerProductDistance): Int = | |
Ordering.Float.compare(this.distance, that.distance) | |
} | |
case class EditDistance(distance: Float) extends AnyVal with Distance[EditDistance] | |
def compare(that: EditDistance): Int = | |
Ordering.Float.compare(this.distance, that.distance) | |
} | |
object Metric | |
def fromThrift(metric: DistanceMetric): Metric[_ <: Distance[_]] | |
def toThrift(metric: Metric[_ <: Distance[_]]): DistanceMetric | |
def fromString(metricName: String): Metric[_ <: Distance[_]] | |
with Injection[_, ServiceDistance] | |
trait Metric[D <: Distance[D]] | |
def distance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): D | |
def absoluteDistance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): Float | |
def fromAbsoluteDistance(distance: Float): D | |
} | |
case object L2 extends Metric[L2Distance] with Injection[L2Distance, ServiceDistance] | |
def distance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): L2Distance | |
def fromAbsoluteDistance(distance: Float): L2Distance | |
def absoluteDistance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): Float = distance(embedding1, embedding2).distance | |
override def apply(scalaDistance: L2Distance): ServiceDistance | |
def invert(serviceDistance: ServiceDistance): Try[L2Distance] | |
object Cosine extends Metric[CosineDistance] with Injection[CosineDistance, ServiceDistance] | |
def distance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): CosineDistance | |
def fromAbsoluteDistance(distance: Float): CosineDistance | |
def absoluteDistance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): Float = distance(embedding1, embedding2).distance | |
override def apply(scalaDistance: CosineDistance): ServiceDistance | |
def invert(serviceDistance: ServiceDistance): Try[CosineDistance] | |
object InnerProduct | |
extends Metric[InnerProductDistance] | |
with Injection[InnerProductDistance, ServiceDistance] | |
def distance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): InnerProductDistance | |
def fromAbsoluteDistance(distance: Float): InnerProductDistance | |
def absoluteDistance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): Float = distance(embedding1, embedding2).distance | |
override def apply(scalaDistance: InnerProductDistance): ServiceDistance | |
def invert( | |
serviceDistance: ServiceDistance | |
): Try[InnerProductDistance] | |
object Edit extends Metric[EditDistance] with Injection[EditDistance, ServiceDistance] | |
def intDistance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector, | |
pos1: Int, | |
pos2: Int, | |
precomputedDistances: scala.collection.mutable.Map[(Int, Int), Int] | |
): Int | |
def distance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): EditDistance | |
def fromAbsoluteDistance(distance: Float): EditDistance | |
def absoluteDistance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): Float = distance(embedding1, embedding2).distance | |
override def apply(scalaDistance: EditDistance): ServiceDistance | |
def invert( | |
serviceDistance: ServiceDistance | |
): Try[EditDistance] | |
object MetricUtil | |
def dot( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): Float | |
def l2distance( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): Double | |
def cosineSimilarity( | |
embedding1: EmbeddingVector, | |
embedding2: EmbeddingVector | |
): Float | |
def norm( | |
embedding: EmbeddingVector | |
): EmbeddingVector | |
.\ann\src\main\scala\com\twitter\ann\common\QueryableById.scala | |
trait that allows you to query for nearest neighbors given an arbitrary type T1. This is | |
* in contrast to a regular com.twitter.ann.common.Appendable, which takes an embedding as the input | |
* argument. | |
* | |
* This interface uses the Stitch API for batching. See go/stitch for details on how to use it. | |
* | |
* @tparam T1 type of the query. | |
* @tparam T2 type of the result. | |
* @tparam P runtime parameters supported by the index. | |
* @tparam D distance function used in the index. | |
*/ | |
trait QueryableById[T1, T2, P <: RuntimeParams, D <: Distance[D]] | |
.\ann\src\main\scala\com\twitter\ann\common\QueryableByIdImplementation.scala | |
class QueryableByIdImplementation[T1, T2, P <: RuntimeParams, D <: Distance[D]]( | |
embeddingProducer: EmbeddingProducer[T1], | |
queryable: Queryable[T2, P, D]) | |
extends QueryableById[T1, T2, P, D] | |
def queryById( | |
id: T1, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Stitch[List[T2]] | |
def queryByIdWithDistance( | |
id: T1, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Stitch[List[NeighborWithDistance[T2, D]]] | |
def batchQueryById( | |
ids: Seq[T1], | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Stitch[List[NeighborWithSeed[T1, T2]]] | |
def batchQueryWithDistanceById( | |
ids: Seq[T1], | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Stitch[List[NeighborWithDistanceWithSeed[T1, T2, D]]] | |
.\ann\src\main\scala\com\twitter\ann\common\QueryableOperations.scala | |
object QueryableOperations | |
class Map[T, P <: RuntimeParams, D <: Distance[D]]( | |
val q: Queryable[T, P, D]) | |
def mapRuntimeParameters(f: P => P): Queryable[T, P, D] | |
.\ann\src\main\scala\com\twitter\ann\common\ReadWriteFuturePool.scala | |
trait ReadWriteFuturePool | |
def read[T](f: => T): Future[T] | |
def write[T](f: => T): Future[T] | |
} | |
object ReadWriteFuturePool | |
def apply(readPool: FuturePool, writePool: FuturePool): ReadWriteFuturePool | |
def apply(commonPool: FuturePool): ReadWriteFuturePool | |
class ReadWriteFuturePoolANN(readPool: FuturePool, writePool: FuturePool) | |
extends ReadWriteFuturePool | |
def read[T](f: => T): Future[T] | |
def write[T](f: => T): Future[T] | |
.\ann\src\main\scala\com\twitter\ann\common\Serialization.scala | |
trait Serialization | |
def toDirectory( | |
serializationDirectory: AbstractFile | |
): Unit | |
def toDirectory( | |
serializationDirectory: ResourceId | |
): Unit | |
} | |
/** | |
* Interface for reading a Queryable from a directory | |
* @tparam T the id of the embeddings | |
* @tparam Q type of the Queryable that is deserialized. | |
*/ | |
trait QueryableDeserialization[T, P <: RuntimeParams, D <: Distance[D], Q <: Queryable[T, P, D]] | |
.\ann\src\main\scala\com\twitter\ann\common\ServiceClientQueryable.scala | |
class ServiceClientQueryable[T, P <: RuntimeParams, D <: Distance[D]]( | |
service: Service[NearestNeighborQuery, NearestNeighborResult], | |
runtimeParamInjection: Injection[P, ServiceRuntimeParams], | |
distanceInjection: Injection[D, ServiceDistance], | |
idInjection: Injection[T, Array[Byte]]) | |
extends Queryable[T, P, D] | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[T]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[NeighborWithDistance[T, D]]] = | |
service | |
.apply( | |
NearestNeighborQuery( | |
embeddingSerDe.toThrift(embedding), | |
withDistance = true, | |
runtimeParamInjection(runtimeParams), | |
numOfNeighbors | |
) | |
) | |
.map | |
.\ann\src\main\scala\com\twitter\ann\common\ShardApi.scala | |
trait ShardFunction[T] | |
def apply(shards: Int, entity: EntityEmbedding[T]): Int | |
} | |
/** | |
* Randomly shards the embeddings based on number of total shards. | |
*/ | |
class RandomShardFunction[T] extends ShardFunction[T] | |
def apply(shards: Int, entity: EntityEmbedding[T]): Int | |
class ShardedAppendable[T, P <: RuntimeParams, D <: Distance[D]]( | |
indices: Seq[Appendable[T, P, D]], | |
shardFn: ShardFunction[T], | |
shards: Int) | |
extends Appendable[T, P, D] | |
def append(entity: EntityEmbedding[T]): Future[Unit] | |
def toQueryable: Queryable[T, P, D] | |
class ComposedQueryable[T, P <: RuntimeParams, D <: Distance[D]]( | |
indices: Seq[Queryable[T, P, D]]) | |
extends Queryable[T, P, D] | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[T]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[NeighborWithDistance[T, D]]] | |
.\ann\src\main\scala\com\twitter\ann\common\ShardedSerialization.scala | |
object ShardConstants | |
class ShardedSerialization( | |
shards: Seq[Serialization]) | |
extends Serialization | |
def toDirectory(directory: AbstractFile): Unit | |
def toDirectory(directory: ResourceId): Unit | |
def toDirectory(directory: IndexOutputFile): Unit | |
class ComposedQueryableDeserialization[T, P <: RuntimeParams, D <: Distance[D]]( | |
deserializationFn: (AbstractFile) => Queryable[T, P, D]) | |
extends QueryableDeserialization[T, P, D, Queryable[T, P, D]] | |
def fromDirectory(directory: AbstractFile): Queryable[T, P, D] | |
def accept(file: AbstractFile): Boolean = | |
file.getName.startsWith(ShardConstants.ShardPrefix) | |
}) | |
.asScala | |
.toList | |
val indices = shardDirs | |
.map | |
class ShardedIndexBuilderWithSerialization[T, P <: RuntimeParams, D <: Distance[D]]( | |
shardedIndex: ShardedAppendable[T, P, D], | |
shardedSerialization: ShardedSerialization) | |
extends Appendable[T, P, D] | |
with Serialization | |
def append(entity: EntityEmbedding[T]): Future[Unit] | |
def toDirectory(directory: AbstractFile): Unit | |
def toDirectory(directory: ResourceId): Unit | |
def toQueryable: Queryable[T, P, D] | |
.\ann\src\main\scala\com\twitter\ann\common\Task.scala | |
trait Task extends Closable | |
def run(): Future[Unit] | |
def task(): Future[Unit] | |
// Task interval | |
protected def taskInterval: Duration | |
/** | |
* Start the task after random jitter | |
*/ | |
final def jitteredStart(): Unit = synchronized | |
def startImmediately(): Unit = synchronized | |
def close(deadline: Time): Future[Unit] | |
.\ann\src\main\scala\com\twitter\ann\dataflow\offline\ANNIndexBuilderBeamJob.scala | |
trait ANNOptions extends DateRangeOptions | |
def getOutputPath(): String | |
def setOutputPath(value: String): Unit | |
@Description("If set, the index is grouped") | |
@Default.Boolean(false) | |
def getGrouped: Boolean | |
def setGrouped(value: Boolean): Unit | |
@Description( | |
"If set, a segment will be registered for the provided DAL dataset module which will trigger " + | |
"DAL registration.") | |
@Default.Boolean(false) | |
def getEnableDalRegistration: Boolean | |
def setEnableDalRegistration(value: Boolean): Unit | |
@Description( | |
"Output GCS path for the generated index. The OutputPath should be of the format " + | |
"'gs://user. | |
def getOutputDALPath: String | |
def setOutputDALPath(value: String): Unit | |
@Description("Get ANN index dataset name") | |
def getDatasetModuleName: String | |
def setDatasetModuleName(value: String): Unit | |
@Description("Get ANN index dataset owner role") | |
def getDatasetOwnerRole: String | |
def setDatasetOwnerRole(value: String): Unit | |
@Description("If set, index is written in <output>/<timestamp>") | |
@Default.Boolean(false) | |
def getOutputWithTimestamp: Boolean | |
def setOutputWithTimestamp(value: Boolean): Unit | |
@Description("File which contains a SQL query to retrieve embeddings from BQ") | |
def getDatasetSqlPath: String | |
def setDatasetSqlPath(value: String): Unit | |
@Description("Dimension of embedding in the input data. See go/ann") | |
def getDimension: Int | |
def setDimension(value: Int): Unit | |
@Description("The type of entity ID that is used with the embeddings. See go/ann") | |
def getEntityKind: String | |
def setEntityKind(value: String): Unit | |
@Description("The kind of index you want to generate (HNSW/Annoy/Brute Force/faiss). See go/ann") | |
def getAlgo: String | |
def setAlgo(value: String): Unit | |
@Description("Distance metric (InnerProduct/Cosine/L2). See go/ann") | |
def getMetric: String | |
def setMetric(value: String): Unit | |
@Description("Specifies how many parallel inserts happen to the index. See go/ann") | |
def getConcurrencyLevel: Int | |
def setConcurrencyLevel(value: Int): Unit | |
@Description( | |
"Used by HNSW algo. Larger value increases build time but will give better recall. See go/ann") | |
def getEfConstruction: Int | |
def setEfConstruction(value: Int): Unit | |
@Description( | |
"Used by HNSW algo. Larger value increases the index size but will give better recall. " + | |
"See go/ann") | |
def getMaxM: Int | |
def setMaxM(value: Int): Unit | |
@Description("Used by HNSW algo. Approximate number of elements that will be indexed. See go/ann") | |
def getExpectedElements: Int | |
def setExpectedElements(value: Int): Unit | |
@Description( | |
"Used by Annoy. num_trees is provided during build time and affects the build time and the " + | |
"index size. A larger value will give more accurate results, but larger indexes. See go/ann") | |
def getAnnoyNumTrees: Int | |
def setAnnoyNumTrees(value: Int): Unit | |
@Description( | |
"FAISS factory string determines the ANN algorithm and compression. " + | |
"See https://github.com/facebookresearch/faiss/wiki/The-index-factory") | |
def getFAISSFactoryString: String | |
def setFAISSFactoryString(value: String): Unit | |
@Description("Sample rate for training during creation of FAISS index. Default is 0.05f") | |
@Default.Float(0.05f) | |
def getTrainingSampleRate: Float | |
def setTrainingSampleRate(value: Float): Unit | |
} | |
/** | |
* Builds ANN index. | |
* | |
* The input embeddings are read from BigQuery using the input SQL query. The output from this SQL | |
* query needs to have two columns, "entityID" [Long] and "embedding" [List[Double]] | |
* | |
* Output directory supported is GCS bucket | |
*/ | |
object ANNIndexBuilderBeamJob extends ScioBeamJob[ANNOptions] | |
def configurePipeline(sc: ScioContext, opts: ANNOptions): Unit | |
def transformTableRowToKeyVal( | |
data: BaseEmbeddingData | |
): Option[KV[String, KV[Long, TEmbedding]]] | |
class OutputSink( | |
outDir: ResourceId, | |
isFaiss: Boolean, | |
outputDALPath: String, | |
enableDalRegistration: Boolean, | |
datasetModuleName: String, | |
datasetOwnerRole: String, | |
instant: Instant, | |
date: DateRange, | |
counterNameSpace: String) | |
extends PTransform[PCollection[Map[String, Iterable[KV[Long, TEmbedding]]]], PDone] | |
def expand(input: PCollection[Map[String, Iterable[KV[Long, TEmbedding]]]]): PDone | |
class BuildANNIndex(outDir: ResourceId, counterNameSpace: String) | |
extends DoFn[Map[String, Iterable[KV[Long, TEmbedding]]], Unit] | |
def transformKeyValToEmbeddingWithEntity[T <: EntityId]( | |
entityKind: EntityKind[T] | |
)( | |
keyVal: KV[Long, TEmbedding] | |
): EntityEmbedding[T] | |
def processElement[T <: EntityId, D <: Distance[D]]( | |
@Element dataGrouped: Map[String, Iterable[KV[Long, TEmbedding]]], | |
context: ProcessContext | |
): Unit | |
class BuildFaissANNIndex(outDir: ResourceId, counterNameSpace: String) | |
extends DoFn[Map[String, Iterable[KV[Long, TEmbedding]]], Unit] | |
def processElement[D <: Distance[D]]( | |
@Element dataGrouped: Map[String, Iterable[KV[Long, TEmbedding]]], | |
context: ProcessContext | |
): Unit | |
.\ann\src\main\scala\com\twitter\ann\dataflow\offline\BaseEmbeddingData.scala | |
trait BaseEmbeddingData | |
.\ann\src\main\scala\com\twitter\ann\experimental\Runner.scala | |
object Runner | |
def main(args: Array[String]): Unit | |
def time[T](fn: => T): (T, Long) | |
.\ann\src\main\scala\com\twitter\ann\faiss\FaissCommon.scala | |
object FaissCommon | |
def apply(scalaParams: FaissParams): ServiceRuntimeParams | |
def invert(thriftParams: ServiceRuntimeParams): Try[FaissParams] = | |
thriftParams match | |
def isValidFaissIndex(path: AbstractFile): Boolean | |
.\ann\src\main\scala\com\twitter\ann\faiss\FaissIndex.scala | |
class FaissParams( | |
nprobe: Option[Int], | |
quantizerEf: Option[Int], | |
quantizerKFactorRF: Option[Int], | |
quantizerNprobe: Option[Int], | |
ht: Option[Int]) | |
extends RuntimeParams | |
def toString: String = s"FaissParams($ | |
def toLibraryString: String = | |
Seq( | |
nprobe.map | |
object FaissIndex | |
def loadIndex[T, D <: Distance[D]]( | |
outerDimension: Int, | |
outerMetric: Metric[D], | |
directory: AbstractFile | |
): Queryable[T, FaissParams, D] | |
.\ann\src\main\scala\com\twitter\ann\faiss\FaissIndexer.scala | |
trait FaissIndexer extends Logging | |
def build[D <: Distance[D]]( | |
pipe: TypedPipe[EntityEmbedding[Long]], | |
sampleRate: Float, | |
factoryString: String, | |
metric: Metric[D], | |
outputDirectory: AbstractFile | |
): Execution[Unit] | |
def buildAndWriteFaissIndex[D <: Distance[D]]( | |
entities: Iterable[EntityEmbedding[Long]], | |
sampleRate: Float, | |
factoryString: String, | |
metricType: Metric[D], | |
outputDirectory: IndexOutputFile | |
): Unit | |
def copyToOutputAndCreateSuccess( | |
tmpFile: AbstractFile, | |
outputDirectory: IndexOutputFile | |
) | |
def toFloatVector( | |
dimensions: Int, | |
entities: Iterable[EntityEmbedding[Long]] | |
): FloatVector | |
def toIndexVector(embeddings: Iterable[EntityEmbedding[Long]]): LongVector | |
def parseMetric[D <: Distance[D]](metric: Metric[D]): MetricType = metric match | |
def l2Normalize[D <: Distance[D]](metric: Metric[D]): Boolean = metric match | |
object FaissIndexer extends FaissIndexer | |
.\ann\src\main\scala\com\twitter\ann\faiss\HourlyDirectoryWithSuccessFileListing.scala | |
object HourlyDirectoryWithSuccessFileListing extends Logging | |
def listHourlyIndexDirectories( | |
root: AbstractFile, | |
startingFrom: Time, | |
count: Int, | |
lookbackInterval: Int | |
): Seq[AbstractFile] = listingStep(root, startingFrom, count, lookbackInterval) | |
private def listingStep( | |
root: AbstractFile, | |
startingFrom: Time, | |
remainingDirectoriesToFind: Int, | |
remainingAttempts: Int | |
): List[AbstractFile] | |
def getSuccessfulDirectoryForDate( | |
root: AbstractFile, | |
date: Time | |
): Try[AbstractFile] | |
.\ann\src\main\scala\com\twitter\ann\faiss\HourlyShardedIndex.scala | |
object HourlyShardedIndex | |
def loadIndex[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
directory: AbstractFile, | |
shardsToLoad: Int, | |
shardWatchInterval: Duration, | |
lookbackInterval: Int, | |
statsReceiver: StatsReceiver | |
): HourlyShardedIndex[T, D] | |
class HourlyShardedIndex[T, D <: Distance[D]]( | |
outerMetric: Metric[D], | |
outerDimension: Int, | |
directory: AbstractFile, | |
shardsToLoad: Int, | |
shardWatchInterval: Duration, | |
lookbackInterval: Int, | |
override protected val statsReceiver: StatsReceiver) | |
extends QueryableIndexAdapter[T, D] | |
with Logging | |
with Task | |
def index: Index | |
trait | |
protected def task(): Future[Unit] = Future.value(reloadShards()) | |
protected def taskInterval: Duration = shardWatchInterval | |
private def loadIndex(directory: AbstractFile): Try[Index] = | |
Try(QueryableIndexAdapter.loadJavaIndex(directory)) | |
private val shardsCache = new MemoizedInEpochs[AbstractFile, Index](loadIndex) | |
// Destroying original index invalidate casted index. Keep a reference to both. | |
private val originalIndex = new AtomicReference[IndexShards]() | |
private val castedIndex = new AtomicReference[Index]() | |
private def reloadShards(): Unit | |
.\ann\src\main\scala\com\twitter\ann\faiss\QueryableIndexAdapter.scala | |
object QueryableIndexAdapter extends Logging | |
def loadJavaIndex(directory: AbstractFile): Index | |
trait QueryableIndexAdapter[T, D <: Distance[D]] extends Queryable[T, FaissParams, D] | |
def index: Index | |
protected val metric: Metric[D] | |
protected val dimension: Int | |
private def maybeNormalizeEmbedding(embeddingVector: EmbeddingVector): EmbeddingVector | |
def maybeTranslateToCosineDistanceInplace(array: floatArray, len: Int): Unit | |
def ensuringParams[R](parameterString: String, f: () => R): R | |
def replaceIndex(f: () => Unit): Unit | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: FaissParams | |
): Future[List[T]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: FaissParams | |
): Future[List[NeighborWithDistance[T, D]]] | |
def toFloatArray(emb: EmbeddingVector): floatArray | |
def toSeq(vector: LongVector, len: Long): Seq[Long] | |
def toSeq(array: floatArray, len: Int): Seq[Float] | |
.\ann\src\main\scala\com\twitter\ann\featurestore\FeatureStoreEmbeddingProducer.scala | |
object FeatureStoreEmbeddingProducer | |
def apply[T <: EntityId]( | |
dataset: VersionedOnlineAccessDataset[T, TEmbedding], | |
version: Long, | |
boundFeature: BoundFeature[T, RawFloatTensor], | |
client: Client, | |
statsReceiver: StatsReceiver = new InMemoryStatsReceiver, | |
featureStoreAttributions: Seq[Attribution] = Seq.empty | |
): EmbeddingProducer[EntityWithId[T]] | |
class FeatureStoreEmbeddingProducer[T <: EntityId]( | |
boundFeature: BoundFeature[T, RawFloatTensor], | |
featureStoreClient: FeatureStoreClient) | |
extends EmbeddingProducer[EntityWithId[T]] | |
def produceEmbedding(input: EntityWithId[T]): Stitch[Option[Embedding[Float]]] | |
.\ann\src\main\scala\com\twitter\ann\file_store\ReadableIndexIdFileStore.scala | |
object ReadableIndexIdFileStore | |
def apply[V]( | |
file: AbstractFile, | |
injection: Injection[V, Array[Byte]] | |
): ReadableStore[Long, V] | |
def loadFile(file: AbstractFile): ByteBuffer | |
.\ann\src\main\scala\com\twitter\ann\file_store\WritableIndexIdFileStore.scala | |
object WritableIndexIdFileStore | |
def apply[V]( | |
injection: Injection[V, Array[Byte]] | |
): WritableIndexIdFileStore[V] | |
class WritableIndexIdFileStore[V] private ( | |
map: JConcurrentHashMap[Long, Option[V]], | |
injection: Injection[V, Array[Byte]]) | |
extends Store[Long, V] | |
def get(k: Long): Future[Option[V]] | |
def put(kv: (Long, Option[V])): Future[Unit] | |
def save(file: IndexOutputFile): Unit | |
def getInjection: Injection[V, Array[Byte]] = injection | |
private[this] def toThrift(): FileBasedIndexIdStore | |
def saveThrift(thriftObj: FileBasedIndexIdStore, file: IndexOutputFile): Unit | |
.\ann\src\main\scala\com\twitter\ann\hnsw\DistanceFunctionGenerator.scala | |
object DistanceFunctionGenerator | |
def apply[T, D <: Distance[D]]( | |
metric: Metric[D], | |
idToEmbeddingFn: (T) => EmbeddingVector | |
): DistanceFunctionGenerator[T] | |
def distance(id1: T, id2: T) = | |
updatedMetric.absoluteDistance( | |
idToEmbeddingFn(id1), | |
idToEmbeddingFn(id2) | |
) | |
} | |
val distFnQuery = new DistanceFunction[EmbeddingVector, T] | |
.\ann\src\main\scala\com\twitter\ann\hnsw\Hnsw.scala | |
object Hnsw | |
def apply[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
efConstruction: Int, | |
maxM: Int, | |
expectedElements: Int, | |
futurePool: ReadWriteFuturePool, | |
idEmbeddingMap: IdEmbeddingMap[T] | |
): Hnsw[T, D] | |
def get(): Random = ThreadLocalRandom.current() | |
} | |
val distFn = | |
DistanceFunctionGenerator(metric, (key: T) => idEmbeddingMap.get(key)) | |
val internalIndex = new HnswIndex[T, EmbeddingVector]( | |
distFn.index, | |
distFn.query, | |
efConstruction, | |
maxM, | |
expectedElements, | |
randomProvider | |
) | |
new Hnsw[T, D]( | |
dimension, | |
metric, | |
internalIndex, | |
futurePool, | |
idEmbeddingMap, | |
distFn.shouldNormalize, | |
LockedAccess.apply(expectedElements) | |
) | |
} | |
} | |
private[hnsw] object LockedAccess | |
def apply[T](expectedElements: Int): LockedAccess[T] = | |
DefaultLockedAccess(new ConcurrentHashMap[T, Lock](expectedElements)) | |
protected[hnsw] def apply[T](): LockedAccess[T] = | |
DefaultLockedAccess(new ConcurrentHashMap[T, Lock]()) | |
} | |
private[hnsw] case class DefaultLockedAccess[T](locks: ConcurrentHashMap[T, Lock]) | |
extends LockedAccess[T] | |
def lockProvider(item: T) = locks.computeIfAbsent(item, (_: T) => new ReentrantLock()) | |
} | |
private[hnsw] trait LockedAccess[T] | |
def lockProvider(item: T): Lock | |
def lock[K](item: T)(fn: => K): K | |
class Hnsw[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
hnswIndex: HnswIndex[T, EmbeddingVector], | |
readWriteFuturePool: ReadWriteFuturePool, | |
idEmbeddingMap: IdEmbeddingMap[T], | |
shouldNormalize: Boolean, | |
lockedAccess: LockedAccess[T] = LockedAccess.apply[T]()) | |
extends Appendable[T, HnswParams, D] | |
with Queryable[T, HnswParams, D] | |
with Updatable[T] | |
def append(entity: EntityEmbedding[T]): Future[Unit] | |
def toQueryable: Queryable[T, HnswParams, D] = this | |
override def query( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: HnswParams | |
): Future[List[T]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: HnswParams | |
): Future[List[NeighborWithDistance[T, D]]] | |
def updatedEmbedding(embedding: EmbeddingVector): EmbeddingVector | |
def getIndex: HnswIndex[T, EmbeddingVector] = hnswIndex | |
def getDimen: Int = dimension | |
def getMetric: Metric[D] = metric | |
def getIdEmbeddingMap: IdEmbeddingMap[T] = idEmbeddingMap | |
override def update( | |
entity: EntityEmbedding[T] | |
): Future[Unit] | |
.\ann\src\main\scala\com\twitter\ann\hnsw\HnswCommon.scala | |
object HnswCommon | |
def apply(scalaParams: HnswParams): ServiceRuntimeParams | |
def invert(thriftParams: ServiceRuntimeParams): Try[HnswParams] = | |
thriftParams match | |
def isValidHnswIndex(path: AbstractFile): Boolean | |
class HnswParams(ef: Int) extends RuntimeParams | |
.\ann\src\main\scala\com\twitter\ann\hnsw\HnswIOUtil.scala | |
object HnswIOUtil | |
def loadEmbeddings[T]( | |
embeddingFile: AbstractFile, | |
injection: Injection[T, Array[Byte]], | |
idEmbeddingMap: IdEmbeddingMap[T], | |
): IdEmbeddingMap[T] | |
def saveEmbeddings[T]( | |
stream: OutputStream, | |
injection: Injection[T, Array[Byte]], | |
iter: Iterator[(T, EmbeddingVector)] | |
): Unit | |
def saveIndexMetadata( | |
dimension: Int, | |
metric: Metric[_ <: Distance[_]], | |
numElements: Int, | |
metadataStream: OutputStream | |
): Unit | |
def loadIndexMetadata( | |
metadataFile: AbstractFile | |
): HnswIndexMetadata | |
.\ann\src\main\scala\com\twitter\ann\hnsw\IdEmbeddingMap.scala | |
trait IdEmbeddingMap[T] | |
.\ann\src\main\scala\com\twitter\ann\hnsw\JMapBasedIdEmbeddingMap.scala | |
object JMapBasedIdEmbeddingMap | |
def applyInMemory[T](expectedElements: Int): IdEmbeddingMap[T] = | |
new JMapBasedIdEmbeddingMap[T]( | |
new ConcurrentHashMap[T, EmbeddingVector](expectedElements), | |
Option.empty | |
) | |
/** | |
* Creates in-memory concurrent hashmap based container that can be serialized to disk for storing id embedding mapping. | |
* @param expectedElements: Expected num of elements for sizing hint, need not be exact. | |
* @param injection : Injection for typed Id T to Array[Byte] | |
*/ | |
def applyInMemoryWithSerialization[T]( | |
expectedElements: Int, | |
injection: Injection[T, Array[Byte]] | |
): IdEmbeddingMap[T] = | |
new JMapBasedIdEmbeddingMap[T]( | |
new ConcurrentHashMap[T, EmbeddingVector](expectedElements), | |
Some(injection) | |
) | |
/** | |
* Loads id embedding mapping in in-memory concurrent hashmap. | |
* @param embeddingFile: Local/Hdfs file path for embeddings | |
* @param injection : Injection for typed Id T to Array[Byte] | |
* @param numElements: Expected num of elements for sizing hint, need not be exact | |
*/ | |
def loadInMemory[T]( | |
embeddingFile: AbstractFile, | |
injection: Injection[T, Array[Byte]], | |
numElements: Option[Int] = Option.empty | |
): IdEmbeddingMap[T] | |
class JMapBasedIdEmbeddingMap[T]( | |
map: java.util.concurrent.ConcurrentHashMap[T, EmbeddingVector], | |
injection: Option[Injection[T, Array[Byte]]]) | |
extends IdEmbeddingMap[T] | |
def putIfAbsent(id: T, embedding: EmbeddingVector): EmbeddingVector | |
def put(id: T, embedding: EmbeddingVector): EmbeddingVector | |
def get(id: T): EmbeddingVector | |
def iter(): Iterator[(T, EmbeddingVector)] = | |
map | |
.entrySet() | |
.iterator() | |
.asScala | |
.map(e => (e.getKey, e.getValue)) | |
override def size(): Int = map.size() | |
override def toDirectory(embeddingFile: OutputStream): Unit | |
.\ann\src\main\scala\com\twitter\ann\hnsw\MapDbBasedIdEmbeddingMap.scala | |
class currently only support querying and creates map db on fly from thrift serialized embedding mapping | |
* Implement index creation with this or altogether replace mapdb with some better performing solution as it takes a lot of time to create/query or precreate while serializing thrift embeddings | |
*/ | |
private[hnsw] object MapDbBasedIdEmbeddingMap | |
def loadAsReadonly[T]( | |
embeddingFile: AbstractFile, | |
injection: Injection[T, Array[Byte]] | |
): IdEmbeddingMap[T] | |
class MapDbBasedIdEmbeddingMap[T]( | |
mapDb: HTreeMap[Array[Byte], Array[Float]], | |
injection: Injection[T, Array[Byte]]) | |
extends IdEmbeddingMap[T] | |
def putIfAbsent(id: T, embedding: EmbeddingVector): EmbeddingVector | |
def put(id: T, embedding: EmbeddingVector): EmbeddingVector | |
def get(id: T): EmbeddingVector | |
def iter(): Iterator[(T, EmbeddingVector)] | |
def size(): Int = mapDb.size() | |
override def toDirectory(embeddingFile: OutputStream): Unit | |
.\ann\src\main\scala\com\twitter\ann\hnsw\SerializableHnsw.scala | |
object SerializableHnsw | |
def apply[T, D <: Distance[D]]( | |
index: Hnsw[T, D], | |
injection: Injection[T, Array[Byte]] | |
): SerializableHnsw[T, D] | |
def loadMapBasedQueryableIndex[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
futurePool: ReadWriteFuturePool, | |
directory: AbstractFile | |
): SerializableHnsw[T, D] | |
def loadMMappedBasedQueryableIndex[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
futurePool: ReadWriteFuturePool, | |
directory: AbstractFile | |
): SerializableHnsw[T, D] | |
def loadIndex[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
futurePool: ReadWriteFuturePool, | |
directory: AbstractFile, | |
idEmbeddingMap: IdEmbeddingMap[T], | |
metadata: HnswIndexMetadata | |
): SerializableHnsw[T, D] | |
def get(): Random = ThreadLocalRandom.current() | |
} | |
val internalIndex = HnswIndex.loadHnswIndex[T, EmbeddingVector]( | |
distFn.index, | |
distFn.query, | |
directory.getChild(InternalIndexDir), | |
injection, | |
randomProvider | |
) | |
val index = new Hnsw[T, D]( | |
dimension, | |
metric, | |
internalIndex, | |
futurePool, | |
idEmbeddingMap, | |
distFn.shouldNormalize, | |
LockedAccess.apply(metadata.numElements) | |
) | |
new SerializableHnsw(index, injection) | |
} | |
private[this] def validateMetadata[D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
existingMetadata: HnswIndexMetadata | |
): Unit | |
class SerializableHnsw[T, D <: Distance[D]]( | |
index: Hnsw[T, D], | |
injection: Injection[T, Array[Byte]]) | |
extends Appendable[T, HnswParams, D] | |
with Queryable[T, HnswParams, D] | |
with Serialization | |
with Updatable[T] | |
def append(entity: EntityEmbedding[T]) = index.append(entity) | |
override def toQueryable: Queryable[T, HnswParams, D] = index.toQueryable | |
override def query( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: HnswParams | |
) = index.query(embedding, numOfNeighbours, runtimeParams) | |
override def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbours: Int, | |
runtimeParams: HnswParams | |
) = index.queryWithDistance(embedding, numOfNeighbours, runtimeParams) | |
def toDirectory(directory: ResourceId): Unit | |
def toDirectory(directory: AbstractFile): Unit | |
def toDirectory(indexFile: IndexOutputFile): Unit | |
def update( | |
entity: EntityEmbedding[T] | |
): Future[Unit] | |
.\ann\src\main\scala\com\twitter\ann\hnsw\TypedHnswIndex.scala | |
object TypedHnswIndex | |
def index[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
efConstruction: Int, | |
maxM: Int, | |
expectedElements: Int, | |
readWriteFuturePool: ReadWriteFuturePool | |
): Appendable[T, HnswParams, D] with Queryable[T, HnswParams, D] with Updatable[T] | |
def serializableIndex[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
efConstruction: Int, | |
maxM: Int, | |
expectedElements: Int, | |
injection: Injection[T, Array[Byte]], | |
readWriteFuturePool: ReadWriteFuturePool | |
): Appendable[T, HnswParams, D] | |
with Queryable[T, HnswParams, D] | |
with Updatable[T] | |
with Serialization | |
def loadIndex[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
readWriteFuturePool: ReadWriteFuturePool, | |
directory: AbstractFile | |
): Appendable[T, HnswParams, D] | |
with Queryable[T, HnswParams, D] | |
with Updatable[T] | |
with Serialization | |
def loadMMappedIndex[T, D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
injection: Injection[T, Array[Byte]], | |
readWriteFuturePool: ReadWriteFuturePool, | |
directory: AbstractFile | |
): Appendable[T, HnswParams, D] | |
with Queryable[T, HnswParams, D] | |
with Updatable[T] | |
with Serialization | |
.\ann\src\main\scala\com\twitter\ann\manhattan\ManhattanEmbeddingProducer.scala | |
class ManhattanEmbeddingProducer[T]( | |
keyDescriptor: DescriptorP1L0.DKey[T], | |
valueDescriptor: ValueDescriptor.EmptyValue[EmbeddingVector], | |
manhattanEndpoint: ManhattanKVEndpoint) | |
extends EmbeddingProducer[T] | |
def produceEmbedding(input: T): Stitch[Option[EmbeddingVector]] | |
object ManhattanEmbeddingProducer | |
def keyDescriptor[T]( | |
injection: Injection[T, Array[Byte]], | |
dataset: String | |
): DescriptorP1L0.DKey[T] = | |
ReadOnlyKeyDescriptor(injection.andThen(Bijections.BytesBijection)) | |
.withDataset(dataset) | |
private[manhattan] val EmbeddingDescriptor: ValueDescriptor.EmptyValue[ | |
EmbeddingType.EmbeddingVector | |
] | |
def apply[T]( | |
dataset: String, | |
injection: Injection[T, Array[Byte]], | |
manhattanEndpoint: ManhattanKVEndpoint | |
): EmbeddingProducer[T] | |
.\ann\src\main\scala\com\twitter\ann\scalding\benchmark\Knn.scala | |
class com.twitter.ann.scalding.offline.com.twitter.ann.scalding.benchmark.KnnJob -- \ | |
* --dalEnvironment Prod \ | |
* --search_space_entity_type user \ | |
* --user.feature_store_embedding ConsumerFollowEmbedding300Dataset \ | |
* --user.feature_store_major_version 1569196895 \ | |
* --user.date_range 2019-10-23 \ | |
* --search_space.feature_store_embedding ConsumerFollowEmbedding300Dataset \ | |
* --search_space.feature_store_major_version 1569196895 \ | |
* --search_space.date_range 2019-10-23 \ | |
* --date 2019-10-25 \ | |
* --version "consumer_follower_test" \ | |
* --reducers 10000 \ | |
* --num_of_random_groups 20 \ | |
* --num_replicas 1000 \ | |
* --indexing_strategy.metric InnerProduct \ | |
* --indexing_strategy.type hnsw \ | |
* --indexing_strategy.dimension 300 \ | |
* --indexing_strategy.ef_construction 30 \ | |
* --indexing_strategy.max_m 10 \ | |
* --indexing_strategy.ef_query 50 \ | |
* --search_space_shards 3000 \ | |
* --query_shards 3000 \ | |
* --search_space.read_sample_ratio 0.038 | |
*/ | |
trait KnnJobBase | |
def getKnnDataset[B <: EntityId, D <: Distance[D]]( | |
args: Args | |
)( | |
implicit uniqueID: UniqueID | |
): TypedPipe[Knn] | |
object KnnJob extends TwitterExecutionApp with KnnJobBase | |
def job: Execution[Unit] = Execution.withId | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\IndexingStrategy.scala | |
trait IndexingStrategy[D <: Distance[D]] | |
def buildIndex[T]( | |
indexItems: TraversableOnce[EntityEmbedding[T]] | |
): ParameterlessQueryable[T, _, D] | |
} | |
object IndexingStrategy | |
def parse( | |
args: Args, | |
argumentName: String = "indexing_strategy" | |
): IndexingStrategy[_] | |
def metricArg[D <: Distance[D]] = | |
Metric.fromString(args(s"$argumentName.metric")).asInstanceOf[Metric[D]] | |
args(s"$argumentName.type") match | |
class BruteForceIndexingStrategy[D <: Distance[D]](metric: Metric[D]) | |
extends IndexingStrategy[D] | |
def buildIndex[T]( | |
indexItems: TraversableOnce[EntityEmbedding[T]] | |
): ParameterlessQueryable[T, _, D] | |
class HnswIndexingStrategy[D <: Distance[D]]( | |
dimension: Int, | |
metric: Metric[D], | |
efConstruction: Int, | |
maxM: Int, | |
hnswParams: HnswParams, | |
concurrencyLevel: Int = 1) | |
extends IndexingStrategy[D] | |
def buildIndex[T]( | |
indexItems: TraversableOnce[EntityEmbedding[T]] | |
): ParameterlessQueryable[T, _, D] | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnDebug.scala | |
class ConsumerAssoc(consumerId: UserId, assoc: List[String]) | |
object KnnDebug | |
def getConsumerAssociations( | |
graph: TypedPipe[GraphEdge[UserId, UserId]], | |
usernames: TypedPipe[(UserId, String)], | |
reducers: Int | |
): TypedPipe[ConsumerAssoc] | |
def getDebugTable( | |
neighborsPipe: TypedPipe[(EntityKey, NearestNeighbors)], | |
shards: Int, | |
reducers: Int, | |
limit: Int = 10000, | |
userDataset: Option[TypedPipe[FlatUser]] = None, | |
followDataset: Option[TypedPipe[GraphEdge[UserId, UserId]]] = None, | |
consumerStatesDataset: Option[TypedPipe[CondensedUserState]] = None, | |
minFollows: Int = 25, | |
maxFollows: Int = 50 | |
)( | |
implicit dateRange: DateRange | |
): TypedPipe[(String, String, String, String)] | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnEntityRecoDebugJob.scala | |
object KnnEntityRecoDebugJob extends TwitterExecutionApp | |
def job: Execution[Unit] = Execution.withId | |
def run[A <: EntityId, B <: EntityId, D <: Distance[D]]( | |
uncastQueryEntityKind: EntityKind[_], | |
uncastSearchSpaceEntityKind: EntityKind[_], | |
uncastMetric: Metric[_], | |
args: Args | |
)( | |
implicit uniqueID: UniqueID | |
): Execution[Unit] | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnHelper.scala | |
class Index[T, D <: Distance[D]]( | |
injection: Injection[T, Array[Byte]], | |
metric: Metric[D], | |
dimension: Int, | |
directory: AbstractFile) | |
object KnnHelper | |
def getFilteredUserEmbeddings( | |
args: Args, | |
filterPath: Option[String], | |
reducers: Int, | |
useHashJoin: Boolean | |
)( | |
implicit dateRange: DateRange | |
): TypedPipe[EmbeddingWithEntity[UserId]] | |
def getNeighborsPipe[T <: EntityId, D <: Distance[D]]( | |
args: Args, | |
uncastEntityKind: EntityKind[_], | |
uncastMetric: Metric[_], | |
ef: Int, | |
consumerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]], | |
abstractFile: Option[AbstractFile], | |
reducers: Int, | |
numNeighbors: Int, | |
dimension: Int | |
)( | |
implicit dateRange: DateRange | |
): TypedPipe[(EntityKey, NearestNeighbors)] | |
def bruteForceNearestNeighbors( | |
consumerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]], | |
producerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]], | |
numNeighbors: Int, | |
reducers: Int | |
): TypedPipe[(EntityKey, NearestNeighbors)] | |
def findNearestNeighbours[A <: EntityId, B <: EntityId, D <: Distance[D]]( | |
queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]], | |
searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]], | |
metric: Metric[D], | |
numNeighbors: Int = 10, | |
queryIdsFilter: Option[TypedPipe[A]] = Option.empty, | |
reducers: Int = 100, | |
mappers: Int = 100, | |
isSearchSpaceLarger: Boolean = true, | |
numOfSearchGroups: Int = 1, | |
numReplicas: Int = 1, | |
useCounters: Boolean = true | |
)( | |
implicit ordering: Ordering[A], | |
uid: UniqueID | |
): TypedPipe[(A, Seq[(B, D)])] | |
def findNearestNeighboursWithIndexingStrategy[A <: EntityId, B <: EntityId, D <: Distance[D]]( | |
queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]], | |
searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]], | |
numNeighbors: Int, | |
numOfSearchGroups: Int, | |
indexingStrategy: IndexingStrategy[D], | |
numReplicas: Int = 1, | |
reducersOption: Option[Int] = None, | |
queryShards: Option[Int] = None, | |
searchSpaceShards: Option[Int] = None, | |
useCounters: Boolean = true | |
)( | |
implicit ordering: Ordering[A], | |
uid: UniqueID | |
): UnsortedGrouped[A, Seq[(B, D)]] | |
def shard[T]( | |
pipe: TypedPipe[T], | |
numberOfShards: Option[Int] | |
): TypedPipe[T] | |
def findNearestNeighboursViaCross[A <: EntityId, B <: EntityId, D <: Distance[D]]( | |
queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]], | |
searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]], | |
metric: Metric[D], | |
numNeighbors: Int, | |
reducers: Int, | |
mappers: Int, | |
isSearchSpaceLarger: Boolean | |
)( | |
implicit ordering: Ordering[A] | |
): TypedPipe[(A, Seq[(B, D)])] | |
def nearestNeighborsToString[A <: EntityId, B <: EntityId, D <: Distance[D]]( | |
nearestNeighbors: (A, Seq[(B, D)]), | |
queryEntityKind: EntityKind[A], | |
neighborEntityKind: EntityKind[B], | |
idDistanceSeparator: String = ":", | |
neighborSeparator: String = "\t" | |
): String | |
class NNKey( | |
searchGroup: Int, | |
replica: Int, | |
maxReplica: Option[Int] = None) | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnOfflineJob.scala | |
object KnnOfflineJob extends TwitterExecutionApp | |
def job: Execution[Unit] = Execution.withId | |
object for this job | |
* @param abstractFile: An optional of producer embedding path | |
*/ | |
def execute( | |
args: Args, | |
abstractFile: Option[AbstractFile] | |
)( | |
implicit uniqueID: UniqueID | |
): Execution[Unit] | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\KnnTruthSetGenerator.scala | |
object KnnTruthSetGenerator extends TwitterExecutionApp | |
def job: Execution[Unit] = Execution.withId | |
def run[A <: EntityId, B <: EntityId, D <: Distance[D]]( | |
uncastQueryEntityKind: EntityKind[_], | |
uncastIndexSpaceEntityKind: EntityKind[_], | |
uncastMetric: Metric[_], | |
args: Args | |
)( | |
implicit uniqueID: UniqueID | |
): Execution[Unit] | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\ParameterlessQueryable.scala | |
class ParameterlessQueryable[T, P <: RuntimeParams, D <: Distance[D]]( | |
queryable: Queryable[T, P, D], | |
runtimeParamsForAllQueries: P) | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\faissindexbuilder\IndexBuilder.scala | |
object IndexBuilder extends FaissIndexer with Logging | |
def run[T <: UserId, D <: Distance[D]]( | |
embeddingFormat: EmbeddingFormat[T], | |
embeddingLimit: Option[Int], | |
sampleRate: Float, | |
factoryString: String, | |
metric: Metric[D], | |
outputDirectory: AbstractFile, | |
numDimensions: Int | |
): Execution[Unit] | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\faissindexbuilder\IndexBuilderApp.scala | |
trait IndexBuilderExecutable extends Logging | |
def indexBuilderExecution[T <: UserId, D <: Distance[D]]( | |
args: Args | |
): Execution[Unit] | |
object IndexBuilderApp extends TwitterExecutionApp with IndexBuilderExecutable | |
def job: Execution[Unit] = Execution.getArgs.flatMap | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\indexbuilder\IndexBuilder.scala | |
object IndexBuilder | |
def run[T <: EntityId, _, D <: Distance[D]]( | |
embeddingFormat: EmbeddingFormat[T], | |
embeddingLimit: Option[Int], | |
index: Appendable[T, _, D] with Serialization, | |
concurrencyLevel: Int, | |
outputDirectory: AbstractFile, | |
numDimensions: Int | |
): Execution[Unit] | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\indexbuilder\IndexBuilderApp.scala | |
trait IndexBuilderExecutable | |
def indexBuilderExecution[T <: EntityId, D <: Distance[D]]( | |
args: Args | |
): Execution[Unit] | |
object IndexBuilderApp extends TwitterExecutionApp with IndexBuilderExecutable | |
def job: Execution[Unit] = Execution.getArgs.flatMap | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\indexbuilderfrombq\IndexBuilderFromBQ.scala | |
object IndexBuilder | |
def run[T <: EntityId, _, D <: Distance[D]]( | |
embeddingsPipe: TypedPipe[EmbeddingWithEntity[T]], | |
embeddingLimit: Option[Int], | |
index: Appendable[T, _, D] with Serialization, | |
concurrencyLevel: Int, | |
outputDirectory: AbstractFile, | |
numDimensions: Int | |
): Execution[Unit] | |
.\ann\src\main\scala\com\twitter\ann\scalding\offline\indexbuilderfrombq\IndexBuilderFromBQApp.scala | |
trait IndexBuilderFromBQExecutable | |
def indexBuilderExecution[T <: EntityId, D <: Distance[D]]( | |
args: Args | |
): Execution[Unit] | |
def avroMapper(row: GenericRecord): KeyVal[Long, java.util.List[Double]] | |
def transform[T <: EntityId]( | |
entityKind: EntityKind[T] | |
)( | |
bqRecord: KeyVal[Long, java.util.List[Double]] | |
): EmbeddingWithEntity[T] | |
object IndexBuilderFromBQApp extends TwitterExecutionApp with IndexBuilderFromBQExecutable | |
def job: Execution[Unit] = Execution.getArgs.flatMap | |
.\ann\src\main\scala\com\twitter\ann\serialization\DummyANNIndexInjection.scala | |
object DummyANNIndexInjection | |
.\ann\src\main\scala\com\twitter\ann\serialization\PersistedEmbeddingInjection.scala | |
class PersistedEmbeddingInjection[T]( | |
idByteInjection: Injection[T, Array[Byte]]) | |
extends Injection[EntityEmbedding[T], PersistedEmbedding] | |
def apply(entity: EntityEmbedding[T]): PersistedEmbedding | |
def invert(persistedEmbedding: PersistedEmbedding): Try[EntityEmbedding[T]] | |
.\ann\src\main\scala\com\twitter\ann\serialization\ThriftIteratorIO.scala | |
class can do things lazily so there is no need to have all the object into memory. | |
*/ | |
class ThriftIteratorIO[T <: ThriftStruct]( | |
codec: ThriftStructCodec[T]) | |
def toOutputStream( | |
iterator: Iterator[T], | |
outputStream: OutputStream | |
): Unit | |
def fromInputStream( | |
inputStream: InputStream | |
): Iterator[T] | |
object ThriftIteratorIO | |
def getIterator[T <: ThriftStruct]( | |
codec: ThriftStructCodec[T], | |
inputStream: InputStream | |
): Iterator[T] | |
def getNext: Option[T] = | |
try | |
.\ann\src\main\scala\com\twitter\ann\service\loadtest\AnnLoadTest.scala | |
class AnnIndexQueryLoadTest( | |
worker: AnnLoadTestWorker = new AnnLoadTestWorker()) | |
def performQueries[T, P <: RuntimeParams, D <: Distance[D]]( | |
queryable: Queryable[T, P, D], | |
qps: Int, | |
duration: Duration, | |
queries: Seq[Query[T]], | |
concurrencyLevel: Int, | |
runtimeConfigurations: Seq[QueryTimeConfiguration[T, P]] | |
): Future[Unit] | |
class Query[T](embedding: EmbeddingVector, trueNeighbours: Seq[T] = Seq.empty) | |
class AnnIndexBuildLoadTest( | |
buildRecorder: LoadTestBuildRecorder, | |
embeddingIndexer: EmbeddingIndexer = new EmbeddingIndexer()) | |
def indexEmbeddings[T, P <: RuntimeParams, D <: Distance[D]]( | |
appendable: Appendable[T, P, D], | |
indexSet: Seq[EntityEmbedding[T]], | |
concurrencyLevel: Int | |
): Future[Queryable[T, P, D]] | |
.\ann\src\main\scala\com\twitter\ann\service\loadtest\AnnLoadTestMain.scala | |
object AnnLoadTestMain extends TwitterServer | |
def start(): Unit | |
def getQueries[Q, I]: Seq[Query[I]] | |
def getQueryRuntimeConfig[ | |
T, | |
P <: RuntimeParams | |
]: Seq[QueryTimeConfiguration[T, P]] | |
def toNonZeroOptional(x: Int): Option[Int] = if (x != 0) Some(x) else None | |
for | |
def buildQueryable[T, P <: RuntimeParams, D <: Distance[D]]( | |
inMemoryBuildRecorder: InMemoryLoadTestBuildRecorder | |
): Future[Queryable[T, P, D]] | |
def indexEmbeddingsAndGetQueryable[T, P <: RuntimeParams, D <: Distance[D]]( | |
buildRecorder: LoadTestBuildRecorder, | |
indexSet: Seq[EntityEmbedding[T]] | |
): Future[Queryable[T, P, D]] | |
def performQueries[T, P <: RuntimeParams, D <: Distance[D]]( | |
queryable: Queryable[T, P, D], | |
queryTimeConfig: Seq[QueryTimeConfiguration[T, P]], | |
queries: Seq[Query[T]] | |
): Future[Unit] | |
def getIndexIdInjection[T]: Injection[T, Array[Byte]] | |
def getRuntimeParamInjection[ | |
P <: RuntimeParams | |
]: Injection[P, ServiceRuntimeParams] | |
def getDistanceInjection[D <: Distance[D]]: Injection[D, ServiceDistance] | |
def getDistanceMetric[D <: Distance[D]]: Metric[D] | |
def buildQueryTimeConfig[T, P <: RuntimeParams]( | |
numOfNeighbors: Int, | |
params: P, | |
config: Map[String, String] | |
): QueryTimeConfiguration[T, P] | |
.\ann\src\main\scala\com\twitter\ann\service\loadtest\AnnLoadTestWorker.scala | |
object QueryTimeConfiguration | |
class QueryTimeConfiguration[T, P <: RuntimeParams]( | |
recorder: LoadTestQueryRecorder[T], | |
param: P, | |
numberOfNeighbors: Int, | |
private val results: InMemoryLoadTestQueryRecorder[T]) | |
def toString: String = | |
s"QueryTimeConfiguration(param = $param, numberOfNeighbors = $numberOfNeighbors)" | |
def printResults: String | |
class AnnLoadTestWorker( | |
timer: Timer = DefaultTimer) | |
def runWithQps[T, P <: RuntimeParams, D <: Distance[D]]( | |
queryable: Queryable[T, P, D], | |
queries: Seq[Query[T]], | |
qps: Int, | |
duration: Duration, | |
configuration: QueryTimeConfiguration[T, P], | |
concurrencyLevel: Int | |
): Future[Int] | |
def performQuery[T, P <: RuntimeParams, D <: Distance[D]]( | |
configuration: QueryTimeConfiguration[T, P], | |
queryable: Queryable[T, P, D], | |
query: Query[T] | |
): Future[Try[Unit]] | |
.\ann\src\main\scala\com\twitter\ann\service\loadtest\EmbeddingIndexer.scala | |
class EmbeddingIndexer | |
def indexEmbeddings[T, P <: RuntimeParams, D <: Distance[D]]( | |
appendable: Appendable[T, P, D], | |
recorder: LoadTestBuildRecorder, | |
indexSet: Seq[EntityEmbedding[T]], | |
concurrencyLevel: Int | |
): Future[Queryable[T, P, D]] | |
.\ann\src\main\scala\com\twitter\ann\service\loadtest\LoadTestRecorder.scala | |
trait LoadTestQueryRecorder[T] | |
def recordQueryResult( | |
trueNeighbors: Seq[T], | |
foundNeighbors: Seq[T], | |
queryLatency: Duration | |
): Unit | |
} | |
case class LoadTestQueryResults( | |
numResults: Int, | |
top1Recall: Float, | |
top10Recall: Option[Float], | |
overallRecall: Float) | |
private object LoadTestQueryRecorder | |
def recordQueryResult[T]( | |
trueNeighbors: Seq[T], | |
foundNeighbors: Seq[T] | |
): LoadTestQueryResults | |
class StatsLoadTestQueryRecorder[T]( | |
statsReceiver: StatsReceiver) | |
extends LoadTestQueryRecorder[T] | |
def recordQueryResult( | |
trueNeighbors: Seq[T], | |
foundNeighbors: Seq[T], | |
queryLatency: Duration | |
): Unit | |
trait LoadTestBuildRecorder | |
def recordIndexCreation( | |
indexSize: Int, | |
indexLatency: Duration, | |
toQueryableLatency: Duration | |
): Unit | |
} | |
class StatsLoadTestBuildRecorder( | |
statsReceiver: StatsReceiver) | |
extends LoadTestBuildRecorder | |
def recordIndexCreation( | |
indexSize: Int, | |
indexLatency: Duration, | |
toQueryableLatency: Duration | |
): Unit | |
class QueryRecorderSnapshot(snapshot: Snapshot) | |
def avgQueryLatencyMicros: Double = snapshot.average | |
def p50QueryLatencyMicros: Double = | |
snapshot.percentiles.find(_.quantile == .5).get.value | |
def p90QueryLatencyMicros: Double = | |
snapshot.percentiles.find(_.quantile == .9).get.value | |
def p99QueryLatencyMicros: Double = | |
snapshot.percentiles.find(_.quantile == .99).get.value | |
} | |
class InMemoryLoadTestQueryRecorder[T]( | |
// You have to specify a name of the histogram even though it is not used | |
// Use latch period of bottom. We will compute a new snapshot every time we call computeSnapshot | |
private[this] val latencyHistogram: MetricsBucketedHistogram = | |
new MetricsBucketedHistogram("latencyhistogram", latchPeriod = Duration.Bottom)) | |
extends LoadTestQueryRecorder[T] | |
def computeSnapshot(): QueryRecorderSnapshot | |
def recall: Double = | |
if (counter.get() != 0) | |
def top1Recall: Double = | |
if (counter.get() != 0) | |
def top10Recall: Double = | |
if (countMoreThan10Results.get() != 0) | |
def avgRPS: Double = | |
if (elapsedTime.get() != Duration.Zero) | |
def recordQueryResult( | |
trueNeighbors: Seq[T], | |
foundNeighbors: Seq[T], | |
queryLatency: Duration | |
): Unit | |
class InMemoryLoadTestBuildRecorder extends LoadTestBuildRecorder | |
def recordIndexCreation( | |
size: Int, | |
indexLatencyArg: Duration, | |
toQueryableLatencyArg: Duration | |
): Unit | |
class ComposedLoadTestQueryRecorder[T]( | |
recorders: Seq[LoadTestQueryRecorder[T]]) | |
extends LoadTestQueryRecorder[T] | |
def recordQueryResult( | |
trueNeighbors: Seq[T], | |
foundNeighbors: Seq[T], | |
queryLatency: Duration | |
): Unit = recorders.foreach | |
class ComposedLoadTestBuildRecorder( | |
recorders: Seq[LoadTestBuildRecorder]) | |
extends LoadTestBuildRecorder | |
def recordIndexCreation( | |
indexSize: Int, | |
indexLatency: Duration, | |
toQueryableLatency: Duration | |
): Unit = recorders.foreach | |
.\ann\src\main\scala\com\twitter\ann\service\loadtest\LoadTestUtils.scala | |
object LoadTestUtils | |
def getTruthSetMap[Q, I]( | |
directory: String, | |
queryIdType: String, | |
indexIdType: String | |
): Map[Q, Seq[I]] | |
def getLocalFileHandle( | |
directory: String | |
): AbstractFile | |
def getEmbeddingsSet[T]( | |
directory: String, | |
idType: String | |
): Seq[EntityEmbedding[T]] | |
def loadKnnDirFileToMap[K, V]( | |
directory: AbstractFile, | |
f: Array[String] => Seq[V], | |
converter: String => K | |
): Map[K, Seq[V]] | |
def accept(file: AbstractFile): Boolean = | |
file.getName != AbstractFile.SUCCESS_FILE_NAME | |
}).foreach | |
def getRandomQuerySet( | |
dimension: Int, | |
totalQueries: Int, | |
minValue: Float, | |
maxValue: Float | |
): Seq[EmbeddingVector] | |
def getKeyConverter[T](idType: String): String => T | |
def buildRemoteServiceQueryClient[T, P <: RuntimeParams, D <: Distance[D]]( | |
destination: String, | |
clientId: String, | |
statsReceiver: StatsReceiver, | |
serviceIdentifier: ServiceIdentifier, | |
runtimeParamInjection: Injection[P, ServiceRuntimeParams], | |
distanceInjection: Injection[D, ServiceDistance], | |
indexIdInjection: Injection[T, Array[Byte]] | |
): Future[Queryable[T, P, D]] | |
def apply(request: NearestNeighborQuery): Future[NearestNeighborResult] = | |
client.query(request) | |
} | |
Future.value( | |
new ServiceClientQueryable[T, P, D]( | |
service, | |
runtimeParamInjection, | |
distanceInjection, | |
indexIdInjection | |
) | |
) | |
} | |
// helper method to convert a line in KNN file output format into map | |
@VisibleForTesting | |
def addToMapFromKnnString[K, V]( | |
line: String, | |
f: Array[String] => Seq[V], | |
map: mutable.HashMap[K, Seq[V]], | |
converter: String => K | |
): Unit | |
def printResults( | |
inMemoryBuildRecorder: InMemoryLoadTestBuildRecorder, | |
queryTimeConfigurations: Seq[QueryTimeConfiguration[_, _]] | |
): Seq[String] | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\BaseQueryIndexServer.scala | |
class provides most of the configuration needed for logging, stats, deciders etc. | |
*/ | |
abstract class BaseQueryIndexServer extends ThriftServer with Mtls | |
def additionalModules: Seq[Module] | |
/** | |
* Override this method to add the controller to the thrift router. BaseQueryIndexServer takes | |
* care of most of the other configuration for you. | |
* @param router | |
*/ | |
protected def addController(router: ThriftRouter): Unit | |
override protected final lazy val modules: Seq[Module] = Seq( | |
DeciderModule, | |
new MtlsThriftWebFormsModule[AnnQueryService.MethodPerEndpoint](this) | |
) ++ additionalModules | |
override protected final def configureThrift(router: ThriftRouter): Unit | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\Exceptions.scala | |
object RuntimeExceptionTransform extends ExceptionTransformer | |
def transform | |
def getStatName: PartialFunction[Exception, String] | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\FaissIndexPathProvider.scala | |
class FaissIndexPathProvider( | |
override val minIndexSizeBytes: Long, | |
override val maxIndexSizeBytes: Long, | |
override val statsReceiver: StatsReceiver) | |
extends BaseIndexPathProvider | |
def isValidIndex(dir: AbstractFile): Boolean | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\IndexPathProvider.scala | |
abstract class IndexPathProvider | |
def provideIndexPath(rootPath: AbstractFile, group: Boolean = false): Try[AbstractFile] | |
def provideIndexPathWithGroups(rootPath: AbstractFile): Try[Seq[AbstractFile]] | |
} | |
abstract class BaseIndexPathProvider extends IndexPathProvider | |
def accept(file: AbstractFile): Boolean | |
def findLatestTimeStampValidSuccessDirectory( | |
path: AbstractFile, | |
group: Boolean | |
): AbstractFile | |
def isValidIndex(index: AbstractFile): Boolean | |
override def provideIndexPath( | |
rootPath: AbstractFile, | |
group: Boolean = false | |
): Try[AbstractFile] | |
def provideIndexPathWithGroups( | |
rootPath: AbstractFile | |
): Try[Seq[AbstractFile]] | |
def accept(file: AbstractFile): Boolean = | |
file.isDirectory && file.hasSuccessFile | |
}).asScala.toSeq | |
} | |
} | |
} | |
case class ValidatedIndexPathProvider( | |
override val minIndexSizeBytes: Long, | |
override val maxIndexSizeBytes: Long, | |
override val statsReceiver: StatsReceiver) | |
extends BaseIndexPathProvider | |
def isValidIndex(dir: AbstractFile): Boolean | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\QueryableProvider.scala | |
trait QueryableProvider[T, P <: RuntimeParams, D <: Distance[D]] | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\QueryIndexThriftController.scala | |
class QueryIndexThriftController[T, P <: RuntimeParams, D <: Distance[D]] @Inject() ( | |
statsReceiver: StatsReceiver, | |
queryable: Queryable[T, P, D], | |
runtimeParamInjection: Injection[P, ServiceRuntimeParams], | |
distanceInjection: Injection[D, ServiceDistance], | |
idInjection: Injection[T, Array[Byte]]) | |
extends Controller(AnnQueryService) | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\QueryServerUtil.scala | |
object QueryServerUtil | |
abstract file (directory) size is within the defined limits. | |
* @param dir Hdfs/Local directory | |
* @param minIndexSizeBytes minimum size of file in bytes (Exclusive) | |
* @param maxIndexSizeBytes minimum size of file in bytes (Exclusive) | |
* @return true if file size within minIndexSizeBytes and maxIndexSizeBytes else false | |
*/ | |
def isValidIndexDirSize( | |
dir: AbstractFile, | |
minIndexSizeBytes: Long, | |
maxIndexSizeBytes: Long | |
): Boolean | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\RefreshableQueryable.scala | |
class RefreshableQueryable[T, P <: RuntimeParams, D <: Distance[D]]( | |
grouped: Boolean, | |
rootDir: AbstractFile, | |
queryableProvider: QueryableProvider[T, P, D], | |
indexPathProvider: IndexPathProvider, | |
statsReceiver: StatsReceiver, | |
updateInterval: Duration = 10.minutes) | |
extends QueryableGrouped[T, P, D] | |
def run(): Unit | |
def start(): Unit | |
def buildIndex(indexPath: AbstractFile): Queryable[T, P, D] | |
def innerLoad(): Unit | |
def computeRandomInitDelay(): Duration | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P, | |
key: Option[String] | |
): Future[List[T]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P, | |
key: Option[String] | |
): Future[List[NeighborWithDistance[T, D]]] | |
def getGroupMapping(): Map[Option[String], Queryable[T, P, D]] | |
def query( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[T]] | |
def queryWithDistance( | |
embedding: EmbeddingVector, | |
numOfNeighbors: Int, | |
runtimeParams: P | |
): Future[List[NeighborWithDistance[T, D]]] | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\UnsafeQueryIndexServer.scala | |
class is used when you do not know the generic parameters of the Server at compile time. | |
* If you want compile time checks that your parameters are correct use QueryIndexServer instead. | |
* In particular, when you want to have these id, distance and the runtime params as cli options you | |
* should extend this class. | |
*/ | |
abstract class UnsafeQueryIndexServer[R <: RuntimeParams] extends BaseQueryIndexServer | |
def addController(router: ThriftRouter): Unit | |
def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, R, D] | |
protected val runtimeInjection: Injection[R, ServiceRuntimeParams] | |
private[this] def queryIndexThriftController[ | |
T, | |
D <: Distance[D] | |
]: QueryIndexThriftController[T, R, D] | |
def idInjection[T](): Injection[T, Array[Byte]] | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\throttling\AuroraCPUStatsReader.scala | |
class AuroraCPUStatsReader() | |
def throttledTimeNanos(): Option[Long] = cgroupsCpu.cpuStat.map | |
def cpuQuota: Double = cgroupsCpu.cfsPeriodMicros match | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\throttling\ThrottlingBasedQualityTask.scala | |
object ThrottlingBasedQualityTask | |
class ThrottlingBasedQualityTask( | |
override val statsReceiver: StatsReceiver, | |
// Parameters are taken from OverloadAdmissionController | |
instrument: ThrottlingInstrument = new WindowedThrottlingInstrument(SAMPLING_INTERVAL, 5, | |
new AuroraCPUStatsReader())) | |
extends Task | |
with Logging | |
def task(): Future[Unit] | |
def taskInterval: Duration = SAMPLING_INTERVAL | |
def discountParams[T <: RuntimeParams](params: T): T | |
def applyQualityFactor(param: Int) = math.max(1, math.ceil(param * qualityFactor).toInt) | |
params match | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\throttling\WindowedStats.scala | |
class WindowedStats(window: Int) | |
def add(v: Long): Unit | |
def avg: Double | |
def sum: Long | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\throttling\WindowedThrottlingInstrument.scala | |
trait ThrottlingInstrument | |
def sample(): Unit | |
def percentageOfTimeSpentThrottling(): Double | |
def disabled: Boolean | |
} | |
class WindowedThrottlingInstrument( | |
stepFrequency: Duration, | |
windowLengthInFrequencySteps: Int, | |
reader: AuroraCPUStatsReader) | |
extends ThrottlingInstrument | |
def disabled: Boolean = cpuQuota <= 0 | |
def sample(): Unit = sampleThrottling() match | |
def sampleThrottling(): Option[Long] = reader.throttledTimeNanos().map | |
def percentageOfTimeSpentThrottling(): Double | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\common\warmup\Warmup.scala | |
trait Warmup extends Logging | |
def minSuccessfulTries: Int | |
protected def maxTries: Int | |
protected def randomQueryDimension: Int | |
protected def timeout: Duration | |
@tailrec | |
final protected def run( | |
iteration: Int = 0, | |
successes: Int = 0, | |
name: String, | |
f: => Future[_] | |
): Unit | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\faiss\FaissQueryIndexServer.scala | |
object FaissQueryIndexServer extends FaissQueryableServer | |
class FaissQueryableServer extends UnsafeQueryIndexServer[FaissParams] | |
def queryableProvider[T, D <: Distance[D]]: QueryableProvider[T, FaissParams, D] = | |
new QueryableProvider[T, FaissParams, D] | |
def provideQueryable( | |
directory: AbstractFile | |
): Queryable[T, FaissParams, D] | |
def buildSimpleQueryable[T, D <: Distance[D]]( | |
dir: AbstractFile | |
): Queryable[T, FaissParams, D] | |
def buildShardedQueryable[T, D <: Distance[D]]( | |
dir: AbstractFile | |
): Queryable[T, FaissParams, D] | |
def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, FaissParams, D] | |
def warmup(): Unit = | |
if (warmup_enabled()) | |
new FaissWarmup(unsafeQueryableMap, dimension()).warmup() | |
} | |
class FaissWarmup(faiss: Queryable[_, FaissParams, _], dimension: Int) extends Warmup | |
def minSuccessfulTries: Int = 100 | |
protected def maxTries: Int = 1000 | |
protected def timeout: Duration = 50.milliseconds | |
protected def randomQueryDimension: Int = dimension | |
def warmup(): Unit | |
.\ann\src\main\scala\com\twitter\ann\service\query_server\hnsw\HnswQueryIndexServer.scala | |
object HnswQueryIndexServer extends HnswQueryableServer | |
class HnswQueryableServer extends UnsafeQueryIndexServer[HnswParams] | |
def queryableProvider[T, D <: Distance[D]]: QueryableProvider[T, HnswParams, D] = | |
new QueryableProvider[T, HnswParams, D] | |
def provideQueryable( | |
dir: AbstractFile | |
): Queryable[T, HnswParams, D] | |
def buildQueryable[T, D <: Distance[D]]( | |
dir: AbstractFile, | |
grouped: Boolean | |
): Queryable[T, HnswParams, D] | |
def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, HnswParams, D] | |
def warmup(): Unit = | |
if (warmup_enabled()) new HNSWWarmup(unsafeQueryableMap, dimension()).warmup() | |
} | |
class HNSWWarmup(hnsw: Queryable[_, HnswParams, _], dimension: Int) extends Warmup | |
def minSuccessfulTries: Int = 100 | |
protected def maxTries: Int = 1000 | |
protected def timeout: Duration = 50.milliseconds | |
protected def randomQueryDimension: Int = dimension | |
def warmup(): Unit | |
.\ann\src\main\scala\com\twitter\ann\util\IndexBuilderUtils.scala | |
object IndexBuilderUtils | |
def addToIndex[T]( | |
appendable: Appendable[T, _, _], | |
embeddings: Seq[EntityEmbedding[T]], | |
concurrencyLevel: Int | |
): Future[Int] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\CrMixerHttpServerWarmupHandler.scala | |
class CrMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging | |
def handle(): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\CrMixerServer.scala | |
object CrMixerServerMain extends CrMixerServer | |
class CrMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls | |
def defaultMethodAccess: MethodOptions.Access | |
def configureThrift(router: ThriftRouter): Unit | |
def warmup(): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\CrMixerThriftServerWarmupHandler.scala | |
class CrMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup) | |
extends Handler | |
with Logging | |
def handle(): Unit | |
def warmupQuery(userId: Long): st.CrMixerTweetRequest | |
def assertWarmupResponse( | |
result: Try[Response[st.CrMixer.GetTweetRecommendations.SuccessType]] | |
): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\AdsBlender.scala | |
class AdsBlender @Inject() (globalStats: StatsReceiver) | |
def blend( | |
inputCandidates: Seq[Seq[InitialAdsCandidate]], | |
): Future[Seq[BlendedAdsCandidate]] | |
def buildBlendedAdsCandidate( | |
inputCandidates: Seq[Seq[InitialAdsCandidate]], | |
interleavedCandidates: Seq[InitialAdsCandidate] | |
): Seq[BlendedAdsCandidate] | |
def buildCandidateToCGInfosMap( | |
candidateSeq: Seq[Seq[InitialAdsCandidate]], | |
): Map[TweetId, Seq[CandidateGenerationInfo]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\BlendedCandidatesBuilder.scala | |
object BlendedCandidatesBuilder | |
def build( | |
inputCandidates: Seq[Seq[InitialCandidate]], | |
interleavedCandidates: Seq[InitialCandidate] | |
): Seq[BlendedCandidate] | |
def buildCandidateToCGInfosMap( | |
candidateSeq: Seq[Seq[InitialCandidate]], | |
): Map[TweetId, Seq[CandidateGenerationInfo]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\ContentSignalBlender.scala | |
class ContentSignalBlender @Inject() (globalStats: StatsReceiver) | |
def blend( | |
params: Params, | |
inputCandidates: Seq[Seq[InitialCandidate]], | |
): Future[Seq[BlendedCandidate]] | |
def removeDuplicates(candidates: Seq[InitialCandidate]): Seq[InitialCandidate] | |
def groupByEngineTypeOrFirstContribEngine( | |
candidates: Seq[InitialCandidate] | |
): Map[SimilarityEngineType, Seq[InitialCandidate]] | |
def flattenAndGroupByEngineTypeOrFirstContribEngine( | |
candidates: Seq[Seq[InitialCandidate]] | |
): Seq[Seq[InitialCandidate]] | |
def standardizeAndSortByScore( | |
candidates: Seq[Seq[InitialCandidate]] | |
): Seq[InitialCandidate] | |
def getSnowflakeTimeStamp(tweetId: Long): Time | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\CountWeightedInterleaveBlender.scala | |
class CountWeightedInterleaveBlender @Inject() (globalStats: StatsReceiver) | |
def blend( | |
query: CrCandidateGeneratorQuery, | |
inputCandidates: Seq[Seq[InitialCandidate]] | |
): Future[Seq[BlendedCandidate]] | |
def countWeightedInterleave( | |
query: WeightedBlenderQuery, | |
inputCandidates: Seq[Seq[InitialCandidate]], | |
): Future[Seq[BlendedCandidate]] | |
object CountWeightedInterleaveBlender | |
class WeightedBlenderQuery( | |
rankerWeightShrinkage: Double, | |
maxWeightAdjustments: Int) | |
def paramToQuery(params: Params): WeightedBlenderQuery | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\InterleaveBlender.scala | |
class InterleaveBlender @Inject() (globalStats: StatsReceiver) | |
def blend( | |
inputCandidates: Seq[Seq[InitialCandidate]], | |
): Future[Seq[BlendedCandidate]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\SourceTypeBackFillBlender.scala | |
class SourceTypeBackFillBlender @Inject() (globalStats: StatsReceiver) | |
def blend( | |
params: Params, | |
inputCandidates: Seq[Seq[InitialCandidate]], | |
): Future[Seq[BlendedCandidate]] | |
object ImplicitSignalBackFillBlender | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\blender\SwitchBlender.scala | |
class SwitchBlender @Inject() ( | |
defaultBlender: InterleaveBlender, | |
sourceTypeBackFillBlender: SourceTypeBackFillBlender, | |
adsBlender: AdsBlender, | |
contentSignalBlender: ContentSignalBlender, | |
globalStats: StatsReceiver) | |
def blend( | |
params: Params, | |
userState: UserState, | |
inputCandidates: Seq[Seq[InitialCandidate]], | |
): Future[Seq[BlendedCandidate]] | |
object SwitchBlender | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\AdsCandidateGenerator.scala | |
class AdsCandidateGenerator @Inject() ( | |
ussSourceSignalFetcher: UssSourceSignalFetcher, | |
realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher, | |
adsCandidateSourceRouter: AdsCandidateSourcesRouter, | |
adsBlender: AdsBlender, | |
scribeLogger: AdsRecommendationsScribeLogger, | |
globalStats: StatsReceiver) | |
def get(query: AdsCandidateGeneratorQuery): Future[Seq[RankedAdsCandidate]] | |
def fetchSources( | |
query: AdsCandidateGeneratorQuery | |
): Future[Set[SourceInfo]] | |
def fetchCandidates( | |
query: AdsCandidateGeneratorQuery, | |
sourceSignals: Set[SourceInfo], | |
realGraphSeeds: Map[UserId, Double] | |
): Future[Seq[Seq[InitialAdsCandidate]]] | |
def fetchSeeds( | |
query: AdsCandidateGeneratorQuery | |
): Future[Map[UserId, Double]] | |
def interleave( | |
candidates: Seq[Seq[InitialAdsCandidate]] | |
): Future[Seq[BlendedAdsCandidate]] | |
def rank( | |
candidates: Seq[BlendedAdsCandidate], | |
enableScoreBoost: Boolean, | |
scoreBoostFactor: Double, | |
statsReceiver: StatsReceiver, | |
): Future[Seq[RankedAdsCandidate]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\AdsCandidateSourcesRouter.scala | |
class AdsCandidateSourcesRouter @Inject() ( | |
activePromotedTweetStore: ReadableStore[TweetId, Seq[LineItemInfo]], | |
decider: CrMixerDecider, | |
@Named(ModuleNames.SimClustersANNSimilarityEngine) simClustersANNSimilarityEngine: StandardSimilarityEngine[ | |
Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.TweetBasedUserAdGraphSimilarityEngine) | |
tweetBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedUserAdGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.ConsumersBasedUserAdGraphSimilarityEngine) | |
consumersBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[ | |
ConsumersBasedUserAdGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.ProducerBasedUserAdGraphSimilarityEngine) | |
producerBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[ | |
ProducerBasedUserAdGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) | |
tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, | |
@Named(ModuleNames.ConsumerEmbeddingBasedTwHINANNSimilarityEngine) consumerTwHINANNSimilarityEngine: HnswANNSimilarityEngine, | |
@Named(ModuleNames.ConsumerBasedWalsSimilarityEngine) | |
consumerBasedWalsSimilarityEngine: StandardSimilarityEngine[ | |
ConsumerBasedWalsSimilarityEngine.Query, | |
TweetWithScore | |
], | |
globalStats: StatsReceiver, | |
) | |
def fetchCandidates( | |
requestUserId: UserId, | |
sourceSignals: Set[SourceInfo], | |
realGraphSeeds: Map[UserId, Double], | |
params: configapi.Params | |
): Future[Seq[Seq[InitialAdsCandidate]]] | |
def convertToInitialCandidates( | |
candidates: Seq[TweetWithCandidateGenerationInfo], | |
stats: StatsReceiver | |
): Future[Seq[InitialAdsCandidate]] | |
def getSimClustersANNCandidates( | |
requestUserId: UserId, | |
sourceInfo: Option[SourceInfo], | |
params: configapi.Params, | |
configId: String, | |
minScore: Double | |
) | |
def getProducerBasedUserAdGraphCandidates( | |
sourceInfo: Option[SourceInfo], | |
params: configapi.Params | |
) | |
def getTweetBasedUserAdGraphCandidates( | |
sourceInfo: Option[SourceInfo], | |
params: configapi.Params | |
) | |
def getRealGraphConsumersBasedUserAdGraphCandidates( | |
realGraphSeeds: Map[UserId, Double], | |
params: configapi.Params | |
) | |
def getTwHINAdsCandidates( | |
similarityEngine: HnswANNSimilarityEngine, | |
similarityEngineType: SimilarityEngineType, | |
requestUserId: UserId, | |
sourceInfo: Option[SourceInfo], // if none, then it's consumer-based similarity engine | |
model: String | |
): Future[Seq[TweetWithCandidateGenerationInfo]] | |
def getConsumerBasedWalsCandidates( | |
sourceSignals: Set[SourceInfo], | |
params: configapi.Params | |
): Future[Seq[TweetWithCandidateGenerationInfo]] | |
object AdsCandidateSourcesRouter | |
def getSimClustersANNEmbeddingType( | |
sourceInfo: SourceInfo | |
): EmbeddingType | |
def buildHnswANNQuery(internalId: InternalId, modelId: String): HnswANNEngineQuery | |
def getConsumerBasedWalsSourceInfo( | |
sourceSignals: Set[SourceInfo] | |
): Set[SourceInfo] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\CandidateSourcesRouter.scala | |
class CandidateSourcesRouter @Inject() ( | |
customizedRetrievalCandidateGeneration: CustomizedRetrievalCandidateGeneration, | |
simClustersInterestedInCandidateGeneration: SimClustersInterestedInCandidateGeneration, | |
@Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) | |
tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedUnifiedSimilarityEngine.Query, | |
TweetWithCandidateGenerationInfo | |
], | |
@Named(ModuleNames.ProducerBasedUnifiedSimilarityEngine) | |
producerBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ | |
ProducerBasedUnifiedSimilarityEngine.Query, | |
TweetWithCandidateGenerationInfo | |
], | |
@Named(ModuleNames.ConsumerEmbeddingBasedTripSimilarityEngine) | |
consumerEmbeddingBasedTripSimilarityEngine: StandardSimilarityEngine[ | |
TripEngineQuery, | |
TripTweetWithScore | |
], | |
@Named(ModuleNames.ConsumerEmbeddingBasedTwHINANNSimilarityEngine) | |
consumerBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, | |
@Named(ModuleNames.ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine) | |
consumerBasedTwoTowerSimilarityEngine: HnswANNSimilarityEngine, | |
@Named(ModuleNames.ConsumersBasedUserVideoGraphSimilarityEngine) | |
consumersBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[ | |
ConsumersBasedUserVideoGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.UserTweetEntityGraphSimilarityEngine) userTweetEntityGraphSimilarityEngine: StandardSimilarityEngine[ | |
UserTweetEntityGraphSimilarityEngine.Query, | |
TweetWithScoreAndSocialProof | |
], | |
@Named(ModuleNames.ConsumerBasedWalsSimilarityEngine) | |
consumerBasedWalsSimilarityEngine: StandardSimilarityEngine[ | |
ConsumerBasedWalsSimilarityEngine.Query, | |
TweetWithScore | |
], | |
tweetInfoStore: ReadableStore[TweetId, TweetInfo], | |
globalStats: StatsReceiver, | |
) | |
def fetchCandidates( | |
requestUserId: UserId, | |
sourceSignals: Set[SourceInfo], | |
sourceGraphs: Map[String, Option[GraphSourceInfo]], | |
params: configapi.Params, | |
): Future[Seq[Seq[InitialCandidate]]] | |
def getGraphBasedCandidates[QueryType]( | |
params: configapi.Params, | |
query: EngineQuery[QueryType], | |
engine: StandardSimilarityEngine[QueryType, TweetWithScore], | |
toSimilarityEngineInfo: Double => SimilarityEngineInfo, | |
graphSourceInfoOpt: Option[GraphSourceInfo] = None | |
): Future[Seq[InitialCandidate]] | |
def getCandidates[QueryType]( | |
sourceSignals: Set[SourceInfo], | |
params: configapi.Params, | |
fromParams: (SourceInfo, configapi.Params) => QueryType, | |
getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def getConsumerBasedWalsCandidates( | |
sourceSignals: Set[SourceInfo], | |
params: configapi.Params | |
): Future[Seq[InitialCandidate]] | |
def getSimClustersTripCandidates( | |
params: configapi.Params, | |
query: TripEngineQuery, | |
engine: StandardSimilarityEngine[ | |
TripEngineQuery, | |
TripTweetWithScore | |
], | |
): Future[Seq[InitialCandidate]] | |
def getHnswCandidates( | |
params: configapi.Params, | |
query: HnswANNEngineQuery, | |
engine: HnswANNSimilarityEngine, | |
): Future[Seq[InitialCandidate]] | |
def getCandidatesPerSimilarityEngineModel[QueryType]( | |
requestUserId: UserId, | |
params: configapi.Params, | |
fromParams: (InternalId, configapi.Params) => QueryType, | |
getFunc: QueryType => Future[ | |
Option[Seq[Seq[TweetWithCandidateGenerationInfo]]] | |
] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def convertToInitialCandidates( | |
candidates: Seq[TweetWithCandidateGenerationInfo], | |
): Future[Seq[InitialCandidate]] | |
object CandidateSourcesRouter | |
def getGraphSourceInfoBySourceType( | |
sourceTypeStr: String, | |
sourceGraphs: Map[String, Option[GraphSourceInfo]] | |
): Option[GraphSourceInfo] | |
def getTweetBasedSourceInfo( | |
sourceSignals: Set[SourceInfo] | |
): Set[SourceInfo] | |
def getProducerBasedSourceInfo( | |
sourceSignals: Set[SourceInfo] | |
): Set[SourceInfo] | |
def getConsumerBasedWalsSourceInfo( | |
sourceSignals: Set[SourceInfo] | |
): Set[SourceInfo] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\CrCandidateGenerator.scala | |
class CrCandidateGenerator @Inject() ( | |
sourceInfoRouter: SourceInfoRouter, | |
candidateSourceRouter: CandidateSourcesRouter, | |
switchBlender: SwitchBlender, | |
preRankFilterRunner: PreRankFilterRunner, | |
postRankFilterRunner: PostRankFilterRunner, | |
switchRanker: SwitchRanker, | |
crMixerScribeLogger: CrMixerScribeLogger, | |
timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver) | |
def get(query: CrCandidateGeneratorQuery): Future[Seq[RankedCandidate]] | |
def fetchSources( | |
query: CrCandidateGeneratorQuery | |
): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] | |
def filterSourceInfo( | |
positiveSignals: Set[SourceInfo], | |
negativeSignals: Set[SourceInfo] | |
): Set[SourceInfo] | |
def fetchCandidates( | |
query: CrCandidateGeneratorQuery, | |
sourceSignals: Set[SourceInfo], | |
sourceGraphs: Map[String, Option[GraphSourceInfo]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def preRankFilter( | |
query: CrCandidateGeneratorQuery, | |
candidates: Seq[Seq[InitialCandidate]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def postRankFilter( | |
query: CrCandidateGeneratorQuery, | |
candidates: Seq[RankedCandidate] | |
): Future[Seq[RankedCandidate]] | |
def interleave( | |
query: CrCandidateGeneratorQuery, | |
candidates: Seq[Seq[InitialCandidate]] | |
): Future[Seq[BlendedCandidate]] | |
def rank( | |
query: CrCandidateGeneratorQuery, | |
candidates: Seq[BlendedCandidate], | |
): Future[Seq[RankedCandidate]] | |
def trackResultStats( | |
stats: StatsReceiver | |
)( | |
fn: => Future[Seq[RankedCandidate]] | |
): Future[Seq[RankedCandidate]] | |
def trackReasonChosenSourceTypeStats( | |
candidates: Seq[RankedCandidate], | |
stats: StatsReceiver | |
): Unit | |
def trackReasonChosenSimilarityEngineStats( | |
candidates: Seq[RankedCandidate], | |
stats: StatsReceiver | |
): Unit | |
def trackPotentialReasonsSourceTypeStats( | |
candidates: Seq[RankedCandidate], | |
stats: StatsReceiver | |
): Unit | |
def trackPotentialReasonsSimilarityEngineStats( | |
candidates: Seq[RankedCandidate], | |
stats: StatsReceiver | |
): Unit | |
def trackBlueVerifiedTweetStats( | |
candidates: Seq[RankedCandidate], | |
statsReceiver: StatsReceiver | |
): Unit | |
def trackTopKStats( | |
k: Int, | |
tweetCandidates: Seq[RankedCandidate], | |
isQueryK: Boolean, | |
statsReceiver: StatsReceiver | |
): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\CustomizedRetrievalCandidateGeneration.scala | |
class shall not be thought of as a | |
* Unified Similarity Engine. It is a CG that calls multiple singular Similarity Engines. | |
*/ | |
@Singleton | |
case class CustomizedRetrievalCandidateGeneration @Inject() ( | |
@Named(ModuleNames.TwhinCollabFilterSimilarityEngine) | |
twhinCollabFilterSimilarityEngine: LookupSimilarityEngine[ | |
TwhinCollabFilterSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.DiffusionBasedSimilarityEngine) | |
diffusionBasedSimilarityEngine: LookupSimilarityEngine[ | |
DiffusionBasedSimilarityEngine.Query, | |
TweetWithScore | |
], | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[ | |
Query, | |
Seq[TweetWithCandidateGenerationInfo] | |
] | |
def name: String = this.getClass.getSimpleName | |
private val stats = statsReceiver.scope(name) | |
private val fetchCandidatesStat = stats.scope("fetchCandidates") | |
/** | |
* For each Similarity Engine Model, return a list of tweet candidates | |
*/ | |
override def get( | |
query: Query | |
): Future[Option[Seq[Seq[TweetWithCandidateGenerationInfo]]]] | |
def tweetAgeFilter( | |
candidates: Seq[TweetWithScore], | |
maxTweetAgeHours: Duration | |
): Seq[TweetWithScore] | |
def ageFilterWithStats( | |
offlineInterestedInCandidates: Seq[TweetWithScore], | |
maxTweetAgeHours: Duration, | |
scopedStatsReceiver: StatsReceiver | |
): Seq[TweetWithScore] | |
def getTwhinCollabCandidatesWithCGInfo( | |
tweetCandidates: Option[Seq[TweetWithScore]], | |
maxCandidateNumPerSourceKey: Int, | |
twhinCollabFilterQuery: LookupEngineQuery[ | |
TwhinCollabFilterSimilarityEngine.Query | |
], | |
): Seq[TweetWithCandidateGenerationInfo] | |
def getDiffusionBasedCandidatesWithCGInfo( | |
tweetCandidates: Option[Seq[TweetWithScore]], | |
maxCandidateNumPerSourceKey: Int, | |
diffusionBasedSimilarityEngineQuery: LookupEngineQuery[ | |
DiffusionBasedSimilarityEngine.Query | |
], | |
): Seq[TweetWithCandidateGenerationInfo] | |
object CustomizedRetrievalCandidateGeneration | |
class Query( | |
internalId: InternalId, | |
maxCandidateNumPerSourceKey: Int, | |
maxTweetAgeHours: Duration, | |
// twhinCollabFilter | |
enableTwhinCollabFilter: Boolean, | |
twhinCollabFilterFollowQuery: LookupEngineQuery[ | |
TwhinCollabFilterSimilarityEngine.Query | |
], | |
twhinCollabFilterEngagementQuery: LookupEngineQuery[ | |
TwhinCollabFilterSimilarityEngine.Query | |
], | |
// twhinMultiCluster | |
enableTwhinMultiCluster: Boolean, | |
twhinMultiClusterFollowQuery: LookupEngineQuery[ | |
TwhinCollabFilterSimilarityEngine.Query | |
], | |
twhinMultiClusterEngagementQuery: LookupEngineQuery[ | |
TwhinCollabFilterSimilarityEngine.Query | |
], | |
enableRetweetBasedDiffusion: Boolean, | |
diffusionBasedSimilarityEngineQuery: LookupEngineQuery[ | |
DiffusionBasedSimilarityEngine.Query | |
], | |
) | |
def fromParams( | |
internalId: InternalId, | |
params: configapi.Params | |
): Query | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\FrsTweetCandidateGenerator.scala | |
class FrsTweetCandidateGenerator @Inject() ( | |
@Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]], | |
frsBasedSimilarityEngine: EarlybirdSimilarityEngineRouter, | |
tweetInfoStore: ReadableStore[TweetId, TweetInfo], | |
timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver) | |
def get( | |
frsTweetCandidateGeneratorQuery: FrsTweetCandidateGeneratorQuery | |
): Future[Seq[FrsTweet]] | |
def fetchSeeds( | |
userId: UserId, | |
userDenyList: Set[UserId], | |
languageCodeOpt: Option[String], | |
countryCodeOpt: Option[String], | |
params: Params | |
): Future[Option[Map[UserId, FrsQueryResult]]] | |
def fetchCandidates( | |
searcherUserId: UserId, | |
seedAuthors: Seq[UserId], | |
impressedTweetList: Set[TweetId], | |
frsUserToScores: Map[UserId, Double], | |
params: Params | |
): Future[Option[Seq[TweetWithAuthor]]] | |
def filterCandidates( | |
candidates: Option[Seq[TweetWithAuthor]], | |
params: Params | |
): Future[Option[Seq[TweetWithAuthor]]] | |
def hydrateCandidates( | |
frsAuthorWithScores: Option[Map[UserId, FrsQueryResult]], | |
candidates: Option[Seq[TweetWithAuthor]] | |
): Future[Option[Seq[FrsTweet]]] | |
object FrsTweetCandidateGenerator | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\RelatedTweetCandidateGenerator.scala | |
class RelatedTweetCandidateGenerator @Inject() ( | |
@Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedUnifiedSimilarityEngine.Query, | |
TweetWithCandidateGenerationInfo | |
], | |
@Named(ModuleNames.ProducerBasedUnifiedSimilarityEngine) producerBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ | |
ProducerBasedUnifiedSimilarityEngine.Query, | |
TweetWithCandidateGenerationInfo | |
], | |
preRankFilterRunner: PreRankFilterRunner, | |
relatedTweetScribeLogger: RelatedTweetScribeLogger, | |
tweetInfoStore: ReadableStore[TweetId, TweetInfo], | |
globalStats: StatsReceiver) | |
def get( | |
query: RelatedTweetCandidateGeneratorQuery | |
): Future[Seq[InitialCandidate]] | |
def fetchCandidates( | |
query: RelatedTweetCandidateGeneratorQuery | |
): Future[Seq[Seq[InitialCandidate]]] | |
def getCandidatesFromSimilarityEngine[QueryType]( | |
query: RelatedTweetCandidateGeneratorQuery, | |
fromParamsForRelatedTweet: (InternalId, configapi.Params) => QueryType, | |
getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def preRankFilter( | |
query: RelatedTweetCandidateGeneratorQuery, | |
candidates: Seq[Seq[InitialCandidate]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def convertToInitialCandidates( | |
candidates: Seq[TweetWithCandidateGenerationInfo], | |
): Future[Seq[InitialCandidate]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\RelatedVideoTweetCandidateGenerator.scala | |
class RelatedVideoTweetCandidateGenerator @Inject() ( | |
@Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedUnifiedSimilarityEngine.Query, | |
TweetWithCandidateGenerationInfo | |
], | |
preRankFilterRunner: PreRankFilterRunner, | |
tweetInfoStore: ReadableStore[TweetId, TweetInfo], | |
globalStats: StatsReceiver) | |
def get( | |
query: RelatedVideoTweetCandidateGeneratorQuery | |
): Future[Seq[InitialCandidate]] | |
def fetchCandidates( | |
query: RelatedVideoTweetCandidateGeneratorQuery | |
): Future[Seq[Seq[InitialCandidate]]] | |
def getCandidatesFromSimilarityEngine[QueryType]( | |
query: RelatedVideoTweetCandidateGeneratorQuery, | |
fromParamsForRelatedVideoTweet: (InternalId, configapi.Params) => QueryType, | |
getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def preRankFilter( | |
query: RelatedVideoTweetCandidateGeneratorQuery, | |
candidates: Seq[Seq[InitialCandidate]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def convertToInitialCandidates( | |
candidates: Seq[TweetWithCandidateGenerationInfo], | |
): Future[Seq[InitialCandidate]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\SimClustersInterestedInCandidateGeneration.scala | |
class moving forward. | |
* | |
* After the abstraction improvement (apply SimilarityEngine trait) | |
* these CG will be subjected to change. | |
*/ | |
@Singleton | |
case class SimClustersInterestedInCandidateGeneration @Inject() ( | |
@Named(ModuleNames.SimClustersANNSimilarityEngine) | |
simClustersANNSimilarityEngine: StandardSimilarityEngine[ | |
SimClustersANNSimilarityEngine.Query, | |
TweetWithScore | |
], | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[ | |
SimClustersInterestedInCandidateGeneration.Query, | |
Seq[TweetWithCandidateGenerationInfo] | |
] | |
def name: String = this.getClass.getSimpleName | |
private val stats = statsReceiver.scope(name) | |
private val fetchCandidatesStat = stats.scope("fetchCandidates") | |
override def get( | |
query: SimClustersInterestedInCandidateGeneration.Query | |
): Future[Option[Seq[Seq[TweetWithCandidateGenerationInfo]]]] | |
def simClustersCandidateMinScoreFilter( | |
simClustersAnnCandidates: Seq[TweetWithScore], | |
simClustersInterestedInMinScore: Double, | |
simClustersANNConfigId: String | |
): Seq[TweetWithScore] | |
def getInterestedInCandidateResult( | |
simClustersANNSimilarityEngine: StandardSimilarityEngine[ | |
SimClustersANNSimilarityEngine.Query, | |
TweetWithScore | |
], | |
simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
simClustersInterestedInMinScore: Double, | |
): Future[Option[Seq[TweetWithCandidateGenerationInfo]]] | |
object SimClustersInterestedInCandidateGeneration | |
class Query( | |
internalId: InternalId, | |
enableUserInterestedIn: Boolean, | |
enableUserNextInterestedIn: Boolean, | |
enableAddressBookNextInterestedIn: Boolean, | |
enableProdSimClustersANNSimilarityEngine: Boolean, | |
enableExperimentalSimClustersANNSimilarityEngine: Boolean, | |
enableSimClustersANN1SimilarityEngine: Boolean, | |
enableSimClustersANN2SimilarityEngine: Boolean, | |
enableSimClustersANN3SimilarityEngine: Boolean, | |
enableSimClustersANN5SimilarityEngine: Boolean, | |
enableSimClustersANN4SimilarityEngine: Boolean, | |
simClustersInterestedInMinScore: Double, | |
simClustersNextInterestedInMinScore: Double, | |
simClustersAddressBookInterestedInMinScore: Double, | |
interestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
nextInterestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
addressbookInterestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
interestedInExperimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
nextInterestedInExperimentalSimClustersANNQuery: EngineQuery[ | |
SimClustersANNSimilarityEngine.Query | |
], | |
addressbookInterestedInExperimentalSimClustersANNQuery: EngineQuery[ | |
SimClustersANNSimilarityEngine.Query | |
], | |
interestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
nextInterestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
addressbookInterestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
interestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
nextInterestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
addressbookInterestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
interestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
nextInterestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
addressbookInterestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
interestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
nextInterestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
addressbookInterestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
interestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
nextInterestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
addressbookInterestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
) | |
def fromParams( | |
internalId: InternalId, | |
params: configapi.Params, | |
): Query | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\TopicTweetCandidateGenerator.scala | |
class TopicTweetCandidateGenerator @Inject() ( | |
certoTopicTweetSimilarityEngine: CertoTopicTweetSimilarityEngine, | |
skitTopicTweetSimilarityEngine: SkitTopicTweetSimilarityEngine, | |
skitHighPrecisionTopicTweetSimilarityEngine: SkitHighPrecisionTopicTweetSimilarityEngine, | |
tweetInfoStore: ReadableStore[TweetId, TweetInfo], | |
timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver) | |
def get( | |
query: TopicTweetCandidateGeneratorQuery | |
): Future[Map[Long, Seq[TopicTweet]]] | |
def fetchCandidates( | |
query: TopicTweetCandidateGeneratorQuery | |
): Future[Map[TopicId, Option[Seq[TopicTweetWithScore]]]] | |
def convertToInitialCandidates( | |
candidatesMap: Map[TopicId, Option[Seq[TopicTweetWithScore]]] | |
): Future[Map[TopicId, Seq[InitialCandidate]]] | |
def filterCandidates( | |
topicTweetMap: Map[TopicId, Seq[InitialCandidate]], | |
maxTweetAge: Duration, | |
isVideoOnly: Boolean, | |
excludeTweetIds: Set[TweetId] | |
): Future[Map[TopicId, Seq[InitialCandidate]]] | |
def rankCandidates( | |
tweetCandidatesMap: Map[TopicId, Seq[InitialCandidate]] | |
): Map[TopicId, Seq[InitialCandidate]] | |
def hydrateCandidates( | |
topicCandidatesMap: Map[TopicId, Seq[InitialCandidate]] | |
): Map[Long, Seq[TopicTweet]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\candidate_generation\UtegTweetCandidateGenerator.scala | |
class UtegTweetCandidateGenerator @Inject() ( | |
@Named(ModuleNames.UserTweetEntityGraphSimilarityEngine) userTweetEntityGraphSimilarityEngine: StandardSimilarityEngine[ | |
UserTweetEntityGraphSimilarityEngine.Query, | |
TweetWithScoreAndSocialProof | |
], | |
utegTweetScribeLogger: UtegTweetScribeLogger, | |
tweetInfoStore: ReadableStore[TweetId, TweetInfo], | |
realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher, | |
utegFilterRunner: UtegFilterRunner, | |
globalStats: StatsReceiver) | |
def get( | |
query: UtegTweetCandidateGeneratorQuery | |
): Future[Seq[TweetWithScoreAndSocialProof]] | |
def utegFilter( | |
query: UtegTweetCandidateGeneratorQuery, | |
candidates: Seq[InitialCandidate] | |
): Future[Seq[InitialCandidate]] | |
def fetchSeeds( | |
query: UtegTweetCandidateGeneratorQuery | |
): Future[Map[UserId, Double]] | |
def rankCandidates( | |
query: UtegTweetCandidateGeneratorQuery, | |
filteredCandidates: Seq[InitialCandidate], | |
): Future[Seq[RankedCandidate]] | |
def fetchCandidates( | |
query: UtegTweetCandidateGeneratorQuery, | |
realGraphSeeds: Map[UserId, Double], | |
): Future[Seq[TweetWithScoreAndSocialProof]] | |
def convertToInitialCandidates( | |
candidates: Seq[TweetWithScoreAndSocialProof], | |
): Future[Seq[InitialCandidate]] | |
def convertToTweets( | |
candidates: Seq[RankedCandidate], | |
tweetMap: Map[TweetId, TweetWithScoreAndSocialProof] | |
): Seq[TweetWithScoreAndSocialProof] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\config\SimClustersANNConfig.scala | |
class SimClustersANNConfig( | |
maxNumResults: Int, | |
minScore: Double, | |
candidateEmbeddingType: EmbeddingType, | |
maxTopTweetsPerCluster: Int, | |
maxScanClusters: Int, | |
maxTweetCandidateAge: Duration, | |
minTweetCandidateAge: Duration, | |
annAlgorithm: ScoringAlgorithm) | |
object SimClustersANNConfig | |
def getConfig( | |
embeddingType: String, | |
modelVersion: String, | |
id: String | |
): SimClustersANNConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\controller\CrMixerThriftController.scala | |
class CrMixerThriftController @Inject() ( | |
crCandidateGenerator: CrCandidateGenerator, | |
relatedTweetCandidateGenerator: RelatedTweetCandidateGenerator, | |
relatedVideoTweetCandidateGenerator: RelatedVideoTweetCandidateGenerator, | |
utegTweetCandidateGenerator: UtegTweetCandidateGenerator, | |
frsTweetCandidateGenerator: FrsTweetCandidateGenerator, | |
topicTweetCandidateGenerator: TopicTweetCandidateGenerator, | |
crMixerScribeLogger: CrMixerScribeLogger, | |
relatedTweetScribeLogger: RelatedTweetScribeLogger, | |
utegTweetScribeLogger: UtegTweetScribeLogger, | |
adsRecommendationsScribeLogger: AdsRecommendationsScribeLogger, | |
adsCandidateGenerator: AdsCandidateGenerator, | |
decider: CrMixerDecider, | |
paramsBuilder: ParamsBuilder, | |
endpointLoadShedder: EndpointLoadShedder, | |
signalTimestampStatsUtil: SignalTimestampStatsUtil, | |
tweetRecommendationResultsStore: ReadableWritableStore[UserId, CrMixerTweetResponse], | |
userStateStore: ReadableStore[UserId, UserState], | |
statsReceiver: StatsReceiver) | |
extends Controller(t.CrMixer) | |
def logErrMessage(endpoint: String, e: Throwable): Unit | |
def generateRequestUUID(): Long | |
def getRelatedTweets( | |
endpointName: String, | |
request: RelatedTweetRequest | |
): Future[RelatedTweetResponse] | |
def getRelatedVideoTweets( | |
endpointName: String, | |
request: RelatedVideoTweetRequest | |
): Future[RelatedVideoTweetResponse] | |
def buildCrCandidateGeneratorQuery( | |
thriftRequest: CrMixerTweetRequest, | |
requestUUID: Long, | |
userId: Long | |
): Future[CrCandidateGeneratorQuery] | |
def buildRelatedTweetQuery( | |
thriftRequest: RelatedTweetRequest, | |
requestUUID: Long | |
): Future[RelatedTweetCandidateGeneratorQuery] | |
def buildAdsCandidateGeneratorQuery( | |
thriftRequest: AdsRequest | |
): Future[AdsCandidateGeneratorQuery] | |
def buildRelatedVideoTweetQuery( | |
thriftRequest: RelatedVideoTweetRequest, | |
requestUUID: Long | |
): Future[RelatedVideoTweetCandidateGeneratorQuery] | |
def buildUtegTweetQuery( | |
thriftRequest: UtegTweetRequest, | |
requestUUID: Long | |
): Future[UtegTweetCandidateGeneratorQuery] | |
def buildTopicTweetQuery( | |
thriftRequest: TopicTweetRequest, | |
requestUUID: Long | |
): TopicTweetCandidateGeneratorQuery | |
def buildFrsBasedTweetQuery( | |
thriftRequest: FrsTweetRequest, | |
requestUUID: Long | |
): Future[FrsTweetCandidateGeneratorQuery] | |
def buildThriftResponse( | |
candidates: Seq[RankedCandidate] | |
): CrMixerTweetResponse | |
def scribeTweetScoreFunnelSeries( | |
candidates: Seq[RankedCandidate] | |
): Seq[RankedCandidate] | |
def buildRelatedTweetResponse(candidates: Seq[InitialCandidate]): RelatedTweetResponse | |
def buildRelatedVideoTweetResponse( | |
candidates: Seq[InitialCandidate] | |
): RelatedVideoTweetResponse | |
def buildUtegTweetResponse( | |
candidates: Seq[TweetWithScoreAndSocialProof] | |
): UtegTweetResponse | |
def buildAdsResponse( | |
candidates: Seq[RankedAdsCandidate] | |
): AdsResponse | |
def cacheTweetRecommendationResults( | |
request: CrMixerTweetRequest, | |
response: Future[CrMixerTweetResponse] | |
): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\featureswitch\CrMixerLoggingABDecider.scala | |
class to be used the [[SetImpressedBucketsLocalContextFilter]] must be applied | |
* at the beginning of the request, to initialize a concurrent map used to store impressed buckets. | |
* | |
* Whenever we get an a/b impression, the bucket information is logged to the concurrent hashmap. | |
*/ | |
case class CrMixerLoggingABDecider( | |
loggingAbDecider: LoggingABDecider, | |
statsReceiver: StatsReceiver) | |
extends LoggingABDecider | |
def impression( | |
experimentName: String, | |
recipient: Recipient | |
): Option[Bucket] | |
def track( | |
experimentName: String, | |
eventName: String, | |
recipient: Recipient | |
): Unit | |
def bucket( | |
experimentName: String, | |
recipient: Recipient | |
): Option[Bucket] | |
def experiments: Seq[String] = loggingAbDecider.experiments | |
override def experiment(experimentName: String) = | |
loggingAbDecider.experiment(experimentName) | |
} | |
object CrMixerImpressedBuckets | |
def getAllImpressedBuckets: Option[List[Bucket]] | |
def recordImpressedBucket(bucket: Bucket) | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\featureswitch\ParamsBuilder.scala | |
object for building [[Params]] to override */ | |
@Singleton | |
class ParamsBuilder @Inject() ( | |
globalStats: StatsReceiver, | |
abDecider: LoggingABDecider, | |
featureContextBuilder: FeatureContextBuilder, | |
config: Config) | |
def buildFromClientContext( | |
clientContext: ClientContext, | |
product: t.Product, | |
userState: UserState, | |
userRoleOverride: Option[Set[String]] = None, | |
featureOverrides: Map[String, FeatureValue] = Map.empty, | |
): Params | |
def buildFeatureSwitchRecipientWithGuestId( | |
clientContext: ClientContext, | |
product: t.Product, | |
userState: UserState | |
): FeatureSwitchRecipient | |
def buildFeatureSwitchRecipient( | |
userId: Long, | |
userRolesOverride: Option[Set[String]], | |
clientContext: ClientContext, | |
product: t.Product, | |
userState: UserState | |
): FeatureSwitchRecipient | |
object ParamsBuilder | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\featureswitch\SetImpressedBucketsLocalContextFilter.scala | |
class SetImpressedBucketsLocalContextFilter @Inject() () extends Filter.TypeAgnostic | |
def toFilter[Req, Rep]: Filter[Req, Rep, Req, Rep] = | |
(request: Req, service: Service[Req, Rep]) => | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\FilterBase.scala | |
trait FilterBase | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\ImpressedTweetlistFilter.scala | |
class ImpressedTweetlistFilter() extends FilterBase | |
def filter( | |
candidates: Seq[Seq[InitialCandidate]], | |
config: FilterConfig | |
): Future[Seq[Seq[InitialCandidate]]] | |
def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( | |
request: CGQueryType | |
): FilterConfig | |
object ImpressedTweetlistFilter | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\InNetworkFilter.scala | |
class InNetworkFilter @Inject() ( | |
@Named(ModuleNames.RealGraphInStore) realGraphStoreMh: ReadableStore[UserId, CandidateSeq], | |
globalStats: StatsReceiver) | |
extends FilterBase | |
def filter( | |
candidates: Seq[Seq[InitialCandidate]], | |
filterConfig: FilterConfig, | |
): Future[Seq[Seq[InitialCandidate]]] | |
def filterCandidates( | |
candidates: Seq[Seq[InitialCandidate]], | |
filterConfig: FilterConfig, | |
): Future[Seq[Seq[InitialCandidate]]] | |
def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( | |
request: CGQueryType | |
): FilterConfig | |
object InNetworkFilter | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\PostRankFilterRunner.scala | |
class PostRankFilterRunner @Inject() ( | |
globalStats: StatsReceiver) | |
def run( | |
query: CrCandidateGeneratorQuery, | |
candidates: Seq[RankedCandidate] | |
): Future[Seq[RankedCandidate]] | |
def removeBadRecentNotificationCandidates( | |
candidates: Seq[RankedCandidate] | |
): Seq[RankedCandidate] | |
def isBadQualityRecentNotificationCandidate(candidate: RankedCandidate): Boolean | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\PreRankFilterRunner.scala | |
class PreRankFilterRunner @Inject() ( | |
impressedTweetListFilter: ImpressedTweetlistFilter, | |
tweetAgeFilter: TweetAgeFilter, | |
videoTweetFilter: VideoTweetFilter, | |
tweetReplyFilter: ReplyFilter, | |
globalStats: StatsReceiver) | |
def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery]( | |
request: CGQueryType, | |
candidates: Seq[Seq[InitialCandidate]], | |
): Future[Seq[Seq[InitialCandidate]]] | |
object PreRankFilterRunner | |
def recordCandidateStatsBeforeFilter( | |
candidates: Seq[Seq[InitialCandidate]], | |
statsReceiver: StatsReceiver | |
): Unit | |
def recordCandidateStatsAfterFilter( | |
candidates: Seq[Seq[InitialCandidate]], | |
statsReceiver: StatsReceiver | |
): Unit | |
def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery]( | |
request: CGQueryType, | |
candidates: Seq[Seq[InitialCandidate]], | |
filters: Seq[FilterBase], | |
statsReceiver: StatsReceiver | |
): Future[Seq[Seq[InitialCandidate]]] = | |
filters.foldLeft(Future.value(candidates)) | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\ReplyFilter.scala | |
class ReplyFilter @Inject() () extends FilterBase | |
def name: String = this.getClass.getCanonicalName | |
override type ConfigType = Boolean | |
override def filter( | |
candidates: Seq[Seq[InitialCandidate]], | |
config: ConfigType | |
): Future[Seq[Seq[InitialCandidate]]] | |
def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( | |
query: CGQueryType | |
): ConfigType | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\RetweetFilter.scala | |
class RetweetFilter @Inject() () extends FilterBase | |
def name: String = this.getClass.getCanonicalName | |
override type ConfigType = Boolean | |
override def filter( | |
candidates: Seq[Seq[InitialCandidate]], | |
config: ConfigType | |
): Future[Seq[Seq[InitialCandidate]]] | |
def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( | |
query: CGQueryType | |
): ConfigType | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\TweetAgeFilter.scala | |
class TweetAgeFilter() extends FilterBase | |
def filter( | |
candidates: Seq[Seq[InitialCandidate]], | |
maxTweetAge: Duration | |
): Future[Seq[Seq[InitialCandidate]]] | |
def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( | |
query: CGQueryType | |
): Duration | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\TweetInfoHealthFilterBase.scala | |
trait TweetInfoHealthFilterBase extends FilterBase | |
def name: String = this.getClass.getCanonicalName | |
override type ConfigType = HealthThreshold.Enum.Value | |
def thresholdToPropertyMap: Map[HealthThreshold.Enum.Value, TweetInfo => Option[Boolean]] | |
def getFilterParamFn: CandidateGeneratorQuery => HealthThreshold.Enum.Value | |
override def filter( | |
candidates: Seq[Seq[InitialCandidate]], | |
config: HealthThreshold.Enum.Value | |
): Future[Seq[Seq[InitialCandidate]]] | |
def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( | |
query: CGQueryType | |
): HealthThreshold.Enum.Value | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\UtegFilterRunner.scala | |
class UtegFilterRunner @Inject() ( | |
inNetworkFilter: InNetworkFilter, | |
utegHealthFilter: UtegHealthFilter, | |
retweetFilter: RetweetFilter, | |
globalStats: StatsReceiver) | |
def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery]( | |
request: CGQueryType, | |
candidates: Seq[Seq[InitialCandidate]], | |
): Future[Seq[Seq[InitialCandidate]]] | |
object UtegFilterRunner | |
def recordCandidateStatsBeforeFilter( | |
candidates: Seq[Seq[InitialCandidate]], | |
statsReceiver: StatsReceiver | |
): Unit | |
def recordCandidateStatsAfterFilter( | |
candidates: Seq[Seq[InitialCandidate]], | |
statsReceiver: StatsReceiver | |
): Unit | |
def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery]( | |
request: CGQueryType, | |
candidates: Seq[Seq[InitialCandidate]], | |
filters: Seq[FilterBase], | |
statsReceiver: StatsReceiver | |
): Future[Seq[Seq[InitialCandidate]]] = | |
filters.foldLeft(Future.value(candidates)) | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\UtegHealthFilter.scala | |
class UtegHealthFilter @Inject() () extends FilterBase | |
def name: String = this.getClass.getCanonicalName | |
override type ConfigType = Boolean | |
override def filter( | |
candidates: Seq[Seq[InitialCandidate]], | |
config: ConfigType | |
): Future[Seq[Seq[InitialCandidate]]] | |
def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( | |
query: CGQueryType | |
): ConfigType | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\filter\VideoTweetFilter.scala | |
class VideoTweetFilter() extends FilterBase | |
def filter( | |
candidates: Seq[Seq[InitialCandidate]], | |
config: ConfigType | |
): Future[Seq[Seq[InitialCandidate]]] | |
def checkTweetInfoAttribute(attributeOpt: => Option[Boolean]): Boolean | |
def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( | |
query: CGQueryType | |
): FilterConfig | |
object VideoTweetFilter | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\AdsRecommendationsScribeLogger.scala | |
class AdsRecommendationsScribeLogger @Inject() ( | |
@Named(ModuleNames.AdsRecommendationsLogger) adsRecommendationsScribeLogger: Logger, | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver) | |
def scribeInitialAdsCandidates( | |
query: AdsCandidateGeneratorQuery, | |
getResultFn: => Future[Seq[Seq[InitialAdsCandidate]]], | |
enableScribe: Boolean // controlled by feature switch so that we can scribe for certain DDG | |
): Future[Seq[Seq[InitialAdsCandidate]]] | |
def scribeGetAdsRecommendations( | |
request: AdsRequest, | |
startTime: Long, | |
scribeMetadata: ScribeMetadata, | |
getResultFn: => Future[AdsResponse], | |
enableScribe: Boolean | |
): Future[AdsResponse] | |
def convertFetchCandidatesResult( | |
candidatesSeq: Seq[Seq[InitialAdsCandidate]], | |
requestUserId: UserId | |
): AdsRecommendationsResult | |
def buildScribeMessage( | |
result: AdsRecommendationsResult, | |
scribeMetadata: ScribeMetadata, | |
latencyMs: Long, | |
traceId: Long | |
): GetAdsRecommendationsScribe | |
def scribeResult( | |
scribeMsg: GetAdsRecommendationsScribe | |
): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\CrMixerScribeLogger.scala | |
class CrMixerScribeLogger @Inject() ( | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver, | |
@Named(ModuleNames.TweetRecsLogger) tweetRecsScribeLogger: Logger, | |
@Named(ModuleNames.BlueVerifiedTweetRecsLogger) blueVerifiedTweetRecsScribeLogger: Logger, | |
@Named(ModuleNames.TopLevelApiDdgMetricsLogger) ddgMetricsLogger: Logger, | |
kafkaProducer: KafkaProducerBase[String, GetTweetsRecommendationsScribe]) | |
def scribeSignalSources( | |
query: CrCandidateGeneratorQuery, | |
getResultFn: => Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] | |
): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] | |
def scribeInitialCandidates( | |
query: CrCandidateGeneratorQuery, | |
getResultFn: => Future[Seq[Seq[InitialCandidate]]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def scribePreRankFilterCandidates( | |
query: CrCandidateGeneratorQuery, | |
getResultFn: => Future[Seq[Seq[InitialCandidate]]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def scribeInterleaveCandidates( | |
query: CrCandidateGeneratorQuery, | |
getResultFn: => Future[Seq[BlendedCandidate]] | |
): Future[Seq[BlendedCandidate]] | |
def scribeRankedCandidates( | |
query: CrCandidateGeneratorQuery, | |
getResultFn: => Future[Seq[RankedCandidate]] | |
): Future[Seq[RankedCandidate]] | |
def scribeGetTweetRecommendations( | |
request: CrMixerTweetRequest, | |
startTime: Long, | |
scribeMetadata: ScribeMetadata, | |
getResultFn: => Future[CrMixerTweetResponse] | |
): Future[CrMixerTweetResponse] | |
def scribeGetTweetRecommendationsForBlueVerified( | |
scribeMetadata: ScribeMetadata, | |
getResultFn: => Future[Seq[RankedCandidate]] | |
): Future[Seq[RankedCandidate]] | |
def scribeResultsAndPerformanceMetrics[T]( | |
scribeMetadata: ScribeMetadata, | |
getResultFn: => Future[T], | |
convertToResultFn: (T, UserId) => Result, | |
enableKafkaScribe: Boolean = false | |
): Future[T] | |
def convertTopLevelAPIResult( | |
request: CrMixerTweetRequest, | |
response: CrMixerTweetResponse, | |
startTime: Long | |
): Result | |
def convertFetchSignalSourcesResult( | |
sourceInfoSetTuple: (Set[SourceInfo], Map[String, Option[GraphSourceInfo]]), | |
requestUserId: UserId | |
): Result | |
def convertFetchCandidatesResult( | |
candidatesSeq: Seq[Seq[InitialCandidate]], | |
requestUserId: UserId | |
): Result | |
def convertPreRankFilterResult( | |
candidatesSeq: Seq[Seq[InitialCandidate]], | |
requestUserId: UserId | |
): Result | |
def convertInterleaveResult( | |
blendedCandidates: Seq[BlendedCandidate], | |
requestUserId: UserId | |
): Result | |
def convertRankResult( | |
rankedCandidates: Seq[RankedCandidate], | |
requestUserId: UserId | |
): Result | |
def buildScribeMessage( | |
result: Result, | |
scribeMetadata: ScribeMetadata, | |
latencyMs: Long, | |
traceId: Long | |
): GetTweetsRecommendationsScribe | |
def scribeResult( | |
scribeMsg: GetTweetsRecommendationsScribe | |
): Unit | |
def shouldScribeKafkaMessage( | |
userId: UserId, | |
product: Product | |
): Boolean | |
def downsampleKafkaMessage( | |
scribeMsg: GetTweetsRecommendationsScribe | |
): Seq[GetTweetsRecommendationsScribe] | |
def publishTopLevelDdgMetrics( | |
logger: Logger, | |
topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata, | |
candidateSize: Long, | |
latencyMs: Long, | |
): Unit | |
def getClientData( | |
topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata | |
): ClientDataProvider = | |
MinimalClientDataProvider( | |
userId = topLevelDdgMetricsMetadata.userId, | |
guestId = None, | |
clientApplicationId = topLevelDdgMetricsMetadata.clientApplicationId, | |
countryCode = topLevelDdgMetricsMetadata.countryCode | |
) | |
private def getNamespace( | |
topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata, | |
label: (String, String) | |
): Map[String, String] | |
object CrMixerScribeLogger | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\RelatedTweetScribeLogger.scala | |
class RelatedTweetScribeLogger @Inject() ( | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver, | |
@Named(ModuleNames.RelatedTweetsLogger) relatedTweetsScribeLogger: Logger) | |
def scribeInitialCandidates( | |
query: RelatedTweetCandidateGeneratorQuery, | |
getResultFn: => Future[Seq[Seq[InitialCandidate]]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def scribePreRankFilterCandidates( | |
query: RelatedTweetCandidateGeneratorQuery, | |
getResultFn: => Future[Seq[Seq[InitialCandidate]]] | |
): Future[Seq[Seq[InitialCandidate]]] | |
def scribeGetRelatedTweets( | |
request: RelatedTweetRequest, | |
startTime: Long, | |
relatedTweetScribeMetadata: RelatedTweetScribeMetadata, | |
getResultFn: => Future[RelatedTweetResponse] | |
): Future[RelatedTweetResponse] | |
def scribeResultsAndPerformanceMetrics[T]( | |
relatedTweetScribeMetadata: RelatedTweetScribeMetadata, | |
getResultFn: => Future[T], | |
convertToResultFn: (T, UserId) => RelatedTweetResult | |
): Future[T] | |
def convertTopLevelAPIResult( | |
request: RelatedTweetRequest, | |
response: RelatedTweetResponse, | |
startTime: Long | |
): RelatedTweetResult | |
def convertFetchCandidatesResult( | |
candidatesSeq: Seq[Seq[InitialCandidate]], | |
requestUserId: UserId | |
): RelatedTweetResult | |
def convertPreRankFilterResult( | |
candidatesSeq: Seq[Seq[InitialCandidate]], | |
requestUserId: UserId | |
): RelatedTweetResult | |
def buildScribeMessage( | |
relatedTweetResult: RelatedTweetResult, | |
relatedTweetScribeMetadata: RelatedTweetScribeMetadata, | |
latencyMs: Long, | |
traceId: Long | |
): GetRelatedTweetsScribe | |
def scribeResult( | |
scribeMsg: GetRelatedTweetsScribe | |
): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\ScribeLoggerUtils.scala | |
object ScribeLoggerUtils | |
def publish[T <: ThriftStruct]( | |
logger: Logger, | |
codec: ThriftStructCodec[T], | |
message: T | |
): Unit | |
def getImpressedBuckets( | |
scopedStats: StatsReceiver | |
): Option[List[ImpressesedBucketInfo]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\ScribeMetadata.scala | |
class ScribeMetadata( | |
requestUUID: Long, | |
userId: UserId, | |
product: Product) | |
object ScribeMetadata | |
def from(query: CrCandidateGeneratorQuery): ScribeMetadata | |
def from(query: UtegTweetCandidateGeneratorQuery): ScribeMetadata | |
def from(query: AdsCandidateGeneratorQuery): ScribeMetadata | |
class RelatedTweetScribeMetadata( | |
requestUUID: Long, | |
internalId: InternalId, | |
clientContext: ClientContext, | |
product: Product) | |
object RelatedTweetScribeMetadata | |
def from(query: RelatedTweetCandidateGeneratorQuery): RelatedTweetScribeMetadata | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\TopLevelDdgMetricsMetadata.scala | |
class TopLevelDdgMetricsMetadata( | |
userId: Option[Long], | |
product: Product, | |
clientApplicationId: Option[Long], | |
countryCode: Option[String]) | |
object TopLevelDdgMetricsMetadata | |
def from(request: CrMixerTweetRequest): TopLevelDdgMetricsMetadata | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\logging\UtegTweetScribeLogger.scala | |
class UtegTweetScribeLogger @Inject() ( | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver, | |
@Named(ModuleNames.UtegTweetsLogger) utegTweetScribeLogger: Logger) | |
def scribeInitialCandidates( | |
query: UtegTweetCandidateGeneratorQuery, | |
getResultFn: => Future[Seq[TweetWithScoreAndSocialProof]] | |
): Future[Seq[TweetWithScoreAndSocialProof]] | |
def scribeGetUtegTweetRecommendations( | |
request: UtegTweetRequest, | |
startTime: Long, | |
scribeMetadata: ScribeMetadata, | |
getResultFn: => Future[UtegTweetResponse] | |
): Future[UtegTweetResponse] | |
def convertTopLevelAPIResult( | |
request: UtegTweetRequest, | |
response: UtegTweetResponse, | |
startTime: Long | |
): UtegTweetResult | |
def buildScribeMessage( | |
utegTweetResult: UtegTweetResult, | |
scribeMetadata: ScribeMetadata, | |
latencyMs: Long, | |
traceId: Long | |
): GetUtegTweetsScribe | |
def scribeResult( | |
scribeMsg: GetUtegTweetsScribe | |
): Unit | |
def convertFetchCandidatesResult( | |
candidates: Seq[TweetWithScoreAndSocialProof], | |
requestUserId: UserId | |
): UtegTweetResult | |
def scribeResultsAndPerformanceMetrics[T]( | |
scribeMetadata: ScribeMetadata, | |
getResultFn: => Future[T], | |
convertToResultFn: (T, UserId) => UtegTweetResult | |
): Future[T] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\Candidate.scala | |
trait Candidate | |
def hashCode: Int = tweetId.toInt | |
} | |
case class TweetWithCandidateGenerationInfo( | |
tweetId: TweetId, | |
candidateGenerationInfo: CandidateGenerationInfo) | |
extends Candidate | |
def getSimilarityScore: Double = | |
candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) | |
} | |
case class InitialCandidate( | |
tweetId: TweetId, | |
tweetInfo: TweetInfo, | |
candidateGenerationInfo: CandidateGenerationInfo) | |
extends Candidate | |
def getSimilarityScore: Double = | |
candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) | |
/** | |
* The same candidate can be generated by multiple algorithms. | |
* During blending, candidate deduping happens. In order to retain the candidateGenerationInfo | |
* from different algorithms, we attach them to a list of potentialReasons. | |
*/ | |
def toBlendedCandidate( | |
potentialReasons: Seq[CandidateGenerationInfo], | |
): BlendedCandidate | |
def toRankedCandidate(): RankedCandidate | |
class InitialAdsCandidate( | |
tweetId: TweetId, | |
lineItemInfo: Seq[LineItemInfo], | |
candidateGenerationInfo: CandidateGenerationInfo) | |
extends Candidate | |
def getSimilarityScore: Double = | |
candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) | |
/** | |
* The same candidate can be generated by multiple algorithms. | |
* During blending, candidate deduping happens. In order to retain the candidateGenerationInfo | |
* from different algorithms, we attach them to a list of potentialReasons. | |
*/ | |
def toBlendedAdsCandidate( | |
potentialReasons: Seq[CandidateGenerationInfo], | |
): BlendedAdsCandidate | |
def toRankedAdsCandidate(): RankedAdsCandidate | |
class BlendedCandidate( | |
tweetId: TweetId, | |
tweetInfo: TweetInfo, | |
reasonChosen: CandidateGenerationInfo, | |
potentialReasons: Seq[CandidateGenerationInfo]) | |
extends Candidate | |
def getSimilarityScore: Double = | |
reasonChosen.similarityEngineInfo.score.getOrElse(0.0) | |
assert(potentialReasons.contains(reasonChosen)) | |
def toRankedCandidate(predictionScore: Double): RankedCandidate | |
class BlendedAdsCandidate( | |
tweetId: TweetId, | |
lineItemInfo: Seq[LineItemInfo], | |
reasonChosen: CandidateGenerationInfo, | |
potentialReasons: Seq[CandidateGenerationInfo]) | |
extends Candidate | |
def getSimilarityScore: Double = | |
reasonChosen.similarityEngineInfo.score.getOrElse(0.0) | |
assert(potentialReasons.contains(reasonChosen)) | |
def toRankedAdsCandidate(predictionScore: Double): RankedAdsCandidate | |
class RankedCandidate( | |
tweetId: TweetId, | |
tweetInfo: TweetInfo, | |
predictionScore: Double, | |
reasonChosen: CandidateGenerationInfo, | |
potentialReasons: Seq[CandidateGenerationInfo]) | |
extends Candidate | |
def getSimilarityScore: Double = | |
reasonChosen.similarityEngineInfo.score.getOrElse(0.0) | |
assert(potentialReasons.contains(reasonChosen)) | |
} | |
case class RankedAdsCandidate( | |
tweetId: TweetId, | |
lineItemInfo: Seq[LineItemInfo], | |
predictionScore: Double, | |
reasonChosen: CandidateGenerationInfo, | |
potentialReasons: Seq[CandidateGenerationInfo]) | |
extends Candidate | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\CandidateGeneratorQuery.scala | |
trait CandidateGeneratorQuery | |
trait HasUserId | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\HealthThreshold.scala | |
object HealthThreshold | |
object Enum extends Enumeration | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\ModelConfig.scala | |
class for all Model Based Candidate Sources. | |
* | |
* The Model Name Guideline. Please your modelId as "Algorithm_Product_Date" | |
* If your model is used for multiple product surfaces, name it as all | |
* Don't name your algorithm as MBCG. All the algorithms here are MBCG =.= | |
* | |
* Don't forgot to add your new models into allHnswANNSimilarityEngineModelIds list. | |
*/ | |
object ModelConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\model\ModuleNames.scala | |
object ModuleNames | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\ActivePromotedTweetStoreModule.scala | |
object ActivePromotedTweetStoreModule extends TwitterModule | |
class ActivePromotedTweetStore( | |
activePromotedTweetMHStore: ReadableStore[String, DataRecord], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[TweetId, Seq[LineItemInfo]] | |
def get(tweetId: TweetId): Future[Option[Seq[LineItemInfo]]] | |
def providesActivePromotedTweetStore( | |
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
crMixerStatsReceiver: StatsReceiver | |
): ReadableStore[TweetId, Seq[LineItemInfo]] | |
def statsReceiver: StatsReceiver = | |
crMixerStatsReceiver.scope("active_promoted_tweets_mh") | |
} | |
val mhStore: ReadableStore[String, DataRecord] = | |
ManhattanRO | |
.getReadableStoreWithMtls[String, DataRecord]( | |
mhConfig, | |
manhattanKVClientMtlsParams | |
)( | |
implicitly[Injection[String, Array[Byte]]], | |
CompactThriftCodec[DataRecord] | |
) | |
val underlyingStore = | |
ActivePromotedTweetStore(mhStore, crMixerStatsReceiver.scope("ActivePromotedTweetStore")) | |
val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( | |
backingStore = underlyingStore, | |
cacheClient = crMixerUnifiedCacheClient, | |
ttl = 60.minutes, | |
asyncUpdate = false | |
)( | |
valueInjection = LZ4Injection.compose(SeqObjectInjection[LineItemInfo]()), | |
statsReceiver = crMixerStatsReceiver.scope("memCachedActivePromotedTweetStore"), | |
keyToString | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\BlueVerifiedAnnotationStoreModule.scala | |
object BlueVerifiedAnnotationStoreModule extends TwitterModule | |
def providesBlueVerifiedAnnotationStore( | |
statsReceiver: StatsReceiver, | |
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, | |
): ReadableStore[String, BlueVerifiedAnnotationsV2] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\CertoStratoStoreModule.scala | |
object CertoStratoStoreModule extends TwitterModule | |
def providesCertoStratoStore( | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
stratoClient: Client, | |
statsReceiver: StatsReceiver | |
): ReadableStore[TopicId, Seq[TweetWithScores]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\ConsumersBasedUserAdGraphStoreModule.scala | |
object ConsumersBasedUserAdGraphStoreModule extends TwitterModule | |
def providesConsumerBasedUserAdGraphStore( | |
userAdGraphService: UserAdGraph.MethodPerEndpoint | |
): ReadableStore[ConsumersBasedRelatedAdRequest, RelatedAdResponse] | |
def get( | |
k: ConsumersBasedRelatedAdRequest | |
): Future[Option[RelatedAdResponse]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\ConsumersBasedUserTweetGraphStoreModule.scala | |
object ConsumersBasedUserTweetGraphStoreModule extends TwitterModule | |
def providesConsumerBasedUserTweetGraphStore( | |
userTweetGraphService: UserTweetGraph.MethodPerEndpoint | |
): ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] | |
def get( | |
k: ConsumersBasedRelatedTweetRequest | |
): Future[Option[RelatedTweetResponse]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\ConsumersBasedUserVideoGraphStoreModule.scala | |
object ConsumersBasedUserVideoGraphStoreModule extends TwitterModule | |
def providesConsumerBasedUserVideoGraphStore( | |
userVideoGraphService: UserVideoGraph.MethodPerEndpoint | |
): ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] | |
def get( | |
k: ConsumersBasedRelatedTweetRequest | |
): Future[Option[RelatedTweetResponse]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\CrMixerParamConfigModule.scala | |
object CrMixerParamConfigModule extends TwitterModule | |
def provideConfig(): Config | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\DiffusionStoreModule.scala | |
object DiffusionStoreModule extends TwitterModule | |
def retweetBasedDiffusionRecsMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[Long, TweetsWithScore] | |
def buildTweetRecsStore( | |
serviceIdentifier: ServiceIdentifier, | |
manhattanROConfig: ManhattanROConfig | |
): ReadableStore[Long, TweetsWithScore] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\EarlybirdRecencyBasedCandidateStoreModule.scala | |
object EarlybirdRecencyBasedCandidateStoreModule extends TwitterModule | |
def providesEarlybirdRecencyBasedWithoutRetweetsRepliesCandidateStore( | |
statsReceiver: StatsReceiver, | |
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, | |
@Named(ModuleNames.EarlybirdTweetsCache) earlybirdRecencyBasedTweetsCache: MemcachedClient, | |
timeoutConfig: TimeoutConfig | |
): ReadableStore[UserId, Seq[TweetId]] | |
def get(userId: UserId): Future[Option[Seq[TweetId]]] | |
def providesEarlybirdRecencyBasedWithRetweetsRepliesCandidateStore( | |
statsReceiver: StatsReceiver, | |
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, | |
@Named(ModuleNames.EarlybirdTweetsCache) earlybirdRecencyBasedTweetsCache: MemcachedClient, | |
timeoutConfig: TimeoutConfig | |
): ReadableStore[UserId, Seq[TweetId]] | |
def get(userId: UserId): Future[Option[Seq[TweetId]]] | |
def buildEarlybirdRequest( | |
seedUserId: UserId, | |
filterOutRetweetsAndReplies: Boolean, | |
maxNumTweetsPerSeedUser: Int, | |
processingTimeout: Duration | |
): EarlybirdRequest = | |
EarlybirdRequest( | |
searchQuery = getThriftSearchQuery( | |
seedUserId = seedUserId, | |
filterOutRetweetsAndReplies = filterOutRetweetsAndReplies, | |
maxNumTweetsPerSeedUser = maxNumTweetsPerSeedUser, | |
processingTimeout = processingTimeout | |
), | |
clientId = Some(EarlybirdClientId), | |
timeoutMs = processingTimeout.inMilliseconds.intValue(), | |
getOlderResults = Some(false), | |
adjustedProtectedRequestParams = None, | |
adjustedFullArchiveRequestParams = None, | |
getProtectedTweetsOnly = Some(false), | |
skipVeryRecentTweets = true, | |
) | |
private def getThriftSearchQuery( | |
seedUserId: UserId, | |
filterOutRetweetsAndReplies: Boolean, | |
maxNumTweetsPerSeedUser: Int, | |
processingTimeout: Duration | |
): ThriftSearchQuery = ThriftSearchQuery( | |
serializedQuery = GetEarlybirdQuery( | |
None, | |
None, | |
Set.empty, | |
filterOutRetweetsAndReplies | |
).map(_.serialize), | |
fromUserIDFilter64 = Some(Seq(seedUserId)), | |
numResults = maxNumTweetsPerSeedUser, | |
rankingMode = ThriftSearchRankingMode.Recency, | |
collectorParams = Some( | |
CollectorParams( | |
// numResultsToReturn defines how many results each EB shard will return to search root | |
numResultsToReturn = maxNumTweetsPerSeedUser, | |
// terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. | |
terminationParams = | |
GetCollectorTerminationParams(maxNumTweetsPerSeedUser, processingTimeout) | |
)), | |
facetFieldNames = Some(FacetsToFetch), | |
resultMetadataOptions = Some(MetadataOptions), | |
searchStatusIds = None | |
) | |
private def getEarlybirdSearchResult( | |
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, | |
request: EarlybirdRequest, | |
statsReceiver: StatsReceiver | |
): Future[Option[Seq[TweetId]]] = earlybirdSearchClient | |
.search(request) | |
.map | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\EmbeddingStoreModule.scala | |
object EmbeddingStoreModule extends TwitterModule | |
def twHINEmbeddingRegularUpdateMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[InternalId, api.Embedding] | |
def consumerBasedTwHINEmbeddingRegularUpdateMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[InternalId, api.Embedding] | |
def twoTowerFavConsumerEmbeddingMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[InternalId, api.Embedding] | |
def debuggerDemoUserEmbeddingStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[InternalId, api.Embedding] | |
def debuggerDemoTweetEmbeddingStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[InternalId, api.Embedding] | |
def buildUserEmbeddingStore( | |
serviceIdentifier: ServiceIdentifier, | |
manhattanROConfig: ManhattanROConfig | |
): ReadableStore[InternalId, api.Embedding] | |
def buildTweetEmbeddingStore( | |
serviceIdentifier: ServiceIdentifier, | |
manhattanROConfig: ManhattanROConfig | |
): ReadableStore[InternalId, api.Embedding] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\FrsStoreModule.scala | |
object FrsStoreModule extends TwitterModule | |
def providesFrsStore( | |
frsClient: FollowRecommendationsThriftService.MethodPerEndpoint, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): ReadableStore[FrsStore.Query, Seq[FrsQueryResult]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\MHMtlsParamsModule.scala | |
object MHMtlsParamsModule extends TwitterModule | |
def providesManhattanMtlsParams( | |
serviceIdentifier: ServiceIdentifier | |
): ManhattanKVClientMtlsParams | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\OfflineCandidateStoreModule.scala | |
object OfflineCandidateStoreModule extends TwitterModule | |
def offlineTweet2020CandidateMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def offlineTweet2020Hl0El15CandidateMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def offlineTweet2020Hl2El15CandidateMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def offlineTweet2020Hl2El50CandidateMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def offlineTweet2020Hl8El50CandidateMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def offlineTweetMTSCandidateMhStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def offlineFavDecayedSumCandidateStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def offlineFtrAt5Pop1000RankDecay11CandidateStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def offlineFtrAt5Pop10000RankDecay11CandidateStore( | |
serviceIdentifier: ServiceIdentifier | |
): ReadableStore[UserId, CandidateTweetsList] | |
def buildOfflineCandidateStore( | |
serviceIdentifier: ServiceIdentifier, | |
datasetName: String | |
): ReadableStore[UserId, CandidateTweetsList] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\RealGraphOonStoreModule.scala | |
object RealGraphOonStoreModule extends TwitterModule | |
def providesRealGraphOonStore( | |
stratoClient: StratoClient, | |
statsReceiver: StatsReceiver | |
): ReadableStore[UserId, CandidateSeq] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\RealGraphStoreMhModule.scala | |
object RealGraphStoreMhModule extends TwitterModule | |
def providesRealGraphStoreMh( | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver, | |
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
): ReadableStore[UserId, CandidateSeq] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\RepresentationManagerModule.scala | |
object RepresentationManagerModule extends TwitterModule | |
def providesRepresentationManagerTweetStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[TweetId, SimClustersEmbedding] | |
def providesRepresentationManagerUserFavBasedProducerEmbeddingStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[UserId, SimClustersEmbedding] | |
def providesRepresentationManagerUserLogFavConsumerEmbeddingStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[UserId, SimClustersEmbedding] | |
def providesRepresentationManagerUserFollowInterestedInEmbeddingStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[UserId, SimClustersEmbedding] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\RepresentationScorerModule.scala | |
object RepresentationScorerModule extends TwitterModule | |
def providesRepresentationScorerStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[(UserId, TweetId), Double] | |
def representationScorerStoreKeyMapping(t1: TweetId, t2: TweetId): ListScoreId | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\SampleSimilarityEngineModule.scala | |
object SimpleSimilarityEngineModule extends TwitterModule | |
def providesSimpleSimilarityEngine( | |
timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver | |
): StandardSimilarityEngine[UserId, (TweetId, Double)] | |
object LookupSimilarityEngineModule extends TwitterModule | |
def providesLookupSimilarityEngine( | |
timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver | |
): LookupSimilarityEngine[UserId, (TweetId, Double)] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\SimClustersANNServiceNameToClientMapper.scala | |
object SimClustersANNServiceNameToClientMapper extends TwitterModule | |
def providesSimClustersANNServiceNameToClientMapping( | |
@Named(ModuleNames.ProdSimClustersANNServiceClientName) simClustersANNServiceProd: SimClustersANNService.MethodPerEndpoint, | |
@Named(ModuleNames.ExperimentalSimClustersANNServiceClientName) simClustersANNServiceExperimental: SimClustersANNService.MethodPerEndpoint, | |
@Named(ModuleNames.SimClustersANNServiceClientName1) simClustersANNService1: SimClustersANNService.MethodPerEndpoint, | |
@Named(ModuleNames.SimClustersANNServiceClientName2) simClustersANNService2: SimClustersANNService.MethodPerEndpoint, | |
@Named(ModuleNames.SimClustersANNServiceClientName3) simClustersANNService3: SimClustersANNService.MethodPerEndpoint, | |
@Named(ModuleNames.SimClustersANNServiceClientName5) simClustersANNService5: SimClustersANNService.MethodPerEndpoint, | |
@Named(ModuleNames.SimClustersANNServiceClientName4) simClustersANNService4: SimClustersANNService.MethodPerEndpoint | |
): Map[String, SimClustersANNService.MethodPerEndpoint] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\SkitStratoStoreModule.scala | |
object SkitStratoStoreModule extends TwitterModule | |
def providesSkitStratoStore( | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
stratoClient: Client, | |
statsReceiver: StatsReceiver | |
): ReadableStore[TopicTweetPartitionFlatKey, Seq[TopicTweet]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\StrongTiePredictionStoreModule.scala | |
object StrongTiePredictionStoreModule extends TwitterModule | |
def providesStrongTiePredictionStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[UserId, STPResult] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TripCandidateStoreModule.scala | |
object TripCandidateStoreModule extends TwitterModule | |
def providesSimClustersTripCandidateStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient | |
): ReadableStore[TripDomain, Seq[TripTweet]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TweetInfoStoreModule.scala | |
object TweetInfoStoreModule extends TwitterModule | |
def modules: Seq[Module] = Seq(UnifiedCacheClient) | |
@Provides | |
@Singleton | |
def providesTweetInfoStore( | |
statsReceiver: StatsReceiver, | |
serviceIdentifier: ServiceIdentifier, | |
stratoClient: StratoClient, | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, | |
tweetyPieService: TweetService.MethodPerEndpoint, | |
userTweetGraphPlusService: UserTweetGraphPlus.MethodPerEndpoint, | |
@Named(ModuleNames.BlueVerifiedAnnotationStore) blueVerifiedAnnotationStore: ReadableStore[ | |
String, | |
BlueVerifiedAnnotationsV2 | |
], | |
decider: CrMixerDecider | |
): ReadableStore[TweetId, TweetInfo] | |
def get( | |
k: TweetId | |
): Future[Option[TweetEngagementScores]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TweetRecentEngagedUserStoreModule.scala | |
object TweetRecentEngagedUserStoreModule extends TwitterModule | |
def providesTweetRecentEngagedUserStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[TweetId, TweetRecentEngagedUsers] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TweetRecommendationResultsStoreModule.scala | |
object TweetRecommendationResultsStoreModule extends TwitterModule | |
def providesTweetRecommendationResultsStore( | |
@Named(ModuleNames.TweetRecommendationResultsCache) tweetRecommendationResultsCacheClient: MemcachedClient, | |
statsReceiver: StatsReceiver | |
): ReadableWritableStore[UserId, CrMixerTweetResponse] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TwhinCollabFilterStratoStoreModule.scala | |
object TwhinCollabFilterStratoStoreModule extends TwitterModule | |
def providesTwhinCollabFilterStratoStoreForFollow( | |
stratoClient: StratoClient | |
): ReadableStore[Long, Seq[TweetId]] | |
def providesTwhinCollabFilterStratoStoreForEngagement( | |
stratoClient: StratoClient | |
): ReadableStore[Long, Seq[TweetId]] | |
def providesTwhinMultiClusterStratoStoreForFollow( | |
stratoClient: StratoClient | |
): ReadableStore[Long, Seq[TweetId]] | |
def providesTwhinMultiClusterStratoStoreForEngagement( | |
stratoClient: StratoClient | |
): ReadableStore[Long, Seq[TweetId]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\TwiceClustersMembersStoreModule.scala | |
object TwiceClustersMembersStoreModule extends TwitterModule | |
def providesTweetRecentEngagedUserStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[UserId, OrderedClustersAndMembers] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\UnifiedCacheClient.scala | |
object UnifiedCacheClient extends TwitterModule | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\UserSignalServiceColumnModule.scala | |
object UserSignalServiceColumnModule extends TwitterModule | |
def providesUserSignalServiceStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[BatchSignalRequest, BatchSignalResponse] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\UserSignalServiceStoreModule.scala | |
object UserSignalServiceStoreModule extends TwitterModule | |
def providesUserSignalServiceStore( | |
statsReceiver: StatsReceiver, | |
stratoClient: StratoClient, | |
): ReadableStore[Query, Seq[(SignalType, Seq[UssSignal])]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\UserStateStoreModule.scala | |
object UserStateStoreModule extends TwitterModule | |
class UserStateStore( | |
userStateStore: ReadableStore[UserId, CondensedUserState], | |
timeout: Duration, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[UserId, UserState] | |
def get(userId: UserId): Future[Option[UserState]] | |
def providesUserStateStore( | |
crMixerDecider: CrMixerDecider, | |
statsReceiver: StatsReceiver, | |
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
timeoutConfig: TimeoutConfig | |
): ReadableStore[UserId, UserState] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\ABDeciderModule.scala | |
object ABDeciderModule extends TwitterModule | |
def provideABDecider( | |
@Flag("abdecider.path") abDeciderYmlPath: String, | |
@Named(ModuleNames.AbDeciderLogger) scribeLogger: Logger | |
): LoggingABDecider | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\CrMixerFlagModule.scala | |
object CrMixerFlagName | |
object CrMixerFlagModule extends TwitterModule | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\CrMixerLoggingABDeciderModule.scala | |
object CrMixerLoggingABDeciderModule extends TwitterModule | |
def provideABDecider( | |
loggingABDecider: LoggingABDecider, | |
statsReceiver: StatsReceiver | |
): CrMixerLoggingABDecider | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\FeatureContextBuilderModule.scala | |
object FeatureContextBuilderModule extends TwitterModule | |
def providesFeatureContextBuilder(featureSwitches: FeatureSwitches): FeatureContextBuilder | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\FeatureSwitchesModule.scala | |
object FeatureSwitchesModule extends TwitterModule | |
def providesFeatureSwitches( | |
@Flag("featureswitches.path") featureSwitchDirectory: String, | |
@Flag("use_config_repo_mirror.bool") useConfigRepoMirrorFlag: Boolean, | |
abDecider: CrMixerLoggingABDecider, | |
statsReceiver: StatsReceiver | |
): FeatureSwitches | |
def getConfigRepoAbsPath( | |
useConfigRepoMirrorFlag: Boolean | |
): String | |
def shouldFastRefresh( | |
useConfigRepoMirrorFlag: Boolean | |
): Boolean | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\KafkaProducerModule.scala | |
object KafkaProducerModule extends TwitterModule | |
def provideTweetRecsLoggerFactory( | |
serviceIdentifier: ServiceIdentifier, | |
): KafkaProducerBase[String, GetTweetsRecommendationsScribe] | |
object KafkaProducerFactory | |
def getKafkaProducer( | |
environment: String | |
): KafkaProducerBase[String, GetTweetsRecommendationsScribe] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\LoggerFactoryModule.scala | |
object LoggerFactoryModule extends TwitterModule | |
def provideAbDeciderLogger( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Logger | |
def provideTopLevelApiDdgMetricsLogger( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Logger | |
def provideTweetRecsLogger( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Logger | |
def provideVITTweetRecsLogger( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Logger | |
def provideRelatedTweetsLogger( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Logger | |
def provideUtegTweetsLogger( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Logger | |
def provideAdsRecommendationsLogger( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Logger | |
def buildLoggerFactory( | |
category: ScribeCategory, | |
environment: String, | |
statsReceiver: StatsReceiver | |
): TwitterLoggerFactory | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\MemoizingStatsReceiverModule.scala | |
object MemoizingStatsReceiverModule extends TwitterModule | |
def configure(): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\core\TimeoutConfigModule.scala | |
object TimeoutConfigModule extends TwitterModule | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\grpc_client\NaviGRPCClientModule.scala | |
object NaviGRPCClientModule extends TwitterModule | |
def providesHomeNaviGRPCClient( | |
serviceIdentifier: ServiceIdentifier, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): ManagedChannel | |
def providesAdsFavedNaviGRPCClient( | |
serviceIdentifier: ServiceIdentifier, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): ManagedChannel | |
def providesAdsMonetizableNaviGRPCClient( | |
serviceIdentifier: ServiceIdentifier, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): ManagedChannel | |
def buildClient( | |
serviceIdentifier: ServiceIdentifier, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
dest: String, | |
label: String | |
): ManagedChannel | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\CertoTopicTweetSimilarityEngineModule.scala | |
object CertoTopicTweetSimilarityEngineModule extends TwitterModule | |
def providesCertoTopicTweetSimilarityEngine( | |
@Named(ModuleNames.CertoStratoStoreName) certoStratoStore: ReadableStore[ | |
TopicId, | |
Seq[TweetWithScores] | |
], | |
timeoutConfig: TimeoutConfig, | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver | |
): StandardSimilarityEngine[ | |
EngineQuery[Query], | |
TopicTweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumerBasedWalsSimilarityEngineModule.scala | |
object ConsumerBasedWalsSimilarityEngineModule extends TwitterModule | |
def providesConsumerBasedWalsSimilarityEngine( | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
@Named(ModuleNames.HomeNaviGRPCClient) homeNaviGRPCClient: ManagedChannel, | |
@Named(ModuleNames.AdsFavedNaviGRPCClient) adsFavedNaviGRPCClient: ManagedChannel, | |
@Named(ModuleNames.AdsMonetizableNaviGRPCClient) adsMonetizableNaviGRPCClient: ManagedChannel, | |
): StandardSimilarityEngine[ | |
ConsumerBasedWalsSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumerEmbeddingBasedTripSimilarityEngineModule.scala | |
object ConsumerEmbeddingBasedTripSimilarityEngineModule extends TwitterModule | |
def providesConsumerEmbeddingBasedTripSimilarityEngineModule( | |
@Named(ModuleNames.RmsUserLogFavInterestedInEmbeddingStore) | |
userLogFavInterestedInEmbeddingStore: ReadableStore[UserId, SimClustersEmbedding], | |
@Named(ModuleNames.RmsUserFollowInterestedInEmbeddingStore) | |
userFollowInterestedInEmbeddingStore: ReadableStore[UserId, SimClustersEmbedding], | |
@Named(ModuleNames.TripCandidateStore) | |
tripCandidateStore: ReadableStore[TripDomain, Seq[TripTweet]], | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): StandardSimilarityEngine[TripEngineQuery, TripTweetWithScore] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumerEmbeddingBasedTwHINSimilarityEngineModule.scala | |
object ConsumerEmbeddingBasedTwHINSimilarityEngineModule extends TwitterModule | |
def providesConsumerEmbeddingBasedTwHINANNSimilarityEngine( | |
// MH stores | |
@Named(EmbeddingStoreModule.ConsumerBasedTwHINEmbeddingRegularUpdateMhStoreName) | |
consumerBasedTwHINEmbeddingRegularUpdateMhStore: ReadableStore[InternalId, api.Embedding], | |
@Named(EmbeddingStoreModule.DebuggerDemoUserEmbeddingMhStoreName) | |
debuggerDemoUserEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], | |
@Named(AnnQueryServiceClientModule.TwHINRegularUpdateAnnServiceClientName) | |
twHINRegularUpdateAnnService: AnnQueryService.MethodPerEndpoint, | |
@Named(AnnQueryServiceClientModule.DebuggerDemoAnnServiceClientName) | |
debuggerDemoAnnService: AnnQueryService.MethodPerEndpoint, | |
// Other configs | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver | |
): HnswANNSimilarityEngine | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.scala | |
object ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule extends TwitterModule | |
def providesConsumerEmbeddingBasedTwoTowerANNSimilarityEngine( | |
@Named(EmbeddingStoreModule.TwoTowerFavConsumerEmbeddingMhStoreName) | |
twoTowerFavConsumerEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], | |
@Named(AnnQueryServiceClientModule.TwoTowerFavAnnServiceClientName) | |
twoTowerFavAnnService: AnnQueryService.MethodPerEndpoint, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver | |
): HnswANNSimilarityEngine | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumersBasedUserAdGraphSimilarityEngineModule.scala | |
object ConsumersBasedUserAdGraphSimilarityEngineModule extends TwitterModule | |
def providesConsumersBasedUserAdGraphSimilarityEngine( | |
@Named(ModuleNames.ConsumerBasedUserAdGraphStore) | |
consumersBasedUserAdGraphStore: ReadableStore[ | |
ConsumersBasedRelatedAdRequest, | |
RelatedAdResponse | |
], | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
ConsumersBasedUserAdGraphSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ConsumersBasedUserVideoGraphSimilarityEngineModule.scala | |
object ConsumersBasedUserVideoGraphSimilarityEngineModule extends TwitterModule | |
def providesConsumersBasedUserVideoGraphSimilarityEngine( | |
@Named(ModuleNames.ConsumerBasedUserVideoGraphStore) | |
consumersBasedUserVideoGraphStore: ReadableStore[ | |
ConsumersBasedRelatedTweetRequest, | |
RelatedTweetResponse | |
], | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
ConsumersBasedUserVideoGraphSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\DiffusionBasedSimilarityEngineModule.scala | |
object DiffusionBasedSimilarityEngineModule extends TwitterModule | |
def providesDiffusionBasedSimilarityEngineModule( | |
@Named(ModuleNames.RetweetBasedDiffusionRecsMhStore) | |
retweetBasedDiffusionRecsMhStore: ReadableStore[Long, TweetsWithScore], | |
timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver | |
): LookupSimilarityEngine[Query, TweetWithScore] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\EarlybirdSimilarityEngineModule.scala | |
object EarlybirdSimilarityEngineModule extends TwitterModule | |
def providesRecencyBasedEarlybirdSimilarityEngine( | |
earlybirdRecencyBasedSimilarityEngine: EarlybirdRecencyBasedSimilarityEngine, | |
timeoutConfig: TimeoutConfig, | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver | |
): EarlybirdSimilarityEngine[ | |
EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery, | |
EarlybirdRecencyBasedSimilarityEngine | |
] | |
def providesModelBasedEarlybirdSimilarityEngine( | |
earlybirdModelBasedSimilarityEngine: EarlybirdModelBasedSimilarityEngine, | |
timeoutConfig: TimeoutConfig, | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver | |
): EarlybirdSimilarityEngine[ | |
EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery, | |
EarlybirdModelBasedSimilarityEngine | |
] | |
def providesTensorflowBasedEarlybirdSimilarityEngine( | |
earlybirdTensorflowBasedSimilarityEngine: EarlybirdTensorflowBasedSimilarityEngine, | |
timeoutConfig: TimeoutConfig, | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver | |
): EarlybirdSimilarityEngine[ | |
EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery, | |
EarlybirdTensorflowBasedSimilarityEngine | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ProducerBasedUnifiedSimilarityEngineModule.scala | |
object ProducerBasedUnifiedSimilarityEngineModule extends TwitterModule | |
def providesProducerBasedUnifiedSimilarityEngine( | |
@Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine) | |
producerBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ | |
ProducerBasedUserTweetGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.SimClustersANNSimilarityEngine) | |
simClustersANNSimilarityEngine: StandardSimilarityEngine[ | |
SimClustersANNSimilarityEngine.Query, | |
TweetWithScore | |
], | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): StandardSimilarityEngine[ | |
ProducerBasedUnifiedSimilarityEngine.Query, | |
TweetWithCandidateGenerationInfo | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ProducerBasedUserAdGraphSimilarityEngineModule.scala | |
object ProducerBasedUserAdGraphSimilarityEngineModule extends TwitterModule | |
def providesProducerBasedUserAdGraphSimilarityEngine( | |
userAdGraphService: UserAdGraph.MethodPerEndpoint, | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
ProducerBasedUserAdGraphSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\ProducerBasedUserTweetGraphSimilarityEngineModule.scala | |
object ProducerBasedUserTweetGraphSimilarityEngineModule extends TwitterModule | |
def providesProducerBasedUserTweetGraphSimilarityEngine( | |
userTweetGraphService: UserTweetGraph.MethodPerEndpoint, | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
ProducerBasedUserTweetGraphSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\SimClustersANNSimilarityEngineModule.scala | |
object SimClustersANNSimilarityEngineModule extends TwitterModule | |
def providesProdSimClustersANNSimilarityEngine( | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint], | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver | |
): StandardSimilarityEngine[Query, TweetWithScore] | |
def buildWrapperStore( | |
memCachedStore: ReadableStore[Query, Seq[TweetWithScore]], | |
underlyingStore: ReadableStore[Query, Seq[TweetWithScore]], | |
wrapperStats: StatsReceiver | |
): ReadableStore[Query, Seq[TweetWithScore]] | |
def multiGet[K1 <: Query]( | |
queries: Set[K1] | |
): Map[K1, Future[Option[Seq[TweetWithScore]]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\SkitTopicTweetSimilarityEngineModule.scala | |
object SkitTopicTweetSimilarityEngineModule extends TwitterModule | |
def providesSkitHighPrecisionTopicTweetSimilarityEngine( | |
@Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ | |
TopicTweetPartitionFlatKey, | |
Seq[TopicTweet] | |
], | |
timeoutConfig: TimeoutConfig, | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver | |
): StandardSimilarityEngine[ | |
EngineQuery[Query], | |
TopicTweetWithScore | |
] | |
def providesSkitTfgTopicTweetSimilarityEngine( | |
@Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ | |
TopicTweetPartitionFlatKey, | |
Seq[TopicTweet] | |
], | |
timeoutConfig: TimeoutConfig, | |
decider: CrMixerDecider, | |
statsReceiver: StatsReceiver | |
): StandardSimilarityEngine[ | |
EngineQuery[Query], | |
TopicTweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedQigSimilarityEngineModule.scala | |
object TweetBasedQigSimilarityEngineModule extends TwitterModule | |
def providesTweetBasedQigSimilarTweetsCandidateSource( | |
qigRanker: QigRanker.MethodPerEndpoint, | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
TweetBasedQigSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedTwHINSimlarityEngineModule.scala | |
object TweetBasedTwHINSimlarityEngineModule extends TwitterModule | |
def providesTweetBasedTwHINANNSimilarityEngine( | |
// MH stores | |
@Named(EmbeddingStoreModule.TwHINEmbeddingRegularUpdateMhStoreName) | |
twHINEmbeddingRegularUpdateMhStore: ReadableStore[InternalId, api.Embedding], | |
@Named(EmbeddingStoreModule.DebuggerDemoTweetEmbeddingMhStoreName) | |
debuggerDemoTweetEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], | |
// ANN clients | |
@Named(AnnQueryServiceClientModule.TwHINRegularUpdateAnnServiceClientName) | |
twHINRegularUpdateAnnService: AnnQueryService.MethodPerEndpoint, | |
@Named(AnnQueryServiceClientModule.DebuggerDemoAnnServiceClientName) | |
debuggerDemoAnnService: AnnQueryService.MethodPerEndpoint, | |
// Other configs | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver | |
): HnswANNSimilarityEngine | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedUnifiedSimilarityEngineModule.scala | |
object TweetBasedUnifiedSimilarityEngineModule extends TwitterModule | |
def providesTweetBasedUnifiedSimilarityEngine( | |
@Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) tweetBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedUserTweetGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) tweetBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedUserVideoGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) | |
tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, | |
@Named(ModuleNames.TweetBasedQigSimilarityEngine) tweetBasedQigSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedQigSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.SimClustersANNSimilarityEngine) | |
simClustersANNSimilarityEngine: StandardSimilarityEngine[ | |
SimClustersANNSimilarityEngine.Query, | |
TweetWithScore | |
], | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): StandardSimilarityEngine[ | |
TweetBasedUnifiedSimilarityEngine.Query, | |
TweetWithCandidateGenerationInfo | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedUserAdGraphSimilarityEngineModule.scala | |
object TweetBasedUserAdGraphSimilarityEngineModule extends TwitterModule | |
def providesTweetBasedUserAdGraphSimilarityEngine( | |
userAdGraphService: UserAdGraph.MethodPerEndpoint, | |
tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
TweetBasedUserAdGraphSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedUserTweetGraphSimilarityEngineModule.scala | |
object TweetBasedUserTweetGraphSimilarityEngineModule extends TwitterModule | |
def providesTweetBasedUserTweetGraphSimilarityEngine( | |
userTweetGraphService: UserTweetGraph.MethodPerEndpoint, | |
tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
TweetBasedUserTweetGraphSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TweetBasedUserVideoGraphSimilarityEngineModule.scala | |
object TweetBasedUserVideoGraphSimilarityEngineModule extends TwitterModule | |
def providesTweetBasedUserVideoGraphSimilarityEngine( | |
userVideoGraphService: UserVideoGraph.MethodPerEndpoint, | |
tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], | |
@Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
TweetBasedUserVideoGraphSimilarityEngine.Query, | |
TweetWithScore | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\TwhinCollabFilterLookupSimilarityEngineModule.scala | |
object TwhinCollabFilterLookupSimilarityEngineModule extends TwitterModule | |
def providesTwhinCollabFilterLookupSimilarityEngineModule( | |
@Named(ModuleNames.TwhinCollabFilterStratoStoreForFollow) | |
twhinCollabFilterStratoStoreForFollow: ReadableStore[Long, Seq[TweetId]], | |
@Named(ModuleNames.TwhinCollabFilterStratoStoreForEngagement) | |
twhinCollabFilterStratoStoreForEngagement: ReadableStore[Long, Seq[TweetId]], | |
@Named(ModuleNames.TwhinMultiClusterStratoStoreForFollow) | |
twhinMultiClusterStratoStoreForFollow: ReadableStore[Long, Seq[TweetId]], | |
@Named(ModuleNames.TwhinMultiClusterStratoStoreForEngagement) | |
twhinMultiClusterStratoStoreForEngagement: ReadableStore[Long, Seq[TweetId]], | |
timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver | |
): LookupSimilarityEngine[Query, TweetWithScore] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\similarity_engine\UserTweetEntityGraphSimilarityEngineModule.scala | |
object UserTweetEntityGraphSimilarityEngineModule extends TwitterModule | |
def providesUserTweetEntityGraphSimilarityEngine( | |
userTweetEntityGraphService: UserTweetEntityGraph.MethodPerEndpoint, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider | |
): StandardSimilarityEngine[ | |
UserTweetEntityGraphSimilarityEngine.Query, | |
TweetWithScoreAndSocialProof | |
] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\AnnQueryServiceClientModule.scala | |
object AnnQueryServiceClientModule extends TwitterModule | |
def debuggerDemoAnnServiceClient( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
statsReceiver: StatsReceiver, | |
timeoutConfig: TimeoutConfig, | |
): AnnQueryService.MethodPerEndpoint | |
def twhinUuaAnnServiceClient( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
statsReceiver: StatsReceiver, | |
timeoutConfig: TimeoutConfig, | |
): AnnQueryService.MethodPerEndpoint | |
def twHINRegularUpdateAnnServiceClient( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
statsReceiver: StatsReceiver, | |
timeoutConfig: TimeoutConfig, | |
): AnnQueryService.MethodPerEndpoint | |
def twoTowerFavAnnServiceClient( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
statsReceiver: StatsReceiver, | |
timeoutConfig: TimeoutConfig, | |
): AnnQueryService.MethodPerEndpoint | |
def buildClient( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
dest: String, | |
label: String | |
): AnnQueryService.MethodPerEndpoint | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\EarlybirdSearchClientModule.scala | |
object EarlybirdSearchClientModule | |
extends ThriftMethodBuilderClientModule[ | |
EarlybirdService.ServicePerEndpoint, | |
EarlybirdService.MethodPerEndpoint | |
] | |
with MtlsClient | |
def label: String = "earlybird" | |
override def dest: String = "/s/earlybird-root-superroot/root-superroot" | |
private val requestTimeoutFlag: Flag[Duration] = | |
flag[Duration](EarlybirdClientTimeoutFlagName, "Earlybird client timeout") | |
override protected def requestTimeout: Duration = requestTimeoutFlag() | |
override def retryBudget: RetryBudget = RetryBudget.Empty | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\FrsClientModule.scala | |
object FrsClientModule | |
extends ThriftMethodBuilderClientModule[ | |
FollowRecommendationsThriftService.ServicePerEndpoint, | |
FollowRecommendationsThriftService.MethodPerEndpoint | |
] | |
with MtlsClient | |
def label: String = "follow-recommendations-service" | |
override def dest: String = "/s/follow-recommendations/follow-recos-service" | |
private val frsSignalFetchTimeout: Flag[Duration] = | |
flag[Duration](FrsClientTimeoutFlagName, "FRS signal fetch client timeout") | |
override def requestTimeout: Duration = frsSignalFetchTimeout() | |
override def retryBudget: RetryBudget = RetryBudget.Empty | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\HydraPartitionClientModule.scala | |
object HydraPartitionClientModule | |
extends ThriftMethodBuilderClientModule[ | |
ht.HydraPartition.ServicePerEndpoint, | |
ht.HydraPartition.MethodPerEndpoint | |
] | |
with MtlsClient | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\HydraRootClientModule.scala | |
object HydraRootClientModule | |
extends ThriftMethodBuilderClientModule[ | |
ht.HydraRoot.ServicePerEndpoint, | |
ht.HydraRoot.MethodPerEndpoint | |
] | |
with MtlsClient | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\QigServiceClientModule.scala | |
object QigServiceClientModule | |
extends ThriftMethodBuilderClientModule[ | |
QigRanker.ServicePerEndpoint, | |
QigRanker.MethodPerEndpoint | |
] | |
with MtlsClient | |
def requestTimeout: Duration = qigRankerClientTimeout() | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client = | |
super | |
.configureThriftMuxClient(injector, client) | |
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) | |
.withResponseClassifier | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\SimClustersAnnServiceClientModule.scala | |
object SimClustersAnnServiceClientModule extends TwitterModule | |
def providesProdSimClustersANNServiceClient( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): t.SimClustersANNService.MethodPerEndpoint | |
def providesExperimentalSimClustersANNServiceClient( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): t.SimClustersANNService.MethodPerEndpoint | |
def providesSimClustersANNServiceClient1( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): t.SimClustersANNService.MethodPerEndpoint | |
def providesSimClustersANNServiceClient2( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): t.SimClustersANNService.MethodPerEndpoint | |
def providesSimClustersANNServiceClient3( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): t.SimClustersANNService.MethodPerEndpoint | |
def providesSimClustersANNServiceClient5( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): t.SimClustersANNService.MethodPerEndpoint | |
def providesSimClustersANNServiceClient4( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
): t.SimClustersANNService.MethodPerEndpoint | |
def buildClient( | |
serviceIdentifier: ServiceIdentifier, | |
clientId: ClientId, | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver, | |
dest: String, | |
label: String | |
): t.SimClustersANNService.MethodPerEndpoint | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\TweetyPieClientModule.scala | |
object TweetyPieClientModule | |
extends ThriftMethodBuilderClientModule[ | |
TweetService.ServicePerEndpoint, | |
TweetService.MethodPerEndpoint | |
] | |
with MtlsClient | |
def requestTimeout: Duration = tweetypieClientTimeout() | |
override def retryBudget: RetryBudget = RetryBudget.Empty | |
// We bump the success rate from the default of 0.8 to 0.9 since we're dropping the | |
// consecutive failures part of the default policy. | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client = | |
super | |
.configureThriftMuxClient(injector, client) | |
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) | |
.withSessionQualifier | |
.successRateFailureAccrual(successRate = 0.9, window = 30.seconds) | |
.withResponseClassifier | |
def providesTweetyPie( | |
tweetyPieService: TweetService.MethodPerEndpoint | |
): STweetyPie | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserAdGraphClientModule.scala | |
object UserAdGraphClientModule | |
extends ThriftMethodBuilderClientModule[ | |
UserAdGraph.ServicePerEndpoint, | |
UserAdGraph.MethodPerEndpoint | |
] | |
with MtlsClient | |
def requestTimeout: Duration = userAdGraphClientTimeout() | |
override def retryBudget: RetryBudget = RetryBudget.Empty | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client = | |
super | |
.configureThriftMuxClient(injector, client) | |
.withMutualTls(injector.instance[ServiceIdentifier]) | |
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) | |
.withResponseClassifier | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserTweetEntityGraphClientModule.scala | |
object UserTweetEntityGraphClientModule | |
extends ThriftMethodBuilderClientModule[ | |
UserTweetEntityGraph.ServicePerEndpoint, | |
UserTweetEntityGraph.MethodPerEndpoint | |
] | |
with MtlsClient | |
def requestTimeout: Duration = userTweetEntityGraphClientTimeout() | |
override def retryBudget: RetryBudget = RetryBudget.Empty | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client = | |
super | |
.configureThriftMuxClient(injector, client) | |
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) | |
.withResponseClassifier | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserTweetGraphClientModule.scala | |
object UserTweetGraphClientModule | |
extends ThriftMethodBuilderClientModule[ | |
UserTweetGraph.ServicePerEndpoint, | |
UserTweetGraph.MethodPerEndpoint | |
] | |
with MtlsClient | |
def requestTimeout: Duration = userTweetGraphClientTimeout() | |
override def retryBudget: RetryBudget = RetryBudget.Empty | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client = | |
super | |
.configureThriftMuxClient(injector, client) | |
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) | |
.withResponseClassifier | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserTweetGraphPlusClientModule.scala | |
object UserTweetGraphPlusClientModule | |
extends ThriftMethodBuilderClientModule[ | |
UserTweetGraphPlus.ServicePerEndpoint, | |
UserTweetGraphPlus.MethodPerEndpoint | |
] | |
with MtlsClient | |
def requestTimeout: Duration = userTweetGraphPlusClientTimeout() | |
override def retryBudget: RetryBudget = RetryBudget.Empty | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client = | |
super | |
.configureThriftMuxClient(injector, client) | |
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) | |
.withResponseClassifier | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\module\thrift_client\UserVideoGraphClientModule.scala | |
object UserVideoGraphClientModule | |
extends ThriftMethodBuilderClientModule[ | |
UserVideoGraph.ServicePerEndpoint, | |
UserVideoGraph.MethodPerEndpoint | |
] | |
with MtlsClient | |
def requestTimeout: Duration = userVideoGraphClientTimeout() | |
override def retryBudget: RetryBudget = RetryBudget.Empty | |
override def configureThriftMuxClient( | |
injector: Injector, | |
client: ThriftMux.Client | |
): ThriftMux.Client = | |
super | |
.configureThriftMuxClient(injector, client) | |
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) | |
.withResponseClassifier | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\AdsParams.scala | |
object AdsParams | |
object AdsCandidateGenerationMaxCandidatesNumParam | |
extends FSBoundedParam[Int]( | |
name = "ads_candidate_generation_max_candidates_num", | |
default = 400, | |
min = 0, | |
max = 2000 | |
) | |
object EnableScoreBoost | |
extends FSParam[Boolean]( | |
name = "ads_candidate_generation_enable_score_boost", | |
default = false | |
) | |
object AdsCandidateGenerationScoreBoostFactor | |
extends FSBoundedParam[Double]( | |
name = "ads_candidate_generation_score_boost_factor", | |
default = 10000.0, | |
min = 1.0, | |
max = 100000.0 | |
) | |
object EnableScribe | |
extends FSParam[Boolean]( | |
name = "ads_candidate_generation_enable_scribe", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
AdsCandidateGenerationMaxCandidatesNumParam, | |
EnableScoreBoost, | |
AdsCandidateGenerationScoreBoostFactor | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\BlenderParams.scala | |
object BlenderParams | |
object BlendingAlgorithmEnum extends Enumeration | |
object ContentBasedSortingAlgorithmEnum extends Enumeration | |
object BlendingAlgorithmParam | |
extends FSEnumParam[BlendingAlgorithmEnum.type]( | |
name = "blending_algorithm_id", | |
default = BlendingAlgorithmEnum.RoundRobin, | |
enum = BlendingAlgorithmEnum | |
) | |
object RankingInterleaveWeightShrinkageParam | |
extends FSBoundedParam[Double]( | |
name = "blending_enable_ml_ranking_interleave_weights_shrinkage", | |
default = 1.0, | |
min = 0.0, | |
max = 1.0 | |
) | |
object RankingInterleaveMaxWeightAdjustments | |
extends FSBoundedParam[Int]( | |
name = "blending_interleave_max_weighted_adjustments", | |
default = 3000, | |
min = 0, | |
max = 9999 | |
) | |
object SignalTypeSortingAlgorithmParam | |
extends FSEnumParam[ContentBasedSortingAlgorithmEnum.type]( | |
name = "blending_algorithm_inner_signal_sorting_id", | |
default = ContentBasedSortingAlgorithmEnum.SourceSignalRecency, | |
enum = ContentBasedSortingAlgorithmEnum | |
) | |
object ContentBlenderTypeSortingAlgorithmParam | |
extends FSEnumParam[ContentBasedSortingAlgorithmEnum.type]( | |
name = "blending_algorithm_content_blender_sorting_id", | |
default = ContentBasedSortingAlgorithmEnum.FavoriteCount, | |
enum = ContentBasedSortingAlgorithmEnum | |
) | |
//UserAffinities Algo Param: whether to distributed the source type weights | |
object EnableDistributedSourceTypeWeightsParam | |
extends FSParam[Boolean]( | |
name = "blending_algorithm_enable_distributed_source_type_weights", | |
default = false | |
) | |
object BlendGroupingMethodEnum extends Enumeration | |
object BlendGroupingMethodParam | |
extends FSEnumParam[BlendGroupingMethodEnum.type]( | |
name = "blending_grouping_method_id", | |
default = BlendGroupingMethodEnum.SourceKeyDefault, | |
enum = BlendGroupingMethodEnum | |
) | |
object RecencyBasedRandomSamplingHalfLifeInDays | |
extends FSBoundedParam[Int]( | |
name = "blending_interleave_random_sampling_recency_based_half_life_in_days", | |
default = 7, | |
min = 1, | |
max = 28 | |
) | |
object RecencyBasedRandomSamplingDefaultWeight | |
extends FSBoundedParam[Double]( | |
name = "blending_interleave_random_sampling_recency_based_default_weight", | |
default = 1.0, | |
min = 0.1, | |
max = 2.0 | |
) | |
object SourceTypeBackFillEnableVideoBackFill | |
extends FSParam[Boolean]( | |
name = "blending_enable_video_backfill", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
BlendingAlgorithmParam, | |
RankingInterleaveWeightShrinkageParam, | |
RankingInterleaveMaxWeightAdjustments, | |
EnableDistributedSourceTypeWeightsParam, | |
BlendGroupingMethodParam, | |
RecencyBasedRandomSamplingHalfLifeInDays, | |
RecencyBasedRandomSamplingDefaultWeight, | |
SourceTypeBackFillEnableVideoBackFill, | |
SignalTypeSortingAlgorithmParam, | |
ContentBlenderTypeSortingAlgorithmParam, | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\BypassInterleaveAndRankParams.scala | |
object BypassInterleaveAndRankParams | |
object EnableTwhinCollabFilterBypassParam | |
extends FSParam[Boolean]( | |
name = "bypass_interleave_and_rank_twhin_collab_filter", | |
default = false | |
) | |
object EnableTwoTowerBypassParam | |
extends FSParam[Boolean]( | |
name = "bypass_interleave_and_rank_two_tower", | |
default = false | |
) | |
object EnableConsumerBasedTwhinBypassParam | |
extends FSParam[Boolean]( | |
name = "bypass_interleave_and_rank_consumer_based_twhin", | |
default = false | |
) | |
object EnableConsumerBasedWalsBypassParam | |
extends FSParam[Boolean]( | |
name = "bypass_interleave_and_rank_consumer_based_wals", | |
default = false | |
) | |
object TwhinCollabFilterBypassPercentageParam | |
extends FSBoundedParam[Double]( | |
name = "bypass_interleave_and_rank_twhin_collab_filter_percentage", | |
default = 0.0, | |
min = 0.0, | |
max = 1.0 | |
) | |
object TwoTowerBypassPercentageParam | |
extends FSBoundedParam[Double]( | |
name = "bypass_interleave_and_rank_two_tower_percentage", | |
default = 0.0, | |
min = 0.0, | |
max = 1.0 | |
) | |
object ConsumerBasedTwhinBypassPercentageParam | |
extends FSBoundedParam[Double]( | |
name = "bypass_interleave_and_rank_consumer_based_twhin_percentage", | |
default = 0.0, | |
min = 0.0, | |
max = 1.0 | |
) | |
object ConsumerBasedWalsBypassPercentageParam | |
extends FSBoundedParam[Double]( | |
name = "bypass_interleave_and_rank_consumer_based_wals_percentage", | |
default = 0.0, | |
min = 0.0, | |
max = 1.0 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableTwhinCollabFilterBypassParam, | |
EnableTwoTowerBypassParam, | |
EnableConsumerBasedTwhinBypassParam, | |
EnableConsumerBasedWalsBypassParam, | |
TwhinCollabFilterBypassPercentageParam, | |
TwoTowerBypassPercentageParam, | |
ConsumerBasedTwhinBypassPercentageParam, | |
ConsumerBasedWalsBypassPercentageParam, | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerBasedWalsParams.scala | |
object ConsumerBasedWalsParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "consumer_based_wals_enable_source", | |
default = false | |
) | |
object ModelNameParam | |
extends FSParam[String]( | |
name = "consumer_based_wals_model_name", | |
default = "model_0" | |
) | |
object WilyNsNameParam | |
extends FSParam[String]( | |
name = "consumer_based_wals_wily_ns_name", | |
default = "" | |
) | |
object ModelInputNameParam | |
extends FSParam[String]( | |
name = "consumer_based_wals_model_input_name", | |
default = "examples" | |
) | |
object ModelOutputNameParam | |
extends FSParam[String]( | |
name = "consumer_based_wals_model_output_name", | |
default = "all_tweet_ids" | |
) | |
object ModelSignatureNameParam | |
extends FSParam[String]( | |
name = "consumer_based_wals_model_signature_name", | |
default = "serving_default" | |
) | |
object MaxTweetSignalAgeHoursParam | |
extends FSBoundedParam[Duration]( | |
name = "consumer_based_wals_max_tweet_signal_age_hours", | |
default = 72.hours, | |
min = 1.hours, | |
max = 720.hours | |
) | |
with HasDurationConversion | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerEmbeddingBasedCandidateGenerationParams.scala | |
object ConsumerEmbeddingBasedCandidateGenerationParams | |
object EnableTwHINParam | |
extends FSParam[Boolean]( | |
name = "consumer_embedding_based_candidate_generation_enable_twhin", | |
default = false | |
) | |
object EnableTwoTowerParam | |
extends FSParam[Boolean]( | |
name = "consumer_embedding_based_candidate_generation_enable_two_tower", | |
default = false | |
) | |
object EnableLogFavBasedSimClustersTripParam | |
extends FSParam[Boolean]( | |
name = "consumer_embedding_based_candidate_generation_enable_logfav_based_simclusters_trip", | |
default = false | |
) | |
object EnableFollowBasedSimClustersTripParam | |
extends FSParam[Boolean]( | |
name = "consumer_embedding_based_candidate_generation_enable_follow_based_simclusters_trip", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableTwHINParam, | |
EnableTwoTowerParam, | |
EnableFollowBasedSimClustersTripParam, | |
EnableLogFavBasedSimClustersTripParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerEmbeddingBasedTripParams.scala | |
object ConsumerEmbeddingBasedTripParams | |
object SourceIdParam | |
extends FSParam[String]( | |
name = "consumer_embedding_based_trip_source_id", | |
default = "EXPLR_TOPK_VID_48H_V3") | |
object MaxNumCandidatesParam | |
extends FSBoundedParam[Int]( | |
name = "consumer_embedding_based_trip_max_num_candidates", | |
default = 80, | |
min = 0, | |
max = 200 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
SourceIdParam, | |
MaxNumCandidatesParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerEmbeddingBasedTwHINParams.scala | |
object ConsumerEmbeddingBasedTwHINParams | |
object ModelIdParam | |
extends FSParam[String]( | |
name = "consumer_embedding_based_twhin_model_id", | |
default = ModelConfig.ConsumerBasedTwHINRegularUpdateAll20221024, | |
) // Note: this default value does not match with ModelIds yet. This FS is a placeholder | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
ModelIdParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumerEmbeddingBasedTwoTowerParams.scala | |
object ConsumerEmbeddingBasedTwoTowerParams | |
object ModelIdParam | |
extends FSParam[String]( | |
name = "consumer_embedding_based_two_tower_model_id", | |
default = TwoTowerFavALL20220808, | |
) // Note: this default value does not match with ModelIds yet. This FS is a placeholder | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
ModelIdParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumersBasedUserAdGraphParams.scala | |
object ConsumersBasedUserAdGraphParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "consumers_based_user_ad_graph_enable_source", | |
default = false | |
) | |
// UTG-Lookalike | |
object MinCoOccurrenceParam | |
extends FSBoundedParam[Int]( | |
name = "consumers_based_user_ad_graph_min_co_occurrence", | |
default = 2, | |
min = 0, | |
max = 500 | |
) | |
object MinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "consumers_based_user_ad_graph_min_score", | |
default = 0.0, | |
min = 0.0, | |
max = 10.0 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam, | |
MinCoOccurrenceParam, | |
MinScoreParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumersBasedUserTweetGraphParams.scala | |
object ConsumersBasedUserTweetGraphParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "consumers_based_user_tweet_graph_enable_source", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam, | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ConsumersBasedUserVideoGraphParams.scala | |
object ConsumersBasedUserVideoGraphParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "consumers_based_user_video_graph_enable_source", | |
default = false | |
) | |
// UTG-RealGraphIN | |
object RealGraphInMinCoOccurrenceParam | |
extends FSBoundedParam[Int]( | |
name = "consumers_based_user_video_graph_real_graph_in_min_co_occurrence", | |
default = 3, | |
min = 0, | |
max = 500 | |
) | |
object RealGraphInMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "consumers_based_user_video_graph_real_graph_in_min_score", | |
default = 2.0, | |
min = 0.0, | |
max = 10.0 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam, | |
RealGraphInMinCoOccurrenceParam, | |
RealGraphInMinScoreParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CrMixerParamConfig.scala | |
object CrMixerParamConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CustomizedRetrievalBasedCandidateGenerationParams.scala | |
object CustomizedRetrievalBasedCandidateGenerationParams | |
object EnableOfflineInterestedInParam | |
extends FSParam[Boolean]( | |
name = "customized_retrieval_based_candidate_generation_enable_offline_interestedin", | |
default = false | |
) | |
// Offline SimClusters FTR-based InterestedIn | |
object EnableOfflineFTRInterestedInParam | |
extends FSParam[Boolean]( | |
name = "customized_retrieval_based_candidate_generation_enable_ftr_offline_interestedin", | |
default = false | |
) | |
// TwHin Collab Filter Cluster params | |
object EnableTwhinCollabFilterClusterParam | |
extends FSParam[Boolean]( | |
name = "customized_retrieval_based_candidate_generation_enable_twhin_collab_filter_cluster", | |
default = false | |
) | |
// TwHin Multi Cluster params | |
object EnableTwhinMultiClusterParam | |
extends FSParam[Boolean]( | |
name = "customized_retrieval_based_candidate_generation_enable_twhin_multi_cluster", | |
default = false | |
) | |
object EnableRetweetBasedDiffusionParam | |
extends FSParam[Boolean]( | |
name = "customized_retrieval_based_candidate_generation_enable_retweet_based_diffusion", | |
default = false | |
) | |
object CustomizedRetrievalBasedRetweetDiffusionSource | |
extends FSParam[String]( | |
name = | |
"customized_retrieval_based_candidate_generation_offline_retweet_based_diffusion_model_id", | |
default = ModelConfig.RetweetBasedDiffusion | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableOfflineInterestedInParam, | |
EnableOfflineFTRInterestedInParam, | |
EnableTwhinCollabFilterClusterParam, | |
EnableTwhinMultiClusterParam, | |
EnableRetweetBasedDiffusionParam, | |
CustomizedRetrievalBasedRetweetDiffusionSource | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CustomizedRetrievalBasedFTROfflineInterestedInParams.scala | |
object CustomizedRetrievalBasedFTROfflineInterestedInParams | |
object CustomizedRetrievalBasedFTROfflineInterestedInSource | |
extends FSParam[String]( | |
name = "customized_retrieval_based_ftr_offline_interestedin_model_id", | |
default = ModelConfig.OfflineFavDecayedSum | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
CustomizedRetrievalBasedFTROfflineInterestedInSource) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CustomizedRetrievalBasedOfflineInterestedInParams.scala | |
object CustomizedRetrievalBasedOfflineInterestedInParams | |
object CustomizedRetrievalBasedOfflineInterestedInSource | |
extends FSParam[String]( | |
name = "customized_retrieval_based_offline_interestedin_model_id", | |
default = ModelConfig.OfflineInterestedInFromKnownFor2020 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(CustomizedRetrievalBasedOfflineInterestedInSource) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\CustomizedRetrievalBasedTwhinParams.scala | |
object CustomizedRetrievalBasedTwhinParams | |
object CustomizedRetrievalBasedTwhinCollabFilterFollowSource | |
extends FSParam[String]( | |
name = "customized_retrieval_based_offline_twhin_collab_filter_follow_model_id", | |
default = ModelConfig.TwhinCollabFilterForFollow | |
) | |
object CustomizedRetrievalBasedTwhinCollabFilterEngagementSource | |
extends FSParam[String]( | |
name = "customized_retrieval_based_offline_twhin_collab_filter_engagement_model_id", | |
default = ModelConfig.TwhinCollabFilterForEngagement | |
) | |
object CustomizedRetrievalBasedTwhinMultiClusterFollowSource | |
extends FSParam[String]( | |
name = "customized_retrieval_based_offline_twhin_multi_cluster_follow_model_id", | |
default = ModelConfig.TwhinMultiClusterForFollow | |
) | |
object CustomizedRetrievalBasedTwhinMultiClusterEngagementSource | |
extends FSParam[String]( | |
name = "customized_retrieval_based_offline_twhin_multi_cluster_engagement_model_id", | |
default = ModelConfig.TwhinMultiClusterForEngagement | |
) | |
val AllParams: Seq[Param[_] with FSName] = | |
Seq( | |
CustomizedRetrievalBasedTwhinCollabFilterFollowSource, | |
CustomizedRetrievalBasedTwhinCollabFilterEngagementSource, | |
CustomizedRetrievalBasedTwhinMultiClusterFollowSource, | |
CustomizedRetrievalBasedTwhinMultiClusterEngagementSource, | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\EarlybirdFrsBasedCandidateGenerationParams.scala | |
object EarlybirdFrsBasedCandidateGenerationParams | |
object CandidateGenerationEarlybirdSimilarityEngineType extends Enumeration | |
class SimilarityEngineType(rankingMode: EarlybirdSimilarityEngineType) | |
extends super.Val | |
import scala.language.implicitConversions | |
implicit def valueToEarlybirdRankingMode(x: Value): SimilarityEngineType = | |
x.asInstanceOf[SimilarityEngineType] | |
val EarlybirdRankingMode_RecencyBased: SimilarityEngineType = SimilarityEngineType( | |
EarlybirdSimilarityEngineType_RecencyBased) | |
val EarlybirdRankingMode_ModelBased: SimilarityEngineType = SimilarityEngineType( | |
EarlybirdSimilarityEngineType_ModelBased) | |
val EarlybirdRankingMode_TensorflowBased: SimilarityEngineType = SimilarityEngineType( | |
EarlybirdSimilarityEngineType_TensorflowBased) | |
} | |
object FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam | |
extends FSEnumParam[CandidateGenerationEarlybirdSimilarityEngineType.type]( | |
name = "frs_based_candidate_generation_earlybird_ranking_mode_id", | |
default = | |
CandidateGenerationEarlybirdSimilarityEngineType.EarlybirdRankingMode_RecencyBased, | |
enum = CandidateGenerationEarlybirdSimilarityEngineType | |
) | |
object FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser | |
extends FSBoundedParam[Int]( | |
name = "frs_based_candidate_generation_earlybird_max_tweets_per_user", | |
default = 100, | |
min = 0, | |
/** | |
* Note max should be equal to EarlybirdRecencyBasedCandidateStoreModule.DefaultMaxNumTweetPerUser. | |
* Which is the size of the memcached result list. | |
*/ | |
max = 100 | |
) | |
object FrsBasedCandidateGenerationEarlybirdMaxTweetAge | |
extends FSBoundedParam[Duration]( | |
name = "frs_based_candidate_generation_earlybird_max_tweet_age_hours", | |
default = 24.hours, | |
min = 12.hours, | |
/** | |
* Note max could be related to EarlybirdRecencyBasedCandidateStoreModule.DefaultMaxNumTweetPerUser. | |
* Which is the size of the memcached result list for recency based earlybird candidate source. | |
* E.g. if max = 720.hours, we may want to increase the DefaultMaxNumTweetPerUser. | |
*/ | |
max = 96.hours | |
) | |
with HasDurationConversion | |
object FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies | |
extends FSParam[Boolean]( | |
name = "frs_based_candidate_generation_earlybird_filter_out_retweets_and_replies", | |
default = true | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam, | |
FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser, | |
FrsBasedCandidateGenerationEarlybirdMaxTweetAge, | |
FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies, | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\FrsParams.scala | |
object FrsParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "signal_frs_enable_source", | |
default = false | |
) | |
object EnableSourceGraphParam | |
extends FSParam[Boolean]( | |
name = "graph_frs_enable_source", | |
default = false | |
) | |
object MinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "signal_frs_min_score", | |
default = 0.4, | |
min = 0.0, | |
max = 1.0 | |
) | |
object MaxConsumerSeedsNumParam | |
extends FSBoundedParam[Int]( | |
name = "graph_frs_max_user_seeds_num", | |
default = 200, | |
min = 0, | |
max = 1000 | |
) | |
/** | |
* These params below are only used for FrsTweetCandidateGenerator and shouldn't be used in other endpoints | |
* * FrsBasedCandidateGenerationMaxSeedsNumParam | |
* * FrsCandidateGenerationDisplayLocationParam | |
* * FrsCandidateGenerationDisplayLocation | |
* * FrsBasedCandidateGenerationMaxCandidatesNumParam | |
*/ | |
object FrsBasedCandidateGenerationEnableVisibilityFilteringParam | |
extends FSParam[Boolean]( | |
name = "frs_based_candidate_generation_enable_vf", | |
default = true | |
) | |
object FrsBasedCandidateGenerationMaxSeedsNumParam | |
extends FSBoundedParam[Int]( | |
name = "frs_based_candidate_generation_max_seeds_num", | |
default = 100, | |
min = 0, | |
max = 800 | |
) | |
object FrsBasedCandidateGenerationDisplayLocation extends Enumeration | |
class FrsDisplayLocationValue(displayLocation: DisplayLocation) extends super.Val | |
import scala.language.implicitConversions | |
implicit def valueToDisplayLocationValue(x: Value): FrsDisplayLocationValue = | |
x.asInstanceOf[FrsDisplayLocationValue] | |
val DisplayLocation_ContentRecommender: FrsDisplayLocationValue = FrsDisplayLocationValue( | |
DisplayLocation.ContentRecommender) | |
val DisplayLocation_Home: FrsDisplayLocationValue = FrsDisplayLocationValue( | |
DisplayLocation.HomeTimelineTweetRecs) | |
val DisplayLocation_Notifications: FrsDisplayLocationValue = FrsDisplayLocationValue( | |
DisplayLocation.TweetNotificationRecs) | |
} | |
object FrsBasedCandidateGenerationDisplayLocationParam | |
extends FSEnumParam[FrsBasedCandidateGenerationDisplayLocation.type]( | |
name = "frs_based_candidate_generation_display_location_id", | |
default = FrsBasedCandidateGenerationDisplayLocation.DisplayLocation_Home, | |
enum = FrsBasedCandidateGenerationDisplayLocation | |
) | |
object FrsBasedCandidateGenerationMaxCandidatesNumParam | |
extends FSBoundedParam[Int]( | |
name = "frs_based_candidate_generation_max_candidates_num", | |
default = 100, | |
min = 0, | |
max = 2000 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam, | |
EnableSourceGraphParam, | |
MinScoreParam, | |
MaxConsumerSeedsNumParam, | |
FrsBasedCandidateGenerationMaxSeedsNumParam, | |
FrsBasedCandidateGenerationDisplayLocationParam, | |
FrsBasedCandidateGenerationMaxCandidatesNumParam, | |
FrsBasedCandidateGenerationEnableVisibilityFilteringParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\GlobalParams.scala | |
object GlobalParams | |
object MaxCandidatesPerRequestParam | |
extends FSBoundedParam[Int]( | |
name = "twistly_core_max_candidates_per_request", | |
default = 100, | |
min = 0, | |
max = 9000 | |
) | |
object ModelVersionParam | |
extends FSEnumParam[ModelVersions.Enum.type]( | |
name = "twistly_core_simclusters_model_version_id", | |
default = ModelVersions.Enum.Model20M145K2020, | |
enum = ModelVersions.Enum | |
) | |
object UnifiedMaxSourceKeyNum | |
extends FSBoundedParam[Int]( | |
name = "twistly_core_unified_max_sourcekey_num", | |
default = 15, | |
min = 0, | |
max = 100 | |
) | |
object MaxCandidateNumPerSourceKeyParam | |
extends FSBoundedParam[Int]( | |
name = "twistly_core_candidate_per_sourcekey_max_num", | |
default = 200, | |
min = 0, | |
max = 1000 | |
) | |
// 1 hours to 30 days | |
object MaxTweetAgeHoursParam | |
extends FSBoundedParam[Duration]( | |
name = "twistly_core_max_tweet_age_hours", | |
default = 720.hours, | |
min = 1.hours, | |
max = 720.hours | |
) | |
with HasDurationConversion | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\GoodProfileClickParams.scala | |
object GoodProfileClickParams | |
object ClickMinDwellTimeParam extends Enumeration | |
class SignalTypeValue(signalType: SignalType) extends super.Val | |
import scala.language.implicitConversions | |
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = | |
x.asInstanceOf[SignalTypeValue] | |
val TotalDwellTime10s = SignalTypeValue(SignalType.GoodProfileClick) | |
val TotalDwellTime20s = SignalTypeValue(SignalType.GoodProfileClick20s) | |
val TotalDwellTime30s = SignalTypeValue(SignalType.GoodProfileClick30s) | |
} | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "signal_good_profile_clicks_enable_source", | |
default = false | |
) | |
object ClickMinDwellTimeType | |
extends FSEnumParam[ClickMinDwellTimeParam.type]( | |
name = "signal_good_profile_clicks_min_dwelltime_type_id", | |
default = ClickMinDwellTimeParam.TotalDwellTime10s, | |
enum = ClickMinDwellTimeParam | |
) | |
val AllParams: Seq[Param[_] with FSName] = | |
Seq(EnableSourceParam, ClickMinDwellTimeType) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\GoodTweetClickParams.scala | |
object GoodTweetClickParams | |
object ClickMinDwellTimeParam extends Enumeration | |
class SignalTypeValue(signalType: SignalType) extends super.Val | |
import scala.language.implicitConversions | |
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = | |
x.asInstanceOf[SignalTypeValue] | |
val TotalDwellTime2s = SignalTypeValue(SignalType.GoodTweetClick) | |
val TotalDwellTime5s = SignalTypeValue(SignalType.GoodTweetClick5s) | |
val TotalDwellTime10s = SignalTypeValue(SignalType.GoodTweetClick10s) | |
val TotalDwellTime30s = SignalTypeValue(SignalType.GoodTweetClick30s) | |
} | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "signal_good_tweet_clicks_enable_source", | |
default = false | |
) | |
object ClickMinDwellTimeType | |
extends FSEnumParam[ClickMinDwellTimeParam.type]( | |
name = "signal_good_tweet_clicks_min_dwelltime_type_id", | |
default = ClickMinDwellTimeParam.TotalDwellTime2s, | |
enum = ClickMinDwellTimeParam | |
) | |
object MaxSignalNumParam | |
extends FSBoundedParam[Int]( | |
name = "signal_good_tweet_clicks_max_signal_num", | |
default = 15, | |
min = 0, | |
max = 15 | |
) | |
val AllParams: Seq[Param[_] with FSName] = | |
Seq(EnableSourceParam, ClickMinDwellTimeType, MaxSignalNumParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\InterestedInParams.scala | |
object InterestedInParams | |
object SourceEmbedding extends Enumeration | |
class EmbeddingType(embeddingType: SimClustersEmbeddingType) extends super.Val | |
import scala.language.implicitConversions | |
implicit def valueToEmbeddingtype(x: Value): EmbeddingType = x.asInstanceOf[EmbeddingType] | |
val UserInterestedIn: Value = EmbeddingType(SimClustersEmbeddingType.FilteredUserInterestedIn) | |
val UnfilteredUserInterestedIn: Value = EmbeddingType( | |
SimClustersEmbeddingType.UnfilteredUserInterestedIn) | |
val FromProducerEmbedding: Value = EmbeddingType( | |
SimClustersEmbeddingType.FilteredUserInterestedInFromPE) | |
val LogFavBasedUserInterestedInFromAPE: Value = EmbeddingType( | |
SimClustersEmbeddingType.LogFavBasedUserInterestedInFromAPE) | |
val FollowBasedUserInterestedInFromAPE: Value = EmbeddingType( | |
SimClustersEmbeddingType.FollowBasedUserInterestedInFromAPE) | |
val UserNextInterestedIn: Value = EmbeddingType(SimClustersEmbeddingType.UserNextInterestedIn) | |
// AddressBook based InterestedIn | |
val LogFavBasedUserInterestedAverageAddressBookFromIIAPE: Value = EmbeddingType( | |
SimClustersEmbeddingType.LogFavBasedUserInterestedAverageAddressBookFromIIAPE) | |
val LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( | |
SimClustersEmbeddingType.LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE) | |
val LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( | |
SimClustersEmbeddingType.LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE) | |
val LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( | |
SimClustersEmbeddingType.LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE) | |
val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( | |
SimClustersEmbeddingType.LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE) | |
val LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( | |
SimClustersEmbeddingType.LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE) | |
} | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_enable_source", | |
default = true | |
) | |
object InterestedInEmbeddingIdParam | |
extends FSEnumParam[SourceEmbedding.type]( | |
name = "twistly_interestedin_embedding_id", | |
default = SourceEmbedding.UnfilteredUserInterestedIn, | |
enum = SourceEmbedding | |
) | |
object MinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "twistly_interestedin_min_score", | |
default = 0.072, | |
min = 0.0, | |
max = 1.0 | |
) | |
object EnableSourceSequentialModelParam | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_sequential_model_enable_source", | |
default = false | |
) | |
object NextInterestedInEmbeddingIdParam | |
extends FSEnumParam[SourceEmbedding.type]( | |
name = "twistly_interestedin_sequential_model_embedding_id", | |
default = SourceEmbedding.UserNextInterestedIn, | |
enum = SourceEmbedding | |
) | |
object MinScoreSequentialModelParam | |
extends FSBoundedParam[Double]( | |
name = "twistly_interestedin_sequential_model_min_score", | |
default = 0.0, | |
min = 0.0, | |
max = 1.0 | |
) | |
object EnableSourceAddressBookParam | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_addressbook_enable_source", | |
default = false | |
) | |
object AddressBookInterestedInEmbeddingIdParam | |
extends FSEnumParam[SourceEmbedding.type]( | |
name = "twistly_interestedin_addressbook_embedding_id", | |
default = SourceEmbedding.LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE, | |
enum = SourceEmbedding | |
) | |
object MinScoreAddressBookParam | |
extends FSBoundedParam[Double]( | |
name = "twistly_interestedin_addressbook_min_score", | |
default = 0.0, | |
min = 0.0, | |
max = 1.0 | |
) | |
// Prod SimClusters ANN param | |
// This is used to enable/disable querying of production SANN service. Useful when experimenting | |
// with replacements to it. | |
object EnableProdSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_enable_prod_simclusters_ann", | |
default = true | |
) | |
// Experimental SimClusters ANN params | |
object EnableExperimentalSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_enable_experimental_simclusters_ann", | |
default = false | |
) | |
// SimClusters ANN 1 cluster params | |
object EnableSimClustersANN1Param | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_enable_simclusters_ann_1", | |
default = false | |
) | |
// SimClusters ANN 2 cluster params | |
object EnableSimClustersANN2Param | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_enable_simclusters_ann_2", | |
default = false | |
) | |
// SimClusters ANN 3 cluster params | |
object EnableSimClustersANN3Param | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_enable_simclusters_ann_3", | |
default = false | |
) | |
// SimClusters ANN 5 cluster params | |
object EnableSimClustersANN5Param | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_enable_simclusters_ann_5", | |
default = false | |
) | |
// SimClusters ANN 4 cluster params | |
object EnableSimClustersANN4Param | |
extends FSParam[Boolean]( | |
name = "twistly_interestedin_enable_simclusters_ann_4", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam, | |
EnableSourceSequentialModelParam, | |
EnableSourceAddressBookParam, | |
EnableProdSimClustersANNParam, | |
EnableExperimentalSimClustersANNParam, | |
EnableSimClustersANN1Param, | |
EnableSimClustersANN2Param, | |
EnableSimClustersANN3Param, | |
EnableSimClustersANN5Param, | |
EnableSimClustersANN4Param, | |
MinScoreParam, | |
MinScoreSequentialModelParam, | |
MinScoreAddressBookParam, | |
InterestedInEmbeddingIdParam, | |
NextInterestedInEmbeddingIdParam, | |
AddressBookInterestedInEmbeddingIdParam, | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ProducerBasedCandidateGenerationParams.scala | |
object ProducerBasedCandidateGenerationParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_source", | |
default = false | |
) | |
object UtgCombinationMethodParam | |
extends FSEnumParam[UnifiedSETweetCombinationMethod.type]( | |
name = "producer_based_candidate_generation_utg_combination_method_id", | |
default = UnifiedSETweetCombinationMethod.Frontload, | |
enum = UnifiedSETweetCombinationMethod | |
) | |
// UTG params | |
object EnableUTGParam | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_utg", | |
default = false | |
) | |
object EnableUAGParam | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_uag", | |
default = false | |
) | |
// SimClusters params | |
object EnableSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_simclusters", | |
default = true | |
) | |
// Filter params | |
object SimClustersMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "producer_based_candidate_generation_filter_simclusters_min_score", | |
default = 0.7, | |
min = 0.0, | |
max = 1.0 | |
) | |
// Experimental SimClusters ANN params | |
object EnableExperimentalSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_experimental_simclusters_ann", | |
default = false | |
) | |
// SimClusters ANN cluster 1 params | |
object EnableSimClustersANN1Param | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_simclusters_ann_1", | |
default = false | |
) | |
// SimClusters ANN cluster 2 params | |
object EnableSimClustersANN2Param | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_simclusters_ann_2", | |
default = false | |
) | |
// SimClusters ANN cluster 3 params | |
object EnableSimClustersANN3Param | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_simclusters_ann_3", | |
default = false | |
) | |
// SimClusters ANN cluster 5 params | |
object EnableSimClustersANN5Param | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_simclusters_ann_5", | |
default = false | |
) | |
object EnableSimClustersANN4Param | |
extends FSParam[Boolean]( | |
name = "producer_based_candidate_generation_enable_simclusters_ann_4", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam, | |
EnableUAGParam, | |
EnableUTGParam, | |
EnableSimClustersANNParam, | |
EnableSimClustersANN1Param, | |
EnableSimClustersANN2Param, | |
EnableSimClustersANN3Param, | |
EnableSimClustersANN5Param, | |
EnableSimClustersANN4Param, | |
EnableExperimentalSimClustersANNParam, | |
SimClustersMinScoreParam, | |
UtgCombinationMethodParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ProducerBasedUserAdGraphParams.scala | |
object ProducerBasedUserAdGraphParams | |
object MinCoOccurrenceParam | |
extends FSBoundedParam[Int]( | |
name = "producer_based_user_ad_graph_min_co_occurrence", | |
default = 2, | |
min = 0, | |
max = 500 | |
) | |
object MinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "producer_based_user_ad_graph_min_score", | |
default = 3.0, | |
min = 0.0, | |
max = 10.0 | |
) | |
object MaxNumFollowersParam | |
extends FSBoundedParam[Int]( | |
name = "producer_based_user_ad_graph_max_num_followers", | |
default = 500, | |
min = 100, | |
max = 1000 | |
) | |
val AllParams: Seq[Param[_] with FSName] = | |
Seq(MinCoOccurrenceParam, MaxNumFollowersParam, MinScoreParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\ProducerBasedUserTweetGraphParams.scala | |
object ProducerBasedUserTweetGraphParams | |
object MinCoOccurrenceParam | |
extends FSBoundedParam[Int]( | |
name = "producer_based_user_tweet_graph_min_co_occurrence", | |
default = 4, | |
min = 0, | |
max = 500 | |
) | |
object MinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "producer_based_user_tweet_graph_min_score", | |
default = 3.0, | |
min = 0.0, | |
max = 10.0 | |
) | |
object MaxNumFollowersParam | |
extends FSBoundedParam[Int]( | |
name = "producer_based_user_tweet_graph_max_num_followers", | |
default = 500, | |
min = 100, | |
max = 1000 | |
) | |
val AllParams: Seq[Param[_] with FSName] = | |
Seq(MinCoOccurrenceParam, MaxNumFollowersParam, MinScoreParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RankerParams.scala | |
object RankerParams | |
object MaxCandidatesToRank | |
extends FSBoundedParam[Int]( | |
name = "twistly_core_max_candidates_to_rank", | |
default = 2000, | |
min = 0, | |
max = 9999 | |
) | |
object EnableBlueVerifiedTopK | |
extends FSParam[Boolean]( | |
name = "twistly_core_blue_verified_top_k", | |
default = true | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
MaxCandidatesToRank, | |
EnableBlueVerifiedTopK | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RealGraphInParams.scala | |
object RealGraphInParams | |
object EnableSourceGraphParam | |
extends FSParam[Boolean]( | |
name = "graph_realgraphin_enable_source", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceGraphParam, | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RealGraphOonParams.scala | |
object RealGraphOonParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "signal_realgraphoon_enable_source", | |
default = false | |
) | |
object EnableSourceGraphParam | |
extends FSParam[Boolean]( | |
name = "graph_realgraphoon_enable_source", | |
default = false | |
) | |
object MaxConsumerSeedsNumParam | |
extends FSBoundedParam[Int]( | |
name = "graph_realgraphoon_max_user_seeds_num", | |
default = 200, | |
min = 0, | |
max = 1000 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam, | |
EnableSourceGraphParam, | |
MaxConsumerSeedsNumParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentFollowsParams.scala | |
object RecentFollowsParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_recentfollows_enable_source", | |
default = true | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentNegativeSignalParams.scala | |
object RecentNegativeSignalParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_recentnegativesignals_enable_source", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentNotificationsParams.scala | |
object RecentNotificationsParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_recentnotifications_enable_source", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentOriginalTweetsParams.scala | |
object RecentOriginalTweetsParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_recentoriginaltweets_enable_source", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentReplyTweetsParams.scala | |
object RecentReplyTweetsParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_recentreplytweets_enable_source", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentRetweetsParams.scala | |
object RecentRetweetsParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_recentretweets_enable_source", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RecentTweetFavoritesParams.scala | |
object RecentTweetFavoritesParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_recenttweetfavorites_enable_source", | |
default = true | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedTweetGlobalParams.scala | |
object RelatedTweetGlobalParams | |
object MaxCandidatesPerRequestParam | |
extends FSBoundedParam[Int]( | |
name = "related_tweet_core_max_candidates_per_request", | |
default = 100, | |
min = 0, | |
max = 500 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(MaxCandidatesPerRequestParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedTweetProducerBasedParams.scala | |
object RelatedTweetProducerBasedParams | |
object EnableUTGParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_producer_based_enable_utg", | |
default = false | |
) | |
// SimClusters params | |
object EnableSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_producer_based_enable_simclusters", | |
default = true | |
) | |
// Filter params | |
object SimClustersMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "related_tweet_producer_based_filter_simclusters_min_score", | |
default = 0.0, | |
min = 0.0, | |
max = 1.0 | |
) | |
// Experimental SimClusters ANN params | |
object EnableExperimentalSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_producer_based_enable_experimental_simclusters_ann", | |
default = false | |
) | |
// SimClusters ANN cluster 1 params | |
object EnableSimClustersANN1Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_producer_based_enable_simclusters_ann_1", | |
default = false | |
) | |
// SimClusters ANN cluster 2 params | |
object EnableSimClustersANN2Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_producer_based_enable_simclusters_ann_2", | |
default = false | |
) | |
// SimClusters ANN cluster 3 params | |
object EnableSimClustersANN3Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_producer_based_enable_simclusters_ann_3", | |
default = false | |
) | |
// SimClusters ANN cluster 3 params | |
object EnableSimClustersANN5Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_producer_based_enable_simclusters_ann_5", | |
default = false | |
) | |
// SimClusters ANN cluster 4 params | |
object EnableSimClustersANN4Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_producer_based_enable_simclusters_ann_4", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableUTGParam, | |
EnableSimClustersANNParam, | |
EnableSimClustersANN1Param, | |
EnableSimClustersANN2Param, | |
EnableSimClustersANN3Param, | |
EnableSimClustersANN5Param, | |
EnableSimClustersANN4Param, | |
EnableExperimentalSimClustersANNParam, | |
SimClustersMinScoreParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedTweetTweetBasedParams.scala | |
object RelatedTweetTweetBasedParams | |
object EnableUTGParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_utg", | |
default = false | |
) | |
// UVG params | |
object EnableUVGParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_uvg", | |
default = false | |
) | |
// UAG params | |
object EnableUAGParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_uag", | |
default = false | |
) | |
// SimClusters params | |
object EnableSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_simclusters", | |
default = true | |
) | |
// Experimental SimClusters ANN params | |
object EnableExperimentalSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_experimental_simclusters_ann", | |
default = false | |
) | |
// SimClusters ANN cluster 1 params | |
object EnableSimClustersANN1Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_simclusters_ann_1", | |
default = false | |
) | |
// SimClusters ANN cluster 2 params | |
object EnableSimClustersANN2Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_simclusters_ann_2", | |
default = false | |
) | |
// SimClusters ANN cluster 3 params | |
object EnableSimClustersANN3Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_simclusters_ann_3", | |
default = false | |
) | |
// SimClusters ANN cluster 5 params | |
object EnableSimClustersANN5Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_simclusters_ann_5", | |
default = false | |
) | |
object EnableSimClustersANN4Param | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_simclusters_ann_4", | |
default = false | |
) | |
// TwHIN params | |
object EnableTwHINParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_twhin", | |
default = false | |
) | |
// QIG params | |
object EnableQigSimilarTweetsParam | |
extends FSParam[Boolean]( | |
name = "related_tweet_tweet_based_enable_qig_similar_tweets", | |
default = false | |
) | |
// Filter params | |
object SimClustersMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "related_tweet_tweet_based_filter_simclusters_min_score", | |
default = 0.3, | |
min = 0.0, | |
max = 1.0 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableTwHINParam, | |
EnableQigSimilarTweetsParam, | |
EnableUTGParam, | |
EnableUVGParam, | |
EnableSimClustersANNParam, | |
EnableSimClustersANN2Param, | |
EnableSimClustersANN3Param, | |
EnableSimClustersANN5Param, | |
EnableSimClustersANN4Param, | |
EnableExperimentalSimClustersANNParam, | |
SimClustersMinScoreParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedVideoTweetGlobalParams.scala | |
object RelatedVideoTweetGlobalParams | |
object MaxCandidatesPerRequestParam | |
extends FSBoundedParam[Int]( | |
name = "related_video_tweet_core_max_candidates_per_request", | |
default = 100, | |
min = 0, | |
max = 500 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(MaxCandidatesPerRequestParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RelatedVideoTweetTweetBasedParams.scala | |
object RelatedVideoTweetTweetBasedParams | |
object EnableUTGParam | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_utg", | |
default = false | |
) | |
// SimClusters params | |
object EnableSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_simclusters", | |
default = true | |
) | |
// Experimental SimClusters ANN params | |
object EnableExperimentalSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_experimental_simclusters_ann", | |
default = false | |
) | |
// SimClusters ANN cluster 1 params | |
object EnableSimClustersANN1Param | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_simclusters_ann_1", | |
default = false | |
) | |
// SimClusters ANN cluster 2 params | |
object EnableSimClustersANN2Param | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_simclusters_ann_2", | |
default = false | |
) | |
// SimClusters ANN cluster 3 params | |
object EnableSimClustersANN3Param | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_simclusters_ann_3", | |
default = false | |
) | |
// SimClusters ANN cluster 5 params | |
object EnableSimClustersANN5Param | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_simclusters_ann_5", | |
default = false | |
) | |
// SimClusters ANN cluster 4 params | |
object EnableSimClustersANN4Param | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_simclusters_ann_4", | |
default = false | |
) | |
// TwHIN params | |
object EnableTwHINParam | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_twhin", | |
default = false | |
) | |
// QIG params | |
object EnableQigSimilarTweetsParam | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_qig_similar_tweets", | |
default = false | |
) | |
// Filter params | |
object SimClustersMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "related_video_tweet_tweet_based_filter_simclusters_min_score", | |
default = 0.3, | |
min = 0.0, | |
max = 1.0 | |
) | |
object EnableUVGParam | |
extends FSParam[Boolean]( | |
name = "related_video_tweet_tweet_based_enable_uvg", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableTwHINParam, | |
EnableQigSimilarTweetsParam, | |
EnableUTGParam, | |
EnableUVGParam, | |
EnableSimClustersANNParam, | |
EnableSimClustersANN2Param, | |
EnableSimClustersANN3Param, | |
EnableSimClustersANN5Param, | |
EnableSimClustersANN4Param, | |
EnableExperimentalSimClustersANNParam, | |
SimClustersMinScoreParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\RepeatedProfileVisitsParams.scala | |
object RepeatedProfileVisitsParams | |
object ProfileMinVisitParam extends Enumeration | |
class SignalTypeValue(signalType: SignalType) extends super.Val | |
import scala.language.implicitConversions | |
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = | |
x.asInstanceOf[SignalTypeValue] | |
val TotalVisitsInPast180Days = SignalTypeValue(SignalType.RepeatedProfileVisit180dMinVisit6V1) | |
val TotalVisitsInPast90Days = SignalTypeValue(SignalType.RepeatedProfileVisit90dMinVisit6V1) | |
val TotalVisitsInPast14Days = SignalTypeValue(SignalType.RepeatedProfileVisit14dMinVisit2V1) | |
val TotalVisitsInPast180DaysNoNegative = SignalTypeValue( | |
SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative) | |
val TotalVisitsInPast90DaysNoNegative = SignalTypeValue( | |
SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative) | |
val TotalVisitsInPast14DaysNoNegative = SignalTypeValue( | |
SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative) | |
} | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_repeatedprofilevisits_enable_source", | |
default = true | |
) | |
object MinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "twistly_repeatedprofilevisits_min_score", | |
default = 0.5, | |
min = 0.0, | |
max = 1.0 | |
) | |
object ProfileMinVisitType | |
extends FSEnumParam[ProfileMinVisitParam.type]( | |
name = "twistly_repeatedprofilevisits_min_visit_type_id", | |
default = ProfileMinVisitParam.TotalVisitsInPast14Days, | |
enum = ProfileMinVisitParam | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam, ProfileMinVisitType) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\SimClustersANNParams.scala | |
object SimClustersANNParams | |
object SimClustersANNConfigId | |
extends FSParam[String]( | |
name = "similarity_simclusters_ann_simclusters_ann_config_id", | |
default = "Default" | |
) | |
object SimClustersANN1ConfigId | |
extends FSParam[String]( | |
name = "similarity_simclusters_ann_simclusters_ann_1_config_id", | |
default = "20220810" | |
) | |
object SimClustersANN2ConfigId | |
extends FSParam[String]( | |
name = "similarity_simclusters_ann_simclusters_ann_2_config_id", | |
default = "20220818" | |
) | |
object SimClustersANN3ConfigId | |
extends FSParam[String]( | |
name = "similarity_simclusters_ann_simclusters_ann_3_config_id", | |
default = "20220819" | |
) | |
object SimClustersANN5ConfigId | |
extends FSParam[String]( | |
name = "similarity_simclusters_ann_simclusters_ann_5_config_id", | |
default = "20221221" | |
) | |
object SimClustersANN4ConfigId | |
extends FSParam[String]( | |
name = "similarity_simclusters_ann_simclusters_ann_4_config_id", | |
default = "20221220" | |
) | |
object ExperimentalSimClustersANNConfigId | |
extends FSParam[String]( | |
name = "similarity_simclusters_ann_experimental_simclusters_ann_config_id", | |
default = "20220801" | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
SimClustersANNConfigId, | |
SimClustersANN1ConfigId, | |
SimClustersANN2ConfigId, | |
SimClustersANN3ConfigId, | |
SimClustersANN5ConfigId, | |
ExperimentalSimClustersANNConfigId | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TopicTweetParams.scala | |
object TopicTweetParams | |
object MaxTweetAge | |
extends FSBoundedParam[Duration]( | |
name = "topic_tweet_candidate_generation_max_tweet_age_hours", | |
default = 24.hours, | |
min = 12.hours, | |
max = 48.hours | |
) | |
with HasDurationConversion | |
object MaxTopicTweetCandidatesParam | |
extends FSBoundedParam[Int]( | |
name = "topic_tweet_max_candidates_num", | |
default = 200, | |
min = 0, | |
max = 1000 | |
) | |
object MaxSkitTfgCandidatesParam | |
extends FSBoundedParam[Int]( | |
name = "topic_tweet_skit_tfg_max_candidates_num", | |
default = 100, | |
min = 0, | |
max = 1000 | |
) | |
object MaxSkitHighPrecisionCandidatesParam | |
extends FSBoundedParam[Int]( | |
name = "topic_tweet_skit_high_precision_max_candidates_num", | |
default = 100, | |
min = 0, | |
max = 1000 | |
) | |
object MaxCertoCandidatesParam | |
extends FSBoundedParam[Int]( | |
name = "topic_tweet_certo_max_candidates_num", | |
default = 100, | |
min = 0, | |
max = 1000 | |
) | |
// The min prod score for Certo L2-normalized cosine candidates | |
object CertoScoreThresholdParam | |
extends FSBoundedParam[Double]( | |
name = "topic_tweet_certo_score_threshold", | |
default = 0.015, | |
min = 0, | |
max = 1 | |
) | |
object SemanticCoreVersionIdParam | |
extends FSParam[Long]( | |
name = "semantic_core_version_id", | |
default = 1380520918896713735L | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
CertoScoreThresholdParam, | |
MaxTopicTweetCandidatesParam, | |
MaxTweetAge, | |
MaxCertoCandidatesParam, | |
MaxSkitTfgCandidatesParam, | |
MaxSkitHighPrecisionCandidatesParam, | |
SemanticCoreVersionIdParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedCandidateGenerationParams.scala | |
object TweetBasedCandidateGenerationParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_source", | |
default = false | |
) | |
// UTG params | |
object EnableUTGParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_utg", | |
default = true | |
) | |
// SimClusters params | |
object EnableSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_simclusters", | |
default = true | |
) | |
// Experimental SimClusters ANN params | |
object EnableExperimentalSimClustersANNParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_experimental_simclusters_ann", | |
default = false | |
) | |
// SimClusters ANN cluster 1 params | |
object EnableSimClustersANN1Param | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_simclusters_ann_1", | |
default = false | |
) | |
// SimClusters ANN cluster 2 params | |
object EnableSimClustersANN2Param | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_simclusters_ann_2", | |
default = false | |
) | |
// SimClusters ANN cluster 3 params | |
object EnableSimClustersANN3Param | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_simclusters_ann_3", | |
default = false | |
) | |
// SimClusters ANN cluster 3 params | |
object EnableSimClustersANN5Param | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_simclusters_ann_5", | |
default = false | |
) | |
// SimClusters ANN cluster 4 params | |
object EnableSimClustersANN4Param | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_simclusters_ann_4", | |
default = false | |
) | |
// TwHIN params | |
object EnableTwHINParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_twhin", | |
default = false | |
) | |
// QIG params | |
object EnableQigSimilarTweetsParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_qig_similar_tweets", | |
default = false | |
) | |
object QigMaxNumSimilarTweetsParam | |
extends FSBoundedParam[Int]( | |
name = "tweet_based_candidate_generation_qig_max_num_similar_tweets", | |
default = 100, | |
min = 10, | |
max = 100 | |
) | |
// UVG params | |
object EnableUVGParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_uvg", | |
default = false | |
) | |
// UAG params | |
object EnableUAGParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_candidate_generation_enable_uag", | |
default = false | |
) | |
// Filter params | |
object SimClustersMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "tweet_based_candidate_generation_filter_simclusters_min_score", | |
default = 0.5, | |
min = 0.0, | |
max = 1.0 | |
) | |
// for learning DDG that has a higher threshold for video based SANN | |
object SimClustersVideoBasedMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "tweet_based_candidate_generation_filter_simclusters_video_based_min_score", | |
default = 0.5, | |
min = 0.0, | |
max = 1.0 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableSourceParam, | |
EnableTwHINParam, | |
EnableQigSimilarTweetsParam, | |
EnableUTGParam, | |
EnableUVGParam, | |
EnableUAGParam, | |
EnableSimClustersANNParam, | |
EnableSimClustersANN1Param, | |
EnableSimClustersANN2Param, | |
EnableSimClustersANN3Param, | |
EnableSimClustersANN5Param, | |
EnableSimClustersANN4Param, | |
EnableExperimentalSimClustersANNParam, | |
SimClustersMinScoreParam, | |
SimClustersVideoBasedMinScoreParam, | |
QigMaxNumSimilarTweetsParam, | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedTwHINParams.scala | |
object TweetBasedTwHINParams | |
object ModelIdParam | |
extends FSParam[String]( | |
name = "tweet_based_twhin_model_id", | |
default = ModelConfig.TweetBasedTwHINRegularUpdateAll20221024, | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(ModelIdParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedUserAdGraphParams.scala | |
object TweetBasedUserAdGraphParams | |
object MinCoOccurrenceParam | |
extends FSBoundedParam[Int]( | |
name = "tweet_based_user_ad_graph_min_co_occurrence", | |
default = 1, | |
min = 0, | |
max = 500 | |
) | |
object ConsumersBasedMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "tweet_based_user_ad_graph_consumers_based_min_score", | |
default = 0.0, | |
min = 0.0, | |
max = 10.0 | |
) | |
object MaxConsumerSeedsNumParam | |
extends FSBoundedParam[Int]( | |
name = "tweet_based_user_ad_graph_max_user_seeds_num", | |
default = 100, | |
min = 0, | |
max = 300 | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
MinCoOccurrenceParam, | |
MaxConsumerSeedsNumParam, | |
ConsumersBasedMinScoreParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedUserTweetGraphParams.scala | |
object TweetBasedUserTweetGraphParams | |
object MinCoOccurrenceParam | |
extends FSBoundedParam[Int]( | |
name = "tweet_based_user_tweet_graph_min_co_occurrence", | |
default = 3, | |
min = 0, | |
max = 500 | |
) | |
object TweetBasedMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "tweet_based_user_tweet_graph_tweet_based_min_score", | |
default = 0.5, | |
min = 0.0, | |
max = 10.0 | |
) | |
object ConsumersBasedMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "tweet_based_user_tweet_graph_consumers_based_min_score", | |
default = 4.0, | |
min = 0.0, | |
max = 10.0 | |
) | |
object MaxConsumerSeedsNumParam | |
extends FSBoundedParam[Int]( | |
name = "tweet_based_user_tweet_graph_max_user_seeds_num", | |
default = 100, | |
min = 0, | |
max = 300 | |
) | |
object EnableCoverageExpansionOldTweetParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_user_tweet_graph_enable_coverage_expansion_old_tweet", | |
default = false | |
) | |
object EnableCoverageExpansionAllTweetParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_user_tweet_graph_enable_coverage_expansion_all_tweet", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableCoverageExpansionAllTweetParam, | |
EnableCoverageExpansionOldTweetParam, | |
MinCoOccurrenceParam, | |
MaxConsumerSeedsNumParam, | |
TweetBasedMinScoreParam, | |
ConsumersBasedMinScoreParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetBasedUserVideoGraphParams.scala | |
object TweetBasedUserVideoGraphParams | |
object MinCoOccurrenceParam | |
extends FSBoundedParam[Int]( | |
name = "tweet_based_user_video_graph_min_co_occurrence", | |
default = 5, | |
min = 0, | |
max = 500 | |
) | |
object TweetBasedMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "tweet_based_user_video_graph_tweet_based_min_score", | |
default = 0.0, | |
min = 0.0, | |
max = 100.0 | |
) | |
object ConsumersBasedMinScoreParam | |
extends FSBoundedParam[Double]( | |
name = "tweet_based_user_video_graph_consumers_based_min_score", | |
default = 4.0, | |
min = 0.0, | |
max = 10.0 | |
) | |
object MaxConsumerSeedsNumParam | |
extends FSBoundedParam[Int]( | |
name = "tweet_based_user_video_graph_max_user_seeds_num", | |
default = 200, | |
min = 0, | |
max = 500 | |
) | |
object EnableCoverageExpansionOldTweetParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_user_video_graph_enable_coverage_expansion_old_tweet", | |
default = false | |
) | |
object EnableCoverageExpansionAllTweetParam | |
extends FSParam[Boolean]( | |
name = "tweet_based_user_video_graph_enable_coverage_expansion_all_tweet", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
MinCoOccurrenceParam, | |
MaxConsumerSeedsNumParam, | |
TweetBasedMinScoreParam, | |
EnableCoverageExpansionOldTweetParam, | |
EnableCoverageExpansionAllTweetParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\TweetSharesParams.scala | |
object TweetSharesParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_tweetshares_enable_source", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\UnifiedSETweetCombinationMethod.scala | |
object UnifiedSETweetCombinationMethod extends Enumeration | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\UnifiedUSSSignalParams.scala | |
object UnifiedUSSSignalParams | |
object TweetAggregationTypeParam extends Enumeration | |
class SignalTypeValue(signalType: SignalType) extends super.Val | |
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = | |
x.asInstanceOf[SignalTypeValue] | |
val UniformAggregation = SignalTypeValue(SignalType.TweetBasedUnifiedUniformSignal) | |
val EngagementAggregation = SignalTypeValue( | |
SignalType.TweetBasedUnifiedEngagementWeightedSignal) | |
} | |
object ProducerAggregationTypeParam extends Enumeration | |
class SignalTypeValue(signalType: SignalType) extends super.Val | |
import scala.language.implicitConversions | |
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = | |
x.asInstanceOf[SignalTypeValue] | |
val UniformAggregation = SignalTypeValue(SignalType.ProducerBasedUnifiedUniformSignal) | |
val EngagementAggregation = SignalTypeValue( | |
SignalType.ProducerBasedUnifiedEngagementWeightedSignal) | |
} | |
object ReplaceIndividualUSSSourcesParam | |
extends FSParam[Boolean]( | |
name = "twistly_agg_replace_enable_source", | |
default = false | |
) | |
object EnableTweetAggSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_agg_tweet_agg_enable_source", | |
default = false | |
) | |
object TweetAggTypeParam | |
extends FSEnumParam[TweetAggregationTypeParam.type]( | |
name = "twistly_agg_tweet_agg_type_id", | |
default = TweetAggregationTypeParam.EngagementAggregation, | |
enum = TweetAggregationTypeParam | |
) | |
object UnifiedTweetSourceNumberParam | |
extends FSBoundedParam[Int]( | |
name = "twistly_agg_tweet_agg_source_number", | |
default = 0, | |
min = 0, | |
max = 100, | |
) | |
object EnableProducerAggSourceParam | |
extends FSParam[Boolean]( | |
name = "twistly_agg_producer_agg_enable_source", | |
default = false | |
) | |
object ProducerAggTypeParam | |
extends FSEnumParam[ProducerAggregationTypeParam.type]( | |
name = "twistly_agg_producer_agg_type_id", | |
default = ProducerAggregationTypeParam.EngagementAggregation, | |
enum = ProducerAggregationTypeParam | |
) | |
object UnifiedProducerSourceNumberParam | |
extends FSBoundedParam[Int]( | |
name = "twistly_agg_producer_agg_source_number", | |
default = 0, | |
min = 0, | |
max = 100, | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableTweetAggSourceParam, | |
EnableProducerAggSourceParam, | |
TweetAggTypeParam, | |
ProducerAggTypeParam, | |
UnifiedTweetSourceNumberParam, | |
UnifiedProducerSourceNumberParam, | |
ReplaceIndividualUSSSourcesParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\UtegTweetGlobalParams.scala | |
object UtegTweetGlobalParams | |
object MaxUtegCandidatesToRequestParam | |
extends FSBoundedParam[Int]( | |
name = "max_uteg_candidates_to_request", | |
default = 800, | |
min = 10, | |
max = 200 | |
) | |
object CandidateRefreshSinceTimeOffsetHoursParam | |
extends FSBoundedParam[Duration]( | |
name = "candidate_refresh_since_time_offset_hours", | |
default = 48.hours, | |
min = 1.hours, | |
max = 96.hours | |
) | |
with HasDurationConversion | |
object EnableTLRHealthFilterParam | |
extends FSParam[Boolean]( | |
name = "enable_uteg_tlr_health_filter", | |
default = true | |
) | |
object EnableRepliesToNonFollowedUsersFilterParam | |
extends FSParam[Boolean]( | |
name = "enable_uteg_replies_to_non_followed_users_filter", | |
default = false | |
) | |
object EnableRetweetFilterParam | |
extends FSParam[Boolean]( | |
name = "enable_uteg_retweet_filter", | |
default = true | |
) | |
object EnableInNetworkFilterParam | |
extends FSParam[Boolean]( | |
name = "enable_uteg_in_network_filter", | |
default = true | |
) | |
val AllParams: Seq[Param[_] with FSName] = | |
Seq( | |
MaxUtegCandidatesToRequestParam, | |
CandidateRefreshSinceTimeOffsetHoursParam, | |
EnableTLRHealthFilterParam, | |
EnableRepliesToNonFollowedUsersFilterParam, | |
EnableRetweetFilterParam, | |
EnableInNetworkFilterParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\VideoTweetFilterParams.scala | |
object VideoTweetFilterParams | |
object EnableVideoTweetFilterParam | |
extends FSParam[Boolean]( | |
name = "video_tweet_filter_enable_filter", | |
default = false | |
) | |
val AllParams: Seq[Param[_] with FSName] = Seq( | |
EnableVideoTweetFilterParam | |
) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\VideoViewTweetsParams.scala | |
object VideoViewTweetsParams | |
object EnableSourceParam | |
extends FSParam[Boolean]( | |
name = "signal_videoviewtweets_enable_source", | |
default = false | |
) | |
object EnableSourceImpressionParam | |
extends FSParam[Boolean]( | |
name = "signal_videoviewtweets_enableimpression_source", | |
default = false | |
) | |
object VideoViewTweetType extends Enumeration | |
class SignalTypeValue(signalType: SignalType) extends super.Val | |
import scala.language.implicitConversions | |
implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = | |
x.asInstanceOf[SignalTypeValue] | |
val VideoTweetQualityView: SignalTypeValue = SignalTypeValue(SignalType.VideoView90dQualityV1) | |
val VideoTweetPlayback50: SignalTypeValue = SignalTypeValue(SignalType.VideoView90dPlayback50V1) | |
} | |
object VideoViewTweetTypeParam | |
extends FSEnumParam[VideoViewTweetType.type]( | |
name = "signal_videoviewtweets_videoviewtype_id", | |
default = VideoViewTweetType.VideoTweetQualityView, | |
enum = VideoViewTweetType | |
) | |
val AllParams: Seq[Param[_] with FSName] = | |
Seq(EnableSourceParam, EnableSourceImpressionParam, VideoViewTweetTypeParam) | |
lazy val config: BaseConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\decider\CrMixerDecider.scala | |
class CrMixerDecider @Inject() (decider: Decider) | |
def isAvailable(feature: String, recipient: Option[Recipient]): Boolean | |
def isAvailable(feature: String, useRandomRecipient: Boolean = true): Boolean | |
def isAvailableForId( | |
id: Long, | |
deciderConstants: String | |
): Boolean | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\decider\DeciderKey.scala | |
object DeciderConstants | |
object DeciderKey extends DeciderKeyEnum | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\param\decider\EndpointLoadShedder.scala | |
class EndpointLoadShedder @Inject() ( | |
decider: Decider, | |
statsReceiver: StatsReceiver) | |
def apply[T](endpointName: String, product: String)(serve: => Future[T]): Future[T] | |
object EndpointLoadShedder | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\ranker\DefaultRanker.scala | |
class DefaultRanker() | |
def rank( | |
candidates: Seq[BlendedCandidate], | |
): Future[Seq[RankedCandidate]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\ranker\SwitchRanker.scala | |
class SwitchRanker @Inject() ( | |
defaultRanker: DefaultRanker, | |
globalStats: StatsReceiver) | |
def rank( | |
query: CrCandidateGeneratorQuery, | |
candidates: Seq[BlendedCandidate], | |
): Future[Seq[RankedCandidate]] | |
object SwitchRanker | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\scribe\ScribeCategory.scala | |
object ScribeCategories | |
class ScribeCategory( | |
loggerFactoryNode: String, | |
scribeCategory: String) | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\service\CrMixerAlertNotificationConfig.scala | |
object CrMixerAlertNotificationConfig | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\CertoTopicTweetSimilarityEngine.scala | |
class CertoTopicTweetSimilarityEngine @Inject() ( | |
@Named(ModuleNames.CertoStratoStoreName) certoStratoStore: ReadableStore[ | |
TopicId, | |
Seq[TweetWithScores] | |
], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] | |
def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] | |
object CertoTopicTweetSimilarityEngine | |
class Query( | |
topicId: TopicId, | |
maxCandidates: Int, | |
certoScoreTheshold: Double) | |
def fromParams( | |
topicId: TopicId, | |
isVideoOnly: Boolean, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumerBasedWalsSimilarityEngine.scala | |
object maintain a set of stats that are specific to the Wals Engine. | |
case class WalsStats(scope: String, scopedStats: StatsReceiver) | |
def onFailure(t: Throwable, startTimeMs: Long) | |
def onSuccess(startTimeMs: Long) | |
object and can be expensive | |
// in performance critical paths. | |
object WalsStatsMap | |
def get(scope: String, scopedStats: StatsReceiver): WalsStats | |
class ConsumerBasedWalsSimilarityEngine( | |
homeNaviGRPCClient: ManagedChannel, | |
adsFavedNaviGRPCClient: ManagedChannel, | |
adsMonetizableNaviGRPCClient: ManagedChannel, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: ConsumerBasedWalsSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
def getFeaturesForRecommendations(query: ConsumerBasedWalsSimilarityEngine.Query): Example | |
def getModelInput(query: ConsumerBasedWalsSimilarityEngine.Query): PredictRequest | |
def getModelOutput(query: Query, response: PredictResponse): Seq[TweetWithScore] | |
object ConsumerBasedWalsSimilarityEngine | |
class Query( | |
sourceIds: Seq[SourceInfo], | |
modelName: String, | |
modelInputName: String, | |
modelOutputName: String, | |
modelSignatureName: String, | |
wilyNsName: String, | |
) | |
def fromParams( | |
sourceIds: Seq[SourceInfo], | |
params: configapi.Params, | |
): EngineQuery[Query] | |
def toSimilarityEngineInfo( | |
score: Double | |
): SimilarityEngineInfo | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumerEmbeddingBasedTripSimilarityEngine.scala | |
class TripEngineQuery( | |
modelId: String, | |
sourceId: InternalId, | |
tripSourceId: String, | |
maxResult: Int, | |
params: Params) | |
case class ConsumerEmbeddingBasedTripSimilarityEngine( | |
embeddingStoreLookUpMap: Map[String, ReadableStore[UserId, SimClustersEmbedding]], | |
tripCandidateSource: ReadableStore[TripDomain, Seq[TripTweet]], | |
statsReceiver: StatsReceiver, | |
) extends ReadableStore[TripEngineQuery, Seq[TripTweetWithScore]] | |
def fetchTopClusters(query: TripEngineQuery): Future[Option[Seq[ClusterId]]] | |
def fetchCandidates( | |
topClusters: Seq[ClusterId], | |
tripSourceId: String | |
): Future[Seq[Seq[TripTweetWithScore]]] | |
def get(engineQuery: TripEngineQuery): Future[Option[Seq[TripTweetWithScore]]] | |
object ConsumerEmbeddingBasedTripSimilarityEngine | |
def fromParams( | |
modelId: String, | |
sourceId: InternalId, | |
params: configapi.Params | |
): TripEngineQuery | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumerEmbeddingBasedTwHINSimilarityEngine.scala | |
object ConsumerEmbeddingBasedTwHINSimilarityEngine | |
def fromParams( | |
sourceId: InternalId, | |
params: configapi.Params, | |
): HnswANNEngineQuery | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumerEmbeddingBasedTwoTowerSimilarityEngine.scala | |
object ConsumerEmbeddingBasedTwoTowerSimilarityEngine | |
def fromParams( | |
sourceId: InternalId, | |
params: configapi.Params, | |
): HnswANNEngineQuery | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumersBasedUserAdGraphSimilarityEngine.scala | |
class ConsumersBasedUserAdGraphSimilarityEngine( | |
consumersBasedUserAdGraphStore: ReadableStore[ | |
ConsumersBasedRelatedAdRequest, | |
RelatedAdResponse | |
], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
ConsumersBasedUserAdGraphSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: ConsumersBasedUserAdGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object ConsumersBasedUserAdGraphSimilarityEngine | |
class Query( | |
seedWithScores: Map[UserId, Double], | |
maxResults: Int, | |
minCooccurrence: Int, | |
minScore: Double, | |
maxTweetAgeInHours: Int) | |
def toSimilarityEngineInfo( | |
score: Double | |
): SimilarityEngineInfo | |
def fromParams( | |
seedWithScores: Map[UserId, Double], | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ConsumersBasedUserVideoGraphSimilarityEngine.scala | |
class ConsumersBasedUserVideoGraphSimilarityEngine( | |
consumersBasedUserVideoGraphStore: ReadableStore[ | |
ConsumersBasedRelatedTweetRequest, | |
RelatedTweetResponse | |
], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
ConsumersBasedUserVideoGraphSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: ConsumersBasedUserVideoGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object ConsumersBasedUserVideoGraphSimilarityEngine | |
class Query( | |
seedWithScores: Map[UserId, Double], | |
maxResults: Int, | |
minCooccurrence: Int, | |
minScore: Double, | |
maxTweetAgeInHours: Int) | |
def toSimilarityEngineInfo( | |
score: Double | |
): SimilarityEngineInfo | |
def fromParamsForRealGraphIn( | |
seedWithScores: Map[UserId, Double], | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\DiffusionBasedSimilarityEngine.scala | |
class DiffusionBasedSimilarityEngine( | |
retweetBasedDiffusionRecsMhStore: ReadableStore[Long, TweetsWithScore], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
DiffusionBasedSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: DiffusionBasedSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object DiffusionBasedSimilarityEngine | |
class Query( | |
sourceId: InternalId, | |
) | |
def toSimilarityEngineInfo( | |
query: LookupEngineQuery[Query], | |
score: Double | |
): SimilarityEngineInfo | |
def fromParams( | |
sourceId: InternalId, | |
modelId: String, | |
params: configapi.Params, | |
): LookupEngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdModelBasedSimilarityEngine.scala | |
class EarlybirdModelBasedSimilarityEngine @Inject() ( | |
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, | |
timeoutConfig: TimeoutConfig, | |
stats: StatsReceiver) | |
extends EarlybirdSimilarityEngineBase[EarlybirdModelBasedSearchQuery] | |
def getEarlybirdRequest( | |
query: EarlybirdModelBasedSearchQuery | |
): Option[EarlybirdRequest] = | |
if (query.seedUserIds.nonEmpty) | |
Some( | |
EarlybirdRequest( | |
searchQuery = getThriftSearchQuery(query), | |
clientId = Some(EarlybirdClientId), | |
timeoutMs = timeoutConfig.earlybirdServerTimeout.inMilliseconds.intValue(), | |
clientRequestID = Some(s"$ | |
object EarlybirdModelBasedSimilarityEngine | |
class EarlybirdModelBasedSearchQuery( | |
seedUserIds: Seq[UserId], | |
maxNumTweets: Int, | |
oldestTweetTimestampInSec: Option[UserId], | |
frsUserToScoresForScoreAdjustment: Option[Map[UserId, Double]]) | |
extends EarlybirdSearchQuery | |
/** | |
* Used by Push Service | |
*/ | |
val RealGraphScoringModel = "frigate_unified_engagement_rg" | |
val MaxHitsToProcess = 1000 | |
val MaxConsecutiveSameUser = 1 | |
private def getModelBasedRankingParams( | |
authorSpecificScoreAdjustments: Map[Long, Double] | |
): ThriftRankingParams = ThriftRankingParams( | |
`type` = Some(ThriftScoringFunctionType.ModelBased), | |
selectedModels = Some(Map(RealGraphScoringModel -> 1.0)), | |
applyBoosts = false, | |
authorSpecificScoreAdjustments = Some(authorSpecificScoreAdjustments) | |
) | |
private def getRelevanceOptions( | |
authorSpecificScoreAdjustments: Map[Long, Double], | |
): ThriftSearchRelevanceOptions | |
def getThriftSearchQuery(query: EarlybirdModelBasedSearchQuery): ThriftSearchQuery = | |
ThriftSearchQuery( | |
serializedQuery = Some(f"(* [since_time $ | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdRecencyBasedSimilarityEngine.scala | |
class EarlybirdRecencyBasedSimilarityEngine @Inject() ( | |
@Named(ModuleNames.EarlybirdRecencyBasedWithoutRetweetsRepliesTweetsCache) | |
earlybirdRecencyBasedWithoutRetweetsRepliesTweetsCacheStore: ReadableStore[ | |
UserId, | |
Seq[TweetId] | |
], | |
@Named(ModuleNames.EarlybirdRecencyBasedWithRetweetsRepliesTweetsCache) | |
earlybirdRecencyBasedWithRetweetsRepliesTweetsCacheStore: ReadableStore[ | |
UserId, | |
Seq[TweetId] | |
], | |
timeoutConfig: TimeoutConfig, | |
stats: StatsReceiver) | |
extends ReadableStore[EarlybirdRecencyBasedSearchQuery, Seq[TweetWithAuthor]] | |
def get( | |
query: EarlybirdRecencyBasedSearchQuery | |
): Future[Option[Seq[TweetWithAuthor]]] | |
object EarlybirdRecencyBasedSimilarityEngine | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdSimilarityEngine.scala | |
class EarlybirdSimilarityEngine[ | |
Query, | |
EarlybirdSimilarityEngineStore <: ReadableStore[Query, Seq[TweetWithAuthor]] | |
]( | |
implementingStore: EarlybirdSimilarityEngineStore, | |
override val identifier: SimilarityEngineType, | |
globalStats: StatsReceiver, | |
engineConfig: SimilarityEngineConfig, | |
) extends SimilarityEngine[EngineQuery[Query], TweetWithAuthor] | |
def getScopedStats: StatsReceiver = scopedStats | |
def getCandidates(query: EngineQuery[Query]): Future[Option[Seq[TweetWithAuthor]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdSimilarityEngineBase.scala | |
trait is a base trait for Earlybird similarity engines. All Earlybird similarity | |
* engines extend from it and override the construction method for EarlybirdRequest | |
*/ | |
trait EarlybirdSimilarityEngineBase[EarlybirdSearchQuery] | |
extends ReadableStore[EarlybirdSearchQuery, Seq[TweetWithAuthor]] | |
def earlybirdSearchClient: EarlybirdService.MethodPerEndpoint | |
def statsReceiver: StatsReceiver | |
def getEarlybirdRequest(query: EarlybirdSearchQuery): Option[EarlybirdRequest] | |
override def get(query: EarlybirdSearchQuery): Future[Option[Seq[TweetWithAuthor]]] | |
object EarlybirdSimilarityEngineBase | |
trait EarlybirdSearchQuery | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdSimilarityEngineRouter.scala | |
class EarlybirdSimilarityEngineRouter @Inject() ( | |
earlybirdRecencyBasedSimilarityEngine: EarlybirdSimilarityEngine[ | |
EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery, | |
EarlybirdRecencyBasedSimilarityEngine | |
], | |
earlybirdModelBasedSimilarityEngine: EarlybirdSimilarityEngine[ | |
EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery, | |
EarlybirdModelBasedSimilarityEngine | |
], | |
earlybirdTensorflowBasedSimilarityEngine: EarlybirdSimilarityEngine[ | |
EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery, | |
EarlybirdTensorflowBasedSimilarityEngine | |
], | |
timeoutConfig: TimeoutConfig, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[EarlybirdSimilarityEngineRouter.Query, Seq[TweetWithAuthor]] | |
def get( | |
k: EarlybirdSimilarityEngineRouter.Query | |
): Future[Option[Seq[TweetWithAuthor]]] | |
object EarlybirdSimilarityEngineRouter | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\EarlybirdTensorflowBasedSimilarityEngine.scala | |
class EarlybirdTensorflowBasedSimilarityEngine @Inject() ( | |
earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, | |
timeoutConfig: TimeoutConfig, | |
stats: StatsReceiver) | |
extends EarlybirdSimilarityEngineBase[EarlybirdTensorflowBasedSearchQuery] | |
def getEarlybirdRequest( | |
query: EarlybirdTensorflowBasedSearchQuery | |
): Option[EarlybirdRequest] | |
object EarlybirdTensorflowBasedSimilarityEngine | |
class EarlybirdTensorflowBasedSearchQuery( | |
searcherUserId: Option[UserId], | |
seedUserIds: Seq[UserId], | |
maxNumTweets: Int, | |
beforeTweetIdExclusive: Option[TweetId], | |
afterTweetIdExclusive: Option[TweetId], | |
filterOutRetweetsAndReplies: Boolean, | |
useTensorflowRanking: Boolean, | |
excludedTweetIds: Set[TweetId], | |
maxNumHitsPerShard: Int) | |
extends EarlybirdSearchQuery | |
private def getThriftSearchQuery( | |
query: EarlybirdTensorflowBasedSearchQuery, | |
processingTimeout: Duration | |
): ThriftSearchQuery = | |
ThriftSearchQuery( | |
serializedQuery = GetEarlybirdQuery( | |
query.beforeTweetIdExclusive, | |
query.afterTweetIdExclusive, | |
query.excludedTweetIds, | |
query.filterOutRetweetsAndReplies).map(_.serialize), | |
fromUserIDFilter64 = Some(query.seedUserIds), | |
numResults = query.maxNumTweets, | |
// Whether to collect conversation IDs. Remove it for now. | |
// collectConversationId = Gate.True(), // true for Home | |
rankingMode = ThriftSearchRankingMode.Relevance, | |
relevanceOptions = Some(getRelevanceOptions(query.useTensorflowRanking)), | |
collectorParams = Some( | |
CollectorParams( | |
// numResultsToReturn defines how many results each EB shard will return to search root | |
numResultsToReturn = 1000, | |
// terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. | |
terminationParams = | |
GetCollectorTerminationParams(query.maxNumHitsPerShard, processingTimeout) | |
)), | |
facetFieldNames = Some(FacetsToFetch), | |
resultMetadataOptions = Some(MetadataOptions), | |
searcherId = query.searcherUserId, | |
searchStatusIds = None, | |
namedDisjunctionMap = GetNamedDisjunctions(query.excludedTweetIds) | |
) | |
// The specific values of recap relevance/reranking options correspond to | |
// experiment: enable_recap_reranking_2988,timeline_internal_disable_recap_filter | |
// bucket : enable_rerank,disable_filter | |
private def getRelevanceOptions(useTensorflowRanking: Boolean): ThriftSearchRelevanceOptions | |
def getTensorflowBasedRankingParams: ThriftRankingParams | |
def getLinearRankingParams: ThriftRankingParams | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\FilterUtil.scala | |
object FilterUtil | |
def tweetAgeFilter( | |
candidates: Seq[TweetWithScore], | |
maxTweetAgeHours: Duration | |
): Seq[TweetWithScore] | |
def tweetSourceAgeFilter( | |
candidates: Seq[SourceInfo], | |
maxTweetSignalAgeHoursParam: Duration | |
): Seq[SourceInfo] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\HnswANNSimilarityEngine.scala | |
class HnswANNEngineQuery( | |
modelId: String, | |
sourceId: InternalId, | |
params: Params, | |
) | |
class HnswANNSimilarityEngine( | |
embeddingStoreLookUpMap: Map[String, ReadableStore[InternalId, ThriftEmbedding]], | |
annServiceLookUpMap: Map[String, AnnQueryService.MethodPerEndpoint], | |
globalStats: StatsReceiver, | |
override val identifier: SimilarityEngineType, | |
engineConfig: SimilarityEngineConfig, | |
memCacheConfigOpt: Option[MemCacheConfig[HnswANNEngineQuery]] = None) | |
extends SimilarityEngine[HnswANNEngineQuery, TweetWithScore] | |
def getScopedStats: StatsReceiver = scopedStats | |
private def fetchEmbedding( | |
query: HnswANNEngineQuery, | |
): Future[Option[ThriftEmbedding]] | |
def fetchCandidates( | |
query: HnswANNEngineQuery, | |
embedding: ThriftEmbedding | |
): Future[Seq[TweetWithScore]] | |
def toScore(distance: Distance): Double | |
def getEmbeddingAndCandidates( | |
query: HnswANNEngineQuery | |
): Future[Option[Seq[TweetWithScore]]] | |
def toSimilarityEngineInfo( | |
query: HnswANNEngineQuery, | |
score: Double | |
): SimilarityEngineInfo | |
def getCandidates( | |
engineQuery: HnswANNEngineQuery | |
): Future[Option[Seq[TweetWithScore]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\LookupSimilarityEngine.scala | |
class LookupEngineQuery[Query]( | |
storeQuery: Query, // the actual Query type of the underlying store | |
lookupKey: String, | |
params: Params, | |
) | |
/** | |
* This Engine provides a map interface for looking up different model implementations. | |
* It provides modelId level monitoring for free. | |
* | |
* Example use cases include OfflineSimClusters lookup | |
* | |
* | |
* @param versionedStoreMap A mapping from a modelId to a corresponding implementation | |
* @param memCacheConfigOpt If specified, it will wrap the underlying store with a MemCache layer | |
* You should only enable this for cacheable queries, e.x. TweetIds. | |
* consumer based UserIds are generally not possible to cache. | |
*/ | |
class LookupSimilarityEngine[Query, Candidate <: Serializable]( | |
versionedStoreMap: Map[String, ReadableStore[Query, Seq[Candidate]]], // key = modelId | |
override val identifier: SimilarityEngineType, | |
globalStats: StatsReceiver, | |
engineConfig: SimilarityEngineConfig, | |
memCacheConfigOpt: Option[MemCacheConfig[Query]] = None) | |
extends SimilarityEngine[LookupEngineQuery[Query], Candidate] | |
def getCandidates( | |
engineQuery: LookupEngineQuery[Query] | |
): Future[Option[Seq[Candidate]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ModelBasedANNStore.scala | |
class ModelBasedANNStore( | |
embeddingStoreLookUpMap: Map[String, ReadableStore[InternalId, ThriftEmbedding]], | |
annServiceLookUpMap: Map[String, AnnQueryService.MethodPerEndpoint], | |
globalStats: StatsReceiver) | |
extends ReadableStore[ | |
ModelBasedANNStore.Query, | |
Seq[TweetWithScore] | |
] | |
def get(query: Query): Future[Option[Seq[TweetWithScore]]] | |
def fetchEmbedding(query: Query): Future[Option[ThriftEmbedding]] | |
def fetchCandidates( | |
query: Query, | |
embedding: ThriftEmbedding | |
): Future[Option[NearestNeighborResult]] | |
object ModelBasedANNStore | |
class Query( | |
sourceId: InternalId, | |
modelId: String, | |
similarityEngineType: SimilarityEngineType, | |
ef: Int = 800) | |
def toScore(distance: Distance): Double | |
def toSimilarityEngineInfo(query: Query, score: Double): SimilarityEngineInfo | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ProducerBasedUnifiedSimilarityEngine.scala | |
class ProducerBasedUnifiedSimilarityEngine( | |
@Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine) | |
producerBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ | |
ProducerBasedUserTweetGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
simClustersANNSimilarityEngine: StandardSimilarityEngine[ | |
SimClustersANNSimilarityEngine.Query, | |
TweetWithScore | |
], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ProducerBasedUnifiedSimilarityEngine.Query, Seq[ | |
TweetWithCandidateGenerationInfo | |
]] | |
def get( | |
query: Query | |
): Future[Option[Seq[TweetWithCandidateGenerationInfo]]] | |
def simClustersCandidateMinScoreFilter( | |
simClustersAnnCandidates: Seq[TweetWithScore], | |
simClustersMinScore: Double, | |
simClustersANNConfigId: String | |
): Seq[TweetWithScore] | |
def userTweetGraphFilter( | |
userTweetGraphCandidates: Seq[TweetWithScore] | |
): Seq[TweetWithScore] | |
object ProducerBasedUnifiedSimilarityEngine | |
def getProducerBasedUnifiedCGInfo( | |
sourceInfoOpt: Option[SourceInfo], | |
unifiedScore: Double, | |
contributingSimilarityEngines: Seq[SimilarityEngineInfo] | |
): CandidateGenerationInfo | |
class Query( | |
sourceInfo: SourceInfo, | |
maxCandidateNumPerSourceKey: Int, | |
maxTweetAgeHours: Duration, | |
// SimClusters | |
enableSimClustersANN: Boolean, | |
simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableExperimentalSimClustersANN: Boolean, | |
experimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN1: Boolean, | |
simClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN2: Boolean, | |
simClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN4: Boolean, | |
simClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN3: Boolean, | |
simClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN5: Boolean, | |
simClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
simClustersMinScore: Double, | |
// UTG | |
enableUtg: Boolean, | |
utgCombinationMethod: UnifiedSETweetCombinationMethod.Value, | |
utgQuery: EngineQuery[ProducerBasedUserTweetGraphSimilarityEngine.Query]) | |
def fromParams( | |
sourceInfo: SourceInfo, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
def fromParamsForRelatedTweet( | |
internalId: InternalId, | |
params: configapi.Params | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ProducerBasedUserAdGraphSimilarityEngine.scala | |
class ProducerBasedUserAdGraphSimilarityEngine( | |
userAdGraphService: UserAdGraph.MethodPerEndpoint, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ProducerBasedUserAdGraphSimilarityEngine.Query, Seq[ | |
TweetWithScore | |
]] | |
def get( | |
query: ProducerBasedUserAdGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object ProducerBasedUserAdGraphSimilarityEngine | |
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo | |
class Query( | |
sourceId: InternalId, | |
maxResults: Int, | |
minCooccurrence: Int, // require at least | |
def fromParams( | |
sourceId: InternalId, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\ProducerBasedUserTweetGraphSimilarityEngine.scala | |
class ProducerBasedUserTweetGraphSimilarityEngine( | |
userTweetGraphService: UserTweetGraph.MethodPerEndpoint, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ProducerBasedUserTweetGraphSimilarityEngine.Query, Seq[ | |
TweetWithScore | |
]] | |
def get( | |
query: ProducerBasedUserTweetGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object ProducerBasedUserTweetGraphSimilarityEngine | |
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo | |
class Query( | |
sourceId: InternalId, | |
maxResults: Int, | |
minCooccurrence: Int, // require at least | |
def fromParams( | |
sourceId: InternalId, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SimClustersANNSimilarityEngine.scala | |
class SimClustersANNSimilarityEngine( | |
simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
SimClustersANNSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def getSimClustersANNService( | |
query: SimClustersANNQuery | |
): Option[SimClustersANNService.MethodPerEndpoint] | |
def get( | |
query: SimClustersANNSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object SimClustersANNSimilarityEngine | |
class Query( | |
simClustersANNQuery: SimClustersANNQuery, | |
simClustersANNConfigId: String) | |
def toSimilarityEngineInfo( | |
query: EngineQuery[Query], | |
score: Double | |
): SimilarityEngineInfo | |
def fromParams( | |
internalId: InternalId, | |
embeddingType: EmbeddingType, | |
modelVersion: ModelVersion, | |
simClustersANNConfigId: String, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SimilarityEngine.scala | |
class local this directory only | |
* | |
*/ | |
trait SimilarityEngine[Query, Candidate] | |
def identifier: SimilarityEngineType | |
def getCandidates(query: Query): Future[Option[Seq[Candidate]]] | |
} | |
object SimilarityEngine extends Logging | |
class SimilarityEngineConfig( | |
timeout: Duration, | |
gatingConfig: GatingConfig) | |
/** | |
* Controls for whether or not this Engine is enabled. | |
* In our previous design, we were expecting a Sim Engine will only take one set of Params, | |
* and that’s why we decided to have GatingConfig and the EnableFeatureSwitch in the trait. | |
* However, we now have two candidate generation pipelines: Tweet Rec, Related Tweets | |
* and they are now having their own set of Params, but EnableFeatureSwitch can only put in 1 fixed value. | |
* We need some further refactor work to make it more flexible. | |
* | |
* @param deciderConfig Gate the Engine by a decider. If specified, | |
* @param enableFeatureSwitch. DO NOT USE IT FOR NOW. It needs some refactorting. Please set it to None (SD-20268) | |
*/ | |
case class GatingConfig( | |
deciderConfig: Option[DeciderConfig], | |
enableFeatureSwitch: Option[ | |
FSParam[Boolean] | |
]) // Do NOT use the enableFeatureSwitch. It needs some refactoring. | |
case class DeciderConfig( | |
decider: CrMixerDecider, | |
deciderString: String) | |
case class MemCacheConfig[K]( | |
cacheClient: Client, | |
ttl: Duration, | |
asyncUpdate: Boolean = false, | |
keyToString: K => String) | |
private[similarity_engine] def isEnabled( | |
params: Params, | |
gatingConfig: GatingConfig | |
): Boolean | |
object needs to be cacheable, | |
* i.e. it cannot be a runtime objects or complex objects, for example, configapi.Params | |
* | |
* @param underlyingStore un-cached store implementation | |
* @param keyPrefix a prefix differentiates 2 stores if they share the same key space. | |
* e.x. 2 implementations of ReadableStore[UserId, Seq[Candidiate] ] | |
* can use prefix "store_v1", "store_v2" | |
* @return A ReadableStore with a MemCache wrapper | |
*/ | |
private[similarity_engine] def addMemCache[Query, Candidate <: Serializable]( | |
underlyingStore: ReadableStore[Query, Seq[Candidate]], | |
memCacheConfig: MemCacheConfig[Query], | |
keyPrefix: Option[String] = None, | |
statsReceiver: StatsReceiver | |
): ReadableStore[Query, Seq[Candidate]] | |
def getFromFn[Query, Candidate]( | |
fn: Query => Future[Option[Seq[Candidate]]], | |
storeQuery: Query, | |
engineConfig: SimilarityEngineConfig, | |
params: Params, | |
scopedStats: StatsReceiver | |
): Future[Option[Seq[Candidate]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SimilaritySourceOrderingUtil.scala | |
object SimilaritySourceOrderingUtil | |
def keepGivenOrder( | |
candidates: Seq[Seq[TweetWithCandidateGenerationInfo]], | |
): Seq[TweetWithCandidateGenerationInfo] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SkitHighPrecisionTopicTweetSimilarityEngine.scala | |
class SkitHighPrecisionTopicTweetSimilarityEngine @Inject() ( | |
@Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ | |
TopicTweetPartitionFlatKey, | |
Seq[TopicTweet] | |
], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] | |
def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] | |
def fetch(query: EngineQuery[Query]): Future[Seq[SkitTopicTweet]] | |
def getTweetsForKeys( | |
keys: Seq[TopicTweetPartitionFlatKey], | |
sourceTopic: TopicId | |
): Future[Seq[SkitTopicTweet]] | |
object SkitHighPrecisionTopicTweetSimilarityEngine | |
def fromParams( | |
topicId: TopicId, | |
isVideoOnly: Boolean, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\SkitTopicTweetSimilarityEngine.scala | |
class SkitTopicTweetSimilarityEngine @Inject() ( | |
@Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ | |
TopicTweetPartitionFlatKey, | |
Seq[TopicTweet] | |
], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] | |
def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] | |
def fetch(query: EngineQuery[Query]): Future[Seq[SkitTopicTweet]] | |
def getTweetsForKeys( | |
keys: Seq[TopicTweetPartitionFlatKey], | |
sourceTopic: TopicId | |
): Future[Seq[SkitTopicTweet]] | |
object SkitTopicTweetSimilarityEngine | |
class Query( | |
topicId: TopicId, | |
maxCandidates: Int, | |
maxTweetAge: Duration, | |
semanticCoreVersionId: Long) | |
case class SkitTopicTweet( | |
sourceTopic: TopicId, | |
tweetId: TweetId, | |
favCount: Long, | |
cosineSimilarityScore: Double) | |
def fromParams( | |
topicId: TopicId, | |
isVideoOnly: Boolean, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\StandardSimilarityEngine.scala | |
class EngineQuery[Query]( | |
storeQuery: Query, | |
params: Params, | |
) | |
/** | |
* A straight forward SimilarityEngine implementation that wraps a ReadableStore | |
* | |
* @param implementingStore Provides the candidate retrieval's implementations | |
* @param memCacheConfig If specified, it will wrap the underlying store with a MemCache layer | |
* You should only enable this for cacheable queries, e.x. TweetIds. | |
* consumer based UserIds are generally not possible to cache. | |
* @tparam Query ReadableStore's input type | |
* @tparam Candidate ReadableStore's return type is Seq[[[Candidate]]] | |
*/ | |
class StandardSimilarityEngine[Query, Candidate <: Serializable]( | |
implementingStore: ReadableStore[Query, Seq[Candidate]], | |
override val identifier: SimilarityEngineType, | |
globalStats: StatsReceiver, | |
engineConfig: SimilarityEngineConfig, | |
memCacheConfig: Option[MemCacheConfig[Query]] = None) | |
extends SimilarityEngine[EngineQuery[Query], Candidate] | |
def getScopedStats: StatsReceiver = scopedStats | |
// Add memcache wrapper, if specified | |
private val store | |
def getCandidates( | |
engineQuery: EngineQuery[Query] | |
): Future[Option[Seq[Candidate]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedQigSimilarityEngine.scala | |
class TweetBasedQigSimilarityEngine( | |
qigRanker: QigRanker.MethodPerEndpoint, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
TweetBasedQigSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: TweetBasedQigSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
def getQigSimilarTweetsRequest( | |
tweetId: Long | |
): QigRankerRequest | |
def getCandidatesFromQigResponse( | |
qigSimilarTweetsResponse: QigRankerResponse | |
): Option[Seq[TweetWithScore]] | |
object TweetBasedQigSimilarityEngine | |
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo | |
class Query(sourceId: InternalId) | |
def fromParams( | |
sourceId: InternalId, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedUnifiedSimilarityEngine.scala | |
class TweetBasedUnifiedSimilarityEngine( | |
@Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) | |
tweetBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedUserTweetGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) | |
tweetBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedUserVideoGraphSimilarityEngine.Query, | |
TweetWithScore | |
], | |
simClustersANNSimilarityEngine: StandardSimilarityEngine[ | |
SimClustersANNSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.TweetBasedQigSimilarityEngine) | |
tweetBasedQigSimilarTweetsSimilarityEngine: StandardSimilarityEngine[ | |
TweetBasedQigSimilarityEngine.Query, | |
TweetWithScore | |
], | |
@Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) | |
tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
TweetBasedUnifiedSimilarityEngine.Query, | |
Seq[TweetWithCandidateGenerationInfo] | |
] | |
def get( | |
query: Query | |
): Future[Option[Seq[TweetWithCandidateGenerationInfo]]] | |
def simClustersCandidateMinScoreFilter( | |
simClustersAnnCandidates: Seq[TweetWithScore], | |
simClustersMinScore: Double, | |
simClustersANNConfigId: String | |
): Seq[TweetWithScore] | |
def tweetAgeFilter( | |
candidates: Seq[TweetWithScore], | |
maxTweetAgeHours: Duration | |
): Seq[TweetWithScore] | |
def twhinFilter( | |
twhinCandidates: Seq[TweetWithScore], | |
twhinMaxTweetAgeHours: Duration, | |
simEngineStats: StatsReceiver | |
): Seq[TweetWithScore] | |
def userTweetGraphFilter( | |
userTweetGraphCandidates: Seq[TweetWithScore] | |
): Seq[TweetWithScore] | |
def userVideoGraphFilter( | |
userVideoGraphCandidates: Seq[TweetWithScore] | |
): Seq[TweetWithScore] | |
def qigSimilarTweetsFilter( | |
qigSimilarTweetsCandidates: Seq[TweetWithScore], | |
qigMaxTweetAgeHours: Duration, | |
qigMaxNumSimilarTweets: Int | |
): Seq[TweetWithScore] | |
def getTweetBasedUnifiedCGInfo( | |
sourceInfoOpt: Option[SourceInfo], | |
unifiedScore: Double, | |
contributingSimilarityEngines: Seq[SimilarityEngineInfo] | |
): CandidateGenerationInfo | |
object TweetBasedUnifiedSimilarityEngine | |
class Query( | |
sourceInfo: SourceInfo, | |
maxCandidateNumPerSourceKey: Int, | |
enableSimClustersANN: Boolean, | |
simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableExperimentalSimClustersANN: Boolean, | |
experimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN1: Boolean, | |
simClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN2: Boolean, | |
simClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN3: Boolean, | |
simClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN5: Boolean, | |
simClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
enableSimClustersANN4: Boolean, | |
simClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], | |
simClustersMinScore: Double, | |
simClustersVideoBasedMinScore: Double, | |
twhinModelId: String, | |
enableTwHIN: Boolean, | |
twhinMaxTweetAgeHours: Duration, | |
qigMaxTweetAgeHours: Duration, | |
qigMaxNumSimilarTweets: Int, | |
enableUtg: Boolean, | |
utgQuery: EngineQuery[TweetBasedUserTweetGraphSimilarityEngine.Query], | |
enableUvg: Boolean, | |
uvgQuery: EngineQuery[TweetBasedUserVideoGraphSimilarityEngine.Query], | |
enableQig: Boolean, | |
qigQuery: EngineQuery[TweetBasedQigSimilarityEngine.Query], | |
params: configapi.Params) | |
def fromParams( | |
sourceInfo: SourceInfo, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
def fromParamsForRelatedTweet( | |
internalId: InternalId, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
def fromParamsForRelatedVideoTweet( | |
internalId: InternalId, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedUserAdGraphSimilarityEngine.scala | |
class TweetBasedUserAdGraphSimilarityEngine( | |
userAdGraphService: UserAdGraph.MethodPerEndpoint, | |
tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
TweetBasedUserAdGraphSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: TweetBasedUserAdGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
def getCandidates( | |
tweetId: TweetId, | |
query: TweetBasedUserAdGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object TweetBasedUserAdGraphSimilarityEngine | |
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo | |
def toTweetWithScore( | |
relatedAdResponseFut: Future[Option[RelatedAdResponse]] | |
): Future[Option[Seq[TweetWithScore]]] | |
class Query( | |
sourceId: InternalId, | |
maxResults: Int, | |
minCooccurrence: Int, | |
consumersBasedMinScore: Double, | |
maxTweetAgeInHours: Int, | |
maxConsumerSeedsNum: Int, | |
) | |
def fromParams( | |
sourceId: InternalId, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedUserTweetGraphSimilarityEngine.scala | |
class TweetBasedUserTweetGraphSimilarityEngine( | |
userTweetGraphService: UserTweetGraph.MethodPerEndpoint, | |
tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
TweetBasedUserTweetGraphSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: TweetBasedUserTweetGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
def getCandidates( | |
tweetId: TweetId, | |
query: TweetBasedUserTweetGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
def getCoverageExpansionCandidates( | |
tweetId: TweetId, | |
query: TweetBasedUserTweetGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object TweetBasedUserTweetGraphSimilarityEngine | |
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo | |
def toTweetWithScore( | |
relatedTweetResponseFut: Future[Option[RelatedTweetResponse]] | |
): Future[Option[Seq[TweetWithScore]]] | |
def isOldTweet(tweetId: TweetId): Boolean | |
class Query( | |
sourceId: InternalId, | |
maxResults: Int, | |
minCooccurrence: Int, | |
tweetBasedMinScore: Double, | |
consumersBasedMinScore: Double, | |
maxTweetAgeInHours: Int, | |
maxConsumerSeedsNum: Int, | |
enableCoverageExpansionOldTweet: Boolean, | |
enableCoverageExpansionAllTweet: Boolean, | |
) | |
def fromParams( | |
sourceId: InternalId, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TweetBasedUserVideoGraphSimilarityEngine.scala | |
class TweetBasedUserVideoGraphSimilarityEngine( | |
userVideoGraphService: UserVideoGraph.MethodPerEndpoint, | |
tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
TweetBasedUserVideoGraphSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: TweetBasedUserVideoGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
def getCandidates( | |
tweetId: TweetId, | |
query: TweetBasedUserVideoGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
def getCoverageExpansionCandidates( | |
tweetId: TweetId, | |
query: TweetBasedUserVideoGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object TweetBasedUserVideoGraphSimilarityEngine | |
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo | |
def toTweetWithScore( | |
relatedTweetResponseFut: Future[Option[RelatedTweetResponse]] | |
): Future[Option[Seq[TweetWithScore]]] | |
def isOldTweet(tweetId: TweetId): Boolean | |
class Query( | |
sourceId: InternalId, | |
maxResults: Int, | |
minCooccurrence: Int, | |
tweetBasedMinScore: Double, | |
consumersBasedMinScore: Double, | |
maxTweetAgeInHours: Int, | |
maxConsumerSeedsNum: Int, | |
enableCoverageExpansionOldTweet: Boolean, | |
enableCoverageExpansionAllTweet: Boolean) | |
def fromParams( | |
sourceId: InternalId, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\TwhinCollabFilterSimilarityEngine.scala | |
class TwhinCollabFilterSimilarityEngine( | |
twhinCandidatesStratoStore: ReadableStore[Long, Seq[TweetId]], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
TwhinCollabFilterSimilarityEngine.Query, | |
Seq[TweetWithScore] | |
] | |
def get( | |
query: TwhinCollabFilterSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScore]]] | |
object TwhinCollabFilterSimilarityEngine | |
class TwhinCollabFilterView(clusterVersion: String) | |
case class Query( | |
sourceId: InternalId, | |
) | |
def toSimilarityEngineInfo( | |
query: LookupEngineQuery[Query], | |
score: Double | |
): SimilarityEngineInfo | |
def fromParams( | |
sourceId: InternalId, | |
modelId: String, | |
params: configapi.Params, | |
): LookupEngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\similarity_engine\UserTweetEntityGraphSimilarityEngine.scala | |
class UserTweetEntityGraphSimilarityEngine( | |
userTweetEntityGraph: UserTweetEntityGraph.MethodPerEndpoint, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[ | |
UserTweetEntityGraphSimilarityEngine.Query, | |
Seq[TweetWithScoreAndSocialProof] | |
] | |
def get( | |
query: UserTweetEntityGraphSimilarityEngine.Query | |
): Future[Option[Seq[TweetWithScoreAndSocialProof]]] | |
object UserTweetEntityGraphSimilarityEngine | |
def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo | |
class Query( | |
userId: UserId, | |
seedsWithWeights: Map[UserId, Double], | |
excludedTweetIds: Option[Seq[Long]] = None, | |
maxUtegCandidates: Int, | |
maxTweetAge: Duration, | |
socialProofTypes: Option[Seq[SocialProofType]]) | |
def fromParams( | |
userId: UserId, | |
seedsWithWeights: Map[UserId, Double], | |
excludedTweetIds: Option[Seq[TweetId]] = None, | |
params: configapi.Params, | |
): EngineQuery[Query] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\FrsSourceGraphFetcher.scala | |
class FrsSourceGraphFetcher @Inject() ( | |
@Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]], | |
override val timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver) | |
extends SourceGraphFetcher | |
def isEnabled(query: FetcherQuery): Boolean | |
def fetchAndProcess( | |
query: FetcherQuery, | |
): Future[Option[GraphSourceInfo]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\FrsSourceSignalFetcher.scala | |
class FrsSourceSignalFetcher @Inject() ( | |
@Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]], | |
override val timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver) | |
extends SourceSignalFetcher | |
def isEnabled(query: FetcherQuery): Boolean | |
def fetchAndProcess(query: FetcherQuery): Future[Option[Seq[SourceInfo]]] | |
def convertSourceInfo( | |
sourceType: SourceType, | |
signals: Seq[SignalConvertType] | |
): Seq[SourceInfo] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\FrsStore.scala | |
class FrsStore( | |
frsClient: FollowRecommendationsThriftService.MethodPerEndpoint, | |
statsReceiver: StatsReceiver, | |
decider: CrMixerDecider) | |
extends ReadableStore[Query, Seq[FrsQueryResult]] | |
def get( | |
query: Query | |
): Future[Option[Seq[FrsQueryResult]]] | |
def buildFollowRecommendationRequest( | |
query: Query | |
): RecommendationRequest | |
object FrsStore | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\RealGraphInSourceGraphFetcher.scala | |
class RealGraphInSourceGraphFetcher @Inject() ( | |
@Named(ModuleNames.RealGraphInStore) realGraphStoreMh: ReadableStore[UserId, CandidateSeq], | |
override val timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver) | |
extends SourceGraphFetcher | |
def isEnabled(query: FetcherQuery): Boolean | |
def fetchAndProcess( | |
query: FetcherQuery, | |
): Future[Option[GraphSourceInfo]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\RealGraphOonSourceGraphFetcher.scala | |
class RealGraphOonSourceGraphFetcher @Inject() ( | |
@Named(ModuleNames.RealGraphOonStore) realGraphOonStore: ReadableStore[UserId, CandidateSeq], | |
override val timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver) | |
extends SourceGraphFetcher | |
def isEnabled(query: FetcherQuery): Boolean | |
def fetchAndProcess( | |
query: FetcherQuery, | |
): Future[Option[GraphSourceInfo]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\SourceFetcher.scala | |
trait which, given a [[FetcherQuery]], returns [[ResultType]] | |
* The main purposes of a SourceFetcher is to provide a consistent interface for source fetch | |
* logic, and provides default functions, including: | |
* - Identification | |
* - Observability | |
* - Timeout settings | |
* - Exception Handling | |
*/ | |
trait SourceFetcher[ResultType] extends ReadableStore[FetcherQuery, ResultType] with Logging | |
def identifier: String = this.getClass.getSimpleName | |
protected def stats: StatsReceiver | |
protected def timeoutConfig: TimeoutConfig | |
/*** | |
* Use FeatureSwitch to decide if a specific source is enabled. | |
*/ | |
def isEnabled(query: FetcherQuery): Boolean | |
/*** | |
* This function fetches the raw sources and process them. | |
* Custom stats tracking can be added depending on the type of ResultType | |
*/ | |
def fetchAndProcess( | |
query: FetcherQuery, | |
): Future[Option[ResultType]] | |
/*** | |
* Side-effect function to track stats for signal fetching and processing. | |
*/ | |
def trackStats( | |
query: FetcherQuery | |
)( | |
func: => Future[Option[ResultType]] | |
): Future[Option[ResultType]] | |
/*** | |
* This function is called by the top level class to fetch sources. It executes the pipeline to | |
* fetch raw data, process and transform the sources. Exceptions, Stats, and timeout control are | |
* handled here. | |
*/ | |
override def get( | |
query: FetcherQuery | |
): Future[Option[ResultType]] | |
object SourceFetcher | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\SourceGraphFetcher.scala | |
trait that extends from `SourceFetcher` | |
* and is specialized in tackling User Graph (eg., RealGraphOon, FRS) fetch. | |
* | |
* The [[ResultType]] of a SourceGraphFetcher is a `GraphSourceInfo` which contains a userSeedSet. | |
* When we pass in userId, the underlying store returns one GraphSourceInfo. | |
*/ | |
trait SourceGraphFetcher extends SourceFetcher[GraphSourceInfo] | |
def graphSourceType: SourceType | |
/*** | |
* RawDataType contains a consumers seed UserId and a score (weight) | |
*/ | |
protected type RawDataType = (UserId, Double) | |
def trackStats( | |
query: FetcherQuery | |
)( | |
func: => Future[Option[GraphSourceInfo]] | |
): Future[Option[GraphSourceInfo]] | |
def trackPerItemStats( | |
query: FetcherQuery | |
)( | |
func: => Future[Option[Seq[RawDataType]]] | |
): Future[Option[Seq[RawDataType]]] | |
def convertGraphSourceInfo( | |
userWithScores: Seq[RawDataType] | |
): GraphSourceInfo | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\SourceInfoRouter.scala | |
class SourceInfoRouter @Inject() ( | |
ussSourceSignalFetcher: UssSourceSignalFetcher, | |
frsSourceSignalFetcher: FrsSourceSignalFetcher, | |
frsSourceGraphFetcher: FrsSourceGraphFetcher, | |
realGraphOonSourceGraphFetcher: RealGraphOonSourceGraphFetcher, | |
realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher, | |
) | |
def get( | |
userId: UserId, | |
product: TProduct, | |
userState: UserState, | |
params: configapi.Params | |
): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] | |
def getSourceSignals( | |
fetcherQuery: FetcherQuery | |
): Future[Set[SourceInfo]] | |
def getSourceGraphs( | |
fetcherQuery: FetcherQuery | |
): Future[Map[String, Option[GraphSourceInfo]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\SourceSignalFetcher.scala | |
trait that extends from `SourceFetcher` | |
* and is specialized in tackling Signals (eg., USS, FRS) fetch. | |
* Currently, we define Signals as (but not limited to) a set of past engagements that | |
* the user makes, such as RecentFav, RecentFollow, etc. | |
* | |
* The [[ResultType]] of a SourceSignalFetcher is `Seq[SourceInfo]`. When we pass in userId, | |
* the underlying store returns a list of signals. | |
*/ | |
trait SourceSignalFetcher extends SourceFetcher[Seq[SourceInfo]] | |
def trackStats( | |
query: FetcherQuery | |
)( | |
func: => Future[Option[Seq[SourceInfo]]] | |
): Future[Option[Seq[SourceInfo]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\UssSourceSignalFetcher.scala | |
class UssSourceSignalFetcher @Inject() ( | |
@Named(ModuleNames.UssStore) ussStore: ReadableStore[UssStore.Query, Seq[ | |
(SignalType, Seq[UssSignal]) | |
]], | |
override val timeoutConfig: TimeoutConfig, | |
globalStats: StatsReceiver) | |
extends SourceSignalFetcher | |
def isEnabled(query: FetcherQuery): Boolean = true | |
override def fetchAndProcess( | |
query: FetcherQuery, | |
): Future[Option[Seq[SourceInfo]]] | |
def convertSourceInfo( | |
sourceType: SourceType, | |
signals: Seq[SignalConvertType] | |
): Seq[SourceInfo] | |
def trackUssSignalStatsPerSignalType( | |
query: FetcherQuery, | |
signalType: SignalType, | |
ussSignals: Seq[UssSignal] | |
): Unit | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\source_signal\UssStore.scala | |
class UssStore( | |
stratoStore: ReadableStore[BatchSignalRequest, BatchSignalResponse], | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[Query, Seq[(SignalType, Seq[UssSignal])]] | |
def get(query: Query): Future[Option[Seq[(SignalType, Seq[UssSignal])]]] | |
def buildUserSignalServiceRequests( | |
param: Params, | |
): Seq[SignalRequest] | |
object UssStore | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\CandidateGenerationKeyUtil.scala | |
object CandidateGenerationKeyUtil | |
def toThrift( | |
candidateGenerationInfo: CandidateGenerationInfo, | |
requestUserId: UserId | |
): CandidateGenerationKey | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\CountWeightedInterleaveUtil.scala | |
object CountWeightedInterleaveUtil | |
class GroupingKey( | |
sourceInfoOpt: Option[SourceInfo], | |
similarityEngineTypeOpt: Option[SimilarityEngineType], | |
modelIdOpt: Option[String], | |
authorIdOpt: Option[Long], | |
groupIdOpt: Option[Int]) | |
/** | |
* Converts candidates to grouping key based upon the feature that we interleave with. | |
*/ | |
def toGroupingKey[CandidateType <: Candidate]( | |
candidate: CandidateType, | |
interleaveFeature: Option[BlendGroupingMethodEnum.Value], | |
groupId: Option[Int], | |
): GroupingKey | |
def calculateWeightsKeyByFeature[CandidateType <: Candidate]( | |
candidateSeqKeyByFeature: Map[GroupingKey, Seq[CandidateType]], | |
rankerWeightShrinkage: Double | |
): Map[GroupingKey, Double] | |
def buildRankedCandidatesWithWeightKeyByFeature( | |
rankedCandidateSeq: Seq[RankedCandidate], | |
rankerWeightShrinkage: Double, | |
interleaveFeature: BlendGroupingMethodEnum.Value | |
): Seq[(Seq[RankedCandidate], Double)] | |
def buildInitialCandidatesWithWeightKeyByFeature( | |
initialCandidatesSeqSeq: Seq[Seq[InitialCandidate]], | |
rankerWeightShrinkage: Double, | |
): Seq[(Seq[InitialCandidate], Double)] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\EarlybirdSearchUtil.scala | |
object EarlybirdSearchUtil | |
def GetCollectorTerminationParams( | |
maxNumHitsPerShard: Int, | |
processingTimeout: Duration | |
): Option[CollectorTerminationParams] | |
def GetEarlybirdQuery( | |
beforeTweetIdExclusive: Option[TweetId], | |
afterTweetIdExclusive: Option[TweetId], | |
excludedTweetIds: Set[TweetId], | |
filterOutRetweetsAndReplies: Boolean | |
): Option[EbQuery] = | |
CreateConjunction( | |
Seq( | |
CreateRangeQuery(beforeTweetIdExclusive, afterTweetIdExclusive), | |
CreateExcludedTweetIdsQuery(excludedTweetIds), | |
CreateTweetTypesFilters(filterOutRetweetsAndReplies) | |
).flatten) | |
def CreateRangeQuery( | |
beforeTweetIdExclusive: Option[TweetId], | |
afterTweetIdExclusive: Option[TweetId] | |
): Option[EbQuery] | |
def CreateTweetTypesFilters(filterOutRetweetsAndReplies: Boolean): Option[EbQuery] | |
def CreateConjunction(clauses: Seq[EbQuery]): Option[EbQuery] | |
def CreateExcludedTweetIdsQuery(tweetIds: Set[TweetId]): Option[EbQuery] | |
def GetNamedDisjunctions(excludedTweetIds: Set[TweetId]): Option[Map[String, Seq[Long]]] = | |
if (excludedTweetIds.nonEmpty) | |
createNamedDisjunctionsExcludedTweetIds(excludedTweetIds) | |
else None | |
val EXCLUDE_TWEET_IDS = "exclude_tweet_ids" | |
private def createNamedDisjunctionsExcludedTweetIds( | |
tweetIds: Set[TweetId] | |
): Option[Map[String, Seq[Long]]] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\InterleaveUtil.scala | |
object InterleaveUtil | |
def interleave[CandidateType <: Candidate]( | |
candidates: Seq[Seq[CandidateType]] | |
): Seq[CandidateType] | |
def weightedInterleave[CandidateType <: Candidate]( | |
candidatesAndWeight: Seq[(Seq[CandidateType], Double)], | |
maxWeightAdjustments: Int = 0 | |
): Seq[CandidateType] | |
def buildCandidatesKeyByCGInfo( | |
candidates: Seq[RankedCandidate], | |
): Seq[Seq[RankedCandidate]] | |
class GroupingKey( | |
sourceInfoOpt: Option[SourceInfo], | |
similarityEngineType: SimilarityEngineType, | |
modelId: Option[String]) | |
object GroupingKey | |
def toGroupingKey(candidateGenerationInfo: CandidateGenerationInfo): GroupingKey | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\MetricTagUtil.scala | |
object MetricTagUtil | |
def buildMetricTags(candidate: RankedCandidate): Seq[MetricTag] | |
def toMetricTagFromSource(sourceType: SourceType): Option[MetricTag] | |
def toMetricTagFromSimilarityEngine( | |
seInfo: SimilarityEngineInfo, | |
cseInfo: Seq[SimilarityEngineInfo] | |
): Seq[Option[MetricTag]] | |
def toMetricTagFromModelId( | |
modelId: String | |
): Option[MetricTag] | |
def toMetricTagFromSourceAndSimilarityEngine( | |
sourceInfo: SourceInfo, | |
seInfo: SimilarityEngineInfo | |
): Option[MetricTag] | |
def isFromInterestedIn(candidate: RankedCandidate): Set[MetricTag] | |
.\cr-mixer\server\src\main\scala\com\twitter\cr_mixer\util\SignalTimestampStatsUtil.scala | |
class SignalTimestampStatsUtil @Inject() (statsReceiver: StatsReceiver) | |
def statsSignalTimestamp( | |
tweets: Seq[TweetRecommendation], | |
): Seq[TweetRecommendation] | |
object SignalTimestampStatsUtil | |
def buildLatestSourceSignalTimestamp(candidate: RankedCandidate): Option[Long] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\CandidateSourceRegistry.scala | |
trait CandidateSourceRegistry[Target, Candidate] | |
def sources: Set[CandidateSource[Target, Candidate]] | |
final lazy val candidateSources: Map[ | |
CandidateSourceIdentifier, | |
CandidateSource[Target, Candidate] | |
] | |
def select( | |
identifiers: Set[CandidateSourceIdentifier] | |
): Set[CandidateSource[Target, Candidate]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\EnrichedCandidateSource.scala | |
class EnrichedCandidateSource[Target, Candidate](original: CandidateSource[Target, Candidate]) | |
def gate(predicate: Predicate[Target]): CandidateSource[Target, Candidate] | |
def observe(statsReceiver: StatsReceiver): CandidateSource[Target, Candidate] | |
def apply(target: Target): Stitch[Seq[Candidate]] | |
def stitchMapKey[Target2]( | |
targetMapper: Target2 => Stitch[Option[Target]] | |
): CandidateSource[Target2, Candidate] | |
def stitchMapKeys[Target2]( | |
targetMapper: Target2 => Stitch[Seq[Target]] | |
): CandidateSource[Target2, Candidate] | |
def apply(target: Target2): Stitch[Seq[Candidate]] | |
def mapKeys[Target2]( | |
targetMapper: Target2 => Seq[Target] | |
): CandidateSource[Target2, Candidate] | |
def mapValues[Candidate2]( | |
candidateMapper: Candidate => Stitch[Option[Candidate2]] | |
): CandidateSource[Target, Candidate2] | |
def apply(target: Target): Stitch[Seq[Candidate2]] | |
def mapValue[Candidate2]( | |
candidateMapper: Candidate => Candidate2 | |
): CandidateSource[Target, Candidate2] | |
def within( | |
candidateTimeout: Duration, | |
statsReceiver: StatsReceiver | |
): CandidateSource[Target, Candidate] | |
def apply(target: Target): Stitch[Seq[Candidate]] | |
def failOpenWithin( | |
candidateTimeout: Duration, | |
statsReceiver: StatsReceiver | |
): CandidateSource[Target, Candidate] | |
def apply(target: Target): Stitch[Seq[Candidate]] | |
object EnrichedCandidateSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\ParamPredicate.scala | |
class ParamPredicate[Request <: HasParams](param: Param[Boolean]) extends Predicate[Request] | |
def apply(request: Request): Stitch[PredicateResult] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\Predicate.scala | |
trait Predicate[-Q] | |
def apply(item: Q): Stitch[PredicateResult] | |
def arrow: Arrow[Q, PredicateResult] = Arrow.apply(apply) | |
def map[K](mapper: K => Q): Predicate[K] = Predicate(arrow.contramap(mapper)) | |
/** | |
* check the predicate results for a batch of items for convenience. | |
* | |
* mark it as final to avoid potential abuse usage | |
*/ | |
final def batch(items: Seq[Q]): Stitch[Seq[PredicateResult]] | |
def apply[Q1, Q2](item1: Q1, item2: Q2)(implicit ev: ((Q1, Q2)) => Q): Stitch[PredicateResult] | |
object that represents the logical AND of both predicates | |
*/ | |
def andThen[Q1 <: Q](p: Predicate[Q1]): Predicate[Q1] | |
object that represents the logical OR of both predicates. | |
* if both are invalid, the reason would be the set of all invalid reasons. | |
*/ | |
def or[Q1 <: Q](p: Predicate[Q1]): Predicate[Q1] | |
def gate[Q1 <: Q](gatingPredicate: Predicate[Q1]): Predicate[Q1] | |
def observe(statsReceiver: StatsReceiver): Predicate[Q] = Predicate( | |
StatsUtil.profilePredicateResult(this.arrow, statsReceiver)) | |
def convertToFailOpenWithResultType(resultType: PredicateResult): Predicate[Q] | |
class TruePredicate[Q] extends Predicate[Q] | |
def apply(item: Q): Stitch[PredicateResult] = Predicate.AlwaysTrueStitch | |
} | |
class FalsePredicate[Q](reason: FilterReason) extends Predicate[Q] | |
def apply(item: Q): Stitch[PredicateResult] = InvalidResult | |
} | |
object Predicate | |
def apply[Q](func: Q => Stitch[PredicateResult]): Predicate[Q] = new Predicate[Q] | |
def apply(item: Q): Stitch[PredicateResult] = func(item) | |
override val arrow: Arrow[Q, PredicateResult] = Arrow(func) | |
} | |
def apply[Q](outerArrow: Arrow[Q, PredicateResult]): Predicate[Q] = new Predicate[Q] | |
def apply(item: Q): Stitch[PredicateResult] = arrow(item) | |
override val arrow: Arrow[Q, PredicateResult] = outerArrow | |
} | |
/** | |
* Given some items, this function | |
* 1. chunks them up in groups | |
* 2. lazily applies a predicate on each group | |
* 3. filters based on the predicate | |
* 4. takes first numToTake items. | |
* | |
* If numToTake is satisfied, then any later predicates are not called. | |
* | |
* @param items items of type Q | |
* @param predicate predicate that determines whether an item is acceptable | |
* @param batchSize batch size to call the predicate with | |
* @param numToTake max number of items to return | |
* @param stats stats receiver | |
* @tparam Q type of item | |
* | |
* @return a future of K items | |
*/ | |
def batchFilterTake[Q]( | |
items: Seq[Q], | |
predicate: Predicate[Q], | |
batchSize: Int, | |
numToTake: Int, | |
stats: StatsReceiver | |
): Stitch[Seq[Q]] | |
def take( | |
input: Iterator[Stitch[Seq[Q]]], | |
prev: Seq[Q], | |
takeSize: Int, | |
numOfBatch: Int | |
): Stitch[(Seq[Q], Int)] | |
def filter[Q](items: Seq[Q], predicate: Predicate[Q]): Stitch[Seq[Q]] | |
def filter[T, Q](target: T, items: Seq[Q], predicate: Predicate[(T, Q)]): Stitch[Seq[Q]] | |
object that is the logical "and" of the input predicates | |
*/ | |
def andConcurrently[Q](predicates: Seq[Predicate[Q]]): Predicate[Q] | |
abstract class GatedPredicateBase[Q]( | |
underlyingPredicate: Predicate[Q], | |
stats: StatsReceiver = NullStatsReceiver) | |
extends Predicate[Q] | |
def gate(item: Q): Boolean | |
val underlyingPredicateTotal = stats.counter("underlying_total") | |
val underlyingPredicateValid = stats.counter("underlying_valid") | |
val underlyingPredicateInvalid = stats.counter("underlying_invalid") | |
val notGatedCounter = stats.counter("not_gated") | |
val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) | |
override def apply(item: Q): Stitch[PredicateResult] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\PredicateResult.scala | |
trait PredicateResult | |
def value: Boolean | |
} | |
object PredicateResult | |
object Valid extends PredicateResult | |
class Invalid(reasons: Set[FilterReason] = Set.empty[FilterReason]) extends PredicateResult | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\Ranker.scala | |
trait Ranker[Target, Candidate] extends Transform[Target, Candidate] | |
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] | |
override def transform(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] | |
def observe(statsReceiver: StatsReceiver): Ranker[Target, Candidate] | |
def rank(target: Target, items: Seq[Candidate]): Stitch[Seq[Candidate]] | |
def reverse: Ranker[Target, Candidate] = new Ranker[Target, Candidate] | |
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = | |
ranker.rank(target, candidates).map(_.reverse) | |
} | |
def andThen(other: Ranker[Target, Candidate]): Ranker[Target, Candidate] | |
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] | |
def within(timeout: Duration, statsReceiver: StatsReceiver): Ranker[Target, Candidate] | |
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] | |
object Ranker | |
def chain[Target, Candidate]( | |
transformer: Transform[Target, Candidate], | |
ranker: Ranker[Target, Candidate] | |
): Ranker[Target, Candidate] | |
def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] | |
class IdentityRanker[Target, Candidate] extends Ranker[Target, Candidate] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\RecommendationFlow.scala | |
class RecommendationResultsConfig(desiredCandidateCount: Int, batchForCandidatesCheck: Int) | |
trait BaseRecommendationFlow[Target, Candidate <: UniversalNoun[Long]] | |
def process( | |
pipelineRequest: Target | |
): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] | |
def mapKey[Target2](fn: Target2 => Target): BaseRecommendationFlow[Target2, Candidate] | |
def process( | |
pipelineRequest: Target2 | |
): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] = | |
original.process(fn(pipelineRequest)) | |
} | |
} | |
} | |
/** | |
* Defines a typical recommendation flow to fetch, filter, rank and transform candidates. | |
* | |
* 1. targetEligibility: determine the eligibility of target request | |
* 2. candidateSources: fetch candidates from candidate sources based on target type | |
* 3. preRankerCandidateFilter: light filtering of candidates | |
* 4. ranker: ranking of candidates (could be composed of multiple stages, light ranking, heavy ranking and etc) | |
* 5. postRankerTransform: deduping, grouping, rule based promotion / demotions and etc | |
* 6. validateCandidates: heavy filters to determine the eligibility of the candidates. | |
* will only be applied to candidates that we expect to return. | |
* 7. transformResults: transform the individual candidates into desired format (e.g. hydrate social proof) | |
* | |
* Note that the actual implementations may not need to implement all the steps if not needed | |
* (could just leave to IdentityRanker if ranking is not needed). | |
* | |
* Theoretically, the actual implementation could override the above flow to add | |
* more steps (e.g. add a transform step before ranking). | |
* But it is recommended to add the additional steps into this base flow if the step proves | |
* to have significant justification, or merge it into an existing step if it is a minor change. | |
* | |
* @tparam Target type of target request | |
* @tparam Candidate type of candidate to return | |
*/ | |
trait RecommendationFlow[Target, Candidate <: UniversalNoun[Long]] | |
extends BaseRecommendationFlow[Target, Candidate] | |
with SideEffectsUtil[Target, Candidate] | |
def updateTarget(target: Target): Stitch[Target] = Stitch.value(target) | |
/** | |
* check if the target is eligible for the flow | |
*/ | |
protected def targetEligibility: Predicate[Target] | |
/** | |
* define the candidate sources that should be used for the given target | |
*/ | |
protected def candidateSources(target: Target): Seq[CandidateSource[Target, Candidate]] | |
/** | |
* filter invalid candidates before the ranking phase. | |
*/ | |
protected def preRankerCandidateFilter: Predicate[(Target, Candidate)] | |
/** | |
* rank the candidates | |
*/ | |
protected def selectRanker(target: Target): Ranker[Target, Candidate] | |
/** | |
* transform the candidates after ranking (e.g. dedupping, grouping and etc) | |
*/ | |
protected def postRankerTransform: Transform[Target, Candidate] | |
/** | |
* filter invalid candidates before returning the results. | |
* | |
* Some heavy filters e.g. SGS filter could be applied in this step | |
*/ | |
protected def validateCandidates: Predicate[(Target, Candidate)] | |
/** | |
* transform the candidates into results and return | |
*/ | |
protected def transformResults: Transform[Target, Candidate] | |
/** | |
* configuration for recommendation results | |
*/ | |
protected def resultsConfig(target: Target): RecommendationResultsConfig | |
/** | |
* track the quality factor the recommendation pipeline | |
*/ | |
protected def qualityFactorObserver: Option[QualityFactorObserver] = None | |
def statsReceiver: StatsReceiver | |
/** | |
* high level monitoring for the whole flow | |
* (make sure to add monitoring for each individual component by yourself) | |
* | |
* additional candidates: count, stats, non_empty_count | |
* target eligibility: latency, success, failures, request, count, valid_count, invalid_count, invalid_reasons | |
* candidate generation: latency, success, failures, request, count, non_empty_count, results_stat | |
* pre ranker filter: latency, success, failures, request, count, non_empty_count, results_stat | |
* ranker: latency, success, failures, request, count, non_empty_count, results_stat | |
* post ranker: latency, success, failures, request, count, non_empty_count, results_stat | |
* filter and take: latency, success, failures, request, count, non_empty_count, results_stat, batch count | |
* transform results: latency, success, failures, request, count, non_empty_count, results_stat | |
*/ | |
import RecommendationFlow._ | |
lazy val additionalCandidatesStats = statsReceiver.scope(AdditionalCandidatesStats) | |
lazy val targetEligibilityStats = statsReceiver.scope(TargetEligibilityStats) | |
lazy val candidateGenerationStats = statsReceiver.scope(CandidateGenerationStats) | |
lazy val preRankerFilterStats = statsReceiver.scope(PreRankerFilterStats) | |
lazy val rankerStats = statsReceiver.scope(RankerStats) | |
lazy val postRankerTransformStats = statsReceiver.scope(PostRankerTransformStats) | |
lazy val filterAndTakeStats = statsReceiver.scope(FilterAndTakeStats) | |
lazy val transformResultsStats = statsReceiver.scope(TransformResultsStats) | |
lazy val overallStats = statsReceiver.scope(OverallStats) | |
import StatsUtil._ | |
override def process( | |
pipelineRequest: Target | |
): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] | |
def processValidTarget( | |
target: Target, | |
additionalCandidates: Seq[Candidate] | |
): Stitch[Seq[Candidate]] | |
def take( | |
target: Target, | |
candidates: Seq[Candidate], | |
config: RecommendationResultsConfig | |
): Stitch[Seq[Candidate]] | |
object RecommendationFlow | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\SideEffectsUtil.scala | |
trait SideEffectsUtil[Target, Candidate] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\StatsUtil.scala | |
object StatsUtil | |
def profileStitch[T](stitch: Stitch[T], stat: StatsReceiver): Stitch[T] | |
def profileArrow[T, U](arrow: Arrow[T, U], stat: StatsReceiver): Arrow[T, U] | |
def profileResults[T](results: T, stat: StatsReceiver, size: T => Int): T | |
def profileSeqResults[T](results: Seq[T], stat: StatsReceiver): Seq[T] | |
def profileStitchResults[T](stitch: Stitch[T], stat: StatsReceiver, size: T => Int): Stitch[T] | |
def profileArrowResults[T, U]( | |
arrow: Arrow[T, U], | |
stat: StatsReceiver, | |
size: U => Int | |
): Arrow[T, U] | |
def profileStitchSeqResults[T](stitch: Stitch[Seq[T]], stat: StatsReceiver): Stitch[Seq[T]] | |
def profileStitchOptionalResults[T]( | |
stitch: Stitch[Option[T]], | |
stat: StatsReceiver | |
): Stitch[Option[T]] | |
def profileStitchMapResults[K, V]( | |
stitch: Stitch[Map[K, V]], | |
stat: StatsReceiver | |
): Stitch[Map[K, V]] | |
def getCleanClassName(obj: Object): String = | |
obj.getClass.getSimpleName.stripSuffix("$") | |
/** | |
* Helper function for timing a stitch and count a list of PredicateResult | |
*/ | |
def profilePredicateResults( | |
predicateResult: Stitch[Seq[PredicateResult]], | |
statsReceiver: StatsReceiver | |
): Stitch[Seq[PredicateResult]] | |
def profilePredicateResult( | |
predicateResult: Stitch[PredicateResult], | |
statsReceiver: StatsReceiver | |
): Stitch[PredicateResult] | |
def profilePredicateResults[Q]( | |
predicateResult: Arrow[Q, Seq[PredicateResult]], | |
statsReceiver: StatsReceiver | |
): Arrow[Q, Seq[PredicateResult]] | |
def profilePredicateResult[Q]( | |
predicateResult: Arrow[Q, PredicateResult], | |
statsReceiver: StatsReceiver | |
): Arrow[Q, PredicateResult] | |
def profileStitchSeqResults[T]( | |
stats: StatsReceiver | |
)( | |
block: => Stitch[Seq[T]] | |
): Stitch[Seq[T]] | |
def profileStitch[A](stat: Stat, unit: TimeUnit)(f: => Stitch[A]): Stitch[A] | |
def observeStitchQualityFactor[T]( | |
stitch: Stitch[T], | |
qualityFactorObserverOption: Option[QualityFactorObserver], | |
statsReceiver: StatsReceiver | |
): Stitch[T] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\base\Transform.scala | |
trait Transform[-T, C] | |
def transformItem(target: T, item: C): Stitch[C] | |
def transform(target: T, items: Seq[C]): Stitch[Seq[C]] | |
def mapTarget[T2](mapper: T2 => T): Transform[T2, C] | |
def transformItem(target: T2, item: C): Stitch[C] | |
def transform(target: T2, items: Seq[C]): Stitch[Seq[C]] | |
def andThen[T1 <: T](other: Transform[T1, C]): Transform[T1, C] | |
def transformItem(target: T1, item: C): Stitch[C] = | |
original.transformItem(target, item).flatMap(other.transformItem(target, _)) | |
override def transform(target: T1, items: Seq[C]): Stitch[Seq[C]] = | |
original.transform(target, items).flatMap(other.transform(target, _)) | |
} | |
} | |
def observe(statsReceiver: StatsReceiver): Transform[T, C] | |
def transform(target: T, items: Seq[C]): Stitch[Seq[C]] | |
def transformItem(target: T, item: C): Stitch[C] | |
trait GatedTransform[T <: HasParams, C] extends Transform[T, C] | |
def gated(param: Param[Boolean]): Transform[T, C] | |
object Transform | |
class IdentityTransform[T, C] extends Transform[T, C] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\AddressBookParams.scala | |
object AddressBookParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\ForwardEmailBookSource.scala | |
class ForwardEmailBookSource @Inject() ( | |
forwardEmailBookClientColumn: ForwardEmailBookClientColumn, | |
addressBookClient: AddressbookClient, | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
def apply( | |
target: HasParams with HasClientContext | |
): Stitch[Seq[CandidateUser]] | |
object ForwardEmailBookSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\ForwardPhoneBookSource.scala | |
class ForwardPhoneBookSource @Inject() ( | |
forwardPhoneContactsClientColumn: ForwardPhoneContactsClientColumn, | |
addressBookClient: AddressbookClient, | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] | |
object ForwardPhoneBookSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\ReverseEmailBookSource.scala | |
class ReverseEmailBookSource @Inject() ( | |
reverseEmailContactsClientColumn: ReverseEmailContactsClientColumn, | |
essClient: EmailStorageServiceClient, | |
addressBookClient: AddressbookClient, | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] | |
object ReverseEmailBookSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\addressbook\ReversePhoneBookSource.scala | |
class ReversePhoneBookSource @Inject() ( | |
reversePhoneContactsClientColumn: ReversePhoneContactsClientColumn, | |
pssClient: PhoneStorageServiceClient, | |
addressBookClient: AddressbookClient, | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] | |
object ReversePhoneBookSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\CachedCandidateSource.scala | |
class CachedCandidateSource[K <: Object, V <: Object]( | |
candidateSource: CandidateSource[K, V], | |
maxCacheSize: Int, | |
cacheTTL: Duration, | |
statsReceiver: StatsReceiver, | |
override val identifier: CandidateSourceIdentifier) | |
extends CandidateSource[K, V] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\ExperimentalCandidateSource.scala | |
class ExperimentalCandidateSource[T <: HasParams, V]( | |
baseSource: CandidateSource[T, V], | |
darkreadAlgorithmParam: Param[Boolean], | |
keepCandidatesParam: Param[Boolean], | |
resultCountThresholdParam: Param[Int], | |
baseStatsReceiver: StatsReceiver) | |
extends CandidateSource[T, V] | |
def apply(request: T): Stitch[Seq[V]] | |
def fetchFromCandidateSourceAndProcessResults(request: T): Stitch[Seq[V]] | |
def processResults(results: Seq[V], keepResults: Boolean): Seq[V] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\RealGraphExpansionRepository.scala | |
class InterestExpansionCandidate( | |
userID: Long, | |
score: Double, | |
features: RawYMBIICandidateFeatures) | |
abstract class RealGraphExpansionRepository[Request]( | |
realgraphExpansionStore: Fetcher[ | |
Long, | |
Unit, | |
CandidatesFollowedV1 | |
], | |
override val identifier: CandidateSourceIdentifier, | |
statsReceiver: StatsReceiver = NullStatsReceiver, | |
maxUnderlyingCandidatesToQuery: Int = 50, | |
maxCandidatesToReturn: Int = 40, | |
overrideUnderlyingTimeout: Option[Duration] = None, | |
appendSocialProof: Boolean = false) | |
extends CandidateSource[ | |
Request, | |
CandidateUser | |
] | |
def apply( | |
request: Request, | |
): Stitch[Seq[CandidateUser]] | |
def rerankCandidateExpansions( | |
underlyingCandidatesMap: Map[Long, Double], | |
expansionCandidateMap: Map[Long, Option[CandidatesFollowedV1]] | |
): Seq[InterestExpansionCandidate] | |
object RealGraphExpansionRepository | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SimilarUserExpanderParams.scala | |
object SimilarUserExpanderParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SimilarUserExpanderRepository.scala | |
class SecondDegreeCandidate(userId: Long, score: Double, socialProof: Option[Seq[Long]]) | |
abstract class SimilarUserExpanderRepository[-Request <: HasParams]( | |
override val identifier: CandidateSourceIdentifier, | |
similarToCandidatesFetcher: Fetcher[ | |
Long, | |
Unit, | |
Candidates | |
], | |
expansionInputSizeParam: FSBoundedParam[Int] = DefaultExpansionInputCount, | |
candidatesReturnedSizeParam: FSBoundedParam[Int] = DefaultFinalCandidatesReturnedCount, | |
enableImplicitEngagedExpansion: FSParam[Boolean] = DefaultEnableImplicitEngagedExpansion, | |
thresholdToAvoidExpansion: Int = 30, | |
maxExpansionPerCandidate: Option[Int] = None, | |
includingOriginalCandidates: Boolean = false, | |
scorer: (Double, Double) => Double = SimilarUserExpanderRepository.DefaultScorer, | |
aggregator: (Seq[Double]) => Double = ScoreAggregator.Max, | |
candidateBuilder: (Long, CandidateSourceIdentifier, Double, CandidateUser) => CandidateUser = | |
DefaultCandidateBuilder) | |
extends TwoHopExpansionCandidateSource[ | |
Request, | |
CandidateUser, | |
SecondDegreeCandidate, | |
CandidateUser | |
] | |
def firstDegreeNodes(request: Request): Stitch[Seq[CandidateUser]] | |
def secondaryDegreeNodes( | |
request: Request, | |
firstDegreeCandidate: CandidateUser | |
): Stitch[Seq[SecondDegreeCandidate]] | |
def aggregateAndScore( | |
req: Request, | |
firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SecondDegreeCandidate]] | |
): Stitch[Seq[CandidateUser]] | |
def aggregateCandidateSourceDetails( | |
candidates: Seq[CandidateUser] | |
): Option[UserCandidateSourceDetails] | |
def aggregateAccountSocialProof(candidates: Seq[CandidateUser]): Option[Reason] | |
def getCandidatesAfterImplicitEngagementFiltering( | |
params: Params, | |
firstDegreeCandidatesStitch: Stitch[Seq[CandidateUser]] | |
): Stitch[Seq[CandidateUser]] | |
object SimilarUserExpanderRepository | |
object ScoreAggregator | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SocialProofEnforcedCandidateSource.scala | |
abstract class SocialProofEnforcedCandidateSource( | |
candidateSource: CandidateSource[HasClientContext with HasParams, CandidateUser], | |
modifySocialProof: ModifySocialProof, | |
minNumSocialProofsRequired: Int, | |
override val identifier: CandidateSourceIdentifier, | |
baseStatsReceiver: StatsReceiver) | |
extends CandidateSource[HasClientContext with HasParams, CandidateUser] | |
def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SocialProofEnforcedCandidateSourceFSConfig.scala | |
class SocialProofEnforcedCandidateSourceFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\SocialProofEnforcedCandidateSourceParams.scala | |
object SocialProofEnforcedCandidateSourceParams | |
object MustCallSgs | |
extends FSParam[Boolean]("social_proof_enforced_candidate_source_must_call_sgs", true) | |
case object CallSgsCachedColumn | |
extends FSParam[Boolean]( | |
"social_proof_enforced_candidate_source_call_sgs_cached_column", | |
false) | |
case object QueryIntersectionIdsNum | |
extends FSBoundedParam[Int]( | |
name = "social_proof_enforced_candidate_source_query_intersection_ids_num", | |
default = 3, | |
min = 0, | |
max = Integer.MAX_VALUE) | |
case object MaxNumCandidatesToAnnotate | |
extends FSBoundedParam[Int]( | |
name = "social_proof_enforced_candidate_source_max_num_candidates_to_annotate", | |
default = 50, | |
min = 0, | |
max = Integer.MAX_VALUE) | |
case object GfsIntersectionIdsNum | |
extends FSBoundedParam[Int]( | |
name = "social_proof_enforced_candidate_source_gfs_intersection_ids_num", | |
default = 3, | |
min = 0, | |
max = Integer.MAX_VALUE) | |
case object SgsIntersectionIdsNum | |
extends FSBoundedParam[Int]( | |
name = "social_proof_enforced_candidate_source_sgs_intersection_ids_num", | |
default = 10, | |
min = 0, | |
max = Integer.MAX_VALUE) | |
case object GfsLagDurationInDays | |
extends FSBoundedParam[Duration]( | |
name = "social_proof_enforced_candidate_source_gfs_lag_duration_in_days", | |
default = 14.days, | |
min = 1.days, | |
max = 60.days) | |
with HasDurationConversion | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\StratoFetcherSource.scala | |
abstract class StratoFetcherSource[K, U, V]( | |
fetcher: Fetcher[K, U, V], | |
view: U, | |
override val identifier: CandidateSourceIdentifier) | |
extends CandidateSource[K, CandidateUser] | |
def map(user: K, v: V): Seq[CandidateUser] | |
override def apply(target: K): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\TweetAuthorsCandidateSource.scala | |
trait for tweet authors based algorithms, e.g. topical tweet authors, twistly, ... | |
* | |
* @tparam Target target type | |
* @tparam Candidate output candidate types | |
*/ | |
trait TweetAuthorsCandidateSource[-Target, +Candidate] extends CandidateSource[Target, Candidate] | |
def getTweetCandidates(target: Target): Stitch[Seq[TweetCandidate]] | |
/** | |
* fetch authorId | |
*/ | |
def getTweetAuthorId(tweetCandidate: TweetCandidate): Stitch[Option[Long]] | |
/** | |
* wrap candidate ID and TweetAuthorProof in Candidate | |
*/ | |
def toCandidate(authorId: Long, tweetIds: Seq[Long], score: Option[Double]): Candidate | |
/** | |
* aggregate scores, default to the first score | |
*/ | |
def aggregator(scores: Seq[Double]): Double = | |
scores.headOption.getOrElse(TweetAuthorsCandidateSource.DefaultScore) | |
/** | |
* aggregation method for a group of tweet candidates | |
*/ | |
def aggregateAndScore( | |
target: Target, | |
tweetCandidates: Seq[TweetCandidate] | |
): Seq[Candidate] | |
/** | |
* generate a list of candidates for the target | |
*/ | |
def build( | |
target: Target | |
): Stitch[Seq[Candidate]] | |
def apply(target: Target): Stitch[Seq[Candidate]] = | |
build(target) | |
} | |
object TweetAuthorsCandidateSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\base\TwoHopExpansionCandidateSource.scala | |
trait for two-hop expansion based algorithms, e.g. online_stp, phonebook_prediction, | |
* recent following sims, recent engagement sims, ... | |
* | |
* @tparam Target target type | |
* @tparam FirstDegree type of first degree nodes | |
* @tparam SecondaryDegree type of secondary degree nodes | |
* @tparam Candidate output candidate types | |
*/ | |
trait TwoHopExpansionCandidateSource[-Target, FirstDegree, SecondaryDegree, +Candidate] | |
extends CandidateSource[Target, Candidate] | |
def firstDegreeNodes(req: Target): Stitch[Seq[FirstDegree]] | |
/** | |
* fetch secondary degree nodes given request and first degree nodes | |
*/ | |
def secondaryDegreeNodes(req: Target, node: FirstDegree): Stitch[Seq[SecondaryDegree]] | |
/** | |
* aggregate and score the candidates to generate final results | |
*/ | |
def aggregateAndScore( | |
req: Target, | |
firstDegreeToSecondDegreeNodesMap: Map[FirstDegree, Seq[SecondaryDegree]] | |
): Stitch[Seq[Candidate]] | |
/** | |
* Generate a list of candidates for the target | |
*/ | |
def apply(target: Target): Stitch[Seq[Candidate]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\crowd_search_accounts\CrowdSearchAccountsFSConfig.scala | |
class CrowdSearchAccountsFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\crowd_search_accounts\CrowdSearchAccountsParams.scala | |
object CrowdSearchAccountsParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\crowd_search_accounts\CrowdSearchAccountsSource.scala | |
object AccountsFilteringAndRankingLogicId extends Enumeration | |
object CrowdSearchAccountsSource | |
class CrowdSearchAccountsSource @Inject() ( | |
crowdSearchAccountsClientColumn: CrowdSearchAccountsClientColumn, | |
statsReceiver: StatsReceiver, | |
) extends CandidateSource[CrowdSearchAccountsSource.Target, CandidateUser] | |
with Logging | |
def apply( | |
target: CrowdSearchAccountsSource.Target | |
): Stitch[Seq[CandidateUser]] | |
def transformCrowdSearchAccountsToCandidateSource( | |
crowdSearchAccounts: Seq[Option[CrowdSearchAccounts]] | |
): Seq[CandidateUser] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\BasePopGeoHashSource.scala | |
class BasePopGeohashSource @Inject() ( | |
popGeoSource: CandidateSource[String, CandidateUser], | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[ | |
HasParams with HasClientContext with HasGeohashAndCountryCode, | |
CandidateUser | |
] | |
with BasePopGeohashSourceConfig | |
def apply( | |
target: HasParams with HasClientContext with HasGeohashAndCountryCode | |
): Stitch[Seq[CandidateUser]] | |
trait BasePopGeohashSourceConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopCountryBackFillSource.scala | |
class PopCountryBackFillSource @Inject() (popGeoSource: PopGeoSource) | |
extends CandidateSource[HasClientContext with HasParams, CandidateUser] | |
def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] | |
object PopCountryBackFillSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopCountrySource.scala | |
class PopCountrySource @Inject() ( | |
popGeoSource: PopGeoSource, | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[ | |
HasClientContext with HasParams with HasUserState with HasGeohashAndCountryCode, | |
CandidateUser | |
] | |
def apply( | |
target: HasClientContext with HasParams with HasUserState with HasGeohashAndCountryCode | |
): Stitch[Seq[CandidateUser]] | |
object PopCountrySource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeohashSource.scala | |
class PopGeohashSource @Inject() ( | |
popGeoSource: PopGeoSource, | |
statsReceiver: StatsReceiver) | |
extends BasePopGeohashSource( | |
popGeoSource = popGeoSource, | |
statsReceiver = statsReceiver.scope("PopGeohashSource"), | |
) | |
def candidateSourceEnabled(target: Target): Boolean = true | |
override val identifier: CandidateSourceIdentifier = PopGeohashSource.Identifier | |
override def minGeohashLength(target: Target): Int | |
def maxResults(target: Target): Int | |
def maxGeohashLength(target: Target): Int | |
def returnResultFromAllPrecision(target: Target): Boolean | |
object PopGeohashSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoQualityFollowSource.scala | |
class PopGeohashQualityFollowSource @Inject() ( | |
popGeoSource: PopGeoQualityFollowSource, | |
statsReceiver: StatsReceiver) | |
extends BasePopGeohashSource( | |
popGeoSource = popGeoSource, | |
statsReceiver = statsReceiver.scope("PopGeohashQualityFollowSource"), | |
) | |
def maxResults(target: Target): Int | |
def minGeohashLength(target: Target): Int | |
def maxGeohashLength(target: Target): Int | |
def returnResultFromAllPrecision(target: Target): Boolean | |
def candidateSourceEnabled(target: Target): Boolean | |
object PopGeohashQualityFollowSource | |
object PopGeoQualityFollowSource | |
class PopGeoQualityFollowSource @Inject() ( | |
popGeoQualityFollowClientColumn: UniquePopQualityFollowUsersInPlaceClientColumn, | |
statsReceiver: StatsReceiver, | |
) extends CandidateSource[String, CandidateUser] | |
def apply(target: String): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoQualityFollowSourceFSConfig.scala | |
class PopGeoQualityFollowSourceFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoQualityFollowSourceParams.scala | |
object PopGeoQualityFollowSourceParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoSource.scala | |
class BasePopGeoSource @Inject() ( | |
@Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER) fetcher: Fetcher[ | |
String, | |
Unit, | |
PopUsersInPlace | |
]) extends StratoFetcherWithUnitViewSource[String, PopUsersInPlace]( | |
fetcher, | |
BasePopGeoSource.Identifier) | |
def map(target: String, candidates: PopUsersInPlace): Seq[CandidateUser] = | |
BasePopGeoSource.map(target, candidates) | |
} | |
object BasePopGeoSource | |
def map(target: String, candidates: PopUsersInPlace): Seq[CandidateUser] = | |
candidates.popUsers.sortBy(-_.score).take(BasePopGeoSource.MaxResults).view.map | |
class PopGeoSource @Inject() (basePopGeoSource: BasePopGeoSource, statsReceiver: StatsReceiver) | |
extends CachedCandidateSource[String, CandidateUser]( | |
basePopGeoSource, | |
PopGeoSource.MaxCacheSize, | |
PopGeoSource.CacheTTL, | |
statsReceiver, | |
PopGeoSource.Identifier) | |
object PopGeoSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoSourceFSConfig.scala | |
class PopGeoSourceFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\geo\PopGeoSourceParams.scala | |
object PopGeoSourceParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\ppmi_locale_follow\PPMILocaleFollowSource.scala | |
class PPMILocaleFollowSource @Inject() ( | |
userPreferredLanguagesOnUserClientColumn: UserPreferredLanguagesOnUserClientColumn, | |
localeFollowPpmiClientColumn: LocaleFollowPpmiClientColumn, | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[HasClientContext with HasParams, CandidateUser] | |
def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] | |
def getPPMILocaleFollowCandidates( | |
locales: Seq[String] | |
): Stitch[Seq[CandidateUser]] | |
def getPreferredLocales(userId: Long, countryCode: String): Stitch[Seq[String]] | |
object PPMILocaleFollowSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\ppmi_locale_follow\PPMILocaleFollowSourceFSConfig.scala | |
class PPMILocaleFollowSourceFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\ppmi_locale_follow\PPMILocaleFollowSourceParams.scala | |
class PPMILocaleFollowSourceParams | |
object PPMILocaleFollowSourceParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\promoted_accounts\PromotedAccountsCandidateSource.scala | |
class PromotedCandidateUser( | |
id: Long, | |
position: Int, | |
adImpression: adthrift.AdImpression, | |
followProof: FollowProof, | |
primaryCandidateSource: Option[CandidateSourceIdentifier]) | |
@Singleton | |
class PromotedAccountsCandidateSource @Inject() ( | |
adserverClient: AdserverClient, | |
sgsClient: SocialGraphClient, | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[AdRequest, PromotedCandidateUser] | |
with Logging | |
def apply(request: AdRequest): Stitch[Seq[PromotedCandidateUser]] | |
def shouldShowSocialContext(imp: adthrift.AdImpression): Boolean = | |
imp.experimentValues.exists | |
def getInsertionPositionDefaultValue(isTest: Boolean): Int | |
def profileNumResults(resultsSize: Int, statName: String): Unit | |
object PromotedAccountsCandidateSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\real_graph\RealGraphOonFSConfig.scala | |
class RealGraphOonFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\real_graph\RealGraphOonParams.scala | |
object RealGraphOonParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\real_graph\RealGraphOonV2Source.scala | |
class RealGraphOonV2Source @Inject() ( | |
realGraphClientColumn: UserRealgraphOonV2ClientColumn) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] | |
def parseStratoResults( | |
request: HasParams with HasClientContext, | |
candidateSeqThrift: CandidateSeq | |
): Seq[CandidateUser] | |
object RealGraphOonV2Source | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\real_graph\RealGraphSource.scala | |
class RealGraphSource @Inject() ( | |
realGraph: RealTimeRealGraphClient) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] | |
object RealGraphSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RecentEngagementDirectFollowSource.scala | |
class RecentEngagementDirectFollowSource @Inject() ( | |
realTimeRealGraphClient: RealTimeRealGraphClient) | |
extends CandidateSource[Long, CandidateUser] | |
def apply(targetUserId: Long): Stitch[Seq[CandidateUser]] | |
object RecentEngagementDirectFollowSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RecentEngagementNonDirectFollowSource.scala | |
class RecentEngagementNonDirectFollowSource @Inject() ( | |
realTimeRealGraphClient: RealTimeRealGraphClient) | |
extends CandidateSource[Long, CandidateUser] | |
def apply(targetUserId: Long): Stitch[Seq[CandidateUser]] | |
object RecentEngagementNonDirectFollowSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RepeatedProfileVisitsFSConfig.scala | |
class RepeatedProfileVisitsFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RepeatedProfileVisitsParams.scala | |
object RepeatedProfileVisitsParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\recent_engagement\RepeatedProfileVisitsSource.scala | |
class RepeatedProfileVisitsSource @Inject() ( | |
repeatedProfileVisitsAggregateClientColumn: RepeatedProfileVisitsAggregateClientColumn, | |
realTimeRealGraphClient: RealTimeRealGraphClient, | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
with Logging | |
def applyWithOfflineDataset(targetUserId: Long): Stitch[Map[Long, Int]] | |
def applyWithOnlineData(targetUserId: Long): Stitch[Map[Long, Int]] | |
def getRepeatedVisitedAccounts(params: Params, targetUserId: Long): Stitch[Map[Long, Int]] | |
def getRecommendations(params: Params, userId: Long): Stitch[Seq[CandidateUser]] | |
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] | |
object RepeatedProfileVisitsSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\salsa\RecentEngagementDirectFollowSalsaExpansionSource.scala | |
class RecentEngagementDirectFollowSalsaExpansionSource @Inject() ( | |
realTimeRealGraphClient: RealTimeRealGraphClient, | |
salsaExpander: SalsaExpander) | |
extends SalsaExpansionBasedCandidateSource[Long](salsaExpander) | |
def firstDegreeNodes(target: Long): Stitch[Seq[Long]] = realTimeRealGraphClient | |
.getUsersRecentlyEngagedWith( | |
target, | |
RealTimeRealGraphClient.EngagementScoreMap, | |
includeDirectFollowCandidates = true, | |
includeNonDirectFollowCandidates = false | |
).map | |
def maxResults(target: Long): Int = | |
RecentEngagementDirectFollowSalsaExpansionSource.OutputSize | |
} | |
object RecentEngagementDirectFollowSalsaExpansionSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\salsa\SalsaExpander.scala | |
class SalsaExpandedCandidate( | |
candidateId: Long, | |
numberOfConnections: Int, | |
totalScore: Double, | |
connectingUsers: Seq[Long]) | |
def toCandidateUser: CandidateUser = | |
CandidateUser( | |
id = candidateId, | |
score = Some(totalScore), | |
reason = Some(Reason( | |
Some(AccountProof(followProof = Some(FollowProof(connectingUsers, connectingUsers.size)))))) | |
) | |
} | |
case class SimilarUserCandidate(candidateId: Long, score: Double, similarToCandidate: Long) | |
/** | |
* Salsa expander uses pre-computed lists of candidates for each input user id and returns the highest scored candidates in the pre-computed lists as the expansion for the corresponding input id. | |
*/ | |
@Singleton | |
class SalsaExpander @Inject() ( | |
statsReceiver: StatsReceiver, | |
firstDegreeClient: SalsaFirstDegreeOnUserClientColumn, | |
secondDegreeClient: SalsaSecondDegreeOnUserClientColumn, | |
) | |
def similarUsers( | |
input: Seq[Long], | |
neighbors: Seq[Option[Seq[Candidate]]] | |
): Seq[SalsaExpandedCandidate] | |
def apply( | |
firstDegreeInput: Seq[Long], | |
secondDegreeInput: Seq[Long], | |
maxNumOfCandidatesToReturn: Int | |
): Stitch[Seq[CandidateUser]] | |
object SalsaExpander | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\salsa\SalsaExpansionBasedCandidateSource.scala | |
abstract class SalsaExpansionBasedCandidateSource[Target](salsaExpander: SalsaExpander) | |
extends CandidateSource[Target, CandidateUser] | |
class | |
def firstDegreeNodes(target: Target): Stitch[Seq[Long]] = Stitch.value(Seq()) | |
def secondDegreeNodes(target: Target): Stitch[Seq[Long]] = Stitch.value(Seq()) | |
// max number output results | |
def maxResults(target: Target): Int | |
override def apply(target: Target): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\CacheBasedSimsStore.scala | |
class CacheBasedSimsStore( | |
id: CandidateSourceIdentifier, | |
fetcher: Fetcher[Long, Unit, Candidates], | |
maxCacheSize: Int, | |
cacheTtl: Duration, | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[HasParams with HasSimilarToContext, CandidateUser] | |
def getUsersFromSimsSource(userId: JLong): Stitch[Option[Candidates]] | |
def apply(request: HasParams with HasSimilarToContext): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\DBV2SimsRefreshStore.scala | |
class DBV2SimsRefreshStore @Inject() ( | |
newSimsRefreshOnUserClientColumn: NewSimsRefreshOnUserClientColumn) | |
extends StratoBasedSimsCandidateSourceWithUnitView( | |
fetcher = newSimsRefreshOnUserClientColumn.fetcher, | |
identifier = DBV2SimsRefreshStore.Identifier) | |
@Singleton | |
class CachedDBV2SimsRefreshStore @Inject() ( | |
newSimsRefreshOnUserClientColumn: NewSimsRefreshOnUserClientColumn, | |
statsReceiver: StatsReceiver) | |
extends CacheBasedSimsStore( | |
id = DBV2SimsRefreshStore.Identifier, | |
fetcher = newSimsRefreshOnUserClientColumn.fetcher, | |
maxCacheSize = DBV2SimsRefreshStore.MaxCacheSize, | |
cacheTtl = DBV2SimsRefreshStore.CacheTTL, | |
statsReceiver = statsReceiver.scope("CachedDBV2SimsRefreshStore", "cache") | |
) | |
object DBV2SimsRefreshStore | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\DBV2SimsStore.scala | |
class DBV2SimsStore @Inject() ( | |
@Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates]) | |
extends StratoBasedSimsCandidateSourceWithUnitView( | |
fetcher, | |
identifier = DBV2SimsStore.Identifier) | |
@Singleton | |
class CachedDBV2SimsStore @Inject() ( | |
@Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates], | |
statsReceiver: StatsReceiver) | |
extends CacheBasedSimsStore( | |
id = DBV2SimsStore.Identifier, | |
fetcher = fetcher, | |
maxCacheSize = DBV2SimsStore.MaxCacheSize, | |
cacheTtl = DBV2SimsStore.CacheTTL, | |
statsReceiver = statsReceiver.scope("CachedDBV2SimsStore", "cache") | |
) | |
object DBV2SimsStore | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\Follow2vecNearestNeighborsStore.scala | |
class LinearRegressionFollow2vecNearestNeighborsStore @Inject() ( | |
linearRegressionFollow2vecNearestNeighborsClientColumn: LinearRegressionFollow2vecNearestNeighborsClientColumn) | |
extends StratoBasedSimsCandidateSource[NearestNeighborParamsType]( | |
Follow2vecNearestNeighborsStore.convertFetcher( | |
linearRegressionFollow2vecNearestNeighborsClientColumn.fetcher), | |
view = Follow2vecNearestNeighborsStore.defaultSearchParams, | |
identifier = Follow2vecNearestNeighborsStore.IdentifierF2vLinearRegression | |
) | |
object Follow2vecNearestNeighborsStore | |
def convertFetcher( | |
fetcher: Fetcher[NearestNeighborKeyType, NearestNeighborParamsType, NearestNeighborValueType] | |
): Fetcher[Long, NearestNeighborParamsType, Candidates] | |
def toCandidates( | |
results: Option[NearestNeighborValueType] | |
): Option[Candidates] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SimsExperimentalStore.scala | |
class SimsExperimentalStore @Inject() ( | |
simsExperimentalOnUserClientColumn: SimilarUsersBySimsExperimentalOnUserClientColumn) | |
extends StratoBasedSimsCandidateSourceWithUnitView( | |
fetcher = simsExperimentalOnUserClientColumn.fetcher, | |
identifier = SimsExperimentalStore.Identifier | |
) | |
@Singleton | |
class CachedSimsExperimentalStore @Inject() ( | |
simsExperimentalOnUserClientColumn: SimilarUsersBySimsExperimentalOnUserClientColumn, | |
statsReceiver: StatsReceiver) | |
extends CacheBasedSimsStore( | |
id = SimsExperimentalStore.Identifier, | |
fetcher = simsExperimentalOnUserClientColumn.fetcher, | |
maxCacheSize = SimsExperimentalStore.MaxCacheSize, | |
cacheTtl = SimsExperimentalStore.CacheTTL, | |
statsReceiver = statsReceiver.scope("CachedSimsExperimentalStore", "cache") | |
) | |
object SimsExperimentalStore | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SimsSourceFSConfig.scala | |
class SimsSourceFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SimsSourceParams.scala | |
object SimsSourceParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SimsStore.scala | |
class SimsStore @Inject() ( | |
@Named(GuiceNamedConstants.SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates]) | |
extends StratoBasedSimsCandidateSourceWithUnitView(fetcher, identifier = SimsStore.Identifier) | |
@Singleton | |
class CachedSimsStore @Inject() ( | |
@Named(GuiceNamedConstants.SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates], | |
statsReceiver: StatsReceiver) | |
extends CacheBasedSimsStore( | |
id = SimsStore.Identifier, | |
fetcher = fetcher, | |
maxCacheSize = SimsStore.MaxCacheSize, | |
cacheTtl = SimsStore.CacheTTL, | |
statsReceiver = statsReceiver.scope("CachedSimsStore", "cache") | |
) | |
object SimsStore | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\StratoBasedSimsCandidateSource.scala | |
abstract class StratoBasedSimsCandidateSource[U]( | |
fetcher: Fetcher[Long, U, Candidates], | |
view: U, | |
override val identifier: CandidateSourceIdentifier) | |
extends StratoFetcherSource[Long, U, Candidates](fetcher, view, identifier) | |
def map(target: Long, candidates: Candidates): Seq[CandidateUser] = | |
StratoBasedSimsCandidateSource.map(target, candidates) | |
} | |
object StratoBasedSimsCandidateSource | |
def map(target: Long, candidates: Candidates): Seq[CandidateUser] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims\SwitchingSimsSource.scala | |
class SwitchingSimsSource @Inject() ( | |
cachedDBV2SimsStore: CachedDBV2SimsStore, | |
cachedDBV2SimsRefreshStore: CachedDBV2SimsRefreshStore, | |
cachedSimsExperimentalStore: CachedSimsExperimentalStore, | |
cachedSimsStore: CachedSimsStore, | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
extends CandidateSource[HasParams with HasSimilarToContext, CandidateUser] | |
def apply(request: HasParams with HasSimilarToContext): Stitch[Seq[CandidateUser]] | |
object SwitchingSimsSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\DBV2SimsExpansionParams.scala | |
object DBV2SimsExpansionParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentEngagementSimilarUsersFSConfig.scala | |
class RecentEngagementSimilarUsersFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentEngagementSimilarUsersParams.scala | |
object RecentEngagementSimilarUsersParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentEngagementSimilarUsersSource.scala | |
class RecentEngagementSimilarUsersSource @Inject() ( | |
realTimeRealGraphClient: RealTimeRealGraphClient, | |
switchingSimsSource: SwitchingSimsSource, | |
statsReceiver: StatsReceiver) | |
extends SimsExpansionBasedCandidateSource[HasClientContext with HasParams]( | |
switchingSimsSource) | |
def maxSecondaryDegreeNodes(req: HasClientContext with HasParams): Int = Int.MaxValue | |
override def maxResults(req: HasClientContext with HasParams): Int = | |
RecentEngagementSimilarUsersSource.MaxResults | |
override val identifier: CandidateSourceIdentifier = RecentEngagementSimilarUsersSource.Identifier | |
private val stats = statsReceiver.scope(identifier.name) | |
private val calibratedScoreCounter = stats.counter("calibrated_scores_counter") | |
override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double | |
def calibrateDivisor(req: HasClientContext with HasParams): Double | |
def calibrateScore( | |
candidateScore: Double, | |
req: HasClientContext with HasParams | |
): Double | |
def firstDegreeNodes( | |
target: HasClientContext with HasParams | |
): Stitch[Seq[CandidateUser]] | |
def aggregateAndScore( | |
request: HasClientContext with HasParams, | |
firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SimilarUser]] | |
): Stitch[Seq[CandidateUser]] | |
object RecentEngagementSimilarUsersSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentFollowingSimilarUsersParams.scala | |
object RecentFollowingSimilarUsersParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentFollowingSimilarUsersSource.scala | |
object RecentFollowingSimilarUsersSource | |
class RecentFollowingSimilarUsersSource @Inject() ( | |
socialGraph: SocialGraphClient, | |
switchingSimsSource: SwitchingSimsSource, | |
statsReceiver: StatsReceiver) | |
extends SimsExpansionBasedCandidateSource[ | |
HasParams with HasRecentFollowedUserIds with HasClientContext | |
](switchingSimsSource) | |
def firstDegreeNodes( | |
request: HasParams with HasRecentFollowedUserIds with HasClientContext | |
): Stitch[Seq[CandidateUser]] | |
def maxSecondaryDegreeNodes( | |
req: HasParams with HasRecentFollowedUserIds with HasClientContext | |
): Int | |
def maxResults( | |
req: HasParams with HasRecentFollowedUserIds with HasClientContext | |
): Int | |
def scoreCandidate(sourceScore: Double, similarToScore: Double): Double | |
def calibrateDivisor( | |
req: HasParams with HasRecentFollowedUserIds with HasClientContext | |
): Double | |
def calibrateScore( | |
candidateScore: Double, | |
req: HasParams with HasRecentFollowedUserIds with HasClientContext | |
): Double | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\RecentStrongEngagementDirectFollowSimilarUsersSource.scala | |
class RecentStrongEngagementDirectFollowSimilarUsersSource @Inject() ( | |
realTimeRealGraphClient: RealTimeRealGraphClient, | |
switchingSimsSource: SwitchingSimsSource) | |
extends SimsExpansionBasedCandidateSource[HasClientContext with HasParams]( | |
switchingSimsSource) | |
def firstDegreeNodes( | |
request: HasClientContext with HasParams | |
): Stitch[Seq[CandidateUser]] = request.getOptionalUserId | |
.map | |
def maxSecondaryDegreeNodes(request: HasClientContext with HasParams): Int = Int.MaxValue | |
override def maxResults(request: HasClientContext with HasParams): Int = | |
RecentStrongEngagementDirectFollowSimilarUsersSource.MaxResults | |
override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double | |
def calibrateDivisor(req: HasClientContext with HasParams): Double = 1.0d | |
} | |
object RecentStrongEngagementDirectFollowSimilarUsersSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\SimsExpansionBasedCandidateSource.scala | |
class SimilarUser(candidateId: Long, similarTo: Long, score: Double) | |
abstract class SimsExpansionBasedCandidateSource[-Target <: HasParams]( | |
switchingSimsSource: SwitchingSimsSource) | |
extends TwoHopExpansionCandidateSource[Target, CandidateUser, SimilarUser, CandidateUser] | |
def maxSecondaryDegreeNodes(req: Target): Int | |
// max number output results | |
def maxResults(req: Target): Int | |
// scorer to score candidate based on first and second degree node scores | |
def scoreCandidate(source: Double, similarToScore: Double): Double | |
def calibrateDivisor(req: Target): Double | |
def calibrateScore(candidateScore: Double, req: Target): Double | |
def secondaryDegreeNodes(req: Target, node: CandidateUser): Stitch[Seq[SimilarUser]] | |
def aggregateAndScore( | |
request: Target, | |
firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SimilarUser]] | |
): Stitch[Seq[CandidateUser]] | |
object SimsExpansionBasedCandidateSource | |
object ScoreAggregator | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\SimsExpansionFSConfig.scala | |
class SimsExpansionFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\sims_expansion\SimsExpansionSourceParams.scala | |
object SimsExpansionSourceParams | |
object Aggregator | |
extends FSEnumParam[SimsExpansionSourceAggregatorId.type]( | |
name = "sims_expansion_aggregator_id", | |
default = SimsExpansionSourceAggregatorId.Sum, | |
enum = SimsExpansionSourceAggregatorId) | |
} | |
object SimsExpansionSourceAggregatorId extends Enumeration | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\socialgraph\RecentFollowingRecentFollowingExpansionSource.scala | |
class RecentFollowingRecentFollowingExpansionSource @Inject() ( | |
socialGraphClient: SocialGraphClient, | |
statsReceiver: StatsReceiver) | |
extends TwoHopExpansionCandidateSource[ | |
HasParams with HasRecentFollowedUserIds, | |
Long, | |
Long, | |
CandidateUser | |
] | |
with Logging | |
def firstDegreeNodes( | |
target: HasParams with HasRecentFollowedUserIds | |
): Stitch[Seq[Long]] = Stitch.value( | |
target.recentFollowedUserIds | |
.getOrElse(Nil).take( | |
RecentFollowingRecentFollowingExpansionSource.NumFirstDegreeNodesToRetrieve) | |
) | |
override def secondaryDegreeNodes( | |
target: HasParams with HasRecentFollowedUserIds, | |
node: Long | |
): Stitch[Seq[Long]] = socialGraphClient | |
.getRecentEdgesCached( | |
RecentEdgesQuery( | |
node, | |
Seq(RelationshipType.Following), | |
Some(RecentFollowingRecentFollowingExpansionSource.NumSecondDegreeNodesToRetrieve)), | |
useCachedStratoColumn = | |
target.params(RecentFollowingRecentFollowingExpansionSourceParams.CallSgsCachedColumn) | |
).map( | |
_.take(RecentFollowingRecentFollowingExpansionSource.NumSecondDegreeNodesToRetrieve)).rescue | |
def aggregateAndScore( | |
target: HasParams with HasRecentFollowedUserIds, | |
firstDegreeToSecondDegreeNodesMap: Map[Long, Seq[Long]] | |
): Stitch[Seq[CandidateUser]] | |
object RecentFollowingRecentFollowingExpansionSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\socialgraph\RecentFollowingRecentFollowingExpansionSourceFSConfig.scala | |
class RecentFollowingRecentFollowingExpansionSourceFSConfig @Inject() () | |
extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\socialgraph\RecentFollowingRecentFollowingExpansionSourceParams.scala | |
object RecentFollowingRecentFollowingExpansionSourceParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\BaseOnlineSTPSource.scala | |
abstract class BaseOnlineSTPSource( | |
stpGraphBuilder: STPGraphBuilder, | |
baseStatsReceiver: StatsReceiver) | |
extends CandidateSource[ | |
HasClientContext with HasParams with HasRecentFollowedUserIds, | |
CandidateUser | |
] | |
with Logging | |
def getCandidates( | |
records: Seq[STPRecord], | |
request: HasClientContext with HasParams with HasRecentFollowedUserIds | |
): Stitch[Seq[CandidateUser]] | |
override def apply( | |
request: HasClientContext with HasParams with HasRecentFollowedUserIds | |
): Stitch[Seq[CandidateUser]] = | |
request.getOptionalUserId | |
.map | |
object BaseOnlineSTPSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\Dbv2StpScorer.scala | |
class Dbv2StpScorer @Inject() ( | |
@Named(GuiceNamedConstants.STP_DBV2_SCORER) tfPredictionEngine: TensorflowPredictionEngine) | |
def getScoredResponse(record: STPRecord): Stitch[Option[Double]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\EpStpScorer.scala | |
class ScoredResponse(score: Double, featuresBreakdown: Option[String] = None) | |
/** | |
* STP ML ranker trained using prehistoric ML framework | |
*/ | |
@Singleton | |
class EpStpScorer @Inject() (epScorer: EPScorer) | |
def getScore(responses: List[EPScoringResponse]): Option[ScoredResponse] = | |
responses.headOption | |
.flatMap | |
def getScoredResponse( | |
record: Record, | |
details: Boolean = false | |
): Stitch[Option[ScoredResponse]] | |
object EpStpScorer | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\MutualFollowStrongTiePredictionSource.scala | |
class MutualFollowStrongTiePredictionSource @Inject() ( | |
sgsClient: SocialGraphClient, | |
strongTiePredictionFeaturesOnUserClientColumn: StrongTiePredictionFeaturesOnUserClientColumn) | |
extends CandidateSource[HasClientContext with HasRecentFollowedUserIds, CandidateUser] | |
def apply( | |
target: HasClientContext with HasRecentFollowedUserIds | |
): Stitch[Seq[CandidateUser]] | |
object MutualFollowStrongTiePredictionSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineMutualFollowExpansionSource.scala | |
class OfflineMutualFollowExpansionSource @Inject() ( | |
column: MutualFollowExpansionClientColumn) | |
extends OfflineStrongTiePredictionBaseSource(column.fetcher) | |
object OfflineMutualFollowExpansionSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStpSourceFsConfig.scala | |
class OfflineStpSourceFsConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStpSourceParams.scala | |
object OfflineStpSourceParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStpSourceWithDensePmiMatrix.scala | |
class OfflineStpSourceWithDensePmiMatrix @Inject() ( | |
stpColumn: PpmiDenseMatrixCandidatesClientColumn) | |
extends OfflineStrongTiePredictionBaseSource(stpColumn.fetcher) | |
object OfflineStpSourceWithDensePmiMatrix | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStpSourceWithLegacyPmiMatrix.scala | |
class OfflineStpSourceWithLegacyPmiMatrix @Inject() ( | |
stpColumn: StrongTiePredictionClientColumn) | |
extends OfflineStrongTiePredictionBaseSource(stpColumn.fetcher) | |
object OfflineStpSourceWithLegacyPmiMatrix | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStrongTiePredictionBaseSource.scala | |
class that all variants of our offline stp dataset can extend. Assumes the same STPResult | |
* value in the key and converts the result into the necessary internal model. | |
*/ | |
abstract class OfflineStrongTiePredictionBaseSource( | |
fetcher: Fetcher[Long, Unit, STPResult]) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
def fetch( | |
target: Long, | |
): Stitch[Seq[CandidateUser]] | |
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] | |
object OfflineStrongTiePredictionBaseSource | |
def map(target: Long, candidates: STPResult): Seq[CandidateUser] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OfflineStrongTiePredictionSource.scala | |
object OfflineStpScore extends Feature[UserCandidate, Option[Double]] | |
/** | |
* Main source for strong-tie-prediction candidates generated offline. | |
*/ | |
@Singleton | |
class OfflineStrongTiePredictionSource @Inject() ( | |
offlineStpSourceWithLegacyPmiMatrix: OfflineStpSourceWithLegacyPmiMatrix, | |
offlineStpSourceWithDensePmiMatrix: OfflineStpSourceWithDensePmiMatrix) | |
extends CandidateSource[HasParams with HasClientContext, CandidateUser] | |
with Logging | |
def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] | |
object OfflineStrongTiePredictionSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceFSConfig.scala | |
class OnlineSTPSourceFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceParams.scala | |
object OnlineSTPSourceParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceScorer.scala | |
class OnlineSTPSourceScorer @Inject() ( | |
onlineSTPSourceWithEPScorer: OnlineSTPSourceWithEPScorer) | |
extends CandidateSource[ | |
HasClientContext with HasParams with HasRecentFollowedUserIds, | |
CandidateUser | |
] | |
def apply( | |
request: HasClientContext with HasParams with HasRecentFollowedUserIds | |
): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceWithDeepbirdV2Scorer.scala | |
class OnlineSTPSourceWithDeepbirdV2Scorer @Inject() ( | |
dbv2StpScorer: Dbv2StpScorer, | |
stpGraphBuilder: STPGraphBuilder, | |
baseStatReceiver: StatsReceiver) | |
extends BaseOnlineSTPSource(stpGraphBuilder, baseStatReceiver) | |
def getCandidates( | |
records: Seq[STPRecord], | |
request: HasClientContext with HasParams with HasRecentFollowedUserIds, | |
): Stitch[Seq[CandidateUser]] | |
object OnlineSTPSourceWithDeepbirdV2Scorer | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\OnlineSTPSourceWithEPScorer.scala | |
class OnlineSTPSourceWithEPScorer @Inject() ( | |
epStpScorer: EpStpScorer, | |
stpGraphBuilder: STPGraphBuilder, | |
baseStatReceiver: StatsReceiver) | |
extends BaseOnlineSTPSource(stpGraphBuilder, baseStatReceiver) | |
with Logging | |
def getCandidates( | |
records: Seq[STPRecord], | |
request: HasClientContext with HasParams with HasRecentFollowedUserIds, | |
): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\SocialProofEnforcedOfflineStrongTiePredictionSource.scala | |
class SocialProofEnforcedOfflineStrongTiePredictionSource @Inject() ( | |
offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, | |
modifySocialProof: ModifySocialProof, | |
statsReceiver: StatsReceiver) | |
extends SocialProofEnforcedCandidateSource( | |
offlineStrongTiePredictionSource, | |
modifySocialProof, | |
SocialProofEnforcedOfflineStrongTiePredictionSource.MinNumSocialProofsRequired, | |
SocialProofEnforcedOfflineStrongTiePredictionSource.Identifier, | |
statsReceiver) | |
object SocialProofEnforcedOfflineStrongTiePredictionSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\STPFirstDegreeFetcher.scala | |
class STPFirstDegreeFetcher @Inject() ( | |
realTimeGraphClient: RealTimeRealGraphClient, | |
reversePhoneBookSource: ReversePhoneBookSource, | |
reverseEmailBookSource: ReverseEmailBookSource, | |
forwardEmailBookSource: ForwardEmailBookSource, | |
forwardPhoneBookSource: ForwardPhoneBookSource, | |
mutualFollowStrongTiePredictionSource: MutualFollowStrongTiePredictionSource, | |
lowTweepCredFollowStore: LowTweepCredFollowStore, | |
timer: Timer, | |
statsReceiver: StatsReceiver) | |
extends Logging | |
def getPotentialFirstEdgesFromFetcher( | |
userId: Long, | |
target: HasClientContext with HasParams with HasRecentFollowedUserIds, | |
algorithm: Algorithm, | |
weight: Double | |
): Stitch[Seq[PotentialFirstDegreeEdge]] | |
def getFirstDegreeEdges( | |
target: HasClientContext with HasParams with HasRecentFollowedUserIds | |
): Stitch[Seq[FirstDegreeEdge]] | |
object STPFirstDegreeFetcher | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\STPGraphBuilder.scala | |
class STPGraphBuilder @Inject() ( | |
stpFirstDegreeFetcher: STPFirstDegreeFetcher, | |
stpSecondDegreeFetcher: STPSecondDegreeFetcher, | |
statsReceiver: StatsReceiver) | |
def apply( | |
target: HasClientContext with HasParams with HasRecentFollowedUserIds | |
): Stitch[STPGraph] = stpFirstDegreeFetcher | |
.getFirstDegreeEdges(target).flatMap | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\stp\STPSecondDegreeFetcher.scala | |
class STPSecondDegreeFetcher @Inject() ( | |
strongTiePredictionFeaturesOnUserClientColumn: StrongTiePredictionFeaturesOnUserClientColumn) | |
def scoreSecondDegreeEdge(edge: SecondDegreeEdge): (Int, Int, Int) | |
def bool2int(b: Boolean): Int = if (b) 1 else 0 | |
( | |
-edge.edgeInfo.numMutualFollowPath, | |
-edge.edgeInfo.numLowTweepcredFollowPath, | |
-(bool2int(edge.edgeInfo.forwardEmailPath) + bool2int(edge.edgeInfo.reverseEmailPath) + | |
bool2int(edge.edgeInfo.forwardPhonePath) + bool2int(edge.edgeInfo.reversePhonePath)) | |
) | |
} | |
// Use each first-degree edge(w/ candidateId) to expand and find mutual follows. | |
// Then, with the mutual follows, group-by candidateId and join edge information | |
// to create secondDegree edges. | |
def getSecondDegreeEdges( | |
target: HasClientContext with HasParams, | |
firstDegreeEdges: Seq[FirstDegreeEdge] | |
): Stitch[Seq[SecondDegreeEdge]] | |
object STPSecondDegreeFetcher | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\top_organic_follows_accounts\TopOrganicFollowsAccountsFSConfig.scala | |
class TopOrganicFollowsAccountsFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\top_organic_follows_accounts\TopOrganicFollowsAccountsParams.scala | |
object TopOrganicFollowsAccountsParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\top_organic_follows_accounts\TopOrganicFollowsAccountsSource.scala | |
object AccountsFilteringAndRankingLogicId extends Enumeration | |
object TopOrganicFollowsAccountsSource | |
class TopOrganicFollowsAccountsSource @Inject() ( | |
organicFollowsAccountsClientColumn: OrganicFollowsAccountsClientColumn, | |
statsReceiver: StatsReceiver, | |
) extends CandidateSource[TopOrganicFollowsAccountsSource.Target, CandidateUser] | |
with Logging | |
def apply( | |
target: TopOrganicFollowsAccountsSource.Target | |
): Stitch[Seq[CandidateUser]] | |
def transformOrganicFollowAccountssToCandidateSource( | |
organicFollowsAccounts: Seq[Option[OrganicFollowsAccounts]] | |
): Seq[CandidateUser] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\triangular_loops\TriangularLoopsFSConfig.scala | |
class TriangularLoopsFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\triangular_loops\TriangularLoopsParams.scala | |
object TriangularLoopsParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\triangular_loops\TriangularLoopsSource.scala | |
class TriangularLoopsSource @Inject() ( | |
triangularLoopsV2Column: TriangularLoopsV2OnUserClientColumn) | |
extends CandidateSource[ | |
HasParams with HasClientContext with HasRecentFollowedByUserIds, | |
CandidateUser | |
] | |
def apply( | |
target: HasParams with HasClientContext with HasRecentFollowedByUserIds | |
): Stitch[Seq[CandidateUser]] | |
def filterOutCandidatesNotFollowingTargetUser( | |
candidatesStitch: Stitch[Seq[CandidateUser]], | |
recentFollowings: Option[Seq[Long]] | |
): Stitch[Seq[CandidateUser]] | |
object TriangularLoopsSource | |
def mapCandidatesToCandidateUsers(candidates: Candidates): Seq[CandidateUser] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\two_hop_random_walk\TwoHopRandomWalkSource.scala | |
class TwoHopRandomWalkSource @Inject() ( | |
@Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER) fetcher: Fetcher[ | |
Long, | |
Unit, | |
TCandidateSeq | |
]) extends StratoFetcherWithUnitViewSource[Long, TCandidateSeq]( | |
fetcher, | |
TwoHopRandomWalkSource.Identifier) | |
def map(targetUserId: Long, tCandidateSeq: TCandidateSeq): Seq[CandidateUser] = | |
TwoHopRandomWalkSource.map(targetUserId, tCandidateSeq) | |
} | |
object TwoHopRandomWalkSource | |
def map(targetUserId: Long, tCandidateSeq: TCandidateSeq): Seq[CandidateUser] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\user_user_graph\UserUserGraphCandidateSource.scala | |
class UserUserGraphCandidateSource @Inject() ( | |
@Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER) | |
fetcher: Fetcher[RecommendUserRequest, Unit, RecommendUserResponse], | |
statsReceiver: StatsReceiver) | |
extends CandidateSource[ | |
UserUserGraphCandidateSource.Target, | |
CandidateUser | |
] | |
def apply( | |
target: UserUserGraphCandidateSource.Target | |
): Stitch[Seq[CandidateUser]] | |
def buildRecommendUserRequest( | |
target: UserUserGraphCandidateSource.Target | |
): Option[RecommendUserRequest] | |
def convertToCandidateUsers( | |
recommendedUser: RecommendedUser | |
): CandidateUser | |
object UserUserGraphCandidateSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\user_user_graph\UserUserGraphFSConfig.scala | |
class UserUserGraphFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\candidate_sources\user_user_graph\UserUserGraphParams.scala | |
object UserUserGraphParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\AddressbookClient.scala | |
class AddressbookClient @Inject() ( | |
addressbookService: Addressbook2.MethodPerEndpoint, | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
def getResponseFromService( | |
identifiers: Seq[RecordIdentifier], | |
batchSize: Int, | |
edgeType: EdgeType, | |
maxFetches: Int, | |
queryOption: Option[QueryOption] | |
): Stitch[Seq[AddressBookGetResponse]] | |
def getContactsResponseFromService( | |
identifiers: Seq[RecordIdentifier], | |
batchSize: Int, | |
edgeType: EdgeType, | |
maxFetches: Int, | |
queryOption: Option[QueryOption] | |
): Stitch[Seq[AddressBookGetResponse]] | |
def getUsers( | |
userId: Long, | |
identifiers: Seq[RecordIdentifier], | |
batchSize: Int, | |
edgeType: EdgeType, | |
fetcherOption: Option[Fetcher[Long, Unit, tUserContacts]] = None, | |
maxFetches: Int = 1, | |
queryOption: Option[QueryOption] = None, | |
): Stitch[Seq[Long]] | |
def getHashedContacts( | |
normalizeFn: String => String, | |
extractField: String, | |
)( | |
userId: Long, | |
identifiers: Seq[RecordIdentifier], | |
batchSize: Int, | |
edgeType: EdgeType, | |
fetcherOption: Option[Fetcher[String, Unit, STPResultFeature]] = None, | |
maxFetches: Int = 1, | |
queryOption: Option[QueryOption] = None, | |
): Stitch[Seq[String]] | |
def getEmailContacts = getHashedContacts(normalizeEmail, "emails") _ | |
def getPhoneContacts = getHashedContacts(normalizePhoneNumber, "phoneNumbers") _ | |
private def getUsersFromManhattan( | |
userId: Long, | |
fetcher: Fetcher[Long, Unit, tUserContacts], | |
): Stitch[Seq[Long]] = fetcher | |
.fetch(userId) | |
.map(_.v.map(_.destinationIds).toSeq.flatten.distinct) | |
private def getContactsFromManhattan( | |
userId: Long, | |
fetcher: Fetcher[String, Unit, STPResultFeature], | |
): Stitch[Seq[String]] = fetcher | |
.fetch(userId.toString) | |
.map(_.v.map(_.strongTieUserFeature.map(_.destId)).toSeq.flatten.distinct) | |
} | |
object AddressbookClient | |
def createQueryOption(edgeType: EdgeType, isPhone: Boolean): Option[QueryOption] = | |
(edgeType, isPhone) match | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\AddressbookModule.scala | |
object AddressbookModule extends BaseClientModule[Addressbook2.MethodPerEndpoint] with MtlsClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\models\Contact.scala | |
class Contact( | |
id: Long, | |
emails: Option[Set[String]], | |
phoneNumbers: Option[Set[String]], | |
firstName: Option[String], | |
lastName: Option[String], | |
name: Option[String], | |
appId: Option[Long], | |
appIds: Option[Set[Long]], | |
importedTimestamp: Option[Time]) | |
object Contact | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\models\EdgeType.scala | |
trait EdgeType | |
def toThrift: t.EdgeType | |
} | |
object EdgeType | |
object Forward extends EdgeType | |
object Reverse extends EdgeType | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\models\QueryOption.scala | |
class QueryOption( | |
onlyDiscoverableInExpansion: Boolean, | |
onlyConfirmedInExpansion: Boolean, | |
onlyDiscoverableInResult: Boolean, | |
onlyConfirmedInResult: Boolean, | |
fetchGlobalApiNamespace: Boolean, | |
isDebugRequest: Boolean, | |
resolveEmails: Boolean, | |
resolvePhoneNumbers: Boolean) | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\addressbook\models\RecordIdentifier.scala | |
class RecordIdentifier( | |
userId: Option[Long], | |
email: Option[String], | |
phoneNumber: Option[String]) | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\adserver\AdRequest.scala | |
class AdRequest( | |
clientContext: ClientContext, | |
displayLocation: DisplayLocation, | |
isTest: Option[Boolean], | |
profileUserId: Option[Long]) | |
def toThrift: t.AdRequestParams | |
object AdRequest | |
class MissingAdDisplayLocation(displayLocation: DisplayLocation) | |
extends Exception( | |
s"Display Location $ | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\adserver\AdserverClient.scala | |
class AdserverClient @Inject() (adserverService: NewAdServer.MethodPerEndpoint) | |
def getAdImpressions(adRequest: AdRequest): Stitch[Seq[t.AdImpression]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\adserver\AdserverModule.scala | |
object AdserverModule extends BaseClientModule[NewAdServer.MethodPerEndpoint] with MtlsClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\cache\MemcacheClient.scala | |
object MemcacheClient | |
def apply[V]( | |
client: Client, | |
dest: String, | |
valueBijection: Bijection[Buf, V], | |
ttl: Duration, | |
statsReceiver: StatsReceiver | |
): MemcacheClient[V] | |
class MemcacheClient[V]( | |
client: Client, | |
dest: String, | |
valueBijection: Bijection[Buf, V], | |
ttl: Duration, | |
statsReceiver: StatsReceiver) | |
def readThrough( | |
key: String, | |
underlyingCall: () => Stitch[V] | |
): Stitch[V] | |
def getIfPresent(key: String): Future[Option[V]] | |
def put(key: String, value: V): Future[Unit] | |
def hashString(input: String): String | |
def profileStitch[T](stitch: Stitch[T], stat: StatsReceiver): Stitch[T] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\cache\MemcacheModule.scala | |
object MemcacheModule extends TwitterModule | |
def provideMemcacheClient( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver, | |
): Client | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\cache\ThriftBijection.scala | |
abstract class ThriftBijection[T <: ThriftStruct] extends Bijection[Buf, T] | |
def apply(b: Buf): T | |
def invert(a: T): Buf | |
abstract class ThriftOptionBijection[T <: ThriftStruct] extends Bijection[Buf, Option[T]] | |
def apply(b: Buf): Option[T] | |
def invert(a: Option[T]): Buf | |
class ThriftEnumBijection[T <: ThriftEnum](constructor: Int => T) extends Bijection[Buf, T] | |
def apply(b: Buf): T | |
def invert(a: T): Buf | |
class ThriftEnumOptionBijection[T <: ThriftEnum](constructor: Int => T) extends Bijection[Buf, Option[T]] | |
def apply(b: Buf): Option[T] | |
def invert(a: Option[T]): Buf | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\common\BaseClientModule.scala | |
abstract class BaseClientModule[T: ClassTag] extends ThriftClientModule[T] | |
def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\deepbirdv2\DeepBirdV2PredictionServiceClientModule.scala | |
object DeepBirdV2PredictionServiceClientModule extends TwitterModule | |
def getDeepbirdPredictionServiceClient( | |
clientId: ClientId, | |
label: String, | |
dest: String, | |
statsReceiver: StatsReceiver, | |
serviceIdentifier: ServiceIdentifier | |
): DeepbirdPredictionService.ServiceToClient | |
def providesWtfProdDeepbirdV2PredictionService( | |
clientId: ClientId, | |
statsReceiver: StatsReceiver, | |
serviceIdentifier: ServiceIdentifier | |
): DeepbirdPredictionService.ServiceToClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\dismiss_store\DismissStore.scala | |
class DismissStore @Inject() ( | |
@Named(GuiceNamedConstants.DISMISS_STORE_SCANNER) | |
scanner: Scanner[(Long, Slice[ | |
(Long, Long) | |
]), Unit, (Long, (Long, Long)), WhoToFollowDismissEventDetails], | |
stats: StatsReceiver) | |
extends Logging | |
def get( | |
userId: Long, | |
negStartTimeMs: Long, | |
maxCandidatesToFetchOption: Option[Int] | |
): Stitch[Seq[Long]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\email_storage_service\EmailStorageServiceClient.scala | |
class EmailStorageServiceClient @Inject() ( | |
val emailStorageService: EmailStorageService.MethodPerEndpoint) | |
def getVerifiedEmail( | |
userId: Long, | |
purposeOfProcessing: PurposeOfProcessing | |
): Stitch[Option[String]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\email_storage_service\EmailStorageServiceModule.scala | |
object EmailStorageServiceModule | |
extends BaseClientModule[EmailStorageService.MethodPerEndpoint] | |
with MtlsClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\geoduck\LocationServiceClient.scala | |
class LocationServiceClient @Inject() (locationService: LocationService.MethodPerEndpoint) | |
def getGeohashAndCountryCode(userId: Long): Stitch[GeohashAndCountryCode] | |
def getGeohashFromTransactionLocation( | |
transactionLocation: TransactionLocation | |
): Option[String] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\geoduck\LocationServiceModule.scala | |
object LocationServiceModule | |
extends BaseClientModule[LocationService.MethodPerEndpoint] | |
with MtlsClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\geoduck\ReverseGeocodeClient.scala | |
class ReverseGeocodeClient @Inject() (rgcService: ReverseGeocoder.MethodPerEndpoint) | |
def getGeohashAndCountryCode(ipAddress: String): Stitch[GeohashAndCountryCode] | |
def getGeohashAndCountryCodeFromLocation(location: Location): GeohashAndCountryCode | |
object ReverseGeocodeClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\geoduck\UserLocationFetcher.scala | |
class UserLocationFetcher @Inject() ( | |
locationServiceClient: LocationServiceClient, | |
reverseGeocodeClient: ReverseGeocodeClient, | |
statsReceiver: StatsReceiver) | |
def getGeohashAndCountryCode( | |
userId: Option[Long], | |
ipAddress: Option[String] | |
): Stitch[Option[GeohashAndCountryCode]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\gizmoduck\GizmoduckClient.scala | |
class GizmoduckClient @Inject() (gizmoduckStitchClient: Gizmoduck, statsReceiver: StatsReceiver) | |
def isProtected(userId: Long): Stitch[Boolean] | |
def getUserName(userId: Long, forUserId: Long): Stitch[Option[String]] | |
object GizmoduckClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\gizmoduck\GizmoduckModule.scala | |
object GizmoduckModule extends BaseClientModule[UserService.MethodPerEndpoint] with MtlsClient | |
def provideExtraGizmoduckQueryFields: Set[QueryFields] = Set.empty | |
@Provides | |
@Singleton | |
def providesStitchClient(futureIface: UserService.MethodPerEndpoint): Gizmoduck | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\graph_feature_service\GraphFeatureServiceClient.scala | |
class GraphFeatureServiceClient @Inject() ( | |
graphFeatureService: GraphFeatureService.MethodPerEndpoint) | |
def getIntersections( | |
userId: Long, | |
candidateIds: Seq[Long], | |
numIntersectionIds: Int | |
): Stitch[Map[Long, FollowProof]] | |
object GraphFeatureServiceClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\graph_feature_service\GraphFeatureStoreModule.scala | |
object GraphFeatureStoreModule | |
extends BaseClientModule[GraphFeatureService.MethodPerEndpoint] | |
with MtlsClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\impression_store\ImpressionStoreModule.scala | |
object ImpressionStoreModule extends TwitterModule | |
def providesImpressionStore(stratoClient: Client): WtfImpressionStore | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\impression_store\WtfImpressionStore.scala | |
class WtfImpressionStore @Inject() ( | |
scanner: Scanner[ | |
((Long, TDisplayLocation), Slice[Long]), | |
Unit, | |
((Long, TDisplayLocation), Long), | |
(Long, Int) | |
]) extends Logging | |
def get(userId: Long, dl: DisplayLocation): Stitch[Seq[WtfImpression]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\interests_service\InterestServiceClient.scala | |
class InterestServiceClient @Inject() ( | |
stratoClient: Client, | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
extends Logging | |
def fetchUttInterestIds( | |
userId: Long | |
): Stitch[Seq[Long]] | |
def extractUttInterest( | |
interestRelationShip: InterestRelationship | |
): Option[Long] | |
def fetchCustomInterests( | |
userId: Long | |
): Stitch[Seq[String]] | |
def extractCustomInterest( | |
interestRelationShip: InterestRelationship | |
): Option[String] | |
def fetchInterestRelationships( | |
userId: Long | |
): Stitch[Option[Seq[InterestRelationship]]] | |
def getInterestRelationship( | |
interestData: UserInterestData | |
): Seq[InterestRelationship] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\phone_storage_service\PhoneStorageServiceClient.scala | |
class PhoneStorageServiceClient @Inject() ( | |
val phoneStorageService: PhoneStorageService.MethodPerEndpoint) | |
def getPhoneNumbers( | |
userId: Long, | |
purposeOfProcessing: PurposeOfProcessing, | |
forceCarrierLookup: Option[Boolean] = None | |
): Stitch[Seq[String]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\phone_storage_service\PhoneStorageServiceModule.scala | |
object PhoneStorageServiceModule | |
extends BaseClientModule[PhoneStorageService.MethodPerEndpoint] | |
with MtlsClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\real_time_real_graph\Engagement.scala | |
trait EngagementType | |
// We do not include SoftFollow since it's deprecated | |
object EngagementType | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\real_time_real_graph\EngagementScorer.scala | |
object EngagementScorer | |
def apply( | |
engagements: Map[Long, Seq[Engagement]], | |
engagementScoreMap: Map[EngagementType, Double], | |
minScore: Double = 0.0 | |
): Seq[(Long, Double, Seq[EngagementType])] | |
def score( | |
engagement: Engagement, | |
now: Time, | |
engagementScoreMap: Map[EngagementType, Double] | |
): Double | |
def getEngagementProof( | |
engagements: Seq[Engagement], | |
engagementScoreMap: Map[EngagementType, Double] | |
): Seq[EngagementType] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\real_time_real_graph\RealTimeRealGraphClient.scala | |
class RealTimeRealGraphClient @Inject() ( | |
timelinesUserVertexOnUserClientColumn: TimelinesUserVertexOnUserClientColumn, | |
realGraphScoresMhOnUserClientColumn: RealGraphScoresMhOnUserClientColumn) | |
def mapUserVertexToEngagementAndFilter(userVertex: UserVertex): Map[Long, Seq[Engagement]] | |
def getRecentProfileViewEngagements(userId: Long): Stitch[Map[Long, Seq[Engagement]]] | |
def getUsersRecentlyEngagedWith( | |
userId: Long, | |
engagementScoreMap: Map[EngagementType, Double], | |
includeDirectFollowCandidates: Boolean, | |
includeNonDirectFollowCandidates: Boolean | |
): Stitch[Seq[CandidateUser]] | |
def getRealGraphWeights(userId: Long): Stitch[Map[Long, Double]] = | |
realGraphScoresMhOnUserClientColumn.fetcher | |
.fetch(userId) | |
.map( | |
_.v | |
.map(_.candidates.map(candidate => (candidate.userId, candidate.score)).toMap) | |
.getOrElse(Map.empty[Long, Double])) | |
} | |
object RealTimeRealGraphClient | |
def toEngagement(interaction: Interaction): Option[Engagement] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\socialgraph\SocialGraphClient.scala | |
class RecentEdgesQuery( | |
userId: Long, | |
relations: Seq[RelationshipType], | |
// prefer to default value to better utilize the caching function of stitch | |
count: Option[Int] = Some(SocialGraphClient.MaxQuerySize), | |
performUnion: Boolean = true, | |
recentEdgesWindowOpt: Option[Duration] = None, | |
targets: Option[Seq[Long]] = None) | |
case class EdgeRequestQuery( | |
userId: Long, | |
relation: RelationshipType, | |
count: Option[Int] = Some(SocialGraphClient.MaxQuerySize), | |
performUnion: Boolean = true, | |
recentEdgesWindowOpt: Option[Duration] = None, | |
targets: Option[Seq[Long]] = None) | |
@Singleton | |
class SocialGraphClient @Inject() ( | |
socialGraph: SocialGraph, | |
idsClientColumn: IdsClientColumn, | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
extends Logging | |
def getRecentEdgesCached( | |
rq: RecentEdgesQuery, | |
useCachedStratoColumn: Boolean = true | |
): Stitch[Seq[Long]] | |
def getRecentEdgesCachedInternal(rq: RecentEdgesQuery): Stitch[Seq[Long]] | |
def getRecentEdgesFromCachedColumn(rq: RecentEdgesQuery): Stitch[Seq[Long]] | |
def getRecentEdges(rq: RecentEdgesQuery): Stitch[Seq[Long]] | |
def getRecentEdgesWithTime(rq: EdgeRequestQuery): Stitch[Seq[UserIdWithTimestamp]] | |
def getEdgeCursor(window: Duration): ByteBuffer | |
def getIntersections( | |
userId: Long, | |
candidateIds: Seq[Long], | |
numIntersectionIds: Int | |
): Stitch[Map[Long, FollowProof]] | |
def getIntersectionsFromCachedColumn( | |
userId: Long, | |
candidateIds: Seq[Long], | |
numIntersectionIds: Int | |
): Stitch[Map[Long, FollowProof]] | |
def getInvalidRelationshipUserIds( | |
userId: Long, | |
maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship | |
): Stitch[Seq[Long]] | |
def getInvalidRelationshipUserIdsFromCachedColumn( | |
userId: Long, | |
maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship | |
): Stitch[Seq[Long]] | |
def getRecentFollowedUserIds(userId: Long): Stitch[Seq[Long]] | |
def getRecentFollowedUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] | |
def getRecentFollowedUserIdsWithTime(userId: Long): Stitch[Seq[UserIdWithTimestamp]] | |
def getRecentFollowedByUserIds(userId: Long): Stitch[Seq[Long]] | |
def getRecentFollowedByUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] | |
def getRecentFollowedUserIdsWithTimeWindow( | |
userId: Long, | |
timeWindow: Duration | |
): Stitch[Seq[Long]] | |
object SocialGraphClient | |
def enablePostRankerSgsPredicate(numInvalidRelationshipUsers: Int): Boolean | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\socialgraph\SocialGraphModule.scala | |
object SocialGraphModule | |
extends BaseClientModule[SocialGraphService.MethodPerEndpoint] | |
with MtlsClient | |
def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client = | |
client.withSessionQualifier.noFailFast | |
@Provides | |
@Singleton | |
def providesStitchClient(futureIface: SocialGraphService.MethodPerEndpoint): SocialGraph | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\strato\StratoClientModule.scala | |
object StratoClientModule extends TwitterModule | |
def stratoClient(serviceIdentifier: ServiceIdentifier): Client | |
def cosineFollowFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = | |
stratoClient.fetcher[Long, Unit, HermitCandidates](CosineFollowPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.COSINE_LIST_FETCHER) | |
def cosineListFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = | |
stratoClient.fetcher[Long, Unit, HermitCandidates](CosineListPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) | |
def curatedBlacklistedAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = | |
stratoClient.fetcher[String, Unit, Seq[Long]](CuratedFilteredAccountsPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.CURATED_CANDIDATES_FETCHER) | |
def curatedCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = | |
stratoClient.fetcher[String, Unit, Seq[Long]](CuratedCandidatesPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER) | |
def popUsersInPlaceFetcher(stratoClient: Client): Fetcher[String, Unit, PopUsersInPlace] = | |
stratoClient.fetcher[String, Unit, PopUsersInPlace](PopUsersInPlacePath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.RELATABLE_ACCOUNTS_FETCHER) | |
def relatableAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, RelatableAccounts] = | |
stratoClient.fetcher[String, Unit, RelatableAccounts](RelatableAccountsPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.PROFILE_SIDEBAR_BLACKLIST_SCANNER) | |
def profileSidebarBlacklistScanner( | |
stratoClient: Client | |
): Scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit] = | |
stratoClient.scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit](ProfileSidebarBlacklistPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.REAL_TIME_INTERACTIONS_FETCHER) | |
def realTimeInteractionsFetcher( | |
stratoClient: Client | |
): Fetcher[(Long, Long), Unit, Seq[Interaction]] = | |
stratoClient.fetcher[(Long, Long), Unit, Seq[Interaction]](RealTimeInteractionsPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.SIMS_FETCHER) | |
def simsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = | |
stratoClient.fetcher[Long, Unit, HermitCandidates](SimsPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) | |
def dbv2SimsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = | |
stratoClient.fetcher[Long, Unit, HermitCandidates](DBV2SimsPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.TRIANGULAR_LOOPS_FETCHER) | |
def triangularLoopsFetcher(stratoClient: Client): Fetcher[Long, Unit, TriangularLoopCandidates] = | |
stratoClient.fetcher[Long, Unit, TriangularLoopCandidates](TriangularLoopsPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER) | |
def twoHopRandomWalkFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateSeq] = | |
stratoClient.fetcher[Long, Unit, CandidateSeq](TwoHopRandomWalkPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) | |
def userRecommendabilityFetcher( | |
stratoClient: Client | |
): Fetcher[Long, Unit, UserRecommendabilityFeatures] = | |
stratoClient.fetcher[Long, Unit, UserRecommendabilityFeatures](UserRecommendabilityPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.USER_STATE_FETCHER) | |
def userStateFetcher(stratoClient: Client): Fetcher[Long, Unit, CondensedUserState] = | |
stratoClient.fetcher[Long, Unit, CondensedUserState](UserStatePath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.UTT_ACCOUNT_RECOMMENDATIONS_FETCHER) | |
def uttAccountRecommendationsFetcher( | |
stratoClient: Client | |
): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] = | |
stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations]( | |
UTTAccountRecommendationsPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.UTT_SEED_ACCOUNTS_FETCHER) | |
def uttSeedAccountRecommendationsFetcher( | |
stratoClient: Client | |
): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] = | |
stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations]( | |
UttSeedAccountsRecommendationPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.ELECTION_CANDIDATES_FETCHER) | |
def electionCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = | |
stratoClient.fetcher[String, Unit, Seq[Long]](ElectionCandidatesPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER) | |
def userUserGraphFetcher( | |
stratoClient: Client | |
): Fetcher[RecommendUserRequest, Unit, RecommendUserResponse] = | |
stratoClient.fetcher[RecommendUserRequest, Unit, RecommendUserResponse](UserUserGraphPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.POST_NUX_WTF_FEATURES_FETCHER) | |
def wtfPostNuxFeaturesFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateFeatures] | |
def extendedNetworkFetcher( | |
stratoClient: Client | |
): Fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal] | |
def dismissStoreScanner( | |
stratoClient: Client | |
): Scanner[ | |
(Long, Slice[(Long, Long)]), | |
Unit, | |
(Long, (Long, Long)), | |
WhoToFollowDismissEventDetails | |
] = | |
stratoClient.scanner[ | |
(Long, Slice[(Long, Long)]), // PKEY: userId, LKEY: (-ts, candidateId) | |
Unit, | |
(Long, (Long, Long)), | |
WhoToFollowDismissEventDetails | |
](WtfDissmissEventsPath) | |
@Provides | |
@Singleton | |
@Named(GuiceNamedConstants.LABELED_NOTIFICATION_FETCHER) | |
def labeledNotificationFetcher( | |
stratoClient: Client | |
): Fetcher[Long, Unit, LatestEvents] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\clients\user_state\UserStateClient.scala | |
class UserStateClient @Inject() ( | |
@Named(GuiceNamedConstants.USER_STATE_FETCHER) userStateFetcher: Fetcher[ | |
Long, | |
Unit, | |
CondensedUserState | |
], | |
client: Client, | |
statsReceiver: StatsReceiver, | |
decider: Decider = Decider.False) | |
def getUserState(userId: Long): Stitch[Option[UserState]] | |
def fetchUserState(userId: JLong): Stitch[Option[UserState]] | |
object UserStateClient | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\constants\CandidateAlgorithmTypeConstants.scala | |
object CandidateAlgorithmTypeConstants | |
def getAlgorithmTypes(algoId: String): Set[String] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\constants\GuiceNamedConstants.scala | |
object GuiceNamedConstants | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\constants\ServiceConstants.scala | |
object ServiceConstants | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\features\LocationFeature.scala | |
object LocationFeature | |
extends FeatureWithDefaultOnFailure[PipelineQuery, Option[GeohashAndCountryCode]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\features\TrackingTokenFeature.scala | |
object TrackingTokenFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Int]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\features\UserStateFeature.scala | |
object UserStateFeature extends Feature[PipelineQuery, Option[UserState]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\adapters\CandidateAlgorithmAdapter.scala | |
object CandidateAlgorithmAdapter | |
extends IRecordOneToOneAdapter[Option[UserCandidateSourceDetails]] | |
def remapCandidateSource(a: Algorithm): Algorithm = a match | |
def adaptToDataRecord( | |
userCandidateSourceDetailsOpt: Option[UserCandidateSourceDetails] | |
): DataRecord | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\adapters\ClientContextAdapter.scala | |
object ClientContextAdapter extends IRecordOneToOneAdapter[(ClientContext, DisplayLocation)] | |
def adaptToDataRecord(target: (ClientContext, DisplayLocation)): DataRecord | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\adapters\PostNuxAlgorithmAdapter.scala | |
object PostNuxAlgorithmIdAdapter extends PostNuxAlgorithmAdapter | |
object PostNuxAlgorithmTypeAdapter extends PostNuxAlgorithmAdapter | |
trait PostNuxAlgorithmAdapter extends IRecordOneToOneAdapter[DataRecord] | |
class TransformedAlgorithmFeatures( | |
ratioLog: Continuous) | |
def getFeatures: Seq[Continuous] = Seq(ratioLog) | |
} | |
private def applyFeatureStorePrefix(feature: Continuous) = new Continuous( | |
s"$FeatureStorePrefix$ | |
def addTransformedFeaturesToDataRecordFunc( | |
originalDr: DataRecord, | |
numImpressions: Double, | |
): (DataRecord, Continuous) => DataRecord | |
def adaptToDataRecord(record: DataRecord): DataRecord | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\adapters\PreFetchedFeatureAdapter.scala | |
object PreFetchedFeatureAdapter | |
extends IRecordOneToOneAdapter[ | |
(HasPreFetchedFeature, CandidateUser) | |
] | |
def adaptToDataRecord( | |
record: (HasPreFetchedFeature, CandidateUser) | |
): DataRecord | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\common\FeatureSource.scala | |
trait FeatureSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\common\FeatureSourceId.scala | |
trait FeatureSourceId | |
object FeatureSourceId | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\common\HasPreFetchedFeature.scala | |
trait HasPreFetchedFeature extends HasMutualFollowedUserIds with HasWtfImpressions | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\CandidateAlgorithmSource.scala | |
class CandidateAlgorithmSource @Inject() (stats: StatsReceiver) extends FeatureSource | |
def hydrateFeatures( | |
t: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, // we don't use the target here | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\ClientContextSource.scala | |
class ClientContextSource() extends FeatureSource | |
def hydrateFeatures( | |
t: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureHydrationSourcesFeatureSwitchKeys.scala | |
object FeatureHydrationSourcesFeatureSwitchKeys | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureHydrationSourcesFSConfig.scala | |
class FeatureHydrationSourcesFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreFeatures.scala | |
object FeatureStoreFeatures | |
object FeatureStoreRawFeatures | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreGizmoduckSource.scala | |
class FeatureStoreGizmoduckSource @Inject() ( | |
serviceIdentifier: ServiceIdentifier, | |
stats: StatsReceiver) | |
extends FeatureSource | |
def id: FeatureSourceId = FeatureSourceId.FeatureStoreGizmoduckSourceId | |
override def featureContext: FeatureContext = getFeatureContext | |
val clientConfig: ClientConfig[HasParams] = ClientConfig( | |
dynamicHydrationConfig = dynamicHydrationConfig, | |
featureStoreParamsConfig = | |
FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), | |
/** | |
* The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` | |
* used below takes effect. | |
*/ | |
timeoutProvider = Function.const(800.millis), | |
serviceIdentifier = serviceIdentifier | |
) | |
private val datasetsToCache = Set( | |
UsersourceEntityDataset | |
).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] | |
private val datasetValuesCache: DatasetValuesCache = | |
DatasetValuesCache( | |
Caffeine | |
.newBuilder() | |
.expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) | |
.maximumSize(DefaultCacheMaxKeys) | |
.build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] | |
.asMap, | |
datasetsToCache, | |
DatasetCacheScope | |
) | |
private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( | |
clientConfig, | |
backupSourceStats, | |
Set(datasetValuesCache) | |
) | |
private val adapter: IRecordOneToOneAdapter[PredictionRecord] = | |
PredictionRecordAdapter.oneToOne( | |
BoundFeatureSet(allFeatures), | |
OnlineFeatureGenerationStats(backupSourceStats) | |
) | |
override def hydrateFeatures( | |
target: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
object FeatureStoreGizmoduckSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreParameters.scala | |
object FeatureStoreParameters | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStorePostNuxAlgorithmSource.scala | |
class FeatureStorePostNuxAlgorithmSource @Inject() ( | |
serviceIdentifier: ServiceIdentifier, | |
stats: StatsReceiver) | |
extends FeatureSource | |
def id: FeatureSourceId = FeatureSourceId.FeatureStorePostNuxAlgorithmSourceId | |
override def featureContext: FeatureContext = getFeatureContext | |
private val dataRecordMerger = new DataRecordMerger | |
val clientConfig: ClientConfig[HasParams] = ClientConfig( | |
dynamicHydrationConfig = dynamicHydrationConfig, | |
featureStoreParamsConfig = | |
FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), | |
/** | |
* The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` | |
* used below takes effect. | |
*/ | |
timeoutProvider = Function.const(800.millis), | |
serviceIdentifier = serviceIdentifier | |
) | |
private val datasetsToCache = Set( | |
PostNuxAlgorithmIdAggregateDataset, | |
PostNuxAlgorithmTypeAggregateDataset, | |
).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] | |
private val datasetValuesCache: DatasetValuesCache = | |
DatasetValuesCache( | |
Caffeine | |
.newBuilder() | |
.expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) | |
.maximumSize(DefaultCacheMaxKeys) | |
.build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] | |
.asMap, | |
datasetsToCache, | |
DatasetCacheScope | |
) | |
private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( | |
clientConfig, | |
backupSourceStats, | |
Set(datasetValuesCache) | |
) | |
private val adapterToDataRecord: IRecordOneToOneAdapter[PredictionRecord] = | |
PredictionRecordAdapter.oneToOne( | |
BoundFeatureSet(allFeatures), | |
OnlineFeatureGenerationStats(backupSourceStats) | |
) | |
// These two calculate the rate for each feature by dividing it by the number of impressions, then | |
// apply a log transformation. | |
private val transformAdapters = Seq(PostNuxAlgorithmIdAdapter, PostNuxAlgorithmTypeAdapter) | |
override def hydrateFeatures( | |
target: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
object FeatureStorePostNuxAlgorithmSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreSource.scala | |
class FeatureStoreSource @Inject() ( | |
serviceIdentifier: ServiceIdentifier, | |
stats: StatsReceiver) | |
extends FeatureSource | |
def hydrateFeatures( | |
target: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
object FeatureStoreSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreSourceParams.scala | |
object FeatureStoreSourceParams | |
object EnableTopicAggregateFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTopicAggregateFeatures, | |
default = true | |
) | |
case object EnableAlgorithmAggregateFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAlgorithmAggregateFeatures, | |
default = false | |
) | |
case object EnableAuthorTopicAggregateFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAuthorTopicAggregateFeatures, | |
default = true | |
) | |
case object EnableUserTopicFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserTopicFeatures, | |
default = false | |
) | |
case object EnableTargetUserFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserFeatures, | |
default = true | |
) | |
case object EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature | |
extends FSParam[Boolean]( | |
name = | |
FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature, | |
default = true | |
) | |
case object EnableTargetUserResurrectionFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserResurrectionFeatures, | |
default = true | |
) | |
case object EnableTargetUserWtfImpressionFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserWtfImpressionFeatures, | |
default = true | |
) | |
case object EnableCandidateUserFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserFeatures, | |
default = true | |
) | |
case object EnableCandidateUserAuthorRealTimeAggregateFeatures | |
extends FSParam[Boolean]( | |
name = | |
FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserAuthorRealTimeAggregateFeatures, | |
default = true | |
) | |
case object EnableCandidateUserResurrectionFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserResurrectionFeatures, | |
default = true | |
) | |
case object EnableCandidateUserTimelinesAuthorAggregateFeatures | |
extends FSParam[Boolean]( | |
name = | |
FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserTimelinesAuthorAggregateFeatures, | |
default = true | |
) | |
case object EnableUserCandidateEdgeFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateEdgeFeatures, | |
default = true | |
) | |
case object EnableUserCandidateWtfImpressionCandidateFeatures | |
extends FSParam[Boolean]( | |
name = | |
FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateWtfImpressionCandidateFeatures, | |
default = true | |
) | |
case object EnableUserWtfAlgEdgeFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserWtfAlgEdgeFeatures, | |
default = false | |
) | |
case object EnableSimilarToUserFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableSimilarToUserFeatures, | |
default = true | |
) | |
case object EnableCandidatePrecomputedNotificationFeatures | |
extends FSParam[Boolean]( | |
name = | |
FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidatePrecomputedNotificationFeatures, | |
default = false | |
) | |
case object EnableCandidateClientFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateClientFeatures, | |
default = false | |
) | |
case object EnableUserClientFeatures | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserClientFeatures, | |
default = false | |
) | |
case object EnableSeparateClientForTimelinesAuthors | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForTimelinesAuthor, | |
default = false | |
) | |
case object EnableSeparateClientForMetricCenterUserCounting | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientMetricCenterUserCounting, | |
default = false | |
) | |
case object EnableSeparateClientForNotifications | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForNotifications, | |
default = false | |
) | |
case object EnableSeparateClientForGizmoduck | |
extends FSParam[Boolean]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForGizmoduck, | |
default = false | |
) | |
case object GlobalFetchTimeout | |
extends FSBoundedParam[Duration]( | |
name = FeatureHydrationSourcesFeatureSwitchKeys.FeatureHydrationTimeout, | |
default = 240.millisecond, | |
min = 100.millisecond, | |
max = 400.millisecond) | |
with HasDurationConversion | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreTimelinesAuthorSource.scala | |
class FeatureStoreTimelinesAuthorSource @Inject() ( | |
serviceIdentifier: ServiceIdentifier, | |
stats: StatsReceiver) | |
extends FeatureSource | |
def id: FeatureSourceId = FeatureSourceId.FeatureStoreTimelinesAuthorSourceId | |
override def featureContext: FeatureContext = getFeatureContext | |
val clientConfig: ClientConfig[HasParams] = ClientConfig( | |
dynamicHydrationConfig = dynamicHydrationConfig, | |
featureStoreParamsConfig = | |
FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), | |
/** | |
* The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` | |
* used below takes effect. | |
*/ | |
timeoutProvider = Function.const(800.millis), | |
serviceIdentifier = serviceIdentifier | |
) | |
private val datasetsToCache = Set( | |
AuthorFeaturesEntityDataset | |
).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] | |
private val datasetValuesCache: DatasetValuesCache = | |
DatasetValuesCache( | |
Caffeine | |
.newBuilder() | |
.expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) | |
.maximumSize(DefaultCacheMaxKeys) | |
.build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] | |
.asMap, | |
datasetsToCache, | |
DatasetCacheScope | |
) | |
private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( | |
clientConfig, | |
backupSourceStats, | |
Set(datasetValuesCache) | |
) | |
private val adapter: IRecordOneToOneAdapter[PredictionRecord] = | |
PredictionRecordAdapter.oneToOne( | |
BoundFeatureSet(allFeatures), | |
OnlineFeatureGenerationStats(backupSourceStats) | |
) | |
override def hydrateFeatures( | |
target: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
object FeatureStoreTimelinesAuthorSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\FeatureStoreUserMetricCountsSource.scala | |
class FeatureStoreUserMetricCountsSource @Inject() ( | |
serviceIdentifier: ServiceIdentifier, | |
stats: StatsReceiver) | |
extends FeatureSource | |
def id: FeatureSourceId = FeatureSourceId.FeatureStoreUserMetricCountsSourceId | |
override def featureContext: FeatureContext = getFeatureContext | |
val clientConfig: ClientConfig[HasParams] = ClientConfig( | |
dynamicHydrationConfig = dynamicHydrationConfig, | |
featureStoreParamsConfig = | |
FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), | |
/** | |
* The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` | |
* used below takes effect. | |
*/ | |
timeoutProvider = Function.const(800.millis), | |
serviceIdentifier = serviceIdentifier | |
) | |
private val datasetsToCache = Set( | |
MetricCenterUserCountingFeaturesDataset | |
).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] | |
private val datasetValuesCache: DatasetValuesCache = | |
DatasetValuesCache( | |
Caffeine | |
.newBuilder() | |
.expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) | |
.maximumSize(DefaultCacheMaxKeys) | |
.build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] | |
.asMap, | |
datasetsToCache, | |
DatasetCacheScope | |
) | |
private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( | |
clientConfig, | |
backupSourceStats, | |
Set(datasetValuesCache) | |
) | |
private val adapter: IRecordOneToOneAdapter[PredictionRecord] = | |
PredictionRecordAdapter.oneToOne( | |
BoundFeatureSet(allFeatures), | |
OnlineFeatureGenerationStats(backupSourceStats) | |
) | |
override def hydrateFeatures( | |
target: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
object FeatureStoreUserMetricCountsSource | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\HydrationSourcesModule.scala | |
object HydrationSourcesModule extends TwitterModule | |
def providesAthenaManhattanClient( | |
serviceIdentifier: ServiceIdentifier | |
): ManhattanKVEndpoint | |
def timelinesAuthorStitchCache( | |
manhattanReadOnlyEndpoint: ManhattanKVEndpoint, | |
timelinesAuthorFeaturesColumn: TimelinesAuthorFeaturesOnUserClientColumn, | |
stats: StatsReceiver | |
): StitchCache[JLong, Option[AuthorFeatures]] | |
def metricCenterUserCountingStitchCache( | |
mcUserCountingFeaturesColumn: McUserCountingOnUserClientColumn, | |
stats: StatsReceiver | |
): StitchCache[JLong, Option[MCUserCountingFeatures]] | |
def clearUnsedFieldsForAuthorFeature(entry: AuthorFeatures): AuthorFeatures | |
def randomizedTTL(ttl: Long): Long | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\PreFetchedFeatureSource.scala | |
class PreFetchedFeatureSource @Inject() () extends FeatureSource | |
def id: FeatureSourceId = FeatureSourceId.PreFetchedFeatureSourceId | |
override def featureContext: FeatureContext = PreFetchedFeatureAdapter.getFeatureContext | |
override def hydrateFeatures( | |
target: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\UserScoringFeatureSource.scala | |
class UserScoringFeatureSource @Inject() ( | |
featureStoreSource: FeatureStoreSource, | |
featureStoreGizmoduckSource: FeatureStoreGizmoduckSource, | |
featureStorePostNuxAlgorithmSource: FeatureStorePostNuxAlgorithmSource, | |
featureStoreTimelinesAuthorSource: FeatureStoreTimelinesAuthorSource, | |
featureStoreUserMetricCountsSource: FeatureStoreUserMetricCountsSource, | |
clientContextSource: ClientContextSource, | |
candidateAlgorithmSource: CandidateAlgorithmSource, | |
preFetchedFeatureSource: PreFetchedFeatureSource) | |
extends FeatureSource | |
def hydrateFeatures( | |
target: HasClientContext | |
with HasPreFetchedFeature | |
with HasParams | |
with HasSimilarToContext | |
with HasDisplayLocation, | |
candidates: Seq[CandidateUser] | |
): Stitch[Map[CandidateUser, DataRecord]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\feature_hydration\sources\Utils.scala | |
object Utils | |
def adaptAdditionalFeaturesToDataRecord( | |
record: DataRecord, | |
adapterStats: StatsReceiver, | |
featureAdapters: Seq[IRecordOneToOneAdapter[DataRecord]] | |
): DataRecord | |
def randomizedTTL(ttl: Long): Long | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\AddressBookMetadata.scala | |
class AddressBookMetadata( | |
inForwardPhoneBook: Boolean, | |
inReversePhoneBook: Boolean, | |
inForwardEmailBook: Boolean, | |
inReverseEmailBook: Boolean) | |
object AddressBookMetadata | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\AlgorithmType.scala | |
object AlgorithmType extends Enumeration | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\CandidateUser.scala | |
trait FollowableEntity extends UniversalNoun[Long] | |
trait Recommendation | |
extends FollowableEntity | |
with HasReason | |
with HasAdMetadata | |
with HasTrackingToken | |
def toThrift: t.Recommendation | |
def toOfflineThrift: offline.OfflineRecommendation | |
} | |
case class CandidateUser( | |
override val id: Long, | |
override val score: Option[Double] = None, | |
override val reason: Option[Reason] = None, | |
override val userCandidateSourceDetails: Option[UserCandidateSourceDetails] = None, | |
override val adMetadata: Option[AdMetadata] = None, | |
override val trackingToken: Option[TrackingToken] = None, | |
override val dataRecord: Option[RichDataRecord] = None, | |
override val scores: Option[Scores] = None, | |
override val infoPerRankingStage: Option[scala.collection.Map[String, RankingInfo]] = None, | |
override val params: Params = Params.Invalid, | |
override val engagements: Seq[EngagementType] = Nil, | |
override val recommendationFlowIdentifier: Option[String] = None) | |
extends Recommendation | |
with HasUserCandidateSourceDetails | |
with HasDataRecord | |
with HasScores | |
with HasParams | |
with HasEngagements | |
with HasRecommendationFlowIdentifier | |
with HasInfoPerRankingStage | |
def setFollowProof(followProofOpt: Option[FollowProof]): CandidateUser | |
def addScore(score: Score): CandidateUser | |
object CandidateUser | |
def fromUserRecommendation(candidate: t.UserRecommendation): CandidateUser | |
def fromThriftScoreMap( | |
thriftMapOpt: Option[scala.collection.Map[String, Double]] | |
): Map[CandidateSourceIdentifier, Option[Double]] | |
def fromThriftRankMap( | |
thriftMapOpt: Option[scala.collection.Map[String, Int]] | |
): Map[CandidateSourceIdentifier, Int] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\ClientContextConverter.scala | |
object ClientContextConverter | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\DisplayLocation.scala | |
trait DisplayLocation | |
def toThrift: TDisplayLocation | |
def toOfflineThrift: TOfflineDisplayLocation | |
def toFsName: String | |
// corresponding display location in adserver if available | |
// make sure to be consistent with the definition here | |
def toAdDisplayLocation: Option[AdDisplayLocation] = None | |
} | |
/** | |
* Make sure you add the new DL to the following files and redeploy our attribution jobs | |
* - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift | |
* - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift | |
* - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala | |
*/ | |
object DisplayLocation | |
object ProfileSidebar extends DisplayLocation | |
object HomeTimeline extends DisplayLocation | |
object ReactiveFollow extends DisplayLocation | |
object ExploreTab extends DisplayLocation | |
object MagicRecs extends DisplayLocation | |
object AbUploadInjection extends DisplayLocation | |
object RuxLandingPage extends DisplayLocation | |
object ProfileBonusFollow extends DisplayLocation | |
object ElectionExploreWtf extends DisplayLocation | |
object ClusterFollow extends DisplayLocation | |
object HtlBonusFollow extends DisplayLocation | |
object TopicLandingPageHeader extends DisplayLocation | |
object NewUserSarusBackfill extends DisplayLocation | |
object NuxPymk extends DisplayLocation | |
object NuxInterests extends DisplayLocation | |
object NuxTopicBonusFollow extends DisplayLocation | |
object Sidebar extends DisplayLocation | |
object CampaignForm extends DisplayLocation | |
object ProfileTopFollowers extends DisplayLocation | |
object ProfileTopFollowing extends DisplayLocation | |
object RuxPymk extends DisplayLocation | |
object IndiaCovid19CuratedAccountsWtf extends DisplayLocation | |
object PeoplePlusPlus extends DisplayLocation | |
object TweetNotificationRecs extends DisplayLocation | |
object ProfileDeviceFollow extends DisplayLocation | |
object RecosBackfill extends DisplayLocation | |
object HtlSpaceHosts extends DisplayLocation | |
object PostNuxFollowTask extends DisplayLocation | |
object TopicLandingPage extends DisplayLocation | |
object UserTypeaheadPrefetch extends DisplayLocation | |
object HomeTimelineRelatableAccounts extends DisplayLocation | |
object NuxGeoCategory extends DisplayLocation | |
object NuxInterestsCategory extends DisplayLocation | |
object TopArticles extends DisplayLocation | |
object NuxPymkCategory extends DisplayLocation | |
object HomeTimelineTweetRecs extends DisplayLocation | |
object HtlBulkFriendFollows extends DisplayLocation | |
object NuxAutoFollow extends DisplayLocation | |
object SearchBonusFollow extends DisplayLocation | |
object ContentRecommender extends DisplayLocation | |
object HomeTimelineReverseChron extends DisplayLocation | |
def fromThrift(displayLocation: TDisplayLocation): DisplayLocation = displayLocation match | |
def fromOfflineThrift(displayLocation: TOfflineDisplayLocation): DisplayLocation = | |
displayLocation match | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\EngagementType.scala | |
trait EngagementType | |
def toThrift: TEngagementType | |
def toOfflineThrift: OfflineEngagementType | |
} | |
object EngagementType | |
object Click extends EngagementType | |
object Like extends EngagementType | |
object Mention extends EngagementType | |
object Retweet extends EngagementType | |
object ProfileView extends EngagementType | |
def fromThrift(engagementType: TEngagementType): EngagementType = engagementType match | |
def fromOfflineThrift(engagementType: OfflineEngagementType): EngagementType = | |
engagementType match | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\FilterReason.scala | |
trait FilterReason | |
def reason: String | |
} | |
object FilterReason | |
object NoReason extends FilterReason | |
class ParamReason(paramName: String) extends FilterReason | |
object ExcludedId extends FilterReason | |
object ProfileSidebarBlacklist extends FilterReason | |
object CuratedAccountsCompetitorList extends FilterReason | |
class InvalidRelationshipTypes(relationshipTypes: String) extends FilterReason | |
object ProfileId extends FilterReason | |
object DismissedId extends FilterReason | |
object OptedOutId extends FilterReason | |
object NoUser extends FilterReason | |
object AddressBookUndiscoverable extends FilterReason | |
object PhoneBookUndiscoverable extends FilterReason | |
object Deactivated extends FilterReason | |
object Suspended extends FilterReason | |
object Restricted extends FilterReason | |
object NsfwUser extends FilterReason | |
object NsfwAdmin extends FilterReason | |
object HssSignal extends FilterReason | |
object IsProtected extends FilterReason | |
class CountryTakedown(countryCode: String) extends FilterReason | |
object Blink extends FilterReason | |
object AlreadyFollowed extends FilterReason | |
object InvalidRelationship extends FilterReason | |
object NotFollowingTargetUser extends FilterReason | |
object CandidateSideHoldback extends FilterReason | |
object Inactive extends FilterReason | |
object MissingRecommendabilityData extends FilterReason | |
object HighTweetVelocity extends FilterReason | |
object AlreadyRecommended extends FilterReason | |
object MinStateNotMet extends FilterReason | |
object FailOpen extends FilterReason | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\FlowContext.scala | |
class FlowContext(steps: Seq[RecommendationStep]) | |
def toThrift: t.FlowContext = t.FlowContext(steps = steps.map(_.toThrift)) | |
def toOfflineThrift: offline.OfflineFlowContext = | |
offline.OfflineFlowContext(steps = steps.map(_.toOfflineThrift)) | |
} | |
object FlowContext | |
def fromThrift(flowContext: t.FlowContext): FlowContext | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\FlowRecommendation.scala | |
class FlowRecommendation(userId: Long) | |
def toThrift: t.FlowRecommendation = | |
t.FlowRecommendation(userId = userId) | |
def toOfflineThrift: offline.OfflineFlowRecommendation = | |
offline.OfflineFlowRecommendation(userId = userId) | |
} | |
object FlowRecommendation | |
def fromThrift(flowRecommendation: t.FlowRecommendation): FlowRecommendation | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasAdMetadata.scala | |
class AdMetadata( | |
insertPosition: Int, | |
// use original ad impression info to avoid losing data in domain model translations | |
adImpression: t.AdImpression) | |
trait HasAdMetadata | |
def adMetadata: Option[AdMetadata] | |
def adImpression: Option[t.AdImpression] | |
def insertPosition: Option[Int] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasByfSeedUserIds.scala | |
trait HasByfSeedUserIds | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasDataRecord.scala | |
class RichDataRecord( | |
dataRecord: Option[DataRecord] = None, | |
debugDataRecord: Option[DebugDataRecord] = None, | |
) | |
trait HasDataRecord extends Logging | |
def dataRecord: Option[RichDataRecord] | |
def toDebugDataRecord(dr: DataRecord, featureContext: FeatureContext): DebugDataRecord | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasDebugOptions.scala | |
class DebugOptions( | |
randomizationSeed: Option[Long] = None, | |
fetchDebugInfo: Boolean = false, | |
doNotLog: Boolean = false) | |
object DebugOptions | |
def fromDebugParamsThrift(debugParams: DebugParams): DebugOptions | |
trait HasDebugOptions | |
def debugOptions: Option[DebugOptions] | |
def getRandomizationSeed: Option[Long] = debugOptions.flatMap(_.randomizationSeed) | |
def fetchDebugInfo: Option[Boolean] = debugOptions.map(_.fetchDebugInfo) | |
} | |
trait HasFrsDebugOptions | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasDismissedUserIds.scala | |
trait HasDismissedUserIds | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasDisplayLocation.scala | |
trait HasDisplayLocation | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasEngagements.scala | |
trait HasEngagements | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasExcludedUserIds.scala | |
trait HasExcludedUserIds | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasGeohashAndCountryCode.scala | |
trait HasGeohashAndCountryCode | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasInfoPerRankingStage.scala | |
trait HasInfoPerRankingStage | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasInterestIds.scala | |
trait HasCustomInterests | |
def customInterests: Option[Seq[String]] | |
} | |
trait HasUttInterests | |
def uttInterestIds: Option[Seq[Long]] | |
} | |
trait HasInterestIds extends HasCustomInterests with HasUttInterests | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasInvalidRelationshipUserIds.scala | |
trait HasInvalidRelationshipUserIds | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasIsSoftUser.scala | |
trait HasIsSoftUser | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasMutualFollowedUserIds.scala | |
trait HasMutualFollowedUserIds extends HasRecentFollowedUserIds with HasRecentFollowedByUserIds | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasPreviousRecommendationsContext.scala | |
trait HasPreviousRecommendationsContext | |
def previouslyRecommendedUserIDs: Set[Long] | |
def previouslyFollowedUserIds: Set[Long] | |
def skippedFollows: Set[Long] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasProfileId.scala | |
trait HasProfileId | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasQualityFactor.scala | |
trait HasQualityFactor | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecentFollowedByUserIds.scala | |
trait HasRecentFollowedByUserIds | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecentFollowedUserIds.scala | |
trait HasRecentFollowedUserIds | |
def recentFollowedUserIds: Option[Seq[Long]] | |
// user ids that are recently followed by the target user in set data-structure | |
lazy val recentFollowedUserIdsSet: Option[Set[Long]] = recentFollowedUserIds match | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecentFollowedUserIdsWithTime.scala | |
trait HasRecentFollowedUserIdsWithTime | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecentlyEngagedUserIds.scala | |
trait HasRecentlyEngagedUserIds | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasRecommendationFlowIdentifier.scala | |
trait HasRecommendationFlowIdentifier | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasScores.scala | |
trait HasScores | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasSimilarToContext.scala | |
trait HasSimilarToContext | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasTopicId.scala | |
trait HasTopicId | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasUserCandidateSourceDetails.scala | |
trait HasUserCandidateSourceDetails | |
def userCandidateSourceDetails: Option[UserCandidateSourceDetails] | |
def getAlgorithm: Algorithm | |
def getAllAlgorithms: Seq[Algorithm] | |
def getAddressBookMetadata: Option[AddressBookMetadata] | |
def getCandidateSources: Map[CandidateSourceIdentifier, Option[Double]] | |
def getCandidateRanks: Map[CandidateSourceIdentifier, Int] | |
def getCandidateFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] | |
def getPrimaryCandidateSource: Option[CandidateSourceIdentifier] | |
def withCandidateSource(source: CandidateSourceIdentifier): CandidateUser | |
def withCandidateSourceAndScore( | |
source: CandidateSourceIdentifier, | |
score: Option[Double] | |
): CandidateUser | |
def withCandidateSourceAndFeatures( | |
source: CandidateSourceIdentifier, | |
features: Seq[Feature] | |
): CandidateUser | |
def withCandidateSourceScoreAndFeatures( | |
source: CandidateSourceIdentifier, | |
score: Option[Double], | |
features: Seq[Feature] | |
): CandidateUser | |
def addCandidateSourceScoresMap( | |
scoreMap: Map[CandidateSourceIdentifier, Option[Double]] | |
): CandidateUser | |
def addCandidateSourceRanksMap( | |
rankMap: Map[CandidateSourceIdentifier, Int] | |
): CandidateUser | |
def addInfoPerRankingStage( | |
rankingStage: String, | |
scores: Option[Scores], | |
rank: Int | |
): CandidateUser | |
def addAddressBookMetadataIfAvailable( | |
candidateSources: Seq[CandidateSourceIdentifier] | |
): CandidateUser | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasUserState.scala | |
trait HasUserState | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\HasWtfImpressions.scala | |
trait HasWtfImpressions | |
def wtfImpressions: Option[Seq[WtfImpression]] | |
lazy val numWtfImpressions: Int = wtfImpressions.map(_.size).getOrElse(0) | |
lazy val candidateImpressions: Map[Long, WtfImpression] = wtfImpressions | |
.map | |
def getCandidateImpressionCounts(id: Long): Option[Int] = | |
candidateImpressions.get(id).map(_.counts) | |
def getCandidateLatestTime(id: Long): Option[Time] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\OptimusRequest.scala | |
trait to group together all traits needed for optimus ranking | |
*/ | |
trait OptimusRequest | |
extends HasParams | |
with HasClientContext | |
with HasDisplayLocation | |
with HasInterestIds | |
with HasDebugOptions | |
with HasPreviousRecommendationsContext | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\Product.scala | |
object Product | |
object MagicRecs extends ProductMixerProduct | |
object PlaceholderProductMixerProduct extends ProductMixerProduct | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\RankingInfo.scala | |
class RankingInfo( | |
scores: Option[Scores], | |
rank: Option[Int]) | |
def toThrift: t.RankingInfo | |
def toOfflineThrift: offline.RankingInfo | |
object RankingInfo | |
def fromThrift(rankingInfo: t.RankingInfo): RankingInfo | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\Reason.scala | |
class FollowProof(followedBy: Seq[Long], numIds: Int) | |
def toThrift: t.FollowProof | |
def toOfflineThrift: offline.FollowProof = offline.FollowProof(followedBy, numIds) | |
} | |
object FollowProof | |
def fromThrift(proof: t.FollowProof): FollowProof | |
class SimilarToProof(similarTo: Seq[Long]) | |
def toThrift: t.SimilarToProof | |
def toOfflineThrift: offline.SimilarToProof = offline.SimilarToProof(similarTo) | |
} | |
object SimilarToProof | |
def fromThrift(proof: t.SimilarToProof): SimilarToProof | |
class PopularInGeoProof(location: String) | |
def toThrift: t.PopularInGeoProof | |
def toOfflineThrift: offline.PopularInGeoProof = offline.PopularInGeoProof(location) | |
} | |
object PopularInGeoProof | |
def fromThrift(proof: t.PopularInGeoProof): PopularInGeoProof | |
class TttInterestProof(interestId: Long, interestDisplayName: String) | |
def toThrift: t.TttInterestProof | |
def toOfflineThrift: offline.TttInterestProof = | |
offline.TttInterestProof(interestId, interestDisplayName) | |
} | |
object TttInterestProof | |
def fromThrift(proof: t.TttInterestProof): TttInterestProof | |
class TopicProof(topicId: Long) | |
def toThrift: t.TopicProof | |
def toOfflineThrift: offline.TopicProof = | |
offline.TopicProof(topicId) | |
} | |
object TopicProof | |
def fromThrift(proof: t.TopicProof): TopicProof | |
class CustomInterest(query: String) | |
def toThrift: t.CustomInterestProof | |
def toOfflineThrift: offline.CustomInterestProof = | |
offline.CustomInterestProof(query) | |
} | |
object CustomInterest | |
def fromThrift(proof: t.CustomInterestProof): CustomInterest | |
class TweetsAuthorProof(tweetIds: Seq[Long]) | |
def toThrift: t.TweetsAuthorProof | |
def toOfflineThrift: offline.TweetsAuthorProof = | |
offline.TweetsAuthorProof(tweetIds) | |
} | |
object TweetsAuthorProof | |
def fromThrift(proof: t.TweetsAuthorProof): TweetsAuthorProof | |
class DeviceFollowProof(isDeviceFollow: Boolean) | |
def toThrift: t.DeviceFollowProof | |
def toOfflineThrift: offline.DeviceFollowProof = | |
offline.DeviceFollowProof(isDeviceFollow) | |
} | |
object DeviceFollowProof | |
def fromThrift(proof: t.DeviceFollowProof): DeviceFollowProof | |
class AccountProof( | |
followProof: Option[FollowProof] = None, | |
similarToProof: Option[SimilarToProof] = None, | |
popularInGeoProof: Option[PopularInGeoProof] = None, | |
tttInterestProof: Option[TttInterestProof] = None, | |
topicProof: Option[TopicProof] = None, | |
customInterestProof: Option[CustomInterest] = None, | |
tweetsAuthorProof: Option[TweetsAuthorProof] = None, | |
deviceFollowProof: Option[DeviceFollowProof] = None) | |
def toThrift: t.AccountProof | |
def toOfflineThrift: offline.AccountProof | |
object AccountProof | |
def fromThrift(proof: t.AccountProof): AccountProof | |
class Reason(accountProof: Option[AccountProof]) | |
def toThrift: t.Reason | |
def toOfflineThrift: offline.Reason | |
object Reason | |
def fromThrift(reason: t.Reason): Reason | |
trait HasReason | |
def reason: Option[Reason] | |
// helper methods below | |
def followedBy: Option[Seq[Long]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\RecentlyEngagedUserId.scala | |
class RecentlyEngagedUserId(id: Long, engagementType: EngagementType) | |
def toThrift: t.RecentlyEngagedUserId = | |
t.RecentlyEngagedUserId(id = id, engagementType = engagementType.toThrift) | |
def toOfflineThrift: offline.RecentlyEngagedUserId = | |
offline.RecentlyEngagedUserId(id = id, engagementType = engagementType.toOfflineThrift) | |
} | |
object RecentlyEngagedUserId | |
def fromThrift(recentlyEngagedUserId: t.RecentlyEngagedUserId): RecentlyEngagedUserId | |
def fromOfflineThrift( | |
recentlyEngagedUserId: offline.RecentlyEngagedUserId | |
): RecentlyEngagedUserId | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\RecommendationStep.scala | |
class RecommendationStep( | |
recommendations: Seq[FlowRecommendation], | |
followedUserIds: Set[Long]) | |
def toThrift: t.RecommendationStep = t.RecommendationStep( | |
recommendations = recommendations.map(_.toThrift), | |
followedUserIds = followedUserIds | |
) | |
def toOfflineThrift: offline.OfflineRecommendationStep = | |
offline.OfflineRecommendationStep( | |
recommendations = recommendations.map(_.toOfflineThrift), | |
followedUserIds = followedUserIds) | |
} | |
object RecommendationStep | |
def fromThrift(recommendationStep: t.RecommendationStep): RecommendationStep | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\SafetyLevel.scala | |
trait SafetyLevel | |
def toThrift: ThriftSafetyLevel | |
} | |
object SafetyLevel | |
object Recommendations extends SafetyLevel | |
object TopicsLandingPageTopicRecommendations extends SafetyLevel | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\Score.scala | |
trait so it is possible to add more information for different score types. | |
*/ | |
sealed trait ScoreType | |
def getName: String | |
} | |
/** | |
* Existing Score Types | |
*/ | |
object ScoreType | |
object HeuristicBasedScore extends ScoreType | |
def getName: String = "HeuristicBasedScore" | |
} | |
/** | |
* probability of follow after the candidate is recommended to the user | |
*/ | |
case object PFollowGivenReco extends ScoreType | |
def getName: String = "PFollowGivenReco" | |
} | |
/** | |
* probability of engage after the user follows the candidate | |
*/ | |
case object PEngagementGivenFollow extends ScoreType | |
def getName: String = "PEngagementGivenFollow" | |
} | |
/** | |
* probability of engage per tweet impression | |
*/ | |
case object PEngagementPerImpression extends ScoreType | |
def getName: String = "PEngagementPerImpression" | |
} | |
/** | |
* probability of engage per tweet impression | |
*/ | |
case object PEngagementGivenReco extends ScoreType | |
def getName: String = "PEngagementGivenReco" | |
} | |
def fromScoreTypeString(scoreTypeName: String): ScoreType = scoreTypeName match | |
class Score( | |
value: Double, | |
rankerId: Option[RankerId] = None, | |
scoreType: Option[ScoreType] = None) | |
def toThrift: t.Score = t.Score( | |
value = value, | |
rankerId = rankerId.map(_.toString), | |
scoreType = scoreType.map(_.getName) | |
) | |
def toOfflineThrift: offline.Score = | |
offline.Score( | |
value = value, | |
rankerId = rankerId.map(_.toString), | |
scoreType = scoreType.map(_.getName) | |
) | |
} | |
object Score | |
def optimusScore(score: Double, scoreType: ScoreType): Score | |
def predictionScore(score: Double, rankerId: RankerId): Score | |
def fromThrift(thriftScore: t.Score): Score = | |
Score( | |
value = thriftScore.value, | |
rankerId = thriftScore.rankerId.flatMap(RankerId.getRankerByName), | |
scoreType = thriftScore.scoreType.map(ScoreType.fromScoreTypeString) | |
) | |
} | |
/** | |
* a list of scores | |
*/ | |
final case class Scores( | |
scores: Seq[Score], | |
selectedRankerId: Option[RankerId] = None, | |
isInProducerScoringExperiment: Boolean = false) | |
def toThrift: t.Scores = | |
t.Scores( | |
scores = scores.map(_.toThrift), | |
selectedRankerId = selectedRankerId.map(_.toString), | |
isInProducerScoringExperiment = isInProducerScoringExperiment | |
) | |
def toOfflineThrift: offline.Scores = | |
offline.Scores( | |
scores = scores.map(_.toOfflineThrift), | |
selectedRankerId = selectedRankerId.map(_.toString), | |
isInProducerScoringExperiment = isInProducerScoringExperiment | |
) | |
} | |
object Scores | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\Session.scala | |
object Session | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\SignalData.scala | |
trait SignalData | |
class RecentFollowsSignal( | |
override val userId: Long, | |
override val signalType: SignalType, | |
followedUserId: Long, | |
timestamp: Long) | |
extends SignalData | |
object RecentFollowsSignal | |
def fromUssSignal(targetUserId: Long, signal: Signal): RecentFollowsSignal | |
def getRecentFollowedUserIds( | |
signalDataMap: Option[Map[SignalType, Seq[SignalData]]] | |
): Option[Seq[Long]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\TrackingToken.scala | |
class TrackingToken( | |
sessionId: Long, | |
displayLocation: Option[DisplayLocation], | |
controllerData: Option[ControllerData], | |
algorithmId: Option[Int]) | |
def toThrift: t.TrackingToken | |
def toOfflineThrift: offline.TrackingToken | |
object TrackingToken | |
def serialize(trackingToken: TrackingToken): String | |
def deserialize(trackingTokenStr: String): TrackingToken | |
def fromThrift(token: t.TrackingToken): TrackingToken | |
trait HasTrackingToken | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\models\UserCandidateSourceDetails.scala | |
class UserCandidateSourceDetails( | |
primaryCandidateSource: Option[CandidateSourceIdentifier], | |
candidateSourceScores: Map[CandidateSourceIdentifier, Option[Double]] = Map.empty, | |
candidateSourceRanks: Map[CandidateSourceIdentifier, Int] = Map.empty, | |
addressBookMetadata: Option[AddressBookMetadata] = None, | |
candidateSourceFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] = Map.empty, | |
) | |
def toThrift: t.CandidateSourceDetails | |
def toOfflineThrift: offline.CandidateSourceDetails | |
object UserCandidateSourceDetails | |
def fromThrift(details: t.CandidateSourceDetails): UserCandidateSourceDetails | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\CandidateParamPredicate.scala | |
class CandidateParamPredicate[A <: HasParams]( | |
param: Param[Boolean], | |
reason: FilterReason) | |
extends Predicate[A] | |
def apply(candidate: A): Stitch[PredicateResult] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\CandidateSourceParamPredicate.scala | |
class CandidateSourceParamPredicate( | |
val param: Param[Boolean], | |
val reason: FilterReason, | |
candidateSources: Set[CandidateSourceIdentifier]) | |
extends Predicate[CandidateUser] | |
def apply(candidate: CandidateUser): Stitch[PredicateResult] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\CuratedCompetitorListPredicate.scala | |
class CuratedCompetitorListPredicate @Inject() ( | |
statsReceiver: StatsReceiver, | |
@Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) competitorAccountFetcher: Fetcher[ | |
String, | |
Unit, | |
Seq[Long] | |
]) extends Predicate[CandidateUser] | |
def query(prefix: String): Stitch[Set[Long]] = | |
competitorAccountFetcher.fetch(prefix).map(_.v.getOrElse(Nil).toSet) | |
/** | |
* Caveat here is that though the similarToUserIds allows for a Seq[Long], in practice we would | |
* only return 1 userId. Multiple userId's would result in filtering candidates associated with | |
* a different similarToUserId. For example: | |
* - similarToUser1 -> candidate1, candidate2 | |
* - similarToUser2 -> candidate3 | |
* and in the competitorList store we have: | |
* - similarToUser1 -> candidate3 | |
* we'll be filtering candidate3 on account of similarToUser1, even though it was generated | |
* with similarToUser2. This might still be desirable at a product level (since we don't want | |
* to show these accounts anyway), but might not achieve what you intend to code-wise. | |
*/ | |
override def apply(candidate: CandidateUser): Stitch[PredicateResult] | |
object CuratedCompetitorListPredicate | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\ExcludedUserIdPredicate.scala | |
object ExcludedUserIdPredicate extends Predicate[(HasExcludedUserIds, CandidateUser)] | |
def apply(pair: (HasExcludedUserIds, CandidateUser)): Stitch[PredicateResult] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\InactivePredicate.scala | |
class InactivePredicate @Inject() ( | |
statsReceiver: StatsReceiver, | |
@Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) userRecommendabilityFetcher: Fetcher[ | |
Long, | |
Unit, | |
UserRecommendabilityFeatures | |
]) extends Predicate[(HasParams with HasClientContext with HasUserState, CandidateUser)] | |
def queryUserRecommendable(userId: Long): Stitch[Option[UserRecommendabilityFeatures]] = | |
userRecommendabilityFetcher.fetch(userId).map(_.v) | |
private val userRecommendableCache = | |
StitchCache[JLong, Option[UserRecommendabilityFeatures]]( | |
maxCacheSize = 100000, | |
ttl = 12.hours, | |
statsReceiver = cacheStats.scope("UserRecommendable"), | |
underlyingCall = (userId: JLong) => queryUserRecommendable(userId) | |
) | |
override def apply( | |
targetAndCandidate: (HasParams with HasClientContext with HasUserState, CandidateUser) | |
): Stitch[PredicateResult] | |
def disableInactivityPredicate( | |
target: HasParams, | |
consumerState: Option[UserState], | |
candidateState: Option[UserState] | |
): Boolean | |
object InactivePredicate | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\InactivePredicateParams.scala | |
object InactivePredicateParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\PreviouslyRecommendedUserIdsPredicate.scala | |
class PreviouslyRecommendedUserIdsPredicate | |
extends Predicate[(HasPreviousRecommendationsContext, CandidateUser)] | |
def apply( | |
pair: (HasPreviousRecommendationsContext, CandidateUser) | |
): Stitch[PredicateResult] | |
object PreviouslyRecommendedUserIdsPredicate | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\dismiss\DismissedCandidatePredicate.scala | |
class DismissedCandidatePredicate extends Predicate[(HasDismissedUserIds, CandidateUser)] | |
def apply(pair: (HasDismissedUserIds, CandidateUser)): Stitch[PredicateResult] | |
object DismissedCandidatePredicate | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\dismiss\DismissedCandidatePredicateParams.scala | |
object DismissedCandidatePredicateParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\gizmoduck\GizmoduckPredicate.scala | |
class GizmoduckPredicate @Inject() ( | |
gizmoduck: Gizmoduck, | |
client: Client, | |
statsReceiver: StatsReceiver, | |
decider: Decider = Decider.False) | |
extends Predicate[(HasClientContext with HasParams, CandidateUser)] | |
with Logging | |
def apply( | |
pair: (HasClientContext with HasParams, CandidateUser) | |
): Stitch[PredicateResult] | |
def getGizmoduckPredicateResult( | |
request: HasClientContext with HasParams, | |
candidate: CandidateUser | |
): Stitch[PredicateResult] | |
def logPredicateResultEquality( | |
request: HasClientContext with HasParams, | |
candidate: CandidateUser, | |
predicateResult: PredicateResult | |
): Unit | |
def getPredicateResult( | |
request: HasClientContext with HasParams, | |
candidate: CandidateUser, | |
userResult: UserResult, | |
): PredicateResult | |
def getByUserId(userId: JLong): Stitch[UserResult] | |
object GizmoduckPredicate | |
def getAbPbReason( | |
user: User, | |
abMetadataOpt: Option[AddressBookMetadata] | |
): Set[FilterReason] | |
def getSafetyReasons(user: User): Set[FilterReason] | |
def getCountryTakedownReasons( | |
user: User, | |
countryCodeOpt: Option[String] | |
): Set[FilterReason] | |
def getBlinkReasons(user: User): Set[FilterReason] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\gizmoduck\GizmoduckPredicateCache.scala | |
object GizmoduckPredicateCache | |
class TimeTicker extends Ticker | |
def read(): Long = Time.now.inNanoseconds | |
} | |
def apply[K, V]( | |
maxCacheSize: Int, | |
ttl: Duration, | |
statsReceiver: StatsReceiver | |
): Cache[K, V] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\gizmoduck\GizmoduckPredicateFSConfig.scala | |
class GizmoduckPredicateFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\gizmoduck\GizmoduckPredicateParams.scala | |
object GizmoduckPredicateParams | |
object GizmoduckGetTimeout | |
extends FSBoundedParam[Duration]( | |
name = "gizmoduck_predicate_timeout_in_millis", | |
default = 200.millisecond, | |
min = 1.millisecond, | |
max = 500.millisecond) | |
with HasDurationConversion | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\health\HssPredicate.scala | |
class HssPredicate @Inject() ( | |
healthSignalsOnUserClientColumn: HealthSignalsOnUserClientColumn, | |
statsReceiver: StatsReceiver) | |
extends Predicate[(HasClientContext with HasParams, CandidateUser)] | |
with Logging | |
def apply( | |
pair: (HasClientContext with HasParams, CandidateUser) | |
): Stitch[PredicateResult] | |
def getHssPredicateResult( | |
request: HasClientContext with HasParams, | |
candidate: CandidateUser | |
): Stitch[PredicateResult] | |
def userHealthSignalValueToDoubleOpt(signalValue: Option[SignalValue]): Option[Double] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\health\HssPredicateFSConfig.scala | |
class HssPredicateFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\health\HssPredicateParams.scala | |
object HssPredicateParams | |
object HssCseScoreThreshold | |
extends FSBoundedParam[Double]( | |
"hss_predicate_cse_score_threshold", | |
default = 0.992d, | |
min = 0.0d, | |
max = 1.0d) | |
object HssNsfwScoreThreshold | |
extends FSBoundedParam[Double]( | |
"hss_predicate_nsfw_score_threshold", | |
default = 1.5d, | |
min = -100.0d, | |
max = 100.0d) | |
object HssApiTimeout | |
extends FSBoundedParam[Duration]( | |
name = "hss_predicate_timeout_in_millis", | |
default = 200.millisecond, | |
min = 1.millisecond, | |
max = 500.millisecond) | |
with HasDurationConversion | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\InvalidRelationshipPredicate.scala | |
class InvalidRelationshipPredicate | |
extends Predicate[(HasInvalidRelationshipUserIds, CandidateUser)] | |
def apply( | |
pair: (HasInvalidRelationshipUserIds, CandidateUser) | |
): Stitch[PredicateResult] | |
object InvalidRelationshipPredicate | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\RecentFollowingPredicate.scala | |
class RecentFollowingPredicate extends Predicate[(HasRecentFollowedUserIds, CandidateUser)] | |
def apply(pair: (HasRecentFollowedUserIds, CandidateUser)): Stitch[PredicateResult] | |
object RecentFollowingPredicate | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\SgsPredicateFSConfig.scala | |
class SgsPredicateFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\SgsPredicateParams.scala | |
object SgsPredicateParams | |
object SgsRelationshipsPredicateTimeout | |
extends FSBoundedParam[Duration]( | |
name = "sgs_predicate_relationships_timeout_in_millis", | |
default = 300.millisecond, | |
min = 1.millisecond, | |
max = 1000.millisecond) | |
with HasDurationConversion | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\SgsRelationshipsByUserIdPredicate.scala | |
class SgsRelationshipsByUserIdPredicate( | |
socialGraph: SocialGraph, | |
relationshipMappings: Seq[RelationshipMapping], | |
statsReceiver: StatsReceiver) | |
extends Predicate[(Option[Long], CandidateUser)] | |
with Logging | |
def apply( | |
pair: (Option[Long], CandidateUser) | |
): Stitch[PredicateResult] | |
object SgsRelationshipsByUserIdPredicate | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\sgs\SgsRelationshipsPredicate.scala | |
class RelationshipMapping( | |
relationshipType: RelationshipType, | |
includeBasedOnRelationship: Boolean) | |
class SgsRelationshipsPredicate( | |
socialGraph: SocialGraph, | |
relationshipMappings: Seq[RelationshipMapping], | |
statsReceiver: StatsReceiver = NullStatsReceiver) | |
extends Predicate[(HasClientContext with HasParams, CandidateUser)] | |
with Logging | |
def apply( | |
pair: (HasClientContext with HasParams, CandidateUser) | |
): Stitch[PredicateResult] | |
object SgsRelationshipsPredicate | |
def extractUserId(target: HasClientContext with HasParams): Option[Long] = target match | |
class InvalidTargetCandidateRelationshipTypesPredicate @Inject() ( | |
socialGraph: SocialGraph) | |
extends SgsRelationshipsPredicate( | |
socialGraph, | |
InvalidRelationshipTypesPredicate.InvalidRelationshipTypes) | |
class NoteworthyAccountsSgsPredicate @Inject() ( | |
socialGraph: SocialGraph) | |
extends SgsRelationshipsPredicate( | |
socialGraph, | |
InvalidRelationshipTypesPredicate.NoteworthyAccountsInvalidRelationshipTypes) | |
object InvalidRelationshipTypesPredicate | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\user_activity\UserActivityPredicate.scala | |
abstract case class UserStateActivityPredicate( | |
userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, | |
validCandidateStates: Set[UserState], | |
client: Client, | |
statsReceiver: StatsReceiver, | |
decider: Decider = Decider.False) | |
extends Predicate[(HasParams with HasClientContext, CandidateUser)] | |
def apply( | |
targetAndCandidate: (HasParams with HasClientContext, CandidateUser) | |
): Stitch[PredicateResult] | |
def queryUserRecommendable( | |
userId: Long | |
): Stitch[Option[UserState]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\predicates\user_activity\UserActivityPredicateParams.scala | |
object UserActivityPredicateParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\common\AdhocScoreModificationType.scala | |
object AdhocScoreModificationType extends Enumeration | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\common\DedupCandidates.scala | |
object DedupCandidates | |
def apply[C <: UniversalNoun[Long]](input: Seq[C]): Seq[C] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\common\RankerId.scala | |
object RankerId extends Enumeration | |
def getRankerByName(name: String): Option[RankerId] = | |
RankerId.values.toSeq.find(_.equals(Value(name))) | |
} | |
/** | |
* ML model based heavy ranker ids. | |
*/ | |
object ModelBasedHeavyRankerId | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\fatigue_ranker\ImpressionBasedFatigueRanker.scala | |
class ImpressionBasedFatigueRanker[ | |
Target <: HasClientContext with HasDisplayLocation with HasParams with HasWtfImpressions | |
]( | |
fatigueFactor: Int, | |
statsReceiver: StatsReceiver) | |
extends Ranker[Target, CandidateUser] | |
def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] | |
def trackTimeSinceOldestImpression(impressions: Seq[WtfImpression]): Unit | |
def rankCandidates( | |
target: Target, | |
candidates: Seq[CandidateUser] | |
): Seq[CandidateUser] | |
object ImpressionBasedFatigueRanker | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\fatigue_ranker\ImpressionBasedFatigueRankerFSConfig.scala | |
class ImpressionBasedFatigueRankerFSConfig @Inject() extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\fatigue_ranker\ImpressionBasedFatigueRankerParams.scala | |
object ImpressionBasedFatigueRankerParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\first_n_ranker\FirstNRanker.scala | |
class is meant to filter candidates between stages of our ranker by taking the first N | |
* candidates, merging any candidate source information for candidates with multiple entries. | |
* To allow us to chain this truncation operation any number of times sequentially within the main | |
* ranking builder, we abstract the truncation as a separate Ranker | |
*/ | |
@Singleton | |
class FirstNRanker[Target <: HasClientContext with HasParams with HasQualityFactor] @Inject() ( | |
stats: StatsReceiver) | |
extends Ranker[Target, CandidateUser] | |
class CandidateSourceScore( | |
candidateId: Long, | |
sourceId: CandidateSourceIdentifier, | |
score: Option[Double]) | |
/** | |
* Adds the rank of each candidate based on the primary candidate source's score. | |
* In the event where the provided ordering of candidates do not align with the score, | |
* we will respect the score, since the ordering might have been mixed up due to other previous | |
* steps like the shuffleFn in the `WeightedCandidateSourceRanker`. | |
* @param candidates ordered list of candidates | |
* @return same ordered list of candidates, but with the rank information appended | |
*/ | |
def addRank(candidates: Seq[CandidateUser]): Seq[CandidateUser] | |
def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] | |
def merge(candidates: Seq[CandidateUser]): CandidateUser | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\first_n_ranker\FirstNRankerFeatureSwitchKeys.scala | |
object FirstNRankerFeatureSwitchKeys | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\first_n_ranker\FirstNRankerFSConfig.scala | |
class FirstNRankerFSConfig @Inject() extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\first_n_ranker\FirstNRankerParams.scala | |
object FirstNRankerParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\interleave_ranker\InterleaveRanker.scala | |
class InterleaveRanker[Target <: HasParams] @Inject() ( | |
statsReceiver: StatsReceiver) | |
extends Ranker[Target, CandidateUser] | |
def rank( | |
target: Target, | |
candidates: Seq[CandidateUser] | |
): Stitch[Seq[CandidateUser]] | |
def rankCandidates( | |
target: Target, | |
candidates: Seq[CandidateUser] | |
): Seq[CandidateUser] | |
def interleaveCandidates( | |
candidates: Seq[CandidateUser], | |
rankerIds: Seq[RankerId.RankerId] | |
): Seq[CandidateUser] | |
def resolveConflicts( | |
candidatesWithRank: Seq[(CandidateUser, Int)] | |
): Seq[(CandidateUser, Int)] | |
def getCandidateScoreByRankerId( | |
candidate: CandidateUser, | |
rankerIdOpt: Option[RankerId.RankerId] | |
): Option[Double] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\interleave_ranker\InterleaveRankerFSConfig.scala | |
class InterleaveRankerFSConfig @Inject() extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\interleave_ranker\InterleaveRankerParams.scala | |
object InterleaveRankerParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\ranking\HydrateFeaturesTransform.scala | |
class HydrateFeaturesTransform[ | |
Target <: HasClientContext with HasParams with HasDebugOptions with HasPreFetchedFeature with HasSimilarToContext with HasDisplayLocation] @Inject() ( | |
userScoringFeatureSource: UserScoringFeatureSource, | |
stats: StatsReceiver) | |
extends GatedTransform[Target, CandidateUser] | |
with Logging | |
def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\ranking\MlRanker.scala | |
class has a rank function that will perform 4 steps: | |
* - choose which scorer to use for each candidate | |
* - score candidates given their respective features | |
* - add scoring information to the candidate | |
* - sort candidates by their respective scores | |
* The feature source and scorer will depend on the request's params | |
*/ | |
@Singleton | |
class MlRanker[ | |
Target <: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions] @Inject() ( | |
scorerFactory: ScorerFactory, | |
statsReceiver: StatsReceiver) | |
extends Ranker[Target, CandidateUser] | |
with Logging | |
def rank( | |
target: Target, | |
candidates: Seq[CandidateUser] | |
): Stitch[Seq[CandidateUser]] | |
def chooseRankerByCandidate( | |
candidates: Seq[CandidateUser], | |
requestRankerId: RankerId | |
): Map[CandidateUser, RankerId] | |
def score( | |
candidates: Seq[CandidateUser], | |
rankerIds: Map[CandidateUser, RankerId], | |
requestRankerId: RankerId | |
): Stitch[Seq[CandidateUser]] | |
object | |
case (candidate, scores) => | |
val selectedRankerId = rankerIds(candidate) | |
val useRequestRanker = | |
candidate.params == Params.Invalid || | |
candidate.params == Params.Empty || | |
candidate.params(MlRankerParams.CandidateScorerIdParam) == RankerId.None | |
candidate.copy( | |
score = scores.scores.find(_.rankerId.contains(requestRankerId)).map(_.value), | |
scores = if (scores.scores.nonEmpty) | |
def addMlBaseScoresForAdhocScorers( | |
candidates: Seq[CandidateUser], | |
requestRankerId: RankerId, | |
adhocScorers: Seq[Scorer] | |
): Seq[CandidateUser] | |
def score( | |
dataRecords: Seq[DataRecord], | |
scorers: Seq[Scorer] | |
): Stitch[Seq[Scores]] | |
def sort( | |
target: Target, | |
candidates: Seq[CandidateUser] | |
): Seq[CandidateUser] | |
def scribeCandidates( | |
target: Target, | |
candidates: Seq[CandidateUser] | |
): Seq[CandidateUser] | |
object MlRanker | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\ranking\MlRankerFSConfig.scala | |
class MlRankerFSConfig @Inject() extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\ranking\MlRankerParams.scala | |
object MlRankerParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\AdhocScorer.scala | |
trait AdhocScorer extends Scorer | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\DeepbirdScorer.scala | |
trait that implements the scoring given a deepbirdClient | |
* To test out a new model, create a scorer extending this trait, override the modelName and inject the scorer | |
*/ | |
trait DeepbirdScorer extends Scorer | |
def modelName: String | |
def predictionFeature: Feature.Continuous | |
// Set a default batchSize of 100 when making model prediction calls to the Deepbird V2 prediction server | |
def batchSize: Int = 100 | |
def deepbirdClient: DeepbirdPredictionService.ServiceToClient | |
def baseStats: StatsReceiver | |
def modelSelector: ModelSelector = new ModelSelector().setId(modelName) | |
def stats: StatsReceiver = baseStats.scope(this.getClass.getSimpleName).scope(modelName) | |
private def requestCount = stats.counter("requests") | |
private def emptyRequestCount = stats.counter("empty_requests") | |
private def successCount = stats.counter("success") | |
private def failureCount = stats.counter("failures") | |
private def inputRecordsStat = stats.stat("input_records") | |
private def outputRecordsStat = stats.stat("output_records") | |
// Counters for tracking batch-prediction statistics when making DBv2 prediction calls | |
// | |
// numBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers | |
private def numBatchRequests = stats.counter("batches") | |
// numEmptyBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers | |
// that had an empty input DataRecord | |
private def numEmptyBatchRequests = stats.counter("empty_batches") | |
// numTimedOutBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers | |
// that had timed-out | |
private def numTimedOutBatchRequests = stats.counter("timeout_batches") | |
private def batchPredictionLatency = stats.stat("batch_prediction_latency") | |
private def predictionLatency = stats.stat("prediction_latency") | |
private def numEmptyModelPredictions = stats.counter("empty_model_predictions") | |
private def numNonEmptyModelPredictions = stats.counter("non_empty_model_predictions") | |
private val DefaultPredictionScore = 0.0 | |
/** | |
* NOTE: For instances of [[DeepbirdScorer]] this function SHOULD NOT be used. | |
* Please use [[score(records: Seq[DataRecord])]] instead. | |
*/ | |
@Deprecated | |
def score( | |
target: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions, | |
candidates: Seq[CandidateUser] | |
): Seq[Option[Score]] = | |
throw new UnsupportedOperationException( | |
"For instances of DeepbirdScorer this operation is not defined. Please use " + | |
"`def score(records: Seq[DataRecord]): Stitch[Seq[Score]]` " + | |
"instead.") | |
override def score(records: Seq[DataRecord]): Stitch[Seq[Score]] | |
def batchPredict( | |
dataRecords: Seq[DataRecord], | |
batchSize: Int | |
): Future[Seq[Option[Double]]] | |
def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\PostnuxDeepbirdProdScorer.scala | |
trait when adding new fields to DeepbirdV2 scorers which | |
trait DeepbirdProdScorer extends DeepbirdScorer | |
trait PostNuxV1DeepbirdProdScorer extends DeepbirdProdScorer | |
class PostnuxDeepbirdProdScorer @Inject() ( | |
@Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) | |
override val deepbirdClient: DeepbirdPredictionService.ServiceToClient, | |
override val baseStats: StatsReceiver) | |
extends PostNuxV1DeepbirdProdScorer | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\RandomScorer.scala | |
class RandomScorer @Inject() ( | |
@Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) | |
override val deepbirdClient: DeepbirdPredictionService.ServiceToClient, | |
override val baseStats: StatsReceiver) | |
extends DeepbirdScorer | |
def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\Scorer.scala | |
trait Scorer | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\ml_ranker\scoring\ScorerFactory.scala | |
class ScorerFactory @Inject() ( | |
postnuxProdScorer: PostnuxDeepbirdProdScorer, | |
randomScorer: RandomScorer, | |
stats: StatsReceiver) | |
def getScorers( | |
rankerIds: Seq[RankerId] | |
): Seq[Scorer] | |
def getScorerById(scorerId: RankerId): Scorer = scorerId match | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\utils\Utils.scala | |
object Utils | |
def addRankingInfo(candidates: Seq[CandidateUser], rankingStage: String): Seq[CandidateUser] | |
def getCandidateScoreByRankerId(candidate: CandidateUser, rankerId: RankerId): Option[Score] = | |
candidate.scores.flatMap | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\CandidateShuffle.scala | |
trait CandidateShuffler[T] | |
def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] | |
} | |
class NoShuffle[T]() extends CandidateShuffler[T] | |
def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = input | |
} | |
class RandomShuffler[T]() extends CandidateShuffler[T] | |
def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] | |
trait RankWeightedRandomShuffler[T] extends CandidateShuffler[T] | |
def rankToWeight(rank: Int): Double | |
def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] | |
class ExponentialShuffler[T]() extends RankWeightedRandomShuffler[T] | |
def rankToWeight(rank: Int): Double | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightedCandidateSourceBaseRanker.scala | |
class WeightedCandidateSourceBaseRanker[A, Rec]( | |
candidateSourceWeights: Map[A, Double], | |
weightMethod: WeightMethod = WeightedRandomSampling, | |
randomSeed: Option[Long]) | |
def stream( | |
candidateSources: Set[A], | |
candidateSourceWeights: Map[A, Double], | |
candidates: Map[A, Iterator[Rec]], | |
weightMethod: WeightMethod = WeightedRandomSampling, | |
random: Option[Random] = None | |
): Stream[Rec] | |
def apply(a: A): Double = candidateSourceWeights.getOrElse(a, 0) | |
} | |
/** | |
* Generates a stream of candidates. | |
* | |
* @param candidateSourceIter an iterator over candidate sources returned by the sampling procedure | |
* @return stream of candidates | |
*/ | |
def next(candidateSourceIter: Iterator[A]): Stream[Rec] | |
def apply(input: Map[A, TraversableOnce[Rec]]): Stream[Rec] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightedCandidateSourceRanker.scala | |
class WeightedCandidateSourceRanker[Target <: HasParams]( | |
basedRanker: WeightedCandidateSourceBaseRanker[ | |
CandidateSourceIdentifier, | |
CandidateUser | |
], | |
shuffleFn: Seq[CandidateUser] => Seq[CandidateUser], | |
dedup: Boolean) | |
extends Ranker[Target, CandidateUser] | |
def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] | |
def group( | |
candidates: Seq[CandidateUser] | |
): Map[CandidateSourceIdentifier, Seq[CandidateUser]] | |
def rankCandidates( | |
input: Map[CandidateSourceIdentifier, Seq[CandidateUser]] | |
): Seq[CandidateUser] | |
object WeightedCandidateSourceRanker | |
def build[Target <: HasParams]( | |
candidateSourceWeight: Map[CandidateSourceIdentifier, Double], | |
shuffleFn: Seq[CandidateUser] => Seq[CandidateUser] = identity, | |
dedup: Boolean = false, | |
randomSeed: Option[Long] = None | |
): WeightedCandidateSourceRanker[Target] | |
object WeightedCandidateSourceRankerWithoutRandomSampling | |
def build[Target <: HasParams]( | |
candidateSourceWeight: Map[CandidateSourceIdentifier, Double] | |
): WeightedCandidateSourceRanker[Target] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightedCandidateSourceRankerFSConfig.scala | |
class WeightedCandidateSourceRankerFSConfig @Inject() extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightedCandidateSourceRankerParams.scala | |
object WeightedCandidateSourceRankerParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\rankers\weighted_candidate_source_ranker\WeightMethod.scala | |
object WeightMethod extends Enumeration | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\stores\LowTweepCredFollowStore.scala | |
class LowTweepCredFollowStore @Inject() (tweepCredOnUserClientColumn: TweepCredOnUserClientColumn) | |
def getLowTweepCredUsers(target: HasRecentFollowedUserIds): Stitch[Seq[CandidateUser]] | |
object LowTweepCredFollowStore | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\dedup\DedupTransform.scala | |
class DedupTransform[Request, Candidate <: UniversalNoun[Long]]() | |
extends Transform[Request, Candidate] | |
def transform(target: Request, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\modify_social_proof\ModifySocialProofTransform.scala | |
object ModifySocialProof | |
def addIntersectionIdsToCandidate( | |
candidate: CandidateUser, | |
followProof: FollowProof, | |
stats: StatsReceiver | |
): CandidateUser | |
def addCandidatesWithSocialContextCountStat( | |
statsReceiver: StatsReceiver, | |
count: Int | |
): Unit | |
class makes a request to gfs/sgs for hydrating additional social proof on each of the | |
* provided candidates. | |
*/ | |
@Singleton | |
class ModifySocialProof @Inject() ( | |
gfsClient: GraphFeatureServiceClient, | |
socialGraphClient: SocialGraphClient, | |
statsReceiver: StatsReceiver, | |
decider: Decider = Decider.True) | |
extends Logging | |
def hydrateSocialProof( | |
userId: Long, | |
candidates: Seq[CandidateUser], | |
intersectionIdsNum: Option[Int] = None, | |
mustCallSgs: Boolean = false, | |
callSgsCachedColumn: Boolean = false, | |
gfsLagDuration: Duration = GfsLagDuration, | |
gfsIntersectionIds: Int = GfsIntersectionIds, | |
sgsIntersectionIds: Int = SgsIntersectionIds, | |
): Stitch[Seq[CandidateUser]] | |
class ModifySocialProofTransform @Inject() (modifySocialProof: ModifySocialProof) | |
extends GatedTransform[HasClientContext with HasParams, CandidateUser] | |
with Logging | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\modify_social_proof\RemoveAccountProofTransform.scala | |
class RemoveAccountProofTransform @Inject() (statsReceiver: StatsReceiver) | |
extends GatedTransform[HasClientContext with HasParams, CandidateUser] | |
def transform( | |
target: HasClientContext with HasParams, | |
items: Seq[CandidateUser] | |
): Stitch[Seq[CandidateUser]] = | |
Stitch.value(items.map | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\ranker_id\RandomRankerIdTransform.scala | |
class appends each candidate's rankerIds with the RandomRankerId. | |
* This is primarily for determining if a candidate was generated via random shuffling. | |
*/ | |
@Singleton | |
class RandomRankerIdTransform @Inject() () extends GatedTransform[HasParams, CandidateUser] | |
def transform( | |
target: HasParams, | |
candidates: Seq[CandidateUser] | |
): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\recommendation_flow_identifier\AddRecommendationFlowIdentifierTransform.scala | |
class AddRecommendationFlowIdentifierTransform @Inject() | |
extends Transform[HasRecommendationFlowIdentifier, CandidateUser] | |
def transform( | |
target: HasRecommendationFlowIdentifier, | |
items: Seq[CandidateUser] | |
): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\tracking_token\TrackingTokenTransform.scala | |
class TrackingTokenTransform @Inject() (baseStatsReceiver: StatsReceiver) | |
extends Transform[HasDisplayLocation with HasClientContext, CandidateUser] | |
with Logging | |
def profileResults( | |
target: HasDisplayLocation with HasClientContext, | |
candidates: Seq[CandidateUser] | |
) | |
def transform( | |
target: HasDisplayLocation with HasClientContext, | |
candidates: Seq[CandidateUser] | |
): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\weighted_sampling\SamplingTransform.scala | |
class SamplingTransform @Inject() () | |
extends GatedTransform[HasClientContext with HasParams with HasDebugOptions, CandidateUser] | |
def transform( | |
target: HasClientContext with HasParams with HasDebugOptions, | |
candidates: Seq[CandidateUser] | |
): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\weighted_sampling\SamplingTransformFSConfig.scala | |
class SamplingTransformFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\transforms\weighted_sampling\SamplingTransformParams.scala | |
object SamplingTransformParams | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\CollectionUtil.scala | |
object CollectionUtil | |
def transposeLazy[A](seq: Seq[Seq[A]]): Stream[Seq[A]] = | |
seq.filter(_.nonEmpty) match | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\DisplayLocationProductConverterUtil.scala | |
object DisplayLocationProductConverterUtil | |
def productToDisplayLocation(product: Product): DisplayLocation | |
def displayLocationToProduct(displayLocation: DisplayLocation): Product | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\MergeUtil.scala | |
object MergeUtil | |
def weightedRoundRobin[T]( | |
items: Seq[T] | |
)( | |
implicit weighted: Weighted[T] | |
): Stream[T] | |
def next(): Stream[T] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\RandomUtil.scala | |
object RandomUtil | |
def weightedRandomSamplingWithReplacement[T]( | |
items: Seq[T], | |
random: Option[Random] = None | |
)( | |
implicit weighted: Weighted[T] | |
): Stream[T] | |
def next(): Stream[T] | |
def weightedRandomShuffle[T]( | |
items: Seq[T], | |
random: Option[Random] = None | |
)( | |
implicit weighted: Weighted[T] | |
): Stream[T] | |
def next(it: Seq[T]): Stream[T] | |
def weightedRandomShuffleByRank[T]( | |
items: Seq[T], | |
rankToWeight: Int => Double, | |
random: Option[Random] = None | |
): Stream[T] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\RescueWithStatsUtils.scala | |
object RescueWithStatsUtils | |
def rescueWithStats[T]( | |
s: Stitch[Seq[T]], | |
stats: StatsReceiver, | |
source: String | |
): Stitch[Seq[T]] | |
def rescueOptionalWithStats[T]( | |
s: Stitch[Option[T]], | |
stats: StatsReceiver, | |
source: String | |
): Stitch[Option[T]] | |
def rescueWithStatsWithin[T]( | |
s: Stitch[Seq[T]], | |
stats: StatsReceiver, | |
source: String, | |
timeout: Duration | |
): Stitch[Seq[T]] | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\UserSignupUtil.scala | |
object UserSignupUtil | |
.\follow-recommendations-service\common\src\main\scala\com\twitter\follow_recommendations\common\utils\Weighted.scala | |
class for any Recommendation type that has a weight | |
* | |
*/ | |
trait Weighted[-Rec] | |
def apply(rec: Rec): Double | |
} | |
object Weighted | |
object WeightedTuple extends Weighted[(_, Double)] | |
def apply(rec: (_, Double)): Double = rec._2 | |
} | |
def fromFunction[Rec](f: Rec => Double): Weighted[Rec] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\FollowRecommendationsServiceThriftServer.scala | |
object FollowRecommendationsServiceThriftServerMain extends FollowRecommendationsServiceThriftServer | |
class FollowRecommendationsServiceThriftServer | |
extends ThriftServer | |
with Mtls | |
with HttpServer | |
with HttpMtls | |
def configureThrift(router: ThriftRouter): Unit | |
def configureThriftServer(server: ThriftMux.Server): ThriftMux.Server | |
def configureHttp(router: HttpRouter): Unit = router.add( | |
ProductMixerController[FollowRecommendationsThriftService.MethodPerEndpoint]( | |
this.injector, | |
FollowRecommendationsThriftService.ExecutePipeline)) | |
override def warmup(): Unit | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\Action.scala | |
class Action(text: String, actionURL: String) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\FeedbackAction.scala | |
trait FeedbackAction | |
def toThrift: t.FeedbackAction | |
} | |
case class DismissUserId() extends FeedbackAction | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\Footer.scala | |
class Footer(action: Option[Action]) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\Header.scala | |
class Header(title: Title) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\SocialProof.scala | |
trait SocialProof | |
case class GeoContextProof(popularInCountryText: ExternalString) extends SocialProof | |
case class FollowedByUsersProof(text1: ExternalString, text2: ExternalString, textN: ExternalString) | |
extends SocialProof | |
sealed trait SocialText | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\Title.scala | |
class Title(text: String) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\assembler\models\WTFPresentation.scala | |
trait WTFPresentation | |
def toThrift: t.WTFPresentation | |
} | |
case class UserList( | |
userBioEnabled: Boolean, | |
userBioTruncated: Boolean, | |
userBioMaxLines: Option[Long], | |
feedbackAction: Option[FeedbackAction]) | |
extends WTFPresentation | |
def toThrift: t.WTFPresentation | |
object UserList | |
def fromUserListOptions( | |
userListOptions: UserListOptions | |
): UserList | |
class Carousel( | |
feedbackAction: Option[FeedbackAction]) | |
extends WTFPresentation | |
def toThrift: t.WTFPresentation | |
object Carousel | |
def fromCarouselOptions( | |
carouselOptions: CarouselOptions | |
): Carousel | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\blenders\PromotedAccountsBlender.scala | |
class PromotedAccountsBlender @Inject() (statsReceiver: StatsReceiver) | |
extends Transform[Int, Recommendation] | |
with Logging | |
def transform( | |
maxResults: Int, | |
items: Seq[Recommendation] | |
): Stitch[Seq[Recommendation]] | |
def mergePromotedAccounts( | |
organicUsers: Seq[Recommendation], | |
promotedUsers: Seq[Recommendation] | |
): Seq[Recommendation] | |
def mergeAccountWithIndex( | |
organicUsers: Seq[Recommendation], | |
promotedUsers: Seq[Recommendation], | |
index: Int | |
): Stream[Recommendation] | |
def blendPromotedAccount( | |
organic: Seq[Recommendation], | |
promoted: Seq[Recommendation], | |
maxResults: Int | |
): Seq[Recommendation] | |
def isBlendPromotedNeeded( | |
organicSize: Int, | |
promotedSize: Int, | |
maxResults: Int | |
): Boolean | |
object PromotedAccountsBlender | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\ConfigBuilder.scala | |
class ConfigBuilder @Inject() ( | |
deciderConfigs: DeciderConfigs, | |
featureSwitchConfigs: FeatureSwitchConfigs) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\DeciderConfigs.scala | |
class DeciderConfigs @Inject() (deciderGateBuilder: DeciderGateBuilder) | |
object DeciderConfigs | |
object UserOrGuestOrRequest extends RecipientBuilder | |
def apply(requestContext: BaseRequestContext): Option[Recipient] = requestContext match | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\FeatureSwitchConfigs.scala | |
class FeatureSwitchConfigs @Inject() ( | |
globalFeatureSwitchConfig: GlobalFeatureSwitchConfig, | |
featureHydrationSourcesFSConfig: FeatureHydrationSourcesFSConfig, | |
weightedCandidateSourceRankerFSConfig: WeightedCandidateSourceRankerFSConfig, | |
// Flow related config | |
contentRecommenderFlowFSConfig: ContentRecommenderFlowFSConfig, | |
postNuxMlFlowFSConfig: PostNuxMlFlowFSConfig, | |
// Candidate source related config | |
crowdSearchAccountsFSConfig: CrowdSearchAccountsFSConfig, | |
offlineStpSourceFsConfig: OfflineStpSourceFsConfig, | |
onlineSTPSourceFSConfig: OnlineSTPSourceFSConfig, | |
popGeoSourceFSConfig: PopGeoSourceFSConfig, | |
popGeoQualityFollowFSConfig: PopGeoQualityFollowSourceFSConfig, | |
realGraphOonFSConfig: RealGraphOonFSConfig, | |
repeatedProfileVisitsFSConfig: RepeatedProfileVisitsFSConfig, | |
recentEngagementSimilarUsersFSConfig: RecentEngagementSimilarUsersFSConfig, | |
recentFollowingRecentFollowingExpansionSourceFSConfig: RecentFollowingRecentFollowingExpansionSourceFSConfig, | |
simsExpansionFSConfig: SimsExpansionFSConfig, | |
simsSourceFSConfig: SimsSourceFSConfig, | |
socialProofEnforcedCandidateSourceFSConfig: SocialProofEnforcedCandidateSourceFSConfig, | |
triangularLoopsFSConfig: TriangularLoopsFSConfig, | |
userUserGraphFSConfig: UserUserGraphFSConfig, | |
// Predicate related configs | |
gizmoduckPredicateFSConfig: GizmoduckPredicateFSConfig, | |
hssPredicateFSConfig: HssPredicateFSConfig, | |
sgsPredicateFSConfig: SgsPredicateFSConfig, | |
ppmiLocaleSourceFSConfig: PPMILocaleFollowSourceFSConfig, | |
topOrganicFollowsAccountsFSConfig: TopOrganicFollowsAccountsFSConfig, | |
statsReceiver: StatsReceiver) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\GlobalFeatureSwitchConfig.scala | |
class GlobalFeatureSwitchConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\ParamsFactory.scala | |
class ParamsFactory @Inject() ( | |
config: Config, | |
requestContextFactory: RequestContextFactory, | |
statsReceiver: StatsReceiver) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\RequestContextFactory.scala | |
class RequestContextFactory @Inject() (featureSwitches: FeatureSwitches, decider: Decider) | |
def apply( | |
clientContext: ClientContext, | |
displayLocation: DisplayLocation, | |
featureOverrides: Map[String, FeatureValue] | |
): RequestContext | |
def getFeatureContext( | |
clientContext: ClientContext, | |
displayLocation: DisplayLocation, | |
featureOverrides: Map[String, FeatureValue] | |
): FeatureContext | |
def getFeatureSwitchRecipient( | |
clientContext: ClientContext | |
): FeatureSwitchRecipient | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\candidates\CandidateUserContextFactory.scala | |
class CandidateUserContextFactory @Inject() ( | |
@Named(PRODUCER_SIDE_FEATURE_SWITCHES) featureSwitches: FeatureSwitches, | |
decider: Decider) | |
def apply( | |
candidateUser: CandidateUser, | |
displayLocation: DisplayLocation | |
): CandidateUserContext | |
def getFeatureContext( | |
candidateUser: CandidateUser, | |
displayLocation: DisplayLocation | |
): FeatureContext | |
def getFeatureSwitchRecipient( | |
candidateUser: CandidateUser | |
): FeatureSwitchRecipient | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\candidates\CandidateUserParamsFactory.scala | |
class CandidateUserParamsFactory[T <: HasParams with HasDisplayLocation] @Inject() ( | |
config: Config, | |
candidateContextFactory: CandidateUserContextFactory, | |
statsReceiver: StatsReceiver) | |
def apply(candidateContext: CandidateUser, request: T): CandidateUser | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\candidates\HydrateCandidateParamsTransform.scala | |
class HydrateCandidateParamsTransform[Target <: HasParams with HasDisplayLocation] @Inject() ( | |
candidateParamsFactory: CandidateUserParamsFactory[Target]) | |
extends Transform[Target, CandidateUser] | |
with Logging | |
def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\common\FeatureSwitchConfig.scala | |
trait FeatureSwitchConfig | |
def booleanFSParams: Seq[Param[Boolean] with FSName] = Nil | |
def intFSParams: Seq[FSBoundedParam[Int]] = Nil | |
def longFSParams: Seq[FSBoundedParam[Long]] = Nil | |
def doubleFSParams: Seq[FSBoundedParam[Double]] = Nil | |
def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Nil | |
def optionalDoubleFSParams: Seq[ | |
(BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName) | |
] = Nil | |
def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = Nil | |
/** | |
* Apply overrides in list when the given FS Key is enabled. | |
* This override type does NOT work with experiments. Params here will be evaluated for every | |
* request IMMEDIATELY, not upon param.apply. If you would like to use an experiment pls use | |
* the primitive type or ENUM overrides. | |
*/ | |
def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = Map.empty | |
} | |
object FeatureSwitchConfig | |
def merge(configs: Seq[FeatureSwitchConfig]): FeatureSwitchConfig = new FeatureSwitchConfig | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\deciders\DeciderKey.scala | |
object DeciderKey extends DeciderKeyEnum | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\deciders\DeciderParams.scala | |
object DeciderParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\configapi\params\GlobalParams.scala | |
object GlobalParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\CandidateUserDebugParamsBuilder.scala | |
class CandidateUserDebugParamsBuilder @Inject() (paramsFactory: ParamsFactory) | |
def fromThrift(req: t.ScoringUserRequest): CandidateUserDebugParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\RecommendationRequestBuilder.scala | |
class RecommendationRequestBuilder @Inject() ( | |
requestBuilderUserFetcher: RequestBuilderUserFetcher, | |
statsReceiver: StatsReceiver) | |
def fromThrift(tRequest: t.RecommendationRequest): Stitch[RecommendationRequest] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\RequestBuilderUserFetcher.scala | |
class RequestBuilderUserFetcher @Inject() ( | |
gizmoduck: Gizmoduck, | |
statsReceiver: StatsReceiver, | |
decider: Decider) | |
def fetchUser(userIdOpt: Option[Long]): Stitch[Option[User]] | |
def enableDecider(userId: Long): Boolean | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\ScoringUserRequestBuilder.scala | |
class ScoringUserRequestBuilder @Inject() ( | |
requestBuilderUserFetcher: RequestBuilderUserFetcher, | |
candidateUserDebugParamsBuilder: CandidateUserDebugParamsBuilder, | |
statsReceiver: StatsReceiver) | |
def fromThrift(req: t.ScoringUserRequest): Stitch[ScoringUserRequest] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\controllers\ThriftController.scala | |
class ThriftController @Inject() ( | |
userScoringService: UserScoringService, | |
recommendationRequestBuilder: RecommendationRequestBuilder, | |
scoringUserRequestBuilder: ScoringUserRequestBuilder, | |
productPipelineSelector: ProductPipelineSelector, | |
paramsFactory: ParamsFactory) | |
extends Controller(FollowRecommendationsThriftService) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\ads\PromotedAccountsFlow.scala | |
class PromotedAccountsFlow @Inject() ( | |
promotedAccountsCandidateSource: PromotedAccountsCandidateSource, | |
trackingTokenTransform: TrackingTokenTransform, | |
baseStatsReceiver: StatsReceiver, | |
@Flag("fetch_prod_promoted_accounts") fetchProductionPromotedAccounts: Boolean) | |
extends RecommendationFlow[PromotedAccountsFlowRequest, CandidateUser] | |
def targetEligibility: Predicate[PromotedAccountsFlowRequest] = | |
new ParamPredicate[PromotedAccountsFlowRequest]( | |
PromotedAccountsFlowParams.TargetEligibility | |
) | |
protected override def candidateSources( | |
target: PromotedAccountsFlowRequest | |
): Seq[CandidateSource[PromotedAccountsFlowRequest, CandidateUser]] | |
def preRankerCandidateFilter: Predicate[ | |
(PromotedAccountsFlowRequest, CandidateUser) | |
] | |
def selectRanker( | |
target: PromotedAccountsFlowRequest | |
): Ranker[PromotedAccountsFlowRequest, CandidateUser] | |
def postRankerTransform: Transform[ | |
PromotedAccountsFlowRequest, | |
CandidateUser | |
] | |
def validateCandidates: Predicate[ | |
(PromotedAccountsFlowRequest, CandidateUser) | |
] | |
def transformResults: Transform[PromotedAccountsFlowRequest, CandidateUser] | |
def resultsConfig( | |
target: PromotedAccountsFlowRequest | |
): RecommendationResultsConfig | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\ads\PromotedAccountsFlowParams.scala | |
abstract class PromotedAccountsFlowParams[A](default: A) extends Param[A](default) | |
object PromotedAccountsFlowParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\ads\PromotedAccountsFlowRequest.scala | |
class PromotedAccountsFlowRequest( | |
override val clientContext: ClientContext, | |
override val params: Params, | |
displayLocation: DisplayLocation, | |
profileId: Option[Long], | |
// note we also add userId and profileId to excludeUserIds | |
excludeIds: Seq[Long]) | |
extends HasParams | |
with HasClientContext | |
with HasExcludedUserIds | |
with HasDisplayLocation | |
def toAdsRequest(fetchProductionPromotedAccounts: Boolean): AdRequest | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\ads\PromotedAccountsUtil.scala | |
object PromotedAccountsUtil | |
def toCandidateUser(promotedCandidateUser: PromotedCandidateUser): CandidateUser | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlow.scala | |
class ContentRecommenderFlow @Inject() ( | |
contentRecommenderFlowCandidateSourceRegistry: ContentRecommenderFlowCandidateSourceRegistry, | |
recentFollowingPredicate: RecentFollowingPredicate, | |
gizmoduckPredicate: GizmoduckPredicate, | |
inactivePredicate: InactivePredicate, | |
sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate, | |
invalidRelationshipPredicate: InvalidRelationshipPredicate, | |
trackingTokenTransform: TrackingTokenTransform, | |
baseStatsReceiver: StatsReceiver) | |
extends RecommendationFlow[ContentRecommenderRequest, CandidateUser] | |
with RecommendationFlowBaseSideEffectsUtil[ContentRecommenderRequest, CandidateUser] | |
with CandidateSourceHoldbackUtil | |
def targetEligibility: Predicate[ContentRecommenderRequest] = | |
new ParamPredicate[ContentRecommenderRequest]( | |
ContentRecommenderParams.TargetEligibility | |
) | |
protected override def candidateSources( | |
target: ContentRecommenderRequest | |
): Seq[CandidateSource[ContentRecommenderRequest, CandidateUser]] | |
object recentFollowingGatedPredicate | |
extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)]( | |
recentFollowingPredicate, | |
recentFollowingPredicateStats | |
) | |
def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean = | |
item._1.params(ContentRecommenderParams.EnableRecentFollowingPredicate) | |
} | |
object invalidRelationshipGatedPredicate | |
extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)]( | |
invalidRelationshipPredicate, | |
invalidRelationshipPredicateStats | |
) | |
def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean = | |
item._1.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate) | |
} | |
ExcludedUserIdPredicate | |
.observe(preRankerFilterStats.scope("exclude_user_id_predicate")) | |
.andThen(recentFollowingGatedPredicate.observe(recentFollowingPredicateStats)) | |
.andThen(invalidRelationshipGatedPredicate.observe(invalidRelationshipPredicateStats)) | |
} | |
/** | |
* rank the candidates | |
*/ | |
protected override def selectRanker( | |
target: ContentRecommenderRequest | |
): Ranker[ContentRecommenderRequest, CandidateUser] | |
def postRankerTransform: Transform[ | |
ContentRecommenderRequest, | |
CandidateUser | |
] | |
def validateCandidates: Predicate[ | |
(ContentRecommenderRequest, CandidateUser) | |
] | |
def transformResults: Transform[ContentRecommenderRequest, CandidateUser] | |
def resultsConfig( | |
target: ContentRecommenderRequest | |
): RecommendationResultsConfig | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowCandidateSourceRegistry.scala | |
class ContentRecommenderFlowCandidateSourceRegistry @Inject() ( | |
// social based | |
forwardPhoneBookSource: ForwardPhoneBookSource, | |
forwardEmailBookSource: ForwardEmailBookSource, | |
reversePhoneBookSource: ReversePhoneBookSource, | |
reverseEmailBookSource: ReverseEmailBookSource, | |
offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, | |
triangularLoopsSource: TriangularLoopsSource, | |
userUserGraphCandidateSource: UserUserGraphCandidateSource, | |
realGraphOonSource: RealGraphOonV2Source, | |
recentFollowingRecentFollowingExpansionSource: RecentFollowingRecentFollowingExpansionSource, | |
// activity based | |
recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource, | |
recentEngagementSimilarUsersSource: RecentEngagementSimilarUsersSource, | |
repeatedProfileVisitsSource: RepeatedProfileVisitsSource, | |
// geo based | |
popCountrySource: PopCountrySource, | |
popGeohashSource: PopGeohashSource, | |
popCountryBackFillSource: PopCountryBackFillSource, | |
crowdSearchAccountsSource: CrowdSearchAccountsSource, | |
topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource, | |
ppmiLocaleFollowSource: PPMILocaleFollowSource, | |
baseStatsReceiver: StatsReceiver) | |
extends CandidateSourceRegistry[ContentRecommenderRequest, CandidateUser] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowCandidateSourceWeights.scala | |
object ContentRecommenderFlowCandidateSourceWeights | |
def getWeights( | |
params: Params | |
): Map[CandidateSourceIdentifier, Double] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowCandidateSourceWeightsParams.scala | |
object ContentRecommenderFlowCandidateSourceWeightsParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowFeatureSwitchKeys.scala | |
object ContentRecommenderFlowFeatureSwitchKeys | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderFlowFSConfig.scala | |
class ContentRecommenderFlowFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderParams.scala | |
abstract class ContentRecommenderParams[A](default: A) extends Param[A](default) | |
object ContentRecommenderParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderRequest.scala | |
class ContentRecommenderRequest( | |
override val params: Params, | |
override val clientContext: ClientContext, | |
inputExcludeUserIds: Seq[Long], | |
override val recentFollowedUserIds: Option[Seq[Long]], | |
override val recentFollowedByUserIds: Option[Seq[Long]], | |
override val invalidRelationshipUserIds: Option[Set[Long]], | |
override val displayLocation: DisplayLocation, | |
maxResults: Option[Int] = None, | |
override val debugOptions: Option[DebugOptions] = None, | |
override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None, | |
override val userState: Option[UserState] = None) | |
extends HasParams | |
with HasClientContext | |
with HasDisplayLocation | |
with HasDebugOptions | |
with HasRecentFollowedUserIds | |
with HasRecentFollowedByUserIds | |
with HasInvalidRelationshipUserIds | |
with HasExcludedUserIds | |
with HasUserState | |
with HasGeohashAndCountryCode | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\content_recommender_flow\ContentRecommenderRequestBuilder.scala | |
class ContentRecommenderRequestBuilder @Inject() ( | |
socialGraph: SocialGraphClient, | |
userLocationFetcher: UserLocationFetcher, | |
userStateClient: UserStateClient, | |
statsReceiver: StatsReceiver) | |
def build(req: ProductRequest): Stitch[ContentRecommenderRequest] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlCandidateSourceRegistry.scala | |
class PostNuxMlCandidateSourceRegistry @Inject() ( | |
crowdSearchAccountsCandidateSource: CrowdSearchAccountsSource, | |
topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource, | |
linearRegressionfollow2vecNearestNeighborsStore: LinearRegressionFollow2vecNearestNeighborsStore, | |
forwardEmailBookSource: ForwardEmailBookSource, | |
forwardPhoneBookSource: ForwardPhoneBookSource, | |
offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, | |
onlineSTPSource: OnlineSTPSourceScorer, | |
popCountrySource: PopCountrySource, | |
popCountryBackFillSource: PopCountryBackFillSource, | |
popGeohashSource: PopGeohashSource, | |
recentEngagementDirectFollowSimilarUsersSource: RecentEngagementSimilarUsersSource, | |
recentEngagementNonDirectFollowSource: RecentEngagementNonDirectFollowSource, | |
recentEngagementDirectFollowSalsaExpansionSource: RecentEngagementDirectFollowSalsaExpansionSource, | |
recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource, | |
realGraphOonV2Source: RealGraphOonV2Source, | |
repeatedProfileVisitSource: RepeatedProfileVisitsSource, | |
reverseEmailBookSource: ReverseEmailBookSource, | |
reversePhoneBookSource: ReversePhoneBookSource, | |
triangularLoopsSource: TriangularLoopsSource, | |
userUserGraphCandidateSource: UserUserGraphCandidateSource, | |
ppmiLocaleFollowSource: PPMILocaleFollowSource, | |
popGeohashQualityFollowSource: PopGeohashQualityFollowSource, | |
baseStatsReceiver: StatsReceiver, | |
) extends CandidateSourceRegistry[PostNuxMlRequest, CandidateUser] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlCandidateSourceWeightParams.scala | |
abstract class PostNuxMlCandidateSourceWeightParams[A](default: A) extends Param[A](default) | |
object PostNuxMlCandidateSourceWeightParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlCombinedRankerBuilder.scala | |
class PostNuxMlCombinedRankerBuilder[ | |
T <: HasParams with HasSimilarToContext with HasClientContext with HasExcludedUserIds with HasDisplayLocation with HasDebugOptions with HasPreFetchedFeature with HasDismissedUserIds with HasQualityFactor] @Inject() ( | |
firstNRanker: FirstNRanker[T], | |
hydrateFeaturesTransform: HydrateFeaturesTransform[T], | |
hydrateCandidateParamsTransform: HydrateCandidateParamsTransform[T], | |
mlRanker: MlRanker[T], | |
statsReceiver: StatsReceiver) | |
def build( | |
request: T, | |
candidateSourceWeights: Map[CandidateSourceIdentifier, Double] | |
): Ranker[T, CandidateUser] | |
def buildMainRanker( | |
request: T, | |
isMainRankerPostNuxProd: Boolean, | |
displayLocationStats: StatsReceiver | |
): Ranker[T, CandidateUser] | |
def buildInterleaveRanker( | |
request: T, | |
isMainRankerPostNuxProd: Boolean, | |
interleaveRankerStats: StatsReceiver | |
): Ranker[T, CandidateUser] | |
def buildFatigueRanker( | |
request: T, | |
fatigueRankerStats: StatsReceiver | |
): Ranker[T, CandidateUser] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlow.scala | |
class PostNuxMlFlow @Inject() ( | |
postNuxMlCandidateSourceRegistry: PostNuxMlCandidateSourceRegistry, | |
postNuxMlCombinedRankerBuilder: PostNuxMlCombinedRankerBuilder[PostNuxMlRequest], | |
curatedCompetitorListPredicate: CuratedCompetitorListPredicate, | |
gizmoduckPredicate: GizmoduckPredicate, | |
sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate, | |
hssPredicate: HssPredicate, | |
invalidRelationshipPredicate: InvalidRelationshipPredicate, | |
recentFollowingPredicate: RecentFollowingPredicate, | |
nonNearZeroUserActivityPredicate: NonNearZeroUserActivityPredicate, | |
inactivePredicate: InactivePredicate, | |
dismissedCandidatePredicate: DismissedCandidatePredicate, | |
previouslyRecommendedUserIdsPredicate: PreviouslyRecommendedUserIdsPredicate, | |
modifySocialProofTransform: ModifySocialProofTransform, | |
removeAccountProofTransform: RemoveAccountProofTransform, | |
trackingTokenTransform: TrackingTokenTransform, | |
randomRankerIdTransform: RandomRankerIdTransform, | |
candidateParamsFactory: CandidateUserParamsFactory[PostNuxMlRequest], | |
samplingTransform: SamplingTransform, | |
frsLogger: FrsLogger, | |
baseStatsReceiver: StatsReceiver) | |
extends RecommendationFlow[PostNuxMlRequest, CandidateUser] | |
with RecommendationFlowBaseSideEffectsUtil[PostNuxMlRequest, CandidateUser] | |
with CandidateSourceHoldbackUtil | |
def updateTarget(request: PostNuxMlRequest): Stitch[PostNuxMlRequest] | |
def getCandidateSourceIdentifiers( | |
params: Params | |
): Set[CandidateSourceIdentifier] | |
def candidateSources( | |
request: PostNuxMlRequest | |
): Seq[CandidateSource[PostNuxMlRequest, CandidateUser]] | |
object excludeNearZeroUserPredicate | |
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( | |
nonNearZeroUserActivityPredicate, | |
stats.scope("exclude_near_zero_predicate") | |
) | |
def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = | |
item._1.params(PostNuxMlParams.ExcludeNearZeroCandidates) | |
} | |
object invalidRelationshipGatedPredicate | |
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( | |
invalidRelationshipPredicate, | |
stats.scope("invalid_relationship_predicate") | |
) | |
def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = | |
item._1.params(PostNuxMlParams.EnableInvalidRelationshipPredicate) | |
} | |
ExcludedUserIdPredicate | |
.observe(stats.scope("exclude_user_id_predicate")) | |
.andThen( | |
recentFollowingPredicate.observe(stats.scope("recent_following_predicate")) | |
) | |
.andThen( | |
dismissedCandidatePredicate.observe(stats.scope("dismissed_candidate_predicate")) | |
) | |
.andThen( | |
previouslyRecommendedUserIdsPredicate.observe( | |
stats.scope("previously_recommended_user_ids_predicate")) | |
) | |
.andThen( | |
invalidRelationshipGatedPredicate.observe(stats.scope("invalid_relationship_predicate")) | |
) | |
.andThen( | |
excludeNearZeroUserPredicate.observe(stats.scope("exclude_near_zero_user_state")) | |
) | |
.observe(stats.scope("overall_pre_ranker_candidate_filter")) | |
} | |
override protected def selectRanker( | |
request: PostNuxMlRequest | |
): Ranker[PostNuxMlRequest, CandidateUser] | |
object sgsGatedPredicate | |
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( | |
sgsPredicate.observe(sgsPredicateStats), | |
sgsPredicateStats | |
) | |
def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = | |
item._1.params(PostNuxMlParams.EnableSGSPredicate) || | |
SocialGraphClient.enablePostRankerSgsPredicate( | |
item._1.invalidRelationshipUserIds.getOrElse(Set.empty).size) | |
} | |
val hssPredicateStats = stats.scope("hss_predicate") | |
object hssGatedPredicate | |
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( | |
hssPredicate.observe(hssPredicateStats), | |
hssPredicateStats | |
) | |
def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = | |
item._1.params(PostNuxMlParams.EnableHssPredicate) | |
} | |
Predicate | |
.andConcurrently[(PostNuxMlRequest, CandidateUser)]( | |
Seq( | |
competitorPredicate.observe(stats.scope("curated_competitor_predicate")), | |
gizmoduckPredicate.observe(stats.scope("gizmoduck_predicate")), | |
sgsGatedPredicate, | |
hssGatedPredicate, | |
inactivePredicate.observe(stats.scope("inactive_predicate")), | |
) | |
) | |
// to avoid dilutions, we need to apply the receiver holdback predicates at the very last step | |
.andThen(pymkProducerHoldbackPredicate.observe(stats.scope("pymk_receiver_side_holdback"))) | |
.andThen(producerHoldbackPredicate.observe(stats.scope("receiver_side_holdback"))) | |
.observe(stats.scope("overall_validate_candidates")) | |
} | |
override protected val transformResults: Transform[PostNuxMlRequest, CandidateUser] | |
def resultsConfig(request: PostNuxMlRequest): RecommendationResultsConfig | |
def applySideEffects( | |
target: PostNuxMlRequest, | |
candidateSources: Seq[CandidateSource[PostNuxMlRequest, CandidateUser]], | |
candidatesFromCandidateSources: Seq[CandidateUser], | |
mergedCandidates: Seq[CandidateUser], | |
filteredCandidates: Seq[CandidateUser], | |
rankedCandidates: Seq[CandidateUser], | |
transformedCandidates: Seq[CandidateUser], | |
truncatedCandidates: Seq[CandidateUser], | |
results: Seq[CandidateUser] | |
): Stitch[Unit] | |
object PostNuxMlFlow | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlowCandidateSourceWeights.scala | |
object PostNuxMlFlowCandidateSourceWeights | |
def getWeights(params: Params): Map[CandidateSourceIdentifier, Double] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala | |
object PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlowFeatureSwitchKeys.scala | |
object PostNuxMlFlowFeatureSwitchKeys | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlFlowFSConfig.scala | |
class PostNuxMlFlowFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlParams.scala | |
abstract class PostNuxMlParams[A](default: A) extends Param[A](default) | |
object PostNuxMlParams | |
object FetchCandidateSourceBudget extends PostNuxMlParams[Duration](90.millisecond) | |
// WTF Impression Store has very high tail latency (p9990 or p9999), but p99 latency is pretty good (~100ms) | |
// set the time budget for this step to be 200ms to make the performance of service more predictable | |
case object FatigueRankerBudget extends PostNuxMlParams[Duration](200.millisecond) | |
case object MlRankerBudget | |
extends FSBoundedParam[Duration]( | |
name = PostNuxMlFlowFeatureSwitchKeys.MLRankerBudget, | |
default = 400.millisecond, | |
min = 100.millisecond, | |
max = 800.millisecond) | |
with HasDurationConversion | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlRequest.scala | |
class PostNuxMlRequest( | |
override val params: Params, | |
override val clientContext: ClientContext, | |
override val similarToUserIds: Seq[Long], | |
inputExcludeUserIds: Seq[Long], | |
override val recentFollowedUserIds: Option[Seq[Long]], | |
override val invalidRelationshipUserIds: Option[Set[Long]], | |
override val recentFollowedByUserIds: Option[Seq[Long]], | |
override val dismissedUserIds: Option[Seq[Long]], | |
override val displayLocation: DisplayLocation, | |
maxResults: Option[Int] = None, | |
override val debugOptions: Option[DebugOptions] = None, | |
override val wtfImpressions: Option[Seq[WtfImpression]], | |
override val uttInterestIds: Option[Seq[Long]] = None, | |
override val customInterests: Option[Seq[String]] = None, | |
override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None, | |
inputPreviouslyRecommendedUserIds: Option[Set[Long]] = None, | |
inputPreviouslyFollowedUserIds: Option[Set[Long]] = None, | |
override val isSoftUser: Boolean = false, | |
override val userState: Option[UserState] = None, | |
override val qualityFactor: Option[Double] = None) | |
extends HasParams | |
with HasSimilarToContext | |
with HasClientContext | |
with HasExcludedUserIds | |
with HasDisplayLocation | |
with HasDebugOptions | |
with HasGeohashAndCountryCode | |
with HasPreFetchedFeature | |
with HasDismissedUserIds | |
with HasInterestIds | |
with HasPreviousRecommendationsContext | |
with HasIsSoftUser | |
with HasUserState | |
with HasInvalidRelationshipUserIds | |
with HasQualityFactor | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlRequestBuilder.scala | |
class PostNuxMlRequestBuilder @Inject() ( | |
socialGraph: SocialGraphClient, | |
wtfImpressionStore: WtfImpressionStore, | |
dismissStore: DismissStore, | |
userLocationFetcher: UserLocationFetcher, | |
interestServiceClient: InterestServiceClient, | |
userStateClient: UserStateClient, | |
statsReceiver: StatsReceiver) | |
extends Logging | |
def build( | |
req: ProductRequest, | |
previouslyRecommendedUserIds: Option[Set[Long]] = None, | |
previouslyFollowedUserIds: Option[Set[Long]] = None | |
): Stitch[PostNuxMlRequest] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\flows\post_nux_ml\PostNuxMlRequestBuilderParams.scala | |
object PostNuxMlRequestBuilderParams | |
object TopicIdFetchBudget | |
extends FSBoundedParam[Duration]( | |
name = "post_nux_ml_request_builder_topic_id_fetch_budget_millis", | |
default = 200.millisecond, | |
min = 80.millisecond, | |
max = 400.millisecond) | |
with HasDurationConversion | |
object DismissedIdScanBudget | |
extends FSBoundedParam[Duration]( | |
name = "post_nux_ml_request_builder_dismissed_id_scan_budget_millis", | |
default = 200.millisecond, | |
min = 80.millisecond, | |
max = 400.millisecond) | |
with HasDurationConversion | |
object WTFImpressionsScanBudget | |
extends FSBoundedParam[Duration]( | |
name = "post_nux_ml_request_builder_wtf_impressions_scan_budget_millis", | |
default = 200.millisecond, | |
min = 80.millisecond, | |
max = 400.millisecond) | |
with HasDurationConversion | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\logging\FrsLogger.scala | |
class we use to log data into: | |
* 1) logs.follow_recommendations_logs | |
* | |
* This logger logs data for 2 endpoints: getRecommendations, scoreUserCandidates | |
* All data scribed via this logger have to be converted into the same thrift type: RecommendationLog | |
* | |
* 2) logs.frs_recommendation_flow_logs | |
* | |
* This logger logs recommendation flow data for getRecommendations requests | |
* All data scribed via this logger have to be converted into the same thrift type: FrsRecommendationFlowLog | |
*/ | |
@Singleton | |
class FrsLogger @Inject() ( | |
@Named(GuiceNamedConstants.REQUEST_LOGGER) loggerFactory: LoggerFactory, | |
@Named(GuiceNamedConstants.FLOW_LOGGER) flowLoggerFactory: LoggerFactory, | |
stats: StatsReceiver, | |
@Flag("log_results") serviceShouldLogResults: Boolean) | |
extends ScribeSerialization | |
def logRecommendationResult( | |
request: RecommendationRequest, | |
response: RecommendationResponse | |
): Unit | |
def logScoringResult(request: ScoringUserRequest, response: ScoringUserResponse): Unit | |
def logRecommendationFlowData[Target <: HasClientContext with HasIsSoftUser with HasParams]( | |
request: Target, | |
flowData: RecommendationFlowData[Target] | |
): Unit | |
def shouldLog(debugParamsOpt: Option[DebugParams]): Boolean = | |
debugParamsOpt match | |
object FrsLogger | |
def mkProvider(clientContext: ClientContext) = new ClientDataProvider | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\CandidateSourceType.scala | |
object CandidateSourceType extends Enumeration | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\DebugParams.scala | |
class DebugParams( | |
featureOverrides: Option[Map[String, ConfigApiFeatureValue]], | |
debugOptions: Option[DebugOptions]) | |
object DebugParams | |
def fromThrift(thrift: t.DebugParams): DebugParams = DebugParams( | |
featureOverrides = thrift.featureOverrides.map | |
def toOfflineThrift(model: DebugParams): offline.OfflineDebugParams = | |
offline.OfflineDebugParams(randomizationSeed = model.debugOptions.flatMap(_.randomizationSeed)) | |
} | |
trait HasFrsDebugParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\DisplayContext.scala | |
trait DisplayContext | |
def toOfflineThrift: offline.OfflineDisplayContext | |
} | |
object DisplayContext | |
class Profile(profileId: Long) extends DisplayContext | |
class Search(searchQuery: String) extends DisplayContext | |
class Rux(focalAuthorId: Long) extends DisplayContext | |
class Topic(topicId: Long) extends DisplayContext | |
class ReactiveFollow(followedUserIds: Seq[Long]) extends DisplayContext | |
class NuxInterests(flowContext: Option[FlowContext], uttInterestIds: Option[Seq[Long]]) | |
extends DisplayContext | |
class PostNuxFollowTask(flowContext: Option[FlowContext]) extends DisplayContext | |
class AdCampaignTarget(similarToUserIds: Seq[Long]) extends DisplayContext | |
class ConnectTab( | |
byfSeedUserIds: Seq[Long], | |
similarToUserIds: Seq[Long], | |
engagedUserIds: Seq[RecentlyEngagedUserId]) | |
extends DisplayContext | |
class SimilarToUser(similarToUserId: Long) extends DisplayContext | |
def fromThrift(tDisplayContext: t.DisplayContext): DisplayContext = tDisplayContext match | |
def getDisplayContextAs[T <: DisplayContext: ClassTag](displayContext: DisplayContext): T = | |
displayContext match | |
class UnknownDisplayContextException(name: String) | |
extends Exception(s"Unknown DisplayContext in Thrift: $ | |
class UnexpectedDisplayContextTypeException(displayContext: DisplayContext, expectedType: String) | |
extends Exception(s"DisplayContext $ | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\FeatureValue.scala | |
object FeatureValue | |
def fromThrift(thriftFeatureValue: t.FeatureValue): FeatureValue = thriftFeatureValue match | |
class UnknownFeatureValueException(fieldName: String) | |
extends Exception(s"Unknown FeatureValue name in thrift: $ | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\RecommendationFlowData.scala | |
class RecommendationFlowData[Target <: HasClientContext]( | |
request: Target, | |
recommendationFlowIdentifier: RecommendationPipelineIdentifier, | |
candidateSources: Seq[CandidateSource[Target, CandidateUser]], | |
candidatesFromCandidateSources: Seq[CandidateUser], | |
mergedCandidates: Seq[CandidateUser], | |
filteredCandidates: Seq[CandidateUser], | |
rankedCandidates: Seq[CandidateUser], | |
transformedCandidates: Seq[CandidateUser], | |
truncatedCandidates: Seq[CandidateUser], | |
results: Seq[CandidateUser]) | |
extends HasMarshalling | |
object RecommendationFlowData | |
def userToOfflineRecommendationFlowUserMetadata[Target <: HasClientContext]( | |
request: Target | |
): Option[offline.OfflineRecommendationFlowUserMetadata] | |
def userToOfflineRecommendationFlowSignals[Target <: HasClientContext]( | |
request: Target | |
): Option[offline.OfflineRecommendationFlowSignals] | |
def candidatesToOfflineRecommendationFlowCandidateSourceCandidates[Target <: HasClientContext]( | |
candidateSources: Seq[CandidateSource[Target, CandidateUser]], | |
candidates: Seq[CandidateUser], | |
): Seq[offline.OfflineRecommendationFlowCandidateSourceCandidates] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\RecommendationRequest.scala | |
class RecommendationRequest( | |
clientContext: ClientContext, | |
displayLocation: DisplayLocation, | |
displayContext: Option[DisplayContext], | |
maxResults: Option[Int], | |
cursor: Option[String], | |
excludedIds: Option[Seq[Long]], | |
fetchPromotedContent: Option[Boolean], | |
debugParams: Option[DebugParams] = None, | |
userLocationState: Option[String] = None, | |
isSoftUser: Boolean = false) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\RecommendationResponse.scala | |
class RecommendationResponse(recommendations: Seq[Recommendation]) extends HasMarshalling | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\Request.scala | |
class Request( | |
override val maxResults: Option[Int], | |
override val debugParams: Option[request.DebugParams], | |
override val productContext: Option[ProductContext], | |
override val product: request.Product, | |
override val clientContext: ClientContext, | |
override val serializedRequestCursor: Option[String], | |
override val frsDebugParams: Option[DebugParams], | |
displayLocation: DisplayLocation, | |
excludedIds: Option[Seq[Long]], | |
fetchPromotedContent: Option[Boolean], | |
userLocationState: Option[String] = None) | |
extends ProductMixerRequest | |
with HasFrsDebugParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\ScoringUserRequest.scala | |
class ScoringUserRequest( | |
override val clientContext: ClientContext, | |
override val displayLocation: DisplayLocation, | |
override val params: Params, | |
override val debugOptions: Option[DebugOptions] = None, | |
override val recentFollowedUserIds: Option[Seq[Long]], | |
override val recentFollowedByUserIds: Option[Seq[Long]], | |
override val wtfImpressions: Option[Seq[WtfImpression]], | |
override val similarToUserIds: Seq[Long], | |
candidates: Seq[CandidateUser], | |
debugParams: Option[DebugParams] = None, | |
isSoftUser: Boolean = false) | |
extends HasClientContext | |
with HasDisplayLocation | |
with HasParams | |
with HasDebugOptions | |
with HasPreFetchedFeature | |
with HasSimilarToContext | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\ScoringUserResponse.scala | |
class ScoringUserResponse(candidates: Seq[CandidateUser]) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\models\failures\TimeoutPipelineFailure.scala | |
object TimeoutPipelineFailure | |
def apply(candidateSourceName: String): PipelineFailure | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ABDeciderModule.scala | |
object ABDeciderModule extends TwitterModule | |
def provideABDecider( | |
stats: StatsReceiver, | |
@Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) factory: LoggerFactory | |
): LoggingABDecider | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ConfigApiModule.scala | |
object ConfigApiModule extends TwitterModule | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\DiffyModule.scala | |
object DiffyModule extends TwitterModule | |
def provideDarkTrafficService( | |
serviceIdentifier: ServiceIdentifier | |
): FollowRecommendationsThriftService.ReqRepServicePerEndpoint | |
def provideDarkTrafficFilter( | |
@DarkTrafficService darkService: FollowRecommendationsThriftService.ReqRepServicePerEndpoint, | |
deciderGateBuilder: DeciderGateBuilder, | |
statsReceiver: StatsReceiver, | |
@Flag("environment") env: String | |
): DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\FeatureSwitchesModule.scala | |
object FeaturesSwitchesModule extends TwitterModule | |
def providesFeatureSwitches( | |
abDecider: LoggingABDecider, | |
statsReceiver: StatsReceiver | |
): FeatureSwitches | |
def providesProducerFeatureSwitches( | |
abDecider: LoggingABDecider, | |
statsReceiver: StatsReceiver | |
): FeatureSwitches | |
object ProducerFeatureFilter extends FeatureFilter | |
def filter(feature: Feature): Option[Feature] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\FlagsModule.scala | |
object FlagsModule extends TwitterModule | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ProductRegistryModule.scala | |
object ProductRegistryModule extends TwitterModule | |
def configure(): Unit | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ScorerModule.scala | |
object ScorerModule extends TwitterModule | |
def fileFromResource(resource: String): File | |
def provideEpScorer: EPScorer | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\ScribeModule.scala | |
object ScribeModule extends TwitterModule | |
def provideClientEventsLoggerFactory(stats: StatsReceiver): LoggerFactory | |
def provideFollowRecommendationsLoggerFactory(stats: StatsReceiver): LoggerFactory | |
def provideFrsRecommendationFlowLoggerFactory(stats: StatsReceiver): LoggerFactory | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\modules\TimerModule.scala | |
object TimerModule extends TwitterModule | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\ProdProductRegistry.scala | |
class ProdProductRegistry @Inject() ( | |
exploreTabProduct: ExploreTabProduct, | |
homeTimelineProduct: HomeTimelineProduct, | |
homeTimelineTweetRecsProduct: HomeTimelineTweetRecsProduct, | |
sidebarProduct: SidebarProduct, | |
) extends ProductRegistry | |
def getProductByDisplayLocation(displayLocation: DisplayLocation): common.Product | |
class MissingProductException(displayLocation: DisplayLocation) | |
extends Exception(s"No Product found for $ | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\common\Exceptions.scala | |
abstract class ProductException(message: String) extends Exception(message) | |
class MissingFieldException(productRequest: ProductRequest, fieldName: String) | |
extends ProductException( | |
s"Missing $ | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\common\Product.scala | |
trait Product | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\common\ProductRegistry.scala | |
trait ProductRegistry | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\explore_tab\ExploreTabProduct.scala | |
class ExploreTabProduct @Inject() ( | |
postNuxMlFlow: PostNuxMlFlow, | |
postNuxMlRequestBuilder: PostNuxMlRequestBuilder) | |
extends Product | |
def selectWorkflows( | |
request: ProductRequest | |
): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] | |
def resultsTransformer( | |
request: ProductRequest | |
): Stitch[Transform[ProductRequest, Recommendation]] = | |
Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) | |
override def enabled(request: ProductRequest): Stitch[Boolean] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\explore_tab\configapi\ExploreTabFSConfig.scala | |
class ExploreTabFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\explore_tab\configapi\ExploreTabParams.scala | |
object ExploreTabParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\HomeTimelineProduct.scala | |
class HomeTimelineProduct @Inject() ( | |
postNuxMlFlow: PostNuxMlFlow, | |
postNuxMlRequestBuilder: PostNuxMlRequestBuilder, | |
promotedAccountsFlow: PromotedAccountsFlow, | |
promotedAccountsBlender: PromotedAccountsBlender, | |
productScope: ProductScope, | |
injector: Injector, | |
) extends Product | |
def selectWorkflows( | |
request: ProductRequest | |
): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] | |
def resultsTransformer( | |
request: ProductRequest | |
): Stitch[Transform[ProductRequest, Recommendation]] = Stitch.value(identityTransform) | |
override def enabled(request: ProductRequest): Stitch[Boolean] = | |
Stitch.value(request.params(EnableProduct)) | |
override def layout: Option[Layout] | |
def productMixerProduct: Option[request.Product] = Some(HTLProductMixer) | |
private[home_timeline] def mkPromotedAccountsRequest( | |
req: ProductRequest | |
): PromotedAccountsFlowRequest | |
def getMaxResults(req: ProductRequest): Int | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\HomeTimelineStrings.scala | |
class HomeTimelineStrings @Inject() ( | |
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\HTLProductMixer.scala | |
object HTLProductMixer extends Product | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\configapi\HomeTimelineFSConfig.scala | |
class HomeTimelineFSConfig @Inject() () extends FeatureSwitchConfig | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline\configapi\HomeTimelineParams.scala | |
object HomeTimelineParams | |
object EnableProduct extends Param[Boolean](false) | |
object DefaultMaxResults extends Param[Int](20) | |
object EnableWritingServingHistory | |
extends FSParam[Boolean]("home_timeline_enable_writing_serving_history", false) | |
object DurationGuardrailToForceSuggest | |
extends FSBoundedParam[Duration]( | |
name = "home_timeline_duration_guardrail_to_force_suggest_in_hours", | |
default = 0.hours, | |
min = 0.hours, | |
max = 1000.hours) | |
with HasDurationConversion | |
object SuggestBasedFatigueDuration | |
extends FSBoundedParam[Duration]( | |
name = "home_timeline_suggest_based_fatigue_duration_in_hours", | |
default = 0.hours, | |
min = 0.hours, | |
max = 1000.hours) | |
with HasDurationConversion | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline_tweet_recs\HomeTimelineTweetRecsProduct.scala | |
class HomeTimelineTweetRecsProduct @Inject() ( | |
contentRecommenderFlow: ContentRecommenderFlow, | |
contentRecommenderRequestBuilder: ContentRecommenderRequestBuilder) | |
extends Product | |
def selectWorkflows( | |
request: ProductRequest | |
): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\home_timeline_tweet_recs\configapi\HomeTimelineTweetRecsParams.scala | |
object HomeTimelineTweetRecsParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\sidebar\SidebarProduct.scala | |
class SidebarProduct @Inject() ( | |
postNuxMlFlow: PostNuxMlFlow, | |
postNuxMlRequestBuilder: PostNuxMlRequestBuilder, | |
promotedAccountsFlow: PromotedAccountsFlow, | |
promotedAccountsBlender: PromotedAccountsBlender) | |
extends Product | |
def selectWorkflows( | |
request: ProductRequest | |
): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] | |
def mkPromotedAccountsRequest( | |
req: ProductRequest | |
): PromotedAccountsFlowRequest | |
def getMaxResults(req: ProductRequest): Int | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\products\sidebar\configapi\SidebarParams.scala | |
object SidebarParams | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\FollowRecommendationsServiceWarmupHandler.scala | |
class FollowRecommendationsServiceWarmupHandler @Inject() (warmup: ThriftWarmup) | |
extends Handler | |
with Logging | |
def handle(): Unit | |
def warmupQuery(userId: Long, displayLocation: DisplayLocation): RecommendationRequest | |
def assertWarmupResponse(result: Try[Response[GetRecommendations.SuccessType]]): Unit | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\ProductMixerRecommendationService.scala | |
class ProductMixerRecommendationService @Inject() ( | |
productPipelineRegistry: ProductPipelineRegistry, | |
resultLogger: FrsLogger, | |
baseStats: StatsReceiver) | |
def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse] | |
def convertToProductMixerRequest(frsRequest: RecommendationRequest): Request | |
def convertToProductMixerDebugParams( | |
frsDebugParams: Option[FrsDebugParams] | |
): Option[ProductMixerDebugParams] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\ProductPipelineSelector.scala | |
class ProductPipelineSelector @Inject() ( | |
recommendationsService: RecommendationsService, | |
productMixerRecommendationService: ProductMixerRecommendationService, | |
productPipelineSelectorConfig: ProductPipelineSelectorConfig, | |
baseStats: StatsReceiver) | |
def selectPipeline( | |
request: RecommendationRequest, | |
params: Params | |
): Stitch[RecommendationResponse] | |
def readFromProductMixerPipeline( | |
request: RecommendationRequest, | |
params: Params | |
): Stitch[RecommendationResponse] | |
def readFromOldFrsPipeline( | |
request: RecommendationRequest, | |
params: Params | |
): Stitch[RecommendationResponse] | |
def darkReadAndReturnResult( | |
request: RecommendationRequest, | |
params: Params | |
): Stitch[RecommendationResponse] | |
def compare( | |
request: RecommendationRequest, | |
frsOldPipelineResponse: RecommendationResponse, | |
frsProductMixerResponse: RecommendationResponse | |
): Unit | |
def compareTopNResults( | |
n: Int, | |
frsOldPipelineResponse: RecommendationResponse, | |
frsProductMixerResponse: RecommendationResponse, | |
compareStats: StatsReceiver | |
): Unit | |
def compareUser( | |
oldFrsUser: CandidateUser, | |
productMixerUser: CandidateUser, | |
stats: StatsReceiver | |
): Unit | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\ProductPipelineSelectorConfig.scala | |
class ProductPipelineSelectorConfig | |
def getDarkReadAndExpParams( | |
displayLocation: DisplayLocation | |
): Option[DarkReadAndExpParams] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\ProductRecommenderService.scala | |
class ProductRecommenderService @Inject() ( | |
productRegistry: ProductRegistry, | |
statsReceiver: StatsReceiver) | |
def getRecommendations( | |
request: RecommendationRequest, | |
params: Params | |
): Stitch[Seq[Recommendation]] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\RecommendationsService.scala | |
class RecommendationsService @Inject() ( | |
productRecommenderService: ProductRecommenderService, | |
resultLogger: FrsLogger) | |
def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\UserScoringService.scala | |
class UserScoringService @Inject() ( | |
socialGraph: SocialGraphClient, | |
wtfImpressionStore: WtfImpressionStore, | |
hydrateFeaturesTransform: HydrateFeaturesTransform[ScoringUserRequest], | |
mlRanker: MlRanker[ScoringUserRequest], | |
resultLogger: FrsLogger, | |
stats: StatsReceiver) | |
def get(request: ScoringUserRequest): Stitch[ScoringUserResponse] | |
def hydrate(request: ScoringUserRequest): Stitch[ScoringUserRequest] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\services\exceptions\UnknownExceptionMapper.scala | |
class UnknownLoggingExceptionMapper extends ExceptionMapper[Exception, Throwable] with Logging | |
def handleException(throwable: Exception): Future[Throwable] | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\utils\CandidateSourceHoldbackUtil.scala | |
trait CandidateSourceHoldbackUtil | |
def filterCandidateSources[T <: HasParams]( | |
request: T, | |
sources: Seq[CandidateSource[T, CandidateUser]] | |
): Seq[CandidateSource[T, CandidateUser]] | |
object CandidateSourceHoldbackUtil | |
.\follow-recommendations-service\server\src\main\scala\com\twitter\follow_recommendations\utils\RecommendationFlowBaseSideEffectsUtil.scala | |
trait RecommendationFlowBaseSideEffectsUtil[Target <: HasClientContext, Candidate <: CandidateUser] | |
extends SideEffectsUtil[Target, Candidate] | |
def applySideEffects( | |
target: Target, | |
candidateSources: Seq[CandidateSource[Target, Candidate]], | |
candidatesFromCandidateSources: Seq[Candidate], | |
mergedCandidates: Seq[Candidate], | |
filteredCandidates: Seq[Candidate], | |
rankedCandidates: Seq[Candidate], | |
transformedCandidates: Seq[Candidate], | |
truncatedCandidates: Seq[Candidate], | |
results: Seq[Candidate] | |
): Stitch[Unit] | |
def applySideEffectsCandidateSourceCandidates( | |
target: Target, | |
candidateSources: Seq[CandidateSource[Target, Candidate]], | |
candidatesFromCandidateSources: Seq[Candidate] | |
): Stitch[Unit] | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\common\Configs.scala | |
object Configs | |
def getTimedHdfsShardPath(shardId: Int, path: String, time: Time): String | |
def getHdfsPath(path: String, overrideBaseHdfsPath: Option[String] = None): String | |
def hash(kArr: Array[Byte], seed: Int): Int | |
def hashLong(l: Long, seed: Int): Int | |
def shardForUser(userId: Long): Int | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\Main.scala | |
object Main extends ServerMain | |
class ServerMain extends ThriftServer with Mtls | |
def configureThrift(router: ThriftRouter): Unit | |
def warmup(): Unit | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\controllers\ServerController.scala | |
class ServerController @Inject() ( | |
serverGetIntersectionHandler: ServerGetIntersectionHandler | |
)( | |
implicit statsReceiver: StatsReceiver) | |
extends Controller(thriftscala.Server) | |
def getPresetIntersection: Service[ | |
GetPresetIntersection.Args, | |
GfsIntersectionResponse | |
] | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\handlers\ServerGetIntersectionHandler.scala | |
class ServerGetIntersectionHandler @Inject() ( | |
@Named("ReadThroughGetIntersectionStore") | |
readThroughStore: ReadableStore[GetIntersectionQuery, CachedIntersectionResult], | |
@Named("BypassCacheGetIntersectionStore") | |
readOnlyStore: ReadableStore[GetIntersectionQuery, CachedIntersectionResult] | |
)( | |
implicit statsReceiver: StatsReceiver) | |
extends RequestHandler[GetIntersectionRequest, GfsIntersectionResponse] | |
def apply(request: GetIntersectionRequest): Future[GfsIntersectionResponse] | |
object ServerGetIntersectionHandler | |
class GetIntersectionRequest( | |
userId: Long, | |
candidateUserIds: Seq[Long], | |
featureTypes: Seq[FeatureType], | |
presetFeatureTypes: PresetFeatureTypes, | |
intersectionIdLimit: Option[Int], | |
cacheable: Boolean) | |
object GetIntersectionRequest | |
def fromGfsIntersectionRequest( | |
request: GfsIntersectionRequest, | |
cacheable: Boolean | |
): GetIntersectionRequest | |
def fromGfsPresetIntersectionRequest( | |
request: GfsPresetIntersectionRequest, | |
cacheable: Boolean | |
): GetIntersectionRequest | |
def isZero(opt: Option[Int]): Boolean | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\handlers\ServerWarmupHandler.scala | |
class ServerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler | |
def getRandomRequest: GfsIntersectionRequest | |
def handle(): Unit | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\modules\GetIntersectionStoreModule.scala | |
object GetIntersectionStoreModule extends TwitterModule | |
def provideReadThroughGetIntersectionStore( | |
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, | |
serviceIdentifier: ServiceIdentifier, | |
@Flag(ServerFlagNames.MemCacheClientName) memCacheName: String, | |
@Flag(ServerFlagNames.MemCachePath) memCachePath: String | |
)( | |
implicit statsReceiver: StatsReceiver | |
): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] | |
def provideReadOnlyGetIntersectionStore( | |
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, | |
)( | |
implicit statsReceiver: StatsReceiver | |
): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] | |
def buildMemcacheStore( | |
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, | |
memCacheName: String, | |
memCachePath: String, | |
serviceIdentifier: ServiceIdentifier, | |
)( | |
implicit statsReceiver: StatsReceiver | |
): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\modules\GraphFeatureServiceWorkerClientsModule.scala | |
class GraphFeatureServiceWorkerClients( | |
workers: Seq[thriftscala.Worker.MethodPerEndpoint]) | |
object GraphFeatureServiceWorkerClientsModule extends TwitterModule | |
def provideGraphFeatureServiceWorkerClient( | |
@Flag(ServerFlagNames.NumWorkers) numWorkers: Int, | |
@Flag(ServerFlagNames.ServiceRole) serviceRole: String, | |
@Flag(ServerFlagNames.ServiceEnv) serviceEnv: String, | |
serviceIdentifier: ServiceIdentifier | |
): GraphFeatureServiceWorkerClients | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\modules\LZ4Injection.scala | |
object LZ4Injection extends Injection[Array[Byte], Array[Byte]] | |
def apply(a: Array[Byte]): Array[Byte] = LZ4Injection.fastCompressor.compress(a) | |
override def invert(b: Array[Byte]): Try[Array[Byte]] = Try | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\modules\ServerFlagModule.scala | |
object ServerFlagNames | |
object ServerFlagsModule extends TwitterModule | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\stores\FeatureTypesEncoder.scala | |
object FeatureTypesEncoder | |
def apply(featureTypes: Seq[FeatureType]): String | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\server\stores\GetIntersectionStore.scala | |
class GetIntersectionStore( | |
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[GetIntersectionQuery, CachedIntersectionResult] | |
with Logging | |
def multiGet[K1 <: GetIntersectionQuery]( | |
ks: Set[K1] | |
): Map[K1, Future[Option[CachedIntersectionResult]]] | |
def gfsIntersectionResponseAggregator( | |
responseList: Seq[WorkerIntersectionResponse], | |
features: Seq[FeatureType], | |
candidates: Seq[Long], | |
intersectionIdLimit: Int | |
): Map[Long, CachedIntersectionResult] | |
object GetIntersectionStore | |
class GetIntersectionQuery( | |
userId: Long, | |
candidateId: Long, | |
featureTypes: Seq[FeatureType], | |
presetFeatureTypes: PresetFeatureTypes, | |
featureTypesString: String, | |
calculatedFeatureTypes: Seq[FeatureType], | |
intersectionIdLimit: Int) | |
private[graph_feature_service] object GetIntersectionQuery | |
def buildQueries(request: GetIntersectionRequest): Set[GetIntersectionQuery] | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\util\FeatureTypesCalculator.scala | |
object FeatureTypesCalculator | |
def getFeatureTypes( | |
presetFeatureTypes: PresetFeatureTypes, | |
featureTypes: Seq[FeatureType] | |
): Seq[FeatureType] | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\util\IntersectionValueCalculator.scala | |
object IntersectionValueCalculator | |
def computeArraySize(x: ByteBuffer): Int | |
def apply(x: ByteBuffer, y: ByteBuffer, intersectionIdLimit: Int): WorkerIntersectionValue | |
def computeIntersectionUsingBinarySearchOnLargerByteBuffer( | |
smallArray: ByteBuffer, | |
largeArray: ByteBuffer | |
): Int | |
def computeIntersectionWithIds( | |
smallArray: ByteBuffer, | |
largeArray: ByteBuffer, | |
intersectionLimit: Int | |
): (Int, Seq[Long]) | |
def binarySearch(arr: ByteBuffer, value: Long): Int | |
def apply(x: Array[Long], y: Array[Long], featureType: FeatureType): IntersectionValue | |
def computeIntersectionUsingListMerging[T]( | |
x: Array[T], | |
y: Array[T] | |
)( | |
implicit ordering: Ordering[T] | |
): Int | |
def computeIntersectionUsingBinarySearchOnLargerArray[T]( | |
smallArray: Array[T], | |
largeArray: Array[T] | |
)( | |
implicit ordering: Ordering[T] | |
): Int | |
def binarySearch[T]( | |
arr: Array[T], | |
value: T | |
)( | |
implicit ordering: Ordering[T] | |
): Int | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\Main.scala | |
object Main extends WorkerMain | |
class WorkerMain extends ThriftServer with Mtls | |
def configureThrift(router: ThriftRouter): Unit | |
def warmup(): Unit | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\controllers\WorkerController.scala | |
class WorkerController @Inject() ( | |
workerGetIntersectionHandler: WorkerGetIntersectionHandler | |
)( | |
implicit statsReceiver: StatsReceiver) | |
extends Controller(thriftscala.Worker) | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\handlers\WorkerGetIntersectionHandler.scala | |
class WorkerGetIntersectionHandler @Inject() ( | |
graphContainer: GraphContainer, | |
statsReceiver: StatsReceiver) | |
extends RequestHandler[WorkerIntersectionRequest, WorkerIntersectionResponse] | |
def apply(request: WorkerIntersectionRequest): Future[WorkerIntersectionResponse] | |
object WorkerGetIntersectionHandler | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\handlers\WorkerWarmupHandler.scala | |
class WorkerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging | |
def handle(): Unit | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\modules\GraphContainerProviderModule.scala | |
object GraphContainerProviderModule extends TwitterModule | |
def provideAutoUpdatingGraphs( | |
@Flag(WorkerFlagNames.HdfsCluster) hdfsCluster: String, | |
@Flag(WorkerFlagNames.HdfsClusterUrl) hdfsClusterUrl: String, | |
@Flag(WorkerFlagNames.ShardId) shardId: Int | |
)( | |
implicit statsReceiver: StatsReceiver, | |
timer: Timer | |
): GraphContainer | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\modules\WorkerFlagModule.scala | |
object WorkerFlagNames | |
object WorkerFlagModule extends TwitterModule | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\util\AutoUpdatingGraph.scala | |
class AutoUpdatingGraph( | |
dataPath: String, | |
hdfsCluster: String, | |
hdfsClusterUrl: String, | |
shard: Int, | |
minimumSizeForCompleteGraph: Long, | |
updateIntervalMin: Duration = 1.hour, | |
updateIntervalMax: Duration = 12.hours, | |
deleteInterval: Duration = 2.seconds, | |
sharedSemaphore: Option[AsyncSemaphore] = None | |
)( | |
implicit statsReceiver: StatsReceiver, | |
timer: Timer) | |
extends AutoUpdatingReadOnlyGraph[Long, ByteBuffer]( | |
hdfsCluster, | |
hdfsClusterUrl, | |
shard, | |
minimumSizeForCompleteGraph, | |
updateIntervalMin, | |
updateIntervalMax, | |
deleteInterval, | |
sharedSemaphore | |
) | |
with ConstDBImporter[Long, ByteBuffer] | |
def numGraphShards: Int = Configs.NumGraphShards | |
override def basePath: String = dataPath | |
override val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint | |
override val valueInj: Injection[ByteBuffer, ByteBuffer] = Injection.identity | |
override def get(targetId: Long): Future[Option[ByteBuffer]] = | |
super | |
.get(targetId) | |
.map | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\util\GfsQuery.scala | |
trait GfsQuery | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\util\GraphContainer.scala | |
class GraphContainer( | |
graphs: Map[GraphKey, AutoUpdatingGraph]) | |
def warmup: Future[Unit] | |
.\graph-feature-service\src\main\scala\com\twitter\graph_feature_service\worker\util\GraphKey.scala | |
trait GraphKey | |
def edgeType: EdgeType | |
} | |
sealed trait PartialValueGraph extends GraphKey | |
/** | |
* Follow Graphs | |
*/ | |
object FollowingPartialValueGraph extends PartialValueGraph | |
def edgeType: EdgeType = Following | |
} | |
object FollowedByPartialValueGraph extends PartialValueGraph | |
def edgeType: EdgeType = FollowedBy | |
} | |
/** | |
* Mutual Follow Graphs | |
*/ | |
object MutualFollowPartialValueGraph extends PartialValueGraph | |
.\graph-feature-service\src\main\scalding\com\twitter\graph_feature_service\scalding\GraphFeatureServiceAppBase.scala | |
trait GraphFeatureServiceBaseJob | |
def runOnDateRange( | |
enableValueGraphs: Option[Boolean] = None, | |
enableKeyGraphs: Option[Boolean] = None | |
)( | |
implicit dateRange: DateRange, | |
timeZone: TimeZone, | |
uniqueID: UniqueID | |
): Execution[Unit] | |
/** | |
* Print customized counters in the log | |
*/ | |
def printerCounters[T](execution: Execution[T]): Execution[Unit] | |
trait GraphFeatureServiceAdhocBaseApp extends TwitterExecutionApp with GraphFeatureServiceBaseJob | |
def job: Execution[Unit] = Execution.withId | |
trait GraphFeatureServiceScheduledBaseApp | |
extends TwitterScheduledExecutionApp | |
with GraphFeatureServiceBaseJob | |
def firstTime: RichDate // for example: RichDate("2018-02-21") | |
def batchIncrement: Duration = Days(1) | |
override def scheduledJob: Execution[Unit] = Execution.withId | |
.\graph-feature-service\src\main\scalding\com\twitter\graph_feature_service\scalding\GraphFeatureServiceApps.scala | |
object GraphFeatureServiceAdhocApp | |
extends GraphFeatureServiceMainJob | |
with GraphFeatureServiceAdhocBaseApp | |
object GraphFeatureServiceScheduledApp | |
extends GraphFeatureServiceMainJob | |
with GraphFeatureServiceScheduledBaseApp | |
def firstTime: RichDate = RichDate("2018-05-18") | |
override def runOnDateRange( | |
enableValueGraphs: Option[Boolean], | |
enableKeyGraphs: Option[Boolean] | |
)( | |
implicit dateRange: DateRange, | |
timeZone: TimeZone, | |
uniqueID: UniqueID | |
): Execution[Unit] | |
.\graph-feature-service\src\main\scalding\com\twitter\graph_feature_service\scalding\GraphFeatureServiceMainJob.scala | |
trait GraphFeatureServiceMainJob extends GraphFeatureServiceBaseJob | |
def getShardIdForUser(userId: Long): Int = shardForUser(userId) | |
protected implicit val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint | |
protected implicit val valueInj: Injection[Long, ByteBuffer] = Injections.long2ByteBuffer | |
protected val bufferSize: Int = 1 << 26 | |
protected val maxNumKeys: Int = 1 << 24 | |
protected val numReducers: Int = NumGraphShards | |
protected val outputStreamBufferSize: Int = 1 << 26 | |
protected final val shardingByKey | |
def writeGraphToDB( | |
graph: TypedPipe[(Long, Long)], | |
shardingFunction: (Long, Long) => Int, | |
path: String | |
)( | |
implicit dateRange: DateRange | |
): Execution[TypedPipe[(Int, Unit)]] | |
def extractFeature( | |
featureList: Seq[TEdgeFeature], | |
featureName: FeatureName | |
): Option[Float] | |
def getSubGraph( | |
input: TypedPipe[(Long, Long, EdgeFeature)], | |
edgeFilter: EdgeFeature => Boolean, | |
counter: Stat | |
): TypedPipe[(Long, Long)] | |
def getMauIds()(implicit dateRange: DateRange, uniqueID: UniqueID): TypedPipe[Long] | |
def getRealGraphWithMAUOnly( | |
implicit dateRange: DateRange, | |
timeZone: TimeZone, | |
uniqueID: UniqueID | |
): TypedPipe[(Long, Long, EdgeFeature)] | |
def getTopKFollowGraph( | |
implicit dateRange: DateRange, | |
timeZone: TimeZone, | |
uniqueID: UniqueID | |
): TypedPipe[(Long, Long)] | |
def runOnDateRange( | |
enableValueGraphs: Option[Boolean], | |
enableKeyGraphs: Option[Boolean] | |
)( | |
implicit dateRange: DateRange, | |
timeZone: TimeZone, | |
uniqueID: UniqueID | |
): Execution[Unit] | |
.\graph-feature-service\src\main\scalding\com\twitter\graph_feature_service\scalding\adhoc\RandomRequestGenerationApp.scala | |
object RandomRequestGenerationJob | |
def run( | |
dataSetPath: String, | |
outPutPath: String, | |
numOfPairsToTake: Int | |
)( | |
implicit dateRange: DateRange, | |
uniqueID: UniqueID | |
): Execution[Unit] | |
object RandomRequestGenerationApp extends TwitterExecutionApp | |
def job: Execution[Unit] = Execution.withId | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\HomeMixerHttpServerWarmupHandler.scala | |
class HomeMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging | |
def handle(): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\HomeMixerServer.scala | |
object HomeMixerServerMain extends HomeMixerServer | |
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls | |
def configureThrift(router: ThriftRouter): Unit | |
def configureHttp(router: HttpRouter): Unit = | |
router.add( | |
ProductMixerController[st.HomeMixer.MethodPerEndpoint]( | |
this.injector, | |
st.HomeMixer.ExecutePipeline)) | |
override protected def warmup(): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\HomeMixerThriftServerWarmupHandler.scala | |
class HomeMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup) | |
extends Handler | |
with Logging | |
def handle(): Unit | |
def warmupQuery(userId: Long): st.HomeMixerRequest | |
def assertWarmupResponse( | |
result: Try[Response[st.HomeMixer.GetUrtResponse.SuccessType]] | |
): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\ConversationServiceCandidatePipelineConfig.scala | |
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( | |
conversationServiceCandidateSource: ConversationServiceCandidateSource, | |
tweetypieFeatureHydrator: TweetypieFeatureHydrator, | |
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, | |
namesFeatureHydrator: NamesFeatureHydrator, | |
override val gates: Seq[BaseGate[Query]], | |
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]]) | |
extends DependentCandidatePipelineConfig[ | |
Query, | |
ConversationServiceCandidateSourceRequest, | |
TweetWithConversationMetadata, | |
TweetCandidate | |
] | |
def filters: Seq[Filter[Query, TweetCandidate]] = Seq( | |
RetweetDeduplicationFilter, | |
FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), | |
PredicateFeatureFilter.fromPredicate( | |
FilterIdentifier(QuotedTweetDroppedFilterId), | |
shouldKeepCandidate | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\ConversationServiceCandidatePipelineConfigBuilder.scala | |
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() ( | |
conversationServiceCandidateSource: ConversationServiceCandidateSource, | |
tweetypieFeatureHydrator: TweetypieFeatureHydrator, | |
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, | |
namesFeatureHydrator: NamesFeatureHydrator) | |
def build( | |
gates: Seq[BaseGate[Query]] = Seq.empty, | |
decorator: Option[CandidateDecorator[Query, TweetCandidate]] = None | |
): ConversationServiceCandidatePipelineConfig[Query] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\ConversationServiceResponseFeatureTransformer.scala | |
object ConversationServiceResponseFeatureTransformer | |
extends CandidateFeatureTransformer[TweetWithConversationMetadata] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\EditedTweetsCandidatePipelineConfig.scala | |
class EditedTweetsCandidatePipelineConfig @Inject() ( | |
staleTweetsCacheCandidateSource: StaleTweetsCacheCandidateSource, | |
namesFeatureHydrator: NamesFeatureHydrator, | |
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) | |
extends DependentCandidatePipelineConfig[ | |
PipelineQuery, | |
Seq[Long], | |
Long, | |
TweetCandidate | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\NewTweetsPillCandidatePipelineConfig.scala | |
class NewTweetsPillCandidatePipelineConfig[Query <: PipelineQuery with HasDeviceContext] @Inject() ( | |
) extends DependentCandidatePipelineConfig[ | |
Query, | |
Unit, | |
ShowAlertCandidate, | |
ShowAlertCandidate | |
] | |
def apply( | |
query: Query, | |
candidate: ShowAlertCandidate, | |
features: FeatureMap | |
): Option[Duration] | |
object NewTweetsPillCandidatePipelineConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\candidate_pipeline\TimelineServiceResponseFeatureTransformer.scala | |
object TimelineServiceResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\controller\HomeThriftController.scala | |
class HomeThriftController @Inject() ( | |
homeRequestUnmarshaller: HomeMixerRequestUnmarshaller, | |
urtService: UrtService, | |
scoredTweetsService: ScoredTweetsService, | |
paramsBuilder: ParamsBuilder) | |
extends Controller(t.HomeMixer) | |
with DebugTwitterContext | |
def buildParams(request: HomeMixerRequest): Params | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\candidate_source\EarlybirdCandidateSource.scala | |
object EarlybirdResponseTruncatedFeature | |
extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Boolean] | |
object EarlybirdBottomTweetFeature | |
extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Option[Long]] | |
class EarlybirdCandidateSource @Inject() ( | |
earlybird: t.EarlybirdService.MethodPerEndpoint) | |
extends CandidateSourceWithExtractedFeatures[t.EarlybirdRequest, t.ThriftSearchResult] | |
def apply( | |
request: t.EarlybirdRequest | |
): Stitch[CandidatesWithSourceFeatures[t.ThriftSearchResult]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\candidate_source\SimilarityBasedUsersCandidateSource.scala | |
class SimilarityBasedUsersCandidateSource @Inject() ( | |
similarUsersBySimsOnUserClientColumn: SimilarUsersBySimsOnUserClientColumn) | |
extends CandidateSource[Seq[Long], t.Candidate] | |
def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\candidate_source\StaleTweetsCacheCandidateSource.scala | |
class StaleTweetsCacheCandidateSource @Inject() ( | |
@Named(StaleTweetsCache) staleTweetsCache: MemcachedClient) | |
extends CandidateSource[Seq[Long], Long] | |
def apply(request: Seq[Long]): Stitch[Seq[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\AuthorChildFeedbackActionBuilder.scala | |
class AuthorChildFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings) | |
def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\BlockUserChildFeedbackActionBuilder.scala | |
class BlockUserChildFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings) | |
def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\DontLikeFeedbackActionBuilder.scala | |
class DontLikeFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings, | |
authorChildFeedbackActionBuilder: AuthorChildFeedbackActionBuilder, | |
retweeterChildFeedbackActionBuilder: RetweeterChildFeedbackActionBuilder, | |
notRelevantChildFeedbackActionBuilder: NotRelevantChildFeedbackActionBuilder, | |
unfollowUserChildFeedbackActionBuilder: UnfollowUserChildFeedbackActionBuilder, | |
muteUserChildFeedbackActionBuilder: MuteUserChildFeedbackActionBuilder, | |
blockUserChildFeedbackActionBuilder: BlockUserChildFeedbackActionBuilder, | |
reportTweetChildFeedbackActionBuilder: ReportTweetChildFeedbackActionBuilder) | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[FeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\EngagerSocialContextBuilder.scala | |
class SocialContextIdAndScreenName( | |
socialContextId: Long, | |
screenName: String) | |
object EngagerSocialContextBuilder | |
class EngagerSocialContextBuilder( | |
contextType: GeneralContextType, | |
stringCenter: StringCenter, | |
oneUserString: ExternalString, | |
twoUsersString: ExternalString, | |
moreUsersString: ExternalString, | |
timelineTitle: ExternalString) | |
def apply( | |
socialContextIds: Seq[Long], | |
query: PipelineQuery, | |
candidateFeatures: FeatureMap | |
): Option[SocialContext] | |
def mkOneUserSocialContext(socialContextString: String, userId: Long): GeneralContext | |
def mkManyUserSocialContext( | |
socialContextString: String, | |
viewerId: Long, | |
socialContextIds: Seq[Long] | |
): GeneralContext | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\ExtendedReplySocialContextBuilder.scala | |
class ExtendedReplySocialContextBuilder @Inject() ( | |
externalStrings: HomeMixerExternalStrings, | |
@ProductScoped stringCenterProvider: Provider[StringCenter]) | |
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[SocialContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\FeedbackUtil.scala | |
object FeedbackUtil | |
def buildUserSeeFewerChildFeedbackAction( | |
userId: Long, | |
namesByUserId: Map[Long, String], | |
promptExternalString: ExternalString, | |
confirmationExternalString: ExternalString, | |
engagementType: t.FeedbackEngagementType, | |
stringCenter: StringCenter, | |
injectionType: Option[st.SuggestType] | |
): Option[ChildFeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\FollowedBySocialContextBuilder.scala | |
class FollowedBySocialContextBuilder @Inject() ( | |
externalStrings: HomeMixerExternalStrings, | |
@ProductScoped stringCenterProvider: Provider[StringCenter]) | |
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[SocialContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeAdsClientEventDetailsBuilder.scala | |
class HomeAdsClientEventDetailsBuilder(injectionType: Option[String]) | |
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidate: UniversalNoun[Any], | |
candidateFeatures: FeatureMap | |
): Option[ClientEventDetails] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeClientEventDetailsBuilder.scala | |
object HomeClientEventDetailsBuilder | |
def getRequestJoinId(): Option[Long] = | |
RequestJoinKeyContext.current.flatMap(_.requestJoinId) | |
} | |
case class HomeClientEventDetailsBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
) extends BaseClientEventDetailsBuilder[Query, Candidate] | |
with TweetTypeGenerator[FeatureMap] | |
def apply( | |
query: Query, | |
candidate: Candidate, | |
candidateFeatures: FeatureMap | |
): Option[ClientEventDetails] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeConversationServiceCandidateDecorator.scala | |
object HomeConversationServiceCandidateDecorator | |
def apply( | |
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder | |
): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeFeedbackActionInfoBuilder.scala | |
class HomeFeedbackActionInfoBuilder @Inject() ( | |
notInterestedTopicFeedbackActionBuilder: NotInterestedTopicFeedbackActionBuilder, | |
dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder) | |
extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[FeedbackActionInfo] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeQueryTypePredicates.scala | |
object HomeQueryTypePredicates | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeTimelinesScoreInfoBuilder.scala | |
object HomeTimelinesScoreInfoBuilder | |
extends BaseTimelinesScoreInfoBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[TimelinesScoreInfo] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeTweetSocialContextBuilder.scala | |
class HomeTweetSocialContextBuilder @Inject() ( | |
likedBySocialContextBuilder: LikedBySocialContextBuilder, | |
followedBySocialContextBuilder: FollowedBySocialContextBuilder, | |
topicSocialContextBuilder: TopicSocialContextBuilder, | |
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder, | |
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder) | |
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
features: FeatureMap | |
): Option[SocialContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\HomeTweetTypePredicates.scala | |
object HomeTweetTypePredicates | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\LikedBySocialContextBuilder.scala | |
class LikedBySocialContextBuilder @Inject() ( | |
externalStrings: HomeMixerExternalStrings, | |
@ProductScoped stringCenterProvider: Provider[StringCenter]) | |
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[SocialContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\ListConversationServiceCandidateDecorator.scala | |
object ListConversationServiceCandidateDecorator | |
def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\MuteUserChildFeedbackActionBuilder.scala | |
class MuteUserChildFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings) | |
def apply( | |
candidateFeatures: FeatureMap | |
): Option[ChildFeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\NotInterestedTopicFeedbackActionBuilder.scala | |
class NotInterestedTopicFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings) | |
def apply( | |
candidateFeatures: FeatureMap | |
): Option[FeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\NotRelevantChildFeedbackActionBuilder.scala | |
class NotRelevantChildFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings) | |
def apply( | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[ChildFeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\ReceivedReplySocialContextBuilder.scala | |
class ReceivedReplySocialContextBuilder @Inject() ( | |
externalStrings: HomeMixerExternalStrings, | |
@ProductScoped stringCenterProvider: Provider[StringCenter]) | |
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[SocialContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\ReportTweetChildFeedbackActionBuilder.scala | |
class ReportTweetChildFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings) | |
def apply( | |
candidate: TweetCandidate | |
): Option[ChildFeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\RetweeterChildFeedbackActionBuilder.scala | |
class RetweeterChildFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings) | |
def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\TopicSocialContextBuilder.scala | |
class TopicSocialContextBuilder @Inject() () | |
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[SocialContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\UnfollowUserChildFeedbackActionBuilder.scala | |
class UnfollowUserChildFeedbackActionBuilder @Inject() ( | |
@ProductScoped stringCenter: StringCenter, | |
externalStrings: HomeMixerExternalStrings) | |
def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\YouMightLikeSocialContextBuilder.scala | |
class YouMightLikeSocialContextBuilder @Inject() ( | |
externalStrings: HomeMixerExternalStrings, | |
@ProductScoped stringCenterProvider: Provider[StringCenter]) | |
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[SocialContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\builder\HomeClientEventInfoBuilder.scala | |
class HomeClientEventInfoBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( | |
detailsBuilder: Option[BaseClientEventDetailsBuilder[Query, Candidate]] = None) | |
extends BaseClientEventInfoBuilder[Query, Candidate] | |
def apply( | |
query: Query, | |
candidate: Candidate, | |
candidateFeatures: FeatureMap, | |
element: Option[String] | |
): Option[ClientEventInfo] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\builder\HomeConversationModuleMetadataBuilder.scala | |
class HomeConversationModuleMetadataBuilder[ | |
-Query <: PipelineQuery, | |
-Candidate <: BaseTweetCandidate | |
]() extends BaseModuleMetadataBuilder[Query, Candidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\builder\ListClientEventDetailsBuilder.scala | |
object ListClientEventDetailsBuilder | |
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidate: UniversalNoun[Any], | |
candidateFeatures: FeatureMap | |
): Option[ClientEventDetails] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\urt\builder\AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala | |
class AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder[Query <: PipelineQuery]( | |
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) | |
extends UrtInstructionBuilder[Query, AddEntriesTimelineInstruction] | |
def build( | |
query: Query, | |
entries: Seq[TimelineEntry] | |
): Seq[AddEntriesTimelineInstruction] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\decorator\urt\builder\HomeWhoToFollowFeedbackActionInfoBuilder.scala | |
object HomeWhoToFollowFeedbackActionInfoBuilder | |
class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() ( | |
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry], | |
@ProductScoped stringCenterProvider: Provider[StringCenter]) | |
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\AncestorFeatureHydrator.scala | |
class AncestorFeatureHydrator @Inject() ( | |
conversationServiceClient: tcs.ConversationService.MethodPerEndpoint) | |
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
def getTruncatedRootTweet( | |
ancestors: ta.TweetAncestors, | |
): Option[ta.TweetAncestor] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\AuthorFeatureHydrator.scala | |
object AuthorFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class AuthorFeatureHydrator @Inject() ( | |
@Named(AuthorFeatureRepository) client: KeyValueRepository[Seq[Long], Long, af.AuthorFeatures], | |
override val statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with ObservedKeyValueResultHandler | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord] | |
def extractKeys( | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\DismissInfoQueryFeatureHydrator.scala | |
object DismissInfoQueryFeatureHydrator | |
class DismissInfoQueryFeatureHydrator @Inject() ( | |
dismissInfoClient: InjectionHistoryClient) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] = | |
Stitch.callFuture | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\EarlybirdFeatureHydrator.scala | |
object EarlybirdDataRecordFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class EarlybirdFeatureHydrator @Inject() ( | |
@Named(EarlybirdRepository) client: KeyValueRepository[ | |
(Seq[Long], Long), | |
Long, | |
eb.ThriftSearchResult | |
], | |
override val statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with ObservedKeyValueResultHandler | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def handleResponse( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]], | |
results: KeyValueResult[Long, eb.ThriftSearchResult] | |
): Seq[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\FeedbackHistoryQueryFeatureHydrator.scala | |
class FeedbackHistoryQueryFeatureHydrator @Inject() ( | |
feedbackHistoryClient: FeedbackHistoryManhattanClient) | |
extends QueryFeatureHydrator[PipelineQuery] | |
with Conditionally[PipelineQuery] | |
def onlyIf(query: PipelineQuery): Boolean = | |
query.params(EnableFeedbackFatigueParam) | |
override def hydrate( | |
query: PipelineQuery | |
): Stitch[FeatureMap] = | |
Stitch | |
.callFuture(feedbackHistoryClient.get(query.getRequiredUserId)) | |
.map | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\FocalTweetFeatureHydrator.scala | |
class FocalTweetFeatureHydrator @Inject() () | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\FollowedTopicsQueryFeatureHydrator.scala | |
class FollowedTopicsQueryFeatureHydrator @Inject() ( | |
followedTopicsCandidateSource: FollowedTopicsCandidateSource) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\GizmoduckAuthorSafetyFeatureHydrator.scala | |
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck) | |
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with Conditionally[PipelineQuery] | |
def onlyIf(query: PipelineQuery): Boolean = | |
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam) | |
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety) | |
override def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\GizmoduckUserQueryFeatureHydrator.scala | |
class GizmoduckUserQueryFeatureHydrator @Inject() (gizmoduck: Gizmoduck) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\GraphTwoHopFeatureHydrator.scala | |
object GraphTwoHopFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class GraphTwoHopFeatureHydrator @Inject() ( | |
@Named(GraphTwoHopRepository) client: KeyValueRepository[(Seq[Long], Long), Long, Seq[ | |
gfs.IntersectionValue | |
]], | |
override val statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with ObservedKeyValueResultHandler | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\ImpressionBloomFilterQueryFeatureHydrator.scala | |
class ImpressionBloomFilterQueryFeatureHydrator[ | |
Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( | |
bloomFilter: ImpressionBloomFilter) | |
extends QueryFeatureHydrator[Query] | |
def hydrate(query: Query): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\LastNonPollingTimeQueryFeatureHydrator.scala | |
class LastNonPollingTimeQueryFeatureHydrator @Inject() ( | |
userSessionStore: ReadWriteUserSessionStore) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\ListMembersQueryFeatureHydrator.scala | |
object ListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] | |
class ListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) | |
extends QueryFeatureHydrator[PipelineQuery with HasListId] | |
def hydrate(query: PipelineQuery with HasListId): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\MetricCenterUserCountingFeatureHydrator.scala | |
object MetricCenterUserCountingFeature | |
extends Feature[TweetCandidate, Option[rf.MCUserCountingFeatures]] | |
@Singleton | |
class MetricCenterUserCountingFeatureHydrator @Inject() ( | |
@Named(MetricCenterUserCountingFeatureRepository) client: KeyValueRepository[Seq[ | |
Long | |
], Long, rf.MCUserCountingFeatures], | |
override val statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with ObservedKeyValueResultHandler | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def extractKeys( | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\NamesFeatureHydrator.scala | |
class ProfileNames(screenName: String, realName: String) | |
@Singleton | |
class NamesFeatureHydrator @Inject() (gizmoduck: Gizmoduck) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with Conditionally[PipelineQuery] | |
def onlyIf(query: PipelineQuery): Boolean = query.product match | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\PersistenceStoreQueryFeatureHydrator.scala | |
class PersistenceStoreQueryFeatureHydrator @Inject() ( | |
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\PerspectiveFilteredSocialContextFeatureHydrator.scala | |
class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService: TimelineService) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealGraphInNetworkScoresQueryFeatureHydrator.scala | |
class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( | |
@Named(RealGraphInNetworkScores) store: ReadableStore[Long, Seq[wtf.Candidate]]) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealGraphQueryFeatureHydrator.scala | |
object RealGraphFeatures extends Feature[PipelineQuery, Option[Map[UserId, RealGraphEdgeFeatures]]] | |
@Singleton | |
class RealGraphQueryFeatureHydrator @Inject() ( | |
@Named(RealGraphFeatureRepository) repository: Repository[Long, Option[uss.UserSession]]) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealGraphViewerAuthorFeatureHydrator.scala | |
object RealGraphViewerAuthorDataRecordFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
object RealGraphViewerAuthorsDataRecordFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class RealGraphViewerAuthorFeatureHydrator @Inject() () | |
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
def getRealGraphViewerAuthorFeatures( | |
viewerId: Long, | |
authorId: Long, | |
realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] | |
): rg.UserRealGraphFeatures | |
object RealGraphViewerAuthorFeatureHydrator | |
def getCombinedRealGraphFeatures( | |
userIds: Seq[Long], | |
realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] | |
): rg.RealGraphFeatures | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealGraphViewerRelatedUsersFeatureHydrator.scala | |
object RealGraphViewerRelatedUsersDataRecordFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class RealGraphViewerRelatedUsersFeatureHydrator @Inject() () | |
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
def getRelatedUserIds(features: FeatureMap): Seq[Long] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealTimeInteractionGraphEdgeFeatureHydrator.scala | |
object RealTimeInteractionGraphEdgeFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala | |
object RealTimeInteractionGraphUserVertexQueryFeature | |
extends Feature[PipelineQuery, Option[ig.UserVertex]] | |
@Singleton | |
class RealTimeInteractionGraphUserVertexQueryFeatureHydrator @Inject() ( | |
@Named(RealTimeInteractionGraphUserVertexCache) client: ReadCache[Long, ig.UserVertex], | |
override val statsReceiver: StatsReceiver) | |
extends QueryFeatureHydrator[PipelineQuery] | |
with ObservedKeyValueResultHandler | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\ReplyFeatureHydrator.scala | |
object InReplyToTweetHydratedEarlybirdFeature | |
extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] | |
/** | |
* The purpose of this hydrator is to | |
* 1) hydrate simple features into replies and their ancestor tweets | |
* 2) keep both the normal replies and ancestor source candidates, but hydrate into the candidates | |
* features useful for predicting the quality of the replies and source ancestor tweets. | |
*/ | |
@Singleton | |
class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def hydratedReplyCandidate( | |
replyCandidate: CandidateWithFeatures[TweetCandidate], | |
inReplyToTweetCandidate: CandidateWithFeatures[TweetCandidate] | |
): ( | |
CandidateWithFeatures[TweetCandidate], | |
Option[ConversationFeatures], | |
Option[ThriftTweetFeatures] | |
) | |
def hydrateAncestorTweetCandidate( | |
ancestorTweetCandidate: CandidateWithFeatures[TweetCandidate], | |
descendantReplies: Seq[CandidateWithFeatures[TweetCandidate]], | |
updatedReplyConversationFeatures: Option[ConversationFeatures] | |
): (CandidateWithFeatures[TweetCandidate], Option[ConversationFeatures]) | |
def originalTweetAgeFromSnowflake( | |
candidate: CandidateWithFeatures[TweetCandidate] | |
): Option[Duration] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RequestQueryFeatureHydrator.scala | |
class RequestQueryFeatureHydrator[ | |
Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext] @Inject() ( | |
) extends QueryFeatureHydrator[Query] | |
def getLanguageISOFormatByCode(languageCode: String): String = | |
ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguageUtil.getThriftLanguageOf(languageCode)) | |
private def getRequestJoinId(servedRequestId: Long): Option[Long] = | |
Some(RequestJoinKeyContext.current.flatMap(_.requestJoinId).getOrElse(servedRequestId)) | |
private def hasDarkRequest: Option[Boolean] = ForwardAnnotation.current | |
.getOrElse(Seq[BinaryAnnotation]()) | |
.find(_.key == DarkRequestAnnotation) | |
.map(_.value.asInstanceOf[Boolean]) | |
override def hydrate(query: Query): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\RetweetSourceTweetFeatureHydrator.scala | |
object SourceTweetEarlybirdFeature extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] | |
/** | |
* Feature Hydrator that bulk hydrates source tweets' features to retweet candidates | |
*/ | |
object RetweetSourceTweetFeatureHydrator | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\SGSFollowedUsersQueryFeatureHydrator.scala | |
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]] | |
@Singleton | |
case class SGSFollowedUsersQueryFeatureHydrator @Inject() ( | |
socialGraphStitchClient: SocialGraphStitchClient) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\SGSValidSocialContextFeatureHydrator.scala | |
class SGSValidSocialContextFeatureHydrator @Inject() ( | |
socialGraph: SocialGraph) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def getValidUserIds( | |
viewerId: Long, | |
socialProofUserIds: Seq[Long] | |
): Stitch[Seq[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\SimClustersEngagementSimilarityFeatureHydrator.scala | |
object SimClustersEngagementSimilarityFeature | |
extends DataRecordInAFeature[PipelineQuery] | |
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( | |
simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient, | |
statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with Conditionally[PipelineQuery] | |
def onlyIf(query: PipelineQuery): Boolean | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\SocialGraphServiceFeatureHydrator.scala | |
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def createIdsRequest( | |
userId: Long, | |
relationshipTypes: Set[sg.RelationshipType], | |
targetIds: Option[Seq[Long]] = None | |
): sg.IdsRequest = sg.IdsRequest( | |
relationshipTypes.map | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TimeFeaturesHydrator.scala | |
object TimeFeaturesDataRecordFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
def setTimeFeatures( | |
richDataRecord: RichDataRecord, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap, | |
query: PipelineQuery, | |
): Unit | |
def getTimeFeatures( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap, | |
): Option[TimeFeatures] | |
def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] | |
def addLastEngagementTimeFeatures( | |
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures], | |
timeFeatures: TimeFeatures, | |
timeSinceSourceTweetCreation: Long | |
): Option[TimeFeatures] | |
def addNonPollingTimeFeatures( | |
timeFeatures: TimeFeatures, | |
requestTimestampMs: Long, | |
creationTimeMs: Long, | |
nonPollingTimestampsMs: Option[Seq[Long]] | |
): Option[TimeFeatures] | |
def getTimeSinceLastEngagementHrs( | |
lastEngagementTimeSinceCreationHrsOpt: Option[Double], | |
timeSinceTweetCreation: Long | |
): Option[Double] | |
def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TimelineServiceTweetsQueryFeatureHydrator.scala | |
object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]] | |
@Singleton | |
case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( | |
timelineService: TimelineService, | |
deviceContextMarshaller: DeviceContextMarshaller) | |
extends QueryFeatureHydrator[PipelineQuery with HasDeviceContext] | |
def hydrate(query: PipelineQuery with HasDeviceContext): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TSPInferredTopicFeatureHydrator.scala | |
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]] | |
object TSPInferredTopicDataRecordFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TSPInferredTopicFeatureHydrator @Inject() ( | |
topicSocialProofClientColumn: TopicSocialProofClientColumn, | |
statsReceiver: StatsReceiver, | |
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def getSocialProof( | |
topicWithScores: Seq[tsp.TopicWithScore] | |
): (Option[Long], Option[TopicContextFunctionalityType]) | |
def convertTopicWithScores( | |
topicWithScores: Seq[tsp.TopicWithScore], | |
): Map[Long, Double] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetImpressionsQueryFeatureHydrator.scala | |
class TweetImpressionsQueryFeatureHydrator[ | |
Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( | |
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) | |
extends QueryFeatureHydrator[Query] | |
def hydrate(query: Query): Stitch[FeatureMap] | |
def updateTweetImpressions( | |
tweetImpressionsFromStore: Seq[t.TweetImpressionsEntry], | |
seenIdsFromClient: Seq[Long], | |
currentTime: Long = Time.now.inMilliseconds, | |
tweetImpressionTTL: Duration = TweetImpressionTTL, | |
tweetImpressionCap: Int = TweetImpressionCap, | |
): Seq[t.TweetImpressionsEntry] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetMetaDataFeatureHydrator.scala | |
object TweetMetaDataDataRecord | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
object TweetMetaDataFeatureHydrator | |
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def features: Set[Feature[_, _]] = Set(TweetMetaDataDataRecord) | |
override def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
def setFeatures( | |
richDataRecord: RichDataRecord, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetypieContentFeatureHydrator.scala | |
object TweetypieContentDataRecordFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TweetypieContentFeatureHydrator @Inject() ( | |
@Named(TweetypieContentRepository) client: KeyValueRepository[Seq[Long], Long, tp.Tweet], | |
override val statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with ObservedKeyValueResultHandler | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def parTransformer( | |
result: KeyValueResult[Long, tp.Tweet], | |
candidate: CandidateWithFeatures[TweetCandidate] | |
): Try[(Seq[Long], DataRecord)] | |
def postTransformer( | |
result: Try[Option[tp.Tweet]] | |
): Try[(Seq[Long], DataRecord)] | |
def getCandidateOriginalTweetId( | |
candidate: CandidateWithFeatures[TweetCandidate] | |
): Long | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetypieFeatureHydrator.scala | |
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient) | |
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TweetypieStaticEntitiesFeatureHydrator.scala | |
class TweetypieStaticEntitiesFeatureHydrator @Inject() ( | |
tweetypieStitchClient: TweetypieStitchClient, | |
@Named(TweetypieStaticEntitiesCache) cacheClient: TtlCache[Long, tp.Tweet]) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def createFeatureMap(tweet: tp.Tweet): FeatureMap | |
def readFromTweetypie( | |
query: PipelineQuery, | |
candidate: CandidateWithFeatures[TweetCandidate] | |
): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TwhinAuthorFollow20220101FeatureHydrator.scala | |
object TwhinAuthorFollow20220101Feature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TwhinAuthorFollow20220101FeatureHydrator @Inject() ( | |
@Named(TwhinAuthorFollow20200101FeatureRepository) | |
client: KeyValueRepository[Seq[Long], Long, ml.Embedding], | |
override val statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with ObservedKeyValueResultHandler | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def postTransformer(embedding: Try[Option[ml.Embedding]]): Try[DataRecord] | |
def extractKeys( | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TwhinUserEngagementQueryFeatureHydrator.scala | |
object TwhinUserEngagementFeature | |
extends DataRecordInAFeature[PipelineQuery] | |
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TwhinUserEngagementQueryFeatureHydrator @Inject() ( | |
@Named(TwhinUserEngagementFeatureRepository) | |
client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], | |
statsReceiver: StatsReceiver) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\TwhinUserFollowQueryFeatureHydrator.scala | |
object TwhinUserFollowFeature | |
extends DataRecordInAFeature[PipelineQuery] | |
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TwhinUserFollowQueryFeatureHydrator @Inject() ( | |
@Named(TwhinUserFollowFeatureRepository) | |
client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], | |
statsReceiver: StatsReceiver) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\UserFollowedTopicIdsFeatureHydrator.scala | |
object UserFollowedTopicIdsFeature extends Feature[TweetCandidate, Seq[Long]] | |
@Singleton | |
class UserFollowedTopicIdsFeatureHydrator @Inject() ( | |
@Named(UserFollowedTopicIdsRepository) | |
client: KeyValueRepository[Seq[Long], Long, Seq[Long]], | |
override val statsReceiver: StatsReceiver) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with ObservedKeyValueResultHandler | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def postTransformer(input: Try[Option[Seq[Long]]]): Try[Seq[Long]] | |
def extractKeys( | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\UserLanguagesFeatureHydrator.scala | |
object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguage]] | |
@Singleton | |
case class UserLanguagesFeatureHydrator @Inject() ( | |
@Named(UserLanguagesStore) store: ReadableStore[Long, Seq[scc.ThriftLanguage]]) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\UserStateQueryFeatureHydrator.scala | |
class UserStateQueryFeatureHydrator @Inject() ( | |
userSessionStore: ReadOnlyUserSessionStore) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\UtegFeatureHydrator.scala | |
class UtegFeatureHydrator @Inject() ( | |
@Named(UtegSocialProofRepository) client: KeyValueRepository[ | |
(Seq[Long], (Long, Map[Long, Double])), | |
Long, | |
uteg.TweetRecommendation | |
]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with Conditionally[PipelineQuery] | |
def onlyIf(query: PipelineQuery): Boolean = query.features | |
.exists(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double]).nonEmpty) | |
override def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def handleResponse( | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]], | |
results: KeyValueResult[Long, uteg.TweetRecommendation], | |
): Seq[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\author_features\AuthorFeaturesAdapter.scala | |
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] | |
def getFeatureContext: FeatureContext = featureContext | |
override val commonFeatures: Set[Feature[_]] = Set.empty | |
private val compactDataRecordConverter = new CompactDataRecordConverter() | |
private val drMerger = new DataRecordMerger() | |
override def setFeatures( | |
authorFeaturesOpt: Option[af.AuthorFeatures], | |
richDataRecord: RichDataRecord | |
): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\content\ContentFeatureAdapter.scala | |
object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] | |
def getTweetLengthType(tweetLength: Int): Long | |
def setFeatures( | |
contentFeatures: Option[ContentFeatures], | |
richDataRecord: RichDataRecord | |
): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\earlybird\EarlybirdAdapter.scala | |
object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]] | |
def setFeatures( | |
ebFeatures: Option[sc.ThriftTweetFeatures], | |
richDataRecord: RichDataRecord | |
): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\inferred_topic\InferredTopicAdapter.scala | |
object InferredTopicAdapter extends TimelinesMutatingAdapterBase[Map[Long, Double]] | |
def setFeatures( | |
inferredTopicFeatures: Map[Long, Double], | |
richDataRecord: RichDataRecord | |
): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\non_ml_features\NonMLCandidateFeaturesAdapter.scala | |
class NonMLCandidateFeatures( | |
tweetId: Long, | |
sourceTweetId: Option[Long], | |
originalAuthorId: Option[Long], | |
) | |
/** | |
* define non ml features adapter to create a data record which includes many non ml features | |
* e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. | |
*/ | |
object NonMLCandidateFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCandidateFeatures] | |
def getFeatureContext: FeatureContext = featureContext | |
override val commonFeatures: Set[Feature[_]] = Set.empty | |
override def setFeatures( | |
nonMLCandidateFeatures: NonMLCandidateFeatures, | |
richDataRecord: RichDataRecord | |
): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\non_ml_features\NonMLCommonFeaturesAdapter.scala | |
class NonMLCommonFeatures( | |
userId: Long, | |
predictionRequestId: Option[Long], | |
servedTimestamp: Long, | |
) | |
/** | |
* define non ml features adapter to create a data record which includes many non ml features | |
* e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. | |
*/ | |
object NonMLCommonFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCommonFeatures] | |
def getFeatureContext: FeatureContext = featureContext | |
override val commonFeatures: Set[Feature[_]] = Set( | |
SharedFeatures.USER_ID, | |
TimelinesSharedFeatures.PREDICTION_REQUEST_ID, | |
TimelinesSharedFeatures.SERVED_TIMESTAMP, | |
) | |
override def setFeatures( | |
nonMLCommonFeatures: NonMLCommonFeatures, | |
richDataRecord: RichDataRecord | |
): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\offline_aggregates\PassThroughAdapter.scala | |
object PassThroughAdapter extends IRecordOneToOneAdapter[Seq[DataRecord]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\offline_aggregates\SparseAggregatesToDenseAdapter.scala | |
class SparseAggregatesToDenseAdapter(policy: CombineCountsPolicy) | |
extends TimelinesIRecordAdapter[Seq[DataRecord]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\adapters\twhin_embeddings\TwhinEmbeddingsAdapter.scala | |
trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.Embedding]] | |
def twhinEmbeddingsFeature: Feature.Tensor | |
override def getFeatureContext: FeatureContext = new FeatureContext( | |
twhinEmbeddingsFeature | |
) | |
override def setFeatures( | |
embedding: Option[ml.Embedding], | |
richDataRecord: RichDataRecord | |
): Unit | |
object TwhinEmbeddingsFeatures | |
object TwhinAuthorFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter | |
object TwhinUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter | |
object TwhinUserFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\AggregateFeatureInfo.scala | |
class deriving aggregate feature info from the given configuration parameters. | |
class AggregateFeatureInfo( | |
val aggregateGroups: Set[AggregateGroup], | |
val aggregateType: AggregateType) | |
object AggregateFeatureInfo | |
def pickFeature(aggregateType: AggregateType): BaseAggregateRootFeature | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\AggregateFeaturesToDecodeWithMetadata.scala | |
class AggregateFeaturesToDecodeWithMetadata( | |
metadataOpt: Option[DenseFeatureMetadata], | |
aggregates: UserAggregateInteractions) | |
def toDataRecord(dr: DenseCompactDataRecord): DataRecord = | |
VersionedAggregateFeaturesDecoder.fromJDenseCompact( | |
metadataOpt, | |
dr.versionId, | |
NullStatsReceiver, | |
s"V$ | |
def userAggregatesOpt: Option[DenseCompactDataRecord] | |
def userAuthorAggregates = extract(_.user_author_aggregates) | |
def userEngagerAggregates = extract(_.user_engager_aggregates) | |
def userMentionAggregates = extract(_.user_mention_aggregates) | |
def userOriginalAuthorAggregates = extract(_.user_original_author_aggregates) | |
def userRequestDowAggregates = extract(_.user_request_dow_aggregates) | |
def userRequestHourAggregates = extract(_.user_request_hour_aggregates) | |
def rectweetUserSimclustersTweetAggregates = extract(_.rectweet_user_simclusters_tweet_aggregates) | |
def userTwitterListAggregates = extract(_.user_list_aggregates) | |
def userTopicAggregates = extract(_.user_topic_aggregates) | |
def userInferredTopicAggregates = extract(_.user_inferred_topic_aggregates) | |
def userMediaUnderstandingAnnotationAggregates = extract( | |
_.user_media_understanding_annotation_aggregates) | |
private def extract[T]( | |
v17Fn: V17UserAggregateInteractions => JMap[JLong, DenseCompactDataRecord] | |
): JMap[JLong, DenseCompactDataRecord] | |
object AggregateFeaturesToDecodeWithMetadata | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\BaseAggregateQueryFeatureHydrator.scala | |
abstract class BaseAggregateQueryFeatureHydrator( | |
featureRepository: Repository[Long, Option[UserSession]], | |
metadataRepository: Repository[Int, Option[DenseFeatureMetadata]], | |
feature: Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]]) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
def decodeUserSession( | |
session: UserSession | |
): Option[Future[AggregateFeaturesToDecodeWithMetadata]] | |
def getAggregateFeaturesWithMetadata( | |
versionId: Int, | |
userAggregateInteractions: UserAggregateInteractions, | |
): Future[AggregateFeaturesToDecodeWithMetadata] | |
trait BaseAggregateRootFeature | |
extends Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\BaseEdgeAggregateFeatureHydrator.scala | |
abstract case class BaseEdgeAggregateFeature( | |
aggregateGroups: Set[AggregateGroup], | |
aggregateType: AggregateType, | |
extractMapFn: AggregateFeaturesToDecodeWithMetadata => JMap[JLong, DenseCompactDataRecord], | |
adapter: IRecordOneToOneAdapter[Seq[DataRecord]], | |
getSecondaryKeysFn: CandidateWithFeatures[TweetCandidate] => Seq[Long]) | |
extends DataRecordInAFeature[PipelineQuery] | |
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] | |
def defaultValue: DataRecord = new DataRecord | |
private val rootFeatureInfo = new AggregateFeatureInfo(aggregateGroups, aggregateType) | |
val featureContext: FeatureContext = rootFeatureInfo.featureContext | |
val rootFeature: BaseAggregateRootFeature = rootFeatureInfo.feature | |
} | |
trait BaseEdgeAggregateFeatureHydrator | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def aggregateFeatures: Set[BaseEdgeAggregateFeature] | |
override def features = aggregateFeatures.asInstanceOf[Set[Feature[_, _]]] | |
override def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def hydrateAggregateFeature( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]], | |
feature: BaseEdgeAggregateFeature | |
): Seq[DataRecord] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\EdgeAggregateFeatures.scala | |
object EdgeAggregateFeatures | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\PartAAggregateQueryFeatureHydrator.scala | |
object PartAAggregateRootFeature extends BaseAggregateRootFeature | |
class PartAAggregateQueryFeatureHydrator @Inject() ( | |
@Named(TimelineAggregatePartARepository) | |
repository: Repository[Long, Option[UserSession]], | |
@Named(TimelineAggregateMetadataRepository) | |
metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) | |
extends BaseAggregateQueryFeatureHydrator( | |
repository, | |
metadataRepository, | |
PartAAggregateRootFeature | |
) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\PartBAggregateQueryFeatureHydrator.scala | |
object PartBAggregateRootFeature extends BaseAggregateRootFeature | |
object UserAggregateFeature | |
extends DataRecordInAFeature[PipelineQuery] | |
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class PartBAggregateQueryFeatureHydrator @Inject() ( | |
@Named(TimelineAggregatePartBRepository) | |
repository: Repository[Long, Option[UserSession]], | |
@Named(TimelineAggregateMetadataRepository) | |
metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) | |
extends BaseAggregateQueryFeatureHydrator( | |
repository, | |
metadataRepository, | |
PartBAggregateRootFeature | |
) | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
def mergeDataRecordOpts(dataRecordOpts: Option[DataRecord]*): DataRecord = | |
dataRecordOpts.flatten.foldLeft(new DataRecord) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\Phase1EdgeAggregateFeatureHydrator.scala | |
class Phase1EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\Phase2EdgeAggregateFeatureHydrator.scala | |
class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\offline_aggregates\Utils.scala | |
object Utils | |
def selectAndTransform( | |
idsToSelect: Seq[Long], | |
transform: DenseCompactDataRecord => DataRecord, | |
map: java.util.Map[java.lang.Long, DenseCompactDataRecord], | |
): Map[Long, DataRecord] | |
def filterDataRecord(dr: DataRecord, featureContext: FeatureContext): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala | |
trait BaseRealTimeAggregateBulkCandidateFeatureHydrator[K] | |
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
with BaseRealtimeAggregateHydrator[K] | |
def features: Set[Feature[_, _]] = Set(outputFeature) | |
override lazy val statScope: String = identifier.toString | |
def keysFromQueryAndCandidates( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[K]] | |
override def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\BaseRealtimeAggregateHydrator.scala | |
trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler | |
def postTransformer(dataRecord: Try[Option[DataRecord]]): Try[DataRecord] | |
def fetchAndConstructDataRecords(possiblyKeys: Seq[Option[K]]): Stitch[Seq[Try[DataRecord]]] | |
object BaseRealtimeAggregateHydrator | |
def applyDecay( | |
dataRecord: DataRecord, | |
featureContext: FeatureContext, | |
decay: TimeDecay | |
): DataRecord | |
def time: Long = Time.now.inMillis | |
val richFullDr = new SRichDataRecord(dataRecord, featureContext) | |
val richNewDr = new SRichDataRecord(new DataRecord, featureContext) | |
val featureIterator = featureContext.iterator() | |
featureIterator.forEachRemaining | |
def applyRename( | |
dataRecord: DataRecord, | |
featureContext: FeatureContext, | |
renamedFeatureContext: FeatureContext, | |
featureRenamingMap: Map[MLApiFeature[_], MLApiFeature[_]] | |
): DataRecord | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\BaseRealTimeAggregateQueryFeatureHydrator.scala | |
trait BaseRealTimeAggregateQueryFeatureHydrator[K] | |
extends QueryFeatureHydrator[PipelineQuery] | |
with BaseRealtimeAggregateHydrator[K] | |
def features: Set[Feature[_, _]] = Set(outputFeature) | |
override lazy val statScope: String = identifier.toString | |
def keysFromQueryAndCandidates( | |
query: PipelineQuery | |
): Option[K] | |
override def hydrate( | |
query: PipelineQuery | |
): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala | |
object EngagementsReceivedByAuthorRealTimeAggregateFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator @Inject() ( | |
@Named(EngagementsReceivedByAuthorCache) override val client: ReadCache[Long, DataRecord], | |
override val statsReceiver: StatsReceiver) | |
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\RealTimeAggregateTimeDecay.scala | |
class RealTimeAggregateTimeDecay( | |
featureIdToHalfLife: Map[Long, Duration], | |
timestampFeatureId: Long = TIMESTAMP.getFeatureId) | |
def apply(record: DataRecord, timeNow: Long): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala | |
object TopicCountryEngagementRealTimeAggregateFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TopicCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( | |
@Named(TopicCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], | |
override val statsReceiver: StatsReceiver) | |
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] | |
def keysFromQueryAndCandidates( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[(Long, String)]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TopicEngagementRealTimeAggregateFeatureHydrator.scala | |
object TopicEngagementRealTimeAggregateFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TopicEngagementRealTimeAggregateFeatureHydrator @Inject() ( | |
@Named(TopicEngagementCache) override val client: ReadCache[Long, DataRecord], | |
override val statsReceiver: StatsReceiver) | |
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] | |
def keysFromQueryAndCandidates( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala | |
object TweetCountryEngagementRealTimeAggregateFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( | |
@Named(TweetCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], | |
override val statsReceiver: StatsReceiver) | |
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] | |
def keysFromQueryAndCandidates( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[(Long, String)]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TweetEngagementRealTimeAggregateFeatureHydrator.scala | |
object TweetEngagementRealTimeAggregateFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() ( | |
@Named(TweetEngagementCache) override val client: ReadCache[Long, DataRecord], | |
override val statsReceiver: StatsReceiver) | |
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] | |
def keysFromQueryAndCandidates( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\TwitterListEngagementRealTimeAggregateFeatureHydrator.scala | |
object TwitterListEngagementRealTimeAggregateFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class TwitterListEngagementRealTimeAggregateFeatureHydrator @Inject() ( | |
@Named(TwitterListEngagementCache) override val client: ReadCache[Long, DataRecord], | |
override val statsReceiver: StatsReceiver) | |
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] | |
def keysFromQueryAndCandidates( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala | |
object UserAuthorEngagementRealTimeAggregateFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class UserAuthorEngagementRealTimeAggregateFeatureHydrator @Inject() ( | |
@Named(UserAuthorEngagementCache) override val client: ReadCache[(Long, Long), DataRecord], | |
override val statsReceiver: StatsReceiver) | |
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long)] | |
def keysFromQueryAndCandidates( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Seq[Option[(Long, Long)]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\feature_hydrator\real_time_aggregates\UserEngagementRealTimeAggregatesFeatureHydrator.scala | |
object UserEngagementRealTimeAggregateFeature | |
extends DataRecordInAFeature[PipelineQuery] | |
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
@Singleton | |
class UserEngagementRealTimeAggregatesFeatureHydrator @Inject() ( | |
@Named(UserEngagementCache) override val client: ReadCache[Long, DataRecord], | |
override val statsReceiver: StatsReceiver) | |
extends BaseRealTimeAggregateQueryFeatureHydrator[Long] | |
def keysFromQueryAndCandidates(query: PipelineQuery): Option[Long] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\DropMaxCandidatesFilter.scala | |
class DropMaxCandidatesFilter[Candidate <: UniversalNoun[Any]]( | |
maxCandidatesParam: FSBoundedParam[Int]) | |
extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\FeedbackFatigueFilter.scala | |
object FeedbackFatigueFilter | |
extends Filter[PipelineQuery, TweetCandidate] | |
with Filter.Conditionally[PipelineQuery, TweetCandidate] | |
def onlyIf( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Boolean = | |
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) | |
private val DurationForFiltering = 14.days | |
override def apply( | |
query: pipeline.PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
def getUserIds( | |
feedbackEntries: Seq[FeedbackEntry], | |
): Set[Long] = | |
feedbackEntries.collect | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\InvalidConversationModuleFilter.scala | |
object InvalidConversationModuleFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\KeepBestOutOfNetworkTweetPerAuthorFilter.scala | |
object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\OutOfNetworkCompetitorFilter.scala | |
object OutOfNetworkCompetitorFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
def isOutOfNetworkTweetFromCompetitor( | |
candidate: CandidateWithFeatures[TweetCandidate], | |
competitorAuthors: Set[Long] | |
): Boolean | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\OutOfNetworkCompetitorURLFilter.scala | |
object OutOfNetworkCompetitorURLFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
def hasOutOfNetworkUrlFromCompetitor( | |
candidate: CandidateWithFeatures[TweetCandidate], | |
competitorUrls: Set[String] | |
): Boolean | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PredicateFeatureFilter.scala | |
trait ShouldKeepCandidate | |
def apply(features: FeatureMap): Boolean | |
} | |
object PredicateFeatureFilter | |
def fromPredicate[Candidate <: UniversalNoun[Any]]( | |
identifier: FilterIdentifier, | |
shouldKeepCandidate: ShouldKeepCandidate | |
): Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PredicateGatedFilter.scala | |
trait FilterPredicate[-Query <: PipelineQuery] | |
def apply(query: Query): Boolean | |
} | |
/** | |
* A [[Filter]] with [[Conditionally]] based on a [[FilterPredicate]] | |
* | |
* @param predicate the predicate to turn this filter on and off | |
* @param filter the underlying filter to run when `predicate` is true | |
* @tparam Query The domain model for the query or request | |
* @tparam Candidate The type of the candidates | |
*/ | |
case class PredicateGatedFilter[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( | |
predicate: FilterPredicate[Query], | |
filter: Filter[Query, Candidate]) | |
extends Filter[Query, Candidate] | |
with Filter.Conditionally[Query, Candidate] | |
def onlyIf(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): Boolean = | |
Conditionally.and(Filter.Input(query, candidates), filter, predicate(query)) | |
override def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] = filter.apply(query, candidates) | |
} | |
object PredicateGatedFilter | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PreviouslySeenTweetsFilter.scala | |
object PreviouslySeenTweetsFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PreviouslyServedAncestorsFilter.scala | |
object PreviouslyServedAncestorsFilter | |
extends Filter[PipelineQuery, TweetCandidate] | |
with TimelinePersistenceUtils | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\PreviouslyServedTweetsFilter.scala | |
object PreviouslyServedTweetsFilter | |
extends Filter[PipelineQuery, TweetCandidate] | |
with Filter.Conditionally[PipelineQuery, TweetCandidate] | |
def onlyIf( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Boolean | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\RejectTweetFromViewerFilter.scala | |
object RejectTweetFromViewerFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\RetweetDeduplicationFilter.scala | |
object RetweetDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\RetweetSourceTweetRemovingFilter.scala | |
object RetweetSourceTweetRemovingFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\filter\SocialContextFilter.scala | |
object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[FilterResult[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\DismissFatigueGate.scala | |
object DismissFatigueGate | |
class DismissFatigueGate( | |
suggestType: SuggestType, | |
dismissInfoFeature: Feature[PipelineQuery, Map[SuggestType, Option[DismissInfo]]], | |
baseDismissDuration: Duration = DismissFatigueGate.DefaultBaseDismissDuration, | |
) extends Gate[PipelineQuery] | |
def shouldContinue(query: PipelineQuery): Stitch[Boolean] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\ExcludeSoftUserGate.scala | |
object ExcludeSoftUserGate extends Gate[PipelineQuery] | |
def shouldContinue(query: PipelineQuery): Stitch[Boolean] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\MinCachedTweetsGate.scala | |
class MinCachedTweetsGate( | |
candidatePipelineIdentifier: CandidatePipelineIdentifier, | |
minCachedTweetsParam: Param[Int]) | |
extends Gate[PipelineQuery] | |
def shouldContinue(query: PipelineQuery): Stitch[Boolean] | |
object MinCachedTweetsGate | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\NonEmptySeqFeatureGate.scala | |
class NonEmptySeqFeatureGate[T: TypeTag]( | |
feature: Feature[PipelineQuery, Seq[T]]) | |
extends Gate[PipelineQuery] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\RequestContextGate.scala | |
class RequestContextGate(requestContexts: Seq[RequestContext.Value]) | |
extends Gate[PipelineQuery with HasDeviceContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\RequestContextNotGate.scala | |
class RequestContextNotGate(requestContexts: Seq[RequestContext.Value]) | |
extends Gate[PipelineQuery with HasDeviceContext] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\SupportedLanguagesGate.scala | |
object SupportedLanguagesGate extends Gate[PipelineQuery] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\TimelinesPersistenceStoreLastInjectionGate.scala | |
class TimelinesPersistenceStoreLastInjectionGate( | |
minInjectionIntervalParam: Param[Duration], | |
persistenceEntriesFeature: Feature[PipelineQuery, Seq[TimelineResponseV3]], | |
entityIdType: EntityIdType.Value) | |
extends Gate[PipelineQuery] | |
with TimelinePersistenceUtils | |
def shouldContinue(query: PipelineQuery): Stitch[Boolean] = | |
Stitch( | |
query.queryTime.since(getLastInjectionTime(query)) > query.params(minInjectionIntervalParam)) | |
private def getLastInjectionTime(query: PipelineQuery) = query.features | |
.flatMap | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\gate\ViewerIsListOwnerGate.scala | |
class ViewerIsListOwnerGate @Inject() (socialGraph: SocialGraph) | |
extends Gate[PipelineQuery with HasListId] | |
def shouldContinue(query: PipelineQuery with HasListId): Stitch[Boolean] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\query_transformer\EditedTweetsCandidatePipelineQueryTransformer.scala | |
object EditedTweetsCandidatePipelineQueryTransformer | |
extends CandidatePipelineQueryTransformer[PipelineQuery, Seq[Long]] | |
def transform(query: PipelineQuery): Seq[Long] | |
def getApplicableCandidates(query: PipelineQuery): Seq[PersistenceStoreEntry] | |
class PersistenceStoreEntry( | |
entryWithItemIds: EntryWithItemIds, | |
servedTime: Time, | |
clientPlatform: ClientPlatform, | |
requestType: RequestType) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\scorer\FeedbackFatigueScorer.scala | |
object FeedbackFatigueScorer | |
extends Scorer[PipelineQuery, TweetCandidate] | |
with Conditionally[PipelineQuery] | |
def features: Set[Feature[_, _]] = Set(ScoreFeature) | |
override def onlyIf(query: PipelineQuery): Boolean = | |
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) | |
private val DurationForFiltering = 14.days | |
private val DurationForDiscounting = 140.days | |
private val ScoreMultiplierLowerBound = 0.2 | |
private val ScoreMultiplierUpperBound = 1.0 | |
private val ScoreMultiplierIncrementsCount = 4 | |
private val ScoreMultiplierIncrement = | |
(ScoreMultiplierUpperBound - ScoreMultiplierLowerBound) / ScoreMultiplierIncrementsCount | |
private val ScoreMultiplierIncrementDurationInDays = | |
DurationForDiscounting.inDays / ScoreMultiplierIncrementsCount.toDouble | |
override def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def getUserDiscounts( | |
queryTime: Time, | |
feedbackEntries: Seq[FeedbackEntry], | |
): Map[Long, Double] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\scorer\OONTweetScalingScorer.scala | |
object OONTweetScalingScorer extends Scorer[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\scorer\VerifiedAuthorScalingScorer.scala | |
object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def getUpdatedScore( | |
score: Option[Double], | |
candidate: CandidateWithFeatures[TweetCandidate], | |
query: PipelineQuery | |
): Option[Double] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\selector\DebunchCandidates.scala | |
trait MustDebunch | |
def apply(candidate: CandidateWithDetails): Boolean | |
} | |
object DebunchCandidates | |
class DebunchCandidates( | |
override val pipelineScope: CandidateScope, | |
mustDebunch: MustDebunch, | |
maxBunchSize: Int) | |
extends Selector[PipelineQuery] | |
def apply( | |
query: PipelineQuery, | |
remainingCandidates: Seq[CandidateWithDetails], | |
result: Seq[CandidateWithDetails] | |
): SelectorResult | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\selector\UpdateConversationModuleId.scala | |
class UpdateConversationModuleId( | |
override val pipelineScope: CandidateScope) | |
extends Selector[PipelineQuery] | |
def apply( | |
query: PipelineQuery, | |
remainingCandidates: Seq[CandidateWithDetails], | |
result: Seq[CandidateWithDetails] | |
): SelectorResult | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\selector\UpdateHomeClientEventDetails.scala | |
class UpdateHomeClientEventDetails(candidatePipelines: Set[CandidatePipelineIdentifier]) | |
extends Selector[PipelineQuery] | |
def apply( | |
query: PipelineQuery, | |
remainingCandidates: Seq[CandidateWithDetails], | |
result: Seq[CandidateWithDetails] | |
): SelectorResult | |
def updateItemPresentation( | |
query: PipelineQuery, | |
item: ItemCandidateWithDetails, | |
resultCandidateFeatures: FeatureMap, | |
resultFeatures: FeatureMap, | |
): ItemCandidateWithDetails | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\selector\UpdateNewTweetsPillDecoration.scala | |
object UpdateNewTweetsPillDecoration | |
class UpdateNewTweetsPillDecoration[Query <: PipelineQuery with HasDeviceContext]( | |
override val pipelineScope: CandidateScope, | |
stringCenter: StringCenter, | |
seeNewTweetsString: ExternalString, | |
tweetedString: ExternalString) | |
extends Selector[Query] | |
def apply( | |
query: Query, | |
remainingCandidates: Seq[CandidateWithDetails], | |
result: Seq[CandidateWithDetails] | |
): SelectorResult | |
def useAvatars(query: Query, userIds: Seq[Long]): Boolean | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ClientEventsBuilder.scala | |
trait ClientEventsBuilder | |
def section(query: PipelineQuery): Option[String] | |
def count( | |
candidates: Seq[CandidateWithDetails], | |
predicate: FeatureMap => Boolean = _ => true, | |
queryFeatures: FeatureMap = FeatureMap.empty | |
): Option[Long] = Some(candidates.view.count(item => predicate(item.features ++ queryFeatures))) | |
protected def sum( | |
candidates: Seq[CandidateWithDetails], | |
predicate: FeatureMap => Option[Int], | |
queryFeatures: FeatureMap = FeatureMap.empty | |
): Option[Long] = | |
Some(candidates.view.flatMap(item => predicate(item.features ++ queryFeatures)).sum) | |
} | |
private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder | |
def build( | |
query: PipelineQuery, | |
injectedTweets: Seq[ItemCandidateWithDetails], | |
promotedTweets: Seq[ItemCandidateWithDetails], | |
whoToFollowUsers: Seq[ItemCandidateWithDetails] | |
): Seq[ClientEvent] | |
object EmptyTimelineEventsBuilder extends ClientEventsBuilder | |
def build( | |
query: PipelineQuery, | |
injectedTweets: Seq[ItemCandidateWithDetails] | |
): Seq[ClientEvent] | |
object QueryEventsBuilder extends ClientEventsBuilder | |
def build( | |
query: PipelineQuery, | |
injectedTweets: Seq[ItemCandidateWithDetails] | |
): Seq[ClientEvent] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\HomeScribeClientEventSideEffect.scala | |
class HomeScribeClientEventSideEffect( | |
override val logPipelinePublisher: EventPublisher[LogEvent], | |
injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier], | |
adsCandidatePipelineIdentifier: CandidatePipelineIdentifier, | |
whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, | |
) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] | |
def buildClientEvents( | |
query: PipelineQuery, | |
selectedCandidates: Seq[CandidateWithDetails], | |
remainingCandidates: Seq[CandidateWithDetails], | |
droppedCandidates: Seq[CandidateWithDetails], | |
response: Timeline | |
): Seq[ScribeClientEventSideEffect.ClientEvent] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\HomeScribeServedEntriesSideEffect.scala | |
class HomeScribeServedEntriesSideEffect @Inject() ( | |
scribeEventPublisher: EventPublisher[thrift.Timeline]) | |
extends PipelineResultSideEffect[ | |
PipelineQuery with HasSeenTweetIds with HasDeviceContext, | |
Timeline | |
] | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[ | |
PipelineQuery with HasSeenTweetIds with HasDeviceContext, | |
Timeline | |
] | |
): Stitch[Unit] | |
def buildTimeline( | |
inputs: PipelineResultSideEffect.Inputs[ | |
PipelineQuery with HasSeenTweetIds with HasDeviceContext, | |
Timeline | |
] | |
): thrift.Timeline | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\PublishClientSentImpressionsEventBusSideEffect.scala | |
object PublishClientSentImpressionsEventBusSideEffect | |
class PublishClientSentImpressionsEventBusSideEffect @Inject() ( | |
eventBusPublisher: EventBusPublisher[PublishedImpressionList]) | |
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] | |
with PipelineResultSideEffect.Conditionally[ | |
PipelineQuery with HasSeenTweetIds, | |
HasMarshalling | |
] | |
def onlyIf( | |
query: PipelineQuery with HasSeenTweetIds, | |
selectedCandidates: Seq[CandidateWithDetails], | |
remainingCandidates: Seq[CandidateWithDetails], | |
droppedCandidates: Seq[CandidateWithDetails], | |
response: HasMarshalling | |
): Boolean = query.seenTweetIds.exists(_.nonEmpty) | |
def buildEvents( | |
query: PipelineQuery with HasSeenTweetIds, | |
currentTime: Long | |
): Option[Seq[Impression]] | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] | |
): Stitch[Unit] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\PublishClientSentImpressionsManhattanSideEffect.scala | |
class PublishClientSentImpressionsManhattanSideEffect @Inject() ( | |
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) | |
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] | |
with PipelineResultSideEffect.Conditionally[ | |
PipelineQuery with HasSeenTweetIds, | |
HasMarshalling | |
] | |
def onlyIf( | |
query: PipelineQuery with HasSeenTweetIds, | |
selectedCandidates: Seq[CandidateWithDetails], | |
remainingCandidates: Seq[CandidateWithDetails], | |
droppedCandidates: Seq[CandidateWithDetails], | |
response: HasMarshalling | |
): Boolean = query.seenTweetIds.exists(_.nonEmpty) | |
def buildEvents(query: PipelineQuery): Option[(Long, t.TweetImpressionsEntries)] | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] | |
): Stitch[Unit] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateFeatureKeysKafkaSideEffect.scala | |
class ServedCandidateFeatureKeysKafkaSideEffect( | |
topic: String, | |
sourceIdentifiers: Set[identifier.CandidatePipelineIdentifier]) | |
extends KafkaPublishingSideEffect[ | |
sc.CandidateFeatureKey, | |
pldr.PolyDataRecord, | |
PipelineQuery, | |
Timeline | |
] | |
with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] | |
def onlyIf( | |
query: PipelineQuery, | |
selectedCandidates: Seq[CandidateWithDetails], | |
remainingCandidates: Seq[CandidateWithDetails], | |
droppedCandidates: Seq[CandidateWithDetails], | |
response: Timeline | |
): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam) | |
override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" | |
override val keySerde: Serializer[sc.CandidateFeatureKey] = | |
CandidateFeatureKeySerde().serializer() | |
override val valueSerde: Serializer[pldr.PolyDataRecord] = | |
TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer | |
override val clientId: String = "home_mixer_served_candidate_feature_keys_producer" | |
override def buildRecords( | |
query: PipelineQuery, | |
selectedCandidates: Seq[CandidateWithDetails], | |
remainingCandidates: Seq[CandidateWithDetails], | |
droppedCandidates: Seq[CandidateWithDetails], | |
response: Timeline | |
): Seq[ProducerRecord[sc.CandidateFeatureKey, pldr.PolyDataRecord]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala | |
class ServedCandidateFeatureKeysKafkaSideEffectBuilder @Inject() ( | |
injectedServiceIdentifier: ServiceIdentifier) | |
def build( | |
sourceIdentifiers: Set[CandidatePipelineIdentifier] | |
): ServedCandidateFeatureKeysKafkaSideEffect | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateKafkaSideEffect.scala | |
object ServedCandidateKafkaSideEffect | |
def extractCandidates( | |
query: PipelineQuery, | |
selectedCandidates: Seq[CandidateWithDetails], | |
sourceIdentifiers: Set[CandidatePipelineIdentifier] | |
): Seq[ItemCandidateWithDetails] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateKeysKafkaSideEffect.scala | |
class ServedCandidateKeysKafkaSideEffect( | |
topic: String, | |
sourceIdentifiers: Set[CandidatePipelineIdentifier]) | |
extends KafkaPublishingSideEffect[ | |
sc.ServedCandidateKey, | |
pldr.PolyDataRecord, | |
PipelineQuery, | |
Timeline | |
] | |
with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] | |
def onlyIf( | |
query: PipelineQuery, | |
selectedCandidates: Seq[CandidateWithDetails], | |
remainingCandidates: Seq[CandidateWithDetails], | |
droppedCandidates: Seq[CandidateWithDetails], | |
response: Timeline | |
): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam) | |
override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" | |
override val keySerde: Serializer[sc.ServedCandidateKey] = ServedCandidateKeySerde.serializer() | |
override val valueSerde: Serializer[pldr.PolyDataRecord] = | |
TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer | |
override val clientId: String = "home_mixer_served_candidate_keys_producer" | |
override def buildRecords( | |
query: PipelineQuery, | |
selectedCandidates: Seq[CandidateWithDetails], | |
remainingCandidates: Seq[CandidateWithDetails], | |
droppedCandidates: Seq[CandidateWithDetails], | |
response: Timeline | |
): Seq[ProducerRecord[sc.ServedCandidateKey, pldr.PolyDataRecord]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedCandidateKeysKafkaSideEffectBuilder.scala | |
class ServedCandidateKeysKafkaSideEffectBuilder @Inject() ( | |
injectedServiceIdentifier: ServiceIdentifier) | |
def build( | |
sourceIdentifiers: Set[CandidatePipelineIdentifier] | |
): ServedCandidateKeysKafkaSideEffect | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\ServedStatsSideEffect.scala | |
class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver) | |
extends PipelineResultSideEffect[PipelineQuery, Timeline] | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] | |
): Stitch[Unit] | |
def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit | |
def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit | |
def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\TruncateTimelinesPersistenceStoreSideEffect.scala | |
class TruncateTimelinesPersistenceStoreSideEffect @Inject() ( | |
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) | |
extends PipelineResultSideEffect[PipelineQuery, Timeline] | |
def getResponsesToDelete(query: PipelineQuery): Seq[TimelineResponseV3] | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] | |
): Stitch[Unit] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\UpdateImpressionBloomFilterSideEffect.scala | |
class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter) | |
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline] | |
with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline] | |
def onlyIf( | |
query: PipelineQuery with HasSeenTweetIds, | |
selectedCandidates: Seq[CandidateWithDetails], | |
remainingCandidates: Seq[CandidateWithDetails], | |
droppedCandidates: Seq[CandidateWithDetails], | |
response: Timeline | |
): Boolean = query.seenTweetIds.exists(_.nonEmpty) | |
def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq] | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline] | |
): Stitch[Unit] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\UpdateLastNonPollingTimeSideEffect.scala | |
class UpdateLastNonPollingTimeSideEffect[ | |
Query <: PipelineQuery with HasDeviceContext, | |
ResponseType <: HasMarshalling] @Inject() ( | |
override val userSessionStore: ReadWriteUserSessionStore) | |
extends UserSessionStoreUpdateSideEffect[ | |
WriteRequest, | |
Query, | |
ResponseType | |
] | |
def buildWriteRequest(query: Query): Option[WriteRequest] | |
def makeLastNonPollingTimestamps(query: Query): NonPollingTimestamps | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\functional_component\side_effect\UpdateTimelinesPersistenceStoreSideEffect.scala | |
object UpdateTimelinesPersistenceStoreSideEffect | |
class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( | |
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) | |
extends PipelineResultSideEffect[PipelineQuery, Timeline] | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] | |
): Stitch[Unit] | |
def buildTweetEntryWithItemIds( | |
candidate: ItemCandidateWithDetails, | |
sortIndex: Long | |
): EntryWithItemIds | |
def requestTypeFromQuery(query: PipelineQuery): persistence.RequestType | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\DeviceContextUnmarshaller.scala | |
class DeviceContextUnmarshaller @Inject() () | |
def apply(deviceContext: t.DeviceContext): DeviceContext | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\HomeMixerDebugParamsUnmarshaller.scala | |
class HomeMixerDebugParamsUnmarshaller @Inject() ( | |
featureValueUnmarshaller: FeatureValueUnmarshaller) | |
def apply(debugParams: t.DebugParams): DebugParams | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\HomeMixerProductContextUnmarshaller.scala | |
class HomeMixerProductContextUnmarshaller @Inject() ( | |
deviceContextUnmarshaller: DeviceContextUnmarshaller) | |
def apply(productContext: t.ProductContext): ProductContext = productContext match | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\HomeMixerProductUnmarshaller.scala | |
class HomeMixerProductUnmarshaller @Inject() () | |
def apply(product: t.Product): Product = product match | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\request\HomeMixerRequestUnmarshaller.scala | |
class HomeMixerRequestUnmarshaller @Inject() ( | |
clientContextUnmarshaller: ClientContextUnmarshaller, | |
homeProductUnmarshaller: HomeMixerProductUnmarshaller, | |
homeProductContextUnmarshaller: HomeMixerProductContextUnmarshaller, | |
homeDebugParamsUnmarshaller: HomeMixerDebugParamsUnmarshaller) | |
def apply(homeRequest: t.HomeMixerRequest): HomeMixerRequest | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\ChronologicalCursorMarshaller.scala | |
object ChronologicalCursorMarshaller | |
def apply(cursor: UrtOrderedCursor): Option[t.ChronologicalCursor] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\ChronologicalCursorUnmarshaller.scala | |
object ChronologicalCursorUnmarshaller | |
def apply(requestCursor: t.RequestCursor): Option[UrtOrderedCursor] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\DeviceContextMarshaller.scala | |
class DeviceContextMarshaller @Inject() () | |
def apply(deviceContext: DeviceContext, clientContext: ClientContext): t.DeviceContext | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\RecommendedUsersCursorUnmarshaller.scala | |
object RecommendedUsersCursorUnmarshaller | |
def apply(requestCursor: t.RequestCursor): Option[UrtUnorderedExcludeIdsCursor] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\TimelineServiceCursorMarshaller.scala | |
object TimelineServiceCursorMarshaller | |
def apply(cursor: UrtOrderedCursor): Option[t.Cursor2] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timelines\TopicContextFunctionalityTypeUnmarshaller.scala | |
object TopicContextFunctionalityTypeUnmarshaller | |
def apply( | |
topicContextFunctionalityType: urt.TopicContextFunctionalityType | |
): TopicContextFunctionalityType = topicContextFunctionalityType match | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timeline_logging\ConversationEntryMarshaller.scala | |
object ConversationEntryMarshaller | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timeline_logging\PromotedTweetEntryMarshaller.scala | |
object PromotedTweetEntryMarshaller | |
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetEntry | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timeline_logging\TweetEntryMarshaller.scala | |
object TweetEntryMarshaller | |
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetEntry | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\marshaller\timeline_logging\WhoToFollowEntryMarshaller.scala | |
object WhoToFollowEntryMarshaller | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\ClearCacheIncludeInstruction.scala | |
class ClearCacheIncludeInstruction( | |
enableParam: FSParam[Boolean], | |
minEntriesParam: FSBoundedParam[Int]) | |
extends IncludeInstruction[PipelineQuery with HasDeviceContext] | |
def apply( | |
query: PipelineQuery with HasDeviceContext, | |
entries: Seq[TimelineEntry] | |
): Boolean | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\ContentFeatures.scala | |
object ContentFeatures | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\GapIncludeInstruction.scala | |
object GapIncludeInstruction | |
extends IncludeInstruction[PipelineQuery with HasPipelineCursor[UrtOrderedCursor]] | |
def apply( | |
query: PipelineQuery with HasPipelineCursor[UrtOrderedCursor], | |
entries: Seq[TimelineEntry] | |
): Boolean | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\HomeAdsQuery.scala | |
trait HomeAdsQuery | |
extends AdsQuery | |
with PipelineQuery | |
with HasDeviceContext | |
with HasPipelineCursor[UrtOrderedCursor] | |
def requestTriggerType: Option[RequestTriggerType] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\HomeFeatures.scala | |
object HomeFeatures | |
object AncestorsFeature extends Feature[TweetCandidate, Seq[ta.TweetAncestor]] | |
object AudioSpaceMetaDataFeature extends Feature[TweetCandidate, Option[AudioSpaceMetaData]] | |
object TwitterListIdFeature extends Feature[TweetCandidate, Option[Long]] | |
/** | |
* For Retweets, this should refer to the retweeting user. Use [[SourceUserIdFeature]] if you want to know | |
* who created the Tweet that was retweeted. | |
*/ | |
object AuthorIdFeature | |
extends Feature[TweetCandidate, Option[Long]] | |
with LongDiscreteDataRecordCompatible | |
object AuthorIsEligibleForConnectBoostFeature extends Feature[TweetCandidate, Boolean] | |
object AuthorIsBlueVerifiedFeature extends Feature[TweetCandidate, Boolean] | |
object AuthoredByContextualUserFeature extends Feature[TweetCandidate, Boolean] | |
object CachedCandidatePipelineIdentifierFeature extends Feature[TweetCandidate, Option[String]] | |
object CandidateSourceIdFeature | |
extends Feature[TweetCandidate, Option[cts.CandidateTweetSourceId]] | |
object ConversationFeature extends Feature[TweetCandidate, Option[ConversationFeatures]] | |
/** | |
* This field should be set to the focal Tweet's tweetId for all tweets which are expected to | |
* be rendered in the same convo module. For non-convo module Tweets, this will be | |
* set to None. Note this is different from how TweetyPie defines ConversationId which is defined | |
* on all Tweets and points to the root tweet. This feature is used for grouping convo modules together. | |
*/ | |
object ConversationModuleFocalTweetIdFeature extends Feature[TweetCandidate, Option[Long]] | |
/** | |
* This field should always be set to the root Tweet in a conversation for all Tweets. For replies, this will | |
* point back to the root Tweet. For non-replies, this will be the candidate's Tweet id. This is consistent with | |
* the TweetyPie definition of ConversationModuleId. | |
*/ | |
object ConversationModuleIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object DirectedAtUserIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object EarlybirdFeature extends Feature[TweetCandidate, Option[sc.ThriftTweetFeatures]] | |
object EarlybirdScoreFeature extends Feature[TweetCandidate, Option[Double]] | |
object EntityTokenFeature extends Feature[TweetCandidate, Option[String]] | |
object ExclusiveConversationAuthorIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object FavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] | |
object FeedbackHistoryFeature extends Feature[TweetCandidate, Seq[FeedbackEntry]] | |
object RetweetedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] | |
object RepliedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] | |
object FollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] | |
object TopicIdSocialContextFeature extends Feature[TweetCandidate, Option[Long]] | |
object TopicContextFunctionalityTypeFeature | |
extends Feature[TweetCandidate, Option[TopicContextFunctionalityType]] | |
object FromInNetworkSourceFeature extends Feature[TweetCandidate, Boolean] | |
object FullScoringSucceededFeature extends Feature[TweetCandidate, Boolean] | |
object HasDisplayedTextFeature extends Feature[TweetCandidate, Boolean] | |
object InReplyToTweetIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object InReplyToUserIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object IsAncestorCandidateFeature extends Feature[TweetCandidate, Boolean] | |
object IsExtendedReplyFeature | |
extends DataRecordFeature[TweetCandidate, Boolean] | |
with BoolDataRecordCompatible | |
object IsRandomTweetFeature | |
extends Feature[TweetCandidate, Boolean] | |
with BoolDataRecordCompatible | |
object IsReadFromCacheFeature extends Feature[TweetCandidate, Boolean] | |
object IsRetweetFeature extends Feature[TweetCandidate, Boolean] | |
object IsRetweetedReplyFeature extends Feature[TweetCandidate, Boolean] | |
object LastScoredTimestampMsFeature extends Feature[TweetCandidate, Option[Long]] | |
object NonSelfFavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] | |
object NumImagesFeature extends Feature[TweetCandidate, Option[Int]] | |
object OriginalTweetCreationTimeFromSnowflakeFeature extends Feature[TweetCandidate, Option[Time]] | |
object PositionFeature extends Feature[TweetCandidate, Option[Int]] | |
// Internal id generated per prediction service request | |
object PredictionRequestIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object QuotedUserIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object ScoreFeature extends Feature[TweetCandidate, Option[Double]] | |
object SemanticCoreIdFeature extends Feature[TweetCandidate, Option[Long]] | |
// Key for kafka logging | |
object ServedIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object SimclustersTweetTopKClustersWithScoresFeature | |
extends Feature[TweetCandidate, Map[String, Double]] | |
object SocialContextFeature extends Feature[TweetCandidate, Option[tst.SocialContext]] | |
object SourceTweetIdFeature | |
extends Feature[TweetCandidate, Option[Long]] | |
with LongDiscreteDataRecordCompatible | |
object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object StreamToKafkaFeature extends Feature[TweetCandidate, Boolean] | |
object SuggestTypeFeature extends Feature[TweetCandidate, Option[st.SuggestType]] | |
object TSPMetricTagFeature extends Feature[TweetCandidate, Set[tsp.MetricTag]] | |
object TweetLanguageFeature extends Feature[TweetCandidate, Option[String]] | |
object TweetUrlsFeature extends Feature[TweetCandidate, Seq[String]] | |
object VideoDurationMsFeature extends Feature[TweetCandidate, Option[Int]] | |
object ViewerIdFeature | |
extends Feature[TweetCandidate, Long] | |
with LongDiscreteDataRecordCompatible | |
def featureName: String = SharedFeatures.USER_ID.getFeatureName | |
override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) | |
} | |
object WeightedModelScoreFeature extends Feature[TweetCandidate, Option[Double]] | |
object MentionUserIdFeature extends Feature[TweetCandidate, Seq[Long]] | |
object MentionScreenNameFeature extends Feature[TweetCandidate, Seq[String]] | |
object SemanticAnnotationFeature extends Feature[TweetCandidate, Seq[esb.TweetEntityAnnotation]] | |
object HasImageFeature extends Feature[TweetCandidate, Boolean] | |
object HasVideoFeature extends Feature[TweetCandidate, Boolean] | |
// Tweetypie VF Features | |
object IsHydratedFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Boolean] | |
object IsNsfwFeature extends Feature[TweetCandidate, Boolean] | |
object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean] | |
// Raw Tweet Text from Tweetypie | |
object TweetTextFeature extends Feature[TweetCandidate, Option[String]] | |
// SGS Features | |
/** | |
* By convention, this is set to true for retweets of non-followed authors | |
* E.g. where somebody the viewer follows retweets a Tweet from somebody the viewer doesn't follow | |
*/ | |
object InNetworkFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Boolean] | |
object AccountAgeFeature extends Feature[PipelineQuery, Option[Time]] | |
object ClientIdFeature | |
extends Feature[PipelineQuery, Option[Long]] | |
with LongDiscreteDataRecordCompatible | |
def featureName: String = SharedFeatures.CLIENT_ID.getFeatureName | |
override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.ClientType) | |
} | |
object CachedScoredTweetsFeature extends Feature[PipelineQuery, Seq[hmt.CachedScoredTweet]] | |
object DeviceLanguageFeature extends Feature[PipelineQuery, Option[String]] | |
object DismissInfoFeature | |
extends FeatureWithDefaultOnFailure[PipelineQuery, Map[st.SuggestType, Option[DismissInfo]]] | |
def defaultValue: Map[st.SuggestType, Option[DismissInfo]] = Map.empty | |
} | |
object FollowingLastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] | |
object GetInitialFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible | |
def featureName: String = RequestContextFeatures.IS_GET_INITIAL.getFeatureName | |
override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty | |
} | |
object GetMiddleFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible | |
def featureName: String = RequestContextFeatures.IS_GET_MIDDLE.getFeatureName | |
override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty | |
} | |
object GetNewerFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible | |
def featureName: String = RequestContextFeatures.IS_GET_NEWER.getFeatureName | |
override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty | |
} | |
object GetOlderFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible | |
def featureName: String = RequestContextFeatures.IS_GET_OLDER.getFeatureName | |
override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty | |
} | |
object GuestIdFeature | |
extends Feature[PipelineQuery, Option[Long]] | |
with LongDiscreteDataRecordCompatible | |
def featureName: String = SharedFeatures.GUEST_ID.getFeatureName | |
override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.GuestId) | |
} | |
object HasDarkRequestFeature extends Feature[TweetCandidate, Option[Boolean]] | |
object ImpressionBloomFilterFeature | |
extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq] | |
def defaultValue: blm.ImpressionBloomFilterSeq = | |
blm.ImpressionBloomFilterSeq(Seq.empty) | |
} | |
object IsForegroundRequestFeature extends Feature[PipelineQuery, Boolean] | |
object IsLaunchRequestFeature extends Feature[PipelineQuery, Boolean] | |
object LastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] | |
object NonPollingTimesFeature extends Feature[PipelineQuery, Seq[Long]] | |
object PersistenceEntriesFeature extends Feature[PipelineQuery, Seq[TimelineResponseV3]] | |
object PollingFeature extends Feature[PipelineQuery, Boolean] | |
object PullToRefreshFeature extends Feature[PipelineQuery, Boolean] | |
// Scores from Real Graph representing the relationship between the viewer and another user | |
object RealGraphInNetworkScoresFeature extends Feature[PipelineQuery, Map[UserId, Double]] | |
object RequestJoinIdFeature extends Feature[TweetCandidate, Option[Long]] | |
// Internal id generated per request, mainly to deduplicate re-served cached tweets in logging | |
object ServedRequestIdFeature extends Feature[PipelineQuery, Option[Long]] | |
object ServedTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]] | |
object TweetImpressionsFeature extends Feature[PipelineQuery, Seq[imp.TweetImpressionsEntry]] | |
object UserFollowedTopicsCountFeature extends Feature[PipelineQuery, Option[Int]] | |
object UserFollowingCountFeature extends Feature[PipelineQuery, Option[Int]] | |
object UserScreenNameFeature extends Feature[PipelineQuery, Option[String]] | |
object UserStateFeature extends Feature[PipelineQuery, Option[um.UserState]] | |
object UserTypeFeature extends Feature[PipelineQuery, Option[gt.UserType]] | |
object WhoToFollowExcludedUserIdsFeature | |
extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\request\DeviceContext.scala | |
class DeviceContext( | |
isPolling: Option[Boolean], | |
requestContext: Option[String], | |
latestControlAvailable: Option[Boolean], | |
autoplayEnabled: Option[Boolean]) | |
def toTimelineServiceDeviceContext(clientContext: ClientContext): tls.DeviceContext = | |
tls.DeviceContext( | |
countryCode = clientContext.countryCode, | |
languageCode = clientContext.languageCode, | |
clientAppId = clientContext.appId, | |
ipAddress = clientContext.ipAddress, | |
guestId = clientContext.guestId, | |
sessionId = None, | |
timezone = None, | |
userAgent = clientContext.userAgent, | |
deviceId = clientContext.deviceId, | |
isPolling = isPolling, | |
requestProvenance = requestContext, | |
referrer = None, | |
tfeAuthHeader = None, | |
mobileDeviceId = clientContext.mobileDeviceId, | |
isSessionStart = None, | |
displaySize = None, | |
isURTRequest = Some(true), | |
latestControlAvailable = latestControlAvailable, | |
guestIdMarketing = clientContext.guestIdMarketing, | |
isInternalOrTwoffice = clientContext.isTwoffice, | |
browserNotificationPermission = None, | |
guestIdAds = clientContext.guestIdAds, | |
) | |
} | |
object DeviceContext | |
object RequestContext extends Enumeration | |
trait HasDeviceContext | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\request\HasListId.scala | |
trait HasListId | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\request\HasSeenTweetIds.scala | |
trait HasSeenTweetIds | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\model\request\HomeMixerProduct.scala | |
object FollowingProduct extends Product | |
object ForYouProduct extends Product | |
object ScoredTweetsProduct extends Product | |
object ListTweetsProduct extends Product | |
object ListRecommendedUsersProduct extends Product | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\AdvertiserBrandSafetySettingsStoreModule.scala | |
object AdvertiserBrandSafetySettingsStoreModule extends TwitterModule | |
def providesAdvertiserBrandSafetySettingsStore( | |
injectedServiceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): ReadableStore[Long, ads.AdvertiserBrandSafetySettings] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ClientSentImpressionsPublisherModule.scala | |
object ClientSentImpressionsPublisherModule extends TwitterModule with ConfigUtils | |
def providesClientSentImpressionsPublisher( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): EventBusPublisher[PublishedImpressionList] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ConversationServiceModule.scala | |
object ConversationServiceModule | |
extends ThriftMethodBuilderClientModule[ | |
ConversationService.ServicePerEndpoint, | |
ConversationService.MethodPerEndpoint | |
] | |
with MtlsClient | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\FeedbackHistoryClientModule.scala | |
object FeedbackHistoryClientModule extends TwitterModule | |
def providesFeedbackHistoryClient( | |
serviceId: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\HomeAdsCandidateSourceModule.scala | |
object HomeAdsCandidateSourceModule | |
extends ThriftMethodBuilderClientModule[ | |
NewAdServer.ServicePerEndpoint, | |
NewAdServer.MethodPerEndpoint | |
] | |
with MtlsClient | |
def configureMethodBuilder( | |
injector: Injector, | |
methodBuilder: MethodBuilder | |
): MethodBuilder | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\HomeMixerFlagsModule.scala | |
object HomeMixerFlagsModule extends TwitterModule | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\HomeMixerResourcesModule.scala | |
object HomeMixerResourcesModule extends TwitterModule | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\HomeNaviModelClientModule.scala | |
object HomeNaviModelClientModule extends TwitterModule | |
def providesPredictionGRPCService( | |
serviceIdentifier: ServiceIdentifier, | |
): PredictionGRPCService | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ImpressionBloomFilterModule.scala | |
object ImpressionBloomFilterModule extends TwitterModule | |
def providesImpressionBloomFilter( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): ImpressionBloomFilter | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\InjectionHistoryClientModule.scala | |
object InjectionHistoryClientModule extends TwitterModule | |
def providesInjectionHistoryClient( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ManhattanClientsModule.scala | |
object ManhattanClientsModule extends TwitterModule with ConfigUtils | |
def providesRealGraphManhattanEndpoint( | |
serviceIdentifier: ServiceIdentifier | |
): ManhattanKVEndpoint | |
def providesUserMetadataManhattanEndpoint( | |
serviceIdentifier: ServiceIdentifier | |
): ManhattanKVEndpoint | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ManhattanFeatureRepositoryModule.scala | |
object ManhattanFeatureRepositoryModule extends TwitterModule | |
def to(userId: Long): Try[ByteBuffer] | |
def from(b: ByteBuffer): Try[Long] = ??? | |
} | |
val FloatTensorTransformer = new Transformer[ByteBuffer, FloatTensor] | |
def to(input: ByteBuffer): Try[FloatTensor] | |
def from(b: FloatTensor): Try[ByteBuffer] = ??? | |
} | |
// manhattan clients | |
@Provides | |
@Singleton | |
@Named(ManhattanApolloClient) | |
def providesManhattanApolloClient( | |
serviceIdentifier: ServiceIdentifier | |
): mh.ManhattanCoordinator.MethodPerEndpoint | |
def providesManhattanAthenaClient( | |
serviceIdentifier: ServiceIdentifier | |
): mh.ManhattanCoordinator.MethodPerEndpoint | |
def providesManhattanOmegaClient( | |
serviceIdentifier: ServiceIdentifier | |
): mh.ManhattanCoordinator.MethodPerEndpoint | |
def providesManhattanStarbuckClient( | |
serviceIdentifier: ServiceIdentifier | |
): mh.ManhattanCoordinator.MethodPerEndpoint | |
def providesMetricCenterUserCountingFeatureRepository( | |
@Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint | |
): KeyValueRepository[Seq[Long], Long, rf.MCUserCountingFeatures] | |
def providesTimelineAggregateMetadataRepository( | |
@Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint | |
): Repository[Int, Option[DenseFeatureMetadata]] | |
def to(buffer: ByteBuffer): Try[DenseFeatureMetadata] = Try | |
def from(metadata: DenseFeatureMetadata): Try[ByteBuffer] = ??? | |
} | |
val inProcessCache: Cache[Int, Cached[DenseFeatureMetadata]] = InProcessLruCacheFactory( | |
ttl = Duration.fromMinutes(20), | |
lruSize = 30 | |
).apply(serializer = Transformer(_ => ???, _ => ???)) // Serialization is not necessary here. | |
val keyValueRepository = new ManhattanKeyValueRepository( | |
client = client, | |
keyTransformer = keyTransformer, | |
valueTransformer = valueTransformer, | |
appId = "timelines_dense_aggregates_encoding_metadata", // Expected QPS is negligible. | |
dataset = "user_session_dense_feature_metadata", | |
timeoutInMillis = 100 | |
) | |
KeyValueRepository | |
.singular( | |
new CachingKeyValueRepository[Seq[Int], Int, DenseFeatureMetadata]( | |
keyValueRepository, | |
new NonLockingCache(inProcessCache), | |
keysAsQuery[Int] | |
) | |
) | |
} | |
@Provides | |
@Singleton | |
@Named(RealGraphFeatureRepository) | |
def providesRealGraphFeatureRepository( | |
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint | |
): Repository[Long, Option[UserSession]] | |
def providesAuthorFeatureRepository( | |
@Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, | |
@Named(HomeAuthorFeaturesCacheClient) cacheClient: Memcache | |
): KeyValueRepository[Seq[Long], Long, af.AuthorFeatures] | |
def providesTwhinAuthorFollow20200101FeatureRepository( | |
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, | |
@Named(TwhinAuthorFollow20200101FeatureCacheClient) cacheClient: Memcache | |
): KeyValueRepository[Seq[Long], Long, ml.Embedding] | |
def providesTwhinUserFollowFeatureRepository( | |
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint | |
): KeyValueRepository[Seq[Long], Long, FloatTensor] | |
def providesTimelineAggregatePartARepository( | |
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, | |
): Repository[Long, Option[uss.UserSession]] = | |
timelineAggregateRepository( | |
mhClient = client, | |
mhDataset = "timelines_aggregates_v2_features_by_user_part_a_apollo", | |
mhAppId = "timelines_aggregates_v2_features_by_user_part_a_apollo" | |
) | |
@Provides | |
@Singleton | |
@Named(TimelineAggregatePartBRepository) | |
def providesTimelineAggregatePartBRepository( | |
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, | |
): Repository[Long, Option[uss.UserSession]] = | |
timelineAggregateRepository( | |
mhClient = client, | |
mhDataset = "timelines_aggregates_v2_features_by_user_part_b_apollo", | |
mhAppId = "timelines_aggregates_v2_features_by_user_part_b_apollo" | |
) | |
@Provides | |
@Singleton | |
@Named(TwhinUserEngagementFeatureRepository) | |
def providesTwhinUserEngagementFeatureRepository( | |
@Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint | |
): KeyValueRepository[Seq[Long], Long, FloatTensor] | |
def buildMemCachedRepository[K, V]( | |
keyValueRepository: KeyValueRepository[Seq[K], K, V], | |
cacheClient: Memcache, | |
cachePrefix: String, | |
ttl: Duration, | |
valueInjection: Injection[V, Array[Byte]] | |
): CachingKeyValueRepository[Seq[K], K, V] | |
def buildInProcessCachedRepository[K, V]( | |
keyValueRepository: KeyValueRepository[Seq[K], K, V], | |
ttl: Duration, | |
size: Int, | |
valueInjection: Injection[V, Array[Byte]] | |
): CachingKeyValueRepository[Seq[K], K, V] | |
def batchedManhattanKeyValueRepository[K, V]( | |
client: ManhattanCoordinator.MethodPerEndpoint, | |
keyTransformer: Transformer[K, ByteBuffer], | |
valueTransformer: Transformer[ByteBuffer, V], | |
appId: String, | |
dataset: String, | |
timeoutInMillis: Int, | |
chunkSize: Int = DEFAULT_RPC_CHUNK_SIZE | |
): KeyValueRepository[Seq[K], K, V] = | |
KeyValueRepository.chunked( | |
new ManhattanKeyValueRepository( | |
client = client, | |
keyTransformer = keyTransformer, | |
valueTransformer = valueTransformer, | |
appId = appId, | |
dataset = dataset, | |
timeoutInMillis = timeoutInMillis | |
), | |
chunker = ChunkingStrategy.equalSize(chunkSize) | |
) | |
private def transportFromByteBuffer(buffer: ByteBuffer): TTransport = | |
new TMemoryInputTransport( | |
buffer.array(), | |
buffer.arrayOffset() + buffer.position(), | |
buffer.remaining()) | |
private def timelineAggregateRepository( | |
mhClient: mh.ManhattanCoordinator.MethodPerEndpoint, | |
mhDataset: String, | |
mhAppId: String | |
): Repository[Long, Option[uss.UserSession]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ManhattanTweetImpressionStoreModule.scala | |
object ManhattanTweetImpressionStoreModule extends TwitterModule | |
def providesManhattanTweetImpressionStoreClient( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): ManhattanTweetImpressionStoreClient | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\MemcachedFeatureRepositoryModule.scala | |
object MemcachedFeatureRepositoryModule extends TwitterModule | |
def providesTimelinesRealTimeAggregateClient( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Memcache | |
def providesHomeAuthorFeaturesCacheClient( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Memcache | |
def providesTwhinAuthorFollow20200101FeatureCacheClient( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Memcache | |
def providesRealTimeInteractionGraphUserVertexClient( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Memcache | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\OptimizedStratoClientModule.scala | |
object OptimizedStratoClientModule extends TwitterModule | |
def mkRetryPolicy(tries: Int): RetryPolicy[Try[Nothing]] = | |
RetryPolicy.tries(tries, DefaultRetryPartialFunction) | |
@Singleton | |
@Provides | |
@Named(BatchedStratoClientWithModerateTimeout) | |
def providesStratoClient( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): Client | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\PeopleDiscoveryServiceModule.scala | |
object PeopleDiscoveryServiceModule | |
extends ThriftMethodBuilderClientModule[ | |
ThriftPeopleDiscoveryService.ServicePerEndpoint, | |
ThriftPeopleDiscoveryService.MethodPerEndpoint | |
] | |
with MtlsClient | |
def configureMethodBuilder( | |
injector: Injector, | |
methodBuilder: MethodBuilder | |
): MethodBuilder | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\PipelineFailureExceptionMapper.scala | |
class PipelineFailureExceptionMapper | |
extends ExceptionMapper[PipelineFailure, ThriftException] | |
with Logging | |
def handleException(throwable: PipelineFailure): Future[ThriftException] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\RealGraphInNetworkScoresModule.scala | |
object RealGraphInNetworkScoresModule extends TwitterModule | |
def providesRealGraphInNetworkScoresFeaturesStore( | |
@Named(RealGraphManhattanEndpoint) realGraphInNetworkScoresManhattanKVEndpoint: ManhattanKVEndpoint | |
): ReadableStore[ViewerId, Seq[Candidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\RealtimeAggregateFeatureRepositoryModule.scala | |
object RealtimeAggregateFeatureRepositoryModule | |
extends TwitterModule | |
with RealtimeAggregateHelpers | |
def providesUserTopicEngagementForNewUserCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[(Long, Long), ml.DataRecord] | |
def providesTwitterListEngagementCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[Long, ml.DataRecord] | |
def providesTopicEngagementCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[Long, ml.DataRecord] | |
def providesUserAuthorEngagementCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[(Long, Long), ml.DataRecord] | |
def providesUserEngagementCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[Long, ml.DataRecord] | |
def providesTweetCountryEngagementCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[(Long, String), ml.DataRecord] | |
def providesTweetEngagementCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[Long, ml.DataRecord] | |
def providesEngagementsReceivedByAuthorCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[Long, ml.DataRecord] | |
def providesTopicCountryEngagementCache( | |
@Named(TimelinesRealTimeAggregateClient) client: Memcache | |
): ReadCache[(Long, String), ml.DataRecord] | |
def providesRealTimeInteractionGraphUserVertexCache( | |
@Named(RealTimeInteractionGraphUserVertexClient) client: Memcache | |
): ReadCache[Long, ig.UserVertex] | |
trait RealtimeAggregateHelpers | |
def customKeyBuilder[K](prefix: String, f: K => Array[Byte]): K => String | |
def g(arr: Array[Byte]) = ??? | |
MemcacheHelper.keyEncoder(prefix)(Injection.build(f)(g)) | |
} | |
private val keyEncoder: AggregationKey => String | |
def keyTransformD1(f1: Feature.Discrete)(key: Long): String | |
def keyTransformD2( | |
f1: Feature.Discrete, | |
f2: Feature.Discrete | |
)( | |
keys: (Long, Long) | |
): String | |
def keyTransformD1T1( | |
f1: Feature.Discrete, | |
f2: Feature.Text | |
)( | |
keys: (Long, String) | |
): String | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ScoredTweetsMemcacheModule.scala | |
object ScoredTweetsMemcacheModule extends TwitterModule | |
def providesScoredTweetsCache( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): TtlCache[UserId, t.CachedScoredTweets] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ScribeEventPublisherModule.scala | |
object ScribeEventPublisherModule extends TwitterModule | |
def providesClientEventsScribeEventPublisher( | |
@Flag(ScribeClientEventsFlag) sendToScribe: Boolean | |
): EventPublisher[ca.LogEvent] | |
def providesCommonFeaturesScribeEventPublisher( | |
@Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean | |
): EventPublisher[pldr.PolyDataRecord] | |
def providesCandidateFeaturesScribeEventPublisher( | |
@Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean | |
): EventPublisher[pldr.PolyDataRecord] | |
def providesMinimumFeaturesScribeEventPublisher( | |
@Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean | |
): EventPublisher[pldr.PolyDataRecord] | |
def providesServedEntriesScribeEventPublisher( | |
@Flag(ScribeServedEntriesFlag) sendToScribe: Boolean | |
): EventPublisher[tl.Timeline] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\SimClustersRecentEngagementsClientModule.scala | |
object SimClustersRecentEngagementsClientModule extends TwitterModule | |
def providesSimilarityClient( | |
@Named(BatchedStratoClientWithModerateTimeout) | |
stratoClient: Client, | |
statsReceiver: StatsReceiver | |
): SimClustersRecentEngagementSimilarityClient | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\StaleTweetsCacheModule.scala | |
object StaleTweetsCacheModule extends TwitterModule | |
def providesCache( | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): MemcachedClient | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\ThriftFeatureRepositoryModule.scala | |
object ThriftFeatureRepositoryModule extends TwitterModule | |
def providesInterestsThriftServiceClient( | |
clientId: ClientId, | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): int.InterestsThriftService.MethodPerEndpoint | |
def providesUserFollowedTopicIdsRepository( | |
@Named(InterestsThriftServiceClient) client: int.InterestsThriftService.MethodPerEndpoint | |
): KeyValueRepository[Seq[Long], Long, Seq[Long]] | |
def lookup(userId: Long): Future[Seq[Long]] | |
def providesUtegSocialProofRepository( | |
clientId: ClientId, | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): KeyValueRepository[UtegQuery, Long, uteg.TweetRecommendation] | |
def lookup( | |
tweetIds: Seq[Long], | |
view: (Long, Map[Long, Double]) | |
): Future[Seq[Option[uteg.TweetRecommendation]]] | |
def providesTweetypieContentRepository( | |
clientId: ClientId, | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): KeyValueRepository[Seq[Long], Long, tp.Tweet] | |
def lookup(tweetIds: Seq[Long]): Future[Seq[Option[tp.Tweet]]] | |
def providesGraphTwoHopRepository( | |
clientId: ClientId, | |
serviceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): KeyValueRepository[(Seq[Long], Long), Long, Seq[gfs.IntersectionValue]] | |
def lookup( | |
userIds: Seq[Long], | |
viewerId: Long | |
): Future[Seq[Option[Seq[gfs.IntersectionValue]]]] | |
def providesEarlybirdSearchRepository( | |
client: eb.EarlybirdService.MethodPerEndpoint, | |
clientId: ClientId | |
): KeyValueRepository[EarlybirdQuery, Long, eb.ThriftSearchResult] | |
def lookup( | |
tweetIds: Seq[Long], | |
viewerId: Long | |
): Future[Seq[Option[eb.ThriftSearchResult]]] | |
def toRepository[K, V]( | |
hydrate: K => Future[V] | |
): KeyValueRepository[Seq[K], K, V] | |
def asRepository(keys: Seq[K]): Future[KeyValueResult[K, V]] | |
def toRepositoryBatch[K, V]( | |
hydrate: Seq[K] => Future[Seq[Option[V]]], | |
chunkSize: Int = DefaultRPCChunkSize | |
): KeyValueRepository[Seq[K], K, V] | |
def repository(keys: Seq[K]): Future[KeyValueResult[K, V]] = | |
batchRepositoryProcess(keys, hydrate(keys)) | |
KeyValueRepository.chunked(repository, ChunkingStrategy.equalSize(chunkSize)) | |
} | |
protected def toRepositoryBatchWithView[K, T, V]( | |
hydrate: (Seq[K], T) => Future[Seq[Option[V]]], | |
chunkSize: Int = DefaultRPCChunkSize | |
): KeyValueRepository[(Seq[K], T), K, V] | |
def repository(input: (Seq[K], T)): Future[KeyValueResult[K, V]] | |
def batchRepositoryProcess[K, V]( | |
keys: Seq[K], | |
f: Future[Seq[Option[V]]] | |
): Future[KeyValueResult[K, V]] | |
object CustomChunkingStrategy | |
def equalSizeWithView[K, T](maxSize: Int): ((Seq[K], T)) => Seq[(Seq[K], T)] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\TimelinesPersistenceStoreClientModule.scala | |
object TimelinesPersistenceStoreClientModule extends TwitterModule | |
def providesTimelinesPersistenceStoreClient( | |
injectedServiceIdentifier: ServiceIdentifier, | |
statsReceiver: StatsReceiver | |
): TimelineResponseBatchesClient[TimelineResponseV3] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\TweetyPieClientModule.scala | |
object TweetyPieClientModule | |
extends ThriftMethodBuilderClientModule[ | |
TweetService.ServicePerEndpoint, | |
TweetService.MethodPerEndpoint | |
] | |
with MtlsClient | |
def providesTweetypieStitchClient(tweetService: TweetService.MethodPerEndpoint): TweetyPie = | |
new TweetyPie(tweetService) | |
/** | |
* TweetyPie client id must be in the form of | |
def clientId(injector: Injector): ClientId | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\TweetypieStaticEntitiesCacheClientModule.scala | |
object TweetypieStaticEntitiesCacheClientModule extends TwitterModule | |
def providesTweetypieStaticEntitiesCache( | |
statsReceiver: StatsReceiver, | |
serviceIdentifier: ServiceIdentifier | |
): TtlCache[Long, tp.Tweet] | |
def mkCache( | |
finagleMemcache: FinagleMemcache, | |
statsReceiver: StatsReceiver | |
): TtlCache[Long, tp.Tweet] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\module\UserMetadataStoreModule.scala | |
object UserMetadataStoreModule extends TwitterModule | |
def providesUserLanguagesFeaturesStore( | |
@Named(UserMetadataManhattanEndpoint) UserMetadataManhattanKVEndpoint: ManhattanKVEndpoint, | |
statsReceiver: StatsReceiver | |
): ReadableStore[Long, Seq[scc.ThriftLanguage]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\GlobalParamConfigModule.scala | |
object GlobalParamConfigModule extends TwitterModule | |
def configure(): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\HomeGlobalParamConfig.scala | |
class HomeGlobalParamConfig @Inject() () extends GlobalParamConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\HomeGlobalParams.scala | |
object HomeGlobalParams | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\HomeMixerFlagName.scala | |
object HomeMixerFlagName | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\HomeMixerInjectionNames.scala | |
object HomeMixerInjectionNames | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\param\decider\DeciderKey.scala | |
object DeciderKey extends DeciderKeyEnum | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\HomeMixerProductModule.scala | |
object HomeMixerProductModule extends TwitterModule | |
def configure(): Unit | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\HomeProductPipelineRegistryConfig.scala | |
class HomeProductPipelineRegistryConfig @Inject() ( | |
injector: Injector, | |
productScope: ProductScope) | |
extends ProductPipelineRegistryConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingAdsCandidatePipelineBuilder.scala | |
class FollowingAdsCandidatePipelineBuilder @Inject() ( | |
adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, | |
adsCandidateSource: AdsProdThriftCandidateSource, | |
advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ | |
FollowingQuery, | |
AdsCandidate | |
]) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingEarlybirdCandidatePipelineConfig.scala | |
class FollowingEarlybirdCandidatePipelineConfig @Inject() ( | |
earlybirdCandidateSource: EarlybirdCandidateSource, | |
followingEarlybirdQueryTransformer: FollowingEarlybirdQueryTransformer) | |
extends CandidatePipelineConfig[ | |
FollowingQuery, | |
t.EarlybirdRequest, | |
t.ThriftSearchResult, | |
TweetCandidate | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingEarlybirdQueryTransformer.scala | |
class FollowingEarlybirdQueryTransformer @Inject() (clientId: ClientId) | |
extends CandidatePipelineQueryTransformer[FollowingQuery, t.EarlybirdRequest] | |
def transform(query: FollowingQuery): t.EarlybirdRequest | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingEarlybirdResponseFeatureTransformer.scala | |
object FollowingEarlybirdResponseFeatureTransformer | |
extends CandidateFeatureTransformer[t.ThriftSearchResult] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingMixerPipelineConfig.scala | |
class FollowingMixerPipelineConfig @Inject() ( | |
followingEarlybirdCandidatePipelineConfig: FollowingEarlybirdCandidatePipelineConfig, | |
conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ | |
FollowingQuery | |
], | |
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, | |
followingAdsCandidatePipelineBuilder: FollowingAdsCandidatePipelineBuilder, | |
followingWhoToFollowArmCandidatePipelineConfigBuilder: FollowingWhoToFollowArmCandidatePipelineConfigBuilder, | |
flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder, | |
editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, | |
newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery], | |
dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, | |
gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, | |
persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, | |
realGraphInNetworkSourceQueryHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, | |
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery], | |
sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, | |
tweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[FollowingQuery], | |
lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, | |
adsInjector: AdsInjector, | |
updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline], | |
publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, | |
publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, | |
updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, | |
truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, | |
homeTimelineServedEntriesSideEffect: HomeScribeServedEntriesSideEffect, | |
clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], | |
externalStrings: HomeMixerExternalStrings, | |
@ProductScoped stringCenterProvider: Provider[StringCenter], | |
urtTransportMarshaller: UrtTransportMarshaller) | |
extends MixerPipelineConfig[FollowingQuery, Timeline, urt.TimelineResponse] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingProductPipelineConfig.scala | |
class FollowingProductPipelineConfig @Inject() ( | |
followingMixerPipelineConfig: FollowingMixerPipelineConfig, | |
followingParamConfig: FollowingParamConfig) | |
extends ProductPipelineConfig[HomeMixerRequest, FollowingQuery, urt.TimelineResponse] | |
def pipelineQueryTransformer( | |
request: HomeMixerRequest, | |
params: Params | |
): FollowingQuery | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\FollowingWhoToFollowArmCandidatePipelineConfigBuilder.scala | |
class FollowingWhoToFollowArmCandidatePipelineConfigBuilder @Inject() ( | |
whoToFollowArmDependentCandidatePipelineConfigBuilder: WhoToFollowArmDependentCandidatePipelineConfigBuilder, | |
homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) | |
def build( | |
requiredNonEmptyPipelines: CandidateScope | |
): WhoToFollowArmDependentCandidatePipelineConfig[FollowingQuery] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\model\FollowingQuery.scala | |
class FollowingQuery( | |
override val params: Params, | |
override val clientContext: ClientContext, | |
override val pipelineCursor: Option[UrtOrderedCursor], | |
override val requestedMaxResults: Option[Int], | |
override val debugOptions: Option[DebugOptions], | |
override val features: Option[FeatureMap], | |
override val deviceContext: Option[DeviceContext], | |
override val seenTweetIds: Option[Seq[Long]], | |
override val dspClientContext: Option[dsp.DspClientContext]) | |
extends PipelineQuery | |
with HasPipelineCursor[UrtOrderedCursor] | |
with HasDeviceContext | |
with HasSeenTweetIds | |
with HasFlipInjectionParams | |
with HomeAdsQuery | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\model\HomeMixerExternalStrings.scala | |
class HomeMixerExternalStrings @Inject() ( | |
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\param\FollowingParam.scala | |
object FollowingParam | |
object ServerMaxResultsParam | |
extends FSBoundedParam[Int]( | |
name = "following_server_max_results", | |
default = 100, | |
min = 1, | |
max = 500 | |
) | |
object EnableWhoToFollowCandidatePipelineParam | |
extends FSParam[Boolean]( | |
name = "following_enable_who_to_follow", | |
default = true | |
) | |
object EnableAdsCandidatePipelineParam | |
extends FSParam[Boolean]( | |
name = "following_enable_ads", | |
default = true | |
) | |
object EnableFlipInjectionModuleCandidatePipelineParam | |
extends FSParam[Boolean]( | |
name = "following_enable_flip_inline_injection_module", | |
default = true | |
) | |
object FlipInlineInjectionModulePosition | |
extends FSBoundedParam[Int]( | |
name = "following_flip_inline_injection_module_position", | |
default = 0, | |
min = 0, | |
max = 1000 | |
) | |
object WhoToFollowPositionParam | |
extends FSBoundedParam[Int]( | |
name = "following_who_to_follow_position", | |
default = 5, | |
min = 0, | |
max = 99 | |
) | |
object WhoToFollowMinInjectionIntervalParam | |
extends FSBoundedParam[Duration]( | |
"following_who_to_follow_min_injection_interval_in_minutes", | |
default = 1800.minutes, | |
min = 0.minutes, | |
max = 6000.minutes) | |
with HasDurationConversion | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\following\param\FollowingParamConfig.scala | |
class FollowingParamConfig @Inject() () extends ProductParamConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouAdsCandidatePipelineBuilder.scala | |
class ForYouAdsCandidatePipelineBuilder @Inject() ( | |
adsCandidatePipelineConfigBuilder: AdsCandidatePipelineConfigBuilder, | |
adsCandidateSource: AdsProdThriftCandidateSource, | |
advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ | |
ForYouQuery, | |
AdsCandidate | |
]) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouConversationServiceCandidatePipelineConfig.scala | |
class ForYouConversationServiceCandidatePipelineConfig @Inject() ( | |
forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, | |
forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, | |
conversationServiceCandidateSource: ConversationServiceCandidateSource, | |
tweetypieFeatureHydrator: TweetypieFeatureHydrator, | |
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, | |
namesFeatureHydrator: NamesFeatureHydrator, | |
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) | |
extends DependentCandidatePipelineConfig[ | |
ForYouQuery, | |
ConversationServiceCandidateSourceRequest, | |
TweetWithConversationMetadata, | |
TweetCandidate | |
] | |
def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( | |
PreviouslyServedTweetsFilter, | |
PreviouslySeenTweetsFilter, | |
RetweetDeduplicationFilter, | |
FeatureFilter.fromFeature(FilterIdentifier("TweetypieHydrated"), IsHydratedFeature), | |
PredicateFeatureFilter.fromPredicate( | |
FilterIdentifier("QuotedTweetDropped"), | |
shouldKeepCandidate | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouProductPipelineConfig.scala | |
class ForYouProductPipelineConfig @Inject() ( | |
forYouTimelineScorerMixerPipelineConfig: ForYouTimelineScorerMixerPipelineConfig, | |
forYouScoredTweetsMixerPipelineConfig: ForYouScoredTweetsMixerPipelineConfig, | |
forYouParamConfig: ForYouParamConfig) | |
extends ProductPipelineConfig[HomeMixerRequest, ForYouQuery, urt.TimelineResponse] | |
def pipelineQueryTransformer( | |
request: HomeMixerRequest, | |
params: Params | |
): ForYouQuery | |
def pipelineSelector( | |
query: ForYouQuery | |
): ComponentIdentifier | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouScoredTweetsCandidatePipelineConfig.scala | |
class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( | |
scoredTweetsProductCandidateSource: ScoredTweetsProductCandidateSource, | |
tweetypieFeatureHydrator: TweetypieFeatureHydrator, | |
namesFeatureHydrator: NamesFeatureHydrator, | |
sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, | |
perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, | |
focalTweetFeatureHydrator: FocalTweetFeatureHydrator, | |
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, | |
homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) | |
extends CandidatePipelineConfig[ | |
ForYouQuery, | |
ForYouQuery, | |
ScoredTweetWithConversationMetadata, | |
TweetCandidate | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouScoredTweetsMixerPipelineConfig.scala | |
class ForYouScoredTweetsMixerPipelineConfig @Inject() ( | |
forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, | |
forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, | |
forYouAdsCandidatePipelineBuilder: ForYouAdsCandidatePipelineBuilder, | |
forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, | |
flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, | |
editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, | |
newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], | |
dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, | |
gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, | |
persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, | |
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], | |
feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, | |
timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, | |
adsInjector: AdsInjector, | |
servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, | |
servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, | |
updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, | |
truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, | |
homeScribeServedEntriesSideEffect: HomeScribeServedEntriesSideEffect, | |
servedStatsSideEffect: ServedStatsSideEffect, | |
clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], | |
externalStrings: HomeMixerExternalStrings, | |
@ProductScoped stringCenterProvider: Provider[StringCenter], | |
urtTransportMarshaller: UrtTransportMarshaller) | |
extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouScoredTweetsResponseFeatureTransformer.scala | |
object ForYouScoredTweetsResponseFeatureTransformer | |
extends CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] | |
def transform(input: ScoredTweetWithConversationMetadata): FeatureMap = | |
FeatureMapBuilder() | |
.add(AncestorsFeature, input.ancestors.getOrElse(Seq.empty)) | |
.add(AuthorIdFeature, Some(input.authorId)) | |
.add(ConversationModuleIdFeature, input.conversationId) | |
.add(ConversationModuleFocalTweetIdFeature, input.conversationFocalTweetId) | |
.add(DirectedAtUserIdFeature, input.directedAtUserId) | |
.add(FavoritedByUserIdsFeature, input.favoritedByUserIds.getOrElse(Seq.empty)) | |
.add(FollowedByUserIdsFeature, input.followedByUserIds.getOrElse(Seq.empty)) | |
.add(InNetworkFeature, input.inNetwork.getOrElse(false)) | |
.add(InReplyToTweetIdFeature, input.inReplyToTweetId) | |
.add(InReplyToUserIdFeature, input.inReplyToUserId) | |
.add(IsReadFromCacheFeature, input.isReadFromCache.getOrElse(false)) | |
.add(IsRetweetFeature, input.sourceTweetId.isDefined) | |
.add(QuotedTweetIdFeature, input.quotedTweetId) | |
.add(QuotedUserIdFeature, input.quotedUserId) | |
.add(ScoreFeature, input.score) | |
.add(SourceTweetIdFeature, input.sourceTweetId) | |
.add(SourceUserIdFeature, input.sourceUserId) | |
.add(StreamToKafkaFeature, input.streamToKafka.getOrElse(false)) | |
.add(SuggestTypeFeature, input.suggestType) | |
.add( | |
TopicContextFunctionalityTypeFeature, | |
input.topicFunctionalityType.collect | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouTimelineScorerCandidatePipelineConfig.scala | |
class ForYouTimelineScorerCandidatePipelineConfig @Inject() ( | |
timelineScorerCandidateSource: TimelineScorerCandidateSource, | |
deviceContextMarshaller: DeviceContextMarshaller, | |
tweetypieFeatureHydrator: TweetypieFeatureHydrator, | |
sgsFeatureHydrator: SocialGraphServiceFeatureHydrator, | |
sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, | |
perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, | |
namesFeatureHydrator: NamesFeatureHydrator, | |
focalTweetFeatureHydrator: FocalTweetFeatureHydrator, | |
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, | |
homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) | |
extends CandidatePipelineConfig[ | |
ForYouQuery, | |
t.ScoredTweetsRequest, | |
ScoredTweetCandidateWithFocalTweet, | |
TweetCandidate | |
] | |
def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( | |
RetweetDeduplicationFilter, | |
FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), | |
PredicateFeatureFilter.fromPredicate( | |
FilterIdentifier(QuotedTweetDroppedFilterId), | |
shouldKeepCandidate | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouTimelineScorerMixerPipelineConfig.scala | |
class ForYouTimelineScorerMixerPipelineConfig @Inject() ( | |
forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, | |
forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, | |
forYouAdsCandidatePipelineBuilder: ForYouAdsCandidatePipelineBuilder, | |
forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, | |
flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, | |
editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, | |
newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], | |
dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, | |
gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, | |
manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ForYouQuery], | |
memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, | |
persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, | |
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], | |
timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, | |
lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, | |
feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, | |
adsInjector: AdsInjector, | |
servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, | |
servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, | |
updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ForYouQuery, Timeline], | |
publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, | |
publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, | |
updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, | |
truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, | |
homeScribeServedEntriesSideEffect: HomeScribeServedEntriesSideEffect, | |
servedStatsSideEffect: ServedStatsSideEffect, | |
clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], | |
externalStrings: HomeMixerExternalStrings, | |
@ProductScoped stringCenterProvider: Provider[StringCenter], | |
urtTransportMarshaller: UrtTransportMarshaller) | |
extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] | |
def fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( | |
requestQueryFeatureHydrator, | |
persistenceStoreQueryFeatureHydrator, | |
timelineServiceTweetsQueryFeatureHydrator, | |
feedbackHistoryQueryFeatureHydrator, | |
AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), | |
AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), | |
AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), | |
AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), | |
AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) | |
) | |
private val forYouAdsCandidatePipelineConfig = forYouAdsCandidatePipelineBuilder.build() | |
private val forYouWhoToFollowCandidatePipelineConfig = | |
forYouWhoToFollowCandidatePipelineConfigBuilder.build() | |
private val flipPromptCandidatePipelineConfig = | |
flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( | |
supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) | |
) | |
override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( | |
forYouTimelineScorerCandidatePipelineConfig, | |
forYouAdsCandidatePipelineConfig, | |
forYouWhoToFollowCandidatePipelineConfig, | |
flipPromptCandidatePipelineConfig | |
) | |
override val dependentCandidatePipelines: Seq[ | |
DependentCandidatePipelineConfig[ForYouQuery, _, _, _] | |
] = Seq( | |
forYouConversationServiceCandidatePipelineConfig, | |
editedTweetsCandidatePipelineConfig, | |
newTweetsPillCandidatePipelineConfig | |
) | |
override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( | |
forYouTimelineScorerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, | |
forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, | |
forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, | |
flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, | |
editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, | |
newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, | |
) | |
override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( | |
UpdateSortCandidates( | |
ordering = CandidatesUtil.reverseChronTweetsOrdering, | |
candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier | |
), | |
UpdateSortCandidates( | |
ordering = CandidatesUtil.scoreOrdering, | |
candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier | |
), | |
UpdateSortModuleItemCandidates( | |
candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier, | |
ordering = CandidatesUtil.conversationModuleTweetsOrdering | |
), | |
DebunchCandidates( | |
pipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier), | |
mustDebunch | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouTimelineScorerResponseFeatureTransformer.scala | |
object ForYouTimelineScorerResponseFeatureTransformer | |
extends CandidateFeatureTransformer[ScoredTweetCandidateWithFocalTweet] | |
def getLanguageISOFormatByValue(languageCodeValue: Int): String = | |
ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguage.findByValue(languageCodeValue)) | |
override def transform( | |
candidateWithFocalTweet: ScoredTweetCandidateWithFocalTweet | |
): FeatureMap | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\ForYouWhoToFollowCandidatePipelineConfigBuilder.scala | |
class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( | |
whoToFollowCandidatePipelineConfigBuilder: WhoToFollowCandidatePipelineConfigBuilder, | |
homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) | |
def build(): WhoToFollowCandidatePipelineConfig[ForYouQuery] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\candidate_source\ScoredTweetsProductCandidateSource.scala | |
class ScoredTweetWithConversationMetadata( | |
tweetId: Long, | |
authorId: Long, | |
score: Option[Double] = None, | |
suggestType: Option[st.SuggestType] = None, | |
sourceTweetId: Option[Long] = None, | |
sourceUserId: Option[Long] = None, | |
quotedTweetId: Option[Long] = None, | |
quotedUserId: Option[Long] = None, | |
inReplyToTweetId: Option[Long] = None, | |
inReplyToUserId: Option[Long] = None, | |
directedAtUserId: Option[Long] = None, | |
inNetwork: Option[Boolean] = None, | |
favoritedByUserIds: Option[Seq[Long]] = None, | |
followedByUserIds: Option[Seq[Long]] = None, | |
ancestors: Option[Seq[ta.TweetAncestor]] = None, | |
topicId: Option[Long] = None, | |
topicFunctionalityType: Option[tl.TopicContextFunctionalityType] = None, | |
conversationId: Option[Long] = None, | |
conversationFocalTweetId: Option[Long] = None, | |
isReadFromCache: Option[Boolean] = None, | |
streamToKafka: Option[Boolean] = None) | |
@Singleton | |
class ScoredTweetsProductCandidateSource @Inject() ( | |
override val productPipelineRegistry: Provider[ProductPipelineRegistry], | |
override val paramsBuilder: Provider[ParamsBuilder]) | |
extends ProductPipelineCandidateSource[ | |
ForYouQuery, | |
HomeMixerRequest, | |
t.ScoredTweetsResponse, | |
ScoredTweetWithConversationMetadata | |
] | |
def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest | |
def productPipelineResultTransformer( | |
productPipelineResult: t.ScoredTweetsResponse | |
): Seq[ScoredTweetWithConversationMetadata] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\model\ForYouQuery.scala | |
class ForYouQuery( | |
override val params: Params, | |
override val clientContext: ClientContext, | |
override val pipelineCursor: Option[UrtOrderedCursor], | |
override val requestedMaxResults: Option[Int], | |
override val debugOptions: Option[DebugOptions], | |
override val features: Option[FeatureMap], | |
override val deviceContext: Option[DeviceContext], | |
override val seenTweetIds: Option[Seq[Long]], | |
override val dspClientContext: Option[dsp.DspClientContext]) | |
extends PipelineQuery | |
with HasPipelineCursor[UrtOrderedCursor] | |
with HasDeviceContext | |
with HasSeenTweetIds | |
with HasFlipInjectionParams | |
with HomeAdsQuery | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\param\ForYouParam.scala | |
object ForYouParam | |
object EnableTimelineScorerCandidatePipelineParam | |
extends FSParam[Boolean]( | |
name = "for_you_enable_timeline_scorer_candidate_pipeline", | |
default = true | |
) | |
object EnableScoredTweetsCandidatePipelineParam | |
extends BooleanDeciderParam(DeciderKey.EnableForYouScoredTweetsCandidatePipeline) | |
object EnableWhoToFollowCandidatePipelineParam | |
extends FSParam[Boolean]( | |
name = "for_you_enable_who_to_follow", | |
default = true | |
) | |
object EnableScoredTweetsMixerPipelineParam | |
extends FSParam[Boolean]( | |
name = "for_you_enable_scored_tweets_mixer_pipeline", | |
default = true | |
) | |
object ServerMaxResultsParam | |
extends FSBoundedParam[Int]( | |
name = "for_you_server_max_results", | |
default = 35, | |
min = 1, | |
max = 500 | |
) | |
object TimelineServiceMaxResultsParam | |
extends FSBoundedParam[Int]( | |
name = "for_you_timeline_service_max_results", | |
default = 800, | |
min = 1, | |
max = 800 | |
) | |
object AdsNumOrganicItemsParam | |
extends FSBoundedParam[Int]( | |
name = "for_you_ads_num_organic_items", | |
default = 35, | |
min = 1, | |
max = 100 | |
) | |
object WhoToFollowPositionParam | |
extends FSBoundedParam[Int]( | |
name = "for_you_who_to_follow_position", | |
default = 5, | |
min = 0, | |
max = 99 | |
) | |
object WhoToFollowMinInjectionIntervalParam | |
extends FSBoundedParam[Duration]( | |
"for_you_who_to_follow_min_injection_interval_in_minutes", | |
default = 1800.minutes, | |
min = 0.minutes, | |
max = 6000.minutes) | |
with HasDurationConversion | |
object WhoToFollowDisplayTypeIdParam | |
extends FSEnumParam[WhoToFollowModuleDisplayType.type]( | |
name = "for_you_enable_who_to_follow_display_type_id", | |
default = WhoToFollowModuleDisplayType.Vertical, | |
enum = WhoToFollowModuleDisplayType | |
) | |
object EnableFlipInjectionModuleCandidatePipelineParam | |
extends FSParam[Boolean]( | |
name = "for_you_enable_flip_inline_injection_module", | |
default = true | |
) | |
object FlipInlineInjectionModulePosition | |
extends FSBoundedParam[Int]( | |
name = "for_you_flip_inline_injection_module_position", | |
default = 0, | |
min = 0, | |
max = 1000 | |
) | |
object ClearCacheOnPtr | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\for_you\param\ForYouParamConfig.scala | |
class ForYouParamConfig @Inject() () extends ProductParamConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\ListMemberBasedUsersCandidatePipelineConfig.scala | |
class ListMemberBasedUsersCandidatePipelineConfig @Inject() ( | |
similarityBasedUsersCandidateSource: SimilarityBasedUsersCandidateSource, | |
gizmoduckUserFeatureHydrator: GizmoduckUserFeatureHydrator, | |
isListMemberFeatureHydrator: IsListMemberFeatureHydrator) | |
extends CandidatePipelineConfig[ | |
ListRecommendedUsersQuery, | |
Seq[Long], | |
t.Candidate, | |
UserCandidate | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\ListMemberBasedUsersResponseFeatureTransfromer.scala | |
object ListMemberBasedUsersResponseFeatureTransfromer | |
extends CandidateFeatureTransformer[t.Candidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\ListRecommendedUsersMixerPipelineConfig.scala | |
class ListRecommendedUsersMixerPipelineConfig @Inject() ( | |
listMemberBasedUsersCandidatePipelineConfig: ListMemberBasedUsersCandidatePipelineConfig, | |
viewerIsListOwnerGate: ViewerIsListOwnerGate, | |
listMembersQueryFeatureHydrator: ListMembersQueryFeatureHydrator, | |
urtTransportMarshaller: UrtTransportMarshaller) | |
extends MixerPipelineConfig[ListRecommendedUsersQuery, Timeline, urt.TimelineResponse] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\ListRecommendedUsersProductPipelineConfig.scala | |
class ListRecommendedUsersProductPipelineConfig @Inject() ( | |
listRecommendedUsersMixerPipelineConfig: ListRecommendedUsersMixerPipelineConfig, | |
listRecommendedUsersParamConfig: ListRecommendedUsersParamConfig) | |
extends ProductPipelineConfig[ | |
HomeMixerRequest, | |
ListRecommendedUsersQuery, | |
urt.TimelineResponse | |
] | |
def pipelineQueryTransformer( | |
request: HomeMixerRequest, | |
params: Params | |
): ListRecommendedUsersQuery | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\feature_hydrator\GizmoduckUserFeatureHydrator.scala | |
class GizmoduckUserFeatureHydrator @Inject() (gizmoduck: Gizmoduck) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, UserCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[UserCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\feature_hydrator\IsListMemberFeatureHydrator.scala | |
class IsListMemberFeatureHydrator @Inject() (socialGraph: SocialGraph) | |
extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate] | |
def apply( | |
query: PipelineQuery with HasListId, | |
candidates: Seq[CandidateWithFeatures[UserCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\filter\DropMaxCandidatesByScoreFilter.scala | |
object DropMaxCandidatesByScoreFilter extends Filter[PipelineQuery, UserCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[UserCandidate]] | |
): Stitch[FilterResult[UserCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\filter\PreviouslyServedUsersFilter.scala | |
object PreviouslyServedUsersFilter extends Filter[ListRecommendedUsersQuery, UserCandidate] | |
def apply( | |
query: ListRecommendedUsersQuery, | |
candidates: Seq[CandidateWithFeatures[UserCandidate]] | |
): Stitch[FilterResult[UserCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\model\ListFeatures.scala | |
object ListFeatures | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\model\ListRecommendedUsersQuery.scala | |
class ListRecommendedUsersQuery( | |
override val listId: Long, | |
override val params: Params, | |
override val clientContext: ClientContext, | |
override val pipelineCursor: Option[UrtUnorderedExcludeIdsCursor], | |
override val requestedMaxResults: Option[Int], | |
override val debugOptions: Option[DebugOptions], | |
override val features: Option[FeatureMap], | |
selectedUserIds: Option[Seq[Long]], | |
excludedUserIds: Option[Seq[Long]]) | |
extends PipelineQuery | |
with HasPipelineCursor[UrtUnorderedExcludeIdsCursor] | |
with HasListId | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\param\ListRecommendedUsersParam.scala | |
object ListRecommendedUsersParam | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_recommended_users\param\ListRecommendedUsersParamConfig.scala | |
class ListRecommendedUsersParamConfig @Inject() () extends ProductParamConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\ListTweetsAdsCandidatePipelineBuilder.scala | |
class ListTweetsAdsCandidatePipelineBuilder @Inject() ( | |
adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, | |
adsCandidateSource: AdsProdThriftCandidateSource, | |
advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ | |
ListTweetsQuery, | |
AdsCandidate | |
]) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\ListTweetsMixerPipelineConfig.scala | |
class ListTweetsMixerPipelineConfig @Inject() ( | |
listTweetsTimelineServiceCandidatePipelineConfig: ListTweetsTimelineServiceCandidatePipelineConfig, | |
conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ | |
ListTweetsQuery | |
], | |
listTweetsAdsCandidatePipelineBuilder: ListTweetsAdsCandidatePipelineBuilder, | |
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ListTweetsQuery], | |
adsInjector: AdsInjector, | |
clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], | |
urtTransportMarshaller: UrtTransportMarshaller) | |
extends MixerPipelineConfig[ListTweetsQuery, Timeline, urt.TimelineResponse] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\ListTweetsProductPipelineConfig.scala | |
class ListTweetsProductPipelineConfig @Inject() ( | |
listTweetsMixerPipelineConfig: ListTweetsMixerPipelineConfig, | |
listTweetsParamConfig: ListTweetsParamConfig) | |
extends ProductPipelineConfig[HomeMixerRequest, ListTweetsQuery, urt.TimelineResponse] | |
def pipelineQueryTransformer( | |
request: HomeMixerRequest, | |
params: Params | |
): ListTweetsQuery | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\ListTweetsTimelineServiceCandidatePipelineConfig.scala | |
class ListTweetsTimelineServiceCandidatePipelineConfig @Inject() ( | |
timelineServiceTweetCandidateSource: TimelineServiceTweetCandidateSource) | |
extends CandidatePipelineConfig[ListTweetsQuery, t.TimelineQuery, t.Tweet, TweetCandidate] | |
def candidateSource: BaseCandidateSource[t.TimelineQuery, t.Tweet] = | |
timelineServiceTweetCandidateSource | |
override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\model\ListTweetsQuery.scala | |
class ListTweetsQuery( | |
override val params: Params, | |
override val clientContext: ClientContext, | |
override val pipelineCursor: Option[UrtOrderedCursor], | |
override val requestedMaxResults: Option[Int], | |
override val debugOptions: Option[DebugOptions], | |
override val features: Option[FeatureMap], | |
override val listId: Long, | |
override val deviceContext: Option[DeviceContext], | |
override val dspClientContext: Option[dsp.DspClientContext]) | |
extends PipelineQuery | |
with HasPipelineCursor[UrtOrderedCursor] | |
with HasListId | |
with HomeAdsQuery | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\param\ListTweetsParam.scala | |
object ListTweetsParam | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\list_tweets\param\ListTweetsParamConfig.scala | |
class ListTweetsParamConfig @Inject() () extends ProductParamConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\ScoredTweetsProductPipelineConfig.scala | |
class ScoredTweetsProductPipelineConfig @Inject() ( | |
scoredTweetsRecommendationPipelineConfig: ScoredTweetsRecommendationPipelineConfig, | |
scoredTweetsParamConfig: ScoredTweetsParamConfig) | |
extends ProductPipelineConfig[HomeMixerRequest, ScoredTweetsQuery, t.ScoredTweets] | |
def pipelineQueryTransformer( | |
request: HomeMixerRequest, | |
params: Params | |
): ScoredTweetsQuery | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\ScoredTweetsRecommendationPipelineConfig.scala | |
class ScoredTweetsRecommendationPipelineConfig @Inject() ( | |
scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, | |
scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, | |
scoredTweetsCrMixerCandidatePipelineConfig: ScoredTweetsCrMixerCandidatePipelineConfig, | |
scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, | |
cachedScoredTweetsCandidatePipelineConfig: CachedScoredTweetsCandidatePipelineConfig, | |
requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ScoredTweetsQuery], | |
lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, | |
realTimeInteractionGraphUserVertexQueryFeatureHydrator: RealTimeInteractionGraphUserVertexQueryFeatureHydrator, | |
userStateQueryFeatureHydrator: UserStateQueryFeatureHydrator, | |
userEngagementRealTimeAggregatesFeatureHydrator: UserEngagementRealTimeAggregatesFeatureHydrator, | |
twhinUserEngagementQueryFeatureHydrator: TwhinUserEngagementQueryFeatureHydrator, | |
twhinUserFollowQueryFeatureHydrator: TwhinUserFollowQueryFeatureHydrator, | |
cachedScoredTweetsQueryFeatureHydrator: CachedScoredTweetsQueryFeatureHydrator, | |
scoredTweetsScoringPipelineConfig: ScoredTweetsScoringPipelineConfig, | |
scoredTweetsWeightedScoresSumScoringPipelineConfig: ScoredTweetsWeightedScoresSumScoringPipelineConfig, | |
manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ | |
ScoredTweetsQuery | |
], | |
memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, | |
publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, | |
publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, | |
realGraphInNetworkScoresQueryFeatureHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, | |
realGraphQueryFeatureHydrator: RealGraphQueryFeatureHydrator, | |
userLanguagesFeatureHydrator: UserLanguagesFeatureHydrator, | |
partAAggregateQueryFeatureHydrator: PartAAggregateQueryFeatureHydrator, | |
partBAggregateQueryFeatureHydrator: PartBAggregateQueryFeatureHydrator, | |
cachedScoredTweetsSideEffect: CachedScoredTweetsSideEffect, | |
scribeServedCommonFeaturesAndCandidateFeaturesSideEffect: ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect, | |
updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ | |
ScoredTweetsQuery, | |
ScoredTweetsResponse | |
], | |
@Flag(TargetFetchLatency) targetFetchLatency: Duration, | |
@Flag(TargetScoringLatency) targetScoringLatency: Duration) | |
extends RecommendationPipelineConfig[ | |
ScoredTweetsQuery, | |
TweetCandidate, | |
ScoredTweetsResponse, | |
t.ScoredTweetsResponse | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\CachedScoredTweetsCandidatePipelineConfig.scala | |
class CachedScoredTweetsCandidatePipelineConfig @Inject() ( | |
cachedScoredTweetsCandidateSource: CachedScoredTweetsCandidateSource) | |
extends CandidatePipelineConfig[ | |
ScoredTweetsQuery, | |
ScoredTweetsQuery, | |
hmt.CachedScoredTweet, | |
TweetCandidate | |
] | |
object CachedScoredTweetsCandidatePipelineConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\ScoredTweetsCrMixerCandidatePipelineConfig.scala | |
class ScoredTweetsCrMixerCandidatePipelineConfig @Inject() ( | |
crMixerTweetRecommendationsCandidateSource: CrMixerTweetRecommendationsCandidateSource, | |
tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) | |
extends CandidatePipelineConfig[ | |
ScoredTweetsQuery, | |
t.CrMixerTweetRequest, | |
t.TweetRecommendation, | |
TweetCandidate | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\ScoredTweetsFrsCandidatePipelineConfig.scala | |
class ScoredTweetsFrsCandidatePipelineConfig @Inject() ( | |
timelineRankerRecapCandidateSource: TimelineRankerRecapCandidateSource, | |
frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator) | |
extends CandidatePipelineConfig[ | |
ScoredTweetsQuery, | |
tlr.RecapQuery, | |
tlr.CandidateTweet, | |
TweetCandidate | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\ScoredTweetsInNetworkCandidatePipelineConfig.scala | |
class ScoredTweetsInNetworkCandidatePipelineConfig @Inject() ( | |
timelineRankerInNetworkCandidateSource: TimelineRankerInNetworkCandidateSource, | |
replyFeatureHydrator: ReplyFeatureHydrator) | |
extends CandidatePipelineConfig[ | |
ScoredTweetsQuery, | |
t.RecapQuery, | |
t.CandidateTweet, | |
TweetCandidate | |
] | |
def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( | |
RetweetSourceTweetRemovingFilter | |
) | |
override val postFilterFeatureHydration: Seq[ | |
BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] | |
] = Seq(replyFeatureHydrator) | |
override val featuresFromCandidateSourceTransformers: Seq[ | |
CandidateFeatureTransformer[t.CandidateTweet] | |
] = Seq(ScoredTweetsInNetworkResponseFeatureTransformer) | |
override val resultTransformer: CandidatePipelineResultsTransformer[ | |
t.CandidateTweet, | |
TweetCandidate | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_pipeline\ScoredTweetsUtegCandidatePipelineConfig.scala | |
class ScoredTweetsUtegCandidatePipelineConfig @Inject() ( | |
timelineRankerUtegCandidateSource: TimelineRankerUtegCandidateSource) | |
extends CandidatePipelineConfig[ | |
ScoredTweetsQuery, | |
t.UtegLikedByTweetsQuery, | |
t.CandidateTweet, | |
TweetCandidate | |
] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\candidate_source\CachedScoredTweetsCandidateSource.scala | |
class CachedScoredTweetsCandidateSource @Inject() () | |
extends CandidateSource[PipelineQuery, hmt.CachedScoredTweet] | |
def apply(request: PipelineQuery): Stitch[Seq[hmt.CachedScoredTweet]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\feature_hydrator\CachedScoredTweetsQueryFeatureHydrator.scala | |
class CachedScoredTweetsQueryFeatureHydrator @Inject() ( | |
scoredTweetsCache: TtlCache[UserId, hmt.CachedScoredTweets]) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\marshaller\ScoredTweetsResponseDomainMarshaller.scala | |
object ScoredTweetsResponseDomainMarshaller | |
extends DomainMarshaller[ScoredTweetsQuery, ScoredTweetsResponse] | |
def apply( | |
query: ScoredTweetsQuery, | |
selections: Seq[CandidateWithDetails] | |
): ScoredTweetsResponse = ScoredTweetsResponse( | |
scoredTweets = selections.collect | |
def mkScoredTweet(tweetId: Long, features: FeatureMap): ScoredTweet | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\marshaller\ScoredTweetsResponseTransportMarshaller.scala | |
object ScoredTweetsResponseTransportMarshaller | |
extends TransportMarshaller[ScoredTweetsResponse, t.ScoredTweetsResponse] | |
def apply(input: ScoredTweetsResponse): t.ScoredTweetsResponse | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\model\ScoredTweetsQuery.scala | |
class ScoredTweetsQuery( | |
override val params: Params, | |
override val clientContext: ClientContext, | |
override val pipelineCursor: Option[UrtOrderedCursor], | |
override val requestedMaxResults: Option[Int], | |
override val debugOptions: Option[DebugOptions], | |
override val features: Option[FeatureMap], | |
override val deviceContext: Option[DeviceContext], | |
override val seenTweetIds: Option[Seq[Long]], | |
override val qualityFactorStatus: Option[QualityFactorStatus]) | |
extends PipelineQuery | |
with HasPipelineCursor[UrtOrderedCursor] | |
with HasDeviceContext | |
with HasSeenTweetIds | |
with HasQualityFactorStatus | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\param\ScoredTweetsParam.scala | |
object ScoredTweetsParam | |
object CrMixerSource | |
object EnableCandidatePipelineParam | |
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsCrMixerCandidatePipeline) | |
} | |
object FrsTweetSource | |
object EnableCandidatePipelineParam | |
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsFrsCandidatePipeline) | |
} | |
object InNetworkSource | |
object EnableCandidatePipelineParam | |
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsInNetworkCandidatePipeline) | |
} | |
object QualityFactor | |
object MaxTweetsToScoreParam | |
extends FSBoundedParam[Int]( | |
name = "scored_tweets_quality_factor_max_tweets_to_score", | |
default = 1100, | |
min = 0, | |
max = 10000 | |
) | |
object CrMixerMaxTweetsToScoreParam | |
extends FSBoundedParam[Int]( | |
name = "scored_tweets_quality_factor_cr_mixer_max_tweets_to_score", | |
default = 500, | |
min = 0, | |
max = 10000 | |
) | |
} | |
object ServerMaxResultsParam | |
extends FSBoundedParam[Int]( | |
name = "scored_tweets_server_max_results", | |
default = 120, | |
min = 1, | |
max = 500 | |
) | |
object UtegSource | |
object EnableCandidatePipelineParam | |
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsUtegCandidatePipeline) | |
} | |
object CachedScoredTweets | |
object TTLParam | |
extends FSBoundedParam[Duration]( | |
name = "scored_tweets_cached_scored_tweets_ttl_minutes", | |
default = 3.minutes, | |
min = 0.minute, | |
max = 60.minutes | |
) | |
with HasDurationConversion | |
object MinCachedTweetsParam | |
extends FSBoundedParam[Int]( | |
name = "scored_tweets_cached_scored_tweets_min_cached_tweets", | |
default = 30, | |
min = 0, | |
max = 1000 | |
) | |
} | |
object Scoring | |
object HomeModelParam | |
extends FSParam[String](name = "scored_tweets_home_model", default = "Home") | |
object ModelWeights | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\param\ScoredTweetsParamConfig.scala | |
class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_feature_hydrator\FrsSeedUsersQueryFeatureHydrator.scala | |
object FrsSeedUserIdsFeature extends Feature[TweetCandidate, Option[Seq[Long]]] | |
object FrsUserToFollowedByUserIdsFeature extends Feature[TweetCandidate, Map[Long, Seq[Long]]] | |
@Singleton | |
case class FrsSeedUsersQueryFeatureHydrator @Inject() ( | |
userFollowRecommendationsCandidateSource: UserFollowRecommendationsCandidateSource) | |
extends QueryFeatureHydrator[ScoredTweetsQuery] | |
def features: Set[Feature[_, _]] = Set( | |
FrsSeedUserIdsFeature, | |
FrsUserToFollowedByUserIdsFeature | |
) | |
override def hydrate(query: ScoredTweetsQuery): Stitch[FeatureMap] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_transformer\TimelineRankerFrsQueryTransformer.scala | |
object TimelineRankerFrsQueryTransformer | |
class TimelineRankerFrsQueryTransformer[ | |
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext | |
]( | |
override val candidatePipelineIdentifier: CandidatePipelineIdentifier, | |
override val maxTweetsToFetch: Int = MaxTweetsToFetch, | |
override val sinceDuration: Duration = SinceDuration) | |
extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] | |
with TimelineRankerQueryTransformer[Query] | |
def seedAuthorIds(query: Query): Option[Seq[Long]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_transformer\TimelineRankerInNetworkQueryTransformer.scala | |
object TimelineRankerInNetworkQueryTransformer | |
class TimelineRankerInNetworkQueryTransformer[ | |
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext | |
]( | |
override val candidatePipelineIdentifier: CandidatePipelineIdentifier, | |
override val maxTweetsToFetch: Int = MaxTweetsToFetch, | |
override val sinceDuration: Duration = SinceDuration) | |
extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] | |
with TimelineRankerQueryTransformer[Query] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_transformer\TimelineRankerQueryTransformer.scala | |
object TimelineRankerQueryTransformer | |
trait TimelineRankerQueryTransformer[ | |
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] | |
def maxTweetsToFetch: Int | |
def sinceDuration: Duration | |
def options: TweetKindOption.ValueSet = TweetKindOption.Default | |
def candidateTweetSourceId: CandidateTweetSourceId.Value | |
def skipVeryRecentTweets: Boolean | |
def utegLikedByTweetsOptions(query: Query): Option[tlr.UtegLikedByTweetsOptions] = None | |
def seedAuthorIds(query: Query): Option[Seq[Long]] = None | |
def candidatePipelineIdentifier: CandidatePipelineIdentifier | |
def earlybirdModels: Seq[EarlybirdScoringModelConfig] = | |
EarlybirdScoringModels.fromEnum(EarlybirdScoringModelsId.UnifiedEngagementProd) | |
def tensorflowModel: Option[String] = None | |
def buildTimelineRankerQuery(query: Query): tlr.RecapQuery | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\query_transformer\TimelineRankerUtegQueryTransformer.scala | |
object TimelineRankerUtegQueryTransformer | |
def utegEarlybirdModels = | |
EarlybirdScoringModels.fromEnum(EarlybirdScoringModelsId.UnifiedEngagementRectweet) | |
} | |
case class TimelineRankerUtegQueryTransformer[ | |
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext | |
]( | |
override val candidatePipelineIdentifier: CandidatePipelineIdentifier, | |
override val maxTweetsToFetch: Int = MaxTweetsToFetch, | |
override val sinceDuration: Duration = SinceDuration) | |
extends CandidatePipelineQueryTransformer[Query, t.UtegLikedByTweetsQuery] | |
with TimelineRankerQueryTransformer[Query] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\CachedScoredTweetsResponseFeatureTransformer.scala | |
object CachedScoredTweetsResponseFeatureTransformer | |
extends CandidateFeatureTransformer[hmt.CachedScoredTweet] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\ScoredTweetsCrMixerResponseFeatureTransformer.scala | |
object ScoredTweetsCrMixerResponseFeatureTransformer | |
extends CandidateFeatureTransformer[crm.TweetRecommendation] | |
def transform(candidate: crm.TweetRecommendation): FeatureMap | |
def CrMixerMetricTagToTspMetricTag( | |
crMixerMetricTag: crm.MetricTag | |
): Option[tsp.MetricTag] = crMixerMetricTag match | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\ScoredTweetsFrsResponseFeatureTransformer.scala | |
object ScoredTweetsFrsResponseFeatureTransformer | |
extends CandidateFeatureTransformer[tlr.CandidateTweet] | |
def transform(candidate: tlr.CandidateTweet): FeatureMap | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\ScoredTweetsInNetworkResponseFeatureTransformer.scala | |
object ScoredTweetsInNetworkResponseFeatureTransformer | |
extends CandidateFeatureTransformer[tlr.CandidateTweet] | |
def transform(candidate: tlr.CandidateTweet): FeatureMap | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\ScoredTweetsUtegResponseFeatureTransformer.scala | |
object ScoredTweetsUtegResponseFeatureTransformer | |
extends CandidateFeatureTransformer[tlr.CandidateTweet] | |
def transform(candidate: tlr.CandidateTweet): FeatureMap | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\response_transformer\TimelineRankerResponseTransformer.scala | |
object TimelineRankerResponseTransformer | |
def transform(candidate: tlr.CandidateTweet): FeatureMap | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scorer\DiversityDiscountProvider.scala | |
trait DiversityDiscountProvider | |
def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] | |
/** | |
* Compute the discounted score for the position | |
* @param score the previous score for the candidate | |
* @param position zero-based position for the candidate for the given entity | |
* @return the discounted score for the candidate | |
*/ | |
def discount(score: Double, position: Int): Double | |
} | |
object AuthorDiversityDiscountProvider extends DiversityDiscountProvider | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scorer\DiversityScorer.scala | |
class DiversityScorer(diversityDiscountProvider: DiversityDiscountProvider) | |
extends Scorer[ScoredTweetsQuery, TweetCandidate] | |
def apply( | |
query: ScoredTweetsQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scorer\HomeNaviModelDataRecordScorer.scala | |
object CommonFeaturesDataRecordFeature | |
extends DataRecordInAFeature[PipelineQuery] | |
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
object CandidateFeaturesDataRecordFeature | |
extends DataRecordInAFeature[TweetCandidate] | |
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] | |
def defaultValue: DataRecord = new DataRecord() | |
} | |
case class HomeNaviModelDataRecordScorer[ | |
Query <: PipelineQuery, | |
Candidate <: UniversalNoun[Any], | |
CandidateFeatures <: BaseDataRecordFeature[Candidate, _], | |
ResultFeatures <: BaseDataRecordFeature[Candidate, _] | |
]( | |
override val identifier: ScorerIdentifier, | |
modelClient: PredictionServiceGRPCClient, | |
candidateFeatures: FeaturesScope[CandidateFeatures], | |
resultFeatures: Set[ResultFeatures], | |
statsReceiver: StatsReceiver) | |
extends Scorer[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[Seq[FeatureMap]] | |
object HomeNaviModelDataRecordScorer | |
trait PredictedScoreFeature | |
extends DataRecordOptionalFeature[TweetCandidate, Double] | |
with DoubleDataRecordCompatible | |
def statName: String | |
def modelWeightParam: FSBoundedParam[Double] | |
} | |
object PredictedFavoriteScoreFeature extends PredictedScoreFeature | |
object PredictedReplyScoreFeature extends PredictedScoreFeature | |
object PredictedRetweetScoreFeature extends PredictedScoreFeature | |
object PredictedReplyEngagedByAuthorScoreFeature extends PredictedScoreFeature | |
object PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PredictedScoreFeature | |
object PredictedGoodClickConvoDescUamGt2ScoreFeature extends PredictedScoreFeature | |
object PredictedNegativeFeedbackV2ScoreFeature extends PredictedScoreFeature | |
object PredictedGoodProfileClickScoreFeature extends PredictedScoreFeature | |
object PredictedReportedScoreFeature extends PredictedScoreFeature | |
object PredictedVideoPlayback50ScoreFeature extends PredictedScoreFeature | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scorer\WeightedScoresSumScorer.scala | |
class WeightedScoresSumScorer @Inject() (statsReceiver: StatsReceiver) | |
extends Scorer[ScoredTweetsQuery, TweetCandidate] | |
def apply( | |
query: ScoredTweetsQuery, | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
def weightedModelScore( | |
query: PipelineQuery, | |
features: FeatureMap | |
): Double | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsDiversityScoringPipelineConfig.scala | |
object ScoredTweetsDiversityScoringPipelineConfig | |
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsRescoreOONScoringPipelineConfig.scala | |
object ScoredTweetsRescoreOONScoringPipelineConfig | |
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig.scala | |
object ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig | |
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsScoringPipelineConfig.scala | |
class ScoredTweetsScoringPipelineConfig @Inject() ( | |
scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, | |
scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, | |
scoredTweetsCrMixerCandidatePipelineConfig: ScoredTweetsCrMixerCandidatePipelineConfig, | |
scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, | |
predictionGRPCService: PredictionGRPCService, | |
ancestorFeatureHydrator: AncestorFeatureHydrator, | |
authorFeatureHydrator: AuthorFeatureHydrator, | |
earlybirdFeatureHydrator: EarlybirdFeatureHydrator, | |
metricCenterUserCountingFeatureHydrator: MetricCenterUserCountingFeatureHydrator, | |
tweetypieContentFeatureHydrator: TweetypieContentFeatureHydrator, | |
gizmoduckAuthorSafetyFeatureHydrator: GizmoduckAuthorSafetyFeatureHydrator, | |
graphTwoHopFeatureHydrator: GraphTwoHopFeatureHydrator, | |
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, | |
twhinAuthorFollow20220101FeatureHydrator: TwhinAuthorFollow20220101FeatureHydrator, | |
userFollowedTopicIdsFeatureHydrator: UserFollowedTopicIdsFeatureHydrator, | |
utegFeatureHydrator: UtegFeatureHydrator, | |
realGraphViewerAuthorFeatureHydrator: RealGraphViewerAuthorFeatureHydrator, | |
realGraphViewerRelatedUsersFeatureHydrator: RealGraphViewerRelatedUsersFeatureHydrator, | |
realTimeInteractionGraphEdgeFeatureHydrator: RealTimeInteractionGraphEdgeFeatureHydrator, | |
engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator: EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, | |
topicCountryEngagementRealTimeAggregateFeatureHydrator: TopicCountryEngagementRealTimeAggregateFeatureHydrator, | |
topicEngagementRealTimeAggregateFeatureHydrator: TopicEngagementRealTimeAggregateFeatureHydrator, | |
tspInferredTopicFeatureHydrator: TSPInferredTopicFeatureHydrator, | |
tweetCountryEngagementRealTimeAggregateFeatureHydrator: TweetCountryEngagementRealTimeAggregateFeatureHydrator, | |
tweetEngagementRealTimeAggregateFeatureHydrator: TweetEngagementRealTimeAggregateFeatureHydrator, | |
twitterListEngagementRealTimeAggregateFeatureHydrator: TwitterListEngagementRealTimeAggregateFeatureHydrator, | |
userAuthorEngagementRealTimeAggregateFeatureHydrator: UserAuthorEngagementRealTimeAggregateFeatureHydrator, | |
simClustersEngagementSimilarityFeatureHydrator: SimClustersEngagementSimilarityFeatureHydrator, | |
phase1EdgeAggregateFeatureHydrator: Phase1EdgeAggregateFeatureHydrator, | |
phase2EdgeAggregateFeatureHydrator: Phase2EdgeAggregateFeatureHydrator, | |
statsReceiver: StatsReceiver) | |
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\scoring_pipeline\ScoredTweetsWeightedScoresSumScoringPipelineConfig.scala | |
class ScoredTweetsWeightedScoresSumScoringPipelineConfig @Inject() ( | |
weightedScoresSumScorer: WeightedScoresSumScorer) | |
extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\side_effect\CachedScoredTweetsSideEffect.scala | |
class CachedScoredTweetsSideEffect @Inject() ( | |
scoredTweetsCache: TtlCache[Long, hmt.CachedScoredTweets]) | |
extends PipelineResultSideEffect[PipelineQuery, ScoredTweetsResponse] | |
def buildCachedScoredTweets( | |
candidates: Seq[CandidateWithDetails] | |
): hmt.CachedScoredTweets | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, ScoredTweetsResponse] | |
): Stitch[Unit] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\product\scored_tweets\side_effect\ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala | |
class ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect @Inject() ( | |
@Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, | |
@Named(CommonFeaturesScribeEventPublisher) commonFeaturesScribeEventPublisher: EventPublisher[ | |
pldr.PolyDataRecord | |
], | |
@Named(CandidateFeaturesScribeEventPublisher) candidateFeaturesScribeEventPublisher: EventPublisher[ | |
pldr.PolyDataRecord | |
], | |
@Named(MinimumFeaturesScribeEventPublisher) minimumFeaturesScribeEventPublisher: EventPublisher[ | |
pldr.PolyDataRecord | |
], | |
statsReceiver: StatsReceiver, | |
) extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] | |
with Logging | |
def apply( | |
inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse] | |
): Stitch[Unit] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\service\HomeMixerAccessPolicy.scala | |
object HomeMixerAccessPolicy | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\service\HomeMixerAlertConfig.scala | |
object HomeMixerAlertConfig | |
object BusinessHours | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\service\ScoredTweetsService.scala | |
class ScoredTweetsService @Inject() (productPipelineRegistry: ProductPipelineRegistry) | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\store\RealGraphInNetworkScoresStore.scala | |
object ManhattanRealGraphKVDescriptor | |
class RealGraphInNetworkScoresStore(manhattanKVEndpoint: ManhattanKVEndpoint) | |
extends ReadableStore[Long, Seq[wtf.Candidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\store\UserLanguagesStore.scala | |
object ManhattanUserLanguagesKVDescriptor | |
class UserLanguagesStore( | |
manhattanKVEndpoint: ManhattanKVEndpoint, | |
statsReceiver: StatsReceiver) | |
extends ReadableStore[Long, Seq[scc.ThriftLanguage]] | |
def get(viewerId: Long): Future[Option[Seq[scc.ThriftLanguage]]] = | |
Stitch | |
.run( | |
manhattanKVEndpoint.get(key = userLanguagesDatasetKey.withPkey(viewerId), valueDescriptor)) | |
.map | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\CachedScoredTweetsHelper.scala | |
object CachedScoredTweetsHelper | |
def tweetImpressionsAndCachedScoredTweets( | |
features: FeatureMap, | |
candidatePipelineIdentifier: CandidatePipelineIdentifier | |
): Seq[Long] | |
def tweetImpressionsAndCachedScoredTweetsInRange( | |
features: FeatureMap, | |
candidatePipelineIdentifier: CandidatePipelineIdentifier, | |
maxNumImpressions: Int, | |
sinceTime: Time, | |
untilTime: Time | |
): Seq[Long] = | |
tweetImpressionsAndCachedScoredTweets(features, candidatePipelineIdentifier) | |
.filter | |
def unseenCachedScoredTweets( | |
features: FeatureMap | |
): Seq[hmt.CachedScoredTweet] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\CandidatesUtil.scala | |
object CandidatesUtil | |
def getItemCandidates(candidates: Seq[CandidateWithDetails]): Seq[ItemCandidateWithDetails] | |
def getItemCandidatesWithOnlyModuleLast( | |
candidates: Seq[CandidateWithDetails] | |
): Seq[ItemCandidateWithDetails] | |
def containsType[CandidateType <: UniversalNoun[_]]( | |
candidates: Seq[CandidateWithDetails] | |
)( | |
implicit tag: ClassTag[CandidateType] | |
): Boolean = candidates.exists | |
def getOriginalAuthorId(candidateFeatures: FeatureMap): Option[Long] = | |
if (candidateFeatures.getOrElse(IsRetweetFeature, false)) | |
candidateFeatures.getOrElse(SourceUserIdFeature, None) | |
else candidateFeatures.getOrElse(AuthorIdFeature, None) | |
def getEngagerUserIds( | |
candidateFeatures: FeatureMap | |
): Seq[Long] | |
def getMediaUnderstandingAnnotationIds( | |
candidateFeatures: FeatureMap | |
): Seq[Long] | |
def getTweetIdAndSourceId(candidate: CandidateWithFeatures[TweetCandidate]): Seq[Long] = | |
Seq(candidate.candidate.id) ++ candidate.features.getOrElse(SourceTweetIdFeature, None) | |
def isAuthoredByViewer(query: PipelineQuery, candidateFeatures: FeatureMap): Boolean = | |
candidateFeatures.getOrElse(AuthorIdFeature, None).contains(query.getRequiredUserId) || | |
(candidateFeatures.getOrElse(IsRetweetFeature, false) && | |
candidateFeatures.getOrElse(SourceUserIdFeature, None).contains(query.getRequiredUserId)) | |
val reverseChronTweetsOrdering: Ordering[CandidateWithDetails] = | |
Ordering.by[CandidateWithDetails, Long] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\InjectionTransformer.scala | |
object InjectionTransformerImplicits | |
class ByteArrayInjectionToByteBufferTransformer[A](baInj: Injection[A, Array[Byte]]) | |
def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) | |
def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) | |
} | |
implicit class BufInjectionToByteBufferTransformer[A](bufInj: Injection[A, Buf]) | |
def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) | |
def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) | |
} | |
implicit class ByteBufferInjectionToByteBufferTransformer[A](bbInj: Injection[A, ByteBuffer]) | |
def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) | |
def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) | |
} | |
} | |
class InjectionTransformer[A, B](inj: Injection[A, B]) extends Transformer[A, B] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\LanguageUtil.scala | |
object LanguageUtil | |
def computeLanguages( | |
userLanguages: smg.UserLanguages, | |
minProducedLanguageRatio: Double = DafaultMinProducedLanguageRatio, | |
minConsumedLanguageConfidence: Double = DefaultMinConsumedLanguageConfidence | |
): Seq[scc.ThriftLanguage] | |
def computeLanguageConfidenceMap( | |
userLanguages: smg.UserLanguages, | |
minProducedLanguageRatio: Double, | |
minConsumedLanguageConfidence: Double | |
): Map[scc.ThriftLanguage, Double] | |
def getLanguageMap( | |
scoredLanguages: Seq[smg.ScoredString] | |
): Map[scc.ThriftLanguage, Double] | |
def getThriftLanguage(languageName: String): Option[scc.ThriftLanguage] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\MissingKeyException.scala | |
object MissingKeyException extends Exception("Missing key") | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\ObservedKeyValueResultHandler.scala | |
trait ObservedKeyValueResultHandler | |
def observedGet[K, V]( | |
key: Option[K], | |
keyValueResult: KeyValueResult[K, V], | |
): Try[Option[V]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\ReplyRetweetUtil.scala | |
object ReplyRetweetUtil | |
def isEligibleReply(candidate: CandidateWithFeatures[TweetCandidate]): Boolean | |
def replyToAncestorTweetCandidatesMap( | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Map[Long, Seq[CandidateWithFeatures[TweetCandidate]]] | |
def ancestorTweetIdToDescendantRepliesMap( | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Map[Long, Seq[CandidateWithFeatures[TweetCandidate]]] | |
def replyTweetIdToInReplyToTweetMap( | |
candidates: Seq[CandidateWithFeatures[TweetCandidate]] | |
): Map[Long, CandidateWithFeatures[TweetCandidate]] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\TensorFlowUtil.scala | |
object TensorFlowUtil | |
def skipEmbeddingBBHeader(bb: ByteBuffer): ByteBuffer | |
def byteBufferToFloatIterator( | |
bb: ByteBuffer | |
): Iterator[Float] | |
def embeddingByteBufferToFloatTensor( | |
bb: ByteBuffer | |
): FloatTensor | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\TweetImpressionsHelper.scala | |
object TweetImpressionsHelper | |
def tweetImpressions(features: FeatureMap): Set[Long] | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\earlybird\EarlybirdRequestUtil.scala | |
object EarlybirdRequestUtil | |
def getTweetsEBFeaturesRequest( | |
userId: Option[Long], | |
tweetIds: Option[Seq[Long]], | |
clientId: Option[String], | |
getTweetsFromArchiveIndex: Boolean = false, | |
getOnlyProtectedTweets: Boolean = false, | |
): eb.EarlybirdRequest | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\earlybird\EarlybirdResponseUtil.scala | |
object EarlybirdResponseUtil | |
def getTweetCountByAuthorId( | |
searchResults: Seq[eb.ThriftSearchResult] | |
): Map[Long, Int] | |
def getLanguage(uiLanguageCode: Option[String]): Option[scc.ThriftLanguage] | |
def getMentions(result: eb.ThriftSearchResult): Seq[String] | |
def getHashtags(result: eb.ThriftSearchResult): Seq[String] | |
def getFacets( | |
facetLabels: Seq[eb.ThriftFacetLabel], | |
facetName: String, | |
charsToRemove: Set[Char] | |
): Seq[String] | |
def isUserMentioned( | |
screenName: Option[String], | |
mentions: Seq[String] | |
): Boolean | |
def isUsersMainLanguage( | |
tweetLanguage: scc.ThriftLanguage, | |
userLanguages: Seq[scc.ThriftLanguage] | |
): Boolean | |
def isUsersLanguage( | |
tweetLanguage: scc.ThriftLanguage, | |
userLanguages: Seq[scc.ThriftLanguage] | |
): Boolean | |
def isUILanguage( | |
tweetLanguage: scc.ThriftLanguage, | |
uiLanguage: Option[scc.ThriftLanguage] | |
): Boolean | |
def getBooleanOptFeature( | |
featureName: EarlybirdFieldConstant, | |
resultMapOpt: Option[scala.collection.Map[Int, Boolean]], | |
defaultValue: Boolean = false, | |
): Option[Boolean] | |
def getDoubleAsIntOptFeature( | |
featureName: EarlybirdFieldConstant, | |
resultMapOpt: Option[scala.collection.Map[Int, Double]] | |
): Option[Int] | |
def getIntOptFeature( | |
featureName: EarlybirdFieldConstant, | |
resultMapOpt: Option[scala.collection.Map[Int, Int]] | |
): Option[Int] | |
def getOONTweetThriftFeaturesByTweetId( | |
searcherUserId: Long, | |
screenName: Option[String], | |
userLanguages: Seq[scc.ThriftLanguage], | |
uiLanguageCode: Option[String] = None, | |
searchResults: Seq[eb.ThriftSearchResult], | |
): Map[Long, sc.ThriftTweetFeatures] | |
def getOONThriftTweetFeaturesFromSearchResult( | |
searcherUserId: Long, | |
screenName: Option[String], | |
userLanguages: Seq[scc.ThriftLanguage], | |
uiLanguage: Option[scc.ThriftLanguage], | |
tweetCountByAuthorId: Map[Long, Int], | |
searchResult: eb.ThriftSearchResult | |
): sc.ThriftTweetFeatures | |
def applyUserIndependentFeatures( | |
result: eb.ThriftSearchResult | |
)( | |
thriftTweetFeatures: sc.ThriftTweetFeatures | |
): sc.ThriftTweetFeatures | |
def applyOONUserDependentFeatures( | |
searcherUserId: Long, | |
screenName: Option[String], | |
userLanguages: Seq[scc.ThriftLanguage], | |
uiLanguage: Option[scc.ThriftLanguage], | |
tweetCountByAuthorId: Map[Long, Int], | |
result: eb.ThriftSearchResult | |
)( | |
thriftTweetFeatures: sc.ThriftTweetFeatures | |
): sc.ThriftTweetFeatures | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\earlybird\RelevanceSearchUtil.scala | |
object RelevanceSearchUtil | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\tweetypie\RequestFields.scala | |
object RequestFields | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\tweetypie\content\FeatureExtractionHelper.scala | |
object FeatureExtractionHelper | |
def extractFeatures( | |
tweet: tp.Tweet | |
): ContentFeatures | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\tweetypie\content\TweetMediaFeaturesExtractor.scala | |
object TweetMediaFeaturesExtractor | |
def hasImage(tweet: tp.Tweet): Boolean = hasMediaByCategory(tweet, ImageCategories) | |
def hasVideo(tweet: tp.Tweet): Boolean = hasMediaByCategory(tweet, VideoCategories) | |
private def hasMediaByCategory(tweet: tp.Tweet, categories: Set[Int]): Boolean | |
def addMediaFeaturesFromTweet( | |
inputFeatures: ContentFeatures, | |
tweet: tp.Tweet, | |
): ContentFeatures | |
def getSizeFeatures(mediaEntities: Seq[tp.MediaEntity]): Seq[MediaSizeFeatures] | |
def getPlaybackFeatures(mediaEntities: Seq[tp.MediaEntity]): PlaybackFeatures | |
def getMediaTagScreenNames(tagMap: Map[Long, Seq[tp.MediaTag]]): Seq[String] = | |
tagMap.values | |
.flatMap(seqMediaTag => seqMediaTag.flatMap(_.screenName)) | |
.toSeq | |
// Areas of the faces identified in the media entities | |
private def getFaceMapAreas(mediaEntities: Seq[tp.MediaEntity]): Seq[Int] | |
def getSortedColorPalette( | |
mediaEntities: Seq[tp.MediaEntity] | |
): Seq[mi.ColorPaletteItem] | |
def getStickerFeatures(mediaEntities: Seq[tp.MediaEntity]): Seq[Long] | |
def getMediaOriginProviders(mediaEntities: Seq[tp.MediaEntity]): Seq[String] = | |
for | |
def getIsManaged(mediaEntities: Seq[tp.MediaEntity]): Boolean | |
def getIs360(mediaEntities: Seq[tp.MediaEntity]): Boolean | |
def getViewCount(mediaEntities: Seq[tp.MediaEntity]): Option[Long] | |
def getUserDefinedProductMetadataFeatures( | |
mediaEntities: Seq[tp.MediaEntity] | |
): Seq[UserDefinedProductMetadataFeatures] = | |
for | |
.\home-mixer\server\src\main\scala\com\twitter\home_mixer\util\tweetypie\content\TweetTextFeaturesExtractor.scala | |
object TweetTextFeaturesExtractor | |
def addTextFeaturesFromTweet( | |
inputFeatures: ContentFeatures, | |
tweet: tp.Tweet | |
): ContentFeatures | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\account_recommendations_mixer\AccountRecommendationsMixerCandidateSource.scala | |
object WhoToFollowModuleHeaderFeature extends Feature[UserCandidate, t.Header] | |
object WhoToFollowModuleFooterFeature extends Feature[UserCandidate, Option[t.Footer]] | |
object WhoToFollowModuleDisplayOptionsFeature | |
extends Feature[UserCandidate, Option[t.DisplayOptions]] | |
@Singleton | |
class AccountRecommendationsMixerCandidateSource @Inject() ( | |
accountRecommendationsMixer: t.AccountRecommendationsMixer.MethodPerEndpoint) | |
extends CandidateSourceWithExtractedFeatures[ | |
t.AccountRecommendationsMixerRequest, | |
t.RecommendedUser | |
] | |
def apply( | |
request: t.AccountRecommendationsMixerRequest | |
): Stitch[CandidatesWithSourceFeatures[t.RecommendedUser]] | |
def responseToCandidatesWithSourceFeatures( | |
userRecommendations: Seq[t.RecommendedUser], | |
header: t.Header, | |
footer: Option[t.Footer], | |
displayOptions: Option[t.DisplayOptions], | |
): CandidatesWithSourceFeatures[t.RecommendedUser] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\ads\AdsProdStratoCandidateSource.scala | |
class AdsProdStratoCandidateSource @Inject() (adsClient: MakeAdRequestClientColumn) | |
extends StratoKeyFetcherSource[ | |
AdRequestParams, | |
AdRequestResponse, | |
AdImpression | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\ads\AdsProdThriftCandidateSource.scala | |
class AdsProdThriftCandidateSource @Inject() ( | |
adServerClient: NewAdServer.MethodPerEndpoint) | |
extends CandidateSource[AdRequestParams, AdImpression] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\ads\AdsStagingCandidateSource.scala | |
class AdsStagingCandidateSource @Inject() (adsClient: MakeAdRequestStagingClientColumn) | |
extends StratoKeyFetcherSource[ | |
AdRequestParams, | |
AdRequestResponse, | |
AdImpression | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\ann\AnnCandidateSource.scala | |
class AnnCandidateSource[T1, T2, P <: RuntimeParams, D <: Distance[D]]( | |
val annQueryableById: QueryableById[T1, T2, P, D], | |
val batchSize: Int, | |
val timeoutPerRequest: Duration, | |
override val identifier: CandidateSourceIdentifier) | |
extends CandidateSource[AnnIdQuery[T1, P], NeighborWithDistanceWithSeed[T1, T2, D]] | |
def apply( | |
request: AnnIdQuery[T1, P] | |
): Stitch[Seq[NeighborWithDistanceWithSeed[T1, T2, D]]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\audiospace\CreatedSpacesCandidateSource.scala | |
class CreatedSpacesCandidateSource @Inject() ( | |
column: CreatedSpacesSliceOnUserClientColumn) | |
extends StratoKeyViewFetcherWithSourceFeaturesSource[ | |
Long, | |
CreatedSpacesView, | |
SpaceSlice, | |
String | |
] | |
def stratoResultTransformer( | |
stratoKey: Long, | |
stratoResult: SpaceSlice | |
): Seq[String] = | |
stratoResult.items | |
override protected def extractFeaturesFromStratoResult( | |
stratoKey: Long, | |
stratoResult: SpaceSlice | |
): FeatureMap | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\business_profiles\TeamMembersCandidateSource.scala | |
class TeamMembersCandidateSource @Inject() ( | |
column: BusinessProfileTeamMembersOnUserClientColumn) | |
extends StratoKeyViewFetcherWithSourceFeaturesSource[ | |
Long, | |
TeamMembersView, | |
TeamMembersSlice, | |
Long | |
] | |
def stratoResultTransformer( | |
stratoKey: Long, | |
stratoResult: TeamMembersSlice | |
): Seq[Long] = | |
stratoResult.members | |
override protected def extractFeaturesFromStratoResult( | |
stratoKey: Long, | |
stratoResult: TeamMembersSlice | |
): FeatureMap | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\cr_mixer\CrMixerFrsBasedTweetRecommendationsCandidateSource.scala | |
class CrMixerFrsBasedTweetRecommendationsCandidateSource @Inject() ( | |
crMixerClient: t.CrMixer.MethodPerEndpoint) | |
extends CandidateSource[t.FrsTweetRequest, t.FrsTweet] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\cr_mixer\CrMixerTweetRecommendationsCandidateSource.scala | |
class CrMixerTweetRecommendationsCandidateSource @Inject() ( | |
crMixerClient: t.CrMixer.MethodPerEndpoint) | |
extends CandidateSource[t.CrMixerTweetRequest, t.TweetRecommendation] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\earlybird\EarlybirdTweetCandidateSource.scala | |
class EarlybirdTweetCandidateSource @Inject() ( | |
earlybirdService: t.EarlybirdService.MethodPerEndpoint) | |
extends CandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] | |
with Logging | |
def apply(request: t.EarlybirdRequest): Stitch[Seq[t.ThriftSearchResult]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\explore_ranker\ExploreRankerCandidateSource.scala | |
class ExploreRankerCandidateSource @Inject() ( | |
exploreRankerService: t.ExploreRanker.MethodPerEndpoint) | |
extends CandidateSource[t.ExploreRankerRequest, t.ImmersiveRecsResult] | |
def apply( | |
request: t.ExploreRankerRequest | |
): Stitch[Seq[t.ImmersiveRecsResult]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\flexible_injection_pipeline\PromptCandidateSource.scala | |
class PromptCandidateSource @Inject() (taskService: servicethrift.TaskService.MethodPerEndpoint) | |
extends CandidateSource[servicethrift.GetInjectionsRequest, IntermediatePrompt] | |
with Logging | |
def apply( | |
request: servicethrift.GetInjectionsRequest | |
): Stitch[Seq[IntermediatePrompt]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\hermit\UsersSimilarToMeCandidateSource.scala | |
class UsersSimilarToMeCandidateSource @Inject() ( | |
column: HermitRecommendUsersClientColumn) | |
extends StratoKeyViewFetcherSource[ | |
Long, | |
RecommendationRequest, | |
RecommendationResponse, | |
RelatedUser | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\interest_discovery\RelatedTopicsCandidateSource.scala | |
class RelatedTopicsCandidateSource @Inject() ( | |
interestDiscoveryService: t.InterestsDiscoveryService.MethodPerEndpoint) | |
extends CandidateSource[t.RelatedTopicsRequest, t.RelatedTopic] | |
with Logging | |
def apply( | |
request: t.RelatedTopicsRequest | |
): Stitch[Seq[t.RelatedTopic]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\lists\OrganicPopGeoListsCandidateSource.scala | |
class OrganicPopGeoListsCandidateSource @Inject() ( | |
organicPopgeoListsClientColumn: OrganicPopgeoListsClientColumn) | |
extends StratoKeyFetcherSource[ | |
OrganicPopgeoListsClientColumn.Key, | |
OrganicPopgeoListsClientColumn.Value, | |
TwitterListCandidate | |
] | |
def stratoResultTransformer( | |
stratoResult: OrganicPopgeoListsClientColumn.Value | |
): Seq[TwitterListCandidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\people_discovery\PeopleDiscoveryCandidateSource.scala | |
object WhoToFollowModuleHeaderFeature extends Feature[UserCandidate, t.Header] | |
object WhoToFollowModuleDisplayOptionsFeature | |
extends Feature[UserCandidate, Option[t.DisplayOptions]] | |
object WhoToFollowModuleShowMoreFeature extends Feature[UserCandidate, Option[t.ShowMore]] | |
@Singleton | |
class PeopleDiscoveryCandidateSource @Inject() ( | |
peopleDiscoveryService: t.ThriftPeopleDiscoveryService.MethodPerEndpoint) | |
extends CandidateSourceWithExtractedFeatures[t.GetModuleRequest, t.RecommendedUser] | |
with Logging | |
def apply( | |
request: t.GetModuleRequest | |
): Stitch[CandidatesWithSourceFeatures[t.RecommendedUser]] | |
def layoutToCandidatesWithSourceFeatures( | |
userRecommendations: Seq[t.RecommendedUser], | |
header: t.Header, | |
displayOptions: Option[t.DisplayOptions], | |
showMore: Option[t.ShowMore], | |
): CandidatesWithSourceFeatures[t.RecommendedUser] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\recommendations\UserFollowRecommendationsCandidateSource.scala | |
class UserFollowRecommendationsCandidateSource @Inject() ( | |
getRecommendationsClientColumn: GetRecommendationsClientColumn) | |
extends StratoKeyViewFetcherSource[ | |
fr.RecommendationRequest, | |
Unit, | |
fr.RecommendationResponse, | |
fr.UserRecommendation | |
] | |
def stratoResultTransformer( | |
stratoKey: fr.RecommendationRequest, | |
stratoResult: fr.RecommendationResponse | |
): Seq[fr.UserRecommendation] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\social_graph\SocialgraphCandidateSource.scala | |
trait SocialgraphResponse | |
case class SocialgraphResult(id: Long) extends SocialgraphResponse | |
case class SocialgraphCursor(cursor: Long, cursorType: CursorType) extends SocialgraphResponse | |
@Singleton | |
class SocialgraphCandidateSource @Inject() ( | |
override val fetcher: Fetcher[thriftscala.IdsRequest, Option[ | |
thriftscala.RequestContext | |
], thriftscala.IdsResult]) | |
extends StratoKeyViewFetcherSource[ | |
thriftscala.IdsRequest, | |
Option[thriftscala.RequestContext], | |
thriftscala.IdsResult, | |
SocialgraphResponse | |
] | |
def stratoResultTransformer( | |
stratoKey: IdsRequest, | |
stratoResult: IdsResult | |
): Seq[SocialgraphResponse] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\social_graph\SocialgraphCursorConstants.scala | |
object SocialgraphCursorConstants | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timelines_impression_store\TimelinesImpressionStoreCandidateSourceV2.scala | |
class TimelinesImpressionStoreCandidateSourceV2 @Inject() ( | |
client: TweetImpressionStoreManhattanV2OnUserClientColumn) | |
extends StratoKeyFetcherSource[ | |
Long, | |
t.TweetImpressionsEntries, | |
t.TweetImpressionsEntry | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_ranker\TimelineRankerInNetworkCandidateSource.scala | |
object TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature | |
extends Feature[PipelineQuery, Map[Long, t.CandidateTweet]] | |
@Singleton | |
class TimelineRankerInNetworkCandidateSource @Inject() ( | |
timelineRankerClient: t.TimelineRanker.MethodPerEndpoint) | |
extends CandidateSourceWithExtractedFeatures[t.RecapQuery, t.CandidateTweet] | |
def apply( | |
request: t.RecapQuery | |
): Stitch[CandidatesWithSourceFeatures[t.CandidateTweet]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_ranker\TimelineRankerRecapCandidateSource.scala | |
class TimelineRankerRecapCandidateSource @Inject() ( | |
timelineRankerClient: t.TimelineRanker.MethodPerEndpoint) | |
extends CandidateSource[t.RecapQuery, t.CandidateTweet] | |
def apply( | |
request: t.RecapQuery | |
): Stitch[Seq[t.CandidateTweet]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_ranker\TimelineRankerUtegCandidateSource.scala | |
object TimelineRankerUtegSourceTweetsFeature | |
extends Feature[PipelineQuery, Seq[t.CandidateTweet]] | |
@Singleton | |
class TimelineRankerUtegCandidateSource @Inject() ( | |
timelineRankerClient: t.TimelineRanker.MethodPerEndpoint) | |
extends CandidateSourceWithExtractedFeatures[t.UtegLikedByTweetsQuery, t.CandidateTweet] | |
def apply( | |
request: t.UtegLikedByTweetsQuery | |
): Stitch[CandidatesWithSourceFeatures[t.CandidateTweet]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_scorer\TimelineScorerCandidateSource.scala | |
class ScoredTweetCandidateWithFocalTweet( | |
candidate: v1.ScoredTweetCandidate, | |
focalTweetIdOpt: Option[Long]) | |
case object TimelineScorerCandidateSourceSucceededFeature extends Feature[PipelineQuery, Boolean] | |
@Singleton | |
class TimelineScorerCandidateSource @Inject() ( | |
timelineScorerClient: t.TimelineScorer.MethodPerEndpoint) | |
extends CandidateSourceWithExtractedFeatures[ | |
t.ScoredTweetsRequest, | |
ScoredTweetCandidateWithFocalTweet | |
] | |
def apply( | |
request: t.ScoredTweetsRequest | |
): Stitch[CandidatesWithSourceFeatures[ScoredTweetCandidateWithFocalTweet]] | |
def isEligibleReply(candidate: ct.ScoredTweetCandidateAliases.V1Alias): Boolean | |
def originalTweetId(candidate: ct.ScoredTweetCandidateAliases.V1Alias): Long | |
def toScoredTweetCandidateFromAncestor( | |
ancestor: Ancestor, | |
inReplyToTweetId: Option[Long], | |
conversationId: Option[Long], | |
ancestors: Option[Seq[Ancestor]], | |
candidateTweetSourceId: Option[CandidateTweetSourceId] | |
): ct.ScoredTweetCandidateAliases.V1Alias | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\timeline_service\TimelineServiceTweetCandidateSource.scala | |
object TimelineServiceResponseWasTruncatedFeature | |
extends FeatureWithDefaultOnFailure[PipelineQuery, Boolean] | |
class TimelineServiceTweetCandidateSource @Inject() ( | |
timelineService: TimelineService) | |
extends CandidateSourceWithExtractedFeatures[t.TimelineQuery, t.Tweet] | |
def apply(request: t.TimelineQuery): Stitch[CandidatesWithSourceFeatures[t.Tweet]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\topics\FollowedTopicsCandidateSource.scala | |
class FollowedTopicsCandidateSource @Inject() ( | |
column: FollowedTopicsGetterClientColumn) | |
extends StratoKeyViewFetcherSeqSource[ | |
Long, | |
Unit, | |
Long | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\tweetconvosvc\ConversationServiceCandidateSource.scala | |
class ConversationServiceCandidateSourceRequest( | |
tweetsWithConversationMetadata: Seq[TweetWithConversationMetadata]) | |
case class TweetWithConversationMetadata( | |
tweetId: Long, | |
userId: Option[Long], | |
sourceTweetId: Option[Long], | |
sourceUserId: Option[Long], | |
inReplyToTweetId: Option[Long], | |
conversationId: Option[Long], | |
ancestors: Seq[ta.TweetAncestor]) | |
/** | |
* Candidate source that fetches ancestors of input candidates from Tweetconvosvc and | |
* returns a flattened list of input and ancestor candidates. | |
*/ | |
@Singleton | |
class ConversationServiceCandidateSource @Inject() ( | |
conversationServiceClient: tcs.ConversationService.MethodPerEndpoint) | |
extends CandidateSourceWithExtractedFeatures[ | |
ConversationServiceCandidateSourceRequest, | |
TweetWithConversationMetadata | |
] | |
def apply( | |
request: ConversationServiceCandidateSourceRequest | |
): Stitch[CandidatesWithSourceFeatures[TweetWithConversationMetadata]] | |
def getTweetsInThread( | |
focalTweet: TweetWithConversationMetadata, | |
ancestors: ta.TweetAncestors | |
): Seq[TweetWithConversationMetadata] | |
def getTruncatedRootTweet( | |
ancestors: ta.TweetAncestors, | |
focalTweetId: Long | |
): Option[TweetWithConversationMetadata] | |
def dedupeCandidates( | |
inputTweetsWithConversationMetadata: Seq[TweetWithConversationMetadata], | |
ancestors: Seq[TweetWithConversationMetadata] | |
): Seq[TweetWithConversationMetadata] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\tweetconvosvc\ConversationServiceResponseFeatureTransformer.scala | |
object AuthorIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object AncestorIdsFeature extends Feature[TweetCandidate, Seq[Long]] | |
object ConversationModuleFocalTweetIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object InReplyToFeature extends Feature[TweetCandidate, Option[Long]] | |
object IsRetweetFeature extends Feature[TweetCandidate, Boolean] | |
object SourceTweetIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object SuggestTypeFeature extends Feature[TweetCandidate, Option[SuggestType]] | |
object ConversationServiceResponseFeatureTransformer | |
extends CandidateFeatureTransformer[TweetWithConversationMetadata] | |
def transform(candidate: TweetWithConversationMetadata): FeatureMap | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\candidate_source\tweetconvosvc\DropMaxConversationModuleItemCandidates.scala | |
class DropMaxConversationModuleItemCandidates[-Query <: PipelineQuery]( | |
override val pipelineScope: CandidateScope, | |
includeRootTweet: Boolean) | |
extends Selector[Query] | |
def apply( | |
query: Query, | |
remainingCandidates: Seq[CandidateWithDetails], | |
result: Seq[CandidateWithDetails] | |
): SelectorResult | |
def updateConversationModule( | |
module: ModuleCandidateWithDetails, | |
includeRootTweet: Boolean | |
): ModuleCandidateWithDetails | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\slice\SliceItemCandidateDecorator.scala | |
class SliceItemCandidateDecorator[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( | |
cursorBuilder: CandidateSliceItemBuilder[Query, CursorCandidate, CursorItem], | |
override val identifier: DecoratorIdentifier = DecoratorIdentifier("SliceItemCandidate")) | |
extends CandidateDecorator[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[Seq[Decoration]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\slice\builder\CursorCandidateSliceItemBuilder.scala | |
class CursorCandidateSliceItemBuilder() | |
extends CandidateSliceItemBuilder[PipelineQuery, CursorCandidate, CursorItem] | |
def apply( | |
query: PipelineQuery, | |
candidate: CursorCandidate, | |
featureMap: FeatureMap | |
): CursorItem = | |
candidate.cursorType match | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\UrtConversationItemCandidateDecorator.scala | |
class UrtConversationItemCandidateDecorator[ | |
Query <: PipelineQuery, | |
Candidate <: BaseTweetCandidate | |
]( | |
tweetCandidateUrtItemBuilder: TweetCandidateUrtItemBuilder[Query, Candidate], | |
override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtConversationItem")) | |
extends CandidateDecorator[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[Seq[Decoration]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\UrtItemCandidateDecorator.scala | |
class UrtItemCandidateDecorator[ | |
Query <: PipelineQuery, | |
BuilderInput <: UniversalNoun[Any], | |
BuilderOutput <: TimelineItem | |
]( | |
builder: CandidateUrtEntryBuilder[Query, BuilderInput, BuilderOutput], | |
override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtItemCandidate")) | |
extends CandidateDecorator[Query, BuilderInput] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[BuilderInput]] | |
): Stitch[Seq[Decoration]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\UrtItemInModuleDecorator.scala | |
class UrtItemInModuleDecorator[ | |
Query <: PipelineQuery, | |
BuilderInput <: UniversalNoun[Any], | |
BuilderOutput <: TimelineItem | |
]( | |
urtItemCandidateDecorator: CandidateDecorator[Query, BuilderInput], | |
moduleBuilder: BaseTimelineModuleBuilder[Query, BuilderInput], | |
override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtItemInModule")) | |
extends CandidateDecorator[Query, BuilderInput] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[BuilderInput]] | |
): Stitch[Seq[Decoration]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\UrtMultipleModulesDecorator.scala | |
trait GroupByKey[-Query <: PipelineQuery, -BuilderInput <: UniversalNoun[Any], Key] | |
def apply(query: Query, candidate: BuilderInput, candidateFeatures: FeatureMap): Option[Key] | |
} | |
/** | |
* Similar to [[UrtItemInModuleDecorator]] except that this decorator can assign items to different | |
* modules based on the provided [[GroupByKey]]. | |
* | |
* @param urtItemCandidateDecorator decorates individual item candidates | |
* @param moduleBuilder builds a module from a particular candidate group | |
* @param groupByKey assigns each candidate a module group. Returning [[None]] will result in the | |
* candidate not being assigned to a module | |
*/ | |
case class UrtMultipleModulesDecorator[ | |
-Query <: PipelineQuery, | |
-BuilderInput <: UniversalNoun[Any], | |
GroupKey | |
]( | |
urtItemCandidateDecorator: CandidateDecorator[Query, BuilderInput], | |
moduleBuilder: BaseTimelineModuleBuilder[Query, BuilderInput], | |
groupByKey: GroupByKey[Query, BuilderInput, GroupKey], | |
override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtMultipleModules")) | |
extends CandidateDecorator[Query, BuilderInput] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[BuilderInput]] | |
): Stitch[Seq[Decoration]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\contextual_ref\ContextualTweetRefBuilder.scala | |
class ContextualTweetRefBuilder[-Candidate <: BaseTweetCandidate]( | |
tweetHydrationContext: TweetHydrationContext) | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\conversations\ConversationModuleMetadataBuilder.scala | |
class ConversationModuleMetadataBuilder[ | |
Query <: PipelineQuery, | |
Candidate <: BaseTweetCandidate | |
]( | |
ancestorIdsFeature: Feature[_, Seq[Long]], | |
allIdsOrdering: Ordering[Long]) | |
extends BaseModuleMetadataBuilder[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): ModuleMetadata | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\FlipPromptCandidateUrtItemBuilder.scala | |
object FlipPromptCandidateUrtItemBuilder | |
class FlipPromptCandidateUrtItemBuilder[-Query <: PipelineQuery]() | |
extends CandidateUrtEntryBuilder[Query, BasePromptCandidate[Any], TimelineItem] | |
def apply( | |
query: Query, | |
promptCandidate: BasePromptCandidate[Any], | |
candidateFeatures: FeatureMap | |
): TimelineItem | |
def getInlinePromptMessageContent( | |
candidate: onboardingthrift.InlinePrompt | |
): MessageContent | |
def getFullCoverContent( | |
candidate: onboardingthrift.FullCover | |
): FullCoverContent = | |
FullCoverContent( | |
displayType = CoverFullCoverDisplayType, | |
primaryText = convertRichText(candidate.primaryText), | |
primaryCoverCta = convertCoverCta(candidate.primaryButtonAction), | |
secondaryCoverCta = candidate.secondaryButtonAction.map(convertCoverCta), | |
secondaryText = candidate.secondaryText.map(convertRichText), | |
imageVariant = candidate.image.map(img => convertImageVariant(img.image)), | |
details = candidate.detailText.map(convertRichText), | |
dismissInfo = candidate.dismissInfo.map(convertDismissInfo), | |
imageDisplayType = candidate.image.map(img => convertImageDisplayType(img.imageDisplayType)), | |
impressionCallbacks = candidate.impressionCallbacks.map(_.map(convertCallback).toList) | |
) | |
private def getHalfCoverContent( | |
candidate: onboardingthrift.HalfCover | |
): HalfCoverContent = | |
HalfCoverContent( | |
displayType = | |
candidate.displayType.map(convertHalfCoverDisplayType).getOrElse(CoverHalfCoverDisplayType), | |
primaryText = convertRichText(candidate.primaryText), | |
primaryCoverCta = convertCoverCta(candidate.primaryButtonAction), | |
secondaryCoverCta = candidate.secondaryButtonAction.map(convertCoverCta), | |
secondaryText = candidate.secondaryText.map(convertRichText), | |
coverImage = candidate.image.map(convertCoverImage), | |
dismissible = candidate.dismissible, | |
dismissInfo = candidate.dismissInfo.map(convertDismissInfo), | |
impressionCallbacks = candidate.impressionCallbacks.map(_.map(convertCallback).toList) | |
) | |
private def buildClientEventInfo( | |
injection: Injection | |
): Option[ClientEventInfo] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\FlipPromptModuleGrouping.scala | |
object FlipPromptModuleGrouping extends GroupByKey[PipelineQuery, UniversalNoun[Any], Int] | |
def apply( | |
query: PipelineQuery, | |
candidate: UniversalNoun[Any], | |
candidateFeatures: FeatureMap | |
): Option[Int] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\FlipPromptUrtModuleBuilder.scala | |
class FlipPromptUrtModuleBuilder[-Query <: PipelineQuery]( | |
moduleIdGeneration: ModuleIdGeneration = AutomaticUniqueModuleId()) | |
extends BaseTimelineModuleBuilder[Query, BasePromptCandidate[Any]] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[BasePromptCandidate[Any]]] | |
): TimelineModule | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\OnboardingInjectionConversions.scala | |
class to convert onboarding thrift to product-mixer models | |
*/ | |
object OnboardingInjectionConversions | |
def convertFeedbackInfo( | |
feedbackInfo: onboardingthrift.FeedbackInfo | |
): FeedbackActionInfo | |
def convertClientEventInfo(input: onboardingthrift.ClientEventInfo): ClientEventInfo = | |
ClientEventInfo( | |
component = input.component, | |
element = input.element, | |
details = None, | |
action = input.action, | |
entityToken = None) | |
def convertCallback(callback: onboardingthrift.Callback): Callback = | |
Callback(callback.endpoint) | |
def convertImage(image: onboardingthrift.Image): MessageImage = | |
MessageImage( | |
Set(convertImageVariant(image.image)), | |
backgroundColor = | |
None | |
) | |
def convertCoverImage(image: onboardingthrift.Image): CoverImage = | |
CoverImage( | |
convertImageVariant(image.image), | |
imageDisplayType = convertImageDisplayType(image.imageDisplayType), | |
imageAnimationType = image.imageAnimationType.map(convertImageAnimationType), | |
) | |
def convertImageDisplayType( | |
imageDisplayType: onboardingthrift.ImageDisplayType | |
): ImageDisplayType = | |
imageDisplayType match | |
def convertImageAnimationType( | |
imageAnimationType: onboardingthrift.ImageAnimationType | |
): ImageAnimationType = | |
imageAnimationType match | |
def convertImageVariant(imageVariant: onboardingthrift.ImageVariant): ImageVariant = | |
ImageVariant( | |
url = imageVariant.url, | |
width = imageVariant.width, | |
height = imageVariant.height, | |
palette = None) | |
def convertButtonAction( | |
buttonAction: onboardingthrift.ButtonAction | |
): MessageTextAction = | |
MessageTextAction( | |
buttonAction.text, | |
MessageAction( | |
dismissOnClick = buttonAction.dismissOnClick.getOrElse(true), | |
url = getActionUrl(buttonAction), | |
clientEventInfo = Some(convertClientEventInfo(buttonAction.clientEventInfo)), | |
onClickCallbacks = buttonAction.callbacks.map(_.map(convertCallback).toList) | |
) | |
) | |
private def getActionUrl(buttonAction: onboardingthrift.ButtonAction) = | |
buttonAction.buttonBehavior match | |
def convertRichText( | |
richText: com.twitter.onboarding.injections.thriftscala.RichText | |
): RichText | |
def convertAlignment(alignment: onboardingthrift.RichTextAlignment): RichTextAlignment = | |
alignment match | |
def convertRef(ref: onboardingthrift.ReferenceObject): ReferenceObject = | |
ref match | |
def convertFormat(format: onboardingthrift.RichTextFormat): RichTextFormat = | |
format match | |
def convertSocialContext(socialContext: onboardingthrift.RichText): SocialContext = | |
GeneralContext( | |
contextType = FollowGeneralContextType, | |
text = socialContext.text, | |
url = None, | |
contextImageUrls = None, | |
landingUrl = None) | |
def convertUserFacePile( | |
userFacepile: onboardingthrift.PromptUserFacepile | |
): UserFacepile = | |
UserFacepile( | |
userIds = userFacepile.userIds.toList, | |
featuredUserIds = userFacepile.featuredUserIds.toList, | |
action = userFacepile.action.map(convertButtonAction), | |
actionType = userFacepile.actionType.map(convertUserFacePileActionType), | |
displaysFeaturingText = userFacepile.displaysFeaturingText, | |
displayType = Some(LargeUserFacepileDisplayType) | |
) | |
private def convertUserFacePileActionType( | |
actionType: onboardingthrift.FacepileActionType | |
): MessageActionType = | |
actionType match | |
def convertHalfCoverDisplayType( | |
displayType: onboardingthrift.HalfCoverDisplayType | |
): HalfCoverDisplayType = | |
displayType match | |
def convertDismissInfo(dismissInfo: onboardingthrift.DismissInfo): DismissInfo = | |
DismissInfo(dismissInfo.callbacks.map(_.map(convertCallback))) | |
def convertCoverCta( | |
buttonAction: onboardingthrift.ButtonAction | |
): CoverCta = | |
CoverCta( | |
buttonAction.text, | |
ctaBehavior = convertCoverCtaBehavior(buttonAction.buttonBehavior), | |
callbacks = buttonAction.callbacks.map(_.map(convertCallback).toList), | |
clientEventInfo = Some(convertClientEventInfo(buttonAction.clientEventInfo)), | |
icon = buttonAction.icon.map(covertHorizonIcon), | |
buttonStyle = buttonAction.buttonStyle.map(covertButtonStyle) | |
) | |
private def convertCoverCtaBehavior( | |
behavior: onboardingthrift.ButtonBehavior | |
): CoverCtaBehavior = | |
behavior match | |
def covertButtonStyle(bStyle: onboardingthrift.CtaButtonStyle): ButtonStyle = | |
bStyle match | |
def covertHorizonIcon(icon: onboardingthrift.HorizonIcon): HorizonIcon = | |
icon match | |
def convertUrl(url: onboardingthrift.Url): Url | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\RelevancePromptConversions.scala | |
class to convert Relevance Prompt related onboarding thrift to product-mixer models | |
*/ | |
object RelevancePromptConversions | |
def convertContent( | |
candidate: onboardingthrift.RelevancePrompt | |
): RelevancePromptContent = | |
RelevancePromptContent( | |
displayType = convertDisplayType(candidate.displayType), | |
title = candidate.title.text, | |
confirmation = buildConfirmation(candidate), | |
isRelevantText = candidate.isRelevantButton.text, | |
notRelevantText = candidate.notRelevantButton.text, | |
isRelevantCallback = convertCallbacks(candidate.isRelevantButton.callbacks), | |
notRelevantCallback = convertCallbacks(candidate.notRelevantButton.callbacks), | |
isRelevantFollowUp = None, | |
notRelevantFollowUp = None | |
) | |
// Based on com.twitter.timelinemixer.injection.model.candidate.OnboardingRelevancePromptDisplayType#fromThrift | |
def convertDisplayType( | |
displayType: onboardingthrift.RelevancePromptDisplayType | |
): RelevancePromptDisplayType = | |
displayType match | |
def buildConfirmation(candidate: onboardingthrift.RelevancePrompt): String | |
def buttonToDismissFeedbackText(button: onboardingthrift.ButtonAction): Option[String] | |
def convertCallbacks(onboardingCallbacks: Option[Seq[onboardingthrift.Callback]]): Callback | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\flexible_injection_pipeline\TilesCarouselConversions.scala | |
object TilesCarouselConversions | |
def convertTile(tile: onboardingthrift.Tile, id: Long): TileItem | |
def convertTileBadge(badge: onboardingthrift.Badge): Badge | |
def convertModuleHeader(header: onboardingthrift.TilesCarouselHeader): ModuleHeader | |
def convertRosettaColor(color: onboardingthrift.RosettaColor): RosettaColor = | |
color match | |
class UnknownThriftEnumException(enumName: String) | |
extends Exception(s"Unknown Thrift Enum Found: $ | |
class UnsupportedTileCarouselConversionException(UnsupportedTileType: String) | |
extends Exception(s"Unsupported Tile Type Found: $ | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\icon\HorizonIconBuilder.scala | |
class HorizonIconBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
icon: HorizonIcon) | |
extends BaseHorizonIconBuilder[Query, Candidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\ad\AdsCandidateUrtItemBuilder.scala | |
class AdsCandidateUrtItemBuilder[Query <: PipelineQuery]( | |
tweetClientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, AdsTweetCandidate]] = None, | |
contextualTweetRefBuilder: Option[ContextualTweetRefBuilder[AdsTweetCandidate]] = None, | |
tweetDisplayType: TweetDisplayType = Tweet) | |
extends CandidateUrtEntryBuilder[Query, AdsCandidate, TimelineItem] | |
def apply( | |
pipelineQuery: Query, | |
candidate: AdsCandidate, | |
candidateFeatures: FeatureMap | |
): TimelineItem | |
def promotedMetadata(impression: adserver.AdImpression) | |
def convertDisclosureType( | |
disclosureType: adserver.DisclosureType | |
): DisclosureType = disclosureType match | |
def convertClickTrackingInfo( | |
clickTracking: adserver.ClickTrackingInfo | |
): ClickTrackingInfo = ClickTrackingInfo( | |
urlParams = clickTracking.urlParams.getOrElse(Map.empty), | |
urlOverride = clickTracking.urlOverride, | |
urlOverrideType = clickTracking.urlOverrideType.map | |
def prerollMetadata(adImpression: adserver.AdImpression): Option[PrerollMetadata] | |
def adMetadataContainer( | |
adImpression: adserver.AdImpression | |
): Option[AdMetadataContainer] | |
def convertSponsorshipType( | |
sponsorshipType: ads.SponsorshipType | |
): SponsorshipType = sponsorshipType match | |
def convertDisclaimerType( | |
disclaimerType: ads.DisclaimerType | |
): DisclaimerType = disclaimerType match | |
def convertDynamicPrerollType( | |
dynamicPrerollType: ads.DynamicPrerollType | |
): DynamicPrerollType = | |
dynamicPrerollType match | |
def convertMediaInfo(mediaInfo: ads.MediaInfo): MediaInfo | |
def convertVideoVariants(videoVariants: Seq[ads.VideoVariant]): Seq[VideoVariant] | |
def convertCallToAction(callToAction: ads.CallToAction): CallToAction | |
def convertPreroll( | |
preroll: ads.Preroll | |
): Preroll | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\DurationParamBuilder.scala | |
class DurationParamBuilder( | |
durationParam: Param[Duration]) | |
extends BaseDurationBuilder[PipelineQuery] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\ShowAlertCandidateUrtItemBuilder.scala | |
object ShowAlertCandidateUrtItemBuilder | |
class ShowAlertCandidateUrtItemBuilder[-Query <: PipelineQuery]( | |
alertType: ShowAlertType, | |
displayLocationBuilder: BaseShowAlertDisplayLocationBuilder[Query], | |
colorConfigBuilder: BaseShowAlertColorConfigurationBuilder[Query], | |
triggerDelayBuilder: Option[BaseDurationBuilder[Query]] = None, | |
displayDurationBuilder: Option[BaseDurationBuilder[Query]] = None, | |
clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, ShowAlertCandidate]] = None, | |
collapseDelayBuilder: Option[BaseDurationBuilder[Query]] = None, | |
userIdsBuilder: Option[BaseShowAlertUserIdsBuilder[Query]] = None, | |
richTextBuilder: Option[BaseRichTextBuilder[Query, ShowAlertCandidate]] = None, | |
iconDisplayInfoBuilder: Option[BaseShowAlertIconDisplayInfoBuilder[Query]] = None, | |
navigationMetadataBuilder: Option[BaseShowAlertNavigationMetadataBuilder[Query]] = None) | |
extends CandidateUrtEntryBuilder[ | |
Query, | |
ShowAlertCandidate, | |
ShowAlert | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\StaticShowAlertColorConfigurationBuilder.scala | |
class StaticShowAlertColorConfigurationBuilder[-Query <: PipelineQuery]( | |
configuration: ShowAlertColorConfiguration) | |
extends BaseShowAlertColorConfigurationBuilder[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\StaticShowAlertDisplayLocationBuilder.scala | |
class StaticShowAlertDisplayLocationBuilder[-Query <: PipelineQuery]( | |
location: ShowAlertDisplayLocation) | |
extends BaseShowAlertDisplayLocationBuilder[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\alert\StaticShowAlertIconDisplayInfoBuilder.scala | |
class StaticShowAlertIconDisplayInfoBuilder[-Query <: PipelineQuery]( | |
iconDisplayInfo: ShowAlertIconDisplayInfo) | |
extends BaseShowAlertIconDisplayInfoBuilder[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\article\ArticleCandidateUrtItemBuilder.scala | |
object ArticleCandidateUrtItemBuilder | |
class ArticleCandidateUrtItemBuilder[ | |
-Query <: PipelineQuery, | |
Candidate <: BaseArticleCandidate | |
]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate], | |
articleSeedType: ArticleSeedType = FollowingListSeed, | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, Candidate] | |
] = None, | |
displayType: Option[ArticleDisplayType] = None, | |
socialContextBuilder: Option[BaseSocialContextBuilder[Query, Candidate]] = None, | |
) extends CandidateUrtEntryBuilder[Query, Candidate, ArticleItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\audio_space\AudioSpaceCandidateUrtItemBuilder.scala | |
object AudioSpaceCandidateUrtItemBuilder | |
class AudioSpaceCandidateUrtItemBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UniversalNoun[Any]], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, UniversalNoun[Any]] | |
] = None) | |
extends CandidateUrtEntryBuilder[Query, AudioSpaceCandidate, AudioSpaceItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\card\CardCandidateUtrItemBuilder.scala | |
object CardCandidateUtrItemBuilder | |
class CardCandidateUtrItemBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CardCandidate], | |
cardUrlBuilder: BaseStr[Query, CardCandidate], | |
textBuilder: Option[BaseStr[Query, CardCandidate]], | |
subtextBuilder: Option[BaseStr[Query, CardCandidate]], | |
urlBuilder: Option[BaseUrlBuilder[Query, CardCandidate]], | |
cardDisplayType: Option[CardDisplayType], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, CardCandidate], | |
] = None) | |
extends CandidateUrtEntryBuilder[Query, CardCandidate, CardItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\commerce\CommerceProductCandidateUrtItemBuilder.scala | |
object CommerceProductCandidateUrtItemBuilder | |
class CommerceProductCandidateUrtItemBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CommerceProductCandidate], | |
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, CommerceProductCandidate]]) | |
extends CandidateUrtEntryBuilder[ | |
Query, | |
CommerceProductCandidate, | |
CommerceProductItem | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\commerce\CommerceProductGroupCandidateUrtItemBuilder.scala | |
object CommerceProductGroupCandidateUrtItemBuilder | |
class CommerceProductGroupCandidateUrtItemBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CommerceProductGroupCandidate], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, CommerceProductGroupCandidate] | |
]) extends CandidateUrtEntryBuilder[ | |
Query, | |
CommerceProductGroupCandidate, | |
CommerceProductGroupItem | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\event_summary\EventCandidateUrtItemBuilder.scala | |
object EventCandidateUrtItemBuilder | |
class EventCandidateUrtItemBuilder[Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UnifiedEventCandidate], | |
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, UnifiedEventCandidate]] = | |
None) | |
extends CandidateUrtEntryBuilder[Query, UnifiedEventCandidate, TimelineItem] | |
def apply( | |
query: Query, | |
candidate: UnifiedEventCandidate, | |
candidateFeatures: FeatureMap | |
): TimelineItem | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\generic_summary\GenericSummaryActionBuilder.scala | |
object GenericSummaryActionBuilder | |
class GenericSummaryActionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
urlBuilder: BaseUrlBuilder[Query, Candidate], | |
clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, Candidate]] = None) | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\generic_summary\GenericSummaryCandidateUrtItemBuilder.scala | |
object GenericSummaryCandidateUrtItemBuilder | |
class GenericSummaryCandidateUrtItemBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, GenericSummaryCandidate], | |
headlineRichTextBuilder: BaseRichTextBuilder[Query, GenericSummaryCandidate], | |
displayType: GenericSummaryItemDisplayType, | |
genericSummaryContextCandidateUrtItemBuilder: Option[ | |
GenericSummaryContextBuilder[Query, GenericSummaryCandidate] | |
] = None, | |
genericSummaryActionCandidateUrtItemBuilder: Option[ | |
GenericSummaryActionBuilder[Query, GenericSummaryCandidate] | |
] = None, | |
timestamp: Option[Time] = None, | |
userAttributionIds: Option[Seq[Long]] = None, | |
media: Option[Media] = None, | |
promotedMetadata: Option[PromotedMetadata] = None, | |
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, GenericSummaryCandidate]] = | |
None) | |
extends CandidateUrtEntryBuilder[Query, GenericSummaryCandidate, GenericSummaryItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\generic_summary\GenericSummaryContextBuilder.scala | |
class GenericSummaryContextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
richTextBuilder: BaseRichTextBuilder[Query, Candidate], | |
icon: Option[HorizonIcon] = None) | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\icon_label\IconLabelCandidateUrtItemBuilder.scala | |
object IconLabelCandidateUrtItemBuilder | |
class IconLabelCandidateUrtItemBuilder[-Query <: PipelineQuery, Candidate <: LabelCandidate]( | |
richTextBuilder: BaseRichTextBuilder[Query, Candidate], | |
icon: Option[HorizonIcon] = None, | |
entities: Option[List[RichTextEntity]] = None, | |
clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, Candidate]] = None, | |
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, Candidate]] = None) | |
extends CandidateUrtEntryBuilder[Query, Candidate, IconLabelItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\message\CompactPromptCandidateUrtItemStringCenterBuilder.scala | |
object CompactPromptCandidateUrtItemStringCenterBuilder | |
class CompactPromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CompactPromptCandidate], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, CompactPromptCandidate] | |
] = None, | |
headerTextBuilder: BaseStr[Query, CompactPromptCandidate], | |
bodyTextBuilder: Option[BaseStr[Query, CompactPromptCandidate]] = None, | |
headerRichTextBuilder: Option[BaseRichTextBuilder[Query, CompactPromptCandidate]] = None, | |
bodyRichTextBuilder: Option[BaseRichTextBuilder[Query, CompactPromptCandidate]] = None) | |
extends CandidateUrtEntryBuilder[Query, CompactPromptCandidate, MessagePromptItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\message\InlinePromptCandidateUrtItemStringCenterBuilder.scala | |
object InlinePromptCandidateUrtItemStringCenterBuilder | |
class InlinePromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, InlinePromptCandidate], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, InlinePromptCandidate] | |
] = None, | |
headerTextBuilder: BaseStr[Query, InlinePromptCandidate], | |
bodyTextBuilder: Option[BaseStr[Query, InlinePromptCandidate]] = None, | |
headerRichTextBuilder: Option[BaseRichTextBuilder[Query, InlinePromptCandidate]] = None, | |
bodyRichTextBuilder: Option[BaseRichTextBuilder[Query, InlinePromptCandidate]] = None, | |
primaryMessageTextActionBuilder: Option[ | |
MessageTextActionBuilder[Query, InlinePromptCandidate] | |
] = None, | |
secondaryMessageTextActionBuilder: Option[ | |
MessageTextActionBuilder[Query, InlinePromptCandidate] | |
] = None, | |
socialContextBuilder: Option[BaseSocialContextBuilder[Query, InlinePromptCandidate]] = None, | |
userFacePileBuilder: Option[ | |
UserFacePileBuilder | |
] = None) | |
extends CandidateUrtEntryBuilder[Query, InlinePromptCandidate, MessagePromptItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\message\MessageTextActionBuilder.scala | |
object MessageTextActionBuilder | |
class MessageTextActionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
textBuilder: BaseStr[Query, Candidate], | |
dismissOnClick: Boolean, | |
url: Option[String] = None, | |
clientEventInfo: Option[ClientEventInfo] = None, | |
onClickCallbacks: Option[List[Callback]] = None) | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\message\UserFacePileBuilder.scala | |
class UserFacePileBuilder( | |
userIds: Seq[Long], | |
featuredUserIds: Seq[Long], | |
action: Option[MessageTextAction], | |
actionType: Option[MessageActionType], | |
displaysFeaturingText: Option[Boolean], | |
displayType: Option[UserFacepileDisplayType]) | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\moment\MomentAnnotationCandidateUrtItemBuilder.scala | |
object MomentAnnotationCandidateUrtItemBuilder | |
class MomentAnnotationCandidateUrtItemBuilder[Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, MomentAnnotationCandidate], | |
annotationTextRichTextBuilder: BaseRichTextBuilder[Query, MomentAnnotationCandidate], | |
annotationHeaderRichTextBuilder: BaseRichTextBuilder[Query, MomentAnnotationCandidate], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, MomentAnnotationCandidate] | |
] = None, | |
) extends CandidateUrtEntryBuilder[Query, MomentAnnotationCandidate, MomentAnnotationItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\relevance_prompt\RelevancePromptCandidateUrtItemStringCenterBuilder.scala | |
object RelevancePromptCandidateUrtItemStringCenterBuilder | |
class RelevancePromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, RelevancePromptCandidate], | |
titleTextBuilder: BaseStr[Query, RelevancePromptCandidate], | |
confirmationTextBuilder: BaseStr[Query, RelevancePromptCandidate], | |
isRelevantTextBuilder: BaseStr[Query, RelevancePromptCandidate], | |
notRelevantTextBuilder: BaseStr[Query, RelevancePromptCandidate], | |
displayType: RelevancePromptDisplayType, | |
isRelevantCallback: Callback, | |
notRelevantCallback: Callback, | |
isRelevantFollowUp: Option[RelevancePromptFollowUpFeedbackType] = None, | |
notRelevantFollowUp: Option[RelevancePromptFollowUpFeedbackType] = None, | |
impressionCallbacks: Option[List[Callback]] = None) | |
extends CandidateUrtEntryBuilder[Query, RelevancePromptCandidate, PromptItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\suggestion\SpellingSuggestionCandidateUrtItemBuilder.scala | |
object SpellingSuggestionCandidateUrtItemBuilder | |
class SpellingSuggestionCandidateUrtItemBuilder[Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, SpellingSuggestionCandidate], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, SpellingSuggestionCandidate] | |
] = None, | |
) extends CandidateUrtEntryBuilder[Query, SpellingSuggestionCandidate, SpellingItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\tile\TileCandidateUrtItemBuilder.scala | |
object TileCandidateUrtItemBuilder | |
class TileCandidateUrtItemBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, PromptCarouselTileCandidate], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, PromptCarouselTileCandidate] | |
] = None) | |
extends CandidateUrtEntryBuilder[Query, PromptCarouselTileCandidate, TileItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\ParamTopicDisplayTypeBuilder.scala | |
object TopicCandidateDisplayType extends Enumeration | |
class ParamTopicDisplayTypeBuilder( | |
displayTypeParam: FSEnumParam[TopicCandidateDisplayType.type]) | |
extends BaseTopicDisplayTypeBuilder[PipelineQuery, TopicCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TopicCandidate, | |
candidateFeatures: FeatureMap | |
): Option[TopicDisplayType] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\ParamTopicFunctionalityTypeBuilder.scala | |
object TopicFunctionalityTypeParamValue extends Enumeration | |
class ParamTopicFunctionalityTypeBuilder( | |
functionalityTypeParam: FSEnumParam[TopicFunctionalityTypeParamValue.type]) | |
extends BaseTopicFunctionalityTypeBuilder[PipelineQuery, TopicCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TopicCandidate, | |
candidateFeatures: FeatureMap | |
): Option[TopicFunctionalityType] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\StaticTopicDisplayTypeBuilder.scala | |
class StaticTopicDisplayTypeBuilder( | |
displayType: TopicDisplayType) | |
extends BaseTopicDisplayTypeBuilder[PipelineQuery, BaseTopicCandidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\StaticTopicFunctionalityTypeBuilder.scala | |
class StaticTopicFunctionalityTypeBuilder( | |
functionalityType: TopicFunctionalityType) | |
extends BaseTopicFunctionalityTypeBuilder[PipelineQuery, BaseTopicCandidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\TopicCandidateUrtItemBuilder.scala | |
object TopicCandidateUrtItemBuilder | |
class TopicCandidateUrtItemBuilder[-Query <: PipelineQuery, Candidate <: BaseTopicCandidate]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate], | |
topicFunctionalityTypeBuilder: Option[BaseTopicFunctionalityTypeBuilder[Query, Candidate]] = None, | |
topicDisplayTypeBuilder: Option[BaseTopicDisplayTypeBuilder[Query, Candidate]] = None, | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, Candidate] | |
] = None) | |
extends CandidateUrtEntryBuilder[Query, Candidate, TopicItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\topic\VerticalGridTopicCandidateUrtItemBuilder.scala | |
class VerticalGridTopicCandidateUrtItemBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, TopicCandidate], | |
verticalGridItemTopicFunctionalityType: VerticalGridItemTopicFunctionalityType, | |
verticalGridItemTileStyle: VerticalGridItemTileStyle, | |
urlBuilder: Option[BaseUrlBuilder[Query, TopicCandidate]] = None, | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, TopicCandidate] | |
] = None) | |
extends CandidateUrtEntryBuilder[Query, TopicCandidate, VerticalGridItem] | |
def apply( | |
query: Query, | |
topicCandidate: TopicCandidate, | |
candidateFeatures: FeatureMap | |
): VerticalGridItem | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\trend\TrendCandidateUrtItemBuilder.scala | |
object TrendCandidateUrtItemBuilder | |
class TrendCandidateUrtItemBuilder[Query <: PipelineQuery]( | |
trendMetaDescriptionBuilder: TrendMetaDescriptionBuilder[Query, UnifiedTrendCandidate], | |
promotedMetadataBuilder: BasePromotedMetadataBuilder[Query, UnifiedTrendCandidate], | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UnifiedTrendCandidate], | |
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, UnifiedTrendCandidate]] = | |
None) | |
extends CandidateUrtEntryBuilder[Query, UnifiedTrendCandidate, TimelineItem] | |
def apply( | |
query: Query, | |
candidate: UnifiedTrendCandidate, | |
candidateFeatures: FeatureMap | |
): TimelineItem | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\trend\TrendMetaDescriptionBuilder.scala | |
class TrendMetaDescriptionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
promotedByMetaDescriptionStr: Str[PipelineQuery, UniversalNoun[Any]], | |
tweetCountMetaDescriptionStr: Str[PipelineQuery, UniversalNoun[Any]], | |
compactingNumberLocalizer: CompactingNumberLocalizer) | |
def apply( | |
query: Query, | |
candidate: Candidate, | |
candidateFeatures: FeatureMap | |
): Option[String] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\trend\TrendPromotedMetadataBuilder.scala | |
object TrendPromotedMetadataBuilder | |
extends BasePromotedMetadataBuilder[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidate: UniversalNoun[Any], | |
candidateFeatures: FeatureMap | |
): Option[PromotedMetadata] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\tweet\TweetCandidateUrtItemBuilder.scala | |
object TweetCandidateUrtItemBuilder | |
class TweetCandidateUrtItemBuilder[Query <: PipelineQuery, Candidate <: BaseTweetCandidate]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate], | |
displayType: TweetDisplayType = Tweet, | |
entryIdToReplaceBuilder: Option[BaseEntryIdToReplaceBuilder[Query, Candidate]] = None, | |
socialContextBuilder: Option[BaseSocialContextBuilder[Query, Candidate]] = None, | |
highlightsBuilder: Option[BaseTweetHighlightsBuilder[Query, Candidate]] = None, | |
innerTombstoneInfo: Option[TombstoneInfo] = None, | |
timelinesScoreInfoBuilder: Option[BaseTimelinesScoreInfoBuilder[Query, Candidate]] = None, | |
hasModeratedReplies: Option[Boolean] = None, | |
forwardPivot: Option[ForwardPivot] = None, | |
innerForwardPivot: Option[ForwardPivot] = None, | |
feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, Candidate]] = None, | |
promotedMetadata: Option[PromotedMetadata] = None, | |
conversationAnnotation: Option[ConversationAnnotation] = None, | |
contextualTweetRefBuilder: Option[ContextualTweetRefBuilder[Candidate]] = None, | |
prerollMetadata: Option[PrerollMetadata] = None, | |
replyBadge: Option[Badge] = None, | |
destinationBuilder: Option[BaseUrlBuilder[Query, Candidate]] = None) | |
extends CandidateUrtEntryBuilder[Query, Candidate, TweetItem] | |
def apply( | |
pipelineQuery: Query, | |
tweetCandidate: Candidate, | |
candidateFeatures: FeatureMap | |
): TweetItem | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\twitter_list\TwitterListCandidateUrtItemBuilder.scala | |
object TwitterListCandidateUrtItemBuilder | |
class TwitterListCandidateUrtItemBuilder[-Query <: PipelineQuery]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, TwitterListCandidate], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, TwitterListCandidate] | |
] = None, | |
displayType: Option[TwitterListDisplayType] = None) | |
extends CandidateUrtEntryBuilder[Query, TwitterListCandidate, TwitterListItem] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\unified_trend_event\UnifiedTrendEventCandidateUrtItemBuilder.scala | |
class UnifiedTrendEventCandidateUrtItemBuilder[Query <: PipelineQuery]( | |
eventCandidateUrtItemBuilder: EventCandidateUrtItemBuilder[Query], | |
trendCandidateUrtItemBuilder: TrendCandidateUrtItemBuilder[Query]) | |
extends CandidateUrtEntryBuilder[Query, UnifiedTrendEventCandidate[Any], TimelineItem] | |
def apply( | |
query: Query, | |
candidate: UnifiedTrendEventCandidate[Any], | |
candidateFeatures: FeatureMap | |
): TimelineItem | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\item\user\UserCandidateUrtItemBuilder.scala | |
object UserCandidateUrtItemBuilder | |
class UserCandidateUrtItemBuilder[Query <: PipelineQuery, UserCandidate <: BaseUserCandidate]( | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UserCandidate], | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, UserCandidate] | |
] = None, | |
displayType: UserDisplayType = User, | |
promotedMetadataBuilder: Option[BasePromotedMetadataBuilder[Query, UserCandidate]] = None, | |
socialContextBuilder: Option[BaseSocialContextBuilder[Query, UserCandidate]] = None, | |
reactiveTriggersBuilder: Option[BaseUserReactiveTriggersBuilder[Query, UserCandidate]] = None, | |
enableReactiveBlending: Option[Boolean] = None) | |
extends CandidateUrtEntryBuilder[Query, UserCandidate, UserItem] | |
def apply( | |
query: Query, | |
userCandidate: UserCandidate, | |
candidateFeatures: FeatureMap | |
): UserItem | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\ClientEventInfoBuilder.scala | |
class ClientEventInfoBuilder[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( | |
component: String, | |
detailsBuilder: Option[BaseClientEventDetailsBuilder[Query, Candidate]] = None) | |
extends BaseClientEventInfoBuilder[Query, Candidate] | |
def apply( | |
query: Query, | |
candidate: Candidate, | |
candidateFeatures: FeatureMap, | |
element: Option[String] | |
): Option[ClientEventInfo] = | |
Some( | |
ClientEventInfo( | |
component = Some(component), | |
element = element, | |
details = detailsBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), | |
action = None, | |
entityToken = None) | |
) | |
} | |
/** | |
* In rare cases you might not want to send client event info. For | |
* example, this might be set already on the client for some legacy | |
* timelines. | |
*/ | |
object EmptyClientEventInfoBuilder | |
extends BaseClientEventInfoBuilder[PipelineQuery, UniversalNoun[Any]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\ConversationTweetClientEventDetailsBuilder.scala | |
object ConversationTweetClientEventDetailsBuilder | |
class ConversationTweetClientEventDetailsBuilder[-Query <: PipelineQuery]( | |
injectionType: Option[String]) | |
extends BaseClientEventDetailsBuilder[Query, BaseTweetCandidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\StaticUrlBuilder.scala | |
class StaticUrlBuilder(url: String, urlType: UrlType) | |
extends BaseUrlBuilder[PipelineQuery, UniversalNoun[Any]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\TopicClientEventDetailsBuilder.scala | |
object TopicClientEventDetailsBuilder | |
class TopicClientEventDetailsBuilder[-Query <: PipelineQuery]() | |
extends BaseClientEventDetailsBuilder[Query, BaseTopicCandidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\TopicNotInterestedFeedbackActionInfoBuilder.scala | |
class TopicNotInterestedFeedbackActionInfoBuilder[-Query <: PipelineQuery]() | |
extends BaseFeedbackActionInfoBuilder[Query, BaseTopicCandidate] | |
def apply( | |
query: Query, | |
topicCandidate: BaseTopicCandidate, | |
candidateFeatures: FeatureMap | |
): Option[FeedbackActionInfo] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\TopicsToFollowModuleMetadataBuilder.scala | |
object TopicsToFollowModuleMetadataBuilder | |
def getCarouselRowCount(topicsCount: Int, maxCarouselRows: Int): Int = | |
Math.min(maxCarouselRows, (topicsCount / TopicsPerRow) + 1) | |
} | |
case class TopicsToFollowModuleMetadataBuilder(maxCarouselRowsParam: Param[Int]) | |
extends BaseModuleMetadataBuilder[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[UniversalNoun[Any]]] | |
): ModuleMetadata | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\TopicTweetClientEventDetailsBuilder.scala | |
object TopicTweetClientEventDetailsBuilder | |
class TopicTweetClientEventDetailsBuilder[-Query <: PipelineQuery]() | |
extends BaseClientEventDetailsBuilder[Query, TweetCandidate] | |
def apply( | |
query: Query, | |
topicTweetCandidate: TweetCandidate, | |
candidateFeatures: FeatureMap | |
): Option[ClientEventDetails] = | |
Some( | |
ClientEventDetails( | |
conversationDetails = None, | |
timelinesDetails = Some( | |
TimelinesDetails( | |
injectionType = None, | |
controllerData = buildControllerData(getTopicId(query)), | |
sourceData = None)), | |
articleDetails = None, | |
liveEventDetails = None, | |
commerceDetails = None | |
)) | |
private def getTopicId(query: Query): Option[Long] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\metadata\WhoToFollowFeedbackActionInfoBuilder.scala | |
class WhoToFollowFeedbackActionInfoBuilder[ | |
-Query <: PipelineQuery, | |
-Candidate <: UniversalNoun[Any] | |
]( | |
externalStringRegistry: ExternalStringRegistry, | |
stringCenter: StringCenter, | |
encodedFeedbackRequest: Option[String]) | |
extends BaseFeedbackActionInfoBuilder[Query, Candidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\operation\CursorCandidateUrtOperationBuilder.scala | |
class CursorCandidateUrtOperationBuilder[-Query <: PipelineQuery]( | |
cursorType: CursorType, | |
displayTreatment: Option[CursorDisplayTreatment] = None, | |
idToReplace: Option[Long] = None) | |
extends CandidateUrtEntryBuilder[Query, CursorCandidate, CursorOperation] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\promoted\FeaturePromotedMetadataBuilder.scala | |
class FeaturePromotedMetadataBuilder(adImpressionFeature: Feature[_, Option[ad.AdImpression]]) | |
extends BasePromotedMetadataBuilder[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidate: UniversalNoun[Any], | |
candidateFeatures: FeatureMap | |
): Option[PromotedMetadata] | |
def convertAdMetadataContainer( | |
serializedAdMetadataContainer: ac.SerializedThrift | |
): Option[AdMetadataContainer] = | |
AdMetadataContainerSerializer.deserialize(serializedAdMetadataContainer).map | |
def convertDisclosureType(disclosureType: ad.DisclosureType): DisclosureType = | |
disclosureType match | |
def convertSponsorshipType(sponsorshipType: ads.SponsorshipType): SponsorshipType = | |
sponsorshipType match | |
def convertDisclaimerType(disclaimerType: ads.DisclaimerType): DisclaimerType = | |
disclaimerType match | |
def convertSkAdNetworkDataList( | |
skAdNetworkDataList: Seq[ads.SkAdNetworkData] | |
): Seq[SkAdNetworkData] = skAdNetworkDataList.map | |
def convertClickTrackingInfo(clickTracking: ad.ClickTrackingInfo): ClickTrackingInfo = | |
ClickTrackingInfo( | |
urlParams = clickTracking.urlParams.getOrElse(Map.empty), | |
urlOverride = clickTracking.urlOverride, | |
urlOverrideType = clickTracking.urlOverrideType.map | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\RichTextBuilder.scala | |
class RichTextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
textBuilder: BaseStr[Query, Candidate], | |
linkMap: Map[String, String], | |
rtl: Option[Boolean], | |
alignment: Option[RichTextAlignment], | |
linkTypeMap: Map[String, UrlType] = Map.empty) | |
extends BaseRichTextBuilder[Query, Candidate] | |
def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): RichText | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\RichTextMarkupUtil.scala | |
object out of | |
* a string with inline XML markup. | |
* | |
* This allows us to use a string like "Our system <a href="#promix">Product Mixer</a> is the <b>best</b>". Using | |
* inline markup like this is advantageous since the string can go through translation/localization and the | |
* translators will move the tags around in each language as appropriate. | |
* | |
* This class is derived from the OCF (onboarding/serve)'s RichTextUtil, but they diverge because: | |
* - We generate ProMix URT structures, not OCF URT structures | |
* - The OCF supports some internal OCF tags, like <data> | |
* - The OCF has additional legacy support and processing that we don't need | |
*/ | |
object RichTextMarkupUtil | |
def richTextFromMarkup( | |
text: String, | |
linkMap: Map[String, String], | |
rtl: Option[Boolean] = None, | |
alignment: Option[RichTextAlignment] = None, | |
linkTypeMap: Map[String, UrlType] = Map.empty | |
): RichText | |
def adjustEntities( | |
entities: scala.collection.mutable.ArrayBuffer[RichTextEntity], | |
start: Int, | |
length: Int | |
): Unit | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\RichTextReferenceObjectBuilder.scala | |
trait RichTextReferenceObjectBuilder | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\RichTextRtlOptionBuilder.scala | |
trait RichTextRtlOptionBuilder[-Query <: PipelineQuery] | |
def apply(query: Query): Option[Boolean] | |
} | |
case class StaticRichTextRtlOptionBuilder[-Query <: PipelineQuery](rtlOption: Option[Boolean]) | |
extends RichTextRtlOptionBuilder[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\StaticRichTextBuilder.scala | |
class StaticRichTextBuilder(richText: RichText) | |
extends BaseRichTextBuilder[PipelineQuery, UniversalNoun[Any]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextEntityProcessor.scala | |
object TwitterTextEntityProcessor | |
object DefaultReferenceObjectBuilder extends RichTextReferenceObjectBuilder | |
def apply(twitterEntity: Extractor.Entity): Option[ReferenceObject] | |
class TwitterTextEntityProcessor( | |
twitterTextReferenceObjectBuilder: RichTextReferenceObjectBuilder = DefaultReferenceObjectBuilder) | |
extends TwitterTextRendererProcessor | |
def process( | |
twitterTextRenderer: TwitterTextRenderer | |
): TwitterTextRenderer | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextFormatProcessor.scala | |
object TwitterTextFormatProcessor | |
class TwitterTextFormatProcessor( | |
formats: Set[RichTextFormat] = Set(Plain, Strong), | |
) extends TwitterTextRendererProcessor | |
def renderText(text: String): RichText | |
def process(richTextBuilder: TwitterTextRenderer): TwitterTextRenderer | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextRenderer.scala | |
object TwitterTextRenderer | |
def apply( | |
text: String, | |
rtl: Option[Boolean] = None, | |
alignment: Option[RichTextAlignment] = None | |
): TwitterTextRenderer | |
def fromRichText(richText: RichText): TwitterTextRenderer | |
def buildRichTextEntity( | |
text: String, | |
entity: TwitterTextRendererEntity[_] | |
): RichTextEntity | |
class TwitterTextRenderer( | |
rtl: Option[Boolean], | |
alignment: Option[RichTextAlignment], | |
) | |
def append( | |
string: String, | |
format: Option[RichTextFormat] = None, | |
refObject: Option[ReferenceObject] = None | |
): TwitterTextRenderer | |
def build: RichText | |
def transform(twitterTextProcessor: TwitterTextRendererProcessor): TwitterTextRenderer | |
def entities: Seq[TwitterTextRendererEntity[_]] | |
def mergeFormat(start: Int, end: Int, format: RichTextFormat): TwitterTextRenderer | |
def remove(start: Int, end: Int): TwitterTextRenderer = replace(start, end, "") | |
/** | |
* Replaces text, formatting, and refObject information in the given range. | |
* @param start Start index to apply formatting (inclusive) | |
* @param end End index to apply formatting (exclusive) | |
* @param string The new text to insert | |
* @param format The [[RichTextFormat]] assigned to the new text | |
* @param refObject The [[ReferenceObject]] assigned to the new text | |
* @return this | |
*/ | |
def replace( | |
start: Int, | |
end: Int, | |
string: String, | |
format: Option[RichTextFormat] = None, | |
refObject: Option[ReferenceObject] = None | |
): TwitterTextRenderer | |
def setFormat(start: Int, end: Int, format: RichTextFormat): TwitterTextRenderer | |
def removeAndOffsetFormats(start: Int, end: Int, newSize: Int): Int | |
def validateRange(start: Int, end: Int): Unit | |
def setRefObject(start: Int, end: Int, refObject: ReferenceObject): TwitterTextRenderer | |
def removeAndOffsetRefObjects(start: Int, end: Int, newSize: Int): Int | |
def text: String | |
def buildEntities( | |
formats: List[TwitterTextRendererEntity[RichTextFormat]], | |
refs: List[TwitterTextRendererEntity[ReferenceObject]], | |
acc: List[TwitterTextRendererEntity[_]] = List() | |
): Seq[TwitterTextRendererEntity[_]] | |
class TwitterTextRendererEntity[+T] private[richtext] ( | |
startIndex: Int, | |
endIndex: Int, | |
value: T) | |
def nonEmpty: Boolean = !isEmpty | |
def isEmpty: Boolean = startIndex == endIndex | |
private[richtext] def enclosedIn(start: Int, end: Int): Boolean | |
def encloses(start: Int, end: Int): Boolean | |
def endsBetween(start: Int, end: Int): Boolean | |
def offset(num: Int): TwitterTextRendererEntity[T] | |
def startsBetween(start: Int, end: Int): Boolean | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextRendererProcessor.scala | |
trait TwitterTextRendererProcessor | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\richtext\twitter_text\TwitterTextRichTextBuilder.scala | |
class TwitterTextRichTextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
stringBuilder: BaseStr[Query, Candidate], | |
alignment: Option[RichTextAlignment] = None, | |
formats: Set[RichTextFormat] = Set(Plain, Strong), | |
twitterTextRtlOptionBuilder: RichTextRtlOptionBuilder[Query] = | |
StaticRichTextRtlOptionBuilder[Query](None), | |
twitterTextReferenceObjectBuilder: RichTextReferenceObjectBuilder = DefaultReferenceObjectBuilder) | |
extends BaseRichTextBuilder[Query, Candidate] | |
def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): RichText | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\social_context\FeatureSocialContextBuilder.scala | |
class FeatureSocialContextBuilder( | |
socialContextFeature: Feature[_, Option[t.SocialContext]]) | |
extends BaseSocialContextBuilder[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidate: UniversalNoun[Any], | |
candidateFeatures: FeatureMap | |
): Option[SocialContext] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\social_context\GeneralModuleSocialContextBuilder.scala | |
class works the same as [[GeneralSocialContextBuilder]] but passes a list of candidates | |
* into [[BaseModuleStr]] when rendering the string. | |
*/ | |
case class GeneralModuleSocialContextBuilder[ | |
-Query <: PipelineQuery, | |
-Candidate <: UniversalNoun[Any] | |
]( | |
textBuilder: BaseModuleStr[Query, Candidate], | |
contextType: GeneralContextType, | |
url: Option[String] = None, | |
contextImageUrls: Option[List[String]] = None, | |
landingUrl: Option[Url] = None) | |
extends BaseModuleSocialContextBuilder[Query, Candidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\social_context\GeneralSocialContextBuilder.scala | |
class GeneralSocialContextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
textBuilder: BaseStr[Query, Candidate], | |
contextType: GeneralContextType, | |
url: Option[String] = None, | |
contextImageUrls: Option[List[String]] = None, | |
landingUrl: Option[Url] = None) | |
extends BaseSocialContextBuilder[Query, Candidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\social_context\WhoToFollowSocialContextBuilder.scala | |
class WhoToFollowSocialContextBuilder( | |
socialTextFeature: Feature[_, Option[String]], | |
contextTypeFeature: Feature[_, Option[h.ContextType]]) | |
extends BaseSocialContextBuilder[PipelineQuery, UserCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: UserCandidate, | |
candidateFeatures: FeatureMap | |
): Option[GeneralContext] | |
def convertContextType(contextType: Option[h.ContextType]): Option[GeneralContextType] = | |
contextType match | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\stringcenter\ModuleStr.scala | |
class works the same as [[Str]] but passes in a list of candidates to the | |
* [[BaseModuleStringCenterPlaceholderBuilder]] when building the placeholders. | |
*/ | |
case class ModuleStr[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
text: ExternalString, | |
stringCenter: StringCenter, | |
stringCenterPlaceholderBuilder: Option[ | |
BaseModuleStringCenterPlaceholderBuilder[Query, Candidate] | |
] = None) | |
extends BaseModuleStr[Query, Candidate] | |
def apply(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): String | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\stringcenter\Str.scala | |
class StrStatic( | |
text: String) | |
extends BaseStr[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidate: UniversalNoun[Any], | |
candidateFeatures: FeatureMap | |
): String = text | |
} | |
case class Str[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
text: ExternalString, | |
stringCenter: StringCenter, | |
stringCenterPlaceholderBuilder: Option[BaseStringCenterPlaceholderBuilder[Query, Candidate]] = | |
None) | |
extends BaseStr[Query, Candidate] | |
def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): String | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\FeatureModuleDisplayTypeBuilder.scala | |
class FeatureModuleDisplayTypeBuilder( | |
displayTypeFeature: Feature[_, Option[ModuleDisplayType]], | |
defaultDisplayType: ModuleDisplayType = VerticalConversation) | |
extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleDynamicShowMoreBehaviorRevealByCountBuilder.scala | |
class ModuleDynamicShowMoreBehaviorRevealByCountBuilder( | |
initialItemsCount: Int, | |
showMoreItemsCount: Int) | |
extends BaseModuleShowMoreBehaviorBuilder[PipelineQuery, UniversalNoun[Any]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleFooterBuilder.scala | |
class ModuleFooterBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
textBuilder: BaseStr[Query, Candidate], | |
urlBuilder: Option[BaseUrlBuilder[Query, Candidate]]) | |
extends BaseModuleFooterBuilder[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Option[ModuleFooter] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleHeaderBuilder.scala | |
class ModuleHeaderBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
textBuilder: BaseStr[Query, Candidate], | |
isSticky: Option[Boolean] = None, | |
moduleHeaderIconBuilder: Option[BaseHorizonIconBuilder[Query, Candidate]] = None, | |
customIcon: Option[ImageVariant] = None, | |
moduleSocialContextBuilder: Option[BaseModuleSocialContextBuilder[Query, Candidate]] = None, | |
moduleHeaderDisplayTypeBuilder: BaseModuleHeaderDisplayTypeBuilder[Query, Candidate] = | |
ModuleHeaderDisplayTypeBuilder(Classic)) | |
extends BaseModuleHeaderBuilder[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Option[ModuleHeader] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleHeaderDisplayTypeBuilder.scala | |
class ModuleHeaderDisplayTypeBuilder[ | |
-Query <: PipelineQuery, | |
-Candidate <: UniversalNoun[Any] | |
]( | |
moduleHeaderDisplayType: ModuleHeaderDisplayType = Classic) | |
extends BaseModuleHeaderDisplayTypeBuilder[Query, Candidate] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleIdGeneration.scala | |
trait is used for Module ID generation. Clients are safe to ignore this code unless they | |
* have a specific use case that requires hard-coded, specific, module ids. In that scenario, | |
* they can use the [[ManualModuleId]] case class. | |
*/ | |
sealed trait ModuleIdGeneration | |
object ModuleIdGeneration | |
def apply(moduleId: Long): ModuleIdGeneration = moduleId match | |
class AutomaticUniqueModuleId private (moduleId: Long = 0L) extends ModuleIdGeneration | |
def withOffset(offset: Long): AutomaticUniqueModuleId = copy( | |
AutomaticUniqueModuleId.idRange.min + offset) | |
} | |
object AutomaticUniqueModuleId | |
def apply(): AutomaticUniqueModuleId = AutomaticUniqueModuleId(idRange.min) | |
def isAutomaticUniqueModuleId(moduleId: Long): Boolean = idRange.contains(moduleId) | |
} | |
/** | |
* ManualModuleId should normally not be required, but is helpful if the | |
* entryId of the module must be controlled. A scenario where this may be | |
* required is if a single candidate source returns multiple modules, and | |
* each module has the same presentation (e.g. Header, Footer). By setting | |
* different IDs, we signal to the platform that each module should be separate | |
* by using a different manual Id. | |
*/ | |
case class ManualModuleId(override val moduleId: Long) extends ModuleIdGeneration | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ModuleShowMoreBehaviorRevealByCountBuilder.scala | |
class ModuleShowMoreBehaviorRevealByCountBuilder( | |
initialItemsCountParam: Param[Int], | |
showMoreItemsCountParam: Param[Int]) | |
extends BaseModuleShowMoreBehaviorBuilder[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidate: Seq[CandidateWithFeatures[UniversalNoun[Any]]] | |
): ModuleShowMoreBehavior | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ParamGatedModuleFooterBuilder.scala | |
class ParamGatedModuleFooterBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
enableParam: Param[Boolean], | |
enabledBuilder: BaseModuleFooterBuilder[Query, Candidate], | |
defaultBuilder: Option[BaseModuleFooterBuilder[Query, Candidate]] = None) | |
extends BaseModuleFooterBuilder[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Option[ModuleFooter] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ParamGatedModuleHeaderBuilder.scala | |
class ParamGatedModuleHeaderBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
enableParam: Param[Boolean], | |
enabledBuilder: BaseModuleHeaderBuilder[Query, Candidate], | |
defaultBuilder: Option[BaseModuleHeaderBuilder[Query, Candidate]] = None) | |
extends BaseModuleHeaderBuilder[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Option[ModuleHeader] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\ParamWhoToFollowModuleDisplayTypeBuilder.scala | |
object WhoToFollowModuleDisplayType extends Enumeration | |
class ParamWhoToFollowModuleDisplayTypeBuilder( | |
displayTypeParam: Param[WhoToFollowModuleDisplayType.Value] = | |
StaticParam(WhoToFollowModuleDisplayType.Vertical)) | |
extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[UniversalNoun[Any]]] | |
): ModuleDisplayType | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\StaticModuleDisplayTypeBuilder.scala | |
class StaticModuleDisplayTypeBuilder(displayType: ModuleDisplayType) | |
extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\decorator\urt\builder\timeline_module\TimelineModuleBuilder.scala | |
class TimelineModuleBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( | |
entryNamespace: EntryNamespace, | |
displayTypeBuilder: BaseModuleDisplayTypeBuilder[Query, Candidate], | |
clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate], | |
moduleIdGeneration: ModuleIdGeneration = AutomaticUniqueModuleId(), | |
feedbackActionInfoBuilder: Option[ | |
BaseFeedbackActionInfoBuilder[Query, Candidate] | |
] = None, | |
headerBuilder: Option[BaseModuleHeaderBuilder[Query, Candidate]] = None, | |
footerBuilder: Option[BaseModuleFooterBuilder[Query, Candidate]] = None, | |
metadataBuilder: Option[BaseModuleMetadataBuilder[Query, Candidate]] = None, | |
showMoreBehaviorBuilder: Option[BaseModuleShowMoreBehaviorBuilder[Query, Candidate]] = None) | |
extends BaseTimelineModuleBuilder[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): TimelineModule | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\MetricDefinitions.scala | |
object MetricDefinition | |
class for all metric definitions | |
*/ | |
sealed trait MetricDefinition | |
def toCsvField: Seq[String] | |
val metricDefinitionType: String | |
} | |
/** | |
* Pattern Metric Definition | |
* @param pattern the regex pattern for this metric | |
*/ | |
case class NamedPatternMetricDefinition( | |
pattern: Seq[String]) | |
extends MetricDefinition | |
def toCsvField: Seq[String] = pattern | |
override val metricDefinitionType: String = "NAMED_PATTERN" | |
} | |
/** | |
* Strainer Metric Definition | |
* @param strainerExpression a filter on top of client events | |
*/ | |
case class StrainerMetricDefinition( | |
strainerExpression: String) | |
extends MetricDefinition | |
def toCsvField: Seq[String] | |
class LambdaMetricDefinition( | |
lambdaExpression: String) | |
extends MetricDefinition | |
def toCsvField: Seq[String] | |
class BucketRatioMetricDefinition( | |
numerator: String, | |
denominator: String) | |
extends MetricDefinition | |
def toCsvField: Seq[String] | |
object Metric | |
def fromLine(line: String): Metric | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\MetricGroup.scala | |
class MetricGroup( | |
id: Option[Long], | |
name: String, | |
description: String, | |
metrics: ListSet[Metric]) | |
def toCsv: String | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\MetricTemplateCLIRunner.scala | |
class MetricTemplateCLIConfig( | |
// default values required for OptionParser | |
templateFileName: String = null, | |
outputFileName: String = null, | |
metricGroupName: String = null, | |
metricGroupDesc: String = null, | |
metricGroupId: Option[Long] = None, | |
absolutePath: Option[String] = None) | |
trait MetricTemplateCLIRunner | |
def templateDir: String | |
def placeholders: PlaceholdersMap | |
private val ProgramName = "Metric Template CLI" | |
private val VersionNumber = "1.0" | |
private def mkPath(fileName: String, absolutePath: Option[String]): String | |
def main(args: Array[String]): Unit | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\MetricTemplates.scala | |
class MatchedPlaceholder(outerKey: String, innerKey: Option[String] = None) | |
object MetricTemplates | |
def interpolate(inputTemplate: String, placeholders: PlaceholdersMap): Seq[String] | |
def getMatchedPlaceholders(inputTemplate: String): Seq[MatchedPlaceholder] | |
def unwrap(str: String): String = | |
str.stripPrefix("$ | |
def wrap(str: String): String = | |
"\\$\\ | |
def getPlaceholderKeyValues( | |
groupedPlaceholders: Map[String, Seq[MatchedPlaceholder]], | |
placeholders: PlaceholdersMap | |
): Seq[(String, Seq[Named])] | |
def crossProduct[T](seqOfSeqOfItems: Seq[Seq[T]]): Seq[List[T]] | |
def generateTemplateKey(matched: MatchedPlaceholder): String | |
def getFieldValue[T: ClassTag]( | |
mirror: Mirror, | |
cls: T, | |
accessors: Map[String, MethodSymbol], | |
fieldName: String | |
): String | |
def caseAccessors[T: ClassTag](mirror: Mirror, cls: T): Map[String, MethodSymbol] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\experiments\metrics\PlaceholderConfig.scala | |
trait for all placeholder values | |
sealed trait Named | |
def name: String | |
} | |
case class Const(override val name: String) extends Named | |
// contains only client event patterns | |
case class CEPattern( | |
override val name: String, | |
client: String = "", | |
page: String = "", | |
section: String = "", | |
component: String = "", | |
element: String = "", | |
action: String = "", | |
strainer: String = "") | |
extends Named | |
def toString: String | |
class Topic( | |
override val name: String, | |
topicId: String = "") | |
extends Named | |
object PlaceholderConfig | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1QueryUserIdFeature.scala | |
object FeatureStoreV1QueryUserIdFeature | |
def apply[Query <: PipelineQuery, Value]( | |
feature: FSv1Feature[UserId, Value], | |
legacyName: Option[String] = None, | |
defaultValue: Option[Value] = None, | |
enabledParam: Option[FSParam[Boolean]] = None | |
): FeatureStoreV1Feature[Query, Query, _ <: EntityId, Value] | |
with FeatureStoreV1QueryFeature[Query, _ <: EntityId, Value] = | |
FeatureStoreV1QueryFeature(feature, QueryUserIdEntity, legacyName, defaultValue, enabledParam) | |
} | |
object FeatureStoreV1QueryUserIdAggregateFeature | |
def apply[Query <: PipelineQuery]( | |
featureGroup: TimelinesAggregationFrameworkFeatureGroup[UserId], | |
enabledParam: Option[FSParam[Boolean]] = None, | |
keepLegacyNames: Boolean = false, | |
featureNameTransform: Option[FeatureRenameTransform] = None | |
): FeatureStoreV1QueryFeatureGroup[Query, _ <: EntityId] = | |
FeatureStoreV1QueryFeatureGroup( | |
featureGroup, | |
QueryUserIdEntity, | |
enabledParam, | |
keepLegacyNames, | |
featureNameTransform)((implicitly[ClassTag[UserId]])) | |
} | |
object QueryUserIdEntity extends FeatureStoreV1QueryEntity[PipelineQuery, UserId] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.scala | |
object FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature | |
def apply[Query <: PipelineQuery, Value]( | |
feature: FSv1Feature[EdgeEntityId[UserId, UserId], Value], | |
legacyName: Option[String] = None, | |
defaultValue: Option[Value] = None, | |
enabledParam: Option[FSParam[Boolean]] = None | |
): FeatureStoreV1CandidateFeature[Query, TweetCandidate, _ <: EntityId, Value] = | |
FeatureStoreV1CandidateFeature( | |
feature, | |
QueryUserIdTweetCandidateAuthorIdEntity, | |
legacyName, | |
defaultValue, | |
enabledParam) | |
} | |
object FeatureStoreV1QueryUserIdTweetCandidateAuthorIdAggregateFeature | |
def apply[Query <: PipelineQuery]( | |
featureGroup: TimelinesAggregationFrameworkFeatureGroup[EdgeEntityId[UserId, UserId]], | |
enabledParam: Option[FSParam[Boolean]] = None, | |
keepLegacyNames: Boolean = false, | |
featureNameTransform: Option[FeatureRenameTransform] = None | |
): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] = | |
FeatureStoreV1CandidateFeatureGroup( | |
featureGroup, | |
QueryUserIdTweetCandidateAuthorIdEntity, | |
enabledParam, | |
keepLegacyNames, | |
featureNameTransform | |
)(implicitly[ClassTag[EdgeEntityId[UserId, UserId]]]) | |
} | |
object QueryUserIdTweetCandidateAuthorIdEntity | |
extends FeatureStoreV1CandidateEntity[ | |
PipelineQuery, | |
TweetCandidate, | |
EdgeEntityId[UserId, UserId] | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.scala | |
object FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature | |
def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate, Value]( | |
feature: FSv1Feature[EdgeEntityId[UserId, TweetId], Value], | |
legacyName: Option[String] = None, | |
defaultValue: Option[Value] = None, | |
enabledParam: Option[FSParam[Boolean]] = None | |
): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] = | |
FeatureStoreV1CandidateFeature( | |
feature, | |
QueryUserIdTweetCandidateTweetIdEntity, | |
legacyName, | |
defaultValue, | |
enabledParam) | |
} | |
object FeatureStoreV1QueryUserIdTweetCandidateTweetIdAggregateFeature | |
def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate]( | |
featureGroup: TimelinesAggregationFrameworkFeatureGroup[EdgeEntityId[UserId, TweetId]], | |
enabledParam: Option[FSParam[Boolean]] = None, | |
keepLegacyNames: Boolean = false, | |
featureNameTransform: Option[FeatureRenameTransform] = None | |
): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] = | |
FeatureStoreV1CandidateFeatureGroup( | |
featureGroup, | |
QueryUserIdTweetCandidateTweetIdEntity, | |
enabledParam, | |
keepLegacyNames, | |
featureNameTransform | |
)(implicitly[ClassTag[EdgeEntityId[UserId, TweetId]]]) | |
} | |
object QueryUserIdTweetCandidateTweetIdEntity | |
extends FeatureStoreV1CandidateEntity[ | |
PipelineQuery, | |
BaseTweetCandidate, | |
EdgeEntityId[UserId, TweetId] | |
] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1TweetCandidateAuthorIdFeature.scala | |
object FeatureStoreV1TweetCandidateAuthorIdFeature | |
def apply[Query <: PipelineQuery, Value]( | |
feature: FSv1Feature[UserId, Value], | |
legacyName: Option[String] = None, | |
defaultValue: Option[Value] = None, | |
enabledParam: Option[FSParam[Boolean]] = None | |
): FeatureStoreV1CandidateFeature[Query, TweetCandidate, _ <: EntityId, Value] = | |
FeatureStoreV1CandidateFeature( | |
feature, | |
TweetCandidateAuthorIdEntity, | |
legacyName, | |
defaultValue, | |
enabledParam) | |
} | |
object FeatureStoreV1TweetCandidateAuthorIdAggregateFeature | |
def apply[Query <: PipelineQuery]( | |
featureGroup: TimelinesAggregationFrameworkFeatureGroup[UserId], | |
enabledParam: Option[FSParam[Boolean]] = None, | |
keepLegacyNames: Boolean = false, | |
featureNameTransform: Option[FeatureRenameTransform] = None | |
): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] = | |
FeatureStoreV1CandidateFeatureGroup( | |
featureGroup, | |
TweetCandidateAuthorIdEntity, | |
enabledParam, | |
keepLegacyNames, | |
featureNameTransform | |
)(implicitly[ClassTag[UserId]]) | |
} | |
object TweetCandidateAuthorIdEntity | |
extends FeatureStoreV1CandidateEntity[PipelineQuery, TweetCandidate, UserId] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1TweetCandidateTweetIdFeature.scala | |
object FeatureStoreV1TweetCandidateTweetIdFeature | |
def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate, Value]( | |
feature: FSv1Feature[TweetId, Value], | |
legacyName: Option[String] = None, | |
defaultValue: Option[Value] = None, | |
enabledParam: Option[FSParam[Boolean]] = None | |
): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] = | |
FeatureStoreV1CandidateFeature( | |
feature, | |
TweetCandidateTweetIdEntity, | |
legacyName, | |
defaultValue, | |
enabledParam) | |
} | |
object FeatureStoreV1TweetCandidateTweetIdAggregateFeature | |
def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate]( | |
featureGroup: TimelinesAggregationFrameworkFeatureGroup[TweetId], | |
enabledParam: Option[FSParam[Boolean]] = None, | |
keepLegacyNames: Boolean = false, | |
featureNameTransform: Option[FeatureRenameTransform] = None | |
): FeatureStoreV1CandidateFeatureGroup[Query, Candidate, _ <: EntityId] = | |
FeatureStoreV1CandidateFeatureGroup( | |
featureGroup, | |
TweetCandidateTweetIdEntity, | |
enabledParam, | |
keepLegacyNames, | |
featureNameTransform | |
) | |
} | |
object TweetCandidateTweetIdEntity | |
extends FeatureStoreV1CandidateEntity[PipelineQuery, BaseTweetCandidate, TweetId] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature\featurestorev1\FeatureStoreV1UserCandidateUserIdFeature.scala | |
object FeatureStoreV1UserCandidateUserIdFeature | |
def apply[Query <: PipelineQuery, Candidate <: BaseUserCandidate, Value]( | |
feature: FSv1Feature[UserId, Value], | |
legacyName: Option[String] = None, | |
defaultValue: Option[Value] = None, | |
enabledParam: Option[FSParam[Boolean]] = None | |
): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] = | |
FeatureStoreV1CandidateFeature( | |
feature, | |
UserCandidateUserIdEntity, | |
legacyName, | |
defaultValue, | |
enabledParam) | |
} | |
object UserCandidateUserIdEntity | |
extends FeatureStoreV1CandidateEntity[PipelineQuery, BaseUserCandidate, UserId] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\ads\AdvertiserBrandSafetySettingsFeatureHydrator.scala | |
object AdvertiserBrandSafetySettingsFeature | |
extends FeatureWithDefaultOnFailure[AdsCandidate, Option[ad.AdvertiserBrandSafetySettings]] | |
class AdvertiserBrandSafetySettingsFeatureHydrator[ | |
Query <: PipelineQuery with AdsQuery, | |
Candidate <: AdsCandidate] @Inject() ( | |
advertiserBrandSafetySettingsStore: ReadableStore[Long, ad.AdvertiserBrandSafetySettings]) | |
extends CandidateFeatureHydrator[Query, Candidate] | |
def apply( | |
query: Query, | |
candidate: Candidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\decay\DecayCandidateFeatureHydrator.scala | |
object DecayScore extends Feature[UniversalNoun[Long], Double] | |
/** | |
* Hydrates snowflake ID candidates with a decay score: | |
* | |
* It is using exponential decay formula to calculate the score | |
* exp(k * age) | |
* where k = ln(0.5) / half-life | |
* | |
* Here is an example for half-life = 1 day | |
* For the brand new tweet it will be exp((ln(0.5)/1)*0) = 1 | |
* For the tweet which was created 1 day ago it will be exp((ln(0.5)/1)*1) = 0.5 | |
* For the tweet which was created 10 day ago it will be exp((ln(0.5)/1)*10) = 0.00097 | |
* | |
* Reference: https://www.cuemath.com/exponential-decay-formula/ | |
* | |
* @note This penalizes but does not filter out the candidate, so "stale" candidates can still appear. | |
*/ | |
case class DecayCandidateFeatureHydrator[Candidate <: UniversalNoun[Long]]( | |
halfLife: Param[Duration] = StaticParam[Duration](2.days), | |
resultFeature: Feature[UniversalNoun[Long], Double] = DecayScore) | |
extends CandidateFeatureHydrator[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: Candidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\param_gated\ParamGatedBulkCandidateFeatureHydrator.scala | |
class ParamGatedBulkCandidateFeatureHydrator[ | |
-Query <: PipelineQuery, | |
Result <: UniversalNoun[Any] | |
]( | |
enabledParam: Param[Boolean], | |
bulkCandidateFeatureHydrator: BulkCandidateFeatureHydrator[Query, Result]) | |
extends BulkCandidateFeatureHydrator[Query, Result] | |
with Conditionally[Query] | |
def onlyIf(query: Query): Boolean = | |
Conditionally.and(query, bulkCandidateFeatureHydrator, query.params(enabledParam)) | |
override def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Result]] | |
): Stitch[Seq[FeatureMap]] = bulkCandidateFeatureHydrator(query, candidates) | |
} | |
object ParamGatedBulkCandidateFeatureHydrator | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\param_gated\ParamGatedCandidateFeatureHydrator.scala | |
class ParamGatedCandidateFeatureHydrator[ | |
-Query <: PipelineQuery, | |
-Result <: UniversalNoun[Any] | |
]( | |
enabledParam: Param[Boolean], | |
candidateFeatureHydrator: CandidateFeatureHydrator[Query, Result]) | |
extends CandidateFeatureHydrator[Query, Result] | |
with Conditionally[Query] | |
def onlyIf(query: Query): Boolean = | |
Conditionally.and(query, candidateFeatureHydrator, query.params(enabledParam)) | |
override def apply( | |
query: Query, | |
candidate: Result, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] = candidateFeatureHydrator.apply(query, candidate, existingFeatures) | |
} | |
object ParamGatedCandidateFeatureHydrator | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\param_gated\featurestorev1\ParamGatedFeatureStoreV1CandidateFeatureHydrator.scala | |
class ParamGatedFeatureStoreV1CandidateFeatureHydrator[ | |
Query <: PipelineQuery, | |
Candidate <: UniversalNoun[Any] | |
]( | |
enabledParam: Param[Boolean], | |
candidateFeatureHydrator: FeatureStoreV1CandidateFeatureHydrator[Query, Candidate]) | |
extends FeatureStoreV1CandidateFeatureHydrator[Query, Candidate] | |
with Conditionally[Query] | |
def onlyIf(query: Query): Boolean = | |
Conditionally.and(query, candidateFeatureHydrator, query.params(enabledParam)) | |
override def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[Seq[FeatureMap]] = candidateFeatureHydrator(query, candidates) | |
} | |
object ParamGatedFeatureStoreV1CandidateFeatureHydrator | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\qualityfactor_gated\QualityFactorGatedCandidateFeatureHydrator.scala | |
object QualityFactorGatedCandidateFeatureHydrator | |
class QualityFactorGatedCandidateFeatureHydrator[ | |
-Query <: PipelineQuery with HasQualityFactorStatus, | |
Result <: UniversalNoun[Any] | |
]( | |
pipelineIdentifier: ComponentIdentifier, | |
qualityFactorInclusiveThreshold: Param[Double], | |
candidateFeatureHydrator: CandidateFeatureHydrator[Query, Result]) | |
extends CandidateFeatureHydrator[Query, Result] | |
with Conditionally[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\tweet_is_nsfw\TweetIsNsfwCandidateFeatureHydrator.scala | |
object IsNsfw extends FeatureWithDefaultOnFailure[TweetCandidate, Option[Boolean]] | |
def apply( | |
hasNsfwHighPrecisionLabel: Option[Boolean], | |
isNsfwUser: Option[Boolean], | |
isNsfwAdmin: Option[Boolean] | |
): Boolean | |
class TweetIsNsfwCandidateFeatureHydrator( | |
tweetypieStitchClient: TweetypieStitchClient, | |
tweetVisibilityPolicy: t.TweetVisibilityPolicy) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] | |
with Logging | |
def features: Set[Feature[_, _]] = Set(IsNsfw) | |
private val NsfwLabelFields: Set[t.TweetInclude] = Set[t.TweetInclude]( | |
// Tweet fields containing NSFW related attributes, in addition to what exists in coreData. | |
t.TweetInclude.TweetFieldId(t.Tweet.NsfwHighPrecisionLabelField.id), | |
t.TweetInclude.TweetFieldId(t.Tweet.CoreDataField.id) | |
) | |
override def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[BaseTweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
class IsNsfwFeatureHydrationFailure(message: String) | |
extends Exception(s"IsNsfwFeatureHydrationFailure($ | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\tweet_tlx\TweetTLXScoreCandidateFeatureHydrator.scala | |
class TweetTLXScoreCandidateFeatureHydrator @Inject() ( | |
column: TimelineScorerTweetScoresV1ClientColumn) | |
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: TweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\tweet_tweetypie\TweetTweetypieCandidateFeatureHydrator.scala | |
object IsCommunityTweetFeature extends Feature[TweetCandidate, Boolean] | |
// Tweetypie VF Features | |
object HasTakedownFeature extends Feature[TweetCandidate, Boolean] | |
object HasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean] | |
object IsHydratedFeature extends Feature[TweetCandidate, Boolean] | |
object IsNarrowcastFeature extends Feature[TweetCandidate, Boolean] | |
object IsNsfwAdminFeature extends Feature[TweetCandidate, Boolean] | |
object IsNsfwFeature extends Feature[TweetCandidate, Boolean] | |
object IsNsfwUserFeature extends Feature[TweetCandidate, Boolean] | |
object IsNullcastFeature extends Feature[TweetCandidate, Boolean] | |
object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean] | |
object QuotedTweetHasTakedownFeature extends Feature[TweetCandidate, Boolean] | |
object QuotedTweetHasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean] | |
object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]] | |
object SourceTweetHasTakedownFeature extends Feature[TweetCandidate, Boolean] | |
object SourceTweetHasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean] | |
object TakedownCountryCodesFeature extends Feature[TweetCandidate, Set[String]] | |
object IsReplyFeature extends Feature[TweetCandidate, Boolean] | |
object InReplyToFeature extends Feature[TweetCandidate, Option[Long]] | |
object IsRetweetFeature extends Feature[TweetCandidate, Boolean] | |
object TweetTweetypieCandidateFeatureHydrator | |
class TweetTweetypieCandidateFeatureHydrator( | |
tweetypieStitchClient: TweetypieStitchClient, | |
safetyLevelPredicate: PipelineQuery => SafetyLevel) | |
extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] | |
def apply( | |
query: PipelineQuery, | |
candidate: BaseTweetCandidate, | |
existingFeatures: FeatureMap | |
): Stitch[FeatureMap] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\candidate\tweet_visibility_reason\TweetVisibilityReasonBulkCandidateFeatureHydrator.scala | |
object VisibilityReason | |
extends FeatureWithDefaultOnFailure[TweetCandidate, Option[SPAM.FilteredReason]] | |
class TweetVisibilityReasonBulkCandidateFeatureHydrator @Inject() ( | |
tweetypieStitchClient: TweetypieStitchClient, | |
safetyLevel: SPAM.SafetyLevel) | |
extends BulkCandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] | |
with Logging | |
def features: Set[Feature[_, _]] = Set(VisibilityReason) | |
override def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[BaseTweetCandidate]] | |
): Stitch[Seq[FeatureMap]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\async\AsyncQueryFeatureHydrator.scala | |
class AsyncQueryFeatureHydrator[-Query <: PipelineQuery] private[async] ( | |
override val hydrateBefore: PipelineStepIdentifier, | |
queryFeatureHydrator: QueryFeatureHydrator[Query]) | |
extends QueryFeatureHydrator[Query] | |
with AsyncHydrator | |
def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query) | |
} | |
/** | |
* A [[FeatureStoreV1QueryFeatureHydrator]] with [[AsyncHydrator]] that hydrated asynchronously for features | |
* to be before the step identified in [[hydrateBefore]]. We need a standalone class for feature store, | |
* different from the above as FStore hydrators are exempt from validations at run time. | |
* | |
* @param hydrateBefore the [[PipelineStepIdentifier]] step to make sure this feature is hydrated before. | |
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run asynchronously | |
* @tparam Query The domain model for the query or request | |
*/ | |
case class AsyncFeatureStoreV1QueryFeatureHydrator[Query <: PipelineQuery] private[async] ( | |
override val hydrateBefore: PipelineStepIdentifier, | |
featureStoreV1QueryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query]) | |
extends FeatureStoreV1QueryFeatureHydrator[ | |
Query | |
] | |
with AsyncHydrator | |
object AsyncQueryFeatureHydrator | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\cr_ml_ranker\CrMlRankerCommonQueryFeatureHydrator.scala | |
object CrMlRankerCommonFeatures extends Feature[PipelineQuery, t.CommonFeatures] | |
object CrMlRankerRankingConfig extends Feature[PipelineQuery, t.RankingConfig] | |
private[cr_ml_ranker] class CrMlRankerCommonQueryFeatureHydrator( | |
crMlRanker: t.CrMLRanker.MethodPerEndpoint, | |
rankingConfigSelector: RankingConfigBuilder) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\cr_ml_ranker\CrMlRankerCommonQueryFeatureHydratorBuilder.scala | |
class CrMlRankerCommonQueryFeatureHydratorBuilder @Inject() ( | |
crMlRanker: t.CrMLRanker.MethodPerEndpoint) | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\cr_ml_ranker\RankingConfigBuilder.scala | |
trait RankingConfigBuilder | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\impressed_tweets\ImpressedTweetsQueryFeatureHydrator.scala | |
object ImpressedTweets extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] | |
class ImpressedTweetsQueryFeatureHydrator @Inject() ( | |
tweetImpressionStore: ReadableStore[Long, ImpressionList]) | |
extends QueryFeatureHydrator[PipelineQuery] | |
def hydrate(query: PipelineQuery): Stitch[FeatureMap] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\logged_in_only\LoggedInOnlyQueryFeatureHydrator.scala | |
class LoggedInOnlyQueryFeatureHydrator[-Query <: PipelineQuery, Result <: UniversalNoun[Any]]( | |
queryFeatureHydrator: QueryFeatureHydrator[Query]) | |
extends QueryFeatureHydrator[Query] | |
with Conditionally[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\param_gated\AsyncParamGatedQueryFeatureHydrator.scala | |
class AsyncParamGatedQueryFeatureHydrator[ | |
-Query <: PipelineQuery, | |
Result <: UniversalNoun[Any] | |
]( | |
enabledParam: Param[Boolean], | |
override val hydrateBefore: PipelineStepIdentifier, | |
queryFeatureHydrator: QueryFeatureHydrator[Query]) | |
extends QueryFeatureHydrator[Query] | |
with Conditionally[Query] | |
with AsyncHydrator | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\param_gated\ParamGatedQueryFeatureHydrator.scala | |
class ParamGatedQueryFeatureHydrator[-Query <: PipelineQuery, Result <: UniversalNoun[Any]]( | |
enabledParam: Param[Boolean], | |
queryFeatureHydrator: QueryFeatureHydrator[Query]) | |
extends QueryFeatureHydrator[Query] | |
with Conditionally[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\param_gated\featurestorev1\AsyncParamGatedFeatureStoreV1QueryFeatureHydrator.scala | |
class AsyncParamGatedFeatureStoreV1QueryFeatureHydrator[ | |
Query <: PipelineQuery, | |
Result <: UniversalNoun[Any] | |
]( | |
enabledParam: Param[Boolean], | |
override val hydrateBefore: PipelineStepIdentifier, | |
queryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query]) | |
extends FeatureStoreV1QueryFeatureHydrator[Query] | |
with Conditionally[Query] | |
with AsyncHydrator | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\param_gated\featurestorev1\ParamGatedFeatureStoreV1QueryFeatureHydrator.scala | |
class ParamGatedFeatureStoreV1QueryFeatureHydrator[ | |
Query <: PipelineQuery, | |
Result <: UniversalNoun[Any] | |
]( | |
enabledParam: Param[Boolean], | |
queryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query]) | |
extends FeatureStoreV1QueryFeatureHydrator[Query] | |
with Conditionally[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\feature_hydrator\query\qualityfactor_gated\QualityFactorGatedQueryFeatureHydrator.scala | |
object QualityFactorGatedQueryFeatureHydrator | |
class QualityFactorGatedQueryFeatureHydrator[ | |
-Query <: PipelineQuery with HasQualityFactorStatus, | |
Result <: UniversalNoun[Any] | |
]( | |
pipelineIdentifier: ComponentIdentifier, | |
qualityFactorInclusiveThreshold: Param[Double], | |
queryFeatureHydrator: QueryFeatureHydrator[Query]) | |
extends QueryFeatureHydrator[Query] | |
with Conditionally[Query] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\AdaptiveLongIntBloomFilterDedupFilter.scala | |
trait GetAdaptiveLongIntBloomFilter[Query <: PipelineQuery] | |
def apply(query: Query): Option[AdaptiveLongIntBloomFilter] | |
} | |
case class AdaptiveLongIntBloomFilterDedupFilter[ | |
Query <: PipelineQuery, | |
Candidate <: UniversalNoun[Long] | |
]( | |
getBloomFilter: GetAdaptiveLongIntBloomFilter[Query]) | |
extends Filter[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\ExcludedIdsFilter.scala | |
class ExcludedIdsFilter[ | |
Query <: PipelineQuery with HasExcludedIds, | |
Candidate <: UniversalNoun[Long] | |
]() extends Filter[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\FeatureFilter.scala | |
object FeatureFilter | |
def fromFeature[Candidate <: UniversalNoun[Any]]( | |
feature: Feature[Candidate, Boolean] | |
): Filter[PipelineQuery, Candidate] = | |
FeatureFilter.fromFeature(FilterIdentifier(feature.toString), feature) | |
/** | |
* Builds a Filter that keeps candidates when the provided Boolean Feature is present and True. | |
* If the Feature is missing or False, the candidate is removed. | |
* | |
* | |
def fromFeature[Candidate <: UniversalNoun[Any]]( | |
identifier: FilterIdentifier, | |
feature: Feature[Candidate, Boolean] | |
): Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\FeatureValueConditionalFilter.scala | |
trait ShouldApplyFilter[FeatureValue] | |
def apply(feature: FeatureValue): Boolean | |
} | |
/** | |
* A filter that applies the [[filter]] for candidates for which [[shouldApplyFilter]] is true, and keeps the others | |
* @param feature feature to determine whether to apply underyling filter | |
* @param shouldApplyFilter function to determine whether to apply filter | |
* @param filter the actual filter to apply if shouldApplyFilter is True | |
* @tparam Query The domain model for the query or request | |
* @tparam Candidate The type of the candidates | |
* @tparam FeatureValueType | |
*/ | |
case class FeatureValueConditionalFilter[ | |
-Query <: PipelineQuery, | |
Candidate <: UniversalNoun[Any], | |
FeatureValueType | |
]( | |
feature: Feature[Candidate, FeatureValueType], | |
shouldApplyFilter: ShouldApplyFilter[FeatureValueType], | |
filter: Filter[Query, Candidate]) | |
extends Filter[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
object FeatureConditionalFilter | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\HasAuthorIdFeatureFilter.scala | |
class HasAuthorIdFeatureFilter[Candidate <: TweetCandidate]() | |
extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\ParamGatedFilter.scala | |
class ParamGatedFilter[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( | |
enabledParam: Param[Boolean], | |
filter: Filter[Query, Candidate]) | |
extends Filter[Query, Candidate] | |
with Filter.Conditionally[Query, Candidate] | |
def onlyIf(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): Boolean = | |
Conditionally.and(Filter.Input(query, candidates), filter, query.params(enabledParam)) | |
override def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] = filter.apply(query, candidates) | |
} | |
object ParamGatedFilter | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\PredicateFilter.scala | |
trait ShouldKeepCandidate[Candidate] | |
def apply(candidate: Candidate): Boolean | |
} | |
object PredicateFilter | |
def fromPredicate[Candidate <: UniversalNoun[Any]]( | |
identifier: FilterIdentifier, | |
shouldKeepCandidate: ShouldKeepCandidate[Candidate] | |
): Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\SnowflakeIdAgeFilter.scala | |
class SnowflakeIdAgeFilter[Candidate <: UniversalNoun[Long]]( | |
maxAgeParam: Param[Duration]) | |
extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetAuthorCountryFilter.scala | |
class TweetAuthorCountryFilter[Candidate <: BaseTweetCandidate]( | |
countryCodeFeature: Feature[Candidate, Option[String]]) | |
extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetAuthorIsSelfFilter.scala | |
class TweetAuthorIsSelfFilter[Candidate <: BaseTweetCandidate]() | |
extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetIsNotReplyFilter.scala | |
class TweetIsNotReplyFilter[Candidate <: BaseTweetCandidate]() | |
extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetLanguageFilter.scala | |
class TweetLanguageFilter[Candidate <: BaseTweetCandidate]( | |
languageCodeFeature: Feature[Candidate, Option[String]]) | |
extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\TweetVisibilityFilter.scala | |
object TweetVisibilityFilter | |
class TweetVisibilityFilter[Candidate <: BaseTweetCandidate]( | |
tweetypieStitchClient: TweetypieStitchClient, | |
tweetVisibilityPolicy: TP.TweetVisibilityPolicy, | |
safetyLevel: SafetyLevel, | |
tweetIncludes: Set[TP.TweetInclude.TweetFieldId] = DefaultTweetIncludes) | |
extends Filter[PipelineQuery, Candidate] | |
with Logging | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\UrtUnorderedExcludeIdsCursorFilter.scala | |
class UrtUnorderedExcludeIdsCursorFilter[ | |
Candidate <: UniversalNoun[Long], | |
Query <: PipelineQuery with HasPipelineCursor[UrtUnorderedExcludeIdsCursor] | |
]() extends Filter[Query, Candidate] | |
def apply( | |
query: Query, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\list_visibility\ListVisibilityFilter.scala | |
class ListVisibilityFilter[Candidate <: UniversalNoun[Long]]( | |
listsColumn: CoreOnListClientColumn) | |
extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\filter\tweet_impression\TweetImpressionFilter.scala | |
class TweetImpressionFilter[Candidate <: BaseTweetCandidate]( | |
) extends Filter[PipelineQuery, Candidate] | |
def apply( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithFeatures[Candidate]] | |
): Stitch[FilterResult[Candidate]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\DefinedCountryCodeGate.scala | |
object DefinedCountryCodeGate extends Gate[PipelineQuery] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\FeatureGate.scala | |
trait ShouldContinue[Value] | |
def apply(featureValue: Value): Boolean | |
/** If the [[Feature]] is a failure, use this value */ | |
def onFailedFeature(t: Throwable): GateResult = GateResult.Stop | |
/** | |
* If the [[Feature]], or [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]], | |
* is missing use this value | |
*/ | |
def onMissingFeature: GateResult = GateResult.Stop | |
} | |
object FeatureGate | |
def fromFeature( | |
feature: Feature[_, Boolean] | |
): FeatureGate[Boolean] = | |
FeatureGate.fromFeature(GateIdentifier(feature.toString), feature) | |
def fromNegatedFeature( | |
feature: Feature[_, Boolean] | |
): FeatureGate[Boolean] = | |
FeatureGate.fromNegatedFeature(GateIdentifier(feature.toString), feature) | |
def fromFeature( | |
gateIdentifier: GateIdentifier, | |
feature: Feature[_, Boolean] | |
): FeatureGate[Boolean] = | |
FeatureGate[Boolean](gateIdentifier, feature, identity) | |
def fromNegatedFeature( | |
gateIdentifier: GateIdentifier, | |
feature: Feature[_, Boolean] | |
): FeatureGate[Boolean] = | |
FeatureGate[Boolean](gateIdentifier, feature, !identity(_)) | |
} | |
/** | |
* A [[Gate]] that is actuated based upon the value of the provided feature | |
*/ | |
case class FeatureGate[Value]( | |
gateIdentifier: GateIdentifier, | |
feature: Feature[_, Value], | |
continue: ShouldContinue[Value]) | |
extends Gate[PipelineQuery] | |
def shouldContinue(query: PipelineQuery): Stitch[Boolean] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\FirstPageGate.scala | |
object FirstPageGate extends Gate[PipelineQuery with HasPipelineCursor[_]] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\NoCandidatesGate.scala | |
class NoCandidatesGate(scope: CandidateScope) extends QueryAndCandidateGate[PipelineQuery] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\NonEmptyAdsQueryStringGate.scala | |
object NonEmptyAdsQueryStringGate extends Gate[PipelineQuery with AdsQuery] | |
def shouldContinue(query: PipelineQuery with AdsQuery): Stitch[Boolean] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\NonEmptyCandidatesGate.scala | |
class NonEmptyCandidatesGate(scope: CandidateScope) | |
extends QueryAndCandidateGate[PipelineQuery] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\QualityFactorGate.scala | |
class QualityFactorGate(pipelineIdentifier: ComponentIdentifier, threshold: Double) | |
extends Gate[PipelineQuery with HasQualityFactorStatus] | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\gate\any_candidates_without_feature\AnyCandidatesWithoutFeatureGate.scala | |
class AnyCandidatesWithoutFeatureGate( | |
override val identifier: GateIdentifier, | |
scope: CandidateScope, | |
missingFeature: Feature[_, _]) | |
extends QueryAndCandidateGate[PipelineQuery] | |
def shouldContinue( | |
query: PipelineQuery, | |
candidates: Seq[CandidateWithDetails] | |
): Stitch[Boolean] = | |
Stitch.value(scope.partition(candidates).candidatesInScope.exists | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\ArticleCandidate.scala | |
trait BaseArticleCandidate extends UniversalNoun[Int] | |
/** | |
* Canonical ArticleCandidate model. Always prefer this version over all other variants. | |
* | |
* @note Any additional fields should be added as a [[com.twitter.product_mixer.core.feature.Feature]] | |
* on the candidate's [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]]. If the | |
* features come from the candidate source itself (as opposed to hydrated via a | |
* [[com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator]]), | |
* then [[com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig.featuresFromCandidateSourceTransformers]] | |
* can be used to extract features from the candidate source response. | |
* | |
* @note This class should always remain `final`. If for any reason the `final` modifier is removed, | |
* the equals() implementation must be updated in order to handle class inheritor equality | |
* (see note on the equals method below) | |
*/ | |
final class ArticleCandidate private ( | |
override val id: Int) | |
extends BaseArticleCandidate | |
def canEqual(that: Any): Boolean = that.isInstanceOf[ArticleCandidate] | |
/** | |
* High performance implementation of equals method that leverages: | |
* - Referential equality short circuit | |
* - Cached hashcode equality short circuit | |
* - Field values are only checked if the hashCodes are equal to handle the unlikely case | |
* of a hashCode collision | |
* - Removal of check for `that` being an equals-compatible descendant since this class is final | |
* | |
* @note `candidate.canEqual(this)` is not necessary because this class is final | |
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala, | |
* Chapter 28]] for discussion and design. | |
*/ | |
override def equals(that: Any): Boolean = | |
that match | |
object construction. This prevents the | |
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the | |
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field | |
* object mutability and hashCode implementations. | |
* | |
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode | |
* are immutable. This includes: | |
* - Inability to mutate the object reference on for an existing instantiated candidate | |
* (i.e. each field is a val) | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* data structure), assuming stable hashCode implementations for these objects | |
* | |
* @note In order for the hashCode to be consistent with object equality, `##` must be used for | |
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`. | |
*/ | |
override val hashCode: Int = id.## | |
} | |
object ArticleCandidate | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\AudioSpaceCandidate.scala | |
class should always remain `final`. If for any reason the `final` modifier is removed, | |
* the equals() implementation must be updated in order to handle class inheritor equality | |
* (see note on the equals method below) | |
*/ | |
final class AudioSpaceCandidate private ( | |
override val id: String) | |
extends UniversalNoun[String] | |
def canEqual(that: Any): Boolean = that.isInstanceOf[AudioSpaceCandidate] | |
/** | |
* High performance implementation of equals method that leverages: | |
* - Referential equality short circuit | |
* - Cached hashcode equality short circuit | |
* - Field values are only checked if the hashCodes are equal to handle the unlikely case | |
* of a hashCode collision | |
* - Removal of check for `that` being an equals-compatible descendant since this class is final | |
* | |
* @note `candidate.canEqual(this)` is not necessary because this class is final | |
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala, | |
* Chapter 28]] for discussion and design. | |
*/ | |
override def equals(that: Any): Boolean = | |
that match | |
object construction. This prevents the | |
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the | |
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field | |
* object mutability and hashCode implementations. | |
* | |
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode | |
* are immutable. This includes: | |
* - Inability to mutate the object reference on for an existing instantiated candidate | |
* (i.e. each field is a val) | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* data structure), assuming stable hashCode implementations for these objects | |
* | |
* @note In order for the hashCode to be consistent with object equality, `##` must be used for | |
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`. | |
*/ | |
override val hashCode: Int = id.## | |
} | |
object AudioSpaceCandidate | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\CardCandidate.scala | |
class should always remain `final`. If for any reason the `final` modifier is removed, | |
* the equals() implementation must be updated in order to handle class inheritor equality | |
* (see note on the equals method below) | |
*/ | |
final class CardCandidate private ( | |
override val id: String) | |
extends UniversalNoun[String] | |
def canEqual(that: Any): Boolean = that.isInstanceOf[CardCandidate] | |
/** | |
* High performance implementation of equals method that leverages: | |
* - Referential equality short circuit | |
* - Cached hashcode equality short circuit | |
* - Field values are only checked if the hashCodes are equal to handle the unlikely case | |
* of a hashCode collision | |
* - Removal of check for `that` being an equals-compatible descendant since this class is final | |
* | |
* @note `candidate.canEqual(this)` is not necessary because this class is final | |
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala, | |
* Chapter 28]] for discussion and design. | |
*/ | |
override def equals(that: Any): Boolean = | |
that match | |
object construction. This prevents the | |
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the | |
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field | |
* object mutability and hashCode implementations. | |
* | |
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode | |
* are immutable. This includes: | |
* - Inability to mutate the object reference on for an existing instantiated candidate | |
* (i.e. each field is a val) | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* data structure), assuming stable hashCode implementations for these objects | |
* | |
* @note In order for the hashCode to be consistent with object equality, `##` must be used for | |
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`. | |
*/ | |
override val hashCode: Int = id.## | |
} | |
object CardCandidate | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\CommerceItemCandidate.scala | |
class should always remain `final`. If for any reason the `final` modifier is removed, | |
* the equals() implementation must be updated in order to handle class inheritor equality | |
* (see note on the equals method below) | |
*/ | |
final class CommerceProductCandidate private ( | |
override val id: Long) | |
extends UniversalNoun[Long] | |
def canEqual(that: Any): Boolean = that.isInstanceOf[CommerceProductCandidate] | |
/** | |
* High performance implementation of equals method that leverages: | |
* - Referential equality short circuit | |
* - Cached hashcode equality short circuit | |
* - Field values are only checked if the hashCodes are equal to handle the unlikely case | |
* of a hashCode collision | |
* - Removal of check for `that` being an equals-compatible descendant since this class is final | |
* | |
* @note `candidate.canEqual(this)` is not necessary because this class is final | |
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala, | |
* Chapter 28]] for discussion and design. | |
*/ | |
override def equals(that: Any): Boolean = | |
that match | |
object construction. This prevents the | |
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the | |
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field | |
* object mutability and hashCode implementations. | |
* | |
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode | |
* are immutable. This includes: | |
* - Inability to mutate the object reference on for an existing instantiated candidate | |
* (i.e. each field is a val) | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* data structure), assuming stable hashCode implementations for these objects | |
* | |
* @note In order for the hashCode to be consistent with object equality, `##` must be used for | |
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`. | |
*/ | |
override val hashCode: Int = id.## | |
} | |
object CommerceProductCandidate | |
def apply(id: Long): CommerceProductCandidate = new CommerceProductCandidate(id) | |
} | |
/** | |
* Canonical CommerceProductGroupCandidate model which encapsulates information about a Single | |
* Product Type and its corresponding versions. Always prefer this version over all other variants. | |
* For example: | |
* iPhone 14 | |
* - 128 GB, White | |
* - 128 GB, Blue | |
* - 1TB, Grey | |
* When a user clicks on a Product Group, they will be shown information about all of the possible | |
* versions of the top level product. | |
* | |
* @note Both CommerceProduct (above) and CommerceProductGroups can be shown in the same | |
* TimelineModule (i.e Carousel) | |
* | |
* @note Any additional fields should be added as a [[com.twitter.product_mixer.core.feature.Feature]] | |
* on the candidate's [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]]. If the | |
* features come from the candidate source itself (as opposed to hydrated via a | |
* [[com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator]]), | |
* then [[com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig.featuresFromCandidateSourceTransformers]] | |
* can be used to extract features from the candidate source response. | |
* | |
* @note This class should always remain `final`. If for any reason the `final` modifier is removed, | |
* the equals() implementation must be updated in order to handle class inheritor equality | |
* (see note on the equals method below) | |
*/ | |
final class CommerceProductGroupCandidate private ( | |
override val id: Long) | |
extends UniversalNoun[Long] | |
def canEqual(that: Any): Boolean = that.isInstanceOf[CommerceProductGroupCandidate] | |
/** | |
* High performance implementation of equals method that leverages: | |
* - Referential equality short circuit | |
* - Cached hashcode equality short circuit | |
* - Field values are only checked if the hashCodes are equal to handle the unlikely case | |
* of a hashCode collision | |
* - Removal of check for `that` being an equals-compatible descendant since this class is final | |
* | |
* @note `candidate.canEqual(this)` is not necessary because this class is final | |
* @see [[http://www.artima.com/pins1ed/object-equality.html Programming in Scala, | |
* Chapter 28]] for discussion and design. | |
*/ | |
override def equals(that: Any): Boolean = | |
that match | |
object construction. This prevents the | |
* need to recompute the hashCode on each hashCode() invocation, which is the behavior of the | |
* Scala compiler case class-generated hashCode() since it cannot make assumptions regarding field | |
* object mutability and hashCode implementations. | |
* | |
* @note Caching the hashCode is only safe if all of the fields used to construct the hashCode | |
* are immutable. This includes: | |
* - Inability to mutate the object reference on for an existing instantiated candidate | |
* (i.e. each field is a val) | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* - Inability to mutate the field object instance itself (i.e. each field is an immutable | |
* data structure), assuming stable hashCode implementations for these objects | |
* @note In order for the hashCode to be consistent with object equality, `##` must be used for | |
* boxed numeric types and null. As such, always prefer `.##` over `.hashCode()`. | |
*/ | |
override val hashCode: Int = id.## | |
} | |
object CommerceProductGroupCandidate | |
.\product-mixer\component-library\src\main\scala\com\twitter\product_mixer\component_library\model\candidate\CursorCandidate.scala | |
trait CursorType | |
case object PreviousCursor extends CursorType | |
case object NextCursor extends CursorType | |
/** | |
* Canonical CursorCandidate model. Always prefer this version over all other variants. | |
* | |
* @note Any additional fields should be added as a [[com.twitter.product_mixer.core.feature.Feature]] | |
* on the candidate's [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]]. If the | |
* features come from the candidate source itself (as opposed to hydrated via a | |
* [[com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator]]), | |
* then [[com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig.featuresFromCandidateSourceTransformers]] | |
* can be used to extract features from the candidate source response. | |
* | |
* @note This class should always remain `final`. If for any reason the `final` modifier is removed, | |
* the equals() implementation must be updated in order to handle cl |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment