Skip to content

Instantly share code, notes, and snippets.

Marc-Olivier Arsenault marcolivierarsenault

Block or report user

Report or block marcolivierarsenault

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
@marcolivierarsenault
marcolivierarsenault / Alt Coin Scraper
Created Jan 15, 2018
lamda function to get all the lambda function and store them in a database
View Alt Coin Scraper
import json
import decimal
import urllib.request
import boto3
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('AltCoinHistory')
def lambda_handler(event, context):
View defaultSiamese.py
def triplet_loss(y_true, y_pred, alpha = 0.4):
"""
Implementation of the triplet loss function
Arguments:
y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.
y_pred -- python list containing three objects:
anchor -- the encodings for the anchor data
positive -- the encodings for the positive data (similar to anchor)
negative -- the encodings for the negative data (different from anchor)
View lossless_triplet_loss.py
def lossless_triplet_loss(y_true, y_pred, N = 3, beta=N, epsilon=1e-8):
"""
Implementation of the triplet loss function
Arguments:
y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.
y_pred -- python list containing three objects:
anchor -- the encodings for the anchor data
positive -- the encodings for the positive data (similar to anchor)
negative -- the encodings for the negative data (different from anchor)
View UDF.scala
def findMatchingPatterns(regexes: ArrayList[String]): UserDefinedFunction = {
udf((value: String) => {
for {
text <- Option(value)
matches = regexes.asScala.filter(r => Pattern.matches(r, text))
if matches.nonEmpty
} yield matches
}, ArrayType(StringType))
}
View regex.py
from utils.scala_functions import find_matching_patterns
from pyspark.sql import functions as F
regexes = regex.agg(F.collect_list(F.col("pattern"))).collect()[0][0]
regexes = sc.broadcast(regexes)
articles = articles \
.withColumn("patterns", find_matching_patterns(F.col("text"), regexes.value)
.withColumn("patterns", F.when(F.col("patterns").isNull(), F.array(F.lit(None))).otherwise(F.col("patterns"))) \
.withColumn("pattern", F.explode(F.col("patterns")))
View rlike.py
wiki.join(regex, expr("text rlike pattern") how='left_outer')
You can’t perform that action at this time.