Skip to content

Instantly share code, notes, and snippets.

Avatar

Bernhard Schäfer bernhardschaefer

View GitHub Profile
View trim_maskrcnn_benchmark_model.py
keys_to_remove = [
'module.roi_heads.box.predictor.cls_score.weight',
'module.roi_heads.box.predictor.cls_score.bias',
'module.roi_heads.box.predictor.bbox_pred.weight',
'module.roi_heads.box.predictor.bbox_pred.bias',
'module.roi_heads.mask.predictor.mask_fcn_logits.weight', # mask
'module.roi_heads.mask.predictor.mask_fcn_logits.bias' # mask
]
def trim_maskrcnn_benchmark_model(model_path: str, trimmed_model_path: str):
View TerminalVim.scpt
-- TerminalVim.app
-- This creates a shim Application that will enable you to open files from the Finder in vim using iTerm
-- To use this script:
-- 1. Open Automator and create a new Application
-- 2. Add the "Run Applescript" action
-- 3. Paste this script into the Run Applescript section
-- 4. Save the application as TerminalVim.app in your Applications folder
-- 5. In the Finder, right click on a file and select "Open With". In that window you can set TerminalVim as a default
on run {input, parameters}
View spark-submit-streaming-yarn.sh
#!/bin/bash
# Minimum TODOs on a per job basis:
# 1. define name, application jar path, main class, queue and log4j-yarn.properties path
# 2. remove properties not applicable to your Spark version (Spark 1.x vs. Spark 2.x)
# 3. tweak num_executors, executor_memory (+ overhead), and backpressure settings
# the two most important settings:
num_executors=6
executor_memory=3g
View log4j-yarn.properties
# Spark Streaming Logging Configuration
# See also: http://spark.apache.org/docs/2.0.2/running-on-yarn.html#debugging-your-application
log4j.rootLogger=INFO, stderr
# application namespace configuration
log4j.logger.de.inovex.mysparkapp=stderr, stdout
# Write all logs to standard Spark stderr file
log4j.appender.stderr=org.apache.log4j.RollingFileAppender
View SparkTimerExample.scala
import org.apache.spark.groupon.metrics.UserMetricsSystem
import org.apache.spark.groupon.metrics.SparkTimer
import org.apache.spark.rdd.RDD
class SparkTimerExample[T] {
lazy val timer: SparkTimer = UserMetricsSystem.timer("BulkPutTimer")
def bulkPut(rdd: RDD[T]) = {
rdd.foreachPartition(partitionOfRecords => {
val timerCtx = timer.time()
View spark-streaming-hdfs-marker-file.scala
val checkpointDirectory = "hdfs:///path/to/checkpoint/dir"
def main(args: Array[String]) {
// Get StreamingContext from checkpoint data or create a new one
val context = StreamingContext.getOrCreate(checkpointDirectory, functionToCreateContext _)
// Start the context
context.start()