Holger Brandl holgerbrandl
holgerbrandl
/ kshell_0.2_kscript_header.kts
Created
August 21, 2018 08:51
kshell-0.2-Snaphot dependency heaer for kscript
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| @file:DependsOn("org.apache.hadoop:hadoop-common:2.7.0") | |
| @file:DependsOn("com.github.khud:kshell-repl-api:0.2.2-1.2.41") | |
| @file:DependsOn("sparklin:jline3-shaded:0.2-SNAPSHOT") | |
| @file:DependsOn("sparklin:kshell:0.2-SNAPSHOT") |
holgerbrandl
/ kafka_string_serializer_not_found.kts
Created
June 18, 2018 08:07
Example to illustrate issue when running kafka code with kotlin
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env kscript | |
| @file:DependsOn("org.apache.kafka:kafka-clients:1.1.0") | |
| import java.util.Properties | |
| import org.apache.kafka.clients.producer.KafkaProducer | |
| import org.apache.kafka.clients.producer.ProducerRecord | |
| import org.apache.kafka.common.serialization.StringSerializer | |
| val properties = Properties() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // repl class-path | |
| @file:DependsOn("org.apache.hadoop:hadoop-common:2.7.0") | |
| @file:DependsOn("com.github.khud:kshell-repl-api:0.2.2-1.2.41") | |
| @file:DependsOn("sparklin:jline3-shaded:0.2-SNAPSHOT") | |
| @file:DependsOn("sparklin:kshell:0.2-SNAPSHOT") | |
| @file:DependsOn("de.mpicbg.scicomp:kutils:0.10") | |
| @file:DependsOn("com.github.holgerbrandl:kscript-support:1.2.4") | |
| @file:DependsOn("de.mpicbg.scicomp.joblist:joblist-kotlin:1.2") |
holgerbrandl
/ kshell_launcher.kts
Created
June 6, 2018 08:09
Launch kshell with a maven resolved classpath
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env kscript | |
| // repl class-path | |
| @file:DependsOn("org.apache.hadoop:hadoop-common:2.7.0") | |
| @file:DependsOn("com.github.khud:kshell-repl-api:0.2.2-1.2.41") | |
| @file:DependsOn("sparklin:jline3-shaded:0.2-SNAPSHOT") | |
| @file:DependsOn("sparklin:kshell:0.2-SNAPSHOT") | |
| // user classpath | |
| @file:DependsOn("com.offbytwo:docopt:0.6.0.20150202","log4j:log4j:1.2.14") |
holgerbrandl
/ IrisShape.kt
Last active
May 15, 2018 16:44
example to illustrate issue with RecordReaderDataSetIterator
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package org.deeplearning4j.datasets.datavec | |
| import org.datavec.api.records.reader.impl.csv.CSVRecordReader | |
| import org.datavec.api.split.FileSplit | |
| import org.nd4j.linalg.io.ClassPathResource | |
| fun main(args: Array<String>) { | |
| val recordReader = CSVRecordReader(0, ',').apply{ | |
| // initialize(FileSplit(File("/Users/brandl/projects/deep_learning/dl4j/dl4j-test-resources/target/classes/iris.txt"))) |
holgerbrandl
/ RecorderReaderExample.kt
Last active
May 9, 2018 08:53
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package playground | |
| import DATA_ROOT | |
| import org.datavec.api.io.labels.PathLabelGenerator | |
| import org.datavec.api.split.CollectionInputSplit | |
| import org.datavec.api.writable.IntWritable | |
| import org.datavec.api.writable.Writable | |
| import org.datavec.image.recordreader.ImageRecordReader | |
| import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator | |
| import java.io.File |
holgerbrandl
/ gist:095c8eb7a486626676882c94269bc741
Created
April 25, 2018 04:56
Thread dump of deadlock in org.deeplearning4j.streaming.kafka.NDArrayPublisherTests#testPublish
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Full thread dump | |
| "main@1" prio=5 tid=0x1 nid=NA waiting | |
| java.lang.Thread.State: WAITING | |
| at sun.misc.Unsafe.park(Unsafe.java:-1) | |
| at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) | |
| at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) | |
| at org.apache.camel.component.seda.SedaPollingConsumer.receive(SedaPollingConsumer.java:41) | |
| at org.apache.camel.impl.ConsumerCache.receive(ConsumerCache.java:202) |
holgerbrandl
/ gist:821313db8ecaf17134b2b3ce089eb111
Created
April 24, 2018 16:05
thread dump of deadlock in org.deeplearning4j.bagofwords.vectorizer.TfidfVectorizerTest#testTfIdfVectorizer
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| "main@1" prio=5 tid=0x1 nid=NA waiting | |
| java.lang.Thread.State: WAITING | |
| at java.lang.Object.wait(Object.java:-1) | |
| at java.lang.Object.wait(Object.java:502) | |
| at org.deeplearning4j.models.word2vec.wordstore.VocabConstructor$VocabRunnable.awaitDone(VocabConstructor.java:536) | |
| at org.deeplearning4j.models.word2vec.wordstore.VocabConstructor.buildJointVocabulary(VocabConstructor.java:221) | |
| at org.deeplearning4j.bagofwords.vectorizer.BaseTextVectorizer.buildVocab(BaseTextVectorizer.java:53) | |
| at org.deeplearning4j.bagofwords.vectorizer.BaseTextVectorizer.fit(BaseTextVectorizer.java:58) | |
| at org.deeplearning4j.bagofwords.vectorizer.TfidfVectorizerTest.testTfIdfVectorizer(TfidfVectorizerTest.java:71) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-1) |
holgerbrandl
/ shuffle_fasta.kts
Last active
April 9, 2018 14:41
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| @file:DependsOn("de.mpicbg.scicomp:kutils:0.10") | |
| @file:KotlinOpts("-J-Xmx5g") | |
| import de.mpicbg.scicomp.bioinfo.openFasta | |
| import java.io.File | |
| //val inputFasta = File("test.fasta") | |
| val inputFasta = File(args[0]) | |
| openFasta(inputFasta).shuffled().forEach{ println(it.toEntryString())} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| //DEPS de.mpicbg.scicomp:kutils:0.7 | |
| //KOTLIN_OPTS -J-Xmx10g -J-server | |
| import de.mpicbg.scicomp.bioinfo.openFastq | |
| import java.io.File | |
| //val args = listOf("ID.txt", "../lanereps_pooled/s1_WT_A_R1_paired.fastq") | |
| val filterIds = File(args[0]).readLines().map{ "@" + it} | |