Skip to content

Instantly share code, notes, and snippets.

geoHeil geoHeil

Block or report user

Report or block geoHeil

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
@geoHeil
geoHeil / foo.scala
Last active Sep 20, 2019
spark custom File Stream Source
View foo.scala
package org.apache.spark.sql.execution.streaming
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
class StatefulFileStreamOptions(parameters: CaseInsensitiveMap[String])
extends FileStreamOptions(parameters) {
def this(parameters: Map[String, String]) =
this(CaseInsensitiveMap(parameters))
@geoHeil
geoHeil / foo.scala
Last active Aug 24, 2019
azure event hub captured avro file parsing in spark
View foo.scala
// using from_json
import org.apache.spark.sql.functions.from_json
import org.apache.spark.sql.functions.col
import org.apache.spark.sql.types.StringType
val schema = spark.read.json(df.select("Body").as[String]).schema
val otherColumns = df.drop("Body").columns.map(col)
val combined = otherColumns :+ from_json(col("Body").cast(StringType), schema).alias("Body_parsed")
val result = df.select(combined:_*)
@geoHeil
geoHeil / foo.scala
Created Aug 1, 2019
Validate sequence of numbers within allowed range / bounds
View foo.scala
val valuesRight = Seq(1, 2, 3, 4, 5, 6, 7)
val valuesWrong = Seq(1, 2, 5, 6, 7, 8, 9)
val allowedValues = Range(1, 8)
def containsNotAllowedValues(allowed: Range, input: Seq[Int]): Boolean = {
!allowed.containsSlice(input)
}
containsNotAllowedValues(allowedValues, valuesRight) // expected false as no wrong element contained
// result: false ==> correct
@geoHeil
geoHeil / assertion failed
Last active Apr 11, 2019
Assertion failed
View assertion failed
19/04/11 09:11:13 ERROR Executor: Exception in task 0.0 in stage 11.0 (TID 11)
java.lang.AssertionError: assertion failed
at scala.Predef$.assert(Predef.scala:156)
at org.apache.spark.sql.geosparksql.expressions.ST_GeomFromWKT.eval(Constructors.scala:123)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificUnsafeProjection.writeFields_0_0$(Unknown Source)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificUnsafeProjection.apply(Unknown Source)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificUnsafeProjection.apply(Unknown Source)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:410)
at org.apache.spark.sql.execution.columnar.CachedRDDBuilder$$anonfun$1$$anon$1.next(InMemoryRelation.scala:100)
at org.apache.spark.sql.execution.columnar.CachedRDDBuilder$$anonfun$1$$anon$1.next(InMemoryRelation.scala:90)
View gist:bec24156ae2485144d467386d19384ca
org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:226) at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(BroadcastExchangeExec.scala:146) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeBroadcast$1.apply(SparkPlan.scala:144) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeBroadcast$1.apply(SparkPlan.scala:140) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152) at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:140) at org.apache.spark.sql.execution.joins.BroadcastNestedLoopJoinExec.doExecute(BroadcastNestedLoopJoinExec.scala:343) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131) at org.apache.spark.sql.ex
View gist:e684c14a7fe5731738beaa7084b2d3af
> install.packages("sf")
Installing package into ‘/Users/geoheil/my_project/packrat/lib/x86_64-apple-darwin18.2.0/3.5.3’
(as ‘lib’ is unspecified)
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 7975k 100 7975k 0 0 2506k 0 0:00:03 0:00:03 --:--:-- 2507k
* installing *source* package ‘sf’ ...
** Paket ‘sf’ erfolgreich entpackt und MD5 Summen überprüft
configure: CC: clang
configure: CXX: clang++ -std=gnu++11
@geoHeil
geoHeil / keplergl(2).json
Created Dec 11, 2018
kepler.gl earthquakes example data earthquakes
View keplergl(2).json
{
"version": "v1",
"config": {
"visState": {
"filters": [
{
"dataId": "earthquakes",
"id": "vo18yorx",
"name": "DateTime",
"type": "timeRange",
View gist:42951311cea4c2cc3b6f2230f19ee70d
/usr/local/opt/qt/lib/QtSql.framework/Headers -isystem /usr/local/opt/qt/lib/QtConcurrent.framework/Headers -isystem /usr/local/opt/qt/lib/QtPositioning.framework/Headers -isystem /usr/local/Cellar/grass7/7.4.1/grass-base/include -Wall -Wextra -Wno-long-long -Wformat-security -Wno-strict-aliasing -Wno-return-type-c-linkage -Wno-overloaded-virtual -Qunused-arguments -O2 -g -DNDEBUG -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk -fPIC -fvisibility=hidden -F/usr/local/opt/qca/lib -fPIC -std=gnu++11 -MD -MT src/core/CMakeFiles/qgis_core.dir/processing/models/qgsprocessingmodeloutput.cpp.o -MF src/core/CMakeFiles/qgis_core.dir/processing/models/qgsprocessingmodeloutput.cpp.o.d -o src/core/CMakeFiles/qgis_core.dir/processing/models/qgsprocessingmodeloutput.cpp.o -c ../src/core/processing/models/qgsprocessingmodeloutput.cpp
[1990/4818] /usr/local/Homebrew/Library/Homebrew/shims/mac/super/clang++ -DQT_CONCURRENT_LIB -DQT_CORE_LIB -DQT_DISABLE_DEPRECAT
@geoHeil
geoHeil / foo.sh
Last active Jul 29, 2018
scala class path
View foo.sh
gradle dependencyInsight --dependency scala-reflect [±65-home-night-location ●●]
> Task :proj1:dependencyInsight
org.scala-lang:scala-reflect:2.11.11
variant "runtime" [
org.gradle.status = release (not requested)
Requested attributes not found in the selected variant:
org.gradle.usage = java-api
]
Selection reasons:
View gist:6470acd6e2bdc1b1894d14b385d65110
scala.reflect.internal.Types$TypeError: value <none> is not a member of $line23.$read
at scala.tools.nsc.typechecker.Contexts$ThrowingReporter.handleError(Contexts.scala:1402)
at scala.tools.nsc.typechecker.Contexts$ContextReporter.issue(Contexts.scala:1254)
at scala.tools.nsc.typechecker.Contexts$Context.issue(Contexts.scala:573)
at scala.tools.nsc.typechecker.Typers$Typer$$anonfun$normalTypedApply$1$1.apply(Typers.scala:4549)
at scala.tools.nsc.typechecker.Typers$Typer$$anonfun$normalTypedApply$1$1.apply(Typers.scala:4548)
at scala.tools.nsc.typechecker.Typers$Typer.onError$3(Typers.scala:4522)
at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply$1(Typers.scala:4548)
at scala.tools.nsc.typechecker.Typers$Typer.typedApply$1(Typers.scala:4580)
at scala.tools.nsc.typechecker.Typers$Typer.typedInAnyMode$1(Typers.scala:5343)
You can’t perform that action at this time.