blog-lookup-table-7-base-plus-lookup.scala

import org.apache.spark.sql.Window

val competitorWindow = Window
  .partitionBy("date", "competitor")
  .orderBy(levenshtein($"competitor", $"short_name"))

val scheduleRich = schedule
  .join(
    teams, levenshtein($"competitor", $"short_name") < 5, "left_outer"
  )
  .withColumn("_rank", row_number().over(competitorWindow))
  .filter($"_rank" === 1)
  .drop("_rank")

scheduleRich.drop("short_name").show(2)

+----------+----------+---------------+--------+------------------+
|      date|competitor|           team|division|        conference|
+----------+----------+---------------+--------+------------------+
|2016-10-05|   Anaheim|  Anaheim Ducks| Pacific|Western Conference|
|2016-10-09|   Anaheim|  Anaheim Ducks| Pacific|Western Conference|
+----------+----------+---------------+--------+------------------+