Hello.scala

object LoadData extends SparkSqlJob {
  def runJob(sc: SQLContext, jobConfig: Config): Any = {
    val data = sc.read.format("jdbc").options(Map(
      "url" -> "connection details",
      "dbtable" -> "query",
      "driver" -> "org.postgresql.Driver",
      "partitionColumn" -> "id",
      "lowerBound" -> "0",
      "upperBound" -> "89306",
      "numPartitions" -> "10"
    )).load()

    data.registerTempTable("data")
    sc.cacheTable("data")
  }

  def validate(sc: SQLContext, config: Config): SparkJobValidation = SparkJobValid
}

object QueryData extends SparkSqlJob {
  def runJob(sc: SQLContext, jobConfig: Config): Any = {
    val data = sc.table("data")

    val query = "0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00123096027609 0.000473046781683 0.0 0.0 0.0 0.0009488330825 0.0 0.0 0.0 0.0 0.0 0.0 0.0137392766078 0.0 0.0 0.0 0.0191767597593 0.0 0.0 0.0 0.0 0.000731426233752 0.0 0.0 0.00435365929053 0.0 0.0 0.00295592675776 0.0 0.0 0.0 0.0 0.0 0.0 0.180746677081 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.02610323854 0.0 0.0 0.0 0.0 0.0 0.0462299070678 0.0 0.0 0.0 0.0 0.000637902847294 0.0 0.0 0.0 0.0 0.0101081124241 0.0 0.0 0.0 0.0 0.0487236424484 0.0 0.0 0.0 0.0 0.0 0.0 0.000287902480496 0.0 0.0 0.00132419761976 0.0 0.0 0.0 0.0 0.0352488409208 0.00491106672027 0.0 0.0 0.0369623466178 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.205633647203 0.0141100799644 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00324107901134 0.0387467485404 0.0122842871101 0.136911815335 0.00521904127299 0.0 0.0 0.0 0.0 0.0 0.0 0.0316491986631 0.0 0.0 0.116035059531 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.07693036768 0.0 0.0 0.0 0.00173134073514 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.114135870184 0.0 0.0 0.0 0.0 0.0467395225428 0.0 0.0 0.0 0.0 0.0 0.0 0.0259483415483 0.0 0.0 0.0 0.0 0.0167253094104 0.0 0.0 0.0 0.00738929901697 0.0 0.00342665759117 0.0 0.0570243472801 0.19436930311 0.0 0.0980731547624 0.0 0.190623378272 0.0 0.0 0.0 0.0 0.0 0.000206974931431 0.0 0.0 0.0 0.0 0.0 0.000830180929477 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0131384932911 0.0 0.0 0.0 0.0 0.0 0.00711158911813 0.0 0.0 0.0 0.0 0.00588456696354 0.0 0.0113746842182 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0009321336828 0.0 0.0 0.0 0.0680734185514 0.0 0.0 0.0 0.0 0.0 0.000385862681096 0.0 0.0628536603032 0.0 0.0 0.0 0.0 0.00697003134604 0.0 0.0 0.0 0.0 0.0 0.0471929400079 0.0 0.0281585919663 0.0 0.0 0.0 0.0167220060596 0.0 0.0864424403349 0.0 0.0 0.0 0.0 0.0 0.0 0.0436454767725 0.0 0.0 0.0 0.0 0.000336024340904 0.0 0.0 0.0 0.028890352874 0.0 0.0 0.0 0.0 0.00874431664527 0.0 0.0858322466624 0.0 0.0 0.0 0.0 0.00159247797796 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0599345475696 0.00370547061464 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0996452856119 0.000475912786494 0.0 0.0 0.0 0.011700922678 0.0 0.0 0.0 0.0 0.000442778725852 0.0 0.0 0.0203650005578 0.0 0.0 0.0 0.0 0.0 0.0421325492504 0.0 0.0 0.0 0.415417659314 0.0 0.00113677607447 0.0 0.0 0.0 0.00445690684895 0.0 0.0 0.0 0.0 0.0 0.0853438575808 0.000711050105131 0.0 0.0 0.0 0.0177522785176 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000381763468824 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0368837943281 0.0 0.0 0.0519460067104 0.0 0.0029726128285 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0138733451786 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0632603561456 0.0 0.0 0.0 0.288229623764 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00389770854264 0.0149997217658 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00519569206174 0.0213788990323 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0635546274141 0.000654888234009 0.0 0.0 0.0 0.0 0.0 0.0 0.000600471006369 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0138094670119 0.0 0.0 0.0267196570032 0.0 0.0179215578453 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00500407897744 0.148294501836 0.00720312013211 0.0 0.0 0.000360399714611 0.0928538248254 0.0 0.0 0.000557436095364 0.0 0.0 0.0 0.0 0.0 0.0 0.0405868701919 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0195579889266 0.0 0.0 0.0 0.0 0.00630700101488 0.0 0.0 0.000446989616945 0.0 0.0 0.0105847265146 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0176688809564 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0846420766784 0.0 0.0 0.0 0.0 0.0307973303479 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.000602464494245 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00099768454999 0.00717811790699 0.0338753358605 0.0 0.197224626016 0.0 3.87455624026e-05 0.0 0.0 0.0612866334964 0.0 0.0 0.0 0.000168674616874 0.0 0.0 0.0309610630444 0.0 0.0 0.00260002261067 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0247249310394 0.0 0.0276168263754 0.0 0.029380241045 0.0033774307936 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0053702066026 0.0 0.0148745736698 0.0 0.0 0.0881998729235 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.028654506834 0.0 0.0 0.0 0.0 0.0 0.0161957453373 0.0 0.0275963352497 0.0 0.0 0.0 0.0221736038526 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0108980174811 0.000650896015904 0.0 0.0 0.0 0.0 0.190557618212 0.0 0.0116607194176 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0861238481239 0.0 0.0 0.0 0.0130341075846 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.216109384557 2.19044884966e-05 0.0 0.0 0.0 0.0619378736671 0.00245634871313 0.0 0.0 0.1406755722 0.0 0.0 0.0 0.0 0.0185194768302 0.0465857088217 0.0 0.0 0.0 0.0802824174607 0.0 0.0 0.0171049234871 0.0 0.0128603265269 0.00256795101855 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0374171453868 0.0 0.0 0.000966296136684 5.4431225835e-07 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.109049887827 0.0 0.0 0.0 0.0 0.0 0.0 0.106060724783 0.00498137044848 0.0 0.0 0.0 0.0 0.0 0.0459195848338 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0933268446703 0.0 0.00138154314697 0.0 0.0 0.0722743099143 0.0 0.0 0.0161127600258 0.0 0.0 0.125571845747 0.0 0.0 0.0 0.0137959136367 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 9.33280374029e-06 0.0 0.0 0.049354673465 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00758299342642 0.0892172121653 0.0 0.0 0.0 0.00698627639 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00380089279203 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00431186681687 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0234355034844 0.0 0.0 0.0 0.0513738078181 0.0693086111899 0.368580306905 0.0 0.0921587826873 0.0 0.0811292673953 0.0 0.0 0.0 0.0".split(" ").map(s => s.toFloat)

    val result1 = data.map { row =>
      val (id, hash) = row(0).asInstanceOf[Long] -> row(1).asInstanceOf[String]
      id -> hash.split(" ").map(s => s.toFloat).zip(hash1).map(pair => pair._1 * pair._2).sum
    }.sortBy(_._2, ascending = false)

    result1.take(10)
  }

  def validate(sc: SQLContext, config: Config): SparkJobValidation = SparkJobValid
}