Target specific (NOD2) coding variants (GWAS + ClinVar pathogenic)

targetMissenseVariants.py

import pyspark.sql.functions as F
from pyspark import SparkConf
from pyspark.sql import SparkSession


sparkConf = SparkConf()

spark = (
    SparkSession.builder
    .config(conf=sparkConf)
    .master('yarn')
    .getOrCreate()
)

# Platform evidence data
evidence = spark.read.parquet("gs://open-targets-data-releases/21.06/output/etl/parquet/evidence")

out = (
    evidence
    .filter(F.col("targetId") == "ENSG00000167207")
    .filter(F.col("variantId").isNotNull())
    .filter((F.col("variantFunctionalConsequenceId") == "SO_0001583") | (F.col("variantFunctionalConsequenceId") =="SO:0001587"))
    # .withColumn("clinicalSignificances", F.explode("clinicalSignificances"))
    .filter((F.col("datasourceId") == "ot_genetics_portal") | (F.array_contains(F.col("clinicalSignificances"), "pathogenic")))
    .persist()
    .select("datasourceId", "diseaseId", "variantId", "variantRsId", "variantFunctionalConsequenceId", "clinicalSignificances", "literature")
    .sort("variantId")
)