This is an useful function helps in feature selection process of your ML workflow.

Find null count for all columns in pysark

from pyspark.sql.functions import col

from pyspark.sql.functions import sum as spark_sum

def count_null(col_name):

    return spark_sum(col(col_name).isNull().cast('integer')).alias(col_name)

# Build up a list of column expressions, one per column.

exprs = [count_null(col_name) for col_name in logs_df.columns]

# Run the aggregation. The *exprs converts the list of expressions into

# variable function arguments.

logs_df.agg(*exprs).show()