PYTHON

pyspark check all columns for null values

from pyspark.sql.functions import isnan, when, count, col

df.select([count(when(isnan(c), c)).alias(c) for c in df.columns]).show()
+-------+----------+---+
|session|timestamp1|id2|
+-------+----------+---+
|      0|         0|  3|
+-------+----------+---+

check null all column pyspark

data_indexed.select([count(when(col(c).isNull(),c)).alias(c) for c in data_indexed.columns]).toPandas().T