pandas-pyspark-all-val.py

# pandas
row_ct = df.shape[0]
num_ct = pd.to_numeric(df[col3], errors='coerce').count() # coerce makes nan, count drops nan
# another check using regex
num_regex = r"^((-)?[0-9]+)(,[0-9]+)*(\.[0-9]+)?$|(^$)"
all_are_nums = all(df[col3].fillna('').astype(str).apply(lambda x: re.match(num_regex, x)))

if (num_ct == row_ct) or all_are_nums:
    df[col3] = pd.to_numeric(df[col3], errors='coerce')

# pyspark
row_ct = df.count()
rowswith_col3float = df.filter(F.col(col3).cast("float").isNotNull()).count()
if row_ct == rowswith_col3float:
    df = df.withColumn(col3, F.col(col3).cast("float"))