#apache-spark #pyspark #info

info.py

"""Information Script for Apache Spark.

How to use?
 > spark-submit info.py
"""
import os
import platform
import sys
from contextlib import contextmanager

from pyspark.sql import SparkSession


@contextmanager
def spark_session() -> SparkSession:
    spark = SparkSession.builder.appName("info").getOrCreate()
    yield spark
    spark.stop()


def main() -> None:
    with spark_session() as spark:
        ctx = spark.sparkContext
        print("\n 🌎 Environment Info")
        print(" ===================")
        print(f"  🐍 Python => {platform.python_version()} ")
        print(f"  🍺 CPUs   => {os.cpu_count()}")
        print(f"  💥 Spark  => {spark.version}")
        print(f"  🐘 Hadoop => {ctx._jvm.org.apache.hadoop.util.VersionInfo.getVersion()}")

        print("\n 🔧 Spark Config")
        print(" ===============")
        configurations = ctx.getConf().getAll()
        for conf in configurations:
            print(f"  {conf[0]} => {conf[1]}")
        print("")


if __name__ == "__main__":
    main()
    sys.exit(0)