[filter RDD transformation] filter RDD 변환 예시 #pyspark #pyspark101

part_4_filter_transformation_pyspark101_demo.py

# Importing Spark Related Packages
from pyspark.sql import SparkSession

if __name__ == "__main__":
    print("PySpark 101 Tutorial")
    print("Part 4. How to use filter RDD transformation in PySpark using PyCharm IDE")

    spark = SparkSession \
            .builder \
            .appName("Part 4. How to use filter RDD transformation in PySpark using PyCharm IDE") \
            .master("local[*]") \
            .enableHiveSupport() \
            .getOrCreate()

    py_number_list = [1,2,3,4,5]
    print("Printing Python Number List: ")
    print(py_number_list)

    print("Creating First RDD from Python Number List")

    # 숫자 3의 의미는 파티션을 갯수
    number_rdd = spark.sparkContext.parallelize(py_number_list)
    number_even_rdd = number_rdd.filter(lambda n: n % 2 == 0)

    print(number_even_rdd.collect())

    py_str_list = ["Arun", "Arvind", "Arjun", "Anna"]
    print(py_str_list)

    str_rdd = spark.sparkContext.parallelize(py_str_list, 2)
    str_rdd_result = str_rdd.filter(lambda name: "r" in name).collect()
    print(str_rdd_result)

    input_file_path = "file:///g:/WS/data/pyspark101/tech.txt"
    tech_rdd = spark.sparkContext.textFile(input_file_path)
    tech_lower_rdd = tech_rdd.filter(lambda ele: "park" in ele)
    tech_lower_rdd_list = tech_lower_rdd.collect()

    for element in tech_lower_rdd_list:
        print(element)

    print("Stopping the SparkSession object")
    spark.stop()