pyspark_forward_rate.py

# Implementation with PySpark
from pyspark.sql import SparkSession
from pyspark.sql import Window
from pyspark.sql import functions as f

# Define Spark settings
builder = SparkSession.builder.appName("Forward_Rate_Calculation")
spark = builder.getOrCreate()

# Read in the spot rate data as csv
data = spark.read.csv(rate_path, header=True, inferSchema=True)

# Shift the spot rate down along the maturity axis and calculate forward rate
window = Window.orderBy("Maturity")
data = data.withColumn("Spot_Rate_Shift", f.lag(f.col("Spot_Rate"), 1).over(window)) \
           .fillna(0, subset=["Spot_Rate_Shift"])
data = data.withColumn("Forward_Rate", (1 + f.col("Spot_Rate"))**f.col("Maturity") /
                                       ((1 + f.col("Spot_Rate_Shift"))**(f.col("Maturity") - 1)) - 1)