Skip to content

Instantly share code, notes, and snippets.

@mattppal
Last active July 6, 2023 15:22
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save mattppal/033b1081f82d028ac92f36121299531e to your computer and use it in GitHub Desktop.
Demo from "What on Earth is Delta Lake?" article
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "e34bb3d0",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import pyspark\n",
"from delta import *"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1947d33a",
"metadata": {},
"outputs": [],
"source": [
"builder = pyspark.sql.SparkSession.builder.appName(\"MyApp\") \\\n",
" .config(\"spark.sql.extensions\", \"io.delta.sql.DeltaSparkSessionExtension\") \\\n",
" .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.spark.sql.delta.catalog.DeltaCatalog\")\n",
"\n",
"spark = configure_spark_with_delta_pip(builder).getOrCreate()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8ed1cc0e",
"metadata": {},
"outputs": [],
"source": [
"delta_path = './delta-playground/delta-table'\n",
"data = spark.range(0, 5)\n",
"data.write.mode(\"overwrite\").format(\"delta\").save(delta_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d7c805a1",
"metadata": {},
"outputs": [],
"source": [
"df = spark.read.format(\"delta\").load(delta_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "aea58f10",
"metadata": {},
"outputs": [],
"source": [
"df = df.union(spark.range(5, 10))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "28116862",
"metadata": {},
"outputs": [],
"source": [
"df.show()\n",
"df.write.mode(\"overwrite\").format(\"delta\").save(delta_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d37755a3",
"metadata": {},
"outputs": [],
"source": [
"df = df.union(spark.range(10, 100))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8a69c3c4",
"metadata": {},
"outputs": [],
"source": [
"df.show()\n",
"df.write.mode(\"overwrite\").format(\"delta\").save(delta_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b027200f",
"metadata": {},
"outputs": [],
"source": [
"for i in range(100,110):\n",
" df = df.union(spark.range(i, i+1))\n",
" df.write.mode(\"overwrite\").format(\"delta\").save(delta_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "afce9eee",
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"df.show()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "py3-default",
"language": "python",
"name": "pyenv_py3-default"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment