HoodieDeltaStreamerCoW.sh

DATA_LAKE_BUCKET="<your_data_lake_s3_bucket>"
TARGET_TABLE="tickit.ecomm.sale"

spark-submit \
  --name %{TARGET_TABLE} \
  --jars /usr/lib/spark/jars/spark-avro.jar,/usr/lib/hudi/hudi-utilities-bundle.jar \
  --conf spark.sql.catalogImplementation=hive \
  --conf spark.yarn.submit.waitAppCompletion=false \
  --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer `ls /usr/lib/hudi/hudi-utilities-bundle.jar` \
  --props file://${PWD}/${TARGET_TABLE}.properties \
  --table-type COPY_ON_WRITE \
  --source-ordering-field __source_ts_ms \
  --source-class org.apache.hudi.utilities.sources.AvroDFSSource \
  --schemaprovider-class org.apache.hudi.utilities.schema.SchemaRegistryProvider \
  --target-table ${TARGET_TABLE} \
  --target-base-path s3://${DATA_LAKE_BUCKET}/cdc_hudi_data_lake/silver/${TARGET_TABLE}/ \
  --enable-sync \
  --continuous \
  --op UPSERT \
  > ${TARGET_TABLE}.log 2>&1 &