Aditya Goenka ad1happy2go
ad1happy2go
/ 13.1_rc1_spark_schema_evol_failes.sql
Created
May 17, 2023 09:37
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| create table table1 ( | |
| id int, | |
| dt string, | |
| name string, | |
| price double, | |
| ts long | |
| ) using hudi | |
| tblproperties ( | |
| primaryKey = 'id', | |
| type = 'cow', |
ad1happy2go
/ 13.1_rc1_deltastreamer_test_spark3.2
Last active
May 17, 2023 15:53
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| hadoop fs -cp s3://rxusandbox-us-west-2/testcases/stocks/data/schema.avsc /tmp/ | |
| hadoop fs -cp s3://rxusandbox-us-west-2/testcases/stocks/data/source /tmp/source_parquet | |
| NOW=$(date '+%Y%m%dt%H%M%S') | |
| bin/spark-submit --master yarn --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer \ | |
| ~/hudi-utilities-bundle_2.12-0.13.0.jar --target-base-path /tmp/deltastreamertest/stocks${NOW} \ | |
| --target-table stocks${NOW} --table-type COPY_ON_WRITE --base-file-format PARQUET \ | |
| --source-class org.apache.hudi.utilities.sources.JsonDFSSource \ | |
| --source-ordering-field ts --payload-class org.apache.hudi.common.model.DefaultHoodieRecordPayload \ | |
| --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider \ |
ad1happy2go
/ issue_8636_deltastreamer_cleaningissue
Created
May 17, 2023 11:41
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ~/spark/spark-3.2.3-bin-hadoop3.2/bin/spark-submit \ | |
| --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer \ | |
| ~/spark/code/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-0.13.0.jar --table-type MERGE_ON_READ \ | |
| --source-class org.apache.hudi.utilities.sources.JsonKafkaSource \ | |
| --source-ordering-field ts \ | |
| --target-base-path /tmp/stock_ticks_cow \ | |
| --target-table stock_ticks_cow --props /var/demo/config/kafka-source.properties \ | |
| --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider \ | |
| --continuous \ | |
| --min-sync-interval-seconds 5 |
ad1happy2go
/ partial_updates
Created
May 17, 2023 15:12
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from pyspark.sql.types import * | |
| from pyspark.sql.functions import * | |
| import time | |
| from datetime import datetime | |
| from pyspark.sql import SparkSession | |
| from pyspark.sql import Row | |
| from datetime import date | |
| spark = SparkSession \ | |
| .builder \ |
ad1happy2go
/ 13.1_rc1_deltastreamer_test_Spark_3.1
Last active
May 17, 2023 16:31
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| hadoop fs -cp s3://rxusandbox-us-west-2/testcases/stocks/data/schema.avsc /tmp/ | |
| hadoop fs -cp s3://rxusandbox-us-west-2/testcases/stocks/data/source /tmp/source_parquet | |
| NOW=$(date '+%Y%m%dt%H%M%S') | |
| bin/spark-submit --master local --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer \ | |
| --jars /home/hadoop/v_3.1/hudi-hive-sync-bundle-0.13.0.jar,/home/hadoop/v_3.1/hudi-spark3.1-bundle_2.12-0.13.0.jar \ | |
| /home/hadoop/v_3.1/hudi-utilities-slim-bundle_2.12-0.13.0.jar \ | |
| --target-base-path /tmp/deltastreamertest/stocks${NOW} \ | |
| --target-table stocks${NOW} --table-type COPY_ON_WRITE --base-file-format PARQUET \ | |
| --source-class org.apache.hudi.utilities.sources.JsonDFSSource \ |
ad1happy2go
/ v_13.1_rc1_delatastreamer_test_spark3.3
Last active
May 17, 2023 16:17
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| hadoop fs -cp s3://rxusandbox-us-west-2/testcases/stocks/data/schema.avsc /tmp/ | |
| hadoop fs -cp s3://rxusandbox-us-west-2/testcases/stocks/data/source /tmp/source_parquet | |
| NOW=$(date '+%Y%m%dt%H%M%S') | |
| bin/spark-submit --master local --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer \ | |
| --jars /home/hadoop//v_3.3/hudi-hive-sync-bundle-0.13.0.jar,/home/hadoop/v_3.3/hudi-spark3.3-bundle_2.12-0.13.0.jar \ | |
| /home/hadoop/v_3.3/hudi-utilities-slim-bundle_2.12-0.13.0.jar \ | |
| --target-base-path /tmp/deltastreamertest/stocks${NOW} \ | |
| --target-table stocks${NOW} --table-type COPY_ON_WRITE --base-file-format PARQUET \ | |
| --source-class org.apache.hudi.utilities.sources.JsonDFSSource \ |
ad1happy2go
/ v_13.2_deltastreamertest_spark3.2
Last active
May 17, 2023 16:57
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| hadoop fs -cp s3://rxusandbox-us-west-2/testcases/stocks/data/schema.avsc /tmp/ | |
| hadoop fs -cp s3://rxusandbox-us-west-2/testcases/stocks/data/source /tmp/source_parquet | |
| NOW=$(date '+%Y%m%dt%H%M%S') | |
| bin/spark-submit --master local --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer \ | |
| --jars /home/hadoop/v_3.2/hudi-hive-sync-bundle-0.13.0.jar,/home/hadoop/v_3.2/hudi-spark3.2-bundle_2.12-0.13.0.jar \ | |
| /home/hadoop/v_3.2/hudi-utilities-slim-bundle_2.12-0.13.0.jar \ | |
| --target-base-path /tmp/deltastreamertest/stocks${NOW} \ | |
| --target-table stocks${NOW} --table-type COPY_ON_WRITE --base-file-format PARQUET \ | |
| --source-class org.apache.hudi.utilities.sources.JsonDFSSource \ |
ad1happy2go
/ issue_ 8756_schemaevolution
Last active
May 22, 2023 14:55
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import org.apache.hudi.QuickstartUtils._ | |
| import scala.collection.JavaConversions._ | |
| import org.apache.spark.sql.SaveMode | |
| import org.apache.hudi.DataSourceReadOptions._ | |
| import org.apache.hudi.DataSourceWriteOptions._ | |
| import org.apache.hudi.config.HoodieWriteConfig._ | |
| import org.apache.hudi.config._ | |
| import org.apache.hudi.common.model.HoodieRecord | |
| import org.apache.hudi.keygen.constant._ |
ad1happy2go
/ issue_5916_deltastreamer_compaction_fsview
Last active
May 23, 2023 09:47
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ~/spark/spark-3.2.3-bin-hadoop3.2/bin/spark-submit \ | |
| --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer \ | |
| ~/spark/code/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-0.13.0.jar --table-type MERGE_ON_READ \ | |
| --source-class org.apache.hudi.utilities.sources.JsonKafkaSource \ | |
| --source-ordering-field ts \ | |
| --target-base-path /tmp/issue_5916 \ | |
| --target-table stock_ticks_cow --props /var/demo/config/kafka-source.properties \ | |
| --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider \ | |
| --continuous \ | |
| --min-sync-interval-seconds 5 |
ad1happy2go
/ issue_7191_athena_data_missing
Created
May 23, 2023 16:09
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ======GLUE SCRIPT========= | |
| import sys | |
| import os | |
| from pyspark.context import SparkContext | |
| from pyspark.sql.session import SparkSession | |
| from awsglue.context import GlueContext | |
| from awsglue.job import Job | |
| from awsglue.dynamicframe import DynamicFrame | |
| from pyspark.sql.functions import col, to_timestamp, monotonically_increasing_id, to_date, when | |
| from pyspark.sql.functions import * |
OlderNewer