Skip to content

Instantly share code, notes, and snippets.

@ad1happy2go
Last active May 8, 2024 09:44
Show Gist options
  • Save ad1happy2go/68ccc93f3327a38c5a2bfb1d79e1e187 to your computer and use it in GitHub Desktop.
Save ad1happy2go/68ccc93f3327a38c5a2bfb1d79e1e187 to your computer and use it in GitHub Desktop.
CREATE TABLE merge_source (
ts BIGINT,
uuid STRING,
rider STRING,
driver STRING,
fare DOUBLE,
city STRING
) USING PARQUET;
INSERT INTO merge_source
VALUES
(1695159649087,'334e26e9-8355-45cc-97c6-c31daf0df330','rider-A','driver-K',19.10,'san_francisco'),
(1695091554788,'e96c4396-3fad-413a-a942-4cb36106d721','rider-C','driver-M',27.70 ,'san_francisco'),
(1695046462179,'9909a8b1-2d15-4d3d-8ec9-efc48c536a00','rider-D','driver-L',33.90 ,'san_francisco'),
(1695332066204,'1dced545-862b-4ceb-8b43-d2a568f6616b','rider-E','driver-O',93.50,'san_francisco'),
(1695516137016,'e3cf430c-889d-4015-bc98-59bdce1e530c','rider-F','driver-P',34.15,'sao_paulo' ),
(1695376420876,'7a84095f-737f-40bc-b62f-6b69664712d2','rider-G','driver-Q',43.40 ,'sao_paulo' ),
(1695173887231,'3eeb61f7-c2b0-4636-99bd-5d7a5a1d2c04','rider-I','driver-S',41.06 ,'chennai' ),
(1695115999911,'c8abbe79-8d89-47ea-b4ce-4d224bae5bfa','rider-J','driver-T',17.85,'chennai');
CREATE TABLE hudi_table (
ts BIGINT,
uuid STRING,
rider STRING,
driver STRING,
fare DOUBLE,
city STRING
) USING HUDI
PARTITIONED BY (city)
OPTIONS (
primaryKey 'uuid',
hoodie.datasource.write.operation 'upsert',
hoodie.datasource.write.precombine.field 'ts',
hoodie.datasource.write.recordkey.field 'uuid',
hoodie.table.name 'issue_11138',
hoodie.datasource.write.table.type 'MERGE_ON_READ'
);
insert into hudi_table
select * from merge_source;
merge into hudi_table as target
using merge_source as source
on target.uuid = source.uuid
when matched then update set target.city = source.city ,
target.fare = source.fare, target.ts = source.ts;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment