-
-
Save yihua/e3afe0f34400e60f81f6da925560118e to your computer and use it in GitHub Desktop.
Duplicates are deleted with Bloom Index in Apache Hudi
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
An example demonstrating the following scenario with the Bloom Index: | |
(1) Write duplicates to a Hudi table from two commits, so that primary key “0246b883-2a11-4b92-97a1-f2a5eee76004” exists | |
in two data files: “57ea9a3a-8957-4f7c-87c8-4e1f94d22e17-0_0-68-186_20230406141013228.parquet” | |
and “c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_0-15-71_20230406140955410.parquet”; | |
(2) Delete the record; | |
(3) After the deletion, the primary key does not exist anymore in the table, both records are deleted, | |
and the two data files are re-written (“57ea9a3a-8957-4f7c-87c8-4e1f94d22e17-0_0-146-380_20230406141019422.parquet”, | |
and “c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_1-146-381_20230406141019422.parquet”). | |
org.apache.hudi#hudi-spark3.3-bundle_2.12 added as a dependency | |
:: resolving dependencies :: org.apache.spark#spark-submit-parent-60267dc5-1ea6-4aca-9390-4eb347656dbc;1.0 | |
confs: [default] | |
found org.apache.hudi#hudi-spark3.3-bundle_2.12;0.13.0 in central | |
:: resolution report :: resolve 121ms :: artifacts dl 3ms | |
:: modules in use: | |
org.apache.hudi#hudi-spark3.3-bundle_2.12;0.13.0 from central in [default] | |
--------------------------------------------------------------------- | |
| | modules || artifacts | | |
| conf | number| search|dwnlded|evicted|| number|dwnlded| | |
--------------------------------------------------------------------- | |
| default | 1 | 0 | 0 | 0 || 1 | 0 | | |
--------------------------------------------------------------------- | |
:: retrieving :: org.apache.spark#spark-submit-parent-60267dc5-1ea6-4aca-9390-4eb347656dbc | |
confs: [default] | |
0 artifacts copied, 1 already retrieved (0kB/3ms) | |
23/04/06 14:09:38 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | |
Setting default log level to "WARN". | |
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). | |
Spark context Web UI available at http://192.168.1.101:5555 | |
Spark context available as 'sc' (master = local[8], app id = local-1680815384230). | |
Spark session available as 'spark'. | |
Welcome to | |
____ __ | |
/ __/__ ___ _____/ /__ | |
_\ \/ _ \/ _ `/ __/ '_/ | |
/___/ .__/\_,_/_/ /_/\_\ version 3.3.1 | |
/_/ | |
Using Scala version 2.12.15 (OpenJDK 64-Bit Server VM, Java 1.8.0_292) | |
Type in expressions to have them evaluated. | |
Type :help for more information. | |
scala> import org.apache.hudi.QuickstartUtils._ | |
import org.apache.hudi.QuickstartUtils._ | |
scala> import scala.collection.JavaConversions._ | |
import scala.collection.JavaConversions._ | |
scala> val tableName = "hudi_trips_cow" | |
tableName: String = hudi_trips_cow | |
scala> val basePath = "/tmp/hudi_trips_cow" | |
basePath: String = /tmp/hudi_trips_cow | |
scala> val dataGen = new DataGenerator | |
dataGen: org.apache.hudi.QuickstartUtils.DataGenerator = org.apache.hudi.QuickstartUtils$DataGenerator@247dad0f | |
scala> val inserts = convertToStringList(dataGen.generateInserts(10)) | |
inserts: java.util.List[String] = [{"ts": 1680336472738, "uuid": "0246b883-2a11-4b92-97a1-f2a5eee76004", "rider": "rider-213", "driver": "driver-213", "begin_lat": 0.4726905879569653, "begin_lon": 0.46157858450465483, "end_lat": 0.754803407008858, "end_lon": 0.9671159942018241, "fare": 34.158284716382845, "partitionpath": "americas/brazil/sao_paulo"}, "partitionpath": "americas/brazil/sao_paulo"}, {"ts": 1680532340694, "uuid": "a3babedf-afb5-4a41-b8ee-ae482ec033d0", "rider": "rider-213", "driver": "driver-213", "begin_lat": 0.6100070562136587, "begin_lon": 0.8779402295427752, "end_lat": 0.3407870505929602, "end_lon": 0.5030798142293655, "fare": 43.4923811219014, "partitionpath": "americas/brazil/sao_paulo"}, "partitionpath": "americas/brazil/sao_paulo"}, {"ts":... | |
scala> val df = spark.read.json(spark.sparkContext.parallelize(inserts)) | |
df: org.apache.spark.sql.DataFrame = [begin_lat: double, begin_lon: double ... 8 more fields] | |
scala> df.write.format("hudi"). | |
| option("hoodie.datasource.write.precombine.field", "ts"). | |
| option("hoodie.datasource.write.recordkey.field", "uuid"). | |
| option("hoodie.datasource.write.partitionpath.field", "partitionpath"). | |
| option("hoodie.table.name", tableName). | |
| mode("overwrite"). | |
| save(basePath) | |
scala> spark.read.format("hudi").load(basePath).count() | |
res1: Long = 10 | |
scala> spark.read.format("hudi").load(basePath).select("_hoodie_record_key").distinct().count() | |
res2: Long = 10 | |
scala> spark.read.format("hudi").load(basePath).orderBy("_hoodie_partition_path", "_hoodie_record_key").show(false) | |
+-------------------+---------------------+------------------------------------+------------------------------------+------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ | |
|_hoodie_commit_time|_hoodie_commit_seqno |_hoodie_record_key |_hoodie_partition_path |_hoodie_file_name |begin_lat |begin_lon |driver |end_lat |end_lon |fare |rider |ts |uuid |partitionpath | | |
+-------------------+---------------------+------------------------------------+------------------------------------+------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ | |
|20230406140955410 |20230406140955410_0_0|0246b883-2a11-4b92-97a1-f2a5eee76004|americas/brazil/sao_paulo |c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_0-15-71_20230406140955410.parquet|0.4726905879569653 |0.46157858450465483|driver-213|0.754803407008858 |0.9671159942018241 |34.158284716382845|rider-213|1680336472738|0246b883-2a11-4b92-97a1-f2a5eee76004|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_0_2|8bfc444a-b5a4-4c5d-82cc-b1555e8cb207|americas/brazil/sao_paulo |c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_0-15-71_20230406140955410.parquet|0.0750588760043035 |0.03844104444445928|driver-213|0.04376353354538354|0.6346040067610669 |66.62084366450246 |rider-213|1680257010805|8bfc444a-b5a4-4c5d-82cc-b1555e8cb207|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_0_1|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo |c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_0-15-71_20230406140955410.parquet|0.6100070562136587 |0.8779402295427752 |driver-213|0.3407870505929602 |0.5030798142293655 |43.4923811219014 |rider-213|1680532340694|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_1_2|098af893-3e91-4f6f-94be-880d9dc5d4fa|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet|0.8742041526408587 |0.7528268153249502 |driver-213|0.9197827128888302 |0.362464770874404 |19.179139106643607|rider-213|1680303958541|098af893-3e91-4f6f-94be-880d9dc5d4fa|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_3|284aea0d-fd32-4d58-87f2-717407b4f65b|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet|0.21624150367601136|0.14285051259466197|driver-213|0.5890949624813784 |0.0966823831927115 |93.56018115236618 |rider-213|1680376044322|284aea0d-fd32-4d58-87f2-717407b4f65b|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_1|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet|0.5731835407930634 |0.4923479652912024 |driver-213|0.08988581780930216|0.42520899698713666|64.27696295884016 |rider-213|1680432389599|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_0|7d641875-87ee-4cf7-abb0-219d2c1d1d61|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet|0.11488393157088261|0.6273212202489661 |driver-213|0.7454678537511295 |0.3954939864908973 |27.79478688582596 |rider-213|1680467994066|7d641875-87ee-4cf7-abb0-219d2c1d1d61|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_4|91841d56-7516-4764-8974-ce128c243d98|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet|0.1856488085068272 |0.9694586417848392 |driver-213|0.38186367037201974|0.25252652214479043|33.92216483948643 |rider-213|1680697484636|91841d56-7516-4764-8974-ce128c243d98|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_2_0|7da4abd7-0b01-4d65-a996-9d4ebc6f60b7|asia/india/chennai |0bce535e-b5b9-433f-9a5f-bce4be0e357a-0_2-15-73_20230406140955410.parquet|0.40613510977307 |0.5644092139040959 |driver-213|0.798706304941517 |0.02698359227182834|17.851135255091155|rider-213|1680468252048|7da4abd7-0b01-4d65-a996-9d4ebc6f60b7|asia/india/chennai | | |
|20230406140955410 |20230406140955410_2_1|e5dedb56-9ae3-43d7-b181-65e1d8ed0a1d|asia/india/chennai |0bce535e-b5b9-433f-9a5f-bce4be0e357a-0_2-15-73_20230406140955410.parquet|0.651058505660742 |0.8192868687714224 |driver-213|0.20714896002914462|0.06224031095826987|41.06290929046368 |rider-213|1680573462843|e5dedb56-9ae3-43d7-b181-65e1d8ed0a1d|asia/india/chennai | | |
+-------------------+---------------------+------------------------------------+------------------------------------+------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ | |
scala> val df2 = spark.read.json(spark.sparkContext.parallelize(inserts.slice(0, 3))) | |
df2: org.apache.spark.sql.DataFrame = [begin_lat: double, begin_lon: double ... 8 more fields] | |
scala> df2.write.format("hudi"). | |
| option("hoodie.datasource.write.operation", "insert"). | |
| option("hoodie.datasource.write.precombine.field", "ts"). | |
| option("hoodie.datasource.write.recordkey.field", "uuid"). | |
| option("hoodie.datasource.write.partitionpath.field", "partitionpath"). | |
| option("hoodie.table.name", tableName). | |
| option("hoodie.parquet.small.file.limit", "0"). | |
| mode("append"). | |
| save(basePath) | |
scala> spark.read.format("hudi").load(basePath).count() | |
res5: Long = 13 | |
scala> spark.read.format("hudi").load(basePath).select("_hoodie_record_key").distinct().count() | |
res6: Long = 10 | |
scala> spark.read.format("hudi").load(basePath).orderBy("_hoodie_partition_path", "_hoodie_record_key").show(false) | |
+-------------------+---------------------+------------------------------------+------------------------------------+-------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ | |
|_hoodie_commit_time|_hoodie_commit_seqno |_hoodie_record_key |_hoodie_partition_path |_hoodie_file_name |begin_lat |begin_lon |driver |end_lat |end_lon |fare |rider |ts |uuid |partitionpath | | |
+-------------------+---------------------+------------------------------------+------------------------------------+-------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ | |
|20230406141013228 |20230406141013228_0_0|0246b883-2a11-4b92-97a1-f2a5eee76004|americas/brazil/sao_paulo |57ea9a3a-8957-4f7c-87c8-4e1f94d22e17-0_0-68-186_20230406141013228.parquet|0.4726905879569653 |0.46157858450465483|driver-213|0.754803407008858 |0.9671159942018241 |34.158284716382845|rider-213|1680336472738|0246b883-2a11-4b92-97a1-f2a5eee76004|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_0_0|0246b883-2a11-4b92-97a1-f2a5eee76004|americas/brazil/sao_paulo |c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_0-15-71_20230406140955410.parquet |0.4726905879569653 |0.46157858450465483|driver-213|0.754803407008858 |0.9671159942018241 |34.158284716382845|rider-213|1680336472738|0246b883-2a11-4b92-97a1-f2a5eee76004|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_0_2|8bfc444a-b5a4-4c5d-82cc-b1555e8cb207|americas/brazil/sao_paulo |c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_0-15-71_20230406140955410.parquet |0.0750588760043035 |0.03844104444445928|driver-213|0.04376353354538354|0.6346040067610669 |66.62084366450246 |rider-213|1680257010805|8bfc444a-b5a4-4c5d-82cc-b1555e8cb207|americas/brazil/sao_paulo | | |
|20230406141013228 |20230406141013228_0_1|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo |57ea9a3a-8957-4f7c-87c8-4e1f94d22e17-0_0-68-186_20230406141013228.parquet|0.6100070562136587 |0.8779402295427752 |driver-213|0.3407870505929602 |0.5030798142293655 |43.4923811219014 |rider-213|1680532340694|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_0_1|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo |c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_0-15-71_20230406140955410.parquet |0.6100070562136587 |0.8779402295427752 |driver-213|0.3407870505929602 |0.5030798142293655 |43.4923811219014 |rider-213|1680532340694|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_1_2|098af893-3e91-4f6f-94be-880d9dc5d4fa|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.8742041526408587 |0.7528268153249502 |driver-213|0.9197827128888302 |0.362464770874404 |19.179139106643607|rider-213|1680303958541|098af893-3e91-4f6f-94be-880d9dc5d4fa|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_3|284aea0d-fd32-4d58-87f2-717407b4f65b|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.21624150367601136|0.14285051259466197|driver-213|0.5890949624813784 |0.0966823831927115 |93.56018115236618 |rider-213|1680376044322|284aea0d-fd32-4d58-87f2-717407b4f65b|americas/united_states/san_francisco| | |
|20230406141013228 |20230406141013228_1_0|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco|bab49cd1-b09e-43a3-b694-76628b3c7a60-0_1-68-187_20230406141013228.parquet|0.5731835407930634 |0.4923479652912024 |driver-213|0.08988581780930216|0.42520899698713666|64.27696295884016 |rider-213|1680432389599|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_1|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.5731835407930634 |0.4923479652912024 |driver-213|0.08988581780930216|0.42520899698713666|64.27696295884016 |rider-213|1680432389599|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_0|7d641875-87ee-4cf7-abb0-219d2c1d1d61|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.11488393157088261|0.6273212202489661 |driver-213|0.7454678537511295 |0.3954939864908973 |27.79478688582596 |rider-213|1680467994066|7d641875-87ee-4cf7-abb0-219d2c1d1d61|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_4|91841d56-7516-4764-8974-ce128c243d98|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.1856488085068272 |0.9694586417848392 |driver-213|0.38186367037201974|0.25252652214479043|33.92216483948643 |rider-213|1680697484636|91841d56-7516-4764-8974-ce128c243d98|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_2_0|7da4abd7-0b01-4d65-a996-9d4ebc6f60b7|asia/india/chennai |0bce535e-b5b9-433f-9a5f-bce4be0e357a-0_2-15-73_20230406140955410.parquet |0.40613510977307 |0.5644092139040959 |driver-213|0.798706304941517 |0.02698359227182834|17.851135255091155|rider-213|1680468252048|7da4abd7-0b01-4d65-a996-9d4ebc6f60b7|asia/india/chennai | | |
|20230406140955410 |20230406140955410_2_1|e5dedb56-9ae3-43d7-b181-65e1d8ed0a1d|asia/india/chennai |0bce535e-b5b9-433f-9a5f-bce4be0e357a-0_2-15-73_20230406140955410.parquet |0.651058505660742 |0.8192868687714224 |driver-213|0.20714896002914462|0.06224031095826987|41.06290929046368 |rider-213|1680573462843|e5dedb56-9ae3-43d7-b181-65e1d8ed0a1d|asia/india/chennai | | |
+-------------------+---------------------+------------------------------------+------------------------------------+-------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ | |
scala> val deletes = dataGen.generateDeletes(df2.collectAsList().slice(0, 1)) | |
deletes: java.util.List[String] = [{"ts": "1680336472738","uuid": "0246b883-2a11-4b92-97a1-f2a5eee76004","partitionpath": "americas/brazil/sao_paulo"}] | |
scala> val deleteDf = spark.read.json(spark.sparkContext.parallelize(deletes)) | |
deleteDf: org.apache.spark.sql.DataFrame = [partitionpath: string, ts: string ... 1 more field] | |
scala> deleteDf.write.format("hudi"). | |
| option("hoodie.datasource.write.operation", "delete"). | |
| option("hoodie.datasource.write.precombine.field", "ts"). | |
| option("hoodie.datasource.write.recordkey.field", "uuid"). | |
| option("hoodie.datasource.write.partitionpath.field", "partitionpath"). | |
| option("hoodie.table.name", tableName). | |
| option("hoodie.index.type", "BLOOM"). | |
| mode("append"). | |
| save(basePath) | |
scala> spark.read.format("hudi").load(basePath).count() | |
res9: Long = 11 | |
scala> spark.read.format("hudi").load(basePath).select("_hoodie_record_key").distinct().count() | |
res10: Long = 9 | |
scala> spark.read.format("hudi").load(basePath).orderBy("_hoodie_partition_path", "_hoodie_record_key").show(false) | |
+-------------------+---------------------+------------------------------------+------------------------------------+--------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ | |
|_hoodie_commit_time|_hoodie_commit_seqno |_hoodie_record_key |_hoodie_partition_path |_hoodie_file_name |begin_lat |begin_lon |driver |end_lat |end_lon |fare |rider |ts |uuid |partitionpath | | |
+-------------------+---------------------+------------------------------------+------------------------------------+--------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ | |
|20230406140955410 |20230406140955410_0_2|8bfc444a-b5a4-4c5d-82cc-b1555e8cb207|americas/brazil/sao_paulo |c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_1-146-381_20230406141019422.parquet|0.0750588760043035 |0.03844104444445928|driver-213|0.04376353354538354|0.6346040067610669 |66.62084366450246 |rider-213|1680257010805|8bfc444a-b5a4-4c5d-82cc-b1555e8cb207|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_0_1|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo |c099eab3-0a43-47af-8ed5-b51ba6f488cf-0_1-146-381_20230406141019422.parquet|0.6100070562136587 |0.8779402295427752 |driver-213|0.3407870505929602 |0.5030798142293655 |43.4923811219014 |rider-213|1680532340694|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo | | |
|20230406141013228 |20230406141013228_0_1|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo |57ea9a3a-8957-4f7c-87c8-4e1f94d22e17-0_0-146-380_20230406141019422.parquet|0.6100070562136587 |0.8779402295427752 |driver-213|0.3407870505929602 |0.5030798142293655 |43.4923811219014 |rider-213|1680532340694|a3babedf-afb5-4a41-b8ee-ae482ec033d0|americas/brazil/sao_paulo | | |
|20230406140955410 |20230406140955410_1_2|098af893-3e91-4f6f-94be-880d9dc5d4fa|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.8742041526408587 |0.7528268153249502 |driver-213|0.9197827128888302 |0.362464770874404 |19.179139106643607|rider-213|1680303958541|098af893-3e91-4f6f-94be-880d9dc5d4fa|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_3|284aea0d-fd32-4d58-87f2-717407b4f65b|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.21624150367601136|0.14285051259466197|driver-213|0.5890949624813784 |0.0966823831927115 |93.56018115236618 |rider-213|1680376044322|284aea0d-fd32-4d58-87f2-717407b4f65b|americas/united_states/san_francisco| | |
|20230406141013228 |20230406141013228_1_0|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco|bab49cd1-b09e-43a3-b694-76628b3c7a60-0_1-68-187_20230406141013228.parquet |0.5731835407930634 |0.4923479652912024 |driver-213|0.08988581780930216|0.42520899698713666|64.27696295884016 |rider-213|1680432389599|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_1|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.5731835407930634 |0.4923479652912024 |driver-213|0.08988581780930216|0.42520899698713666|64.27696295884016 |rider-213|1680432389599|503879f9-e3b2-499f-9c7c-e18f3f44f6d1|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_0|7d641875-87ee-4cf7-abb0-219d2c1d1d61|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.11488393157088261|0.6273212202489661 |driver-213|0.7454678537511295 |0.3954939864908973 |27.79478688582596 |rider-213|1680467994066|7d641875-87ee-4cf7-abb0-219d2c1d1d61|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_1_4|91841d56-7516-4764-8974-ce128c243d98|americas/united_states/san_francisco|ab9e923e-6210-4f85-9fbb-516204be8353-0_1-15-72_20230406140955410.parquet |0.1856488085068272 |0.9694586417848392 |driver-213|0.38186367037201974|0.25252652214479043|33.92216483948643 |rider-213|1680697484636|91841d56-7516-4764-8974-ce128c243d98|americas/united_states/san_francisco| | |
|20230406140955410 |20230406140955410_2_0|7da4abd7-0b01-4d65-a996-9d4ebc6f60b7|asia/india/chennai |0bce535e-b5b9-433f-9a5f-bce4be0e357a-0_2-15-73_20230406140955410.parquet |0.40613510977307 |0.5644092139040959 |driver-213|0.798706304941517 |0.02698359227182834|17.851135255091155|rider-213|1680468252048|7da4abd7-0b01-4d65-a996-9d4ebc6f60b7|asia/india/chennai | | |
|20230406140955410 |20230406140955410_2_1|e5dedb56-9ae3-43d7-b181-65e1d8ed0a1d|asia/india/chennai |0bce535e-b5b9-433f-9a5f-bce4be0e357a-0_2-15-73_20230406140955410.parquet |0.651058505660742 |0.8192868687714224 |driver-213|0.20714896002914462|0.06224031095826987|41.06290929046368 |rider-213|1680573462843|e5dedb56-9ae3-43d7-b181-65e1d8ed0a1d|asia/india/chennai | | |
+-------------------+---------------------+------------------------------------+------------------------------------+--------------------------------------------------------------------------+-------------------+-------------------+----------+-------------------+-------------------+------------------+---------+-------------+------------------------------------+------------------------------------+ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment