Created
November 30, 2021 18:36
-
-
Save alexeykudinkin/8ec70674f23c797ab285fb6d2e2f14ad to your computer and use it in GitHub Desktop.
Hudi Marker exception
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| org.apache.spark.SparkException: Writing job failed. | |
| at org.apache.spark.sql.errors.QueryExecutionErrors$.writingJobFailedError(QueryExecutionErrors.scala:606) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2(WriteToDataSourceV2Exec.scala:381) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2$(WriteToDataSourceV2Exec.scala:330) | |
| at org.apache.spark.sql.execution.datasources.v2.AppendDataExec.writeWithV2(WriteToDataSourceV2Exec.scala:236) | |
| at org.apache.spark.sql.execution.datasources.v2.V2ExistingTableWriteExec.run(WriteToDataSourceV2Exec.scala:309) | |
| at org.apache.spark.sql.execution.datasources.v2.V2ExistingTableWriteExec.run$(WriteToDataSourceV2Exec.scala:308) | |
| at org.apache.spark.sql.execution.datasources.v2.AppendDataExec.run(WriteToDataSourceV2Exec.scala:236) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:43) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:43) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.executeCollect(V2CommandExec.scala:49) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:110) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:110) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:106) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481) | |
| at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:82) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:481) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:457) | |
| at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:106) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:93) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:91) | |
| at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:128) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:848) | |
| at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:311) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:247) | |
| at org.apache.hudi.HoodieSparkSqlWriter$.bulkInsertAsRow(HoodieSparkSqlWriter.scala:480) | |
| at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:162) | |
| at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:164) | |
| at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:110) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:110) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:106) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481) | |
| at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:82) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:481) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:457) | |
| at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:106) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:93) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:91) | |
| at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:128) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:848) | |
| at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:382) | |
| at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:355) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) | |
| ... 57 elided | |
| Caused by: org.apache.hudi.exception.HoodieException: Clustering failed to write to files:92c21c81-f2a6-468a-8569-716e3f77af64-0 | |
| at org.apache.hudi.internal.DataSourceInternalWriterHelper.commit(DataSourceInternalWriterHelper.java:86) | |
| at org.apache.hudi.spark3.internal.HoodieDataSourceInternalBatchWrite.commit(HoodieDataSourceInternalBatchWrite.java:93) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2(WriteToDataSourceV2Exec.scala:369) | |
| ... 121 more | |
| Caused by: org.apache.hudi.exception.HoodieClusteringException: Clustering failed to write to files:92c21c81-f2a6-468a-8569-716e3f77af64-0 | |
| at org.apache.hudi.client.SparkRDDWriteClient.completeClustering(SparkRDDWriteClient.java:378) | |
| at org.apache.hudi.client.SparkRDDWriteClient.completeTableService(SparkRDDWriteClient.java:470) | |
| at org.apache.hudi.client.SparkRDDWriteClient.cluster(SparkRDDWriteClient.java:364) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.lambda$inlineCluster$15(AbstractHoodieWriteClient.java:1073) | |
| at org.apache.hudi.common.util.Option.ifPresent(Option.java:96) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.inlineCluster(AbstractHoodieWriteClient.java:1071) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.runTableServicesInline(AbstractHoodieWriteClient.java:480) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.commitStats(AbstractHoodieWriteClient.java:206) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.commitStats(AbstractHoodieWriteClient.java:176) | |
| at org.apache.hudi.internal.DataSourceInternalWriterHelper.commit(DataSourceInternalWriterHelper.java:83) | |
| ... 123 more | |
| Suppressed: org.apache.hudi.exception.HoodieRollbackException: Failed to rollback file:///tmp/hudi/amazon_reviews_z-order commits 20211130102229352 | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.rollback(AbstractHoodieWriteClient.java:650) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.rollback(AbstractHoodieWriteClient.java:592) | |
| at org.apache.hudi.internal.DataSourceInternalWriterHelper.abort(DataSourceInternalWriterHelper.java:94) | |
| at org.apache.hudi.spark3.internal.HoodieDataSourceInternalBatchWrite.abort(HoodieDataSourceInternalBatchWrite.java:98) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2(WriteToDataSourceV2Exec.scala:376) | |
| at org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec.writeWithV2$(WriteToDataSourceV2Exec.scala:330) | |
| at org.apache.spark.sql.execution.datasources.v2.AppendDataExec.writeWithV2(WriteToDataSourceV2Exec.scala:236) | |
| at org.apache.spark.sql.execution.datasources.v2.V2ExistingTableWriteExec.run(WriteToDataSourceV2Exec.scala:309) | |
| at org.apache.spark.sql.execution.datasources.v2.V2ExistingTableWriteExec.run$(WriteToDataSourceV2Exec.scala:308) | |
| at org.apache.spark.sql.execution.datasources.v2.AppendDataExec.run(WriteToDataSourceV2Exec.scala:236) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result$lzycompute(V2CommandExec.scala:43) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result(V2CommandExec.scala:43) | |
| at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.executeCollect(V2CommandExec.scala:49) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:110) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:110) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:106) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481) | |
| at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:82) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:481) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:457) | |
| at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:106) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:93) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:91) | |
| at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:128) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:848) | |
| at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:311) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:247) | |
| at org.apache.hudi.HoodieSparkSqlWriter$.bulkInsertAsRow(HoodieSparkSqlWriter.scala:480) | |
| at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:162) | |
| at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:164) | |
| at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:110) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) | |
| at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) | |
| at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) | |
| at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) | |
| at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:110) | |
| at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:106) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481) | |
| at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:82) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:481) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267) | |
| at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) | |
| at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:457) | |
| at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:106) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:93) | |
| at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:91) | |
| at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:128) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:848) | |
| at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:382) | |
| at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:355) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239) | |
| Caused by: java.lang.IllegalArgumentException: Cannot use marker based rollback strategy on completed instant:[20211130102229352__commit__COMPLETED] | |
| at org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40) | |
| at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.<init>(BaseRollbackActionExecutor.java:90) | |
| at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.<init>(BaseRollbackActionExecutor.java:71) | |
| at org.apache.hudi.table.action.rollback.CopyOnWriteRollbackActionExecutor.<init>(CopyOnWriteRollbackActionExecutor.java:48) | |
| at org.apache.hudi.table.HoodieSparkCopyOnWriteTable.rollback(HoodieSparkCopyOnWriteTable.java:342) | |
| at org.apache.hudi.client.AbstractHoodieWriteClient.rollback(AbstractHoodieWriteClient.java:635) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment