Spark configuration changes in Dagster 1.10.7

spark_configuration_changes.md

2,11c2,4
< spark.blacklist.application.fetchFailure.enabled
< spark.blacklist.application.maxFailedExecutorsPerNode
< spark.blacklist.application.maxFailedTasksPerExecutor
< spark.blacklist.enabled
< spark.blacklist.killBlacklistedExecutors
< spark.blacklist.stage.maxFailedExecutorsPerNode
< spark.blacklist.stage.maxFailedTasksPerExecutor
< spark.blacklist.task.maxTaskAttemptsPerExecutor
< spark.blacklist.task.maxTaskAttemptsPerNode
< spark.blacklist.timeout
---
> spark.appStatusStore.diskStoreDir
> spark.archives
> spark.barrier.sync.timeout
12a6
> spark.broadcast.UDFCompressionThreshold
15a10
> spark.checkpoint.compress
21d15
< spark.core.connection.ack.wait.timeout
22a17
> spark.decommission.enabled
29a25
> spark.driver.defaultJavaOptions
33a30,33
> spark.driver.log.allowErasureCoding
> spark.driver.log.dfsDir
> spark.driver.log.layout
> spark.driver.log.persistToDfs.enabled
36a37
> spark.driver.memoryOverheadFactor
37a39,41
> spark.driver.resource.{resourceName}.amount
> spark.driver.resource.{resourceName}.discoveryScript
> spark.driver.resource.{resourceName}.vendor
47a52,53
> spark.dynamicAllocation.shuffleTracking.enabled
> spark.dynamicAllocation.shuffleTracking.timeout
50a57
> spark.eventLog.compression.codec
52a60,62
> spark.eventLog.erasureCoding.enabled
> spark.eventLog.gcMetrics.oldGenerationGarbageCollectors
> spark.eventLog.gcMetrics.youngGenerationGarbageCollectors
53a64
> spark.eventLog.logStageExecutorMetrics
55a67,78
> spark.eventLog.rolling.enabled
> spark.eventLog.rolling.maxFileSize
> spark.excludeOnFailure.application.fetchFailure.enabled
> spark.excludeOnFailure.application.maxFailedExecutorsPerNode
> spark.excludeOnFailure.application.maxFailedTasksPerExecutor
> spark.excludeOnFailure.enabled
> spark.excludeOnFailure.killExcludedExecutors
> spark.excludeOnFailure.stage.maxFailedExecutorsPerNode
> spark.excludeOnFailure.stage.maxFailedTasksPerExecutor
> spark.excludeOnFailure.task.maxTaskAttemptsPerExecutor
> spark.excludeOnFailure.task.maxTaskAttemptsPerNode
> spark.excludeOnFailure.timeout
56a80,83
> spark.executor.decommission.forceKillTimeout
> spark.executor.decommission.killInterval
> spark.executor.decommission.signal
> spark.executor.defaultJavaOptions
59a87
> spark.executor.failuresValidityInterval
65a94
> spark.executor.maxNumFailures
67a97,100
> spark.executor.memoryOverheadFactor
> spark.executor.metrics.fileSystemSchemes
> spark.executor.metrics.pollingInterval
> spark.executor.processTreeMetrics.enabled
68a102,104
> spark.executor.resource.{resourceName}.amount
> spark.executor.resource.{resourceName}.discoveryScript
> spark.executor.resource.{resourceName}.vendor
71a108,111
> spark.files.ignoreCorruptFiles
> spark.files.ignoreMissingFiles
> spark.files.io.connectionCreationTimeout
> spark.files.io.connectionTimeout
83a124
> spark.io.compression.zstd.bufferPool.enabled
104a146
> spark.log.level
107d148
< spark.maxRemoteBlockSizeFetchToMem
112c153,154
< spark.memory.useLegacyMode
---
> spark.network.io.preferDirectBufs
> spark.network.maxRemoteBlockSizeFetchToMem
113a156
> spark.network.timeoutInterval
128a172
> spark.redaction.string.regex
131a176
> spark.resources.discoveryPlugin
132a178,180
> spark.rpc.io.backLog
> spark.rpc.io.connectionCreationTimeout
> spark.rpc.io.connectionTimeout
135,136c183,186
< spark.rpc.numRetries
< spark.rpc.retry.wait
---
> spark.scheduler.barrier.maxConcurrentTasksCheck.interval
> spark.scheduler.barrier.maxConcurrentTasksCheck.maxFailures
> spark.scheduler.excludeOnFailure.unschedulableTaskSetTimeout
> spark.scheduler.listenerbus.eventqueue.appStatus.capacity
137a188,191
> spark.scheduler.listenerbus.eventqueue.eventLog.capacity
> spark.scheduler.listenerbus.eventqueue.executorManagement.capacity
> spark.scheduler.listenerbus.eventqueue.shared.capacity
> spark.scheduler.listenerbus.eventqueue.streams.capacity
140a195
> spark.scheduler.resource.profileMergeConflicts
144a200,201
> spark.shuffle.checksum.algorithm
> spark.shuffle.checksum.enabled
145a203,204
> spark.shuffle.detectCorrupt.root
> spark.shuffle.detectCorrupt.useExtraMemory
146a206,208
> spark.shuffle.io.backLog
> spark.shuffle.io.connectionCreationTimeout
> spark.shuffle.io.connectionTimeout
150a213
> spark.shuffle.mapOutput.minSizeForBroadcast
152c215,216
< spark.shuffle.memoryFraction
---
> spark.shuffle.readHostLocalDisk
> spark.shuffle.reduceLocality.enabled
154a219,220
> spark.shuffle.service.db.backend
> spark.shuffle.service.db.enabled
155a222
> spark.shuffle.service.fetch.rdd.enabled
156a224
> spark.shuffle.service.name
157a226
> spark.shuffle.service.removeShuffle
158a228
> spark.shuffle.sort.io.plugin.class
159a230,235
> spark.shuffle.spill.diskWriteBufferSize
> spark.shuffle.unsafe.file.output.buffer
> spark.shuffle.useOldFetchProtocol
> spark.speculation.efficiency.enabled
> spark.speculation.efficiency.longRunTaskFactor
> spark.speculation.efficiency.processRateMultiplier
160a237
> spark.speculation.minTaskRuntime
163a241
> spark.speculation.task.duration.threshold
164a243
> spark.stage.ignoreDecommissionFetchFailure
166c245,253
< spark.storage.memoryFraction
---
> spark.standalone.submit.waitAppCompletion
> spark.storage.decommission.enabled
> spark.storage.decommission.fallbackStorage.cleanUp
> spark.storage.decommission.fallbackStorage.path
> spark.storage.decommission.rddBlocks.enabled
> spark.storage.decommission.shuffleBlocks.enabled
> spark.storage.decommission.shuffleBlocks.maxDiskSize
> spark.storage.decommission.shuffleBlocks.maxThreads
> spark.storage.localDiskByExecutors.cacheSize
169c256
< spark.storage.unrollFraction
---
> spark.storage.unrollMemoryThreshold
175d261
< spark.streaming.kafka.maxRetries
190a277,278
> spark.task.resource.{resourceName}.amount
> spark.ui.custom.executor.log.url
194a283
> spark.ui.liveUpdate.minFlushPeriod
196a286,287
> spark.ui.proxyRedirectUri
> spark.ui.requestHeaderSize
203a295,300
> spark.ui.store.path
> spark.ui.timeline.executors.maximum
> spark.ui.timeline.jobs.maximum
> spark.ui.timeline.stages.maximum
> spark.ui.timeline.tasks.maximum
> spark.ui.timelineEnabled