microbatch.txt

22/07/25 14:29:04 INFO CheckpointFileManager: Renamed temp file file:/tmp/temporary-1b3adb9c-766a-4aec-97a9-decfd7be10e7/commits/.10.b017bcdc-f142-4b28-8891-d3f2d471b740.tmp to file:/tmp/temporary-1b3adb9c-766a-4aec-97a9-decfd7be10e7/commits/10
22/07/25 14:29:04 INFO MicroBatchExecution: Streaming query made progress: {
  "id" : "bec9640f-ac16-4d00-bd4a-ed8d29b5f768",
  "runId" : "a2e00c3a-924c-4728-b25a-56ee855da7da",
  "name" : "streaming_to_console",
  "timestamp" : "2022-07-25T14:29:00.000Z",
  "batchId" : 10,
  "numInputRows" : 73,
  "inputRowsPerSecond" : 1.2166666666666666,
  "processedRowsPerSecond" : 16.375056078959172,
  "durationMs" : {
    "addBatch" : 4359,
    "getBatch" : 1,
    "latestOffset" : 5,
    "queryPlanning" : 37,
    "triggerExecution" : 4458,
    "walCommit" : 27
  },
  "eventTime" : {
    "avg" : "2022-07-25T14:28:29.947Z",
    "max" : "2022-07-25T14:28:59.290Z",
    "min" : "2022-07-25T14:28:00.559Z",
    "watermark" : "2022-07-25T14:17:59.737Z"
  },
  "stateOperators" : [ {
    "operatorName" : "stateStoreSave",
    "numRowsTotal" : 476,
    "numRowsUpdated" : 132,
    "allUpdatesTimeMs" : 319,
    "numRowsRemoved" : 0,
    "allRemovalsTimeMs" : 0,
    "commitTimeMs" : 14173,
    "memoryUsedBytes" : 262776,
    "numRowsDroppedByWatermark" : 0,
    "numShufflePartitions" : 400,
    "numStateStoreInstances" : 400,
    "customMetrics" : {
      "loadedMapCacheHitCount" : 15600,
      "loadedMapCacheMissCount" : 0,
      "stateOnCurrentVersionSizeBytes" : 154208
    }
  } ],
  "sources" : [ {
    "description" : "KafkaV2[Subscribe[topicC]]",
    "startOffset" : {
      "topicC" : {
        "2" : 101,
        "5" : 96,
        "4" : 88,
        "1" : 84,
        "3" : 112,
        "0" : 100
      }
    },
    "endOffset" : {
      "topicC" : {
        "2" : 114,
        "5" : 110,
        "4" : 98,
        "1" : 98,
        "3" : 124,
        "0" : 110
      }
    },
    "latestOffset" : {
      "topicC" : {
        "2" : 114,
        "5" : 110,
        "4" : 98,
        "1" : 98,
        "3" : 124,
        "0" : 110
      }
    },
    "numInputRows" : 73,
    "inputRowsPerSecond" : 1.2166666666666666,
    "processedRowsPerSecond" : 16.375056078959172,
    "metrics" : {
      "avgOffsetsBehindLatest" : "0.0",
      "maxOffsetsBehindLatest" : "0",
      "minOffsetsBehindLatest" : "0"
    }
  } ],
  "sink" : {
    "description" : "org.apache.spark.sql.execution.streaming.ConsoleTable$@52e344bf",
    "numOutputRows" : 238
  }
}