Output Processor Multi-Threading not working as expected

fluent-bit.yaml

---
service:
    flush: 1
    daemon: off
    log_level: info
    http_server: on
    health_check: on

pipeline:
    inputs:
        - name: dummy
          tag: f322bb90-7780-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7781-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7782-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7783-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7783-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7784-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7785-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7786-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7787-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000
        - name: dummy
          tag: f322bb90-7788-4be6-a8db-b31fa5ee79fb.deployment-75b889d4f5-7bmkq
          dummy: '{"msg": "Thu Nov 30 07:57:00:834 2023 \n DummyMessage"}'
          rate: 1000

    outputs:
        - name: file
          path: output_dir
          match: '*'
          workers: 100
          processors:
              logs:
                - name: lua
                  call: generate_load
                  code: |
                      function generate_load(tag, timestamp, record)
                         new_record = record
                         -- execute important lua code here
                         local result = 0
                         for i = 1, 100000 do
                           result = result + math.sin(i) * math.cos(i)
                         end
                         new_record["output"] = result
                         return 1, timestamp, new_record
                      end