Ingest Actions Flow

iaMasa.md

flowchart TD
subgraph IDX

        TCP("TCP/UDP") -- Uncooked --> pQ
        TCP -- Cooked --> rulesetQ
        TailReader("TailReader") --> pQ
        
        
        fifo("FifoInput") --> pQ
        FSChange("FSChange") --> pQ
        Exec("Exec") --> pQ

        HEC("HEC") --> pQ
        HEC --> aggQ
        HEC --> typingQ
        
        pQ[/parsing/] --> parsingUft8
        rulesetQ[/ruleset*/] --> rsP
        aggQ[/agg/]
        typingQ[/typing/]
        indexQ[/indexing/]
        rfsQ[/rfs*/]


        subgraph rsP["Ruleset Pipeline*"]
            rsRexgex["regexreplacement (RULESET only)"]
        end

        subgraph Parsing["Parsing Pipeline"]
    
            parsingUft8[utf8] --> parseLB[linebreaker] --> parseMetrics[metrics] --> parseHead[header]
        end

        parseHead --> aggQ
        aggQ --> Merging

        subgraph Merging["Merging Pipeline"]
            mergingAgg[aggregator]
        end

        Merging --> typingQ
        typingQ --> Typing

        subgraph Typing["Typing Pipeline"]
            typeTee[tee*] --> typeRegex[regexreplacement] --> typeMetrics[metricschema] --> typeAnn[annotator]
        end

        rsP --> indexQ
        Typing --> indexQ
        indexQ --> indexerPipe
        

        subgraph indexerPipe["Indexing Pipeline"]
            dest(DestinationKey*) -->
            s2sh(S2SOverHttp) -->
            tcpO(TcoOut) -->
            syslog(SyslogOut) -->
            idxF(IndexAndForward) -->
            idx(Index)
        end

        dest --> rfsQ

        idx --> id1
        id1[\bucket/]
        rfsQ --> cjson
        cjson[\"Compressed JSON in S3"/]

    end