syslog-ng python parser

activity.json

{"MESSAGE":"UPDATE system/freeipa/role/correspondant_bioaster_collaboration SUCCESS","openidm":{"operation":"UPDATE","objectPath":"system/freeipa/role/correspondant_bioaster_collaboration","objectType":"system/freeipa/role","objectId":"correspondant_bioaster_collaboration","status":"SUCCESS","before":{"type":"collaboration","name":"correspondant_bioaster_collaboration","members[1]":"jbuffin","members[0]":"pveyre","id":"correspondant_bioaster_collaboration","dn":"cn=correspondant_bioaster_collaboration,cn=roles,cn=accounts,dc=cc,dc=in2p3,dc=fr","collaboration":"bioaster","_id":"correspondant_bioaster_collaboration"},"after":{"type":"collaboration","name":"correspondant_bioaster_collaboration","members[2]":"gboissy","members[1]":"jbuffin","members[0]":"pveyre","id":"correspondant_bioaster_collaboration","dn":"cn=correspondant_bioaster_collaboration,cn=roles,cn=accounts,dc=cc,dc=in2p3,dc=fr","collaboration":"bioaster","_id":"correspondant_bioaster_collaboration"}}}

openidm.py

class UpdateDiff(object):
    def parse(self, log_message):
        """
        Diff between before and after
        """
        if log_message[b'openidm.operation'] != b'UPDATE' and log_message[b'openidm.operation'] != b'PATCH':
            return True
        
        changes = {}
        
        for bkey in log_message.keys():
            # First read before attributes
            if bkey.startswith(b'openidm.before.'):
                attr = bkey.replace(b'openidm.before.',b'')
                akey = b'openidm.after.' + attr
                ckey = b'openidm.changes.' + attr
                if akey in log_message.keys(): # after exists: print before => after only if different
                    if log_message[bkey] != log_message[akey]:
                        changes[ckey] = log_message[bkey] + b' => ' + log_message[akey]
                else: # print before => NULL
                    changes[ckey] = log_message[bkey] + b' => NULL'
        for akey in log_message.keys():
            # Then read after attributes
            if akey.startswith(b'openidm.after.'):
                attr = akey.replace(b'openidm.after.',b'')
                bkey = b'openidm.before.' + attr
                ckey = b'openidm.changes.' + attr
                if bkey not in log_message.keys(): # before does not exist, print NULL => after
                    changes[ckey] = b'NULL => ' + log_message[akey]
        
        for ckey in changes.keys():
            log_message[ckey] = changes[ckey]
        return True

runtime

$ rm -f /tmp/{syslog-ng-debug.log,c,R}; PYTHONPATH=/home/fwernli/syslog_ng-idnum syslog-ng --no-caps -f /home/fwernli/syslog_ng-idnum/syslog-ng.conf -R /tmp/R -c /tmp/c -Fdv
[...]
[2021-01-27T10:56:30.256473] Incoming log entry; line='{"MESSAGE":"UPDATE system/freeipa/role/correspondant_bioaster_collaboration SUCCESS","openidm":{"operation":"UPDATE","objectPath":"system/freeipa/role/correspondant_bioaster_collaboration","objectType":"system/freeipa/role","objectId":"correspondant_bioaster_collaboration","status":"SUCCESS","before":{"type":"collaboration","name":"correspondant_bioaster_collaboration","members[1]":"jbuffin","members[0]":"pveyre","id":"correspondant_bioaster_collaboration","dn":"cn=correspondant_bioaster_collaboration,cn=roles,cn=accounts,dc=cc,dc=in2p3,dc=fr","collaboration":"bioaster","_id":"correspondant_bioaster_collaboration"},"after":{"type":"collaboration","name":"correspondant_bioaster_collaboration","members[2]":"gboissy","members[1]":"jbuffin","members[0]":"pveyre","id":"correspondant_bioaster_collaboration","dn":"cn=correspondant_bioaster_collaboration,cn=roles,cn=accounts,dc=cc,dc=in2p3,dc=fr","collaboration":"bioaster","_id":"correspondant_bioaster_collaboration"}}}'
[2021-01-27T10:56:30.259839] Initializing destination file writer; template='/tmp/syslog-ng-debug.log', filename='/tmp/syslog-ng-debug.log'
[2021-01-27T10:56:30.260066] Outgoing message; message='{"openidm":{"status":"SUCCESS","operation":"UPDATE","objectType":"system/freeipa/role","objectPath":"system/freeipa/role/correspondant_bioaster_collaboration","objectId":"correspondant_bioaster_collaboration","changes":{"userName":"NULL => ","members[2]":"NULL => gboissy"},"before":{"type":"collaboration","name":"correspondant_bioaster_collaboration","members[1]":"jbuffin","members[0]":"pveyre","id":"correspondant_bioaster_collaboration","dn":"cn=correspondant_bioaster_collaboration,cn=roles,cn=accounts,dc=cc,dc=in2p3,dc=fr","collaboration":"bioaster","_id":"correspondant_bioaster_collaboration"},"after":{"type":"collaboration","name":"correspondant_bioaster_collaboration","members[2]":"gboissy","members[1]":"jbuffin","members[0]":"pveyre","id":"correspondant_bioaster_collaboration","dn":"cn=correspondant_bioaster_collaboration,cn=roles,cn=accounts,dc=cc,dc=in2p3,dc=fr","collaboration":"bioaster","_id":"correspondant_bioaster_collaboration"}},"SOURCE":"s_openidm_json","MESSAGE":"UPDATE system/freeipa/role/co'

syslog-ng.conf

#
@version: 3.28

@include scl.conf

options {
    threaded(no);
};

parser p_openidm_json {
    json-parser(
        template(
            "${MESSAGE}"
        ),
        prefix(
            ""
        )
    );
};

rewrite r_program_system_computing {
    set(
        "${openidm.after.userName}",
        value(
            "usracct.username"
        ),
        condition("1" == "2")
    );
};

parser p_openidm_diff {
    python(
        class(
            openidm.UpdateDiff
        )
    );
};

source s_openidm_json {
    file(
        /tmp/activity.json,
        flags(no-parse)
    );
};

destination d_debug {
    file(
        /tmp/syslog-ng-debug.log,
        template(
            "$(format-json -s all-nv-pairs)\n"
        )
    );
};

filter f_system_computing_account {
    match(
        "system/freeipa/account",
        value(
            "openidm.objectType"
        )
    );
};

log {
    source(s_openidm_json);
    parser(p_openidm_json);
    parser(p_openidm_diff);
    #rewrite(r_program_system_computing);
    destination(d_debug);
};