Log to stdout and an API

api_logger.py

# api_logger.py

import logging
import json
import requests
import sys
import os
import argparse


log_ingestion_url = 'https://localhost:5000/api/v1/get/logs' # this will be your api which accepts a payload with your log message and other metadata

class RequestsHandler(logging.Handler):
    def __init__(self, default_metadata=None):
        super(RequestsHandler, self).__init__()
        self.default_metadata = default_metadata or {}

    def emit(self, record):
        log_entry = self.format(record)
        extra_data = record.__dict__.get('extra')
        if record.levelname == "ERROR":
            log_entry['level'] = "error"
        if extra_data:
            log_entry.update(extra_data)
        log_entry.update(self.default_metadata)  # Append default metadata.
        response = requests.post(log_ingestion_url,
                                 data=json.dumps(log_entry), headers={"Content-type": "application/json"})
        return response.content

class FormatterLogger(logging.Formatter):
    def __init__(self, log_source=None):
        super(FormatterLogger, self).__init__()
        self.log_source = log_source
    
    # custom formatting for logging to api
    def format(self, record):
        data = {
            'errormsg': record.msg,
            'scanservice': self.log_source
        }
        return data

def custom_logger(name, log_source, default_metadata=None, log_filename=None):
    
    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)

    # Handler for writing logs to a file
    if log_filename:
        file_handler = logging.FileHandler(filename=log_filename, mode='w')
        file_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        file_handler.setFormatter(file_formatter)
        logger.addHandler(file_handler)

    # Handler for printing logs to stdout
    stream_handler = logging.StreamHandler(sys.stdout)
    stream_formatter = logging.Formatter('%(levelname)s - %(message)s')
    stream_handler.setFormatter(stream_formatter)
    logger.addHandler(stream_handler)

    # Custom handler for sending logs to the specified URL
    custom_handler = RequestsHandler(default_metadata)
    custom_formatter = FormatterLogger(log_source)
    custom_handler.setFormatter(custom_formatter)
    logger.addHandler(custom_handler)

    return logger

if __name__ == "__main__":

    # initialize logger
    logger = custom_logger("http","myapp",
            {
            # metadata to include with every message that's sent to API
                'foo': 'bar',
                'somekey': 'somevalue',
                'env': 'dev'
            },
            log_filename="/path/to/log/file" # optional
    )

    # Read from STDIN when piped to a specific job command.
    log_data = sys.stdin.read()
    logger.info(log_data.strip())


"""
For testing, you can pipe this script to any command like :
somecommand | python3 custom_logger.py

For practical use, you can import the custom_logger() method into your application like:

# in your script/app eg: myapp.py

from api_logger import custom_logger

logger = custom_logger("http","myapp",
    {
    # metadata to include with every message that's sent to API
        'foo': 'bar',
        'somekey': 'somevalue',
        'env': 'dev'
    },
    log_filename="/path/to/log/file.log" # optional
)

logger.info("Some test log")

logger.info("Some test log with extra key val pairs",extra={'extra':{'testkey':'testvar','anotherkey':'anothervar'}})

"""