Skip to content

Instantly share code, notes, and snippets.

@carlosroman
Last active December 11, 2023 15:24
Show Gist options
  • Save carlosroman/8891293a15494761f7ef598f80773edb to your computer and use it in GitHub Desktop.
Save carlosroman/8891293a15494761f7ef598f80773edb to your computer and use it in GitHub Desktop.
Simple journald random logger service
---
version: "3.9"
services:
random-logs-all-multiline:
build:
context: .
dockerfile: Dockerfile.multiline
labels:
com.datadoghq.ad.logs: >-
[{
"source": "random",
"service": "all-multiline",
"log_processing_rules": [{
"type": "multi_line",
"name": "log_start_with_date",
"pattern" : "\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3}"
}]
}]
random-logs-info-only:
build:
context: .
dockerfile: Dockerfile.multiline
labels:
com.datadoghq.ad.logs: >-
[{
"source": "random",
"service": "info-only",
"log_processing_rules": [{
"type": "multi_line",
"name": "log_start_with_date",
"pattern" : "\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3}"
},
{
"type": "include_at_match",
"name": "include_only_warn",
"pattern": "INFO"
}]
}]
random-logs-warning-and-error-only:
build:
context: .
dockerfile: Dockerfile.multiline
labels:
com.datadoghq.ad.logs: >-
[{
"source": "random",
"service": "info-only",
"log_processing_rules": [{
"type": "multi_line",
"name": "log_start_with_date",
"pattern" : "\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3}"
},
{
"type": "include_at_match",
"name": "include_only_warn",
"pattern": "WARNING|ERROR"
}]
}]
# syntax=docker/dockerfile:1
FROM python:3.11-alpine
COPY random-logger.py /usr/bin/
ENTRYPOINT [ "python" ]
CMD [ "/usr/bin/random-logger.py" ]
# syntax=docker/dockerfile:1
FROM python:3.11-alpine
COPY random-multiline-logger.py /usr/bin/
ENTRYPOINT [ "python" ]
CMD [ "/usr/bin/random-multiline-logger.py" ]
#!/usr/bin/env python3
import logging
import random
import threading
import time
import concurrent.futures
from logging.handlers import RotatingFileHandler
_charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
class Info:
def __init__(self):
self.value = {'stop': False}
self._lock = threading.Lock()
def stop(self):
with self._lock:
self.value['stop'] = True
def not_stopped(self):
self._lock.acquire(timeout=20)
try:
answer = self.value['stop']
finally:
self._lock.release()
return not answer
def worker(arg):
while arg.not_stopped():
rnd = ''.join(random.sample(_charset*9, 9))
match random.choice(['debug', 'info', 'warn', 'error']):
case 'debug':
logging.debug(f'This is a debug log that shows a log that can be ignored. {rnd}',
extra={'identifier': f'{rnd}'})
case 'info':
logging.info(f'This is less important than debug log and is often used to provide context in the current task. {rnd}',
extra={'identifier': f'{rnd}'})
case 'warn':
logging.warning(f'A warning that should be ignored is usually at this level and should be actionable. {rnd}',
extra={'identifier': f'{rnd}'})
case 'error':
logging.error(f'An error is usually an exception that has been caught and not handled. {rnd}',
extra={'identifier': f'{rnd}'})
time.sleep(random.uniform(0, 5))
def main():
handler = RotatingFileHandler("bob.log", maxBytes=1000000,
backupCount=5)
logging.basicConfig(
handlers=[handler],
format='{"timestamp":"%(asctime)s","severity":"%(levelname)s","message":"%(message)s", "context":{"identifier":"%(identifier)s", "thread":"%(threadName)s"}}',
level=logging.DEBUG)
info = Info()
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
futures = [
executor.submit(worker, info) for i in range(1, 15)
]
while True:
try:
time.sleep(5)
except KeyboardInterrupt:
info.stop()
break
executor.shutdown()
if __name__ == '__main__':
main()
#!/usr/bin/env python3
import logging
import random
from time import sleep
logging.basicConfig(format='%(asctime)s | %(levelname)s | %(message)s', level=logging.DEBUG)
while True:
match random.choice(['debug', 'info', 'warn', 'error']):
case 'debug':
logging.debug('This is a debug log that shows a log that can be ignored."')
case 'info':
logging.info('This is less important than debug log and is often used to provide context in the current task.')
case 'warn':
logging.warning('A warning that should be ignored is usually at this level and should be actionable.')
case 'error':
logging.error('An error is usually an exception that has been caught and not handled.')
sleep(random.uniform(0, 5))
[Unit]
Description=Random logger
[Service]
ExecStart=/usr/bin/random-logger.py
[Install]
WantedBy=multi-user.target
#!/usr/bin/env python3
import argparse
import logging
import random
from time import sleep
parser = argparse.ArgumentParser(description='Random Multiline logger.')
parser.add_argument('--log-file', dest='log_filename', default=None, help='Set logger to output to a logfile rather than stout')
args = parser.parse_args()
logging.basicConfig(filename=args.log_filename, format='%(message)s', level=logging.DEBUG)
mu = 4
sigma = 2
upper = mu + sigma
# For log level
alpha = 1
beta = 1.5
levels = ['debug', 'info', 'warn', 'error', 'critical']
while True:
temp = random.gauss(mu, sigma)
not_multiline = sigma <= temp <= upper
temp = round(random.weibullvariate(alpha, beta))
if temp >= len(levels):
temp = len(levels) -1
level = levels[temp]
match level:
case 'debug':
if not_multiline:
logging.debug('This is a debug log that shows a log that can be ignored."')
else:
logging.debug('bbx This is a \ndebug log that \nshows a log that \ncan be ignored."')
case 'info':
if not_multiline:
logging.info('This is less important than debug log and is often used to provide context in the current task.')
else:
logging.info('ccc This is less \nimportant than debug log \nand is often used to provide context \nin the current task.')
case 'warn':
if not_multiline:
logging.warning('A warning that should be ignored is usually at this level and should be actionable.')
else:
logging.warning('cycc A warning \nthat should be ignored is \nusually at this level and should be actionable.')
case 'error':
if not_multiline:
logging.error('An error is usually an exception that has been caught and not handled.')
else:
logging.error('mcg An error is \nusually an exception that \nhas been caught and not handled.')
case 'critical':
if not_multiline:
logging.critical('A critical error is usually an exception that has not been caught or handled.')
else:
logging.critical('bbx A critical error is \nusually an exception that \nhas not been caught or handled.')
sleep(random.uniform(0, 5))
#!/usr/bin/env sh
# The script can be run as different user by:
# sudo runuser -u dd-agent -- ./random-multiline-logger-other.sh
# The following needs to be added to /etc/datadog-agent/datadog.yaml
# and random-multiline-logger.yaml needs to be copied to /etc/datadog-agent/conf.d/
# Also make sure that /var/log/test exists and is owned by `dd-agent`.
#
# logs_config:
# container_collect_all: true
# auto_multi_line_detection: true
# auto_multi_line_default_match_threshold: 0.1
# auto_multi_line_extra_patterns:
# - (bbx|ccc|cycc|mcg)
python3 -u random-multiline-logger-other.py --log-file=/var/log/test/random-multiline-logger.log
#!/usr/bin/env python3
import argparse
import logging
import random
from time import sleep
parser = argparse.ArgumentParser(description='Random Multiline logger.')
parser.add_argument('--log-file', dest='log_filename', default=None, help='Set logger to output to a logfile rather than stout')
args = parser.parse_args()
logging.basicConfig(filename=args.log_filename, format='%(asctime)s | %(levelname)s | %(message)s', level=logging.DEBUG)
mu = 4
sigma = 2
upper = mu + sigma
# For log level
alpha = 1
beta = 1.5
levels = ['debug', 'info', 'warn', 'error', 'critical']
while True:
temp = random.gauss(mu, sigma)
not_multiline = sigma <= temp <= upper
temp = round(random.weibullvariate(alpha, beta))
if temp >= len(levels):
temp = len(levels) -1
level = levels[temp]
match level:
case 'debug':
if not_multiline:
logging.debug('This is a debug log that shows a log that can be ignored."')
else:
logging.debug('This is a \ndebug log that \nshows a log that \ncan be ignored."')
case 'info':
if not_multiline:
logging.info('This is less important than debug log and is often used to provide context in the current task.')
else:
logging.info('This is less \nimportant than debug log \nand is often used to provide context \nin the current task.')
case 'warn':
if not_multiline:
logging.warning('A warning that should be ignored is usually at this level and should be actionable.')
else:
logging.warning('A warning \nthat should be ignored is \nusually at this level and should be actionable.')
case 'error':
if not_multiline:
logging.error('An error is usually an exception that has been caught and not handled.')
else:
logging.error('An error is \nusually an exception that \nhas been caught and not handled.')
case 'critical':
if not_multiline:
logging.critical('A critical error is usually an exception that has not been caught or handled.')
else:
logging.critical('A critical error is \nusually an exception that \nhas not been caught or handled.')
sleep(random.uniform(0, 5))
#!/usr/bin/env sh
python3 -u random-multiline-logger.py --log-file=/var/log/test/random-multiline-logger.log
---
logs:
- type: file
path: /var/log/test/random-multiline-logger.log
service: random-multiline-logger
source: python
@carlosroman
Copy link
Author

carlosroman commented Sep 13, 2022

  • Copy random-logger.py to /usr/bin/
  • Copy random-logger.service to /etc/systemd/system/
  • Run systemctl daemon-reload
  • To enable and start the new service run systemctl enable --now random-logger.service
  • For multiple instance of this random logger service
    • just copy random-logger.service to /etc/systemd/system/ and name it random-logger-001.service, random-logger-002.service, etc.
    • To enable (after systemctl daemon-reload) it would just be systemctl enable --now random-logger-001.service, systemctl enable --now random-logger-002.service, etc.
    • You might want to cut down the number of journald logs you keep. You can do this by running journalctl --rotate --vacuum-time=1h. This will remove any logs older than an hour and will also make sure journald only keeps logs no older than one hour.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment