yarn_app_logs_splitter.py script

yarn_app_logs_splitter.py

#! /usr/bin/env python

# Simple tool to take the logs generated by yarn logs -applicationId
# and convert them into separate log files.
# Current implementation deconstructs the monolith log dump
# into multiple files in the format <container_id>.<log_file_name>
# For example, container_1400495557557_0001_01_000001.syslog

import argparse
import os
from os.path import isfile,join

parser = argparse.ArgumentParser(description='Apache Dev Setup')
parser.add_argument('--app-logs', required=True,
                   help='Application Logs file')
parser.add_argument('--container-log-dir', required=True,
                   help='Location to where container logs should be generated')

args = parser.parse_args()

print "Using app-logs: " + args.app_logs 
print "Using container-log-dir: " + args.container_log_dir 

base_dir = args.container_log_dir
app_logs_file = args.app_logs

if (not os.path.exists(base_dir)):
  os.makedirs(base_dir, 0755)

current_container_id = ""
current_log_file = ""
current_log_file_handle = None
log_started = False

with open(app_logs_file) as f:
  for line in f:
    l = line.strip()
    if (l.startswith("Container: container")):
      entry = l.split(":")[1].strip()
      current_container_id = entry.expandtabs(1).split(" ")[0].strip()
      log_started = False
      continue

    if (current_container_id == ""):
      continue
        
    if (l.startswith("LogType:")):
      if (current_log_file_handle):
        current_log_file_handle.close()
      file_name = current_container_id + "." + l.split(":")[1].strip()
      current_log_file = os.path.join(base_dir, file_name)
      current_log_file_handle = open(current_log_file, "w")
      log_started = False
      continue

    if (l.startswith("LogLength:")):
      continue
   
    if (l.startswith("Log Contents:")):
      log_started = True
      continue

    if (log_started and current_log_file_handle):
      current_log_file_handle.write(l + os.linesep) 
       
if (current_log_file_handle):
  current_log_file_handle.close()