GCP Log Consolidation with Flask

gistfile2.txt

https://google-cloud-python.readthedocs.io/en/latest/logging/usage.html
https://google-cloud-python.readthedocs.io/en/latest/logging/usage.html#integration-with-python-logging-module
https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry



NOTE:  as of 9/7/18
  https://github.com/GoogleCloudPlatform/google-cloud-python/pull/5878
  https://github.com/GoogleCloudPlatform/google-cloud-python/pull/5885
  
  not yet part of a google-cloud-logging  release

to use:

  curl -v  -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.89 Safari/537.36" -H "X-Cloud-Trace-Context: `python -c "impo uuid; print uuid.uuid4()"`" http://localhost:8080/

  

main.py

#!/usr/bin/python

from flask import Flask
from flask import request, Response, render_template, g, jsonify
from flask.logging import default_handler
from werkzeug.wrappers import Request

import requests, time, os

from google.cloud import logging as gcplogging
client = gcplogging.Client()

import logging
from logging.config import dictConfig
from logging.handlers import HTTPHandler
from logging import StreamHandler


# curl -v  -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.89 Safari/537.36" -H "X-Cloud-Trace-Context: `python -c "impo uuid; print uuid.uuid4()"`" http://localhost:8080/


class GCPHandler(logging.Handler):
    def __init__(self, logName):
        self.logName = logName
        self.logger = client.logger(logName)
        logging.Handler.__init__(self)

    def emit(self, record):
        msg = self.format(record)
        TEXT = msg
        DEFAULT_LABELS = {'foo': 'spam'}
        LABELS = {'foo': 'bar', 'baz': 'qux'}
        SEVERITY = 'INFO'
        TRACE = "projects/{}/traces/{}".format(client.project, request.headers.get('X-Cloud-Trace-Context'))
        self.logger.log_text(TEXT, client=client, labels=LABELS, severity=SEVERITY, trace=TRACE)

class LoggerConfig:
    dictConfig = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'standard': { 'format': '%(asctime)s - %(name)s - %(levelname)s - '
                          '%(message)s - [in %(pathname)s:%(lineno)d]'},
            'short': { 'format': '%(message)s' }
        },
        'handlers': {
            'default': {
                'level': 'INFO',
                'formatter': 'standard',
                'class': 'logging.handlers.RotatingFileHandler',
                'filename': 'app.log',
                'maxBytes': 5000000,
                'backupCount': 10
            },
            'gcp': {
                'level': 'INFO',
                'formatter': 'short',
                'class': 'main.GCPHandler',
                'logName': 'child'
            },
            'debug': {
                'level': 'DEBUG',
                'formatter': 'standard',
                'class': 'logging.StreamHandler'
            },
            'console': {
                'class': 'logging.StreamHandler',
                'level': 'INFO'
            },
        },
        'loggers': {
            'child': {
                'handlers': ['gcp'],
                'level': 'DEBUG',
                'propagate': True},
        },
        # 'root': { 'level': 'DEBUG', 'handlers': ['console'] }
    }

logging.config.dictConfig(LoggerConfig.dictConfig)

class TransitMiddleWare(object):
  def __init__(self, app):
     self.app = app

  def __call__(self, environ, start_response):
    req = Request(environ, shallow=True)
    host = req.headers.get('Host')
    return self.app(environ, start_response)


app = Flask(__name__)
app.wsgi_app = TransitMiddleWare(app.wsgi_app)

parent_logger = client.logger("parent")
logger = logging.getLogger("child")

@app.before_request
def before_request():
    g.request_start_time = time.time()
    g.request_time = lambda: "%.5fs" % (time.time() - g.request_start_time)

@app.after_request
def add_logger(response):

    TEXT = u'TEXT'
    DEFAULT_LABELS = {'foo': 'spam'}
    LABELS = {'foo': 'bar', 'baz': 'qux'}
    SEVERITY = 'INFO'
    TRACE = "projects/{}/traces/{}".format(client.project, request.headers.get('X-Cloud-Trace-Context'))

    REQUEST = {
        'requestMethod': request.method,
        'requestUrl': request.url,
        'status': response.status_code,
        'userAgent': request.headers.get('USER-AGENT'),
        'responseSize': response.content_length,
        'latency': g.request_time(),
        'remoteIp': request.remote_addr
    }

    parent_logger.log_text(TEXT, client=client, labels=LABELS, severity=SEVERITY, http_request=REQUEST, trace=TRACE)

    # optional...similar to envoy: transmit response time back to the user
    response.headers['x-upstream-service-time'] = g.request_time()

    return response

@app.route('/')
def default():
  logger.info("Message 1")
  logger.info("Message 2")
  return 'ok'

@app.route('/_ah/health')
def health():
  return 'ok'

if __name__ == '__main__':
  app.run(host='0.0.0.0', port=8080, debug=False)

requirements.txt

flask
google-cloud-logging==1.6.0
requests

@salrashid123 I am trying out consolidating logs in gcp and this code snippet is very handy and works well.Could you please tell me how to set the resource to "gae_app" instead of the default "global". Is it done in the GCP Handler class or should it be specified in the dictConfig?

You should be able to set the resource type

• https://github.com/salrashid123/flask-gcp-log-groups/blob/master/flask_gcp_log_groups/gcp_logging.py#L18