This is a modified version of treeherder/log_parser/tasks.py that profiles the execution of a parse_log run. It generates one profile file per task run. You can directly use it in your development environment with the process_objects periodic task running

gistfile1.py

import simplejson as json

from celery import task
import time
from treeherder.model.derived import JobsModel
from treeherder.log_parser.artifactbuildercollection import ArtifactBuilderCollection

import profile


def parse_log_func(project, job_id):
    jm = JobsModel(project=project)
    log_references = jm.get_log_references(job_id)

    # we may have many log references per job
    for log in log_references:
        # parse a log given its url
        artifact_builder_collection = ArtifactBuilderCollection(log['url'])
        artifact_builder_collection.parse()

        # store the artifacts generated
        for name, artifact in artifact_builder_collection.artifacts.items():
            jm.insert_job_artifact(job_id, name, 'json', json.dumps(artifact))
    jm.disconnect()


@task(name='parse-log')
def parse_log(project, job_id):
    """
    Call ArtifactBuilderCollection on the given job.
    """
    now = time.time()

    profile.runctx("parse_log_func(p,j)", globals(), {'p': project, 'j': job_id}, "parse_log%s.prof" % now)