Extract values from an arbitrary JSON data structure

sample.json

{
  "name": "\/init.scope\/system.slice\/docker-7f9ed89c7297f99669b6e79d9d8d404d19f160ca40b40f42896506fa7942786b.scope",
  "spec": {
    "creation_time": "2016-06-28T11:00:25.342124476Z",
    "has_cpu": true,
    "cpu": {
      "limit": 2,
      "max_limit": 0,
      "period": 100000
    },
    "has_memory": true,
    "memory": {
      "limit": 9.2233720368548e+18,
      "swap_limit": 9.2233720368548e+18
    },
    "has_network": false,
    "has_filesystem": false,
    "has_diskio": true,
    "has_custom_metrics": false
  },
  "stats": [
    {
      "timestamp": "2016-07-02T13:35:07.076830981Z",
      "cpu": {
        "usage": {
          "total": 1526446336302,
          "per_cpu_usage": [
            94021047521,
            97542835056,
            98011039897,
            98486146966,
            99707826834,
            99277944147,
            101644333338,
            99960172557,
            102285940730,
            102507570541,
            5188873310,
            5413150344,
            5412026428,
            5293455876,
            5437389546,
            5418293319,
            5605345605,
            5373987712,
            5492104407,
            5470533991,
            45149404125,
            48045522503,
            47999321686,
            42784555618,
            47644383609,
            39879844876,
            46733912328,
            47883702727,
            47357584488,
            47870531111,
            1724963668,
            1773475049,
            1740508296,
            1780950200,
            1873195253,
            1631640434,
            1567101536,
            1934580785,
            1708237936,
            1812901949
          ],
          "user": 1161250000000,
          "system": 239280000000
        },
        "load_average": 0
      },
      "diskio": {
        
      },
      "memory": {
        "usage": 49815552,
        "cache": 24576,
        "rss": 49790976,
        "working_set": 49807360,
        "failcnt": 0,
        "container_data": {
          "pgfault": 1500587,
          "pgmajfault": 0
        },
        "hierarchical_data": {
          "pgfault": 1500587,
          "pgmajfault": 0
        }
      },
      "network": {
        "name": "",
        "rx_bytes": 0,
        "rx_packets": 0,
        "rx_errors": 0,
        "rx_dropped": 0,
        "tx_bytes": 0,
        "tx_packets": 0,
        "tx_errors": 0,
        "tx_dropped": 0,
        "tcp": {
          "Established": 0,
          "SynSent": 0,
          "SynRecv": 0,
          "FinWait1": 0,
          "FinWait2": 0,
          "TimeWait": 0,
          "Close": 0,
          "CloseWait": 0,
          "LastAck": 0,
          "Listen": 0,
          "Closing": 0
        },
        "tcp6": {
          "Established": 0,
          "SynSent": 0,
          "SynRecv": 0,
          "FinWait1": 0,
          "FinWait2": 0,
          "TimeWait": 0,
          "Close": 0,
          "CloseWait": 0,
          "LastAck": 0,
          "Listen": 0,
          "Closing": 0
        }
      },
      "task_stats": {
        "nr_sleeping": 0,
        "nr_running": 0,
        "nr_stopped": 0,
        "nr_uninterruptible": 0,
        "nr_io_wait": 0
      }
    }
  ]
}

walk_json.py

#!/usr/bin/env python
# The function `walk()` will walk an arbitrary JSON data file yielding the selected
# "keys".
#
# 2016-07-03
# Daniel Kertesz <daniel@spatof.org>

import json
import sys


# yield tuples: (path, value)
# Example: ("stats.cpu.usage.total", 1526446336302)
def walk(node, paths, keys=None):
    if keys is None:
        keys = []

    if isinstance(node, dict):
        for key, value in node.iteritems():
            if isinstance(value, (dict, list)):
                for x in walk(value, paths, keys + [key]):
                    yield x
            else:
                path = '.'.join(keys + [key])
                if path in paths:
                    yield (path, value)
    elif isinstance(node, list):
        for item in node:
            if isinstance(item, (dict, list)):
                for x in walk(item, paths, keys):
                    yield x
            else:
                path = '.'.join(keys)
                if path in paths:
                    yield (path, value)
    else:
        path = '.'.join(keys)
        if path in paths:
            yield (path, value)


def main():
    values_to_extract = [
        'stats.timestamp',
        'stats.cpu.usage.total',
        'stats.cpu.usage.user',
        'stats.memory.usage',
        'spec.has_network',
        'name',
    ]

    if len(sys.argv) < 2:
        print "Usage: %s <JSON file>" % sys.argv[0]
        sys.exit(1)
        
    with open(sys.argv[1]) as fd:
        data = json.load(fd)

    for result in walk(data, values_to_extract):
        print result

if __name__ == '__main__':
    main()