check_json_pickle_speed.py

import importlib.resources as ires
import xonsh


def xontrib_metadata_json():
    """Loads and returns the xontribs.json file."""
    import json

    with ires.open_text("xonsh", "xontribs.json") as f:
        md = json.load(f)
    return md


def no_res_json():
    """Loads and returns the xontribs.json file."""
    import json, os

    path = os.path.dirname(__file__)
    with open(os.path.join(path, "xontribs.json")) as f:
        md = json.load(f)
    return md


def no_res_multi_json():
    """Loads and returns the xontribs/*.json files."""
    import json, os, glob

    path = os.path.dirname(__file__)
    data = []
    for file in glob.glob(os.path.join(path, "xontribs", "*.json")):
        with open(file) as f:
            data.append(json.load(f))
    return data


def dump_as_pickle():
    import pickle
    from pathlib import Path
    import xonsh

    md = xontrib_metadata_json()
    data = pickle.dumps(md, protocol=4,)  # supported since py3.4
    path = Path(xonsh.__file__).with_name("xontribs.db")
    path.write_bytes(data)


def split_json():
    import json, copy
    from pathlib import Path

    cur = Path(__file__).parent
    md = xontrib_metadata_json()

    for xon in md["xontribs"]:
        xontrib = copy.copy(xon)
        name = xontrib["name"]
        pkg = xontrib["package"]
        if pkg in md["packages"]:
            xontrib["package"] = md["packages"][pkg]
        (cur / "xontribs" / (name + ".json")).write_text(json.dumps(xontrib, indent=2))


def xontrib_metadata_pickle():
    import pickle

    with ires.open_binary("xonsh", "xontribs.db") as f:
        data = pickle.load(f)
    return data


def pickle_plain():
    import pickle, os

    path = os.path.dirname(__file__)

    with open(os.path.join(path, "xontribs.db"), "rb") as f:
        data = pickle.load(f)
    return data


def test_time():
    import timeit

    print("Json: ", timeit.timeit(xontrib_metadata_json, number=10000))
    print("Json without resources: ", timeit.timeit(no_res_json, number=10000))
    print("Json multiple files: ", timeit.timeit(no_res_multi_json, number=10000))

    print("pickle: ", timeit.timeit(xontrib_metadata_pickle, number=10000))
    print(
        "pickle without resources: ", timeit.timeit(pickle_plain, number=10000),
    )



if __name__ == "__main__":
    test_time()
        # results
    # Json:  1.6060721029971319
    # Json without resources:  1.4733050409995485
    # Json multiple files:  17.976261422998505
    # pickle:  1.000752759999159
    # pickle without resources:  0.9649919120020058

a single meta data

{
  "name": "vox",
  "package": {
    "license": "BSD 3-clause",
    "url": "http://xon.sh",
    "install": {
      "conda": "conda install -c conda-forge xonsh",
      "pip": "xpip install xonsh",
      "aura": "sudo aura -A xonsh",
      "yaourt": "yaourt -Sa xonsh"
    }
  },
  "url": "http://xon.sh",
  "description": [
    "Python virtual environment manager for xonsh."
  ]
}