check_json_pickle_speed.py

import importlib.resources as ires
import xonsh


def xontrib_metadata_json():
    """Loads and returns the xontribs.json file."""
    import json

    with ires.open_text("xonsh", "xontribs.json") as f:
        md = json.load(f)
    return md


def no_res_json():
    """Loads and returns the xontribs.json file."""
    import json, os

    path = os.path.dirname(__file__)
    with open(os.path.join(path, "xontribs.json")) as f:
        md = json.load(f)
    return md


def no_res_multi_json():
    """Loads and returns the xontribs/*.json files."""
    import json, os, glob

    path = os.path.dirname(__file__)
    data = []
    for file in glob.glob(os.path.join(path, "xontribs", "*.json")):
        with open(file) as f:
            data.append(json.load(f))
    return data


def dump_as_pickle():
    import pickle
    from pathlib import Path
    import xonsh

    md = xontrib_metadata_json()
    data = pickle.dumps(md, protocol=4,)  # supported since py3.4
    path = Path(xonsh.__file__).with_name("xontribs.db")
    path.write_bytes(data)


def split_json():
    import json, copy
    from pathlib import Path

    cur = Path(__file__).parent
    md = xontrib_metadata_json()

    for xon in md["xontribs"]:
        xontrib = copy.copy(xon)
        name = xontrib["name"]
        pkg = xontrib["package"]
        if pkg in md["packages"]:
            xontrib["package"] = md["packages"][pkg]
        (cur / "xontribs" / (name + ".json")).write_text(json.dumps(xontrib, indent=2))


def xontrib_metadata_pickle():
    import pickle

    with ires.open_binary("xonsh", "xontribs.db") as f:
        data = pickle.load(f)
    return data


def pickle_plain():
    import pickle, os

    path = os.path.dirname(__file__)

    with open(os.path.join(path, "xontribs.db"), "rb") as f:
        data = pickle.load(f)
    return data


def test_time():
    import timeit

    print("Json: ", timeit.timeit(xontrib_metadata_json, number=10000))
    print("Json without resources: ", timeit.timeit(no_res_json, number=10000))
    print("Json multiple files: ", timeit.timeit(no_res_multi_json, number=10000))

    print("pickle: ", timeit.timeit(xontrib_metadata_pickle, number=10000))
    print(
        "pickle without resources: ", timeit.timeit(pickle_plain, number=10000),
    )



if __name__ == "__main__":
    test_time()
        # results
    # Json:  1.6060721029971319
    # Json without resources:  1.4733050409995485
    # Json multiple files:  17.976261422998505
    # pickle:  1.000752759999159
    # pickle without resources:  0.9649919120020058

dir structure

.local.out
|-- check_json_pickle_speed.py
|-- xontribs
|   |-- abbrevs.json
|   |-- apt_tabcomplete.json
|   |-- argcomplete.json
|   |-- autojump.json
|   |-- autovox.json
|   |-- autoxsh.json
|   |-- avox.json
|   |-- back2dir.json
|   |-- base16_shell.json
|   |-- bashisms.json
|   |-- cmd_done.json
|   |-- coreutils.json
|   |-- direnv.json
|   |-- distributed.json
|   |-- docker_tabcomplete.json
|   |-- free_cwd.json
|   |-- fzf-widgets.json
|   |-- gitinfo.json
|   |-- hist_navigator.json
|   |-- histcpy.json
|   |-- jedi.json
|   |-- kitty.json
|   |-- mpl.json
|   |-- onepath.json
|   |-- output_search.json
|   |-- pdb.json
|   |-- pipeliner.json
|   |-- powerline.json
|   |-- powerline2.json
|   |-- powerline_binding.json
|   |-- prompt_bar.json
|   |-- prompt_ret_code.json
|   |-- prompt_vi_mode.json
|   |-- pyenv.json
|   |-- readable-traceback.json
|   |-- schedule.json
|   |-- scrapy_tabcomplete.json
|   |-- sh.json
|   |-- ssh_agent.json
|   |-- vox.json
|   |-- vox_tabcomplete.json
|   |-- whole_word_jumping.json
|   |-- xo.json
|   |-- xog.json
|   |-- xpg.json
|   |-- z.json
|   `-- zoxide.json
|-- xontribs.db
`-- xontribs.json

1 directory, 50 files

a single meta data

{
  "name": "vox",
  "package": {
    "license": "BSD 3-clause",
    "url": "http://xon.sh",
    "install": {
      "conda": "conda install -c conda-forge xonsh",
      "pip": "xpip install xonsh",
      "aura": "sudo aura -A xonsh",
      "yaourt": "yaourt -Sa xonsh"
    }
  },
  "url": "http://xon.sh",
  "description": [
    "Python virtual environment manager for xonsh."
  ]
}