find used and missing samples with m8-js, also collect space usage stats

find_moved_samples.py

#!/usr/bin/env python3
"""
find used and missing samples with m8-js, also collect space usage stats.
also: for every missing sample, show other filenames under Samples/ with the same basename, 
nder the assumption the file may have been moved but not renamed. builds an index of all filenames (slow)
but caches it in a pickle file. note that pickle file is not automatically refreshed when you add/remove samples,
just delete it and re-run in that case.
"""

from pathlib import Path

import pickle
from collections import Counter, defaultdict
import pprint
import json
import subprocess
import argparse


# location of the executable file for m8-js https://github.com/whitlockjc/m8-js
M8JS_BIN_PATH = Path("~/bin/m8").expanduser()

index_cached = Path("wav_index.pickle")


def index_by_fname(m8_root):
    if index_cached.exists():
        with index_cached.open('rb') as f:
            return pickle.load(f)

    index = defaultdict(list)
    for wav_path in m8_root.rglob("*.[wW][aA][vV]"):
        index[wav_path.name].append(wav_path)

    pickle.dump(index, index_cached.open('wb'))
    return index


def main(args):
    m8_root = args.m8_root

    index = index_by_fname(m8_root)

    path_ctr = Counter()

    for m8s_file in sorted((m8_root/"Songs").glob("*.m8s")):
        print(m8s_file)

        m8js_cmd = [M8JS_BIN_PATH, 'export', str(m8s_file)]
        try:
            m8js_output = subprocess.check_output(m8js_cmd, stderr=subprocess.DEVNULL).decode('utf-8')
        except subprocess.CalledProcessError as e:
            print("m8js failed: ", e)
            continue

        song = json.loads(m8js_output)
        version = song['fileMetadata']['version']['majorVersion']
        if version == 3:
            print("M8JS version 3, not supported")
            return

        instruments = song['instruments']
        for instrument in instruments:
            if instrument['kindStr'] != 'SAMPLER':
                continue
            instr_params = instrument['instrParams']
            sample_path = Path(instr_params['samplePath'])
            if sample_path.is_absolute():
                # /foo/bar.wav  -> foo/bar.wav
                sample_path = sample_path.relative_to(sample_path.root)
            sample_path = m8_root / sample_path
            if sample_path.exists():
                path_ctr[sample_path] += 1
            else:
                print(f"  {sample_path.relative_to(m8_root)} missing, ", end='')
                if len(index[sample_path.name]) > 0:
                    print("replacement candidates:")
                    for fname in sorted(index[sample_path.name]):
                        print(f"    {fname.relative_to(m8_root)}")
                else:
                    print("no replacement candidates found")
        print('\n')
    total_sample_size = 0
    for fname in sorted(path_ctr):
        total_sample_size += fname.stat().st_size
    print(f"Total sample size (deduplicated): {total_sample_size/(1 << 20):.2f} MB")



if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("m8_root", type=Path)
    args = parser.parse_args()
    main(args)

find_used_and_missing_samples.py

#!/usr/bin/env python3

""" 
find used and missing samples with m8-js, also collect space usage stats.
"""

import argparse
import json
import subprocess

from collections import Counter
from pathlib import Path

# location of the executable file for m8-js https://github.com/whitlockjc/m8-js
M8JS_BIN_PATH = Path("~/bin/m8").expanduser()


def main(args):
    m8_root = args.m8_root

    path_ctr = Counter()
    missing = set()

    for m8s_file in sorted((m8_root / "Songs").glob("*.m8s")):
        # print(m8s_file)

        m8js_cmd = [M8JS_BIN_PATH, "export", str(m8s_file)]
        try:
            m8js_output = subprocess.check_output(
                m8js_cmd, stderr=subprocess.DEVNULL
            ).decode("utf-8")
        except subprocess.CalledProcessError as e:
            print("m8js failed: ", e)
            continue

        song = json.loads(m8js_output)
        instruments = song["instruments"]
        for instrument in instruments:
            if instrument["kindStr"] != "SAMPLER":
                continue
            instr_params = instrument["instrParams"]
            sample_path = Path(instr_params["samplePath"])
            if sample_path.is_absolute():
                # /foo/bar.wav  -> foo/bar.wav -> m8_root/foo/bar.wav
                sample_path2 = m8_root / sample_path.relative_to(sample_path.root)
            else:
                sample_path2 = m8_root / sample_path
            path_ctr[sample_path2] += 1
            if sample_path2.exists():
                size_mb = (sample_path2.stat().st_size)/(1 << 20)
                print(f"{str(m8s_file.relative_to(m8_root)):40} {str(sample_path):128} {size_mb:.2f} MB")
            else:
                print(f"{str(m8s_file.relative_to(m8_root)):40} {str(sample_path):128} MISSING")
                missing.add(sample_path)

    total_sample_size_unique = 0
    total_sample_size = 0
    for fname, cnt in path_ctr.most_common():
        if fname.exists():
            size = fname.stat().st_size
            total_sample_size_unique += size 
            total_sample_size += (cnt * size)
    print(f"Total sample size: {total_sample_size/(1 << 20):.2f} MB")
    print(f"Total sample size (deduplicated): {total_sample_size_unique/(1 << 20):.2f} MB")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Find used & missing samples in M8 songs, get size stats")
    parser.add_argument(
        "m8_root",
        type=Path,
        help="Path to M8 SD card root directory (containing Songs/ and Samples/ subdirectories)",
    )
    args = parser.parse_args()
    main(args)