Transcode your lossless library to opus in parallel

transcoder.nim

import std/[os, osproc, algorithm, strutils, streams, logging]

const
  SOURCEDIR = "/mnt/bigdrive/media/music/"
  TARGETDIR = "/mnt/bigdrive/media/musicLossy/"
  EXTENSION = "opus"
  TRANSCODE = "opusenc --quiet --bitrate 128"
  COPY = "cp -v --reflink -u $# $#"
  LOGFILE = "transcoder.log"
  TRANSCODE_TYPES = @[".wav", ".flac"]
  COVER_FILE = "cover"
  IGNORE_COVERS = true
  COVER_TYPES = @[".jpg", ".png"]
  COMPLEX_CHECK = true

var fileLog = newFileLogger(LOGFILE)

# Logs errors to log file
proc logOutput(idx: int; p: Process) =
  for line in p.errorStream.lines:
    fileLog.log(lvlInfo, line)

# Logs missing files
proc logDiff(seqA, seqB: seq[string], logFile: string = LOGFILE) =
  fileLog.log(lvlInfo, "\nMissing files:")
  for aItem in seqA:
    if aItem notin seqB:
      fileLog.log(lvlInfo, aItem)

# Returns target file path and command for its transcoding
proc transcode(file, cover, sourceFolder, targetFolder: string, ext: string = EXTENSION): (string, string) =
  var coverArg: string = ""
  if cover != "":
    coverArg = " --picture " & quoteShell(sourceFolder & "/" & cover)
  let
    sourceFile = sourceFolder & "/" & file
    targetFile = targetFolder & "/" & os.changeFileExt(file, ext)
    cmd = TRANSCODE & coverArg & " " & quoteShell(sourceFile) & " " & quoteShell(targetFile)
  return (targetFile, cmd)

# Returns target file path and command for its copying
proc copy(file, sourceFolder, targetFolder: string): (string, string) =
  let
    sourceFile = sourceFolder & "/" & file
    targetFile = targetFolder & "/" & file
    cmd = COPY % [quoteShell(sourceFile), quoteShell(targetFile)]
  return (targetFile, cmd)

# Tries to find a cover file and returns its path if it exists
proc findCover(folder, coverFile: string = COVER_FILE, coverTypes: seq[string] = COVER_TYPES): string =
  for coverType in coverTypes:
    if os.fileExists(coverFile & coverType):
      return coverFile & coverType

proc addProcess(
  cmd, targetFile: string,
  cmds, targetFiles: var seq[string],
  logging, overwrite: bool): (seq[string], seq[string]) =
  if overwrite:
    cmds.add(cmd)
    if logging:
      fileLog.log(lvlInfo, cmd)
    targetFiles.add(targetFile)
  else:
    if not os.fileExists(targetFile):
      cmds.add(cmd)
      if logging:
        fileLog.log(lvlInfo, cmd)
      targetFiles.add(targetFile)
  return (targetFiles, cmds)

proc updateLibrary(
  sourceDir: string = SOURCEDIR,
  targetDir: string = TARGETDIR,
  logging, overwrite: bool = false): (seq[string], seq[string]) =

  var
    cover, targetFile, cmd: string
    targetFolders, sourceFolders, targetFiles, cmds: seq[string]

  for sourceFolder in os.walkDirRec(sourceDir, yieldFilter={pcDir}, followFilter={pcDir}):
    sourceFolders.add(sourceFolder)
    let targetFolder = strutils.replace(sourceFolder, sourceDir, targetDir)
    targetFolders.add(targetFolder)
    discard os.existsOrCreateDir(targetFolder)
    os.setCurrentDir(sourceFolder)
    cover = findCover(sourceFolder)
    for file in os.walkFiles("*"):
      let ext = os.splitFile(file).ext
      if ext in COVER_TYPES:
        if not IGNORE_COVERS:
          (targetFile, cmd) = copy(file, sourceFolder, targetFolder)
          (targetFiles, cmds) = addProcess(cmd, targetFile, cmds, targetFiles, logging, overwrite)
      elif ext in TRANSCODE_TYPES:
        (targetFile, cmd) = transcode(file, cover, sourceFolder, targetFolder)
        (targetFiles, cmds) = addProcess(cmd, targetFile, cmds, targetFiles, logging, overwrite)
      else:
        (targetFile, cmd) = copy(file, sourceFolder, targetFolder)
        (targetFiles, cmds) = addProcess(cmd, targetFile, cmds, targetFiles, logging, overwrite)

  discard osproc.execProcesses(cmds,
                               options = {poStdErrToStdOut},
                               afterRunEvent = logOutput)
  return (targetFolders, targetFiles)

# A check function that offers a basic check and complex check
# basic check will see if the expected number of items is created
# complex check will make sure that each file exists.
# Returns true if succesful, and vice versa
proc check(
  targetFolders, targetFiles: var seq[string],
  sourceDir: string = SOURCEDIR,
  targetDir: string = TARGETDIR,
  complexCheck: bool = COMPLEX_CHECK): bool =

  if targetFiles != @[]:
    var
      createdFolders, createdFiles: seq[string]
      folderAssertion, fileAssertion: bool

    for createdFolder in os.walkDirRec(targetDir, yieldFilter={pcDir}, followFilter={pcDir}):
      createdFolders.add(createdFolder)
      os.setCurrentDir(createdFolder)
      for file in os.walkFiles("*"):
        let createdFile = createdFolder & "/" & file
        createdFiles.add(createdFile)

    if complexCheck:
      targetFolders.sort()
      createdFolders.sort()
      folderAssertion = targetFolders == createdFolders
      targetFiles.sort()
      createdFiles.sort()
      fileAssertion = targetFiles == createdFiles
    else:
      folderAssertion = targetFolders.len == createdFolders.len
      fileAssertion = targetFiles.len == createdFiles.len

    result = false
    if folderAssertion:
      if fileAssertion:
        echo "all folders & files created!"
        result = true
      else:
        if len(createdFiles) != 0:
          logDiff(createdFiles, targetFiles)
          echo "files missing!"
    else:
      if len(createdFolders) != 0:
        logDiff(createdFolders, targetFolders)
        echo "folders missing!"
  else:
    result = true
    echo "nothing to check!"

echo "Updating library..."
var target = updateLibrary(overwrite = false, logging = true)
echo "Releases processed: " & $target[0].len
echo "Tracks processed: " & $target[1].len
echo "Checking..."
discard check(target[0], target[1])

Transcode your lossless library to opus in parallel

A quick and dirty Nim program to help transcoding a large library of music to opus.

The expected library format is:

/Music/
 └─ Artist
    └── Lossless Album
        ├── Track 1.{wav|flac}
        └── cover.{jpg|png}
    └── Lossy Album
        ├── Track 1.mp3
        └── cover.{jpg|png}

The updateLibrary function can be set to overwrite all files regardless of if they exist, otherwise, it will only add new ones.
The program is customisable by changing the value of the constants that you may want to change depending on your setup.
For files that are not lossless (files not in TRANSCODE_TYPES) the program simply copies them. I avoid copying cover files as I want to save space (IGNORE_COVERS).
By default, the program embeds cover art into opus files and expects a cover file (COVER_FILE) to be in the Album folder.