MIST FPGA update scripts

linux_sync

#!/bin/bash
# Pass directory to sync to SD card as parameter 1
# Mounts & syncs any directory in current directory 
# to an SD card that is named the same as the directory.
# Also sets special file settings on the SD card so that
# MIST displays them correctly.

if [ $USER != root ]; then
 echo "Must be run as root!"
 exit 1
fi

# copy from $1 parameter which is the subdirectory from here
# $2 is the Volume to copy to.

if [ -z "$1" ]; then
  echo "Usage: linux_sync Arcade"
  exit 1
fi

VOLUME=${1%/}

mkdir -p /mnt/MIST
#mount /dev/disk/by-uuid/3E42-111B /mnt/MIST
mount -L ${VOLUME@U} /mnt/MIST
if [ ! -f /mnt/MIST/mist.ini ]; then
  echo ERROR Not Mounted!!!
  exit 1
fi

rsync -uvtr --modify-window=1  --delete --exclude='*.ini' --exclude='*.CFG' --exclude='*.RAM' --exclude='*.sav' \
      --exclude='QXL.WIN' \
          --exclude='Next186.vhd' \
      --exclude='.Trashes' --exclude='.fseventsd' --exclude='.Spotlight-V100' --exclude='System Volume Information' \
      --exclude='.DS_Store' \
  $VOLUME/ /mnt/MIST

#Hide RBFs
if [ "$VOLUME" = "Arcade" ]; then
  cd /mnt/MIST
  fatattr +h JT-*/jt*.rbf 
  #Make directories visible
  fatattr +s JT-CPS* JT-SYSTEM16 JT-OTHER Gehstock/IremM72 Gehstock/IremM92 Gehstock/Williams Gehstock/Sega-System-1 Gehstock/Midway-MCR

  cd Gehstock
  fatattr +h Scramble.rbf Galaxian.rbf
  fatattr +h DKong.rbf GBeret.rbf
  fatattr +h Druaga.rbf
  fatattr +h Williams/*.rbf
  fatattr +h Sega-System-1/Segasys1.rbf
  fatattr +h Midway-MCR/mcr*.rbf
  fatattr +h Jailbrek.rbf
  fatattr +h IremM72/*.rbf
  fatattr +h IremM92/*.rbf

  #Remove 64MB games from CPS2
  cd /mnt/MIST/JT-CPS2/
  rm -f 'Dungeons & Dragons_ Shadow over Mystara (Euro 960619).arc' "Night Warriors_ Darkstalkers' Revenge (Euro 950316).arc" 'Street Fighter Alpha 3 (Euro 980904).arc' 'X-Men_ Children of the Atom (Euro 950331).arc' 'Cyberbots_ Fullmetal Madness (Euro 950424).arc' 'X-Men Vs. Street Fighter (Euro 961004).arc' 'X-Men_ Children of the Atom (Euro 950331).arc' 'Vampire Hunter 2_ Darkstalkers Revenge (Japan 970929).arc' ' Vampire Savior 2_ The Lord of Vampire (Japan 970913).arc' 'Vampire Savior_ The Lord of Vampire (Euro 970519).arc' 'Street Fighter Alpha 2 (Euro 960229).arc' "Street Fighter Zero 2 Alpha (Japan 960805).arc" "Hyper Street Fighter II_ The Anniversary Edition (USA 040202).arc" "Mars Matrix_ Hyper Solid Shooting (USA 000412).arc" "Marvel Super Heroes (Euro 951024).arc" "Marvel Super Heroes Vs. Street Fighter (Euro 970625).arc" "Marvel Vs. Capcom_ Clash of Super Heroes (Euro 980123).arc" "Vampire Savior 2_ The Lord of Vampire (Japan 970913).arc"

  cd /tmp

fi

sleep 5s
umount /mnt/MIST

update_cores

#!/bin/bash

# Gets the latest cores and checks all the directories
# in the current for any changes to the cores.
# The directories MUST match the ones in the mist-binaries/cores so just create
# the directories of the cores you want.

#Define this if using Mistica for specific cores
#MISTICA=Y

LOGFILE=update_cores.log

date >$LOGFILE
echo >>$LOGFILE

BASEDIR=_temp
SRCDIR=${BASEDIR}/git/mist-binaries/cores

# clone git repo if it doesn't exist
if [[ ! -d $SRCDIR ]]; then
  mkdir -p ${BASEDIR}/git
  (cd ${BASEDIR}/git; git clone https://github.com/mist-devel/mist-binaries.git)
  #Set timestamps on git files to match repository commit dates
  (cd $SRCDIR; git ls-files | sed "s/'/\\\'/g" | xargs -I{} bash -c 'touch "{}" --date=@$(git log -n1 --pretty=format:%ct -- "{}")')
fi

echo $SRCDIR

# Update the cores from github
(cd $SRCDIR; git pull)

#Check for firmware updates
SRC=`ls -Lt "${SRCDIR}"/../firmware/*.upg 2>/dev/null | head -1`
DST=_firmware/`basename "$SRC"`
mkdir -p _firmware
if [[ ! -f $DST ]]; then
  cp -p "$SRC" "$DST"
  echo ==================
  echo FIRMWARE UPDATED!!! $DST | tee -a $LOGFILE
  echo ==================
  echo 
fi

echo

#Check all the cores in git with current matching directories 
for FILE in `find . -maxdepth 1 -type d | egrep -v '_temp'`
do

  DIR=${FILE:2}

  DST="${DIR}/core.rbf"
  if [[ $DIR = "spectrum" ]];then
    DST="${DIR}/zxspectrum.rbf"
  fi

  SRC=`ls -Lt ${SRCDIR}/$DIR/*.rbf 2>/dev/null | head -1`

  # Mistica currently has core specific SNES
  if [[ $DIR = "snes" && ! -z "$MISTICA" ]]; then
    SRC=`ls -Lt ${SRCDIR}/$DIR/snes_mist_*.rbf 2>/dev/null | head -1`
    if [[ ! -z "$MISTICA" ]]; then
      SRC=`ls -Lt ${SRCDIR}/$DIR/snes_mistica*.rbf 2>/dev/null | head -1`
    fi
  fi

  if [[ $DIR = "plus_too" ]]; then
    SRC=`ls -Lt ${SRCDIR}/$DIR/plusToo_*.rbf 2>/dev/null | head -1`
  fi
  if [[ $DIR = "zxn" ]]; then
    SRC=`ls -Lt ${SRCDIR}/$DIR/ZXN_*.rbf 2>/dev/null | head -1`
  fi

  if [[ -z $SRC || -z $DST ]]; then
    continue
  fi 

  if [[ -f $SRC && ! -f $DST ]]; then
    cp -p "$SRC" "$DST"
    echo Update CORE $DST | tee -a $LOGFILE
  else
    if [[ $SRC -nt $DST ]]; then
      cp -p "$SRC" "$DST"
      echo Update CORE $DST | tee -a $LOGFILE
      echo
    else
      #Check checksum
      MD5SRC=`md5sum "$SRC" | awk '{print $1}'`
      MD5DST=`md5sum "$DST" | awk '{print $1}'`
  
      if [[ $MD5SRC != $MD5DST ]]; then
        cp -p "$SRC" "$DST"
        echo Update CORE $DST | tee -a $LOGFILE
        echo
      fi
    fi
  fi

  #Copy special ROMS
  [[ $DIR = "atari800" ]] && cp -pvu ${SRCDIR}/${DIR}/*.ROM ${DIR}/
  [[ $DIR = "bbc" ]] && cp -pvu ${SRCDIR}/${DIR}/bbc.rom ${DIR}/
  [[ $DIR = "c16" ]] && cp -pvu ${SRCDIR}/${DIR}/c16.rom ${DIR}/
  [[ $DIR = "fpga64" ]] && cp -pvu ${SRCDIR}/${DIR}/*.rom ${SRCDIR}/${DIR}/C64GS.ARC ${DIR}/ 
  [[ $DIR = "ht1080z" ]] && cp -pvu ${SRCDIR}/${DIR}/HT1080Z.ROM ${DIR}/
  [[ $DIR = "ql" ]] && cp -pvu ${SRCDIR}/${DIR}/ql.rom ${DIR}/
  [[ $DIR = "samcoupe" ]] && cp -pvu ${SRCDIR}/${DIR}/samcoupe.rom ${DIR}/
  [[ $DIR = "spectrum" ]] && cp -pvu ${SRCDIR}/${DIR}/spectrum.rom ${DIR}/
  [[ $DIR = "ti994a" ]] && cp -pvu ${SRCDIR}/${DIR}/TI994A.ROM ${DIR}/
  [[ $DIR = "vic20" ]] && cp -pvu ${SRCDIR}/${DIR}/vic20.rom ${DIR}/
  [[ $DIR = "next186" ]] && cp -pvu ${SRCDIR}/${DIR}/Next186.ROM ${DIR}/

  #Move ZXN to spectrum
  [[ $DIR = "zxn" ]] && cp -pvu zxn/core.rbf spectrum/

done

update_gehstock

#!/bin/bash

# Only updates the cores that we have in the Arcade directory
# updated from Gehstock github directory.
# Doesn't add newly added cores from from github, add these manually.
# Names also need to match the Gehstock name

LOGFILE=update_gehstock.log
date >$LOGFILE
echo >$LOGFILE

BASEDIR=_temp
SRCDIR=$BASEDIR/git/Mist_FPGA_Cores/Arcade_MiST

# clone git repo if it doesn't exist
if [[ ! -d $SRCDIR ]]; then
  mkdir -p $BASEDIR/git
  (cd ${BASEDIR}/git; git clone https://github.com/Gehstock/Mist_FPGA_Cores.git)
  #Set timestamps on git files to match repository commit dates
  (cd $SRCDIR; git ls-files | sed "s/'/\\\'/g" | xargs -I{} bash -c 'touch "{}" --date=@$(git log -n1 --pretty=format:%ct -- "{}")')
fi

# Update the cores from github
(cd $SRCDIR/..; git pull)

for FILE in `find Arcade -name '*.rbf'`
do

  DST="$FILE"
  DSTBASE=`basename "$FILE"`

  SRC=`find "$SRCDIR" -name $DSTBASE`

  if [[ -z $SRC || -z $DST ]]; then
    continue
  fi

  if [[ $SRC -nt $DST ]]; then
    cp -p "$SRC" "$DST"
    echo Update CORE $DST | tee -a $LOGFILE
    echo
  else
    #Check checksum
    MD5SRC=`md5sum "$SRC" | awk '{print $1}'`
    MD5DST=`md5sum "$DST" | awk '{print $1}'`

    if [[ $MD5SRC != $MD5DST ]]; then
      cp -p "$SRC" "$DST"
      echo Update CORE $DST | tee -a $LOGFILE
      echo
    fi

  fi

done

update_jtcores

#!/bin/bash

# Updates and gets jt cores
# Retrieves new game ROMS if needed from archive.org
# Don't forget to set the SYSTEM attribute for any subdirectories
# on the SD card - Linux "fatattr" or Windows "attrib +s JT-SYSTEM16".

BASEDIR=_temp
SRCDIR=$BASEDIR/git/jtbin
ZIPDIR=$BASEDIR/mame
MRABIN=./mra

if [[ ! -f $MRABIN ]]; then
  echo "mra executable required"
  exit 1
fi

FLAGFILE=update_jtcores.flag
if [[ ! -f $FLAGFILE ]]; then
  touch -t 197201010000 $FLAGFILE
fi

mkdir -p $ZIPDIR

# clone git repo if it doesn't exist
if [[ ! -d $SRCDIR ]]; then
  mkdir -p $BASEDIR/git
  (cd $BASEDIR/git; git clone https://github.com/jotego/jtbin.git)
  #Set timestamps on git files to match repository commit dates
  (cd $SRCDIR; git ls-files | sed "s/'/\\\'/g" | xargs -I{} bash -c 'touch "{}" --date=@$(git log -n1 --pretty=format:%ct -- "{}")')
fi

echo $SRCDIR

# Update the cores and mra from jtbin
(cd $SRCDIR; git pull)

echo

#Process MRA files first
#Get new MRAs and updated ones since the last run
find $SRCDIR/mra -type f -name '*.mra' -newer $FLAGFILE -print0 | 
while IFS= read -r -d '' MRAFILE
do

  # Exclude what I'm not interested in
  [[ $MRAFILE == *_alternatives* ]] && continue
  #[[ $MRAFILE == *Tokio* ]] && continue
  #[[ $MRAFILE == *Trojan* ]] && continue

  RBF=$($MRABIN -l "$MRAFILE" | grep 'rbf name:' | sed 's/rbf name: //')

  DESTDIR=Arcade/JT-OTHER
  [[ $RBF == *CPS0/* ]] && DESTDIR=Arcade/JT-CPS0
  [[ $RBF == jtcps1 ]] && DESTDIR=Arcade/JT-CPS1
  [[ $RBF == jtcps15 ]] && DESTDIR=Arcade/JT-CPS15
  [[ $RBF == jtcps2 ]] && DESTDIR=Arcade/JT-CPS2
  [[ $RBF == jts16* ]] && DESTDIR=Arcade/JT-SYSTEM16
  mkdir -p $DESTDIR

  # Check ROM exists
  # Get First ROM section only, some MRAs reference two possible ROM options
  ZIPFILE=""
  ZIPFILE2=""
  ZIPFILE=`$MRABIN -l "$MRAFILE" | grep 'zip\[0\]' | head -1 | awk '{print $2}'`
  ZIPFILE2=`$MRABIN -l "$MRAFILE" | grep 'zip\[1\]' | head -1 | awk '{print $2}'`

  # Skip roms not interested in
  #if [[ $ZIPFILE = "qtono2j.zip" ]]; then
  #  continue
  #fi

  # Need qsound for audio
  if [[ ! -f "${ZIPDIR}/qsound.zip" ]]; then 
    (cd "$ZIPDIR";  wget -nc -nv https://archive.org/download/mame-merged/mame-merged/qsound.zip)
  fi

  #echo $ZIPFILE
  basename "$MRAFILE" .mra
  if [[ ! -f "${ZIPDIR}/${ZIPFILE}" ]]; then
    echo Downloading ROM $ZIPFILE
    (cd "$ZIPDIR";  wget -nc -nv https://archive.org/download/mame-merged/mame-merged/${ZIPFILE})
  fi
  if [[ -n "$ZIPFILE2" && ! -f "${ZIPDIR}/${ZIPFILE2}" ]]; then
    echo Downloading ROM $ZIPFILE2
    (cd "$ZIPDIR";  wget -nc -nv https://archive.org/download/mame-merged/mame-merged/${ZIPFILE2})
  fi

  $MRABIN -A -O $DESTDIR/ -z "$ZIPDIR" "$MRAFILE" | grep -v 'expected: None'

  # Fix wrong named RBFs
  #RBF=$(echo $RBF | sed 's/jtcommando/jtcom/')

  if [[ ! -f $SRCDIR/mist/${RBF}.rbf ]]; then
    echo "RBF $RBF is missing for MRA `basename "${MRAFILE}"`, probably BETA"
  else
    RBFBASE=$(basename "$RBFFILE" | sed 's/_[0-9]\+.rbf//')
    rsync -ut "$SRCDIR/mist/${RBF}.rbf" $DESTDIR/${RBF}.rbf
    echo ${RBF}.rbf
  fi

done

#Check for beta RBF cores becoming release, i.e. they would be missed
SRCDIR=$SRCDIR/mist

find $SRCDIR -name '*.rbf' -print0 |
while IFS= read -r -d '' FILE
do
  RBF=`basename "$FILE"`
  DST="Arcade/$FILE"

  SRC=$SRCDIR/$DSTFILE

  if [[ ! $(find Arcade/JT* -name $RBF | grep .) ]]; then
    # Found missing rbf so find out where it belongs
    DESTDIR=Arcade/JT-OTHER
    [[ $RBF == *CPS0/* ]] && DESTDIR=Arcade/JT-CPS0
    [[ $RBF == jtcps1 ]] && DESTDIR=Arcade/JT-CPS1
    [[ $RBF == jtcps15 ]] && DESTDIR=Arcade/JT-CPS15
    [[ $RBF == jtcps2 ]] && DESTDIR=Arcade/JT-CPS2
    [[ $RBF == jts16* ]] && DESTDIR=Arcade/JT-SYSTEM16

    cp -pv $FILE $DESTDIR/
     
  fi
done

#Check for any updated RBFs with no updated MRA
find Arcade/JT* -name '*.rbf' -print0 |
while IFS= read -r -d '' FILE
do
  RBF=`basename "$FILE"`

  if [[ $SRCDIR/$RBF -nt "$FILE" ]]; then
    echo Updated $RBF
    cp -p $SRCDIR/$RBF "$FILE"
  fi
done

touch $FLAGFILE

Create a directory to store your MIST cores and ROMS. In this directory create subdirectories of the cores you want, name based on https://github.com/mist-devel/mist-binaries/tree/master/cores. In addition for the arcade cores create a "Arcade" subdirectory.
Run these scripts to retrieve the latest cores and ROMS.
Copy the directories to individual SD cards or one SD card using linux_sync.
Requires a number of Linux commands: git, wget and the mra executable.

dont forget to use chmod to mark the content as executable after you get the script and mra

wget https://gist.githubusercontent.com/squidrpi/4ce3ea61cbbfa3900e116f9565d45e74/raw/cdbfa6d52a87ebdb7524f8fc8ceed49876c24b1d/update_jtcores
wget https://github.com/mist-devel/mra-tools-c/raw/master/release/linux/mra

chmod u+x ./update_jtcores
chmod u+x ./mra

Thanks for a great script, I had tried and failed a few times to do this manually after reading the doco. (I think I had the wrong version of the ROMS).