Organize pictures in folder by date

Organize_pictures_in_folder_by_date.sh

# All photos and videos are in a folder, from here on called ORIGINALS

# Parallel command can speed up some tasks, specially the conversion ones, check -j for the number of processes to run in parallel

# Before starting, lets clean temporary files to make sure we dont have to deal with them later
find ORIGINALS -maxdepth 9999 -noleaf -type f \( -iname ".DS_Store" -o -name "._*" -o -iname "thumbs.db" -o -iname ".picasa.ini" \) -exec rm -v "{}" \;

# Before continue, maybe search for some file types for some extra cleanup (for quick cleanup copy the -exec part above)
find ORIGINALS -maxdepth 9999 -noleaf -iregex '.*\.\(exe\|tcb\|tcpmd\|txt\|doc\|docx\|xls\|xlsx\|ptp\|7z\|rar\|zip\|iso\|mp3\|m3u\|amr\|xmp\|epp\|xcf\)$'

# Make a backup of your photo folder before proceeding, this is highly important in case something goes wrong
# or you want to confirm how many files were there originally to make sure all was organized. Backup with 
# rsync in order to keep the original files metadata
rsync -av ORIGINALS ORIGINALS_bck

# Convert old and less supported file formats to generalize collection
# This will also help on the process of metadata tagging later, since some formats are not supported by exiftool
# Examples:
#   1. Convert BMP, Xcf and heic images to png (sudo apt-get install imagemagick) - optionally also convert heic
find ORIGINALS_bck -maxdepth 9999 -noleaf -type f -iregex '.*\.\(bmp\|xcf\|heic\)$' -exec convert "{}" "{}".png \; -exec exiftool -ext "*" -m -overwrite_original -tagsfromfile "{}" '-AllDates<Datemodify' '-FileModifyDate<Datemodify' "{}".png \;
# Compare tags with exiftool and see if they are ok, then run the find with just '-exec rm -v "{}" \;' to cleanup the old files
find ORIGINALS_bck -maxdepth 9999 -noleaf -type f -iregex '.*\.\(bmp\|xcf\|heic\)$' -exec rm -v "{}" \;

#   2. Convert mpg,avi,mov,asf... to mp4 (sudo apt-get install ffmpeg)
find ORIGINALS_bck -maxdepth 9999 -noleaf -type f -iregex '.*\.\(asf\|mov\|avi\|mpg\|flv\|ogg\|mts\|vob\|mpeg\|wmf\)$' -exec ffmpeg -i "{}" -map_metadata 0 -movflags use_metadata_tags "{}".mp4 \; 
# Compare tags with exiftool and see if they are ok (for dates `exiftool -time:all -G -a -s FILE`)
# otherwise, fix dates from video files if not ok - CreateDate or otherwise FileModifyDate is taken into account
find ORIGINALS_bck -maxdepth 9999 -noleaf -type f -iregex '.*\.\(asf\|mov\|avi\|mpg\|flv\|ogg\|mts\|vob\|mpeg\|wmf\)$' -exec exiftool -m -ext "*" -overwrite_original -tagsfromfile "{}" "-AllDates<FileModifyDate" "-AllDates<CreateDate" "{}".mp4 \;
# and as last step, cleanup old files (make sure date tags are correct before)
find ORIGINALS_bck -maxdepth 9999 -noleaf -type f -iregex '.*\.\(asf\|mov\|avi\|mpg\|flv\|ogg\|mts\|vob\|mpeg\|wmf\)$' -exec rm -v "{}" \;


# Before proceeding you should tag all the files with both the original folder path and file name so that this valueable information is not lost
#  * -r recursivelly visit all folders in ORIGINALS_bck
#  * -ext "*" processes all file types (.bmps, videos, etc)
#  * we're storing the folders (from local dir on) in ImageDescription tag, comma separated, and the original filename in OriginalRawFileName
#    other tags can be used if there is valuable information on this ones (hint: keywords, subject) https://exiftool.org/TagNames/
exiftool -ext "*" -m -r -overwrite_original '-ImageDescription<${directory;s/\//, /g}' "-OriginalRawFileName<filename" ORIGINALS_bck 2> all.txt
# The ones that returned "Error: XXX" had issues and were not updated
`cat all.txt | grep "Error:" | wc -l` # should match the count given when the command finished for the fail ones
# Go over all of them and make sure to fix them. Run the exiftool again to make sure all were fixed.

# Depending on how much work this all was, maybe its time for another backup of data, just in case
rsync -av ORIGINALS_bck ORIGINALS_bck2

# Moves the file to the organized location from here on refered to as ORGANIZED
# in the end `ORIGINALS_bck` should stay with 0 files and `ORGANIZED` have all of them organized
#  * '/%Y/%m/' creates forder for year and inside folder for month
#  * remove '%Y%m%d_%H%M%S%%-03.c.%%e' to stop renaming the files.
#  * '%%-03.c' part adds incremental 3 digit number for photos created at the same second. Removing this part will make it store only the last photo with that timestamp
#  * "-filemodifydate<CreateDate" "-filecreatedate<CreateDate" sets metadata in the files
exiftool -ext "*" -r -d "ORGANIZED/%Y/%m/%Y%m%d_%H%M%S%%-03.c.%%e" "-Filename<CreateDate" "-filemodifydate<CreateDate" "-filecreatedate<CreateDate" ORIGINALS_bck2

#  Make sure all data was moved by checking how many files are left on the ORIGINALS_bck
#  Remove empty folders on ORIGINALS_bck to help find the files
find ORIGINALS_bck -empty -type d -delete


# Possible ISSUES

# Error: Truncated InteropIFD Directory -- Fix as  `exiftool -all= -tagsfromfile @ -all:all -unsafe -icc_profile $FILE`

# Warning: No writable tags set from ...
#   Some files may fail to be moved with `Warning: No writable tags set from`, this means `CreateDate` is not part of the meta information on that file, although it should
#   do `exiftool -time:all -s PATH/FILE` to confirm the existing dates, and run the command with the new meta info to be used for the file name

# Warning: Missing JPEG SOS ...
# Warning: Invalid EXIF text encoding for UserComment ...
# Warning: File is empty ...
# Warning: GPS pointer references previous InteropIFD directory - ...
#   No need to nothing, they will be moved, if command is run again, they will not be there anymore