Skip to content

Instantly share code, notes, and snippets.

@dbast
Created December 11, 2013 21:12
Show Gist options
  • Save dbast/7918513 to your computer and use it in GitHub Desktop.
Save dbast/7918513 to your computer and use it in GitHub Desktop.
uniform_exposure.py multiprocessing patch
--- uniform_exposure.py 2013-12-11 21:23:03.000000000 +0100
+++ uniform_exposure_par.py 2013-12-11 22:04:49.000000000 +0100
@@ -66,18 +66,16 @@
# =====================================================================================
import os, sys, re, time, datetime, subprocess, shlex, shutil
+from multiprocessing import Pool
from math import *
+
+os.putenv("OMP_NUM_THREADS", "1") # optimizes the parallelisation, by allowing only one OpenMP thread per shell programm
+
log2 = lambda x: log(x) / log(2)
sign = lambda x: x / abs(x) if x != 0 else 0
direrr = False
-try: os.mkdir(out_dir)
-except: print "Warning: could not create output dir '%s'" % out_dir
-
-try: os.mkdir(tmp_dir)
-except: print "Warning: could not create working dir '%s'" % tmp_dir
-
def progress(x, interval=1):
global _progress_first_time, _progress_last_time, _progress_message, _progress_interval
@@ -98,7 +96,6 @@
_progress_last_time = time.time()
def run(cmd):
- f = open("dev.log", "a");
try:
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out = p.communicate()
@@ -108,10 +105,12 @@
print out[1]
raise SystemExit
+ f = open("dev.log", "a");
print >> f, cmd
print >> f, out[0]
print >> f, out[1]
print >> f, ""
+ f.close()
return out[0]
except KeyboardInterrupt:
raise SystemExit
@@ -119,12 +118,11 @@
raise SystemExit
except:
print sys.exc_info()
- f.close()
def change_ext(file, newext):
return os.path.splitext(file)[0] + newext
-def file_number(f):
+def file_number(j):
nr = re.search("([0-9][0-9][0-9][0-9]+)", j)
if nr:
nr = int(nr.groups()[0])
@@ -244,29 +242,13 @@
if median >= 1:
return
gamma = log2(median) / log2(target_median)
- print ("(gamma %.02f)" % gamma), ; sys.stdout.flush()
+ line = ("(gamma %.02f)" % gamma)
cmd = 'mogrify -gamma %f "%s" ' % (gamma, file)
run(cmd)
+ return line
-files = sorted(os.listdir(raw_dir))
-
-# prefer the DNG if there are two files with the same name
-for f in [f for f in files]:
- dng = change_ext(f, ".DNG")
- if f[0] == ".":
- files.remove(f)
- continue
- if dng != f and dng in files:
- files.remove(f)
- continue
-
- # only process CR2 and DNG
- if not(f.endswith(".CR2") or f.endswith(".cr2") or f.endswith(".DNG") or f.endswith(".dng")):
- files.remove(f)
- continue
-
-progress("")
-for k,f in enumerate(files):
+def main(f):
+ global ufraw_options, default_ufraw_options, overall_bias, default_overall_bias, highlight_level, default_highlight_level, midtone_level, default_midtone_level, shadow_level, default_shadow_level, samyang8ff, default_samyang8ff, fullsize, default_fullsize, target_median, default_target_median, ev_step, default_ev_step, enfuse_options, default_enfuse_options
r = os.path.join(raw_dir, f)
j = os.path.join(out_dir, change_ext(f, ".tif" if output_tif16bit else ".jpg"))
jm = os.path.join(tmp_dir, change_ext(f, "-m.tif"))
@@ -277,15 +259,14 @@
# don't overwrite existing jpeg files
if os.path.isfile(j) or os.path.isfile(change_ext(j, "r.jpg")):
- print "%s: output file %s already exists, skipping" % (r, j)
- continue
+ return "%s: output file %s already exists, skipping\n" % (r, j), None
# skip sub-dirs under raw_dir
if not os.path.isfile(r):
- continue
+ return None, None
if f.lower().endswith('.jpg'):
- continue
+ return None, None
rotate_options = ""
if os.path.isfile(lev):
@@ -295,13 +276,13 @@
ar = "2:3" if abs(r90) == 90 else "3:2"
rotate_options = " --rotate=%s --auto-crop --aspect-ratio %s " % (-roll, ar)
- print ""
- print "%s:" % r
+ lines = ""
+ lines += "%s:\n" % r
# override settings
override_settings(r, file_number(r))
ufraw_options = rotate_options + ufraw_options
- print ufraw_options
+ lines += ufraw_options + "\n"
# compute percentiles
mm, mh, ms = get_medians(r)
@@ -328,10 +309,10 @@
else: ecs = [ecs]
# print the levels
- print " midtones: brightness level %5d => exposure %+.2f EV" % (mm, ecm)
- print " highlights: brightness level %5d => exposure %s EV %s" % (mh, ",".join(["%+.2f" % e for e in ech]), "" if needs_highlight_recovery else "(skipping)")
- print " shadows: brightness level %5d => exposure %s EV %s" % (ms, ",".join(["%+.2f" % e for e in ecs]), "" if needs_shadow_recovery else "(skipping)")
- print "", ; sys.stdout.flush()
+ lines += " midtones: brightness level %5d => exposure %+.2f EV\n" % (mm, ecm)
+ lines += " highlights: brightness level %5d => exposure %s EV %s\n" % (mh, ",".join(["%+.2f" % e for e in ech]), "" if needs_highlight_recovery else "(skipping)")
+ lines += " shadows: brightness level %5d => exposure %s EV %s\n" % (ms, ",".join(["%+.2f" % e for e in ecs]), "" if needs_shadow_recovery else "(skipping)")
+ lines += ""
# any ufraw settings file? use it when developing
if os.path.isfile(ufr):
@@ -340,52 +321,51 @@
# develop the raws
shrink = 1 if fullsize == True else (2 if fullsize == False else fullsize)
jpegs = [jm]
- print "(midtones)", ; sys.stdout.flush()
+ lines += "(midtones)"
cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, ecm, r, jm, shrink)
run(cmd)
if needs_highlight_recovery:
# highlight recovery
- print "(highlights", ; sys.stdout.flush()
+ lines += "(highlights"
for ji,e in enumerate(ech):
- if ji > 0: print "\b.", ; sys.stdout.flush()
+ if ji > 0: lines += "\b."
jp = change_ext(jh, "%d.tif" % ji)
cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, e, r, jp, shrink)
run(cmd)
jpegs.append(jp)
- print "\b)", ; sys.stdout.flush()
+ lines += "\b)"
if needs_shadow_recovery:
# shadow recovery
- print "(shadows", ; sys.stdout.flush()
+ lines += "(shadows"
for ji,e in enumerate(ecs):
- if ji > 0: print "\b.", ; sys.stdout.flush()
+ if ji > 0: lines += "\b."
jp = change_ext(js, "%d.tif" % ji)
cmd = 'ufraw-batch --out-type=tiff --out-depth=16 --overwrite %s --exposure=%s "%s" --output="%s" --shrink=%d' % (ufraw_options, e, r, jp, shrink)
run(cmd)
jpegs.append(jp)
- print "\b)", ; sys.stdout.flush()
+ lines += "\b)"
if needs_highlight_recovery or needs_shadow_recovery:
# blend highlights and shadows
- print "(enfuse)", ; sys.stdout.flush()
+ lines += "(enfuse)"
compression = "--compression=DEFLATE" if output_tif16bit else ""
cmd = 'enfuse %s %s -o "%s" %s' % (enfuse_options, compression, j, " ".join(['"%s"' % ji for ji in jpegs]))
run(cmd)
else:
# nothing to blend
if output_tif16bit:
- print "(copy)", ; sys.stdout.flush()
+ lines += "(copy)"
shutil.copy(jm, j)
else:
- print "(convert)", ; sys.stdout.flush()
+ lines += "(convert)"
run("convert %s %s" % (jm, j))
if target_median:
- gamma_correction(j, target_median)
+ lines += gamma_correction(j, target_median)
- cmd = "echo \"%s: overall_bias=%g; highlight_level=%g; midtone_level=%g; shadow_level=%g; ufraw_options='%s'; \" >> settings.log" % (f, overall_bias, highlight_level, midtone_level, shadow_level, ufraw_options)
- run(cmd)
+ settings_cmd = "echo \"%s: overall_bias=%g; highlight_level=%g; midtone_level=%g; shadow_level=%g; ufraw_options='%s'; \" >> settings.log" % (f, overall_bias, highlight_level, midtone_level, shadow_level, ufraw_options)
if 0:
# lossless optimization of the Huffman tables
@@ -406,7 +386,41 @@
for j in jpegs:
os.remove(j)
- print ""
- progress((k+1) / len(files))
-
+ lines += "\n"
+ return lines, settings_cmd
+if __name__ == '__main__':
+ try: os.mkdir(out_dir)
+ except: print "Warning: could not create output dir '%s'" % out_dir
+
+ try: os.mkdir(tmp_dir)
+ except: print "Warning: could not create working dir '%s'" % tmp_dir
+
+ progress("")
+ files = sorted(os.listdir(raw_dir))
+
+ # prefer the DNG if there are two files with the same name
+ for f in [f for f in files]:
+ dng = change_ext(f, ".DNG")
+ if f[0] == ".":
+ files.remove(f)
+ continue
+ if dng != f and dng in files:
+ files.remove(f)
+ continue
+
+ # only process CR2 and DNG
+ if not(f.endswith(".CR2") or f.endswith(".cr2") or f.endswith(".DNG") or f.endswith(".dng")):
+ files.remove(f)
+ continue
+
+ p = Pool()
+ it = p.imap(main, files)
+ for k,f in enumerate(files):
+ line, settings_cmd = it.next()
+ if line:
+ print line
+ if settings_cmd:
+ run(settings_cmd)
+ progress((k+1) / len(files))
+ p.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment