Last active
August 29, 2015 14:16
-
-
Save oeway/60e7b769884a7034d243 to your computer and use it in GitHub Desktop.
This Gist perform the automation of thunderSTORM analysis by monitoring the file change and submit task to the cluster.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <h2>Introduction</h2> | |
| <p>This worksheet is used to monitor data changes in the defined folder, and apply it to another worksheet which created from a template.</p> | |
| <h2>Usage</h2> | |
| <h3>Step 1. Select the folder to be monitored</h3> | |
| <p>Select the "Input" by click on the text "Input:..." in the toolbar(beside Run and Pause button), and input the folder path.</p> | |
| <p>Please note that the path should be absolute path on the server seperated by "/", for example: <strong><em> /pasteur/projets/Imod-grenier/Will/data/dropbox</em></strong> .</p> | |
| <h3>Step 2. Define the template worksheet which will be applied when new data is detected.</h3> | |
| <p>Select the "Next Worksheet" by click the text "Input:..." in the toolbar(beside "Input:..."), and input the path of the next worksheet.</p> | |
| <p>You can get the path of a worksheet by its link address, for example, if the linke of a worksheet is "http://xxxx.com/home/username/9", then <strong>username/9</strong> is the path of the worksheet, witch will be filled into the popup dialog.</p> | |
| <h3>Step 3. Click Run, and it's running, you don't need to keep the browser open.</h3> | |
| <h2>Instructions for designing template worksheet(next worksheet)</h2> | |
| <p>When you design a worksheet for the template of this monitor, you need to notice that, for every new data, this worksheet will copy the next worksheet (template worksheet) first, and then it will set the "Input Directory" of the next worksheet with the path of the data file, then start the computing of the next worksheet.</p> | |
| <p>In the template worksheet, you can use a predefined variable named <strong>INPUT </strong>to access the data folder or data file.</p> | |
| {{{id=1| | |
| %hide | |
| import sys | |
| import time | |
| import os, errno | |
| #from watchdog.observers import Observer | |
| from watchdog.observers.polling import PollingObserver as Observer | |
| from watchdog.events import PatternMatchingEventHandler | |
| import pickle | |
| import time | |
| import PIL.Image | |
| defaultHistory = {'workfolder': INPUT, 'processed':{}, 'processing':{}} | |
| timeout = 5.0 #seconds | |
| try: | |
| print('trying to load from pickle file') | |
| fileHistory = pickle.load(open(DATA+'history.pickle','rb')) | |
| if fileHistory['workfolder']!=INPUT: | |
| fileHistory = defaultHistory | |
| else: | |
| print('file history loaded.') | |
| except: | |
| fileHistory = defaultHistory | |
| #print(fileHistory) | |
| includeList = ['.tiff','.tif'] | |
| def matchTiffFile(path): | |
| d,filename = os.path.split(path) | |
| if os.path.isdir(path): | |
| return False | |
| else: | |
| matched = False | |
| #not dir, try to match extension | |
| for ext in includeList: | |
| if ext.lower() in filename.lower(): | |
| matched = True | |
| break | |
| return matched | |
| def isMultiPageTiff(path): | |
| if os.path.isdir(path): | |
| raise ValueError | |
| with open(path) as f: | |
| img = PIL.Image.open(f) | |
| try: | |
| img.seek(1) | |
| return True | |
| except EOFError: | |
| return False | |
| for root, dirs, files in os.walk(INPUT, topdown=False): | |
| for name in files: | |
| path = os.path.join(root,name) | |
| #if it is tiff file | |
| if matchTiffFile(path): | |
| #if the file is a multi-page tiff file and not included by a folder named *.tif | |
| if isMultiPageTiff(path) and not '.tif' in root and not '.tiff' in root: | |
| if not fileHistory['processed'].has_key(path): | |
| print('add multi-page tiff file to processing:'+name) | |
| fileHistory['processing'][path] = {'size':os.path.getsize(path), 'timeStamp': time.time()} | |
| else: | |
| print('processed file:'+name) | |
| else: | |
| if not fileHistory['processed'].has_key(root): | |
| if not fileHistory['processing'].has_key(root): | |
| print('add folder to "processing":'+ root) | |
| fileHistory['processing'][root] = {'size':os.path.getsize(root), 'timeStamp': time.time()} | |
| else: | |
| print('processed folder:'+root) | |
| break | |
| else: | |
| print('skip file:' + name) | |
| def process(): | |
| for path in fileHistory['processing'].keys(): | |
| d,n = os.path.split(path) | |
| print('waiting for processing: '+n) | |
| pathInfo = fileHistory['processing'][path] | |
| newSize = os.path.getsize(path) | |
| if newSize != pathInfo['size']: | |
| pathInfo['size'] = newSize | |
| pathInfo['timeStamp'] = time.time() | |
| print('file size is changing:%d @ time:%f.'%(pathInfo['size'],pathInfo['timeStamp'])) | |
| if time.time()-pathInfo['timeStamp'] > timeout: | |
| worksheetName = path.lstrip(os.path.abspath(INPUT)) | |
| print('***creating worksheet for ' + worksheetName) | |
| CREATE_NEXT_WORKSHEET(path, worksheetName) | |
| fileHistory['processed'][path] = path | |
| del fileHistory['processing'][path] | |
| pickle.dump(fileHistory,open(DATA+'history.pickle','wb')) | |
| class MyHandler(PatternMatchingEventHandler): | |
| patterns = ["*"] | |
| case_sensitive=True | |
| ignore_patterns = ['.*','.db'] | |
| def process(self, event): | |
| """ | |
| event.event_type | |
| 'modified' | 'created' | 'moved' | 'deleted' | |
| event.is_directory | |
| True | False | |
| event.src_path | |
| path/to/observed/file | |
| """ | |
| path = event.src_path | |
| root,name = os.path.split(path) | |
| if os.path.isdir(path): | |
| return | |
| if matchTiffFile(path): | |
| #if the file is a multi-page tiff file and not included by a folder named *.tif | |
| if isMultiPageTiff(path) and not '.tif' in root and not '.tiff' in root: | |
| if not fileHistory['processed'].has_key(path): | |
| print('add multi-page tiff file to processing:'+name) | |
| fileHistory['processing'][path] = {'size':os.path.getsize(path), 'timeStamp': time.time()} | |
| else: | |
| print('processed file:'+name) | |
| else: | |
| if not fileHistory['processed'].has_key(root): | |
| if not fileHistory['processing'].has_key(root): | |
| print('add folder to processing:'+ root) | |
| fileHistory['processing'][root] = {'size':os.path.getsize(root), 'timeStamp': time.time()} | |
| else: | |
| print('processed folder:'+root) | |
| print name, event.event_type | |
| else: | |
| print('skip file:' + name) | |
| def on_created(self, event): | |
| self.process(event) | |
| if __name__ == '__main__': | |
| observer = Observer() | |
| observer.schedule(MyHandler(), path= INPUT, recursive = True) | |
| observer.start() | |
| print('Data Monitor is running...') | |
| try: | |
| while True: | |
| time.sleep(1) | |
| process() | |
| except KeyboardInterrupt: | |
| observer.stop() | |
| observer.join() | |
| /// | |
| WARNING: Output truncated! | |
| trying to load from pickle file | |
| add multi-page tiff file to processing:six-3D-100msexp-equateur_MMStack.ome.tif | |
| add multi-page tiff file to processing:six-3D-100msexp-equateur_MMStack_1.ome.tif | |
| add multi-page tiff file to processing:neuf_CZ_acquisition3D_pole_100ms_MMStack.ome.tif | |
| add multi-page tiff file to processing:exp_MMStack.ome.tif | |
| add multi-page tiff file to processing:exp_MMStack_1.ome.tif | |
| add multi-page tiff file to processing:five-3D-100ms-exp-poles_MMStack.ome.tif | |
| skip file:Thumbs.db | |
| add multi-page tiff file to processing:five-3D-100ms-exp-poles_MMStack_1.ome.tif | |
| add multi-page tiff file to processing:seven-3D-100ms-exp-equat_MMStack.ome.tif | |
| skip file:Particles Table.xls | |
| add multi-page tiff file to processing:exp_2_MMStack.ome.tif | |
| skip file:Particles Table, .xls | |
| skip file:exp1-TS-ChZ-1-drift-corrected-protocol.txt | |
| skip file:exp1-TS-ChZ-1-protocol.txt | |
| add folder to "processing":/pasteur/projets/Imod-grenier/Christophe/Christophe-yeast-NPC-nanobodies-dilution1pct-28janvier-2015/first-experiment/exp_1/analyses_ChZ | |
| add folder to "processing":/pasteur/projets/Imod-grenier/Christophe/Christophe-yeast-NPC-nanobodies-dilution1pct-28janvier-2015/first-experiment/exp_1 | |
| add folder to "processing":/pasteur/projets/Imod-grenier/Christophe/Christophe-yeast-NPC-nanobodies-dilution1pct-28janvier-2015/fourth-50ms-experiment-poles/exp_1 | |
| add multi-page tiff file to processing:exp_2_MMStack.ome.tif | |
| add folder to "processing":/pasteur/projets/Imod-grenier/Christophe/Christophe-yeast-NPC-nanobodies-dilution1pct-28janvier-2015/third-experiment-poles/exp_1 | |
| skip file:TS-rec-drift-corrected-protocol.txt | |
| skip file:TS-rec-drift-corrected.csv | |
| add folder to "processing":/pasteur/projets/Imod-grenier/Christophe/Christophe-yeast-NPC-nanobodies-dilution1pct-28janvier-2015/third-experiment-poles/exp_3_exp4_50ms | |
| add multi-page tiff file to processing:exp_MMStack.ome.tif | |
| add multi-page tiff file to processing:exp_MMStack_1.ome.tif | |
| skip file:TS-rec-drift-corrected-2-protocol.txt | |
| skip file:Particles Table.xls | |
| skip file:TS-rec-drift-corrected-protocol.txt | |
| skip file:TS-rec-drift-corrected-2.csv | |
| skip file:TS-rec-drift-corrected.csv | |
| add folder to "processing":/pasteur/projets/Imod-grenier/Christophe/Christophe-yeast-NPC-nanobodies-dilution1pct-28janvier-2015/second-experiment-poles/exp_1 | |
| add multi-page tiff file to processing:huit-100ms-exp-poles_contenated.tif | |
| add multi-page tiff file to processing:exp_MMStack.ome.tif | |
| add multi-page tiff file to processing:exp_MMStack_1.ome.tif | |
| add multi-page tiff file to processing:exp_MMStack.ome.tif | |
| add multi-page tiff file to processing:exp_MMStack_1.ome.tif | |
| <ObservedWatch: path=/pasteur/projets/Imod-grenier/Christophe/Christophe-yeast-NPC-nanobodies-dilution1pct-28janvier-2015/, is_recursive=True> | |
| Data Monitor is running... | |
| waiting for processing: exp_3_exp4_50ms | |
| waiting for processing: six-3D-100msexp-equateur_MMStack.ome.tif | |
| waiting for processing: six-3D-100msexp-equateur_MMStack_1.ome.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| waiting for processing: exp_2_MMStack.ome.tif | |
| waiting for processing: exp_1 | |
| waiting for processing: exp_2_MMStack.ome.tif | |
| waiting for processing: five-3D-100ms-exp-poles_MMStack_1.ome.tif | |
| waiting for processing: seven-3D-100ms-exp-equat_MMStack.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| waiting for processing: five-3D-100ms-exp-poles_MMStack.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| waiting for processing: huit-100ms-exp-poles_contenated.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| waiting for processing: analyses_ChZ | |
| waiting for processing: neuf_CZ_acquisition3D_pole_100ms_MMStack.ome.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| waiting for processing: exp_1 | |
| ... | |
| waiting for processing: exp_MMStack.ome.tif | |
| waiting for processing: five-3D-100ms-exp-poles_MMStack.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| waiting for processing: huit-100ms-exp-poles_contenated.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| waiting for processing: analyses_ChZ | |
| waiting for processing: neuf_CZ_acquisition3D_pole_100ms_MMStack.ome.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| waiting for processing: exp_1 | |
| waiting for processing: exp_1 | |
| waiting for processing: exp_1 | |
| waiting for processing: exp_3_exp4_50ms | |
| ***creating worksheet for xperiment-poles/exp_3_exp4_50ms | |
| waiting for processing: six-3D-100msexp-equateur_MMStack.ome.tif | |
| ***creating worksheet for x-3D-100msexp-equateur/six-3D-100msexp-equateur_MMStack.ome.tif | |
| waiting for processing: six-3D-100msexp-equateur_MMStack_1.ome.tif | |
| ***creating worksheet for x-3D-100msexp-equateur/six-3D-100msexp-equateur_MMStack_1.ome.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| ***creating worksheet for xperiment-3D-250ms/exp/exp_MMStack_1.ome.tif | |
| waiting for processing: exp_2_MMStack.ome.tif | |
| ***creating worksheet for xperiment-poles/exp_2/exp_2_MMStack.ome.tif | |
| waiting for processing: exp_1 | |
| ***creating worksheet for xperiment-poles/exp_1 | |
| waiting for processing: exp_2_MMStack.ome.tif | |
| ***creating worksheet for first-experiment/exp_2/exp_2_MMStack.ome.tif | |
| waiting for processing: five-3D-100ms-exp-poles_MMStack_1.ome.tif | |
| ***creating worksheet for five-3D-100ms-exp-poles/five-3D-100ms-exp-poles_MMStack_1.ome.tif | |
| waiting for processing: seven-3D-100ms-exp-equat_MMStack.ome.tif | |
| ***creating worksheet for 3D-100ms-exp-equat/seven-3D-100ms-exp-equat_MMStack.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| ***creating worksheet for D-equatorial-100ms/exp/exp_MMStack.ome.tif | |
| waiting for processing: five-3D-100ms-exp-poles_MMStack.ome.tif | |
| ***creating worksheet for five-3D-100ms-exp-poles/five-3D-100ms-exp-poles_MMStack.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| ***creating worksheet for xperiment-3D-250ms/exp/exp_MMStack.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| ***creating worksheet for xperiment-between-equator-and-pole-3D-100ms/exp/exp_MMStack.ome.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| ***creating worksheet for D-equatorial-100ms/exp/exp_MMStack_1.ome.tif | |
| waiting for processing: huit-100ms-exp-poles_contenated.tif | |
| ***creating worksheet for xp poles/huit-100ms-exp-poles_contenated.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| ***creating worksheet for welveth-2D-100ms-pole/exp/exp_MMStack_1.ome.tif | |
| waiting for processing: analyses_ChZ | |
| ***creating worksheet for first-experiment/exp_1/analyses_ChZ | |
| waiting for processing: neuf_CZ_acquisition3D_pole_100ms_MMStack.ome.tif | |
| ***creating worksheet for f_CZ_acquisition3D_pole_100ms/neuf_CZ_acquisition3D_pole_100ms_MMStack.ome.tif | |
| waiting for processing: exp_MMStack_1.ome.tif | |
| ***creating worksheet for xperiment-between-equator-and-pole-3D-100ms/exp/exp_MMStack_1.ome.tif | |
| waiting for processing: exp_MMStack.ome.tif | |
| ***creating worksheet for welveth-2D-100ms-pole/exp/exp_MMStack.ome.tif | |
| waiting for processing: exp_1 | |
| ***creating worksheet for xperiment-poles/exp_1 | |
| waiting for processing: exp_1 | |
| ***creating worksheet for first-experiment/exp_1 | |
| waiting for processing: exp_1 | |
| ***creating worksheet for fourth-50ms-experiment-poles/exp_1 | |
| }}} | |
| {{{id=2| | |
| /// | |
| }}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <h3>Initialization</h3> | |
| {{{id=11| | |
| %hide | |
| __version__= 23 | |
| macro_template = ''' | |
| //-----------------Generated macro file for ImageJ/Fiji-------------------// | |
| input = "%(input)s" | |
| output_dir = "%(output_dir)s" | |
| if(!File.exists(input)) exit("input file is not exits"); | |
| print("==>loading data..."); | |
| IJ.redirectErrorMessages(); | |
| if(File.isDirectory(input)) | |
| run("Image Sequence...", "open=["+input+"] sort"); | |
| else | |
| open(input); | |
| run("Camera setup", "isemgain=true pixelsize=106.0 gainem=300.0 offset=414.0 photons2adu=3.6"); | |
| print("==>running analysis..."); | |
| IJ.redirectErrorMessages(); | |
| run("Run analysis", "filter=[Wavelet filter (B-Spline)] scale=%(scale).1f order=%(order)d detector=[Local maximum] connectivity=8-neighbourhood threshold=%(threshold)s estimator=[PSF: Integrated Gaussian] sigma=%(sigma).2f method=[Least squares] full_image_fitting=%(full_image_fitting)s fitradius=%(fitradius)d mfaenabled=false renderer=[Averaged shifted histograms] magnification=%(magnification).2f colorizez=%(colorizez)s shifts=%(shifts)s repaint=50 threed=%(threed)s zshifts=%(zshifts)d zrange=%(zrange_min)d:%(zrange_step)d:%(zrange_max)d"); | |
| print("==>exporting results..."); | |
| IJ.redirectErrorMessages(); | |
| run("Export results", "filepath=["+output_dir+"Particle_table.csv] fileformat=[CSV (comma separated)] id=true frame=true sigma=true chi2=true bkgstd=true intensity=true saveprotocol=true offset=true uncertainty=true y=true x=true"); | |
| //print("==>drift correction..."); | |
| //run("Show results table", "action=drift distancethr=40.0 save=false smoothingbandwidth=0.25 ontimeratio=0.1 method=[Fiducial markers]"); | |
| //run("Show results table", "action=drift magnification=5.0 save=false showcorrelations=false method=[Cross correlation] steps=5"); | |
| print("==>saving tiff image..."); | |
| IJ.redirectErrorMessages(); | |
| saveAs("Tiff", output_dir+"Averaged shifted histograms.tif"); | |
| print("==>saving png image..."); | |
| IJ.redirectErrorMessages(); | |
| saveAs("PNG", output_dir+"Averaged shifted histograms.png"); | |
| print("==>script done!"); | |
| //-----------------Generated macro file for ImageJ/Fiji-------------------// | |
| ''' | |
| import time | |
| jobfilepath = os.path.join(DATA,'job.ijm') | |
| injectorpath = os.path.join(DATA,'injector.py') | |
| workdir = '/pasteur/projets/Imod-grenier/Will/bin/ImageJ.app' | |
| PORT = 8888 | |
| import socket | |
| s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) | |
| s.connect(("192.168.0.1",80)) | |
| nodeIP = s.getsockname()[0] | |
| print 'node ip:'+ nodeIP | |
| def listenOnPort(ListenPORT): | |
| # start a socket to listen for image data | |
| serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) | |
| serversock.bind(('', ListenPORT)) | |
| serversock.listen(1) | |
| print('Listening on '+ str(ListenPORT)) | |
| try: | |
| while 1: | |
| clientsock, addr = serversock.accept() | |
| buf = [] | |
| # get all data until the socket closes | |
| try: | |
| while True: | |
| data = clientsock.recv(4096) | |
| if data: | |
| buf.append(data) | |
| else: | |
| break | |
| data = ''.join(buf) | |
| print data | |
| finally: | |
| clientsock.close() | |
| finally: | |
| print('closing socket') | |
| serversock.close() | |
| injector_script = r''' | |
| from ij import IJ | |
| from ij.process import ByteProcessor, ShortProcessor, LUT | |
| import jarray | |
| import socket | |
| import struct | |
| import sys | |
| import traceback | |
| from cStringIO import StringIO | |
| from threading import Thread | |
| import time | |
| from ij import ImageJ | |
| ImageJ() | |
| MACRO_PATH = '' | |
| if len(sys.argv) > 1: | |
| MACRO_PATH = sys.argv[1] | |
| if len(sys.argv) > 2: | |
| IP,PORT = sys.argv[2].split(':') | |
| PORT = int(PORT) | |
| else: | |
| IP,PORT = "127.0.0.1", 8888 | |
| print IP,PORT | |
| if len(sys.argv)>3: | |
| ListenPORT = sys.argv[3] | |
| else: | |
| ListenPORT = None#PORT+1 | |
| EXIT_FLAG = False | |
| class ImagejStatus: | |
| def __init__(self): | |
| from ij import IJ | |
| from ij import ImageJ | |
| ijf = IJ.getDeclaredField('ij') | |
| ijf.setAccessible(True) | |
| imagej = ijf.get(IJ) | |
| from ij import IJ | |
| pb = IJ.getDeclaredField('progressBar') | |
| pb.setAccessible(True) | |
| progressBar = pb.get(imagej) | |
| sl = ImageJ.getDeclaredField('statusLine') | |
| sl.setAccessible(True) | |
| statusLine = sl.get(imagej) | |
| from ij.gui import ProgressBar | |
| fx = ProgressBar.getDeclaredField('fastX') | |
| fx.setAccessible(True) | |
| fastX = fx.get(progressBar) | |
| sx = ProgressBar.getDeclaredField('slowX') | |
| sx.setAccessible(True) | |
| slowX = sx.get(progressBar) | |
| self.progressBar = progressBar | |
| self.sx = sx | |
| self.fx = fx | |
| self.statusLine = statusLine | |
| self.imagej = imagej | |
| self.__status ='' | |
| self.__progress = 0.0 | |
| self.__log = '' | |
| def slowX(self): | |
| return self.sx.get(self.progressBar) | |
| def fastX(self): | |
| return self.fx.get(self.progressBar) | |
| def progress(self): | |
| return self.sx.get(self.progressBar) | |
| def status(self): | |
| return self.statusLine.getText() | |
| def log(self): | |
| return IJ.getLog() | |
| def send(self, data): | |
| global IP,PORT | |
| send_over_socket((IP,PORT), data) | |
| def printStatus(self): | |
| try: | |
| msg = '' | |
| progress_ = self.progress() | |
| if progress_ != self.__progress and progress_ is not None: | |
| self.__progress = progress_ | |
| msg += ('%.1f%%\t'%(self.__progress*100)) | |
| status_ = self.status() | |
| if status_ != self.__status and status_ is not None: | |
| self.__status = status_ | |
| msg += (self.__status+'\t') | |
| log_ = self.log() | |
| if log_ is not None: | |
| if self.logOffset < len(log_): | |
| tmp = log_[self.logOffset:] | |
| self.logOffset = len(log_) | |
| tmp= tmp.replace('Camera base level is set higher than values in the image!\n','').strip() | |
| if tmp !='': | |
| msg += ('\nlog:\n%s\n'%tmp) | |
| if msg !='': | |
| print msg | |
| #self.send(msg) | |
| except: | |
| pass | |
| def monitor(self): | |
| global EXIT_FLAG | |
| self.logOffset = 0 | |
| while not EXIT_FLAG: | |
| self.printStatus() | |
| time.sleep(0.5) | |
| self.printStatus() | |
| def statMonitoring(self): | |
| return Thread(target=self.monitor).start() | |
| def send_over_socket(address, data): | |
| if isinstance(address, (tuple, list)): | |
| sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| else: | |
| sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | |
| try: | |
| sock.connect(address) | |
| except socket.error: | |
| import time | |
| time.sleep(0.1) | |
| try: | |
| sock.connect(address) | |
| except socket.error: | |
| raise socket.error('Could not open "%s"' % (address)) | |
| try: | |
| sock.sendall(data) | |
| finally: | |
| sock.close() | |
| def executor(ListenPORT): | |
| # start a socket to listen for image data | |
| serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) | |
| serversock.bind(('', ListenPORT)) | |
| serversock.listen(1) | |
| print('Listening on '+ str(ListenPORT)) | |
| #IJ.log('Listening on '+ str(ListenPORT)) | |
| try: | |
| while 1: | |
| clientsock, addr = serversock.accept() | |
| tmp = clientsock.recv(32).split(':\n') | |
| cmd = tmp[0] | |
| tmp2 = tmp[1] | |
| buf = [] | |
| buf.append(tmp2) | |
| # get all data until the socket closes | |
| try: | |
| while True: | |
| data = clientsock.recv(4096) | |
| if data: | |
| buf.append(data) | |
| else: | |
| break | |
| data = ''.join(buf) | |
| finally: | |
| clientsock.close() | |
| print cmd,data | |
| if cmd =='cmd': | |
| output = Capturing() | |
| output.__enter__() | |
| try: | |
| exec data | |
| except: | |
| traceback.print_exc(file=sys.stdout) | |
| output.__exit__() | |
| send_over_socket((addr[0],PORT+1),'response:\n'+ ''.join(output)) | |
| finally: | |
| print('closing socket') | |
| serversock.close() | |
| class Capturing(list): | |
| def __enter__(self): | |
| self._stdout = sys.stdout | |
| sys.stdout = self._stringio = StringIO() | |
| return self | |
| def __exit__(self, *args): | |
| self.extend(self._stringio.getvalue().splitlines()) | |
| sys.stdout = self._stdout | |
| def runMacroFile(): | |
| global EXIT_FLAG | |
| try: | |
| EXIT_FLAG = False | |
| ret = IJ.runMacroFile(MACRO_PATH) | |
| print('\n***\nMacro exit with status:%s\n***\n'%ret) | |
| EXIT_FLAG = True | |
| except: | |
| print('Error occured in running macro file!') | |
| EXIT_FLAG = True | |
| try: | |
| if MACRO_PATH: | |
| Thread(target=runMacroFile).start() | |
| print('running macro file') | |
| except: | |
| print('macro running error') | |
| try: | |
| if ListenPORT: | |
| Thread(target=lambda: executor(ListenPORT)).start() | |
| except: | |
| print 'executor is not running' | |
| print('executor running error') | |
| send_over_socket((IP,PORT),'status:\nRunning') | |
| ijStatus = ImagejStatus() | |
| ijStatus.monitor() | |
| #exit imagej | |
| from java.lang import System | |
| System.exit(0) | |
| ''' | |
| jobshell_template = ''' | |
| export DISPLAY=":1" | |
| rm -rf /tmp/.X11-unix | |
| rm -f /tmp/.X11-lock | |
| Xvfb $DISPLAY -auth /dev/null/ & | |
| XVFB_PID=$! | |
| ( | |
| cd {workdir} && | |
| echo "Running in directory: {workdir}" | |
| ./ImageJ-linux64 {injectorpath} {jobfilepath} {nodeIP}:{port} && | |
| wait && | |
| echo Done! | |
| ) | |
| kill -9 $XVFB_PID | |
| rm -rf /tmp/.X11-unix | |
| rm -f /tmp/.X11-lock | |
| ''' | |
| def generate_macro(args,template = macro_template, jobfile = jobfilepath): | |
| import os | |
| macro = template%args | |
| #make a macro file | |
| with open(jobfile,'w') as f: | |
| f.write(macro) | |
| #f.write('\nwait(1)\neval("script", "System.exit(0);");') | |
| return macro | |
| def generate_job_sh(): | |
| with open(injectorpath ,'w') as f: | |
| f.write(injector_script) | |
| with open(DATA+'job.sh','w') as f: | |
| f.write(jobshell_template.format(workdir=workdir, jobfilepath=jobfilepath, injectorpath = injectorpath, nodeIP = nodeIP, port=PORT )) | |
| def run_on_cluster(jobfilepath=jobfilepath, memorysize = '32G'): | |
| html('InputData:\n'+INPUT) | |
| #check if the input file or directory exist | |
| if not os.path.exists(INPUT): | |
| html('\nERROR: Input file is not exist!') | |
| return False | |
| generate_job_sh() | |
| #run it on cluster | |
| ret = sh.eval('time qrsh -q imod -wd {workdir} -l mem_total={memory_size} sh {jobshellpath}'.format(workdir=DATA, jobshellpath=DATA+'job.sh', memory_size= memorysize)) | |
| return True | |
| button_template = ''' | |
| <a class="ui-button ui-widget ui-corner-all ui-state-active ui-button-text" href="%s"> <span style="float:left; font-size: 15px;margin: 10px;" class="ui-icon ui-icon-arrowthickstop-1-s"></span><div style="float:left;font-size: 12px; margin: 10px; width:200px">%s</div></a> | |
| ''' | |
| image_template = ''' | |
| <a class="fancybox" rel="gallery1" href="%(link)s" title="%(title)s"> <img src="%(link)s" alt="%(title)s" style="width:300px;height:300px" /></a> | |
| ''' | |
| update_button_style = ''' | |
| <style> | |
| td input[type="button"] { | |
| width: 100%; | |
| font-size: larger; | |
| color: blue; | |
| } | |
| </style> | |
| ''' | |
| /// | |
| node ip:192.168.0.50 | |
| }}} | |
| <h3>Configurations</h3> | |
| {{{id=14| | |
| %hide | |
| @interact | |
| def _( | |
| input=str(INPUT), | |
| scale=selector(srange(10.0), default = 2.0, label='B-Spline scale'), | |
| order =selector(range(10), default = 3, label='B-Spline order'), | |
| threshold = input_box("1.0*std(Wave.F1)", label='wavelet threshold',type=str), | |
| sigma = input_box(default =1.6,label= 'Sigma', type=float), | |
| full_image_fitting = selector(['false','true'], default='false', label='Full Image Fitting'), | |
| fitradius = input_box(default =3,label= 'fit radius[px]', type=int), | |
| magnification = 4.0, | |
| colorizez = selector(['false','true'], default='false', label='Colorize'), | |
| shifts = 2, | |
| threed = selector(['false','true'], default='false', label='3D'), | |
| zshifts = 2, | |
| zrange_min =-500, | |
| zrange_step = 10, | |
| zrange_max = 500, | |
| output_dir = DATA, | |
| Show_macro_file = True, | |
| Run_on_cluster = False, | |
| memeory_size = '32G', | |
| #auto_update=False | |
| ): | |
| #save arguments | |
| import pickle | |
| d = pickle.dumps(locals()) | |
| with open(DATA+'parameters.pickle','wb') as f: | |
| f.write(d) | |
| macro = generate_macro(locals()) | |
| #html(update_button_style) | |
| if Run_on_cluster: | |
| ret = run_on_cluster(jobfilepath, memeory_size) | |
| if ret: | |
| html(button_template%('data/Particle_table.csv?'+str(time.time()),'Particle Table')) | |
| html(button_template%('data/Averaged shifted histograms.tif?'+str(time.time()),' Tiff File ')) | |
| html(image_template%({'link':'data/Averaged shifted histograms.png?'+str(time.time()),'title':'Averaged shifted histograms.png'})) | |
| SET_ICON_FILE('Averaged shifted histograms.png') | |
| if Show_macro_file: | |
| html(button_template%('data/job.ijm?'+str(time.time()),'Download Macro File')) | |
| html(macro) | |
| /// | |
| <html><!--notruncate--> | |
| <div padding=6 id="div-interact-14"> | |
| <table width=800px height=20px bgcolor="#c5c5c5" cellpadding=15> | |
| <tr> | |
| <td bgcolor="#f9f9f9" valign=top align=left> | |
| <table> | |
| <tr><td colspan=3><table><tr><td align=right><font color="black">input </font></td><td><input type="text" value="/pasteur/projets/Imod-grenier/Will/data/dropbox/TubulinsI-Points/" size=80 onchange="interact(14, {variable: 'input', adapt_number: 77, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">B-Spline scale </font></td><td><select onchange="interact(14, {variable: 'scale', adapt_number: 78, value: encode64(this.options[this.selectedIndex].value)}, 1);"><option value="0" >0.0</option> | |
| <option value="1" >1.0</option> | |
| <option value="2" selected>2.0</option> | |
| <option value="3" >3.0</option> | |
| <option value="4" >4.0</option> | |
| <option value="5" >5.0</option> | |
| <option value="6" >6.0</option> | |
| <option value="7" >7.0</option> | |
| <option value="8" >8.0</option> | |
| <option value="9" >9.0</option> | |
| </select></td> | |
| </tr><tr><td align=right><font color="black">B-Spline order </font></td><td><select onchange="interact(14, {variable: 'order', adapt_number: 79, value: encode64(this.options[this.selectedIndex].value)}, 1);"><option value="0" >0</option> | |
| <option value="1" >1</option> | |
| <option value="2" >2</option> | |
| <option value="3" selected>3</option> | |
| <option value="4" >4</option> | |
| <option value="5" >5</option> | |
| <option value="6" >6</option> | |
| <option value="7" >7</option> | |
| <option value="8" >8</option> | |
| <option value="9" >9</option> | |
| </select></td> | |
| </tr><tr><td align=right><font color="black">wavelet threshold </font></td><td><input type="text" value="1.0*std(Wave.F1)" size=80 onchange="interact(14, {variable: 'threshold', adapt_number: 80, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">Sigma </font></td><td><input type="text" value="1.6" size=80 onchange="interact(14, {variable: 'sigma', adapt_number: 81, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">Full Image Fitting </font></td><td><select onchange="interact(14, {variable: 'full_image_fitting', adapt_number: 82, value: encode64(this.options[this.selectedIndex].value)}, 1);"><option value="0" selected>false</option> | |
| <option value="1" >true</option> | |
| </select></td> | |
| </tr><tr><td align=right><font color="black">fit radius[px] </font></td><td><input type="text" value="3" size=80 onchange="interact(14, {variable: 'fitradius', adapt_number: 83, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">magnification </font></td><td><input type="text" value="4.0" size=80 onchange="interact(14, {variable: 'magnification', adapt_number: 84, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">Colorize </font></td><td><select onchange="interact(14, {variable: 'colorizez', adapt_number: 85, value: encode64(this.options[this.selectedIndex].value)}, 1);"><option value="0" selected>false</option> | |
| <option value="1" >true</option> | |
| </select></td> | |
| </tr><tr><td align=right><font color="black">shifts </font></td><td><input type="text" value="2" size=80 onchange="interact(14, {variable: 'shifts', adapt_number: 86, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">3D </font></td><td><select onchange="interact(14, {variable: 'threed', adapt_number: 87, value: encode64(this.options[this.selectedIndex].value)}, 1);"><option value="0" selected>false</option> | |
| <option value="1" >true</option> | |
| </select></td> | |
| </tr><tr><td align=right><font color="black">zshifts </font></td><td><input type="text" value="2" size=80 onchange="interact(14, {variable: 'zshifts', adapt_number: 88, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">zrange_min </font></td><td><input type="text" value="-500" size=80 onchange="interact(14, {variable: 'zrange_min', adapt_number: 89, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">zrange_step </font></td><td><input type="text" value="10" size=80 onchange="interact(14, {variable: 'zrange_step', adapt_number: 90, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">zrange_max </font></td><td><input type="text" value="500" size=80 onchange="interact(14, {variable: 'zrange_max', adapt_number: 91, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">output_dir </font></td><td><input type="text" value="/pasteur/homes/wouyang/.sage/sage_notebook.sagenb/home/oeway/294/data/" size=80 onchange="interact(14, {variable: 'output_dir', adapt_number: 92, value: encode64(this.value)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">Show_macro_file </font></td><td><input type="checkbox" checked width=200px onchange="interact(14, {variable: 'Show_macro_file', adapt_number: 93, value: encode64(this.checked)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">Run_on_cluster </font></td><td><input type="checkbox" width=200px onchange="interact(14, {variable: 'Run_on_cluster', adapt_number: 94, value: encode64(this.checked)}, 1)"></input></td> | |
| </tr><tr><td align=right><font color="black">memeory_size </font></td><td><input type="text" value="32G" size=80 onchange="interact(14, {variable: 'memeory_size', adapt_number: 95, value: encode64(this.value)}, 1)"></input></td> | |
| </tr></table></td></tr> | |
| <tr><td></td><td style='width: 100%;'> | |
| <div id="cell-interact-14"><?__SAGE__START> | |
| <table border=0 bgcolor="white" width=100%> | |
| <tr> | |
| <td bgcolor="white" align=left valign=top> | |
| <pre><?__SAGE__TEXT></pre> | |
| </td> | |
| </tr> | |
| <tr> | |
| <td align=left valign=top><?__SAGE__HTML></td> | |
| </tr> | |
| </table><?__SAGE__END> | |
| </div></td><td></td></tr> | |
| <tr><td colspan=3></td></tr> | |
| </table></td> | |
| </tr> | |
| </table> | |
| </div></html> | |
| }}} | |
| <h3>Clean data folder</h3> | |
| {{{id=26| | |
| %hide | |
| import pickle | |
| #read parameters | |
| with open(DATA+'parameters.pickle', 'rb') as f: | |
| args = pickle.load(f) | |
| #clean data folder | |
| sh.eval('rm -rf '+DATA+'*') | |
| #save back | |
| d = pickle.dumps(args) | |
| with open(DATA+'parameters.pickle','wb') as f: | |
| f.write(d) | |
| /// | |
| /tmp/tmpbUNhtB | |
| }}} | |
| <h3>Run on Cluster</h3> | |
| {{{id=17| | |
| %hide | |
| with open(DATA+'parameters.pickle', 'rb') as f: | |
| import pickle | |
| args = pickle.load(f) | |
| #update variables | |
| args['input'] = INPUT | |
| args['output_dir'] = DATA | |
| if not args.has_key('memeory_size'): | |
| args['memeory_size'] = '32G' | |
| macro = generate_macro(args) | |
| #from threading import Thread | |
| #lt = Thread(target=lambda: listenOnPort(PORT) ).start() | |
| ret = run_on_cluster(jobfilepath, args['memeory_size'] ) | |
| if ret: | |
| html(button_template%('data/Particle_table.csv?'+str(time.time()),'Particle Table')) | |
| html(button_template%('data/Averaged shifted histograms.tif?'+str(time.time()),' Tiff File ')) | |
| html(image_template%({'link':'data/Averaged shifted histograms.png?'+str(time.time()),'title':'Averaged shifted histograms.png'})) | |
| html(button_template%('data/job.ijm?'+str(time.time()),'Download Macro File')) | |
| html(macro) | |
| SET_ICON_FILE('Averaged shifted histograms.png') | |
| /// | |
| WARNING: Output truncated! | |
| <html><a target='_new' href='/home/oeway/294/cells/17/full_output.txt' class='file_link'>full_output.txt</a></html> | |
| <html><font color='black'>InputData: | |
| /pasteur/projets/Imod-grenier/Will/data/dropbox/TubulinsI-Points/</font></html> | |
| /tmp/tmpQlL7Ie | |
| _XSERVTransmkdir: ERROR: euid != 0,directory /tmp/.X11-unix will not be created. | |
| Initializing built-in extension Generic Event Extension | |
| Initializing built-in extension SHAPE | |
| Initializing built-in extension MIT-SHM | |
| Initializing built-in extension XInputExtension | |
| Initializing built-in extension XTEST | |
| Initializing built-in extension BIG-REQUESTS | |
| Initializing built-in extension SYNC | |
| Initializing built-in extension XKEYBOARD | |
| Initializing built-in extension XC-MISC | |
| Initializing built-in extension SECURITY | |
| Initializing built-in extension XINERAMA | |
| Initializing built-in extension XFIXES | |
| Initializing built-in extension RENDER | |
| Initializing built-in extension RANDR | |
| Initializing built-in extension COMPOSITE | |
| Initializing built-in extension DAMAGE | |
| Initializing built-in extension MIT-SCREEN-SAVER | |
| Initializing built-in extension DOUBLE-BUFFER | |
| Initializing built-in extension RECORD | |
| Initializing built-in extension DPMS | |
| Initializing built-in extension X-Resource | |
| Initializing built-in extension XVideo | |
| Initializing built-in extension XVideo-MotionCompensation | |
| Initializing built-in extension SELinux | |
| Initializing built-in extension GLX | |
| Running in directory: /pasteur/projets/Imod-grenier/Will/bin/ImageJ.app | |
| 192.168.0.50 8888 | |
| running macro file | |
| ImageJ 1.49m; Java 1.7.0_65 [64-bit]; 423 commands; 0 macros | |
| log: | |
| ==>loading data... | |
| 0.2% 5/2401 | |
| 3.4% 80/2401 | |
| 9.1% 217/2401 | |
| 12.5% 300/2401 | |
| 26.2% 627/2401 | |
| 38.8% 931/2401 | |
| 47.7% 1144/2401 | |
| 61.0% 1465/2401 | |
| 75.3% 1807/2401 | |
| 78.9% 1894/2401 | |
| 93.0% 2234/2401 | |
| 0.0% | |
| log: | |
| ==>running analysis... | |
| Run analysis (Stack)... | |
| 0.0% ThunderSTORM processing frame 1 of 2401... | |
| 0.1% ThunderSTORM processing frame 2 of 2401... | |
| 0.5% ThunderSTORM processing frame 13 of 2401... | |
| 0.6% ThunderSTORM processing frame 15 of 2401... | |
| 1.1% ThunderSTORM processing frame 23 of 2401... | |
| 1.5% ThunderSTORM processing frame 37 of 2401... | |
| 1.9% ThunderSTORM processing frame 46 of 2401... | |
| ... | |
| user 0m0.028s | |
| sys 0m0.036s | |
| <html><font color='black'> | |
| <a class="ui-button ui-widget ui-corner-all ui-state-active ui-button-text" href="data/Particle_table.csv?1425932916.88"> <span style="float:left; font-size: 15px;margin: 10px;" class="ui-icon ui-icon-arrowthickstop-1-s"></span><div style="float:left;font-size: 12px; margin: 10px; width:200px">Particle Table</div></a> | |
| </font></html> | |
| <html><font color='black'> | |
| <a class="ui-button ui-widget ui-corner-all ui-state-active ui-button-text" href="data/Averaged shifted histograms.tif?1425932916.88"> <span style="float:left; font-size: 15px;margin: 10px;" class="ui-icon ui-icon-arrowthickstop-1-s"></span><div style="float:left;font-size: 12px; margin: 10px; width:200px"> Tiff File </div></a> | |
| </font></html> | |
| <html><font color='black'> | |
| <a class="fancybox" rel="gallery1" href="data/Averaged shifted histograms.png?1425932916.88" title="Averaged shifted histograms.png"> <img src="data/Averaged shifted histograms.png?1425932916.88" alt="Averaged shifted histograms.png" style="width:300px;height:300px" /></a> | |
| </font></html> | |
| <html><font color='black'> | |
| <a class="ui-button ui-widget ui-corner-all ui-state-active ui-button-text" href="data/job.ijm?1425932916.88"> <span style="float:left; font-size: 15px;margin: 10px;" class="ui-icon ui-icon-arrowthickstop-1-s"></span><div style="float:left;font-size: 12px; margin: 10px; width:200px">Download Macro File</div></a> | |
| </font></html> | |
| <html><font color='black'> | |
| //-----------------Generated macro file for ImageJ/Fiji-------------------// | |
| input = "/pasteur/projets/Imod-grenier/Will/data/dropbox/TubulinsI-Points/" | |
| output_dir = "/pasteur/homes/wouyang/.sage/sage_notebook.sagenb/home/oeway/294/data/" | |
| if(!File.exists(input)) exit("input file is not exits"); | |
| print("==>loading data..."); | |
| IJ.redirectErrorMessages(); | |
| if(File.isDirectory(input)) | |
| run("Image Sequence...", "open=["+input+"] sort"); | |
| else | |
| open(input); | |
| run("Camera setup", "isemgain=true pixelsize=106.0 gainem=300.0 offset=414.0 photons2adu=3.6"); | |
| print("==>running analysis..."); | |
| IJ.redirectErrorMessages(); | |
| run("Run analysis", "filter=[Wavelet filter (B-Spline)] scale=2.0 order=3 detector=[Local maximum] connectivity=8-neighbourhood threshold=1.0*std(Wave.F1) estimator=[PSF: Integrated Gaussian] sigma=1.60 method=[Least squares] full_image_fitting=false fitradius=3 mfaenabled=false renderer=[Averaged shifted histograms] magnification=4.00 colorizez=false shifts=2 repaint=50 threed=false zshifts=2 zrange=-500:10:500"); | |
| print("==>exporting results..."); | |
| IJ.redirectErrorMessages(); | |
| run("Export results", "filepath=["+output_dir+"Particle_table.csv] fileformat=[CSV (comma separated)] id=true frame=true sigma=true chi2=true bkgstd=true intensity=true saveprotocol=true offset=true uncertainty=true y=true x=true"); | |
| //print("==>drift correction..."); | |
| //run("Show results table", "action=drift distancethr=40.0 save=false smoothingbandwidth=0.25 ontimeratio=0.1 method=[Fiducial markers]"); | |
| //run("Show results table", "action=drift magnification=5.0 save=false showcorrelations=false method=[Cross correlation] steps=5"); | |
| print("==>saving tiff image..."); | |
| IJ.redirectErrorMessages(); | |
| saveAs("Tiff", output_dir+"Averaged shifted histograms.tif"); | |
| print("==>saving png image..."); | |
| IJ.redirectErrorMessages(); | |
| saveAs("PNG", output_dir+"Averaged shifted histograms.png"); | |
| print("==>script done!"); | |
| //-----------------Generated macro file for ImageJ/Fiji-------------------// | |
| </font></html> | |
| }}} | |
| {{{id=30| | |
| /// | |
| }}} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment