ZOLA batch processing

zola_batch.ijm

// Andrey Aristov 2018
// aaristov@pasteur.fr
// Institut Pasteur
//
// Select folder containing the data
// Select calibration file for ZOLA PSF
// Specify parameters of localization in the dialog
// Specify prefix for the folders containing super resolution data 
// The script will recursively search for folders with prefix and try to find ome.tiff stacks
// Once files are found it will create ZOLA.lock file to prevent concurrent scripts from processing the same data
// Ome.tif files will be opened in the memory and fused together if dataset is more than 4 GB.
// If dataset contains bright field data to be extracted, select Make substack in the dialog.
// Substack will be extracted from the raw data and saved in the same directofu as BF_Substack...tif
// The rest of the data will be reconstructed by ZOLA using calibration and parameters specified. 
// Localizations will be saved in the data folder into ZOLA_localization_table.csv
// ZOLA.lock will be deleted.
// Processing the next folder.
// One can launch several FiJi instances to reconstruct the datasets. 
// Thanks to the lock file there should not be redundant processing.
// If ZOLA.lock or ZOLA...csv file is found in the folder, the folder is skipped.
// If virtual checkbox selected, bioformats virtual stack will be used. It saves memory, but not compatible with substack.
// At the end of the process, data folder will be scanned again to check for new data. Allows to start reconstruction during acquisition

//=== Change log ===
// 2018-07-05 Using bio-formats import to save memory (not compatible with substack)
// 2018-07-03 checking for new data

//================

// Default values

  zola_csv = "ZOLA_localization_table.csv";
  lock_file = "ZOLA_.lock";
  mounting_medium_refractive=1.330;
  distance_focus_to_coverslip = 0; //um
  patch_size = 24; //px
  expected_axial_range = 4; //um
  min_number_of_photons = 500;
  gpu = true;
  suffix = "ome.tif"; //file extension
  prefix = "sr_"; //folder name filter, can be in any part of the folder name
  make_sub = false; //Make substack and save separately 
  virtual = true;
  silent = true; //batch mode on -- no windows popup
  dataLength = 0; 

  
  print("-== Starting the script ==-");


  print("Please select data folder");
  dir = getDirectory("Choose a Directory With Data to Reconstuct");
  print("-== Folder selected ==-");
  print(dir);
  
  print("Please select calibration file");
  cal_path=File.openDialog("Select a Calibration file");
  print("-== Calibration selected ==-");
  print(cal_path);

  
  title = "ZOLA batch processing";
  
  Dialog.create(title);
  Dialog.addNumber("distance_focus_to_coverslip, um:", distance_focus_to_coverslip);
  Dialog.addNumber("patch_size, px:", patch_size);
  Dialog.addNumber("expected_axial_range:", expected_axial_range);
  Dialog.addNumber("min_number_of_photons:", min_number_of_photons);
  Dialog.addCheckbox("Run_on_GPU:", gpu);
  Dialog.addString("Suffix_for_the_stack:",suffix);
  Dialog.addString("Folder_name_contains:",prefix);
  Dialog.addCheckbox("Make substack:",make_sub);
  Dialog.addNumber("Extract BF frames, start:",10);
  Dialog.addNumber("Extract BF frames, every:",10);
  Dialog.addCheckbox("Virtual stack:",virtual);
  Dialog.addCheckbox("Silent mode:",silent);

  Dialog.show();
  
  distance_focus_to_coverslip = Dialog.getNumber();
  patch_size = Dialog.getNumber();
  expected_axial_range = Dialog.getNumber();
  min_number_of_photons =Dialog.getNumber();
  gpu = Dialog.getCheckbox();
  suffix = Dialog.getString();
 
  prefix = Dialog.getString();
  make_sub = Dialog.getCheckbox();
  print("Make substack: "+make_sub);
  sub_start = Dialog.getNumber();
  sub_skip = Dialog.getNumber();
  virtual = Dialog.getCheckbox();
  silent = Dialog.getCheckbox();
  
  if(silent){
	setBatchMode(true);
  }

  if(gpu){
	print("Using GPU");
  	use_gpu = "run_on_gpu";
  }
  else{
  	use_gpu = "";
	print("Using CPU");
  }


  
  
  len = getDataList(dir);
  while(len > dataLength){
  	print('New data found');
	processSRfolders();
	dataLength = len;
	len = getDataList(dir);
  }
  print('No new data found');
  print("=== END === ");

  function getDataList(dir){
	  print("---------------------");
	  listFiles(dir);
	  list=List.getList();
	  print("Discovered "+List.size+" folders with "+prefix+" prefix");
	  print(list);
	
	  print("---------------------");
	  return List.size
  }  
  
  function doRec(dir,name){
  	new_data = true;
	putLock(dir);
	if(virtual){
		rawID = openVirtualStack(dir+name);
	}
	else{
		rawID = openStack(dir+name);
	}
  	doSubstack(rawID,dir);
  	zolaRec(dir,rawID);
  	closeRaw();
	releaseLock(dir);
	return new_data
  }


  function processSRfolders(){
  	 for(i=0; i<List.size; i++) {
  	 	folder = List.get(i);
  	 	print("-->Looking for stack in:"+folder);
  	 	processSRfolder(folder);
  	 }
  }

  function listFiles(dir) {
  	 count = 0;
  	//List.clear();
     list = getFileList(dir);
     //print("Found "+list.length+" items");
     for (i=0; i<list.length; i++) {
     	//print((i+1) + ": " + dir+ list[i]);
     
        if (indexOf(list[i], prefix)>=0 && !endsWith(list[i],"tif")){
        	
        	List.set(count++, dir+ list[i]);
        	//processSRfolder(dir + list[i])
        }
        else if(endsWith(list[i],"/"))
        	listFiles(""+dir+list[i]);
        //else
        //	return;
     }

  }

  function processSRfolder(dir){
  	list = getFileList(dir);
  	for (i=0; i<list.length; i++) {
  		
  		if (startsWith(list[i], "ZOLA")){
  			print("Found Zola csv, skipping folder");
  			return;
  		}

  	}

  	for (i=0; i<list.length; i++) {
  		if (endsWith(list[i], suffix)){
			print("->processSRfolder: found file: "+list[i]);
  			doRec(dir,list[i]);
  			return;
		}
	}
	print("No "+suffix+" found!");
  }

  function doSubstack(ID,dir){
  	if(make_sub){
  		selectImage(ID);
  	    	nSlices_=nSlices;
  		run("Make Substack...", "delete slices="+sub_start+"-"+nSlices_+"-"+sub_skip);
  		selectWindow("Substack ("+sub_start+"-"+nSlices_+"-"+sub_skip+")");
  		subID = getImageID();
  		print("doSubstack: Made Substack ("+sub_start+"-"+nSlices_+"-"+sub_skip+")");
  		saveSub(subID,dir);
  		closeSub(subID);
		return subID;
  	}
	else{
		return 0;
	}
  }
  
  function closeSub(subID){
  	print("doRec: Closing substack");
  	selectImage(subID);
  	close();
  }

  function closeRaw(){
  	print("doRec: Closing stack");
  	selectImage(rawID);
  	close();
  }

  function saveSub(subID,dir){
    if(make_sub){
	  	print("Saving BF_movie");
  		selectImage(subID);
  		save(dir+"BF_movie_Substack_start_"+sub_start+"_skip_"+sub_skip+".tif");
    }
  }

  function putLock(dir){	
	print("Setting lock");
	f = File.open(dir+lock_file);
	File.close(f);
  }

  function releaseLock(dir){
	print("Releasing lock");
	rel = File.delete(dir+lock_file);
	if(rel){
		print("With Success");
	}
	else{
		print("With a problem");
	}
  }

  function openStack(path){
  	print("openStack: Opening stack "+path);
  	run("Image Sequence...", "open=["+path+"] file="+suffix+" sort ");
	rawID = getImageID();
	return rawID;
  }

  function openVirtualStack(path){
  	print("openVirtualStack: Opening stack "+path);
  	run("Bio-Formats Importer", "open=["+path+"] color_mode=Grayscale split_channels view=Hyperstack stack_order=XYCZT use_virtual_stack");
 	rawID = getImageID();
	return rawID;
 }


  function zolaRec(path,ID){
  	selectImage(ID);
  	print("zolaRec: starting Zola reconstruction of "+nSlices+" frames");
	run("EMCCD", "emccd_camera_adu=1.0000 emccd_camera_gain=1.0000 emccd_camera_offset=0.0000");
  	run(" Localization", use_gpu+" calibration_file=["+cal_path+"] mounting_medium_refractive="+mounting_medium_refractive+" distance_focus_to_coverslip="+distance_focus_to_coverslip+" patch_size="+patch_size+" expected_axial_range="+expected_axial_range+" min_number_of_photons="+min_number_of_photons+" localization_table=["+path+zola_csv+"]");

  }