-
-
Save simone-codeluppi/0396aad8b2f7ce92d55ee5f1f2c09de2 to your computer and use it in GitHub Desktop.
pickle error example
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from typing import * | |
import os | |
import zarr | |
import shutil | |
import yaml | |
import numpy as np | |
from pathlib import Path | |
from collections import OrderedDict | |
import prefect | |
from prefect import Task | |
from prefect import task | |
from prefect.engine import signals | |
###### working ######## | |
@task(name='create_folder_structure') | |
def create_folder_structure(experiment_fpath:str): | |
""" | |
Function used to create the folder structure where to sort the files | |
generated by the machines and the saving the data created during the | |
processing. It creates the backbone structure common to all analysis | |
original_robofish_logs: contains all the original robofish logs. | |
extra_files: contains the extra files acquired during imaging. | |
extra_processing_data: contains extra files used in the analysis | |
like the dark images for flat field correction. | |
pipeline_config: contains all the configuration files. | |
raw_data: contains the renamed .nd2 files and the corresponding | |
pickle configuration files. It is the directory that is | |
backed up on the server. | |
output_figures: contains the reports and visualizations | |
notebooks: will contain potential notebooks used for processing the data | |
probes: will contains the fasta file with the probes used in the experiment | |
tmp: save temporary data | |
Args: | |
experiment_fpath: str | |
folder path of the experiment | |
""" | |
logger = prefect_logging_setup('created_raw_tmp_dir') | |
experiment_fpath = Path(experiment_fpath) | |
folders_list = ['raw_data', | |
'original_robofish_logs', | |
'extra_processing_data', | |
'extra_files', | |
'pipeline_config', | |
'output_figures', | |
'notebooks', | |
'probes', | |
'tmp'] | |
for folder_name in folders_list: | |
try: | |
os.stat(experiment_fpath / folder_name ) | |
logger.info(f'{folder_name} already exist') | |
except FileNotFoundError: | |
os.mkdir(experiment_fpath / folder_name) | |
os.chmod(experiment_fpath / folder_name,0o777) | |
# Not working ##### | |
class create_folder_structure(Task): | |
""" | |
Class used to create the folder structure where to sort the files | |
generated by the machines and the saving the data created during the | |
processing. It creates the backbone structure common to all analysis | |
FOLDER STRUCTURE | |
- original_robofish_logs: contains all the original robofish logs. | |
- extra_files: contains the extra files acquired during imaging. | |
- extra_processing_data: contains extra files used in the analysis | |
like the dark images for flat field correction. | |
- pipeline_config: contains all the configuration files. | |
- raw_data: contains the renamed .nd2 files and the corresponding | |
pickle configuration files. It is the directory that is | |
backed up on the server. | |
- output_figures: contains the reports and visualizations | |
- notebooks: will contain potential notebooks used for processing the data | |
- probes: will contains the fasta file with the probes used in the experiment | |
- tmp: save temporary data | |
Args: | |
experiment_fpath: str | |
folder path of the experiment | |
""" | |
def run(self, experiment_fpath:str): | |
""" | |
Class used to create the folder structure where to sort the files | |
generated by the machines and the saving the data created during the | |
processing. It creates the backbone structure common to all analysis | |
FOLDER STRUCTURE | |
- original_robofish_logs: contains all the original robofish logs. | |
- extra_files: contains the extra files acquired during imaging. | |
- extra_processing_data: contains extra files used in the analysis | |
like the dark images for flat field correction. | |
- pipeline_config: contains all the configuration files. | |
- raw_data: contains the renamed .nd2 files and the corresponding | |
pickle configuration files. It is the directory that is | |
backed up on the server. | |
- output_figures: contains the reports and visualizations | |
- notebooks: will contain potential notebooks used for processing the data | |
- probes: will contains the fasta file with the probes used in the experiment | |
- tmp: save temporary data | |
Args: | |
experiment_fpath: str | |
folder path of the experiment | |
""" | |
experiment_fpath = Path(experiment_fpath) | |
folders_list = ['raw_data', | |
'original_robofish_logs', | |
'extra_processing_data', | |
'extra_files', | |
'pipeline_config', | |
'output_figures', | |
'notebooks', | |
'probes', | |
'tmp'] | |
for folder_name in folders_list: | |
try: | |
os.stat(experiment_fpath / folder_name ) | |
self.logger.info(f'{folder_name} already exist') | |
except FileNotFoundError: | |
os.mkdir(experiment_fpath / folder_name) | |
os.chmod(experiment_fpath / folder_name,0o777) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment