pickle error example

pickle_error.py

from typing import *
import os
import zarr
import shutil
import yaml
import numpy as np
from pathlib import Path
from collections import OrderedDict

import prefect
from prefect import Task
from prefect import task
from prefect.engine import signals

###### working ########
@task(name='create_folder_structure')
def create_folder_structure(experiment_fpath:str):
    """
    Function used to create the folder structure where to sort the files
    generated by the machines and the saving the data created during the
    processing. It creates the backbone structure common to all analysis

    original_robofish_logs: contains all the original robofish logs.
	extra_files: contains the extra files acquired during imaging.
	extra_processing_data: contains extra files used in the analysis 
												like the dark images for flat field correction.
    pipeline_config: contains all the configuration files.
    raw_data: contains the renamed .nd2 files and the corresponding 
						pickle configuration files. It is the directory that is 
						backed up on the server.
	output_figures: contains the reports and visualizations
    notebooks: will contain potential notebooks used for processing the data
    probes: will contains the fasta file with the probes used in the experiment
    tmp: save temporary data
    

    Args:
        experiment_fpath: str
            folder path of the experiment
    """
    logger = prefect_logging_setup('created_raw_tmp_dir')
    experiment_fpath = Path(experiment_fpath)
    folders_list = ['raw_data',
                    'original_robofish_logs',
                    'extra_processing_data',
                    'extra_files',
                    'pipeline_config',
                    'output_figures',
                    'notebooks',
                    'probes',
                    'tmp']
    for folder_name in folders_list:
        try:
            os.stat(experiment_fpath / folder_name )
            logger.info(f'{folder_name} already exist')
        except FileNotFoundError:
            os.mkdir(experiment_fpath / folder_name)
            os.chmod(experiment_fpath / folder_name,0o777)

            
# Not working #####
class create_folder_structure(Task):
  """
  Class used to create the folder structure where to sort the files
  generated by the machines and the saving the data created during the
  processing. It creates the backbone structure common to all analysis

  FOLDER STRUCTURE
  - original_robofish_logs: contains all the original robofish logs.
- extra_files: contains the extra files acquired during imaging.
- extra_processing_data: contains extra files used in the analysis 
                      like the dark images for flat field correction.
  - pipeline_config: contains all the configuration files.
  - raw_data: contains the renamed .nd2 files and the corresponding 
          pickle configuration files. It is the directory that is 
          backed up on the server.
- output_figures: contains the reports and visualizations
  - notebooks: will contain potential notebooks used for processing the data
  - probes: will contains the fasta file with the probes used in the experiment
  - tmp: save temporary data

  Args:
      experiment_fpath: str
          folder path of the experiment
  """

  def run(self, experiment_fpath:str):
      """
      Class used to create the folder structure where to sort the files
      generated by the machines and the saving the data created during the
      processing. It creates the backbone structure common to all analysis

      FOLDER STRUCTURE
      - original_robofish_logs: contains all the original robofish logs.
      - extra_files: contains the extra files acquired during imaging.
      - extra_processing_data: contains extra files used in the analysis 
                                                  like the dark images for flat field correction.
      - pipeline_config: contains all the configuration files.
      - raw_data: contains the renamed .nd2 files and the corresponding 
                          pickle configuration files. It is the directory that is 
                          backed up on the server.
      - output_figures: contains the reports and visualizations
      - notebooks: will contain potential notebooks used for processing the data
      - probes: will contains the fasta file with the probes used in the experiment
      - tmp: save temporary data

      Args:
          experiment_fpath: str
              folder path of the experiment
      """
      experiment_fpath = Path(experiment_fpath)
      folders_list = ['raw_data',
                      'original_robofish_logs',
                      'extra_processing_data',
                      'extra_files',
                      'pipeline_config',
                      'output_figures',
                      'notebooks',
                      'probes',
                      'tmp']
      for folder_name in folders_list:
          try:
              os.stat(experiment_fpath / folder_name )
              self.logger.info(f'{folder_name} already exist')
          except FileNotFoundError:
              os.mkdir(experiment_fpath / folder_name)
              os.chmod(experiment_fpath / folder_name,0o777)