finetune-3.py

            if isOtherThreadIndexBusy:
    
                if not threadHandle_index.is_alive():

                    saved_dataset_shards=[]
                    for address in data_shard_addressses:
                        saved_dataset_shards.append(load_from_disk(address))
              
                    concat=concatenate_datasets(saved_dataset_shards)
                    concat.save_to_disk(self.config.passages_path)
                    print("done saving the dataset to the passage_path")
                    
                    #initializing the RAY workers with newly computed embeddings and index
                    self.trainer.model.module.module.model.rag.retriever.set_new_index()
                    self.trainer.model.module.module.model.rag.retriever.init_retrieval()
                    
                    print("done loading the new index")
                    isEmUpdateBusy = False
                    isOtherThreadIndexBusy =False
                    isAddIndexBusy=False
                        
  self.trainer.accelerator_connector.accelerator.barrier("barrier")