config_clean_medium.jsonnet

local experiment_name = "cnn_edss19";

{	"train_options": { 
        "serialization_dir": "/results/dev/" + experiment_name,
        "file_friendly_logging": false,
        "recover": false,
        "force": true,
        "node_rank": 0,
        "batch_weight_key": "",
        "dry_run": false,
    },
	"params":{
	"dataset_reader": {
            "type": "data_scripts.datasetreaders.ms_edss19_reader",
            "token_indexers": { 
                "tokens": {
                    "type": "pretrained_transformer",
                    "model_name": "/models/base_blue_bert_pt",
                    "namespace": "tokens",
                    "max_length": 512,
                }
            },
        },
        "train_data_path": "data/train_data.csv",
        "validation_data_path": "data/val_data.csv",
        "test_data_path": "data/test_data.csv",
        "unlabeled_data_path": "data/unlabeled_data.csv",
        "model": {
            "type": "models.ms_classifiers.ms_classifier",
            "text_field_embedder": {
                "token_embedders": {
                    "tokens": {
                        "type": "pretrained_transformer",
                        "model_name": "/models/base_blue_bert_pt",
                        "max_length": 512,
                    }
                }
            },
            "seq2vec_encoder": {
                "type": "cnn",
				"embedding_dim": 768,
				"num_filters": 128,
				"ngram_filter_sizes": [2, 3, 4, 5, 6, 10],
            },
            "feedforward": {
                "input_dim": 768,
                "num_layers": 2,
                "hidden_dims": [500, 250],
                "activations": ["relu","relu"]
            },
            "dropout": 0.1,
            "label_namespace": "edss19_labels"
        },
        "data_loader": {
            "batch_sampler": {
                "type": "bucket",
                "batch_size": 5,
                "padding_noise": 0,
            },
        },
        "validation_data_loader": {
            "batch_sampler": {
                "type": "bucket",
                "batch_size": 5,
                "padding_noise": 0,
            },
        },
        "evaluate_on_test": true,
        "trainer": {
            "optimizer": {
                "type": "huggingface_adamw",
                "lr": 5e-4,
                "weight_decay": 0.01,
                "correct_bias": true
            },
            "learning_rate_scheduler": {
                "type": "reduce_on_plateau",
                "min_lr" : 5e-5
            },
            "patience": 5,
            "validation_metric": "+f1",
            "num_epochs": 50,
            "checkpointer": {
                 "num_serialized_models_to_keep": 1,
                 "keep_serialized_model_every_num_seconds": null,
            },
            "model_save_interval": null,
            "grad_norm": 1.0,
            "no_grad": ["embedder"],
            "grad_clipping": 1.0,
            "summary_interval": 1,
            "histogram_interval": 10,
            "should_log_parameter_statistics": true,
            "should_log_learning_rate": true,
            "log_batch_size_period": 100,
            "moving_average": null,
            "distributed": false,
            "local_rank": 0,
            "cuda_device": 3, 
            "world_size": 1,
            "num_gradient_accumulation_steps": 4,
        }
    }
}