Upload 16 files

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fold_0/progress.png filter=lfs diff=lfs merge=lfs -text

fold_0/debug.json

@@ -0,0 +1,87 @@
+{
+    "all_tr_losses": "[]",
+    "all_val_eval_metrics": "[]",
+    "all_val_losses": "[]",
+    "all_val_losses_tr_mode": "[]",
+    "also_val_in_tr_mode": "False",
+    "amp_grad_scaler": "None",
+    "base_num_features": "32",
+    "basic_generator_patch_size": "[205 205 205]",
+    "batch_dice": "False",
+    "batch_size": "4",
+    "best_MA_tr_loss_for_patience": "None",
+    "best_epoch_based_on_MA_tr_loss": "None",
+    "best_val_eval_criterion_MA": "None",
+    "classes": "[1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 21, 22, 23, 24, 3, 4, 5, 6, 7, 8, 9]",
+    "conv_per_stage": "2",
+    "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': False, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([128, 128, 128])}",
+    "dataset_directory": "/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2",
+    "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]",
+    "deterministic": "False",
+    "dl_tr": "<nnunet.training.dataloading.dataset_loading.DataLoader3D object at 0x7fe4c9ceddf0>",
+    "dl_val": "<nnunet.training.dataloading.dataset_loading.DataLoader3D object at 0x7fe4c9cede20>",
+    "do_dummy_2D_aug": "False",
+    "ds_loss_weights": "[0.53333333 0.26666667 0.13333333 0.06666667 0.        ]",
+    "epoch": "0",
+    "experiment_name": "nnUNetTrainerV2_ep16000_nomirror",
+    "fold": "0",
+    "folder_with_preprocessed_data": "/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/nnUNetData_plans_v2.1_stage0",
+    "fp16": "True",
+    "gt_niftis_folder": "/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/gt_segmentations",
+    "inference_pad_border_mode": "constant",
+    "inference_pad_kwargs": "{'constant_values': 0}",
+    "init_args": "('/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/nnUNetPlansv2.1_bs4_plans_3D.pkl', 0, '/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_trained_models/nnUNet/3d_fullres/Task002_SpineV2/nnUNetTrainerV2_ep16000_nomirror__nnUNetPlansv2.1_bs4', '/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2', False, 0, True, False, True)",
+    "initial_lr": "0.01",
+    "log_file": "/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_trained_models/nnUNet/3d_fullres/Task002_SpineV2/nnUNetTrainerV2_ep16000_nomirror__nnUNetPlansv2.1_bs4/fold_0/training_log_2023_4_4_14_36_43.txt",
+    "lr_scheduler": "None",
+    "lr_scheduler_eps": "0.001",
+    "lr_scheduler_patience": "30",
+    "lr_threshold": "1e-06",
+    "max_num_epochs": "16000",
+    "min_region_size_per_class": "None",
+    "min_size_per_class": "None",
+    "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]",
+    "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2]]",
+    "net_pool_per_axis": "[5, 5, 5]",
+    "normalization_schemes": "OrderedDict([(0, 'CT')])",
+    "num_batches_per_epoch": "250",
+    "num_classes": "25",
+    "num_input_channels": "1",
+    "num_val_batches_per_epoch": "50",
+    "online_eval_fn": "[]",
+    "online_eval_foreground_dc": "[]",
+    "online_eval_fp": "[]",
+    "online_eval_tp": "[]",
+    "only_keep_largest_connected_component": "None",
+    "optimizer": "SGD (\nParameter Group 0\n    dampening: 0\n    differentiable: False\n    foreach: None\n    lr: 0.01\n    maximize: False\n    momentum: 0.99\n    nesterov: True\n    weight_decay: 3e-05\n)",
+    "output_folder": "/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_trained_models/nnUNet/3d_fullres/Task002_SpineV2/nnUNetTrainerV2_ep16000_nomirror__nnUNetPlansv2.1_bs4/fold_0",
+    "output_folder_base": "/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_trained_models/nnUNet/3d_fullres/Task002_SpineV2/nnUNetTrainerV2_ep16000_nomirror__nnUNetPlansv2.1_bs4",
+    "oversample_foreground_percent": "0.33",
+    "pad_all_sides": "None",
+    "patch_size": "[128 128 128]",
+    "patience": "50",
+    "pin_memory": "True",
+    "plans_file": "/dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/nnUNetPlansv2.1_bs4_plans_3D.pkl",
+    "regions_class_order": "None",
+    "save_best_checkpoint": "True",
+    "save_every": "50",
+    "save_final_checkpoint": "True",
+    "save_intermediate_checkpoints": "True",
+    "save_latest_only": "True",
+    "stage": "0",
+    "threeD": "True",
+    "tr_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7fe4c9ded8b0>",
+    "train_loss_MA": "None",
+    "train_loss_MA_alpha": "0.93",
+    "train_loss_MA_eps": "0.0005",
+    "transpose_backward": "[0, 1, 2]",
+    "transpose_forward": "[0, 1, 2]",
+    "unpack_data": "True",
+    "use_mask_for_norm": "OrderedDict([(0, False)])",
+    "use_progress_bar": "False",
+    "val_eval_criterion_MA": "None",
+    "val_eval_criterion_alpha": "0.9",
+    "val_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7fe4c9dedeb0>",
+    "was_initialized": "True",
+    "weight_decay": "3e-05"
+}

fold_0/model_best.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e97b1d8e2b2ad80d5387bea88471a0dd126a78a6423d2c671752d818334c50c2
+size 250912138

fold_0/model_best.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711bd55eaf57f67ab0ad167b2fff87a07949f3dda32d3d4b99c585ecbe3acc0e
+size 732008

fold_0/model_latest.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fd91a87029735b30a2c0ab6c9a96b6d4cec2fd94eb5e2040bd1fa3749fded1a
+size 250953044

fold_0/model_latest.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711bd55eaf57f67ab0ad167b2fff87a07949f3dda32d3d4b99c585ecbe3acc0e
+size 732008

fold_0/training_log_2023_3_22_12_03_15.txt

@@ -0,0 +1,239 @@
+Starting... 
+2023-03-22 12:03:15.915841: Using splits from existing split file: /dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/splits_final.pkl 
+2023-03-22 12:03:15.931576: The split file contains 5 splits. 
+2023-03-22 12:03:15.934431: Desired fold for training: 0 
+2023-03-22 12:03:15.937424: This split has 1262 training and 316 validation cases. 
+2023-03-22 12:03:21.972391: TRAINING KEYS:
+ odict_keys(['ts_0001', 'ts_0004', 'ts_0005', 'ts_0006', 'ts_0007', 'ts_0009', 'ts_0010', 'ts_0011', 'ts_0012', 'ts_0013', 'ts_0014', 'ts_0015', 'ts_0016', 'ts_0018', 'ts_0019', 'ts_0021', 'ts_0022', 'ts_0023', 'ts_0025', 'ts_0026', 'ts_0027', 'ts_0028', 'ts_0031', 'ts_0032', 'ts_0033', 'ts_0034', 'ts_0035', 'ts_0036', 'ts_0037', 'ts_0038', 'ts_0039', 'ts_0040', 'ts_0041', 'ts_0042', 'ts_0043', 'ts_0044', 'ts_0045', 'ts_0046', 'ts_0047', 'ts_0048', 'ts_0049', 'ts_0050', 'ts_0051', 'ts_0053', 'ts_0054', 'ts_0055', 'ts_0057', 'ts_0058', 'ts_0059', 'ts_0060', 'ts_0061', 'ts_0062', 'ts_0063', 'ts_0065', 'ts_0066', 'ts_0067', 'ts_0068', 'ts_0069', 'ts_0071', 'ts_0074', 'ts_0075', 'ts_0077', 'ts_0078', 'ts_0079', 'ts_0080', 'ts_0081', 'ts_0082', 'ts_0083', 'ts_0084', 'ts_0085', 'ts_0086', 'ts_0088', 'ts_0089', 'ts_0090', 'ts_0091', 'ts_0093', 'ts_0094', 'ts_0095', 'ts_0096', 'ts_0097', 'ts_0099', 'ts_0100', 'ts_0102', 'ts_0103', 'ts_0105', 'ts_0106', 'ts_0107', 'ts_0108', 'ts_0109', 'ts_0110', 'ts_0112', 'ts_0113', 'ts_0114', 'ts_0116', 'ts_0117', 'ts_0118', 'ts_0119', 'ts_0120', 'ts_0121', 'ts_0122', 'ts_0123', 'ts_0124', 'ts_0125', 'ts_0126', 'ts_0128', 'ts_0129', 'ts_0130', 'ts_0132', 'ts_0133', 'ts_0134', 'ts_0135', 'ts_0138', 'ts_0139', 'ts_0140', 'ts_0141', 'ts_0142', 'ts_0143', 'ts_0144', 'ts_0145', 'ts_0146', 'ts_0148', 'ts_0152', 'ts_0155', 'ts_0156', 'ts_0157', 'ts_0158', 'ts_0159', 'ts_0160', 'ts_0161', 'ts_0163', 'ts_0165', 'ts_0166', 'ts_0168', 'ts_0169', 'ts_0170', 'ts_0171', 'ts_0172', 'ts_0173', 'ts_0174', 'ts_0175', 'ts_0176', 'ts_0177', 'ts_0178', 'ts_0179', 'ts_0181', 'ts_0182', 'ts_0183', 'ts_0184', 'ts_0185', 'ts_0186', 'ts_0189', 'ts_0191', 'ts_0192', 'ts_0194', 'ts_0195', 'ts_0196', 'ts_0197', 'ts_0199', 'ts_0200', 'ts_0202', 'ts_0203', 'ts_0204', 'ts_0205', 'ts_0209', 'ts_0210', 'ts_0211', 'ts_0212', 'ts_0213', 'ts_0214', 'ts_0215', 'ts_0216', 'ts_0217', 'ts_0218', 'ts_0222', 'ts_0224', 'ts_0225', 'ts_0226', 'ts_0228', 'ts_0230', 'ts_0231', 'ts_0232', 'ts_0233', 'ts_0234', 'ts_0235', 'ts_0236', 'ts_0237', 'ts_0239', 'ts_0240', 'ts_0241', 'ts_0242', 'ts_0244', 'ts_0245', 'ts_0246', 'ts_0247', 'ts_0248', 'ts_0249', 'ts_0250', 'ts_0251', 'ts_0252', 'ts_0253', 'ts_0254', 'ts_0255', 'ts_0256', 'ts_0257', 'ts_0258', 'ts_0259', 'ts_0261', 'ts_0263', 'ts_0265', 'ts_0267', 'ts_0269', 'ts_0270', 'ts_0271', 'ts_0272', 'ts_0273', 'ts_0274', 'ts_0275', 'ts_0276', 'ts_0278', 'ts_0279', 'ts_0280', 'ts_0285', 'ts_0286', 'ts_0287', 'ts_0288', 'ts_0289', 'ts_0290', 'ts_0291', 'ts_0292', 'ts_0293', 'ts_0295', 'ts_0296', 'ts_0297', 'ts_0299', 'ts_0300', 'ts_0301', 'ts_0303', 'ts_0304', 'ts_0305', 'ts_0306', 'ts_0307', 'ts_0309', 'ts_0310', 'ts_0312', 'ts_0313', 'ts_0315', 'ts_0316', 'ts_0318', 'ts_0319', 'ts_0320', 'ts_0321', 'ts_0322', 'ts_0323', 'ts_0325', 'ts_0326', 'ts_0327', 'ts_0329', 'ts_0330', 'ts_0331', 'ts_0332', 'ts_0333', 'ts_0334', 'ts_0336', 'ts_0337', 'ts_0339', 'ts_0340', 'ts_0341', 'ts_0342', 'ts_0343', 'ts_0344', 'ts_0345', 'ts_0346', 'ts_0348', 'ts_0349', 'ts_0350', 'ts_0351', 'ts_0352', 'ts_0353', 'ts_0354', 'ts_0355', 'ts_0356', 'ts_0359', 'ts_0360', 'ts_0362', 'ts_0363', 'ts_0364', 'ts_0365', 'ts_0366', 'ts_0367', 'ts_0368', 'ts_0369', 'ts_0370', 'ts_0371', 'ts_0373', 'ts_0374', 'ts_0375', 'ts_0376', 'ts_0377', 'ts_0378', 'ts_0379', 'ts_0381', 'ts_0382', 'ts_0384', 'ts_0385', 'ts_0386', 'ts_0387', 'ts_0388', 'ts_0389', 'ts_0393', 'ts_0396', 'ts_0397', 'ts_0398', 'ts_0399', 'ts_0400', 'ts_0401', 'ts_0402', 'ts_0404', 'ts_0405', 'ts_0406', 'ts_0407', 'ts_0408', 'ts_0409', 'ts_0410', 'ts_0411', 'ts_0412', 'ts_0414', 'ts_0415', 'ts_0416', 'ts_0417', 'ts_0419', 'ts_0420', 'ts_0421', 'ts_0422', 'ts_0424', 'ts_0425', 'ts_0429', 'ts_0430', 'ts_0431', 'ts_0432', 'ts_0433', 'ts_0434', 'ts_0435', 'ts_0437', 'ts_0438', 'ts_0439', 'ts_0440', 'ts_0441', 'ts_0442', 'ts_0443', 'ts_0444', 'ts_0445', 'ts_0446', 'ts_0447', 'ts_0448', 'ts_0449', 'ts_0450', 'ts_0451', 'ts_0452', 'ts_0453', 'ts_0454', 'ts_0457', 'ts_0458', 'ts_0459', 'ts_0460', 'ts_0461', 'ts_0462', 'ts_0464', 'ts_0466', 'ts_0467', 'ts_0468', 'ts_0469', 'ts_0470', 'ts_0472', 'ts_0473', 'ts_0474', 'ts_0475', 'ts_0476', 'ts_0477', 'ts_0478', 'ts_0479', 'ts_0480', 'ts_0481', 'ts_0482', 'ts_0483', 'ts_0484', 'ts_0485', 'ts_0487', 'ts_0488', 'ts_0489', 'ts_0490', 'ts_0491', 'ts_0492', 'ts_0493', 'ts_0494', 'ts_0495', 'ts_0496', 'ts_0497', 'ts_0498', 'ts_0500', 'ts_0501', 'ts_0502', 'ts_0503', 'ts_0504', 'ts_0505', 'ts_0506', 'ts_0507', 'ts_0508', 'ts_0509', 'ts_0512', 'ts_0514', 'ts_0515', 'ts_0516', 'ts_0517', 'ts_0518', 'ts_0519', 'ts_0520', 'ts_0522', 'ts_0524', 'ts_0525', 'ts_0527', 'ts_0528', 'ts_0529', 'ts_0530', 'ts_0531', 'ts_0532', 'ts_0533', 'ts_0534', 'ts_0535', 'ts_0537', 'ts_0538', 'ts_0539', 'ts_0541', 'ts_0542', 'ts_0543', 'ts_0544', 'ts_0546', 'ts_0547', 'ts_0548', 'ts_0552', 'ts_0553', 'ts_0554', 'ts_0555', 'ts_0556', 'ts_0557', 'ts_0559', 'ts_0560', 'ts_0561', 'ts_0562', 'ts_0564', 'ts_0566', 'ts_0567', 'ts_0569', 'ts_0570', 'ts_0572', 'ts_0573', 'ts_0574', 'ts_0575', 'ts_0576', 'ts_0577', 'ts_0578', 'ts_0579', 'ts_0580', 'ts_0582', 'ts_0583', 'ts_0584', 'ts_0585', 'ts_0586', 'ts_0587', 'ts_0588', 'ts_0591', 'ts_0592', 'ts_0594', 'ts_0595', 'ts_0597', 'ts_0599', 'ts_0600', 'ts_0601', 'ts_0602', 'ts_0604', 'ts_0605', 'ts_0606', 'ts_0607', 'ts_0608', 'ts_0609', 'ts_0610', 'ts_0612', 'ts_0613', 'ts_0614', 'ts_0615', 'ts_0616', 'ts_0619', 'ts_0620', 'ts_0621', 'ts_0622', 'ts_0623', 'ts_0625', 'ts_0626', 'ts_0627', 'ts_0628', 'ts_0629', 'ts_0630', 'ts_0631', 'ts_0632', 'ts_0633', 'ts_0634', 'ts_0635', 'ts_0636', 'ts_0637', 'ts_0639', 'ts_0640', 'ts_0642', 'ts_0643', 'ts_0644', 'ts_0645', 'ts_0646', 'ts_0647', 'ts_0649', 'ts_0651', 'ts_0652', 'ts_0653', 'ts_0654', 'ts_0656', 'ts_0657', 'ts_0660', 'ts_0661', 'ts_0662', 'ts_0668', 'ts_0669', 'ts_0670', 'ts_0671', 'ts_0672', 'ts_0673', 'ts_0674', 'ts_0676', 'ts_0677', 'ts_0678', 'ts_0679', 'ts_0680', 'ts_0681', 'ts_0683', 'ts_0684', 'ts_0687', 'ts_0688', 'ts_0689', 'ts_0690', 'ts_0691', 'ts_0693', 'ts_0695', 'ts_0696', 'ts_0697', 'ts_0699', 'ts_0700', 'ts_0701', 'ts_0702', 'ts_0703', 'ts_0704', 'ts_0706', 'ts_0707', 'ts_0708', 'ts_0709', 'ts_0711', 'ts_0712', 'ts_0713', 'ts_0714', 'ts_0715', 'ts_0717', 'ts_0718', 'ts_0719', 'ts_0720', 'ts_0721', 'ts_0722', 'ts_0723', 'ts_0724', 'ts_0725', 'ts_0726', 'ts_0727', 'ts_0729', 'ts_0730', 'ts_0731', 'ts_0734', 'ts_0735', 'ts_0737', 'ts_0738', 'ts_0739', 'ts_0740', 'ts_0741', 'ts_0742', 'ts_0743', 'ts_0744', 'ts_0745', 'ts_0747', 'ts_0748', 'ts_0749', 'ts_0750', 'ts_0751', 'ts_0752', 'ts_0753', 'ts_0754', 'ts_0755', 'ts_0756', 'ts_0757', 'ts_0758', 'ts_0759', 'ts_0760', 'ts_0761', 'ts_0762', 'ts_0764', 'ts_0765', 'ts_0766', 'ts_0769', 'ts_0770', 'ts_0771', 'ts_0772', 'ts_0773', 'ts_0774', 'ts_0775', 'ts_0776', 'ts_0777', 'ts_0778', 'ts_0779', 'ts_0780', 'ts_0781', 'ts_0782', 'ts_0783', 'ts_0784', 'ts_0786', 'ts_0787', 'ts_0788', 'ts_0789', 'ts_0790', 'ts_0791', 'ts_0792', 'ts_0796', 'ts_0797', 'ts_0798', 'ts_0799', 'ts_0800', 'ts_0801', 'ts_0802', 'ts_0803', 'ts_0804', 'ts_0805', 'ts_0807', 'ts_0810', 'ts_0811', 'ts_0812', 'ts_0813', 'ts_0815', 'ts_0816', 'ts_0817', 'ts_0818', 'ts_0819', 'ts_0820', 'ts_0821', 'ts_0822', 'ts_0823', 'ts_0824', 'ts_0825', 'ts_0827', 'ts_0828', 'ts_0830', 'ts_0831', 'ts_0832', 'ts_0833', 'ts_0834', 'ts_0835', 'ts_0836', 'ts_0837', 'ts_0839', 'ts_0841', 'ts_0842', 'ts_0843', 'ts_0844', 'ts_0845', 'ts_0846', 'ts_0847', 'ts_0850', 'ts_0851', 'ts_0852', 'ts_0853', 'ts_0855', 'ts_0856', 'ts_0858', 'ts_0859', 'ts_0860', 'ts_0861', 'ts_0862', 'ts_0863', 'ts_0864', 'ts_0868', 'ts_0869', 'ts_0870', 'ts_0873', 'ts_0874', 'ts_0875', 'ts_0876', 'ts_0878', 'ts_0879', 'ts_0880', 'ts_0881', 'ts_0882', 'ts_0883', 'ts_0884', 'ts_0885', 'ts_0886', 'ts_0888', 'ts_0889', 'ts_0890', 'ts_0891', 'ts_0892', 'ts_0893', 'ts_0894', 'ts_0895', 'ts_0897', 'ts_0898', 'ts_0899', 'ts_0900', 'ts_0902', 'ts_0903', 'ts_0904', 'ts_0906', 'ts_0907', 'ts_0908', 'ts_0909', 'ts_0910', 'ts_0911', 'ts_0912', 'ts_0913', 'ts_0914', 'ts_0916', 'ts_0917', 'ts_0918', 'ts_0919', 'ts_0920', 'ts_0921', 'ts_0922', 'ts_0923', 'ts_0924', 'ts_0926', 'ts_0928', 'ts_0929', 'ts_0930', 'ts_0931', 'ts_0932', 'ts_0933', 'ts_0934', 'ts_0935', 'ts_0936', 'ts_0937', 'ts_0939', 'ts_0941', 'ts_0942', 'ts_0944', 'ts_0945', 'ts_0946', 'ts_0947', 'ts_0948', 'ts_0949', 'ts_0950', 'ts_0951', 'ts_0952', 'ts_0953', 'ts_0956', 'ts_0957', 'ts_0958', 'ts_0960', 'ts_0962', 'ts_0964', 'ts_0967', 'ts_0968', 'ts_0969', 'ts_0970', 'ts_0971', 'ts_0972', 'ts_0973', 'ts_0974', 'ts_0975', 'ts_0976', 'ts_0977', 'ts_0978', 'ts_0979', 'ts_0981', 'ts_0982', 'ts_0983', 'ts_0984', 'ts_0985', 'ts_0986', 'ts_0988', 'ts_0989', 'ts_0990', 'ts_0991', 'ts_0992', 'ts_0995', 'ts_0996', 'ts_0997', 'ts_0999', 'ts_1000', 'ts_1002', 'ts_1003', 'ts_1004', 'ts_1005', 'ts_1006', 'ts_1008', 'ts_1009', 'ts_1010', 'ts_1011', 'ts_1013', 'ts_1014', 'ts_1015', 'ts_1016', 'ts_1017', 'ts_1018', 'ts_1020', 'ts_1021', 'ts_1023', 'ts_1024', 'ts_1025', 'ts_1026', 'ts_1027', 'ts_1029', 'ts_1030', 'ts_1031', 'ts_1032', 'ts_1033', 'ts_1034', 'ts_1035', 'ts_1036', 'ts_1037', 'ts_1038', 'ts_1039', 'ts_1040', 'ts_1041', 'ts_1043', 'ts_1044', 'ts_1045', 'ts_1048', 'ts_1050', 'ts_1051', 'ts_1052', 'ts_1053', 'ts_1054', 'ts_1055', 'ts_1056', 'ts_1058', 'ts_1059', 'ts_1060', 'ts_1061', 'ts_1062', 'ts_1063', 'ts_1064', 'ts_1066', 'ts_1068', 'ts_1070', 'ts_1071', 'ts_1072', 'ts_1074', 'ts_1075', 'ts_1077', 'ts_1079', 'ts_1080', 'ts_1081', 'ts_1082', 'ts_1083', 'ts_1084', 'ts_1086', 'ts_1087', 'ts_1088', 'ts_1089', 'ts_1090', 'ts_1093', 'ts_1094', 'ts_1096', 'ts_1098', 'ts_1099', 'ts_1100', 'ts_1101', 'ts_1102', 'ts_1103', 'ts_1104', 'ts_1106', 'ts_1107', 'ts_1109', 'ts_1110', 'ts_1111', 'ts_1112', 'ts_1114', 'ts_1115', 'ts_1116', 'ts_1118', 'ts_1119', 'ts_1120', 'ts_1121', 'ts_1122', 'ts_1123', 'ts_1124', 'ts_1127', 'ts_1128', 'ts_1130', 'ts_1131', 'ts_1132', 'ts_1133', 'ts_1136', 'ts_1137', 'ts_1138', 'ts_1139', 'ts_1140', 'ts_1141', 'ts_1142', 'ts_1143', 'ts_1144', 'ts_1145', 'ts_1146', 'ts_1147', 'ts_1148', 'ts_1149', 'ts_1150', 'ts_1153', 'ts_1154', 'ts_1155', 'ts_1156', 'ts_1157', 'ts_1160', 'ts_1161', 'ts_1163', 'ts_1164', 'ts_1165', 'ts_1166', 'ts_1167', 'ts_1168', 'ts_1169', 'ts_1170', 'ts_1172', 'ts_1174', 'ts_1175', 'ts_1176', 'ts_1177', 'ts_1178', 'ts_1179', 'ts_1180', 'ts_1183', 'ts_1184', 'ts_1186', 'ts_1189', 'ts_1190', 'ts_1191', 'ts_1192', 'ts_1193', 'ts_1194', 'ts_1195', 'ts_1196', 'ts_1197', 'ts_1198', 'ts_1200', 'ts_1202', 'ts_1203', 'verse_0000', 'verse_0001', 'verse_0002', 'verse_0003', 'verse_0004', 'verse_0005', 'verse_0007', 'verse_0009', 'verse_0010', 'verse_0011', 'verse_0012', 'verse_0013', 'verse_0014', 'verse_0016', 'verse_0017', 'verse_0019', 'verse_0020', 'verse_0021', 'verse_0022', 'verse_0023', 'verse_0024', 'verse_0025', 'verse_0026', 'verse_0027', 'verse_0028', 'verse_0029', 'verse_0030', 'verse_0031', 'verse_0032', 'verse_0034', 'verse_0035', 'verse_0036', 'verse_0037', 'verse_0038', 'verse_0039', 'verse_0042', 'verse_0043', 'verse_0045', 'verse_0047', 'verse_0048', 'verse_0051', 'verse_0052', 'verse_0053', 'verse_0054', 'verse_0055', 'verse_0056', 'verse_0058', 'verse_0059', 'verse_0060', 'verse_0062', 'verse_0063', 'verse_0064', 'verse_0065', 'verse_0066', 'verse_0068', 'verse_0069', 'verse_0070', 'verse_0071', 'verse_0072', 'verse_0073', 'verse_0074', 'verse_0075', 'verse_0076', 'verse_0077', 'verse_0078', 'verse_0079', 'verse_0080', 'verse_0081', 'verse_0082', 'verse_0083', 'verse_0084', 'verse_0085', 'verse_0086', 'verse_0087', 'verse_0088', 'verse_0090', 'verse_0091', 'verse_0092', 'verse_0093', 'verse_0095', 'verse_0098', 'verse_0099', 'verse_0100', 'verse_0101', 'verse_0102', 'verse_0104', 'verse_0105', 'verse_0106', 'verse_0107', 'verse_0108', 'verse_0109', 'verse_0110', 'verse_0111', 'verse_0112', 'verse_0113', 'verse_0114', 'verse_0116', 'verse_0121', 'verse_0122', 'verse_0123', 'verse_0124', 'verse_0125', 'verse_0126', 'verse_0127', 'verse_0128', 'verse_0129', 'verse_0130', 'verse_0131', 'verse_0132', 'verse_0133', 'verse_0134', 'verse_0135', 'verse_0136', 'verse_0137', 'verse_0138', 'verse_0139', 'verse_0142', 'verse_0143', 'verse_0144', 'verse_0146', 'verse_0148', 'verse_0149', 'verse_0151', 'verse_0152', 'verse_0153', 'verse_0154', 'verse_0155', 'verse_0156', 'verse_0157', 'verse_0158', 'verse_0160', 'verse_0162', 'verse_0164', 'verse_0166', 'verse_0168', 'verse_0169', 'verse_0170', 'verse_0171', 'verse_0172', 'verse_0173', 'verse_0174', 'verse_0175', 'verse_0176', 'verse_0177', 'verse_0179', 'verse_0180', 'verse_0181', 'verse_0183', 'verse_0184', 'verse_0185', 'verse_0187', 'verse_0188', 'verse_0189', 'verse_0190', 'verse_0191', 'verse_0192', 'verse_0193', 'verse_0194', 'verse_0195', 'verse_0196', 'verse_0197', 'verse_0198', 'verse_0199', 'verse_0200', 'verse_0201', 'verse_0202', 'verse_0203', 'verse_0204', 'verse_0205', 'verse_0206', 'verse_0207', 'verse_0208', 'verse_0210', 'verse_0212', 'verse_0214', 'verse_0215', 'verse_0216', 'verse_0217', 'verse_0218', 'verse_0219', 'verse_0220', 'verse_0221', 'verse_0223', 'verse_0224', 'verse_0225', 'verse_0226', 'verse_0227', 'verse_0229', 'verse_0230', 'verse_0231', 'verse_0232', 'verse_0234', 'verse_0235', 'verse_0236', 'verse_0237', 'verse_0238', 'verse_0239', 'verse_0240', 'verse_0243', 'verse_0245', 'verse_0247', 'verse_0248', 'verse_0249', 'verse_0250', 'verse_0251', 'verse_0252', 'verse_0254', 'verse_0255', 'verse_0256', 'verse_0257', 'verse_0259', 'verse_0260', 'verse_0261', 'verse_0262', 'verse_0263', 'verse_0265', 'verse_0266', 'verse_0267', 'verse_0268', 'verse_0270', 'verse_0271', 'verse_0272', 'verse_0273', 'verse_0274', 'verse_0275', 'verse_0276', 'verse_0278', 'verse_0280', 'verse_0281', 'verse_0282', 'verse_0283', 'verse_0284', 'verse_0286', 'verse_0287', 'verse_0289', 'verse_0290', 'verse_0291', 'verse_0292', 'verse_0293', 'verse_0294', 'verse_0295', 'verse_0296', 'verse_0297', 'verse_0298', 'verse_0299', 'verse_0302', 'verse_0303', 'verse_0304', 'verse_0305', 'verse_0306', 'verse_0307', 'verse_0308', 'verse_0309', 'verse_0310', 'verse_0311', 'verse_0312', 'verse_0313', 'verse_0314', 'verse_0315', 'verse_0316', 'verse_0317', 'verse_0318', 'verse_0319', 'verse_0320', 'verse_0321', 'verse_0322', 'verse_0323', 'verse_0324', 'verse_0325', 'verse_0326', 'verse_0327', 'verse_0328', 'verse_0329', 'verse_0332', 'verse_0333', 'verse_0334', 'verse_0335', 'verse_0338', 'verse_0339', 'verse_0340', 'verse_0341', 'verse_0342', 'verse_0344', 'verse_0345', 'verse_0346', 'verse_0347', 'verse_0349', 'verse_0351', 'verse_0352', 'verse_0353', 'verse_0355', 'verse_0357', 'verse_0358', 'verse_0359', 'verse_0360', 'verse_0361', 'verse_0363', 'verse_0364', 'verse_0365', 'verse_0366', 'verse_0367', 'verse_0368', 'verse_0369', 'verse_0370', 'verse_0371', 'verse_0372', 'verse_0373']) 
+2023-03-22 12:03:21.975533: VALIDATION KEYS:
+ odict_keys(['ts_0000', 'ts_0002', 'ts_0003', 'ts_0008', 'ts_0017', 'ts_0020', 'ts_0024', 'ts_0029', 'ts_0030', 'ts_0052', 'ts_0056', 'ts_0064', 'ts_0070', 'ts_0072', 'ts_0073', 'ts_0076', 'ts_0087', 'ts_0092', 'ts_0098', 'ts_0101', 'ts_0104', 'ts_0111', 'ts_0115', 'ts_0127', 'ts_0131', 'ts_0136', 'ts_0137', 'ts_0147', 'ts_0149', 'ts_0150', 'ts_0151', 'ts_0153', 'ts_0154', 'ts_0162', 'ts_0164', 'ts_0167', 'ts_0180', 'ts_0187', 'ts_0188', 'ts_0190', 'ts_0193', 'ts_0198', 'ts_0201', 'ts_0206', 'ts_0207', 'ts_0208', 'ts_0219', 'ts_0220', 'ts_0221', 'ts_0223', 'ts_0227', 'ts_0229', 'ts_0238', 'ts_0243', 'ts_0260', 'ts_0262', 'ts_0264', 'ts_0266', 'ts_0268', 'ts_0277', 'ts_0281', 'ts_0282', 'ts_0283', 'ts_0284', 'ts_0294', 'ts_0298', 'ts_0302', 'ts_0308', 'ts_0311', 'ts_0314', 'ts_0317', 'ts_0324', 'ts_0328', 'ts_0335', 'ts_0338', 'ts_0347', 'ts_0357', 'ts_0358', 'ts_0361', 'ts_0372', 'ts_0380', 'ts_0383', 'ts_0390', 'ts_0391', 'ts_0392', 'ts_0394', 'ts_0395', 'ts_0403', 'ts_0413', 'ts_0418', 'ts_0423', 'ts_0426', 'ts_0427', 'ts_0428', 'ts_0436', 'ts_0455', 'ts_0456', 'ts_0463', 'ts_0465', 'ts_0471', 'ts_0486', 'ts_0499', 'ts_0510', 'ts_0511', 'ts_0513', 'ts_0521', 'ts_0523', 'ts_0526', 'ts_0536', 'ts_0540', 'ts_0545', 'ts_0549', 'ts_0550', 'ts_0551', 'ts_0558', 'ts_0563', 'ts_0565', 'ts_0568', 'ts_0571', 'ts_0581', 'ts_0589', 'ts_0590', 'ts_0593', 'ts_0596', 'ts_0598', 'ts_0603', 'ts_0611', 'ts_0617', 'ts_0618', 'ts_0624', 'ts_0638', 'ts_0641', 'ts_0648', 'ts_0650', 'ts_0655', 'ts_0658', 'ts_0659', 'ts_0663', 'ts_0664', 'ts_0665', 'ts_0666', 'ts_0667', 'ts_0675', 'ts_0682', 'ts_0685', 'ts_0686', 'ts_0692', 'ts_0694', 'ts_0698', 'ts_0705', 'ts_0710', 'ts_0716', 'ts_0728', 'ts_0732', 'ts_0733', 'ts_0736', 'ts_0746', 'ts_0763', 'ts_0767', 'ts_0768', 'ts_0785', 'ts_0793', 'ts_0794', 'ts_0795', 'ts_0806', 'ts_0808', 'ts_0809', 'ts_0814', 'ts_0826', 'ts_0829', 'ts_0838', 'ts_0840', 'ts_0848', 'ts_0849', 'ts_0854', 'ts_0857', 'ts_0865', 'ts_0866', 'ts_0867', 'ts_0871', 'ts_0872', 'ts_0877', 'ts_0887', 'ts_0896', 'ts_0901', 'ts_0905', 'ts_0915', 'ts_0925', 'ts_0927', 'ts_0938', 'ts_0940', 'ts_0943', 'ts_0954', 'ts_0955', 'ts_0959', 'ts_0961', 'ts_0963', 'ts_0965', 'ts_0966', 'ts_0980', 'ts_0987', 'ts_0993', 'ts_0994', 'ts_0998', 'ts_1001', 'ts_1007', 'ts_1012', 'ts_1019', 'ts_1022', 'ts_1028', 'ts_1042', 'ts_1046', 'ts_1047', 'ts_1049', 'ts_1057', 'ts_1065', 'ts_1067', 'ts_1069', 'ts_1073', 'ts_1076', 'ts_1078', 'ts_1085', 'ts_1091', 'ts_1092', 'ts_1095', 'ts_1097', 'ts_1105', 'ts_1108', 'ts_1113', 'ts_1117', 'ts_1125', 'ts_1126', 'ts_1129', 'ts_1134', 'ts_1135', 'ts_1151', 'ts_1152', 'ts_1158', 'ts_1159', 'ts_1162', 'ts_1171', 'ts_1173', 'ts_1181', 'ts_1182', 'ts_1185', 'ts_1187', 'ts_1188', 'ts_1199', 'ts_1201', 'verse_0006', 'verse_0008', 'verse_0015', 'verse_0018', 'verse_0033', 'verse_0040', 'verse_0041', 'verse_0044', 'verse_0046', 'verse_0049', 'verse_0050', 'verse_0057', 'verse_0061', 'verse_0067', 'verse_0089', 'verse_0094', 'verse_0096', 'verse_0097', 'verse_0103', 'verse_0115', 'verse_0117', 'verse_0118', 'verse_0119', 'verse_0120', 'verse_0140', 'verse_0141', 'verse_0145', 'verse_0147', 'verse_0150', 'verse_0159', 'verse_0161', 'verse_0163', 'verse_0165', 'verse_0167', 'verse_0178', 'verse_0182', 'verse_0186', 'verse_0209', 'verse_0211', 'verse_0213', 'verse_0222', 'verse_0228', 'verse_0233', 'verse_0241', 'verse_0242', 'verse_0244', 'verse_0246', 'verse_0253', 'verse_0258', 'verse_0264', 'verse_0269', 'verse_0277', 'verse_0279', 'verse_0285', 'verse_0288', 'verse_0300', 'verse_0301', 'verse_0330', 'verse_0331', 'verse_0336', 'verse_0337', 'verse_0343', 'verse_0348', 'verse_0350', 'verse_0354', 'verse_0356', 'verse_0362']) 
+2023-03-22 12:03:25.559617: lr: 0.01 
+2023-03-22 12:03:40.843511: Unable to plot network architecture: 
+2023-03-22 12:03:40.849505: No module named 'hiddenlayer' 
+2023-03-22 12:03:40.855865: 
+printing the network instead:
+ 
+2023-03-22 12:03:40.861621: Generic_UNet(
+  (conv_blocks_localization): ModuleList(
+    (0): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(640, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (1): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(512, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(256, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (2): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(256, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (3): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(128, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (4): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(64, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+  )
+  (conv_blocks_context): ModuleList(
+    (0): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(1, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (1): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(32, 64, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (2): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(64, 128, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (3): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(128, 256, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(256, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (4): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(256, 320, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (5): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+  )
+  (td): ModuleList()
+  (tu): ModuleList(
+    (0): ConvTranspose3d(320, 320, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (1): ConvTranspose3d(320, 256, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (2): ConvTranspose3d(256, 128, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (3): ConvTranspose3d(128, 64, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (4): ConvTranspose3d(64, 32, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+  )
+  (seg_outputs): ModuleList(
+    (0): Conv3d(320, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (1): Conv3d(256, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (2): Conv3d(128, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (3): Conv3d(64, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (4): Conv3d(32, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+  )
+) 
+2023-03-22 12:03:40.870240: 
+ 
+2023-03-22 12:03:40.875478: 
+epoch:  0 
+2023-03-22 12:10:04.570467: train loss : 0.3631 
+2023-03-22 12:10:26.405331: validation loss: 0.1570 
+2023-03-22 12:10:26.437105: Average global foreground Dice: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] 
+2023-03-22 12:10:26.440930: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-03-22 12:10:28.272479: lr: 0.009999 
+2023-03-22 12:10:28.275090: This epoch took 407.393856 s
+ 
+2023-03-22 12:10:28.277601: 
+epoch:  1 

fold_0/training_log_2023_3_31_15_58_42.txt

@@ -0,0 +1,239 @@
+Starting... 
+2023-03-31 15:58:42.830374: Using splits from existing split file: /dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/splits_final.pkl 
+2023-03-31 15:58:42.852679: The split file contains 5 splits. 
+2023-03-31 15:58:42.855808: Desired fold for training: 0 
+2023-03-31 15:58:42.860578: This split has 1262 training and 316 validation cases. 
+2023-03-31 15:58:53.147277: TRAINING KEYS:
+ odict_keys(['ts_0001', 'ts_0004', 'ts_0005', 'ts_0006', 'ts_0007', 'ts_0009', 'ts_0010', 'ts_0011', 'ts_0012', 'ts_0013', 'ts_0014', 'ts_0015', 'ts_0016', 'ts_0018', 'ts_0019', 'ts_0021', 'ts_0022', 'ts_0023', 'ts_0025', 'ts_0026', 'ts_0027', 'ts_0028', 'ts_0031', 'ts_0032', 'ts_0033', 'ts_0034', 'ts_0035', 'ts_0036', 'ts_0037', 'ts_0038', 'ts_0039', 'ts_0040', 'ts_0041', 'ts_0042', 'ts_0043', 'ts_0044', 'ts_0045', 'ts_0046', 'ts_0047', 'ts_0048', 'ts_0049', 'ts_0050', 'ts_0051', 'ts_0053', 'ts_0054', 'ts_0055', 'ts_0057', 'ts_0058', 'ts_0059', 'ts_0060', 'ts_0061', 'ts_0062', 'ts_0063', 'ts_0065', 'ts_0066', 'ts_0067', 'ts_0068', 'ts_0069', 'ts_0071', 'ts_0074', 'ts_0075', 'ts_0077', 'ts_0078', 'ts_0079', 'ts_0080', 'ts_0081', 'ts_0082', 'ts_0083', 'ts_0084', 'ts_0085', 'ts_0086', 'ts_0088', 'ts_0089', 'ts_0090', 'ts_0091', 'ts_0093', 'ts_0094', 'ts_0095', 'ts_0096', 'ts_0097', 'ts_0099', 'ts_0100', 'ts_0102', 'ts_0103', 'ts_0105', 'ts_0106', 'ts_0107', 'ts_0108', 'ts_0109', 'ts_0110', 'ts_0112', 'ts_0113', 'ts_0114', 'ts_0116', 'ts_0117', 'ts_0118', 'ts_0119', 'ts_0120', 'ts_0121', 'ts_0122', 'ts_0123', 'ts_0124', 'ts_0125', 'ts_0126', 'ts_0128', 'ts_0129', 'ts_0130', 'ts_0132', 'ts_0133', 'ts_0134', 'ts_0135', 'ts_0138', 'ts_0139', 'ts_0140', 'ts_0141', 'ts_0142', 'ts_0143', 'ts_0144', 'ts_0145', 'ts_0146', 'ts_0148', 'ts_0152', 'ts_0155', 'ts_0156', 'ts_0157', 'ts_0158', 'ts_0159', 'ts_0160', 'ts_0161', 'ts_0163', 'ts_0165', 'ts_0166', 'ts_0168', 'ts_0169', 'ts_0170', 'ts_0171', 'ts_0172', 'ts_0173', 'ts_0174', 'ts_0175', 'ts_0176', 'ts_0177', 'ts_0178', 'ts_0179', 'ts_0181', 'ts_0182', 'ts_0183', 'ts_0184', 'ts_0185', 'ts_0186', 'ts_0189', 'ts_0191', 'ts_0192', 'ts_0194', 'ts_0195', 'ts_0196', 'ts_0197', 'ts_0199', 'ts_0200', 'ts_0202', 'ts_0203', 'ts_0204', 'ts_0205', 'ts_0209', 'ts_0210', 'ts_0211', 'ts_0212', 'ts_0213', 'ts_0214', 'ts_0215', 'ts_0216', 'ts_0217', 'ts_0218', 'ts_0222', 'ts_0224', 'ts_0225', 'ts_0226', 'ts_0228', 'ts_0230', 'ts_0231', 'ts_0232', 'ts_0233', 'ts_0234', 'ts_0235', 'ts_0236', 'ts_0237', 'ts_0239', 'ts_0240', 'ts_0241', 'ts_0242', 'ts_0244', 'ts_0245', 'ts_0246', 'ts_0247', 'ts_0248', 'ts_0249', 'ts_0250', 'ts_0251', 'ts_0252', 'ts_0253', 'ts_0254', 'ts_0255', 'ts_0256', 'ts_0257', 'ts_0258', 'ts_0259', 'ts_0261', 'ts_0263', 'ts_0265', 'ts_0267', 'ts_0269', 'ts_0270', 'ts_0271', 'ts_0272', 'ts_0273', 'ts_0274', 'ts_0275', 'ts_0276', 'ts_0278', 'ts_0279', 'ts_0280', 'ts_0285', 'ts_0286', 'ts_0287', 'ts_0288', 'ts_0289', 'ts_0290', 'ts_0291', 'ts_0292', 'ts_0293', 'ts_0295', 'ts_0296', 'ts_0297', 'ts_0299', 'ts_0300', 'ts_0301', 'ts_0303', 'ts_0304', 'ts_0305', 'ts_0306', 'ts_0307', 'ts_0309', 'ts_0310', 'ts_0312', 'ts_0313', 'ts_0315', 'ts_0316', 'ts_0318', 'ts_0319', 'ts_0320', 'ts_0321', 'ts_0322', 'ts_0323', 'ts_0325', 'ts_0326', 'ts_0327', 'ts_0329', 'ts_0330', 'ts_0331', 'ts_0332', 'ts_0333', 'ts_0334', 'ts_0336', 'ts_0337', 'ts_0339', 'ts_0340', 'ts_0341', 'ts_0342', 'ts_0343', 'ts_0344', 'ts_0345', 'ts_0346', 'ts_0348', 'ts_0349', 'ts_0350', 'ts_0351', 'ts_0352', 'ts_0353', 'ts_0354', 'ts_0355', 'ts_0356', 'ts_0359', 'ts_0360', 'ts_0362', 'ts_0363', 'ts_0364', 'ts_0365', 'ts_0366', 'ts_0367', 'ts_0368', 'ts_0369', 'ts_0370', 'ts_0371', 'ts_0373', 'ts_0374', 'ts_0375', 'ts_0376', 'ts_0377', 'ts_0378', 'ts_0379', 'ts_0381', 'ts_0382', 'ts_0384', 'ts_0385', 'ts_0386', 'ts_0387', 'ts_0388', 'ts_0389', 'ts_0393', 'ts_0396', 'ts_0397', 'ts_0398', 'ts_0399', 'ts_0400', 'ts_0401', 'ts_0402', 'ts_0404', 'ts_0405', 'ts_0406', 'ts_0407', 'ts_0408', 'ts_0409', 'ts_0410', 'ts_0411', 'ts_0412', 'ts_0414', 'ts_0415', 'ts_0416', 'ts_0417', 'ts_0419', 'ts_0420', 'ts_0421', 'ts_0422', 'ts_0424', 'ts_0425', 'ts_0429', 'ts_0430', 'ts_0431', 'ts_0432', 'ts_0433', 'ts_0434', 'ts_0435', 'ts_0437', 'ts_0438', 'ts_0439', 'ts_0440', 'ts_0441', 'ts_0442', 'ts_0443', 'ts_0444', 'ts_0445', 'ts_0446', 'ts_0447', 'ts_0448', 'ts_0449', 'ts_0450', 'ts_0451', 'ts_0452', 'ts_0453', 'ts_0454', 'ts_0457', 'ts_0458', 'ts_0459', 'ts_0460', 'ts_0461', 'ts_0462', 'ts_0464', 'ts_0466', 'ts_0467', 'ts_0468', 'ts_0469', 'ts_0470', 'ts_0472', 'ts_0473', 'ts_0474', 'ts_0475', 'ts_0476', 'ts_0477', 'ts_0478', 'ts_0479', 'ts_0480', 'ts_0481', 'ts_0482', 'ts_0483', 'ts_0484', 'ts_0485', 'ts_0487', 'ts_0488', 'ts_0489', 'ts_0490', 'ts_0491', 'ts_0492', 'ts_0493', 'ts_0494', 'ts_0495', 'ts_0496', 'ts_0497', 'ts_0498', 'ts_0500', 'ts_0501', 'ts_0502', 'ts_0503', 'ts_0504', 'ts_0505', 'ts_0506', 'ts_0507', 'ts_0508', 'ts_0509', 'ts_0512', 'ts_0514', 'ts_0515', 'ts_0516', 'ts_0517', 'ts_0518', 'ts_0519', 'ts_0520', 'ts_0522', 'ts_0524', 'ts_0525', 'ts_0527', 'ts_0528', 'ts_0529', 'ts_0530', 'ts_0531', 'ts_0532', 'ts_0533', 'ts_0534', 'ts_0535', 'ts_0537', 'ts_0538', 'ts_0539', 'ts_0541', 'ts_0542', 'ts_0543', 'ts_0544', 'ts_0546', 'ts_0547', 'ts_0548', 'ts_0552', 'ts_0553', 'ts_0554', 'ts_0555', 'ts_0556', 'ts_0557', 'ts_0559', 'ts_0560', 'ts_0561', 'ts_0562', 'ts_0564', 'ts_0566', 'ts_0567', 'ts_0569', 'ts_0570', 'ts_0572', 'ts_0573', 'ts_0574', 'ts_0575', 'ts_0576', 'ts_0577', 'ts_0578', 'ts_0579', 'ts_0580', 'ts_0582', 'ts_0583', 'ts_0584', 'ts_0585', 'ts_0586', 'ts_0587', 'ts_0588', 'ts_0591', 'ts_0592', 'ts_0594', 'ts_0595', 'ts_0597', 'ts_0599', 'ts_0600', 'ts_0601', 'ts_0602', 'ts_0604', 'ts_0605', 'ts_0606', 'ts_0607', 'ts_0608', 'ts_0609', 'ts_0610', 'ts_0612', 'ts_0613', 'ts_0614', 'ts_0615', 'ts_0616', 'ts_0619', 'ts_0620', 'ts_0621', 'ts_0622', 'ts_0623', 'ts_0625', 'ts_0626', 'ts_0627', 'ts_0628', 'ts_0629', 'ts_0630', 'ts_0631', 'ts_0632', 'ts_0633', 'ts_0634', 'ts_0635', 'ts_0636', 'ts_0637', 'ts_0639', 'ts_0640', 'ts_0642', 'ts_0643', 'ts_0644', 'ts_0645', 'ts_0646', 'ts_0647', 'ts_0649', 'ts_0651', 'ts_0652', 'ts_0653', 'ts_0654', 'ts_0656', 'ts_0657', 'ts_0660', 'ts_0661', 'ts_0662', 'ts_0668', 'ts_0669', 'ts_0670', 'ts_0671', 'ts_0672', 'ts_0673', 'ts_0674', 'ts_0676', 'ts_0677', 'ts_0678', 'ts_0679', 'ts_0680', 'ts_0681', 'ts_0683', 'ts_0684', 'ts_0687', 'ts_0688', 'ts_0689', 'ts_0690', 'ts_0691', 'ts_0693', 'ts_0695', 'ts_0696', 'ts_0697', 'ts_0699', 'ts_0700', 'ts_0701', 'ts_0702', 'ts_0703', 'ts_0704', 'ts_0706', 'ts_0707', 'ts_0708', 'ts_0709', 'ts_0711', 'ts_0712', 'ts_0713', 'ts_0714', 'ts_0715', 'ts_0717', 'ts_0718', 'ts_0719', 'ts_0720', 'ts_0721', 'ts_0722', 'ts_0723', 'ts_0724', 'ts_0725', 'ts_0726', 'ts_0727', 'ts_0729', 'ts_0730', 'ts_0731', 'ts_0734', 'ts_0735', 'ts_0737', 'ts_0738', 'ts_0739', 'ts_0740', 'ts_0741', 'ts_0742', 'ts_0743', 'ts_0744', 'ts_0745', 'ts_0747', 'ts_0748', 'ts_0749', 'ts_0750', 'ts_0751', 'ts_0752', 'ts_0753', 'ts_0754', 'ts_0755', 'ts_0756', 'ts_0757', 'ts_0758', 'ts_0759', 'ts_0760', 'ts_0761', 'ts_0762', 'ts_0764', 'ts_0765', 'ts_0766', 'ts_0769', 'ts_0770', 'ts_0771', 'ts_0772', 'ts_0773', 'ts_0774', 'ts_0775', 'ts_0776', 'ts_0777', 'ts_0778', 'ts_0779', 'ts_0780', 'ts_0781', 'ts_0782', 'ts_0783', 'ts_0784', 'ts_0786', 'ts_0787', 'ts_0788', 'ts_0789', 'ts_0790', 'ts_0791', 'ts_0792', 'ts_0796', 'ts_0797', 'ts_0798', 'ts_0799', 'ts_0800', 'ts_0801', 'ts_0802', 'ts_0803', 'ts_0804', 'ts_0805', 'ts_0807', 'ts_0810', 'ts_0811', 'ts_0812', 'ts_0813', 'ts_0815', 'ts_0816', 'ts_0817', 'ts_0818', 'ts_0819', 'ts_0820', 'ts_0821', 'ts_0822', 'ts_0823', 'ts_0824', 'ts_0825', 'ts_0827', 'ts_0828', 'ts_0830', 'ts_0831', 'ts_0832', 'ts_0833', 'ts_0834', 'ts_0835', 'ts_0836', 'ts_0837', 'ts_0839', 'ts_0841', 'ts_0842', 'ts_0843', 'ts_0844', 'ts_0845', 'ts_0846', 'ts_0847', 'ts_0850', 'ts_0851', 'ts_0852', 'ts_0853', 'ts_0855', 'ts_0856', 'ts_0858', 'ts_0859', 'ts_0860', 'ts_0861', 'ts_0862', 'ts_0863', 'ts_0864', 'ts_0868', 'ts_0869', 'ts_0870', 'ts_0873', 'ts_0874', 'ts_0875', 'ts_0876', 'ts_0878', 'ts_0879', 'ts_0880', 'ts_0881', 'ts_0882', 'ts_0883', 'ts_0884', 'ts_0885', 'ts_0886', 'ts_0888', 'ts_0889', 'ts_0890', 'ts_0891', 'ts_0892', 'ts_0893', 'ts_0894', 'ts_0895', 'ts_0897', 'ts_0898', 'ts_0899', 'ts_0900', 'ts_0902', 'ts_0903', 'ts_0904', 'ts_0906', 'ts_0907', 'ts_0908', 'ts_0909', 'ts_0910', 'ts_0911', 'ts_0912', 'ts_0913', 'ts_0914', 'ts_0916', 'ts_0917', 'ts_0918', 'ts_0919', 'ts_0920', 'ts_0921', 'ts_0922', 'ts_0923', 'ts_0924', 'ts_0926', 'ts_0928', 'ts_0929', 'ts_0930', 'ts_0931', 'ts_0932', 'ts_0933', 'ts_0934', 'ts_0935', 'ts_0936', 'ts_0937', 'ts_0939', 'ts_0941', 'ts_0942', 'ts_0944', 'ts_0945', 'ts_0946', 'ts_0947', 'ts_0948', 'ts_0949', 'ts_0950', 'ts_0951', 'ts_0952', 'ts_0953', 'ts_0956', 'ts_0957', 'ts_0958', 'ts_0960', 'ts_0962', 'ts_0964', 'ts_0967', 'ts_0968', 'ts_0969', 'ts_0970', 'ts_0971', 'ts_0972', 'ts_0973', 'ts_0974', 'ts_0975', 'ts_0976', 'ts_0977', 'ts_0978', 'ts_0979', 'ts_0981', 'ts_0982', 'ts_0983', 'ts_0984', 'ts_0985', 'ts_0986', 'ts_0988', 'ts_0989', 'ts_0990', 'ts_0991', 'ts_0992', 'ts_0995', 'ts_0996', 'ts_0997', 'ts_0999', 'ts_1000', 'ts_1002', 'ts_1003', 'ts_1004', 'ts_1005', 'ts_1006', 'ts_1008', 'ts_1009', 'ts_1010', 'ts_1011', 'ts_1013', 'ts_1014', 'ts_1015', 'ts_1016', 'ts_1017', 'ts_1018', 'ts_1020', 'ts_1021', 'ts_1023', 'ts_1024', 'ts_1025', 'ts_1026', 'ts_1027', 'ts_1029', 'ts_1030', 'ts_1031', 'ts_1032', 'ts_1033', 'ts_1034', 'ts_1035', 'ts_1036', 'ts_1037', 'ts_1038', 'ts_1039', 'ts_1040', 'ts_1041', 'ts_1043', 'ts_1044', 'ts_1045', 'ts_1048', 'ts_1050', 'ts_1051', 'ts_1052', 'ts_1053', 'ts_1054', 'ts_1055', 'ts_1056', 'ts_1058', 'ts_1059', 'ts_1060', 'ts_1061', 'ts_1062', 'ts_1063', 'ts_1064', 'ts_1066', 'ts_1068', 'ts_1070', 'ts_1071', 'ts_1072', 'ts_1074', 'ts_1075', 'ts_1077', 'ts_1079', 'ts_1080', 'ts_1081', 'ts_1082', 'ts_1083', 'ts_1084', 'ts_1086', 'ts_1087', 'ts_1088', 'ts_1089', 'ts_1090', 'ts_1093', 'ts_1094', 'ts_1096', 'ts_1098', 'ts_1099', 'ts_1100', 'ts_1101', 'ts_1102', 'ts_1103', 'ts_1104', 'ts_1106', 'ts_1107', 'ts_1109', 'ts_1110', 'ts_1111', 'ts_1112', 'ts_1114', 'ts_1115', 'ts_1116', 'ts_1118', 'ts_1119', 'ts_1120', 'ts_1121', 'ts_1122', 'ts_1123', 'ts_1124', 'ts_1127', 'ts_1128', 'ts_1130', 'ts_1131', 'ts_1132', 'ts_1133', 'ts_1136', 'ts_1137', 'ts_1138', 'ts_1139', 'ts_1140', 'ts_1141', 'ts_1142', 'ts_1143', 'ts_1144', 'ts_1145', 'ts_1146', 'ts_1147', 'ts_1148', 'ts_1149', 'ts_1150', 'ts_1153', 'ts_1154', 'ts_1155', 'ts_1156', 'ts_1157', 'ts_1160', 'ts_1161', 'ts_1163', 'ts_1164', 'ts_1165', 'ts_1166', 'ts_1167', 'ts_1168', 'ts_1169', 'ts_1170', 'ts_1172', 'ts_1174', 'ts_1175', 'ts_1176', 'ts_1177', 'ts_1178', 'ts_1179', 'ts_1180', 'ts_1183', 'ts_1184', 'ts_1186', 'ts_1189', 'ts_1190', 'ts_1191', 'ts_1192', 'ts_1193', 'ts_1194', 'ts_1195', 'ts_1196', 'ts_1197', 'ts_1198', 'ts_1200', 'ts_1202', 'ts_1203', 'verse_0000', 'verse_0001', 'verse_0002', 'verse_0003', 'verse_0004', 'verse_0005', 'verse_0007', 'verse_0009', 'verse_0010', 'verse_0011', 'verse_0012', 'verse_0013', 'verse_0014', 'verse_0016', 'verse_0017', 'verse_0019', 'verse_0020', 'verse_0021', 'verse_0022', 'verse_0023', 'verse_0024', 'verse_0025', 'verse_0026', 'verse_0027', 'verse_0028', 'verse_0029', 'verse_0030', 'verse_0031', 'verse_0032', 'verse_0034', 'verse_0035', 'verse_0036', 'verse_0037', 'verse_0038', 'verse_0039', 'verse_0042', 'verse_0043', 'verse_0045', 'verse_0047', 'verse_0048', 'verse_0051', 'verse_0052', 'verse_0053', 'verse_0054', 'verse_0055', 'verse_0056', 'verse_0058', 'verse_0059', 'verse_0060', 'verse_0062', 'verse_0063', 'verse_0064', 'verse_0065', 'verse_0066', 'verse_0068', 'verse_0069', 'verse_0070', 'verse_0071', 'verse_0072', 'verse_0073', 'verse_0074', 'verse_0075', 'verse_0076', 'verse_0077', 'verse_0078', 'verse_0079', 'verse_0080', 'verse_0081', 'verse_0082', 'verse_0083', 'verse_0084', 'verse_0085', 'verse_0086', 'verse_0087', 'verse_0088', 'verse_0090', 'verse_0091', 'verse_0092', 'verse_0093', 'verse_0095', 'verse_0098', 'verse_0099', 'verse_0100', 'verse_0101', 'verse_0102', 'verse_0104', 'verse_0105', 'verse_0106', 'verse_0107', 'verse_0108', 'verse_0109', 'verse_0110', 'verse_0111', 'verse_0112', 'verse_0113', 'verse_0114', 'verse_0116', 'verse_0121', 'verse_0122', 'verse_0123', 'verse_0124', 'verse_0125', 'verse_0126', 'verse_0127', 'verse_0128', 'verse_0129', 'verse_0130', 'verse_0131', 'verse_0132', 'verse_0133', 'verse_0134', 'verse_0135', 'verse_0136', 'verse_0137', 'verse_0138', 'verse_0139', 'verse_0142', 'verse_0143', 'verse_0144', 'verse_0146', 'verse_0148', 'verse_0149', 'verse_0151', 'verse_0152', 'verse_0153', 'verse_0154', 'verse_0155', 'verse_0156', 'verse_0157', 'verse_0158', 'verse_0160', 'verse_0162', 'verse_0164', 'verse_0166', 'verse_0168', 'verse_0169', 'verse_0170', 'verse_0171', 'verse_0172', 'verse_0173', 'verse_0174', 'verse_0175', 'verse_0176', 'verse_0177', 'verse_0179', 'verse_0180', 'verse_0181', 'verse_0183', 'verse_0184', 'verse_0185', 'verse_0187', 'verse_0188', 'verse_0189', 'verse_0190', 'verse_0191', 'verse_0192', 'verse_0193', 'verse_0194', 'verse_0195', 'verse_0196', 'verse_0197', 'verse_0198', 'verse_0199', 'verse_0200', 'verse_0201', 'verse_0202', 'verse_0203', 'verse_0204', 'verse_0205', 'verse_0206', 'verse_0207', 'verse_0208', 'verse_0210', 'verse_0212', 'verse_0214', 'verse_0215', 'verse_0216', 'verse_0217', 'verse_0218', 'verse_0219', 'verse_0220', 'verse_0221', 'verse_0223', 'verse_0224', 'verse_0225', 'verse_0226', 'verse_0227', 'verse_0229', 'verse_0230', 'verse_0231', 'verse_0232', 'verse_0234', 'verse_0235', 'verse_0236', 'verse_0237', 'verse_0238', 'verse_0239', 'verse_0240', 'verse_0243', 'verse_0245', 'verse_0247', 'verse_0248', 'verse_0249', 'verse_0250', 'verse_0251', 'verse_0252', 'verse_0254', 'verse_0255', 'verse_0256', 'verse_0257', 'verse_0259', 'verse_0260', 'verse_0261', 'verse_0262', 'verse_0263', 'verse_0265', 'verse_0266', 'verse_0267', 'verse_0268', 'verse_0270', 'verse_0271', 'verse_0272', 'verse_0273', 'verse_0274', 'verse_0275', 'verse_0276', 'verse_0278', 'verse_0280', 'verse_0281', 'verse_0282', 'verse_0283', 'verse_0284', 'verse_0286', 'verse_0287', 'verse_0289', 'verse_0290', 'verse_0291', 'verse_0292', 'verse_0293', 'verse_0294', 'verse_0295', 'verse_0296', 'verse_0297', 'verse_0298', 'verse_0299', 'verse_0302', 'verse_0303', 'verse_0304', 'verse_0305', 'verse_0306', 'verse_0307', 'verse_0308', 'verse_0309', 'verse_0310', 'verse_0311', 'verse_0312', 'verse_0313', 'verse_0314', 'verse_0315', 'verse_0316', 'verse_0317', 'verse_0318', 'verse_0319', 'verse_0320', 'verse_0321', 'verse_0322', 'verse_0323', 'verse_0324', 'verse_0325', 'verse_0326', 'verse_0327', 'verse_0328', 'verse_0329', 'verse_0332', 'verse_0333', 'verse_0334', 'verse_0335', 'verse_0338', 'verse_0339', 'verse_0340', 'verse_0341', 'verse_0342', 'verse_0344', 'verse_0345', 'verse_0346', 'verse_0347', 'verse_0349', 'verse_0351', 'verse_0352', 'verse_0353', 'verse_0355', 'verse_0357', 'verse_0358', 'verse_0359', 'verse_0360', 'verse_0361', 'verse_0363', 'verse_0364', 'verse_0365', 'verse_0366', 'verse_0367', 'verse_0368', 'verse_0369', 'verse_0370', 'verse_0371', 'verse_0372', 'verse_0373']) 
+2023-03-31 15:58:53.153337: VALIDATION KEYS:
+ odict_keys(['ts_0000', 'ts_0002', 'ts_0003', 'ts_0008', 'ts_0017', 'ts_0020', 'ts_0024', 'ts_0029', 'ts_0030', 'ts_0052', 'ts_0056', 'ts_0064', 'ts_0070', 'ts_0072', 'ts_0073', 'ts_0076', 'ts_0087', 'ts_0092', 'ts_0098', 'ts_0101', 'ts_0104', 'ts_0111', 'ts_0115', 'ts_0127', 'ts_0131', 'ts_0136', 'ts_0137', 'ts_0147', 'ts_0149', 'ts_0150', 'ts_0151', 'ts_0153', 'ts_0154', 'ts_0162', 'ts_0164', 'ts_0167', 'ts_0180', 'ts_0187', 'ts_0188', 'ts_0190', 'ts_0193', 'ts_0198', 'ts_0201', 'ts_0206', 'ts_0207', 'ts_0208', 'ts_0219', 'ts_0220', 'ts_0221', 'ts_0223', 'ts_0227', 'ts_0229', 'ts_0238', 'ts_0243', 'ts_0260', 'ts_0262', 'ts_0264', 'ts_0266', 'ts_0268', 'ts_0277', 'ts_0281', 'ts_0282', 'ts_0283', 'ts_0284', 'ts_0294', 'ts_0298', 'ts_0302', 'ts_0308', 'ts_0311', 'ts_0314', 'ts_0317', 'ts_0324', 'ts_0328', 'ts_0335', 'ts_0338', 'ts_0347', 'ts_0357', 'ts_0358', 'ts_0361', 'ts_0372', 'ts_0380', 'ts_0383', 'ts_0390', 'ts_0391', 'ts_0392', 'ts_0394', 'ts_0395', 'ts_0403', 'ts_0413', 'ts_0418', 'ts_0423', 'ts_0426', 'ts_0427', 'ts_0428', 'ts_0436', 'ts_0455', 'ts_0456', 'ts_0463', 'ts_0465', 'ts_0471', 'ts_0486', 'ts_0499', 'ts_0510', 'ts_0511', 'ts_0513', 'ts_0521', 'ts_0523', 'ts_0526', 'ts_0536', 'ts_0540', 'ts_0545', 'ts_0549', 'ts_0550', 'ts_0551', 'ts_0558', 'ts_0563', 'ts_0565', 'ts_0568', 'ts_0571', 'ts_0581', 'ts_0589', 'ts_0590', 'ts_0593', 'ts_0596', 'ts_0598', 'ts_0603', 'ts_0611', 'ts_0617', 'ts_0618', 'ts_0624', 'ts_0638', 'ts_0641', 'ts_0648', 'ts_0650', 'ts_0655', 'ts_0658', 'ts_0659', 'ts_0663', 'ts_0664', 'ts_0665', 'ts_0666', 'ts_0667', 'ts_0675', 'ts_0682', 'ts_0685', 'ts_0686', 'ts_0692', 'ts_0694', 'ts_0698', 'ts_0705', 'ts_0710', 'ts_0716', 'ts_0728', 'ts_0732', 'ts_0733', 'ts_0736', 'ts_0746', 'ts_0763', 'ts_0767', 'ts_0768', 'ts_0785', 'ts_0793', 'ts_0794', 'ts_0795', 'ts_0806', 'ts_0808', 'ts_0809', 'ts_0814', 'ts_0826', 'ts_0829', 'ts_0838', 'ts_0840', 'ts_0848', 'ts_0849', 'ts_0854', 'ts_0857', 'ts_0865', 'ts_0866', 'ts_0867', 'ts_0871', 'ts_0872', 'ts_0877', 'ts_0887', 'ts_0896', 'ts_0901', 'ts_0905', 'ts_0915', 'ts_0925', 'ts_0927', 'ts_0938', 'ts_0940', 'ts_0943', 'ts_0954', 'ts_0955', 'ts_0959', 'ts_0961', 'ts_0963', 'ts_0965', 'ts_0966', 'ts_0980', 'ts_0987', 'ts_0993', 'ts_0994', 'ts_0998', 'ts_1001', 'ts_1007', 'ts_1012', 'ts_1019', 'ts_1022', 'ts_1028', 'ts_1042', 'ts_1046', 'ts_1047', 'ts_1049', 'ts_1057', 'ts_1065', 'ts_1067', 'ts_1069', 'ts_1073', 'ts_1076', 'ts_1078', 'ts_1085', 'ts_1091', 'ts_1092', 'ts_1095', 'ts_1097', 'ts_1105', 'ts_1108', 'ts_1113', 'ts_1117', 'ts_1125', 'ts_1126', 'ts_1129', 'ts_1134', 'ts_1135', 'ts_1151', 'ts_1152', 'ts_1158', 'ts_1159', 'ts_1162', 'ts_1171', 'ts_1173', 'ts_1181', 'ts_1182', 'ts_1185', 'ts_1187', 'ts_1188', 'ts_1199', 'ts_1201', 'verse_0006', 'verse_0008', 'verse_0015', 'verse_0018', 'verse_0033', 'verse_0040', 'verse_0041', 'verse_0044', 'verse_0046', 'verse_0049', 'verse_0050', 'verse_0057', 'verse_0061', 'verse_0067', 'verse_0089', 'verse_0094', 'verse_0096', 'verse_0097', 'verse_0103', 'verse_0115', 'verse_0117', 'verse_0118', 'verse_0119', 'verse_0120', 'verse_0140', 'verse_0141', 'verse_0145', 'verse_0147', 'verse_0150', 'verse_0159', 'verse_0161', 'verse_0163', 'verse_0165', 'verse_0167', 'verse_0178', 'verse_0182', 'verse_0186', 'verse_0209', 'verse_0211', 'verse_0213', 'verse_0222', 'verse_0228', 'verse_0233', 'verse_0241', 'verse_0242', 'verse_0244', 'verse_0246', 'verse_0253', 'verse_0258', 'verse_0264', 'verse_0269', 'verse_0277', 'verse_0279', 'verse_0285', 'verse_0288', 'verse_0300', 'verse_0301', 'verse_0330', 'verse_0331', 'verse_0336', 'verse_0337', 'verse_0343', 'verse_0348', 'verse_0350', 'verse_0354', 'verse_0356', 'verse_0362']) 
+2023-03-31 15:58:56.169063: lr: 0.01 
+2023-03-31 15:59:18.466001: Unable to plot network architecture: 
+2023-03-31 15:59:18.469075: No module named 'hiddenlayer' 
+2023-03-31 15:59:18.472669: 
+printing the network instead:
+ 
+2023-03-31 15:59:18.490690: Generic_UNet(
+  (conv_blocks_localization): ModuleList(
+    (0): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(640, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (1): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(512, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(256, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (2): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(256, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (3): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(128, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (4): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(64, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+  )
+  (conv_blocks_context): ModuleList(
+    (0): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(1, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (1): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(32, 64, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (2): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(64, 128, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (3): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(128, 256, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(256, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (4): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(256, 320, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (5): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+  )
+  (td): ModuleList()
+  (tu): ModuleList(
+    (0): ConvTranspose3d(320, 320, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (1): ConvTranspose3d(320, 256, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (2): ConvTranspose3d(256, 128, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (3): ConvTranspose3d(128, 64, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (4): ConvTranspose3d(64, 32, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+  )
+  (seg_outputs): ModuleList(
+    (0): Conv3d(320, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (1): Conv3d(256, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (2): Conv3d(128, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (3): Conv3d(64, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (4): Conv3d(32, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+  )
+) 
+2023-03-31 15:59:18.501654: 
+ 
+2023-03-31 15:59:18.505920: 
+epoch:  0 
+2023-03-31 16:05:57.897493: train loss : 0.4308 
+2023-03-31 16:06:16.189684: validation loss: 0.1673 
+2023-03-31 16:06:16.196204: Average global foreground Dice: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] 
+2023-03-31 16:06:16.200414: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-03-31 16:06:17.703125: lr: 0.009999 
+2023-03-31 16:06:17.707587: This epoch took 419.198423 s
+ 
+2023-03-31 16:06:17.719003: 
+epoch:  1 

fold_0/training_log_2023_4_3_21_31_05.txt

@@ -0,0 +1,9 @@
+Starting... 
+2023-04-03 21:31:05.139754: Using splits from existing split file: /dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/splits_final.pkl 
+2023-04-03 21:31:05.147000: The split file contains 5 splits. 
+2023-04-03 21:31:05.148267: Desired fold for training: 0 
+2023-04-03 21:31:05.149477: This split has 1262 training and 316 validation cases. 
+2023-04-03 21:31:09.133807: TRAINING KEYS:
+ odict_keys(['ts_0001', 'ts_0004', 'ts_0005', 'ts_0006', 'ts_0007', 'ts_0009', 'ts_0010', 'ts_0011', 'ts_0012', 'ts_0013', 'ts_0014', 'ts_0015', 'ts_0016', 'ts_0018', 'ts_0019', 'ts_0021', 'ts_0022', 'ts_0023', 'ts_0025', 'ts_0026', 'ts_0027', 'ts_0028', 'ts_0031', 'ts_0032', 'ts_0033', 'ts_0034', 'ts_0035', 'ts_0036', 'ts_0037', 'ts_0038', 'ts_0039', 'ts_0040', 'ts_0041', 'ts_0042', 'ts_0043', 'ts_0044', 'ts_0045', 'ts_0046', 'ts_0047', 'ts_0048', 'ts_0049', 'ts_0050', 'ts_0051', 'ts_0053', 'ts_0054', 'ts_0055', 'ts_0057', 'ts_0058', 'ts_0059', 'ts_0060', 'ts_0061', 'ts_0062', 'ts_0063', 'ts_0065', 'ts_0066', 'ts_0067', 'ts_0068', 'ts_0069', 'ts_0071', 'ts_0074', 'ts_0075', 'ts_0077', 'ts_0078', 'ts_0079', 'ts_0080', 'ts_0081', 'ts_0082', 'ts_0083', 'ts_0084', 'ts_0085', 'ts_0086', 'ts_0088', 'ts_0089', 'ts_0090', 'ts_0091', 'ts_0093', 'ts_0094', 'ts_0095', 'ts_0096', 'ts_0097', 'ts_0099', 'ts_0100', 'ts_0102', 'ts_0103', 'ts_0105', 'ts_0106', 'ts_0107', 'ts_0108', 'ts_0109', 'ts_0110', 'ts_0112', 'ts_0113', 'ts_0114', 'ts_0116', 'ts_0117', 'ts_0118', 'ts_0119', 'ts_0120', 'ts_0121', 'ts_0122', 'ts_0123', 'ts_0124', 'ts_0125', 'ts_0126', 'ts_0128', 'ts_0129', 'ts_0130', 'ts_0132', 'ts_0133', 'ts_0134', 'ts_0135', 'ts_0138', 'ts_0139', 'ts_0140', 'ts_0141', 'ts_0142', 'ts_0143', 'ts_0144', 'ts_0145', 'ts_0146', 'ts_0148', 'ts_0152', 'ts_0155', 'ts_0156', 'ts_0157', 'ts_0158', 'ts_0159', 'ts_0160', 'ts_0161', 'ts_0163', 'ts_0165', 'ts_0166', 'ts_0168', 'ts_0169', 'ts_0170', 'ts_0171', 'ts_0172', 'ts_0173', 'ts_0174', 'ts_0175', 'ts_0176', 'ts_0177', 'ts_0178', 'ts_0179', 'ts_0181', 'ts_0182', 'ts_0183', 'ts_0184', 'ts_0185', 'ts_0186', 'ts_0189', 'ts_0191', 'ts_0192', 'ts_0194', 'ts_0195', 'ts_0196', 'ts_0197', 'ts_0199', 'ts_0200', 'ts_0202', 'ts_0203', 'ts_0204', 'ts_0205', 'ts_0209', 'ts_0210', 'ts_0211', 'ts_0212', 'ts_0213', 'ts_0214', 'ts_0215', 'ts_0216', 'ts_0217', 'ts_0218', 'ts_0222', 'ts_0224', 'ts_0225', 'ts_0226', 'ts_0228', 'ts_0230', 'ts_0231', 'ts_0232', 'ts_0233', 'ts_0234', 'ts_0235', 'ts_0236', 'ts_0237', 'ts_0239', 'ts_0240', 'ts_0241', 'ts_0242', 'ts_0244', 'ts_0245', 'ts_0246', 'ts_0247', 'ts_0248', 'ts_0249', 'ts_0250', 'ts_0251', 'ts_0252', 'ts_0253', 'ts_0254', 'ts_0255', 'ts_0256', 'ts_0257', 'ts_0258', 'ts_0259', 'ts_0261', 'ts_0263', 'ts_0265', 'ts_0267', 'ts_0269', 'ts_0270', 'ts_0271', 'ts_0272', 'ts_0273', 'ts_0274', 'ts_0275', 'ts_0276', 'ts_0278', 'ts_0279', 'ts_0280', 'ts_0285', 'ts_0286', 'ts_0287', 'ts_0288', 'ts_0289', 'ts_0290', 'ts_0291', 'ts_0292', 'ts_0293', 'ts_0295', 'ts_0296', 'ts_0297', 'ts_0299', 'ts_0300', 'ts_0301', 'ts_0303', 'ts_0304', 'ts_0305', 'ts_0306', 'ts_0307', 'ts_0309', 'ts_0310', 'ts_0312', 'ts_0313', 'ts_0315', 'ts_0316', 'ts_0318', 'ts_0319', 'ts_0320', 'ts_0321', 'ts_0322', 'ts_0323', 'ts_0325', 'ts_0326', 'ts_0327', 'ts_0329', 'ts_0330', 'ts_0331', 'ts_0332', 'ts_0333', 'ts_0334', 'ts_0336', 'ts_0337', 'ts_0339', 'ts_0340', 'ts_0341', 'ts_0342', 'ts_0343', 'ts_0344', 'ts_0345', 'ts_0346', 'ts_0348', 'ts_0349', 'ts_0350', 'ts_0351', 'ts_0352', 'ts_0353', 'ts_0354', 'ts_0355', 'ts_0356', 'ts_0359', 'ts_0360', 'ts_0362', 'ts_0363', 'ts_0364', 'ts_0365', 'ts_0366', 'ts_0367', 'ts_0368', 'ts_0369', 'ts_0370', 'ts_0371', 'ts_0373', 'ts_0374', 'ts_0375', 'ts_0376', 'ts_0377', 'ts_0378', 'ts_0379', 'ts_0381', 'ts_0382', 'ts_0384', 'ts_0385', 'ts_0386', 'ts_0387', 'ts_0388', 'ts_0389', 'ts_0393', 'ts_0396', 'ts_0397', 'ts_0398', 'ts_0399', 'ts_0400', 'ts_0401', 'ts_0402', 'ts_0404', 'ts_0405', 'ts_0406', 'ts_0407', 'ts_0408', 'ts_0409', 'ts_0410', 'ts_0411', 'ts_0412', 'ts_0414', 'ts_0415', 'ts_0416', 'ts_0417', 'ts_0419', 'ts_0420', 'ts_0421', 'ts_0422', 'ts_0424', 'ts_0425', 'ts_0429', 'ts_0430', 'ts_0431', 'ts_0432', 'ts_0433', 'ts_0434', 'ts_0435', 'ts_0437', 'ts_0438', 'ts_0439', 'ts_0440', 'ts_0441', 'ts_0442', 'ts_0443', 'ts_0444', 'ts_0445', 'ts_0446', 'ts_0447', 'ts_0448', 'ts_0449', 'ts_0450', 'ts_0451', 'ts_0452', 'ts_0453', 'ts_0454', 'ts_0457', 'ts_0458', 'ts_0459', 'ts_0460', 'ts_0461', 'ts_0462', 'ts_0464', 'ts_0466', 'ts_0467', 'ts_0468', 'ts_0469', 'ts_0470', 'ts_0472', 'ts_0473', 'ts_0474', 'ts_0475', 'ts_0476', 'ts_0477', 'ts_0478', 'ts_0479', 'ts_0480', 'ts_0481', 'ts_0482', 'ts_0483', 'ts_0484', 'ts_0485', 'ts_0487', 'ts_0488', 'ts_0489', 'ts_0490', 'ts_0491', 'ts_0492', 'ts_0493', 'ts_0494', 'ts_0495', 'ts_0496', 'ts_0497', 'ts_0498', 'ts_0500', 'ts_0501', 'ts_0502', 'ts_0503', 'ts_0504', 'ts_0505', 'ts_0506', 'ts_0507', 'ts_0508', 'ts_0509', 'ts_0512', 'ts_0514', 'ts_0515', 'ts_0516', 'ts_0517', 'ts_0518', 'ts_0519', 'ts_0520', 'ts_0522', 'ts_0524', 'ts_0525', 'ts_0527', 'ts_0528', 'ts_0529', 'ts_0530', 'ts_0531', 'ts_0532', 'ts_0533', 'ts_0534', 'ts_0535', 'ts_0537', 'ts_0538', 'ts_0539', 'ts_0541', 'ts_0542', 'ts_0543', 'ts_0544', 'ts_0546', 'ts_0547', 'ts_0548', 'ts_0552', 'ts_0553', 'ts_0554', 'ts_0555', 'ts_0556', 'ts_0557', 'ts_0559', 'ts_0560', 'ts_0561', 'ts_0562', 'ts_0564', 'ts_0566', 'ts_0567', 'ts_0569', 'ts_0570', 'ts_0572', 'ts_0573', 'ts_0574', 'ts_0575', 'ts_0576', 'ts_0577', 'ts_0578', 'ts_0579', 'ts_0580', 'ts_0582', 'ts_0583', 'ts_0584', 'ts_0585', 'ts_0586', 'ts_0587', 'ts_0588', 'ts_0591', 'ts_0592', 'ts_0594', 'ts_0595', 'ts_0597', 'ts_0599', 'ts_0600', 'ts_0601', 'ts_0602', 'ts_0604', 'ts_0605', 'ts_0606', 'ts_0607', 'ts_0608', 'ts_0609', 'ts_0610', 'ts_0612', 'ts_0613', 'ts_0614', 'ts_0615', 'ts_0616', 'ts_0619', 'ts_0620', 'ts_0621', 'ts_0622', 'ts_0623', 'ts_0625', 'ts_0626', 'ts_0627', 'ts_0628', 'ts_0629', 'ts_0630', 'ts_0631', 'ts_0632', 'ts_0633', 'ts_0634', 'ts_0635', 'ts_0636', 'ts_0637', 'ts_0639', 'ts_0640', 'ts_0642', 'ts_0643', 'ts_0644', 'ts_0645', 'ts_0646', 'ts_0647', 'ts_0649', 'ts_0651', 'ts_0652', 'ts_0653', 'ts_0654', 'ts_0656', 'ts_0657', 'ts_0660', 'ts_0661', 'ts_0662', 'ts_0668', 'ts_0669', 'ts_0670', 'ts_0671', 'ts_0672', 'ts_0673', 'ts_0674', 'ts_0676', 'ts_0677', 'ts_0678', 'ts_0679', 'ts_0680', 'ts_0681', 'ts_0683', 'ts_0684', 'ts_0687', 'ts_0688', 'ts_0689', 'ts_0690', 'ts_0691', 'ts_0693', 'ts_0695', 'ts_0696', 'ts_0697', 'ts_0699', 'ts_0700', 'ts_0701', 'ts_0702', 'ts_0703', 'ts_0704', 'ts_0706', 'ts_0707', 'ts_0708', 'ts_0709', 'ts_0711', 'ts_0712', 'ts_0713', 'ts_0714', 'ts_0715', 'ts_0717', 'ts_0718', 'ts_0719', 'ts_0720', 'ts_0721', 'ts_0722', 'ts_0723', 'ts_0724', 'ts_0725', 'ts_0726', 'ts_0727', 'ts_0729', 'ts_0730', 'ts_0731', 'ts_0734', 'ts_0735', 'ts_0737', 'ts_0738', 'ts_0739', 'ts_0740', 'ts_0741', 'ts_0742', 'ts_0743', 'ts_0744', 'ts_0745', 'ts_0747', 'ts_0748', 'ts_0749', 'ts_0750', 'ts_0751', 'ts_0752', 'ts_0753', 'ts_0754', 'ts_0755', 'ts_0756', 'ts_0757', 'ts_0758', 'ts_0759', 'ts_0760', 'ts_0761', 'ts_0762', 'ts_0764', 'ts_0765', 'ts_0766', 'ts_0769', 'ts_0770', 'ts_0771', 'ts_0772', 'ts_0773', 'ts_0774', 'ts_0775', 'ts_0776', 'ts_0777', 'ts_0778', 'ts_0779', 'ts_0780', 'ts_0781', 'ts_0782', 'ts_0783', 'ts_0784', 'ts_0786', 'ts_0787', 'ts_0788', 'ts_0789', 'ts_0790', 'ts_0791', 'ts_0792', 'ts_0796', 'ts_0797', 'ts_0798', 'ts_0799', 'ts_0800', 'ts_0801', 'ts_0802', 'ts_0803', 'ts_0804', 'ts_0805', 'ts_0807', 'ts_0810', 'ts_0811', 'ts_0812', 'ts_0813', 'ts_0815', 'ts_0816', 'ts_0817', 'ts_0818', 'ts_0819', 'ts_0820', 'ts_0821', 'ts_0822', 'ts_0823', 'ts_0824', 'ts_0825', 'ts_0827', 'ts_0828', 'ts_0830', 'ts_0831', 'ts_0832', 'ts_0833', 'ts_0834', 'ts_0835', 'ts_0836', 'ts_0837', 'ts_0839', 'ts_0841', 'ts_0842', 'ts_0843', 'ts_0844', 'ts_0845', 'ts_0846', 'ts_0847', 'ts_0850', 'ts_0851', 'ts_0852', 'ts_0853', 'ts_0855', 'ts_0856', 'ts_0858', 'ts_0859', 'ts_0860', 'ts_0861', 'ts_0862', 'ts_0863', 'ts_0864', 'ts_0868', 'ts_0869', 'ts_0870', 'ts_0873', 'ts_0874', 'ts_0875', 'ts_0876', 'ts_0878', 'ts_0879', 'ts_0880', 'ts_0881', 'ts_0882', 'ts_0883', 'ts_0884', 'ts_0885', 'ts_0886', 'ts_0888', 'ts_0889', 'ts_0890', 'ts_0891', 'ts_0892', 'ts_0893', 'ts_0894', 'ts_0895', 'ts_0897', 'ts_0898', 'ts_0899', 'ts_0900', 'ts_0902', 'ts_0903', 'ts_0904', 'ts_0906', 'ts_0907', 'ts_0908', 'ts_0909', 'ts_0910', 'ts_0911', 'ts_0912', 'ts_0913', 'ts_0914', 'ts_0916', 'ts_0917', 'ts_0918', 'ts_0919', 'ts_0920', 'ts_0921', 'ts_0922', 'ts_0923', 'ts_0924', 'ts_0926', 'ts_0928', 'ts_0929', 'ts_0930', 'ts_0931', 'ts_0932', 'ts_0933', 'ts_0934', 'ts_0935', 'ts_0936', 'ts_0937', 'ts_0939', 'ts_0941', 'ts_0942', 'ts_0944', 'ts_0945', 'ts_0946', 'ts_0947', 'ts_0948', 'ts_0949', 'ts_0950', 'ts_0951', 'ts_0952', 'ts_0953', 'ts_0956', 'ts_0957', 'ts_0958', 'ts_0960', 'ts_0962', 'ts_0964', 'ts_0967', 'ts_0968', 'ts_0969', 'ts_0970', 'ts_0971', 'ts_0972', 'ts_0973', 'ts_0974', 'ts_0975', 'ts_0976', 'ts_0977', 'ts_0978', 'ts_0979', 'ts_0981', 'ts_0982', 'ts_0983', 'ts_0984', 'ts_0985', 'ts_0986', 'ts_0988', 'ts_0989', 'ts_0990', 'ts_0991', 'ts_0992', 'ts_0995', 'ts_0996', 'ts_0997', 'ts_0999', 'ts_1000', 'ts_1002', 'ts_1003', 'ts_1004', 'ts_1005', 'ts_1006', 'ts_1008', 'ts_1009', 'ts_1010', 'ts_1011', 'ts_1013', 'ts_1014', 'ts_1015', 'ts_1016', 'ts_1017', 'ts_1018', 'ts_1020', 'ts_1021', 'ts_1023', 'ts_1024', 'ts_1025', 'ts_1026', 'ts_1027', 'ts_1029', 'ts_1030', 'ts_1031', 'ts_1032', 'ts_1033', 'ts_1034', 'ts_1035', 'ts_1036', 'ts_1037', 'ts_1038', 'ts_1039', 'ts_1040', 'ts_1041', 'ts_1043', 'ts_1044', 'ts_1045', 'ts_1048', 'ts_1050', 'ts_1051', 'ts_1052', 'ts_1053', 'ts_1054', 'ts_1055', 'ts_1056', 'ts_1058', 'ts_1059', 'ts_1060', 'ts_1061', 'ts_1062', 'ts_1063', 'ts_1064', 'ts_1066', 'ts_1068', 'ts_1070', 'ts_1071', 'ts_1072', 'ts_1074', 'ts_1075', 'ts_1077', 'ts_1079', 'ts_1080', 'ts_1081', 'ts_1082', 'ts_1083', 'ts_1084', 'ts_1086', 'ts_1087', 'ts_1088', 'ts_1089', 'ts_1090', 'ts_1093', 'ts_1094', 'ts_1096', 'ts_1098', 'ts_1099', 'ts_1100', 'ts_1101', 'ts_1102', 'ts_1103', 'ts_1104', 'ts_1106', 'ts_1107', 'ts_1109', 'ts_1110', 'ts_1111', 'ts_1112', 'ts_1114', 'ts_1115', 'ts_1116', 'ts_1118', 'ts_1119', 'ts_1120', 'ts_1121', 'ts_1122', 'ts_1123', 'ts_1124', 'ts_1127', 'ts_1128', 'ts_1130', 'ts_1131', 'ts_1132', 'ts_1133', 'ts_1136', 'ts_1137', 'ts_1138', 'ts_1139', 'ts_1140', 'ts_1141', 'ts_1142', 'ts_1143', 'ts_1144', 'ts_1145', 'ts_1146', 'ts_1147', 'ts_1148', 'ts_1149', 'ts_1150', 'ts_1153', 'ts_1154', 'ts_1155', 'ts_1156', 'ts_1157', 'ts_1160', 'ts_1161', 'ts_1163', 'ts_1164', 'ts_1165', 'ts_1166', 'ts_1167', 'ts_1168', 'ts_1169', 'ts_1170', 'ts_1172', 'ts_1174', 'ts_1175', 'ts_1176', 'ts_1177', 'ts_1178', 'ts_1179', 'ts_1180', 'ts_1183', 'ts_1184', 'ts_1186', 'ts_1189', 'ts_1190', 'ts_1191', 'ts_1192', 'ts_1193', 'ts_1194', 'ts_1195', 'ts_1196', 'ts_1197', 'ts_1198', 'ts_1200', 'ts_1202', 'ts_1203', 'verse_0000', 'verse_0001', 'verse_0002', 'verse_0003', 'verse_0004', 'verse_0005', 'verse_0007', 'verse_0009', 'verse_0010', 'verse_0011', 'verse_0012', 'verse_0013', 'verse_0014', 'verse_0016', 'verse_0017', 'verse_0019', 'verse_0020', 'verse_0021', 'verse_0022', 'verse_0023', 'verse_0024', 'verse_0025', 'verse_0026', 'verse_0027', 'verse_0028', 'verse_0029', 'verse_0030', 'verse_0031', 'verse_0032', 'verse_0034', 'verse_0035', 'verse_0036', 'verse_0037', 'verse_0038', 'verse_0039', 'verse_0042', 'verse_0043', 'verse_0045', 'verse_0047', 'verse_0048', 'verse_0051', 'verse_0052', 'verse_0053', 'verse_0054', 'verse_0055', 'verse_0056', 'verse_0058', 'verse_0059', 'verse_0060', 'verse_0062', 'verse_0063', 'verse_0064', 'verse_0065', 'verse_0066', 'verse_0068', 'verse_0069', 'verse_0070', 'verse_0071', 'verse_0072', 'verse_0073', 'verse_0074', 'verse_0075', 'verse_0076', 'verse_0077', 'verse_0078', 'verse_0079', 'verse_0080', 'verse_0081', 'verse_0082', 'verse_0083', 'verse_0084', 'verse_0085', 'verse_0086', 'verse_0087', 'verse_0088', 'verse_0090', 'verse_0091', 'verse_0092', 'verse_0093', 'verse_0095', 'verse_0098', 'verse_0099', 'verse_0100', 'verse_0101', 'verse_0102', 'verse_0104', 'verse_0105', 'verse_0106', 'verse_0107', 'verse_0108', 'verse_0109', 'verse_0110', 'verse_0111', 'verse_0112', 'verse_0113', 'verse_0114', 'verse_0116', 'verse_0121', 'verse_0122', 'verse_0123', 'verse_0124', 'verse_0125', 'verse_0126', 'verse_0127', 'verse_0128', 'verse_0129', 'verse_0130', 'verse_0131', 'verse_0132', 'verse_0133', 'verse_0134', 'verse_0135', 'verse_0136', 'verse_0137', 'verse_0138', 'verse_0139', 'verse_0142', 'verse_0143', 'verse_0144', 'verse_0146', 'verse_0148', 'verse_0149', 'verse_0151', 'verse_0152', 'verse_0153', 'verse_0154', 'verse_0155', 'verse_0156', 'verse_0157', 'verse_0158', 'verse_0160', 'verse_0162', 'verse_0164', 'verse_0166', 'verse_0168', 'verse_0169', 'verse_0170', 'verse_0171', 'verse_0172', 'verse_0173', 'verse_0174', 'verse_0175', 'verse_0176', 'verse_0177', 'verse_0179', 'verse_0180', 'verse_0181', 'verse_0183', 'verse_0184', 'verse_0185', 'verse_0187', 'verse_0188', 'verse_0189', 'verse_0190', 'verse_0191', 'verse_0192', 'verse_0193', 'verse_0194', 'verse_0195', 'verse_0196', 'verse_0197', 'verse_0198', 'verse_0199', 'verse_0200', 'verse_0201', 'verse_0202', 'verse_0203', 'verse_0204', 'verse_0205', 'verse_0206', 'verse_0207', 'verse_0208', 'verse_0210', 'verse_0212', 'verse_0214', 'verse_0215', 'verse_0216', 'verse_0217', 'verse_0218', 'verse_0219', 'verse_0220', 'verse_0221', 'verse_0223', 'verse_0224', 'verse_0225', 'verse_0226', 'verse_0227', 'verse_0229', 'verse_0230', 'verse_0231', 'verse_0232', 'verse_0234', 'verse_0235', 'verse_0236', 'verse_0237', 'verse_0238', 'verse_0239', 'verse_0240', 'verse_0243', 'verse_0245', 'verse_0247', 'verse_0248', 'verse_0249', 'verse_0250', 'verse_0251', 'verse_0252', 'verse_0254', 'verse_0255', 'verse_0256', 'verse_0257', 'verse_0259', 'verse_0260', 'verse_0261', 'verse_0262', 'verse_0263', 'verse_0265', 'verse_0266', 'verse_0267', 'verse_0268', 'verse_0270', 'verse_0271', 'verse_0272', 'verse_0273', 'verse_0274', 'verse_0275', 'verse_0276', 'verse_0278', 'verse_0280', 'verse_0281', 'verse_0282', 'verse_0283', 'verse_0284', 'verse_0286', 'verse_0287', 'verse_0289', 'verse_0290', 'verse_0291', 'verse_0292', 'verse_0293', 'verse_0294', 'verse_0295', 'verse_0296', 'verse_0297', 'verse_0298', 'verse_0299', 'verse_0302', 'verse_0303', 'verse_0304', 'verse_0305', 'verse_0306', 'verse_0307', 'verse_0308', 'verse_0309', 'verse_0310', 'verse_0311', 'verse_0312', 'verse_0313', 'verse_0314', 'verse_0315', 'verse_0316', 'verse_0317', 'verse_0318', 'verse_0319', 'verse_0320', 'verse_0321', 'verse_0322', 'verse_0323', 'verse_0324', 'verse_0325', 'verse_0326', 'verse_0327', 'verse_0328', 'verse_0329', 'verse_0332', 'verse_0333', 'verse_0334', 'verse_0335', 'verse_0338', 'verse_0339', 'verse_0340', 'verse_0341', 'verse_0342', 'verse_0344', 'verse_0345', 'verse_0346', 'verse_0347', 'verse_0349', 'verse_0351', 'verse_0352', 'verse_0353', 'verse_0355', 'verse_0357', 'verse_0358', 'verse_0359', 'verse_0360', 'verse_0361', 'verse_0363', 'verse_0364', 'verse_0365', 'verse_0366', 'verse_0367', 'verse_0368', 'verse_0369', 'verse_0370', 'verse_0371', 'verse_0372', 'verse_0373']) 
+2023-04-03 21:31:09.135754: VALIDATION KEYS:
+ odict_keys(['ts_0000', 'ts_0002', 'ts_0003', 'ts_0008', 'ts_0017', 'ts_0020', 'ts_0024', 'ts_0029', 'ts_0030', 'ts_0052', 'ts_0056', 'ts_0064', 'ts_0070', 'ts_0072', 'ts_0073', 'ts_0076', 'ts_0087', 'ts_0092', 'ts_0098', 'ts_0101', 'ts_0104', 'ts_0111', 'ts_0115', 'ts_0127', 'ts_0131', 'ts_0136', 'ts_0137', 'ts_0147', 'ts_0149', 'ts_0150', 'ts_0151', 'ts_0153', 'ts_0154', 'ts_0162', 'ts_0164', 'ts_0167', 'ts_0180', 'ts_0187', 'ts_0188', 'ts_0190', 'ts_0193', 'ts_0198', 'ts_0201', 'ts_0206', 'ts_0207', 'ts_0208', 'ts_0219', 'ts_0220', 'ts_0221', 'ts_0223', 'ts_0227', 'ts_0229', 'ts_0238', 'ts_0243', 'ts_0260', 'ts_0262', 'ts_0264', 'ts_0266', 'ts_0268', 'ts_0277', 'ts_0281', 'ts_0282', 'ts_0283', 'ts_0284', 'ts_0294', 'ts_0298', 'ts_0302', 'ts_0308', 'ts_0311', 'ts_0314', 'ts_0317', 'ts_0324', 'ts_0328', 'ts_0335', 'ts_0338', 'ts_0347', 'ts_0357', 'ts_0358', 'ts_0361', 'ts_0372', 'ts_0380', 'ts_0383', 'ts_0390', 'ts_0391', 'ts_0392', 'ts_0394', 'ts_0395', 'ts_0403', 'ts_0413', 'ts_0418', 'ts_0423', 'ts_0426', 'ts_0427', 'ts_0428', 'ts_0436', 'ts_0455', 'ts_0456', 'ts_0463', 'ts_0465', 'ts_0471', 'ts_0486', 'ts_0499', 'ts_0510', 'ts_0511', 'ts_0513', 'ts_0521', 'ts_0523', 'ts_0526', 'ts_0536', 'ts_0540', 'ts_0545', 'ts_0549', 'ts_0550', 'ts_0551', 'ts_0558', 'ts_0563', 'ts_0565', 'ts_0568', 'ts_0571', 'ts_0581', 'ts_0589', 'ts_0590', 'ts_0593', 'ts_0596', 'ts_0598', 'ts_0603', 'ts_0611', 'ts_0617', 'ts_0618', 'ts_0624', 'ts_0638', 'ts_0641', 'ts_0648', 'ts_0650', 'ts_0655', 'ts_0658', 'ts_0659', 'ts_0663', 'ts_0664', 'ts_0665', 'ts_0666', 'ts_0667', 'ts_0675', 'ts_0682', 'ts_0685', 'ts_0686', 'ts_0692', 'ts_0694', 'ts_0698', 'ts_0705', 'ts_0710', 'ts_0716', 'ts_0728', 'ts_0732', 'ts_0733', 'ts_0736', 'ts_0746', 'ts_0763', 'ts_0767', 'ts_0768', 'ts_0785', 'ts_0793', 'ts_0794', 'ts_0795', 'ts_0806', 'ts_0808', 'ts_0809', 'ts_0814', 'ts_0826', 'ts_0829', 'ts_0838', 'ts_0840', 'ts_0848', 'ts_0849', 'ts_0854', 'ts_0857', 'ts_0865', 'ts_0866', 'ts_0867', 'ts_0871', 'ts_0872', 'ts_0877', 'ts_0887', 'ts_0896', 'ts_0901', 'ts_0905', 'ts_0915', 'ts_0925', 'ts_0927', 'ts_0938', 'ts_0940', 'ts_0943', 'ts_0954', 'ts_0955', 'ts_0959', 'ts_0961', 'ts_0963', 'ts_0965', 'ts_0966', 'ts_0980', 'ts_0987', 'ts_0993', 'ts_0994', 'ts_0998', 'ts_1001', 'ts_1007', 'ts_1012', 'ts_1019', 'ts_1022', 'ts_1028', 'ts_1042', 'ts_1046', 'ts_1047', 'ts_1049', 'ts_1057', 'ts_1065', 'ts_1067', 'ts_1069', 'ts_1073', 'ts_1076', 'ts_1078', 'ts_1085', 'ts_1091', 'ts_1092', 'ts_1095', 'ts_1097', 'ts_1105', 'ts_1108', 'ts_1113', 'ts_1117', 'ts_1125', 'ts_1126', 'ts_1129', 'ts_1134', 'ts_1135', 'ts_1151', 'ts_1152', 'ts_1158', 'ts_1159', 'ts_1162', 'ts_1171', 'ts_1173', 'ts_1181', 'ts_1182', 'ts_1185', 'ts_1187', 'ts_1188', 'ts_1199', 'ts_1201', 'verse_0006', 'verse_0008', 'verse_0015', 'verse_0018', 'verse_0033', 'verse_0040', 'verse_0041', 'verse_0044', 'verse_0046', 'verse_0049', 'verse_0050', 'verse_0057', 'verse_0061', 'verse_0067', 'verse_0089', 'verse_0094', 'verse_0096', 'verse_0097', 'verse_0103', 'verse_0115', 'verse_0117', 'verse_0118', 'verse_0119', 'verse_0120', 'verse_0140', 'verse_0141', 'verse_0145', 'verse_0147', 'verse_0150', 'verse_0159', 'verse_0161', 'verse_0163', 'verse_0165', 'verse_0167', 'verse_0178', 'verse_0182', 'verse_0186', 'verse_0209', 'verse_0211', 'verse_0213', 'verse_0222', 'verse_0228', 'verse_0233', 'verse_0241', 'verse_0242', 'verse_0244', 'verse_0246', 'verse_0253', 'verse_0258', 'verse_0264', 'verse_0269', 'verse_0277', 'verse_0279', 'verse_0285', 'verse_0288', 'verse_0300', 'verse_0301', 'verse_0330', 'verse_0331', 'verse_0336', 'verse_0337', 'verse_0343', 'verse_0348', 'verse_0350', 'verse_0354', 'verse_0356', 'verse_0362']) 

fold_0/training_log_2023_4_3_21_32_50.txt

@@ -0,0 +1,680 @@
+Starting... 
+2023-04-03 21:32:50.367352: Using splits from existing split file: /dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/splits_final.pkl 
+2023-04-03 21:32:50.373400: The split file contains 5 splits. 
+2023-04-03 21:32:50.374626: Desired fold for training: 0 
+2023-04-03 21:32:50.375816: This split has 1262 training and 316 validation cases. 
+2023-04-03 21:32:54.623561: TRAINING KEYS:
+ odict_keys(['ts_0001', 'ts_0004', 'ts_0005', 'ts_0006', 'ts_0007', 'ts_0009', 'ts_0010', 'ts_0011', 'ts_0012', 'ts_0013', 'ts_0014', 'ts_0015', 'ts_0016', 'ts_0018', 'ts_0019', 'ts_0021', 'ts_0022', 'ts_0023', 'ts_0025', 'ts_0026', 'ts_0027', 'ts_0028', 'ts_0031', 'ts_0032', 'ts_0033', 'ts_0034', 'ts_0035', 'ts_0036', 'ts_0037', 'ts_0038', 'ts_0039', 'ts_0040', 'ts_0041', 'ts_0042', 'ts_0043', 'ts_0044', 'ts_0045', 'ts_0046', 'ts_0047', 'ts_0048', 'ts_0049', 'ts_0050', 'ts_0051', 'ts_0053', 'ts_0054', 'ts_0055', 'ts_0057', 'ts_0058', 'ts_0059', 'ts_0060', 'ts_0061', 'ts_0062', 'ts_0063', 'ts_0065', 'ts_0066', 'ts_0067', 'ts_0068', 'ts_0069', 'ts_0071', 'ts_0074', 'ts_0075', 'ts_0077', 'ts_0078', 'ts_0079', 'ts_0080', 'ts_0081', 'ts_0082', 'ts_0083', 'ts_0084', 'ts_0085', 'ts_0086', 'ts_0088', 'ts_0089', 'ts_0090', 'ts_0091', 'ts_0093', 'ts_0094', 'ts_0095', 'ts_0096', 'ts_0097', 'ts_0099', 'ts_0100', 'ts_0102', 'ts_0103', 'ts_0105', 'ts_0106', 'ts_0107', 'ts_0108', 'ts_0109', 'ts_0110', 'ts_0112', 'ts_0113', 'ts_0114', 'ts_0116', 'ts_0117', 'ts_0118', 'ts_0119', 'ts_0120', 'ts_0121', 'ts_0122', 'ts_0123', 'ts_0124', 'ts_0125', 'ts_0126', 'ts_0128', 'ts_0129', 'ts_0130', 'ts_0132', 'ts_0133', 'ts_0134', 'ts_0135', 'ts_0138', 'ts_0139', 'ts_0140', 'ts_0141', 'ts_0142', 'ts_0143', 'ts_0144', 'ts_0145', 'ts_0146', 'ts_0148', 'ts_0152', 'ts_0155', 'ts_0156', 'ts_0157', 'ts_0158', 'ts_0159', 'ts_0160', 'ts_0161', 'ts_0163', 'ts_0165', 'ts_0166', 'ts_0168', 'ts_0169', 'ts_0170', 'ts_0171', 'ts_0172', 'ts_0173', 'ts_0174', 'ts_0175', 'ts_0176', 'ts_0177', 'ts_0178', 'ts_0179', 'ts_0181', 'ts_0182', 'ts_0183', 'ts_0184', 'ts_0185', 'ts_0186', 'ts_0189', 'ts_0191', 'ts_0192', 'ts_0194', 'ts_0195', 'ts_0196', 'ts_0197', 'ts_0199', 'ts_0200', 'ts_0202', 'ts_0203', 'ts_0204', 'ts_0205', 'ts_0209', 'ts_0210', 'ts_0211', 'ts_0212', 'ts_0213', 'ts_0214', 'ts_0215', 'ts_0216', 'ts_0217', 'ts_0218', 'ts_0222', 'ts_0224', 'ts_0225', 'ts_0226', 'ts_0228', 'ts_0230', 'ts_0231', 'ts_0232', 'ts_0233', 'ts_0234', 'ts_0235', 'ts_0236', 'ts_0237', 'ts_0239', 'ts_0240', 'ts_0241', 'ts_0242', 'ts_0244', 'ts_0245', 'ts_0246', 'ts_0247', 'ts_0248', 'ts_0249', 'ts_0250', 'ts_0251', 'ts_0252', 'ts_0253', 'ts_0254', 'ts_0255', 'ts_0256', 'ts_0257', 'ts_0258', 'ts_0259', 'ts_0261', 'ts_0263', 'ts_0265', 'ts_0267', 'ts_0269', 'ts_0270', 'ts_0271', 'ts_0272', 'ts_0273', 'ts_0274', 'ts_0275', 'ts_0276', 'ts_0278', 'ts_0279', 'ts_0280', 'ts_0285', 'ts_0286', 'ts_0287', 'ts_0288', 'ts_0289', 'ts_0290', 'ts_0291', 'ts_0292', 'ts_0293', 'ts_0295', 'ts_0296', 'ts_0297', 'ts_0299', 'ts_0300', 'ts_0301', 'ts_0303', 'ts_0304', 'ts_0305', 'ts_0306', 'ts_0307', 'ts_0309', 'ts_0310', 'ts_0312', 'ts_0313', 'ts_0315', 'ts_0316', 'ts_0318', 'ts_0319', 'ts_0320', 'ts_0321', 'ts_0322', 'ts_0323', 'ts_0325', 'ts_0326', 'ts_0327', 'ts_0329', 'ts_0330', 'ts_0331', 'ts_0332', 'ts_0333', 'ts_0334', 'ts_0336', 'ts_0337', 'ts_0339', 'ts_0340', 'ts_0341', 'ts_0342', 'ts_0343', 'ts_0344', 'ts_0345', 'ts_0346', 'ts_0348', 'ts_0349', 'ts_0350', 'ts_0351', 'ts_0352', 'ts_0353', 'ts_0354', 'ts_0355', 'ts_0356', 'ts_0359', 'ts_0360', 'ts_0362', 'ts_0363', 'ts_0364', 'ts_0365', 'ts_0366', 'ts_0367', 'ts_0368', 'ts_0369', 'ts_0370', 'ts_0371', 'ts_0373', 'ts_0374', 'ts_0375', 'ts_0376', 'ts_0377', 'ts_0378', 'ts_0379', 'ts_0381', 'ts_0382', 'ts_0384', 'ts_0385', 'ts_0386', 'ts_0387', 'ts_0388', 'ts_0389', 'ts_0393', 'ts_0396', 'ts_0397', 'ts_0398', 'ts_0399', 'ts_0400', 'ts_0401', 'ts_0402', 'ts_0404', 'ts_0405', 'ts_0406', 'ts_0407', 'ts_0408', 'ts_0409', 'ts_0410', 'ts_0411', 'ts_0412', 'ts_0414', 'ts_0415', 'ts_0416', 'ts_0417', 'ts_0419', 'ts_0420', 'ts_0421', 'ts_0422', 'ts_0424', 'ts_0425', 'ts_0429', 'ts_0430', 'ts_0431', 'ts_0432', 'ts_0433', 'ts_0434', 'ts_0435', 'ts_0437', 'ts_0438', 'ts_0439', 'ts_0440', 'ts_0441', 'ts_0442', 'ts_0443', 'ts_0444', 'ts_0445', 'ts_0446', 'ts_0447', 'ts_0448', 'ts_0449', 'ts_0450', 'ts_0451', 'ts_0452', 'ts_0453', 'ts_0454', 'ts_0457', 'ts_0458', 'ts_0459', 'ts_0460', 'ts_0461', 'ts_0462', 'ts_0464', 'ts_0466', 'ts_0467', 'ts_0468', 'ts_0469', 'ts_0470', 'ts_0472', 'ts_0473', 'ts_0474', 'ts_0475', 'ts_0476', 'ts_0477', 'ts_0478', 'ts_0479', 'ts_0480', 'ts_0481', 'ts_0482', 'ts_0483', 'ts_0484', 'ts_0485', 'ts_0487', 'ts_0488', 'ts_0489', 'ts_0490', 'ts_0491', 'ts_0492', 'ts_0493', 'ts_0494', 'ts_0495', 'ts_0496', 'ts_0497', 'ts_0498', 'ts_0500', 'ts_0501', 'ts_0502', 'ts_0503', 'ts_0504', 'ts_0505', 'ts_0506', 'ts_0507', 'ts_0508', 'ts_0509', 'ts_0512', 'ts_0514', 'ts_0515', 'ts_0516', 'ts_0517', 'ts_0518', 'ts_0519', 'ts_0520', 'ts_0522', 'ts_0524', 'ts_0525', 'ts_0527', 'ts_0528', 'ts_0529', 'ts_0530', 'ts_0531', 'ts_0532', 'ts_0533', 'ts_0534', 'ts_0535', 'ts_0537', 'ts_0538', 'ts_0539', 'ts_0541', 'ts_0542', 'ts_0543', 'ts_0544', 'ts_0546', 'ts_0547', 'ts_0548', 'ts_0552', 'ts_0553', 'ts_0554', 'ts_0555', 'ts_0556', 'ts_0557', 'ts_0559', 'ts_0560', 'ts_0561', 'ts_0562', 'ts_0564', 'ts_0566', 'ts_0567', 'ts_0569', 'ts_0570', 'ts_0572', 'ts_0573', 'ts_0574', 'ts_0575', 'ts_0576', 'ts_0577', 'ts_0578', 'ts_0579', 'ts_0580', 'ts_0582', 'ts_0583', 'ts_0584', 'ts_0585', 'ts_0586', 'ts_0587', 'ts_0588', 'ts_0591', 'ts_0592', 'ts_0594', 'ts_0595', 'ts_0597', 'ts_0599', 'ts_0600', 'ts_0601', 'ts_0602', 'ts_0604', 'ts_0605', 'ts_0606', 'ts_0607', 'ts_0608', 'ts_0609', 'ts_0610', 'ts_0612', 'ts_0613', 'ts_0614', 'ts_0615', 'ts_0616', 'ts_0619', 'ts_0620', 'ts_0621', 'ts_0622', 'ts_0623', 'ts_0625', 'ts_0626', 'ts_0627', 'ts_0628', 'ts_0629', 'ts_0630', 'ts_0631', 'ts_0632', 'ts_0633', 'ts_0634', 'ts_0635', 'ts_0636', 'ts_0637', 'ts_0639', 'ts_0640', 'ts_0642', 'ts_0643', 'ts_0644', 'ts_0645', 'ts_0646', 'ts_0647', 'ts_0649', 'ts_0651', 'ts_0652', 'ts_0653', 'ts_0654', 'ts_0656', 'ts_0657', 'ts_0660', 'ts_0661', 'ts_0662', 'ts_0668', 'ts_0669', 'ts_0670', 'ts_0671', 'ts_0672', 'ts_0673', 'ts_0674', 'ts_0676', 'ts_0677', 'ts_0678', 'ts_0679', 'ts_0680', 'ts_0681', 'ts_0683', 'ts_0684', 'ts_0687', 'ts_0688', 'ts_0689', 'ts_0690', 'ts_0691', 'ts_0693', 'ts_0695', 'ts_0696', 'ts_0697', 'ts_0699', 'ts_0700', 'ts_0701', 'ts_0702', 'ts_0703', 'ts_0704', 'ts_0706', 'ts_0707', 'ts_0708', 'ts_0709', 'ts_0711', 'ts_0712', 'ts_0713', 'ts_0714', 'ts_0715', 'ts_0717', 'ts_0718', 'ts_0719', 'ts_0720', 'ts_0721', 'ts_0722', 'ts_0723', 'ts_0724', 'ts_0725', 'ts_0726', 'ts_0727', 'ts_0729', 'ts_0730', 'ts_0731', 'ts_0734', 'ts_0735', 'ts_0737', 'ts_0738', 'ts_0739', 'ts_0740', 'ts_0741', 'ts_0742', 'ts_0743', 'ts_0744', 'ts_0745', 'ts_0747', 'ts_0748', 'ts_0749', 'ts_0750', 'ts_0751', 'ts_0752', 'ts_0753', 'ts_0754', 'ts_0755', 'ts_0756', 'ts_0757', 'ts_0758', 'ts_0759', 'ts_0760', 'ts_0761', 'ts_0762', 'ts_0764', 'ts_0765', 'ts_0766', 'ts_0769', 'ts_0770', 'ts_0771', 'ts_0772', 'ts_0773', 'ts_0774', 'ts_0775', 'ts_0776', 'ts_0777', 'ts_0778', 'ts_0779', 'ts_0780', 'ts_0781', 'ts_0782', 'ts_0783', 'ts_0784', 'ts_0786', 'ts_0787', 'ts_0788', 'ts_0789', 'ts_0790', 'ts_0791', 'ts_0792', 'ts_0796', 'ts_0797', 'ts_0798', 'ts_0799', 'ts_0800', 'ts_0801', 'ts_0802', 'ts_0803', 'ts_0804', 'ts_0805', 'ts_0807', 'ts_0810', 'ts_0811', 'ts_0812', 'ts_0813', 'ts_0815', 'ts_0816', 'ts_0817', 'ts_0818', 'ts_0819', 'ts_0820', 'ts_0821', 'ts_0822', 'ts_0823', 'ts_0824', 'ts_0825', 'ts_0827', 'ts_0828', 'ts_0830', 'ts_0831', 'ts_0832', 'ts_0833', 'ts_0834', 'ts_0835', 'ts_0836', 'ts_0837', 'ts_0839', 'ts_0841', 'ts_0842', 'ts_0843', 'ts_0844', 'ts_0845', 'ts_0846', 'ts_0847', 'ts_0850', 'ts_0851', 'ts_0852', 'ts_0853', 'ts_0855', 'ts_0856', 'ts_0858', 'ts_0859', 'ts_0860', 'ts_0861', 'ts_0862', 'ts_0863', 'ts_0864', 'ts_0868', 'ts_0869', 'ts_0870', 'ts_0873', 'ts_0874', 'ts_0875', 'ts_0876', 'ts_0878', 'ts_0879', 'ts_0880', 'ts_0881', 'ts_0882', 'ts_0883', 'ts_0884', 'ts_0885', 'ts_0886', 'ts_0888', 'ts_0889', 'ts_0890', 'ts_0891', 'ts_0892', 'ts_0893', 'ts_0894', 'ts_0895', 'ts_0897', 'ts_0898', 'ts_0899', 'ts_0900', 'ts_0902', 'ts_0903', 'ts_0904', 'ts_0906', 'ts_0907', 'ts_0908', 'ts_0909', 'ts_0910', 'ts_0911', 'ts_0912', 'ts_0913', 'ts_0914', 'ts_0916', 'ts_0917', 'ts_0918', 'ts_0919', 'ts_0920', 'ts_0921', 'ts_0922', 'ts_0923', 'ts_0924', 'ts_0926', 'ts_0928', 'ts_0929', 'ts_0930', 'ts_0931', 'ts_0932', 'ts_0933', 'ts_0934', 'ts_0935', 'ts_0936', 'ts_0937', 'ts_0939', 'ts_0941', 'ts_0942', 'ts_0944', 'ts_0945', 'ts_0946', 'ts_0947', 'ts_0948', 'ts_0949', 'ts_0950', 'ts_0951', 'ts_0952', 'ts_0953', 'ts_0956', 'ts_0957', 'ts_0958', 'ts_0960', 'ts_0962', 'ts_0964', 'ts_0967', 'ts_0968', 'ts_0969', 'ts_0970', 'ts_0971', 'ts_0972', 'ts_0973', 'ts_0974', 'ts_0975', 'ts_0976', 'ts_0977', 'ts_0978', 'ts_0979', 'ts_0981', 'ts_0982', 'ts_0983', 'ts_0984', 'ts_0985', 'ts_0986', 'ts_0988', 'ts_0989', 'ts_0990', 'ts_0991', 'ts_0992', 'ts_0995', 'ts_0996', 'ts_0997', 'ts_0999', 'ts_1000', 'ts_1002', 'ts_1003', 'ts_1004', 'ts_1005', 'ts_1006', 'ts_1008', 'ts_1009', 'ts_1010', 'ts_1011', 'ts_1013', 'ts_1014', 'ts_1015', 'ts_1016', 'ts_1017', 'ts_1018', 'ts_1020', 'ts_1021', 'ts_1023', 'ts_1024', 'ts_1025', 'ts_1026', 'ts_1027', 'ts_1029', 'ts_1030', 'ts_1031', 'ts_1032', 'ts_1033', 'ts_1034', 'ts_1035', 'ts_1036', 'ts_1037', 'ts_1038', 'ts_1039', 'ts_1040', 'ts_1041', 'ts_1043', 'ts_1044', 'ts_1045', 'ts_1048', 'ts_1050', 'ts_1051', 'ts_1052', 'ts_1053', 'ts_1054', 'ts_1055', 'ts_1056', 'ts_1058', 'ts_1059', 'ts_1060', 'ts_1061', 'ts_1062', 'ts_1063', 'ts_1064', 'ts_1066', 'ts_1068', 'ts_1070', 'ts_1071', 'ts_1072', 'ts_1074', 'ts_1075', 'ts_1077', 'ts_1079', 'ts_1080', 'ts_1081', 'ts_1082', 'ts_1083', 'ts_1084', 'ts_1086', 'ts_1087', 'ts_1088', 'ts_1089', 'ts_1090', 'ts_1093', 'ts_1094', 'ts_1096', 'ts_1098', 'ts_1099', 'ts_1100', 'ts_1101', 'ts_1102', 'ts_1103', 'ts_1104', 'ts_1106', 'ts_1107', 'ts_1109', 'ts_1110', 'ts_1111', 'ts_1112', 'ts_1114', 'ts_1115', 'ts_1116', 'ts_1118', 'ts_1119', 'ts_1120', 'ts_1121', 'ts_1122', 'ts_1123', 'ts_1124', 'ts_1127', 'ts_1128', 'ts_1130', 'ts_1131', 'ts_1132', 'ts_1133', 'ts_1136', 'ts_1137', 'ts_1138', 'ts_1139', 'ts_1140', 'ts_1141', 'ts_1142', 'ts_1143', 'ts_1144', 'ts_1145', 'ts_1146', 'ts_1147', 'ts_1148', 'ts_1149', 'ts_1150', 'ts_1153', 'ts_1154', 'ts_1155', 'ts_1156', 'ts_1157', 'ts_1160', 'ts_1161', 'ts_1163', 'ts_1164', 'ts_1165', 'ts_1166', 'ts_1167', 'ts_1168', 'ts_1169', 'ts_1170', 'ts_1172', 'ts_1174', 'ts_1175', 'ts_1176', 'ts_1177', 'ts_1178', 'ts_1179', 'ts_1180', 'ts_1183', 'ts_1184', 'ts_1186', 'ts_1189', 'ts_1190', 'ts_1191', 'ts_1192', 'ts_1193', 'ts_1194', 'ts_1195', 'ts_1196', 'ts_1197', 'ts_1198', 'ts_1200', 'ts_1202', 'ts_1203', 'verse_0000', 'verse_0001', 'verse_0002', 'verse_0003', 'verse_0004', 'verse_0005', 'verse_0007', 'verse_0009', 'verse_0010', 'verse_0011', 'verse_0012', 'verse_0013', 'verse_0014', 'verse_0016', 'verse_0017', 'verse_0019', 'verse_0020', 'verse_0021', 'verse_0022', 'verse_0023', 'verse_0024', 'verse_0025', 'verse_0026', 'verse_0027', 'verse_0028', 'verse_0029', 'verse_0030', 'verse_0031', 'verse_0032', 'verse_0034', 'verse_0035', 'verse_0036', 'verse_0037', 'verse_0038', 'verse_0039', 'verse_0042', 'verse_0043', 'verse_0045', 'verse_0047', 'verse_0048', 'verse_0051', 'verse_0052', 'verse_0053', 'verse_0054', 'verse_0055', 'verse_0056', 'verse_0058', 'verse_0059', 'verse_0060', 'verse_0062', 'verse_0063', 'verse_0064', 'verse_0065', 'verse_0066', 'verse_0068', 'verse_0069', 'verse_0070', 'verse_0071', 'verse_0072', 'verse_0073', 'verse_0074', 'verse_0075', 'verse_0076', 'verse_0077', 'verse_0078', 'verse_0079', 'verse_0080', 'verse_0081', 'verse_0082', 'verse_0083', 'verse_0084', 'verse_0085', 'verse_0086', 'verse_0087', 'verse_0088', 'verse_0090', 'verse_0091', 'verse_0092', 'verse_0093', 'verse_0095', 'verse_0098', 'verse_0099', 'verse_0100', 'verse_0101', 'verse_0102', 'verse_0104', 'verse_0105', 'verse_0106', 'verse_0107', 'verse_0108', 'verse_0109', 'verse_0110', 'verse_0111', 'verse_0112', 'verse_0113', 'verse_0114', 'verse_0116', 'verse_0121', 'verse_0122', 'verse_0123', 'verse_0124', 'verse_0125', 'verse_0126', 'verse_0127', 'verse_0128', 'verse_0129', 'verse_0130', 'verse_0131', 'verse_0132', 'verse_0133', 'verse_0134', 'verse_0135', 'verse_0136', 'verse_0137', 'verse_0138', 'verse_0139', 'verse_0142', 'verse_0143', 'verse_0144', 'verse_0146', 'verse_0148', 'verse_0149', 'verse_0151', 'verse_0152', 'verse_0153', 'verse_0154', 'verse_0155', 'verse_0156', 'verse_0157', 'verse_0158', 'verse_0160', 'verse_0162', 'verse_0164', 'verse_0166', 'verse_0168', 'verse_0169', 'verse_0170', 'verse_0171', 'verse_0172', 'verse_0173', 'verse_0174', 'verse_0175', 'verse_0176', 'verse_0177', 'verse_0179', 'verse_0180', 'verse_0181', 'verse_0183', 'verse_0184', 'verse_0185', 'verse_0187', 'verse_0188', 'verse_0189', 'verse_0190', 'verse_0191', 'verse_0192', 'verse_0193', 'verse_0194', 'verse_0195', 'verse_0196', 'verse_0197', 'verse_0198', 'verse_0199', 'verse_0200', 'verse_0201', 'verse_0202', 'verse_0203', 'verse_0204', 'verse_0205', 'verse_0206', 'verse_0207', 'verse_0208', 'verse_0210', 'verse_0212', 'verse_0214', 'verse_0215', 'verse_0216', 'verse_0217', 'verse_0218', 'verse_0219', 'verse_0220', 'verse_0221', 'verse_0223', 'verse_0224', 'verse_0225', 'verse_0226', 'verse_0227', 'verse_0229', 'verse_0230', 'verse_0231', 'verse_0232', 'verse_0234', 'verse_0235', 'verse_0236', 'verse_0237', 'verse_0238', 'verse_0239', 'verse_0240', 'verse_0243', 'verse_0245', 'verse_0247', 'verse_0248', 'verse_0249', 'verse_0250', 'verse_0251', 'verse_0252', 'verse_0254', 'verse_0255', 'verse_0256', 'verse_0257', 'verse_0259', 'verse_0260', 'verse_0261', 'verse_0262', 'verse_0263', 'verse_0265', 'verse_0266', 'verse_0267', 'verse_0268', 'verse_0270', 'verse_0271', 'verse_0272', 'verse_0273', 'verse_0274', 'verse_0275', 'verse_0276', 'verse_0278', 'verse_0280', 'verse_0281', 'verse_0282', 'verse_0283', 'verse_0284', 'verse_0286', 'verse_0287', 'verse_0289', 'verse_0290', 'verse_0291', 'verse_0292', 'verse_0293', 'verse_0294', 'verse_0295', 'verse_0296', 'verse_0297', 'verse_0298', 'verse_0299', 'verse_0302', 'verse_0303', 'verse_0304', 'verse_0305', 'verse_0306', 'verse_0307', 'verse_0308', 'verse_0309', 'verse_0310', 'verse_0311', 'verse_0312', 'verse_0313', 'verse_0314', 'verse_0315', 'verse_0316', 'verse_0317', 'verse_0318', 'verse_0319', 'verse_0320', 'verse_0321', 'verse_0322', 'verse_0323', 'verse_0324', 'verse_0325', 'verse_0326', 'verse_0327', 'verse_0328', 'verse_0329', 'verse_0332', 'verse_0333', 'verse_0334', 'verse_0335', 'verse_0338', 'verse_0339', 'verse_0340', 'verse_0341', 'verse_0342', 'verse_0344', 'verse_0345', 'verse_0346', 'verse_0347', 'verse_0349', 'verse_0351', 'verse_0352', 'verse_0353', 'verse_0355', 'verse_0357', 'verse_0358', 'verse_0359', 'verse_0360', 'verse_0361', 'verse_0363', 'verse_0364', 'verse_0365', 'verse_0366', 'verse_0367', 'verse_0368', 'verse_0369', 'verse_0370', 'verse_0371', 'verse_0372', 'verse_0373']) 
+2023-04-03 21:32:54.625574: VALIDATION KEYS:
+ odict_keys(['ts_0000', 'ts_0002', 'ts_0003', 'ts_0008', 'ts_0017', 'ts_0020', 'ts_0024', 'ts_0029', 'ts_0030', 'ts_0052', 'ts_0056', 'ts_0064', 'ts_0070', 'ts_0072', 'ts_0073', 'ts_0076', 'ts_0087', 'ts_0092', 'ts_0098', 'ts_0101', 'ts_0104', 'ts_0111', 'ts_0115', 'ts_0127', 'ts_0131', 'ts_0136', 'ts_0137', 'ts_0147', 'ts_0149', 'ts_0150', 'ts_0151', 'ts_0153', 'ts_0154', 'ts_0162', 'ts_0164', 'ts_0167', 'ts_0180', 'ts_0187', 'ts_0188', 'ts_0190', 'ts_0193', 'ts_0198', 'ts_0201', 'ts_0206', 'ts_0207', 'ts_0208', 'ts_0219', 'ts_0220', 'ts_0221', 'ts_0223', 'ts_0227', 'ts_0229', 'ts_0238', 'ts_0243', 'ts_0260', 'ts_0262', 'ts_0264', 'ts_0266', 'ts_0268', 'ts_0277', 'ts_0281', 'ts_0282', 'ts_0283', 'ts_0284', 'ts_0294', 'ts_0298', 'ts_0302', 'ts_0308', 'ts_0311', 'ts_0314', 'ts_0317', 'ts_0324', 'ts_0328', 'ts_0335', 'ts_0338', 'ts_0347', 'ts_0357', 'ts_0358', 'ts_0361', 'ts_0372', 'ts_0380', 'ts_0383', 'ts_0390', 'ts_0391', 'ts_0392', 'ts_0394', 'ts_0395', 'ts_0403', 'ts_0413', 'ts_0418', 'ts_0423', 'ts_0426', 'ts_0427', 'ts_0428', 'ts_0436', 'ts_0455', 'ts_0456', 'ts_0463', 'ts_0465', 'ts_0471', 'ts_0486', 'ts_0499', 'ts_0510', 'ts_0511', 'ts_0513', 'ts_0521', 'ts_0523', 'ts_0526', 'ts_0536', 'ts_0540', 'ts_0545', 'ts_0549', 'ts_0550', 'ts_0551', 'ts_0558', 'ts_0563', 'ts_0565', 'ts_0568', 'ts_0571', 'ts_0581', 'ts_0589', 'ts_0590', 'ts_0593', 'ts_0596', 'ts_0598', 'ts_0603', 'ts_0611', 'ts_0617', 'ts_0618', 'ts_0624', 'ts_0638', 'ts_0641', 'ts_0648', 'ts_0650', 'ts_0655', 'ts_0658', 'ts_0659', 'ts_0663', 'ts_0664', 'ts_0665', 'ts_0666', 'ts_0667', 'ts_0675', 'ts_0682', 'ts_0685', 'ts_0686', 'ts_0692', 'ts_0694', 'ts_0698', 'ts_0705', 'ts_0710', 'ts_0716', 'ts_0728', 'ts_0732', 'ts_0733', 'ts_0736', 'ts_0746', 'ts_0763', 'ts_0767', 'ts_0768', 'ts_0785', 'ts_0793', 'ts_0794', 'ts_0795', 'ts_0806', 'ts_0808', 'ts_0809', 'ts_0814', 'ts_0826', 'ts_0829', 'ts_0838', 'ts_0840', 'ts_0848', 'ts_0849', 'ts_0854', 'ts_0857', 'ts_0865', 'ts_0866', 'ts_0867', 'ts_0871', 'ts_0872', 'ts_0877', 'ts_0887', 'ts_0896', 'ts_0901', 'ts_0905', 'ts_0915', 'ts_0925', 'ts_0927', 'ts_0938', 'ts_0940', 'ts_0943', 'ts_0954', 'ts_0955', 'ts_0959', 'ts_0961', 'ts_0963', 'ts_0965', 'ts_0966', 'ts_0980', 'ts_0987', 'ts_0993', 'ts_0994', 'ts_0998', 'ts_1001', 'ts_1007', 'ts_1012', 'ts_1019', 'ts_1022', 'ts_1028', 'ts_1042', 'ts_1046', 'ts_1047', 'ts_1049', 'ts_1057', 'ts_1065', 'ts_1067', 'ts_1069', 'ts_1073', 'ts_1076', 'ts_1078', 'ts_1085', 'ts_1091', 'ts_1092', 'ts_1095', 'ts_1097', 'ts_1105', 'ts_1108', 'ts_1113', 'ts_1117', 'ts_1125', 'ts_1126', 'ts_1129', 'ts_1134', 'ts_1135', 'ts_1151', 'ts_1152', 'ts_1158', 'ts_1159', 'ts_1162', 'ts_1171', 'ts_1173', 'ts_1181', 'ts_1182', 'ts_1185', 'ts_1187', 'ts_1188', 'ts_1199', 'ts_1201', 'verse_0006', 'verse_0008', 'verse_0015', 'verse_0018', 'verse_0033', 'verse_0040', 'verse_0041', 'verse_0044', 'verse_0046', 'verse_0049', 'verse_0050', 'verse_0057', 'verse_0061', 'verse_0067', 'verse_0089', 'verse_0094', 'verse_0096', 'verse_0097', 'verse_0103', 'verse_0115', 'verse_0117', 'verse_0118', 'verse_0119', 'verse_0120', 'verse_0140', 'verse_0141', 'verse_0145', 'verse_0147', 'verse_0150', 'verse_0159', 'verse_0161', 'verse_0163', 'verse_0165', 'verse_0167', 'verse_0178', 'verse_0182', 'verse_0186', 'verse_0209', 'verse_0211', 'verse_0213', 'verse_0222', 'verse_0228', 'verse_0233', 'verse_0241', 'verse_0242', 'verse_0244', 'verse_0246', 'verse_0253', 'verse_0258', 'verse_0264', 'verse_0269', 'verse_0277', 'verse_0279', 'verse_0285', 'verse_0288', 'verse_0300', 'verse_0301', 'verse_0330', 'verse_0331', 'verse_0336', 'verse_0337', 'verse_0343', 'verse_0348', 'verse_0350', 'verse_0354', 'verse_0356', 'verse_0362']) 
+2023-04-03 21:32:57.417660: lr: 0.01 
+2023-04-03 21:33:16.369727: Unable to plot network architecture: 
+2023-04-03 21:33:16.371216: No module named 'hiddenlayer' 
+2023-04-03 21:33:16.372611: 
+printing the network instead:
+ 
+2023-04-03 21:33:16.373854: Generic_UNet(
+  (conv_blocks_localization): ModuleList(
+    (0): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(640, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (1): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(512, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(256, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (2): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(256, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (3): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(128, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+    (4): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(64, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+  )
+  (conv_blocks_context): ModuleList(
+    (0): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(1, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (1): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(32, 64, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (2): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(64, 128, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (3): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(128, 256, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(256, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (4): StackedConvLayers(
+      (blocks): Sequential(
+        (0): ConvDropoutNormNonlin(
+          (conv): Conv3d(256, 320, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+        (1): ConvDropoutNormNonlin(
+          (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+          (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+        )
+      )
+    )
+    (5): Sequential(
+      (0): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+      (1): StackedConvLayers(
+        (blocks): Sequential(
+          (0): ConvDropoutNormNonlin(
+            (conv): Conv3d(320, 320, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+            (instnorm): InstanceNorm3d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+            (lrelu): LeakyReLU(negative_slope=0.01, inplace=True)
+          )
+        )
+      )
+    )
+  )
+  (td): ModuleList()
+  (tu): ModuleList(
+    (0): ConvTranspose3d(320, 320, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (1): ConvTranspose3d(320, 256, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (2): ConvTranspose3d(256, 128, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (3): ConvTranspose3d(128, 64, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+    (4): ConvTranspose3d(64, 32, kernel_size=(2, 2, 2), stride=(2, 2, 2), bias=False)
+  )
+  (seg_outputs): ModuleList(
+    (0): Conv3d(320, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (1): Conv3d(256, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (2): Conv3d(128, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (3): Conv3d(64, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+    (4): Conv3d(32, 25, kernel_size=(1, 1, 1), stride=(1, 1, 1), bias=False)
+  )
+) 
+2023-04-03 21:33:16.377836: 
+ 
+2023-04-03 21:33:16.379124: 
+epoch:  0 
+2023-04-03 21:39:37.460981: train loss : -0.1759 
+2023-04-03 21:39:53.967141: validation loss: -0.2126 
+2023-04-03 21:39:53.970520: Average global foreground Dice: [0.9244, 0.9511, 0.9377, 0.9329, 0.9296, 0.9212, 0.9288, 0.9473, 0.9468, 0.9307, 0.8848, 0.8944, 0.9083, 0.9065, 0.8871, 0.8777, 0.8617, 0.8713, 0.881, 0.8843, 0.8551, 0.8393, 0.7982, 0.807] 
+2023-04-03 21:39:53.972148: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 21:39:55.310572: lr: 0.009999 
+2023-04-03 21:39:55.312150: This epoch took 398.931790 s
+ 
+2023-04-03 21:39:55.313447: 
+epoch:  1 
+2023-04-03 21:45:40.010446: train loss : -0.1634 
+2023-04-03 21:46:00.281061: validation loss: -0.1973 
+2023-04-03 21:46:00.301640: Average global foreground Dice: [0.9195, 0.9472, 0.9164, 0.8965, 0.8979, 0.8986, 0.9094, 0.9068, 0.8708, 0.8238, 0.7964, 0.7965, 0.8009, 0.8135, 0.8532, 0.8647, 0.8669, 0.8618, 0.8641, 0.8882, 0.8717, 0.8864, 0.9085, 0.8928] 
+2023-04-03 21:46:00.305038: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 21:46:01.914901: lr: 0.009999 
+2023-04-03 21:46:01.916623: This epoch took 366.602012 s
+ 
+2023-04-03 21:46:01.918043: 
+epoch:  2 
+2023-04-03 21:51:42.159373: train loss : -0.1732 
+2023-04-03 21:51:58.615849: validation loss: -0.2146 
+2023-04-03 21:51:58.619293: Average global foreground Dice: [0.9151, 0.9395, 0.9145, 0.8907, 0.8855, 0.8975, 0.9188, 0.9374, 0.9378, 0.9111, 0.8655, 0.8647, 0.8617, 0.8776, 0.8818, 0.9003, 0.9008, 0.8937, 0.9, 0.9026, 0.9035, 0.8876, 0.846, 0.8282] 
+2023-04-03 21:51:58.620909: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 21:52:00.870646: lr: 0.009998 
+2023-04-03 21:52:00.872342: This epoch took 358.952925 s
+ 
+2023-04-03 21:52:00.873964: 
+epoch:  3 
+2023-04-03 21:58:03.187056: train loss : -0.1699 
+2023-04-03 21:58:18.513868: validation loss: -0.2060 
+2023-04-03 21:58:18.538286: Average global foreground Dice: [0.9244, 0.9483, 0.9216, 0.9111, 0.9101, 0.917, 0.9256, 0.9461, 0.933, 0.8772, 0.839, 0.8342, 0.8358, 0.8153, 0.7853, 0.7908, 0.834, 0.8649, 0.8805, 0.8944, 0.8704, 0.8484, 0.8399, 0.8629] 
+2023-04-03 21:58:18.540026: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 21:58:20.249545: lr: 0.009998 
+2023-04-03 21:58:20.251554: This epoch took 379.376242 s
+ 
+2023-04-03 21:58:20.253508: 
+epoch:  4 
+2023-04-03 22:04:32.562856: train loss : -0.1666 
+2023-04-03 22:04:48.885038: validation loss: -0.1943 
+2023-04-03 22:04:48.903237: Average global foreground Dice: [0.9322, 0.9571, 0.941, 0.9343, 0.9153, 0.9187, 0.9149, 0.9457, 0.9439, 0.9199, 0.878, 0.8029, 0.7758, 0.83, 0.8964, 0.912, 0.9033, 0.8918, 0.9031, 0.9091, 0.8829, 0.8656, 0.8243, 0.8102] 
+2023-04-03 22:04:48.904714: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:04:50.671252: lr: 0.009997 
+2023-04-03 22:04:50.686477: This epoch took 390.431188 s
+ 
+2023-04-03 22:04:50.688668: 
+epoch:  5 
+2023-04-03 22:10:12.619123: train loss : -0.1649 
+2023-04-03 22:10:28.340992: validation loss: -0.2027 
+2023-04-03 22:10:28.355086: Average global foreground Dice: [0.9248, 0.9427, 0.9219, 0.914, 0.9139, 0.9196, 0.9263, 0.9436, 0.9467, 0.9156, 0.89, 0.8996, 0.9156, 0.9186, 0.903, 0.8736, 0.8721, 0.865, 0.8914, 0.9077, 0.879, 0.8589, 0.84, 0.8178] 
+2023-04-03 22:10:28.356611: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:10:30.755664: lr: 0.009997 
+2023-04-03 22:10:30.757861: This epoch took 340.067871 s
+ 
+2023-04-03 22:10:30.759526: 
+epoch:  6 
+2023-04-03 22:16:17.264140: train loss : -0.1654 
+2023-04-03 22:16:37.317280: validation loss: -0.1914 
+2023-04-03 22:16:37.331130: Average global foreground Dice: [0.9306, 0.9504, 0.9361, 0.9347, 0.9284, 0.9241, 0.9364, 0.9535, 0.9563, 0.9454, 0.9224, 0.8944, 0.8628, 0.85, 0.8507, 0.8559, 0.8638, 0.8762, 0.9037, 0.9423, 0.9272, 0.8997, 0.8903, 0.8896] 
+2023-04-03 22:16:37.332728: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:16:39.456258: lr: 0.009996 
+2023-04-03 22:16:39.458088: This epoch took 368.697084 s
+ 
+2023-04-03 22:16:39.459959: 
+epoch:  7 
+2023-04-03 22:22:26.133220: train loss : -0.1709 
+2023-04-03 22:22:41.460770: validation loss: -0.1928 
+2023-04-03 22:22:41.483172: Average global foreground Dice: [0.9215, 0.948, 0.9311, 0.9233, 0.9235, 0.9205, 0.9246, 0.9367, 0.9272, 0.9012, 0.8674, 0.8337, 0.7989, 0.8027, 0.8095, 0.8264, 0.8507, 0.8724, 0.87, 0.8928, 0.8809, 0.8606, 0.8314, 0.8649] 
+2023-04-03 22:22:41.484889: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:22:43.169045: lr: 0.009995 
+2023-04-03 22:22:43.170636: This epoch took 363.709315 s
+ 
+2023-04-03 22:22:43.171957: 
+epoch:  8 
+2023-04-03 22:28:51.572977: train loss : -0.1587 
+2023-04-03 22:29:05.832223: validation loss: -0.1776 
+2023-04-03 22:29:05.846687: Average global foreground Dice: [0.9271, 0.9523, 0.9121, 0.8753, 0.8808, 0.8962, 0.8997, 0.906, 0.8878, 0.874, 0.8621, 0.8498, 0.8328, 0.8182, 0.8439, 0.8683, 0.8861, 0.88, 0.8894, 0.8376, 0.8052, 0.8547, 0.8702, 0.8619] 
+2023-04-03 22:29:05.853475: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:29:07.688439: lr: 0.009995 
+2023-04-03 22:29:07.690402: This epoch took 384.516933 s
+ 
+2023-04-03 22:29:07.691843: 
+epoch:  9 
+2023-04-03 22:35:06.705577: train loss : -0.1757 
+2023-04-03 22:35:22.923309: validation loss: -0.2095 
+2023-04-03 22:35:22.939288: Average global foreground Dice: [0.8943, 0.936, 0.9375, 0.9203, 0.9209, 0.9198, 0.9242, 0.9473, 0.9576, 0.9522, 0.931, 0.9469, 0.9381, 0.9413, 0.9288, 0.9239, 0.9306, 0.9406, 0.9431, 0.9006, 0.842, 0.845, 0.8465, 0.8304] 
+2023-04-03 22:35:22.940980: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:35:25.056950: lr: 0.009994 
+2023-04-03 22:35:25.065417: This epoch took 377.372180 s
+ 
+2023-04-03 22:35:25.067017: 
+epoch:  10 
+2023-04-03 22:41:47.478680: train loss : -0.1699 
+2023-04-03 22:42:04.894636: validation loss: -0.2086 
+2023-04-03 22:42:04.919468: Average global foreground Dice: [0.9167, 0.9419, 0.9202, 0.9082, 0.8997, 0.9013, 0.9103, 0.9228, 0.9088, 0.8858, 0.8891, 0.8767, 0.87, 0.879, 0.8686, 0.8782, 0.8823, 0.865, 0.8945, 0.9033, 0.8445, 0.7944, 0.7955, 0.8322] 
+2023-04-03 22:42:04.921143: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:42:07.388354: lr: 0.009994 
+2023-04-03 22:42:07.390067: This epoch took 402.321569 s
+ 
+2023-04-03 22:42:07.391407: 
+epoch:  11 
+2023-04-03 22:48:12.257129: train loss : -0.1719 
+2023-04-03 22:48:25.894577: validation loss: -0.2087 
+2023-04-03 22:48:25.910935: Average global foreground Dice: [0.9174, 0.94, 0.9108, 0.9112, 0.8928, 0.8793, 0.9019, 0.9349, 0.948, 0.9221, 0.8826, 0.8563, 0.8396, 0.8452, 0.84, 0.8615, 0.8774, 0.8638, 0.8809, 0.8943, 0.8848, 0.8981, 0.8993, 0.902] 
+2023-04-03 22:48:25.912351: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:48:27.220217: lr: 0.009993 
+2023-04-03 22:48:27.221920: This epoch took 379.829142 s
+ 
+2023-04-03 22:48:27.223333: 
+epoch:  12 
+2023-04-03 22:54:22.811424: train loss : -0.1636 
+2023-04-03 22:54:39.559229: validation loss: -0.1961 
+2023-04-03 22:54:39.574977: Average global foreground Dice: [0.9068, 0.9501, 0.943, 0.9383, 0.9348, 0.9171, 0.9155, 0.944, 0.9435, 0.9272, 0.9164, 0.8844, 0.8633, 0.8688, 0.8691, 0.8811, 0.8873, 0.8799, 0.8755, 0.8737, 0.858, 0.8546, 0.8419, 0.8367] 
+2023-04-03 22:54:39.576370: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 22:54:42.162331: lr: 0.009993 
+2023-04-03 22:54:42.164258: This epoch took 374.939443 s
+ 
+2023-04-03 22:54:42.165541: 
+epoch:  13 
+2023-04-03 23:00:28.057051: train loss : -0.1683 
+2023-04-03 23:00:45.886716: validation loss: -0.2052 
+2023-04-03 23:00:45.998142: Average global foreground Dice: [0.9091, 0.9444, 0.9138, 0.9132, 0.92, 0.9197, 0.9147, 0.9209, 0.9186, 0.8951, 0.8854, 0.8696, 0.8198, 0.805, 0.8008, 0.8128, 0.8506, 0.8821, 0.9199, 0.9302, 0.8755, 0.8288, 0.8256, 0.836] 
+2023-04-03 23:00:45.999753: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:00:48.268319: lr: 0.009992 
+2023-04-03 23:00:48.269940: This epoch took 366.103030 s
+ 
+2023-04-03 23:00:48.271241: 
+epoch:  14 
+2023-04-03 23:06:46.055855: train loss : -0.1655 
+2023-04-03 23:07:01.999497: validation loss: -0.2089 
+2023-04-03 23:07:02.009284: Average global foreground Dice: [0.9231, 0.9277, 0.8817, 0.869, 0.8636, 0.8713, 0.9026, 0.9396, 0.9411, 0.9388, 0.9165, 0.9068, 0.9022, 0.8998, 0.9042, 0.9078, 0.9177, 0.9187, 0.9368, 0.9319, 0.9174, 0.9078, 0.9026, 0.8748] 
+2023-04-03 23:07:02.025435: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:07:04.227031: lr: 0.009992 
+2023-04-03 23:07:04.228763: This epoch took 375.956146 s
+ 
+2023-04-03 23:07:04.230174: 
+epoch:  15 
+2023-04-03 23:13:11.721713: train loss : -0.1706 
+2023-04-03 23:13:28.465590: validation loss: -0.2213 
+2023-04-03 23:13:28.490450: Average global foreground Dice: [0.9178, 0.9419, 0.9116, 0.9159, 0.9118, 0.9154, 0.9338, 0.9527, 0.9587, 0.9363, 0.8821, 0.836, 0.8375, 0.8661, 0.8991, 0.9196, 0.9367, 0.9193, 0.9013, 0.9071, 0.8843, 0.8822, 0.8832, 0.8675] 
+2023-04-03 23:13:28.491972: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:13:30.554857: lr: 0.009991 
+2023-04-03 23:13:30.556391: This epoch took 386.324930 s
+ 
+2023-04-03 23:13:30.557819: 
+epoch:  16 
+2023-04-03 23:19:28.867797: train loss : -0.1728 
+2023-04-03 23:19:43.844429: validation loss: -0.2143 
+2023-04-03 23:19:43.860188: Average global foreground Dice: [0.929, 0.9527, 0.9281, 0.9218, 0.9156, 0.909, 0.9217, 0.9379, 0.9198, 0.8942, 0.8629, 0.873, 0.8818, 0.9114, 0.9131, 0.9196, 0.9379, 0.9378, 0.9305, 0.932, 0.9263, 0.9206, 0.9028, 0.8966] 
+2023-04-03 23:19:43.871433: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:19:45.925587: lr: 0.00999 
+2023-04-03 23:19:46.112681: saving checkpoint... 
+2023-04-03 23:19:47.595197: done, saving took 1.66 seconds 
+2023-04-03 23:19:47.746654: This epoch took 377.187525 s
+ 
+2023-04-03 23:19:47.748293: 
+epoch:  17 
+2023-04-03 23:25:12.114978: train loss : -0.1672 
+2023-04-03 23:25:25.962749: validation loss: -0.1930 
+2023-04-03 23:25:25.978662: Average global foreground Dice: [0.9224, 0.9475, 0.9267, 0.9187, 0.9213, 0.9161, 0.8981, 0.8846, 0.8762, 0.8529, 0.8449, 0.8257, 0.7892, 0.7867, 0.8014, 0.798, 0.8129, 0.8273, 0.8308, 0.8481, 0.8441, 0.8756, 0.9066, 0.8623] 
+2023-04-03 23:25:25.981038: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:25:28.001336: lr: 0.00999 
+2023-04-03 23:25:28.003260: This epoch took 340.253613 s
+ 
+2023-04-03 23:25:28.004735: 
+epoch:  18 
+2023-04-03 23:31:22.805643: train loss : -0.1665 
+2023-04-03 23:31:41.526471: validation loss: -0.1916 
+2023-04-03 23:31:41.546930: Average global foreground Dice: [0.9073, 0.9356, 0.9083, 0.9054, 0.9103, 0.9079, 0.9117, 0.9235, 0.908, 0.8714, 0.8472, 0.8408, 0.8113, 0.8103, 0.8182, 0.8312, 0.8482, 0.8542, 0.8941, 0.915, 0.8964, 0.9002, 0.8825, 0.8671] 
+2023-04-03 23:31:41.548864: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:31:44.397947: lr: 0.009989 
+2023-04-03 23:31:44.399800: This epoch took 376.393472 s
+ 
+2023-04-03 23:31:44.401320: 
+epoch:  19 
+2023-04-03 23:37:53.698971: train loss : -0.1666 
+2023-04-03 23:38:09.984562: validation loss: -0.2180 
+2023-04-03 23:38:10.004328: Average global foreground Dice: [0.9182, 0.9465, 0.9257, 0.9201, 0.9181, 0.9157, 0.9335, 0.9454, 0.9409, 0.9119, 0.8797, 0.8636, 0.8755, 0.8765, 0.8626, 0.8542, 0.8458, 0.843, 0.897, 0.911, 0.8861, 0.8925, 0.8897, 0.8839] 
+2023-04-03 23:38:10.014699: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:38:11.920103: lr: 0.009989 
+2023-04-03 23:38:11.922027: This epoch took 387.519427 s
+ 
+2023-04-03 23:38:11.923360: 
+epoch:  20 
+2023-04-03 23:44:33.941956: train loss : -0.1689 
+2023-04-03 23:44:49.621338: validation loss: -0.1805 
+2023-04-03 23:44:49.642569: Average global foreground Dice: [0.9105, 0.9411, 0.9183, 0.8943, 0.8965, 0.9124, 0.9323, 0.9508, 0.9443, 0.9112, 0.8668, 0.8253, 0.7812, 0.7657, 0.7689, 0.8004, 0.8372, 0.8373, 0.8344, 0.8672, 0.8909, 0.9133, 0.8944, 0.8264] 
+2023-04-03 23:44:49.644515: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:44:51.988096: lr: 0.009988 
+2023-04-03 23:44:51.997260: This epoch took 400.072599 s
+ 
+2023-04-03 23:44:51.998558: 
+epoch:  21 
+2023-04-03 23:50:35.276010: train loss : -0.1760 
+2023-04-03 23:50:50.812632: validation loss: -0.2094 
+2023-04-03 23:50:50.831096: Average global foreground Dice: [0.9191, 0.9497, 0.9272, 0.9198, 0.9177, 0.9149, 0.9134, 0.9299, 0.9249, 0.8898, 0.8477, 0.837, 0.8325, 0.8505, 0.875, 0.8954, 0.9095, 0.9036, 0.9002, 0.881, 0.8417, 0.8512, 0.8603, 0.8437] 
+2023-04-03 23:50:50.832661: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:50:53.014424: lr: 0.009988 
+2023-04-03 23:50:53.016204: This epoch took 361.016412 s
+ 
+2023-04-03 23:50:53.017808: 
+epoch:  22 
+2023-04-03 23:56:43.128145: train loss : -0.1784 
+2023-04-03 23:57:01.820741: validation loss: -0.2152 
+2023-04-03 23:57:01.841069: Average global foreground Dice: [0.9185, 0.9442, 0.9234, 0.9084, 0.902, 0.9167, 0.9279, 0.9382, 0.9258, 0.878, 0.8368, 0.836, 0.8473, 0.8809, 0.8952, 0.8589, 0.8602, 0.8825, 0.9188, 0.9356, 0.9161, 0.9116, 0.8944, 0.8832] 
+2023-04-03 23:57:01.846151: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-03 23:57:03.999429: lr: 0.009987 
+2023-04-03 23:57:04.001516: This epoch took 370.982352 s
+ 
+2023-04-03 23:57:04.003543: 
+epoch:  23 
+2023-04-04 00:03:10.321417: train loss : -0.1658 
+2023-04-04 00:03:26.080034: validation loss: -0.1944 
+2023-04-04 00:03:26.102570: Average global foreground Dice: [0.9229, 0.9445, 0.9215, 0.9261, 0.9268, 0.9263, 0.9349, 0.9455, 0.9333, 0.9021, 0.8372, 0.7864, 0.7175, 0.7046, 0.7491, 0.8102, 0.8483, 0.8458, 0.8446, 0.891, 0.8778, 0.8664, 0.8385, 0.8043] 
+2023-04-04 00:03:26.104241: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:03:27.855000: lr: 0.009986 
+2023-04-04 00:03:27.857036: This epoch took 383.851764 s
+ 
+2023-04-04 00:03:27.858513: 
+epoch:  24 
+2023-04-04 00:09:33.037773: train loss : -0.1686 
+2023-04-04 00:09:50.360562: validation loss: -0.1944 
+2023-04-04 00:09:50.392395: Average global foreground Dice: [0.9148, 0.9379, 0.9091, 0.902, 0.9016, 0.8924, 0.9185, 0.9422, 0.9306, 0.8652, 0.7871, 0.7824, 0.776, 0.7714, 0.8149, 0.868, 0.9126, 0.9313, 0.9098, 0.908, 0.8953, 0.9046, 0.8824, 0.8559] 
+2023-04-04 00:09:50.393929: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:09:53.064159: lr: 0.009986 
+2023-04-04 00:09:53.066244: This epoch took 385.206267 s
+ 
+2023-04-04 00:09:53.067773: 
+epoch:  25 
+2023-04-04 00:16:06.223375: train loss : -0.1741 
+2023-04-04 00:16:23.720065: validation loss: -0.2283 
+2023-04-04 00:16:23.738393: Average global foreground Dice: [0.9196, 0.9477, 0.9296, 0.9279, 0.9157, 0.911, 0.9058, 0.9125, 0.9193, 0.9077, 0.8965, 0.908, 0.898, 0.8544, 0.8611, 0.8956, 0.9205, 0.9217, 0.9218, 0.9212, 0.9139, 0.9212, 0.8816, 0.8814] 
+2023-04-04 00:16:23.753197: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:16:25.452540: lr: 0.009985 
+2023-04-04 00:16:25.454599: This epoch took 392.385451 s
+ 
+2023-04-04 00:16:25.455996: 
+epoch:  26 
+2023-04-04 00:22:24.194814: train loss : -0.1647 
+2023-04-04 00:22:40.777901: validation loss: -0.2134 
+2023-04-04 00:22:40.797087: Average global foreground Dice: [0.919, 0.9402, 0.9045, 0.8882, 0.9057, 0.9139, 0.9344, 0.9506, 0.9441, 0.9275, 0.8953, 0.8841, 0.8827, 0.8875, 0.9005, 0.91, 0.9158, 0.9174, 0.9162, 0.8945, 0.8513, 0.8463, 0.8424, 0.8443] 
+2023-04-04 00:22:40.798981: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:22:43.080186: lr: 0.009985 
+2023-04-04 00:22:43.090249: This epoch took 377.632871 s
+ 
+2023-04-04 00:22:43.091749: 
+epoch:  27 
+2023-04-04 00:28:39.280247: train loss : -0.1562 
+2023-04-04 00:28:54.923271: validation loss: -0.2040 
+2023-04-04 00:28:54.938911: Average global foreground Dice: [0.9228, 0.9394, 0.9133, 0.8979, 0.9068, 0.9155, 0.9322, 0.9475, 0.94, 0.8959, 0.8465, 0.8232, 0.808, 0.8182, 0.8294, 0.8371, 0.8507, 0.8631, 0.8872, 0.8924, 0.8779, 0.8477, 0.8191, 0.8023] 
+2023-04-04 00:28:54.940541: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:28:56.796400: lr: 0.009984 
+2023-04-04 00:28:56.798302: This epoch took 373.705230 s
+ 
+2023-04-04 00:28:56.799714: 
+epoch:  28 
+2023-04-04 00:34:46.962346: train loss : -0.1739 
+2023-04-04 00:35:02.756556: validation loss: -0.2001 
+2023-04-04 00:35:02.781336: Average global foreground Dice: [0.9154, 0.9452, 0.9287, 0.9283, 0.9228, 0.9166, 0.9304, 0.9514, 0.9499, 0.9302, 0.8953, 0.8915, 0.9057, 0.9191, 0.8884, 0.8575, 0.8745, 0.8932, 0.8774, 0.8476, 0.7981, 0.7869, 0.8027, 0.7982] 
+2023-04-04 00:35:02.783184: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:35:04.981241: lr: 0.009984 
+2023-04-04 00:35:04.993439: This epoch took 368.180945 s
+ 
+2023-04-04 00:35:04.994718: 
+epoch:  29 
+2023-04-04 00:41:02.248076: train loss : -0.1621 
+2023-04-04 00:41:18.432853: validation loss: -0.2004 
+2023-04-04 00:41:18.458894: Average global foreground Dice: [0.9253, 0.955, 0.9259, 0.9171, 0.9295, 0.9332, 0.9421, 0.9475, 0.9205, 0.8826, 0.8773, 0.8612, 0.8514, 0.8796, 0.8921, 0.9113, 0.9312, 0.9406, 0.9478, 0.9266, 0.8795, 0.8505, 0.8383, 0.8752] 
+2023-04-04 00:41:18.460446: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:41:20.300478: lr: 0.009983 
+2023-04-04 00:41:20.302294: This epoch took 375.306058 s
+ 
+2023-04-04 00:41:20.303562: 
+epoch:  30 
+2023-04-04 00:47:29.097049: train loss : -0.1701 
+2023-04-04 00:47:46.396046: validation loss: -0.1986 
+2023-04-04 00:47:46.412936: Average global foreground Dice: [0.9247, 0.9477, 0.9238, 0.9072, 0.9031, 0.899, 0.9206, 0.9083, 0.8866, 0.8704, 0.8335, 0.8217, 0.7872, 0.7883, 0.7761, 0.8272, 0.8909, 0.9053, 0.9093, 0.919, 0.9008, 0.8712, 0.86, 0.8499] 
+2023-04-04 00:47:46.414625: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:47:48.108251: lr: 0.009983 
+2023-04-04 00:47:48.110271: This epoch took 387.805376 s
+ 
+2023-04-04 00:47:48.111916: 
+epoch:  31 
+2023-04-04 00:53:48.649478: train loss : -0.1755 
+2023-04-04 00:54:04.151864: validation loss: -0.2110 
+2023-04-04 00:54:04.170878: Average global foreground Dice: [0.926, 0.9452, 0.8895, 0.8729, 0.8987, 0.9098, 0.9272, 0.9456, 0.9244, 0.87, 0.7993, 0.7569, 0.7848, 0.8521, 0.8713, 0.8634, 0.885, 0.916, 0.9323, 0.9338, 0.8993, 0.8823, 0.8806, 0.8791] 
+2023-04-04 00:54:04.172471: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 00:54:06.308507: lr: 0.009982 
+2023-04-04 00:54:06.310215: This epoch took 378.196580 s
+ 
+2023-04-04 00:54:06.311597: 
+epoch:  32 
+2023-04-04 01:00:22.118291: train loss : -0.1676 
+2023-04-04 01:00:39.813833: validation loss: -0.2126 
+2023-04-04 01:00:39.835533: Average global foreground Dice: [0.935, 0.9448, 0.8962, 0.9064, 0.9249, 0.9058, 0.8982, 0.9033, 0.9099, 0.8875, 0.8351, 0.8162, 0.8045, 0.8178, 0.8613, 0.8815, 0.8751, 0.8796, 0.9038, 0.9239, 0.8878, 0.8908, 0.8975, 0.8804] 
+2023-04-04 01:00:39.842461: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:00:41.961529: lr: 0.009981 
+2023-04-04 01:00:41.963036: This epoch took 395.650031 s
+ 
+2023-04-04 01:00:41.964604: 
+epoch:  33 
+2023-04-04 01:06:49.282773: train loss : -0.1678 
+2023-04-04 01:07:04.764860: validation loss: -0.2011 
+2023-04-04 01:07:04.779080: Average global foreground Dice: [0.9301, 0.9488, 0.9264, 0.9289, 0.9306, 0.9108, 0.8964, 0.9006, 0.889, 0.8702, 0.825, 0.7617, 0.7708, 0.8438, 0.8809, 0.8768, 0.8836, 0.8815, 0.8954, 0.9263, 0.9488, 0.9486, 0.9436, 0.9273] 
+2023-04-04 01:07:04.780928: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:07:06.932425: lr: 0.009981 
+2023-04-04 01:07:06.934622: This epoch took 384.968686 s
+ 
+2023-04-04 01:07:06.936472: 
+epoch:  34 
+2023-04-04 01:12:53.529304: train loss : -0.1711 
+2023-04-04 01:13:11.523305: validation loss: -0.2002 
+2023-04-04 01:13:11.543066: Average global foreground Dice: [0.9319, 0.946, 0.9246, 0.9174, 0.8999, 0.8892, 0.9079, 0.9177, 0.8951, 0.8519, 0.802, 0.8019, 0.7781, 0.796, 0.8064, 0.8299, 0.8358, 0.8521, 0.8718, 0.8727, 0.8694, 0.8556, 0.8311, 0.7894] 
+2023-04-04 01:13:11.545228: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:13:13.104422: lr: 0.00998 
+2023-04-04 01:13:13.106085: This epoch took 366.168014 s
+ 
+2023-04-04 01:13:13.107355: 
+epoch:  35 
+2023-04-04 01:18:58.873073: train loss : -0.1719 
+2023-04-04 01:19:15.111051: validation loss: -0.2209 
+2023-04-04 01:19:15.114479: Average global foreground Dice: [0.9325, 0.9493, 0.9192, 0.9084, 0.9084, 0.916, 0.9193, 0.9312, 0.9144, 0.8835, 0.8394, 0.8472, 0.8654, 0.8665, 0.8734, 0.8871, 0.9072, 0.926, 0.9365, 0.9367, 0.9076, 0.8565, 0.7926, 0.782] 
+2023-04-04 01:19:15.116196: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:19:16.904485: lr: 0.00998 
+2023-04-04 01:19:16.906435: This epoch took 363.797517 s
+ 
+2023-04-04 01:19:16.907947: 
+epoch:  36 
+2023-04-04 01:25:08.055699: train loss : -0.1688 
+2023-04-04 01:25:22.979308: validation loss: -0.1920 
+2023-04-04 01:25:22.982434: Average global foreground Dice: [0.9193, 0.9444, 0.9291, 0.9043, 0.8996, 0.9143, 0.9263, 0.9333, 0.9404, 0.9196, 0.8805, 0.8543, 0.8172, 0.7977, 0.7746, 0.8056, 0.8295, 0.8482, 0.8657, 0.8756, 0.8528, 0.847, 0.8316, 0.8415] 
+2023-04-04 01:25:22.984175: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:25:24.972442: lr: 0.009979 
+2023-04-04 01:25:24.974307: This epoch took 368.065012 s
+ 
+2023-04-04 01:25:24.976060: 
+epoch:  37 
+2023-04-04 01:31:05.720123: train loss : -0.1661 
+2023-04-04 01:31:21.426927: validation loss: -0.2037 
+2023-04-04 01:31:21.442473: Average global foreground Dice: [0.9149, 0.9395, 0.9044, 0.8965, 0.9045, 0.9094, 0.9221, 0.9243, 0.8981, 0.8423, 0.8218, 0.823, 0.8275, 0.8244, 0.8105, 0.7804, 0.7913, 0.8175, 0.825, 0.8625, 0.8544, 0.8388, 0.8261, 0.8343] 
+2023-04-04 01:31:21.444153: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:31:22.972284: lr: 0.009979 
+2023-04-04 01:31:22.974279: This epoch took 357.996704 s
+ 
+2023-04-04 01:31:22.975794: 
+epoch:  38 
+2023-04-04 01:37:13.642312: train loss : -0.1668 
+2023-04-04 01:37:30.849473: validation loss: -0.1991 
+2023-04-04 01:37:30.874901: Average global foreground Dice: [0.9195, 0.9392, 0.9089, 0.8974, 0.8746, 0.9007, 0.9114, 0.9211, 0.9167, 0.8908, 0.8444, 0.8226, 0.8282, 0.827, 0.8282, 0.8026, 0.7918, 0.834, 0.8614, 0.8925, 0.8842, 0.898, 0.8899, 0.8893] 
+2023-04-04 01:37:30.876623: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:37:34.019162: lr: 0.009978 
+2023-04-04 01:37:34.020961: This epoch took 371.043714 s
+ 
+2023-04-04 01:37:34.022388: 
+epoch:  39 
+2023-04-04 01:43:35.818239: train loss : -0.1727 
+2023-04-04 01:43:53.002667: validation loss: -0.1905 
+2023-04-04 01:43:53.019023: Average global foreground Dice: [0.9182, 0.9424, 0.9301, 0.9355, 0.9307, 0.9206, 0.9235, 0.9479, 0.9431, 0.9378, 0.8904, 0.8459, 0.7774, 0.7729, 0.8187, 0.86, 0.8821, 0.8875, 0.8952, 0.8938, 0.8824, 0.8735, 0.8742, 0.8788] 
+2023-04-04 01:43:53.021049: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:43:55.676369: lr: 0.009977 
+2023-04-04 01:43:55.678720: This epoch took 381.654681 s
+ 
+2023-04-04 01:43:55.680173: 
+epoch:  40 
+2023-04-04 01:50:02.259826: train loss : -0.1738 
+2023-04-04 01:50:19.216763: validation loss: -0.1938 
+2023-04-04 01:50:19.234722: Average global foreground Dice: [0.9232, 0.9561, 0.9359, 0.9315, 0.9284, 0.925, 0.9446, 0.957, 0.953, 0.9293, 0.9018, 0.8539, 0.7725, 0.7782, 0.8267, 0.8346, 0.8641, 0.8804, 0.8609, 0.852, 0.8417, 0.8547, 0.864, 0.8741] 
+2023-04-04 01:50:19.236182: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:50:21.532267: lr: 0.009977 
+2023-04-04 01:50:21.534027: This epoch took 385.852424 s
+ 
+2023-04-04 01:50:21.535333: 
+epoch:  41 
+2023-04-04 01:56:30.658272: train loss : -0.1779 
+2023-04-04 01:56:46.197081: validation loss: -0.2039 
+2023-04-04 01:56:46.222745: Average global foreground Dice: [0.9102, 0.9339, 0.905, 0.8896, 0.8771, 0.8789, 0.8967, 0.9108, 0.8732, 0.8578, 0.8426, 0.8265, 0.8566, 0.8766, 0.8706, 0.8426, 0.8408, 0.8478, 0.84, 0.8517, 0.8188, 0.7996, 0.7857, 0.7517] 
+2023-04-04 01:56:46.224656: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 01:56:48.339925: lr: 0.009976 
+2023-04-04 01:56:48.341841: This epoch took 386.805223 s
+ 
+2023-04-04 01:56:48.343359: 
+epoch:  42 
+2023-04-04 02:02:41.408635: train loss : -0.1609 
+2023-04-04 02:02:56.288207: validation loss: -0.2036 
+2023-04-04 02:02:56.312382: Average global foreground Dice: [0.9348, 0.9566, 0.918, 0.877, 0.872, 0.9113, 0.9137, 0.9202, 0.9227, 0.909, 0.8786, 0.8542, 0.862, 0.8898, 0.8996, 0.8995, 0.9047, 0.9082, 0.8964, 0.8933, 0.8807, 0.8391, 0.7854, 0.7469] 
+2023-04-04 02:02:56.314214: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 02:02:58.546659: lr: 0.009976 
+2023-04-04 02:02:58.549661: This epoch took 370.204986 s
+ 
+2023-04-04 02:02:58.551795: 
+epoch:  43 
+2023-04-04 02:08:52.372625: train loss : -0.1701 
+2023-04-04 02:09:12.658534: validation loss: -0.2139 
+2023-04-04 02:09:12.665433: Average global foreground Dice: [0.9201, 0.9405, 0.9135, 0.9198, 0.9219, 0.9259, 0.9346, 0.9481, 0.9466, 0.9426, 0.9249, 0.9062, 0.896, 0.8849, 0.868, 0.8548, 0.8675, 0.8765, 0.8766, 0.9086, 0.8977, 0.8779, 0.8505, 0.852] 
+2023-04-04 02:09:12.667073: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 02:09:14.960557: lr: 0.009975 
+2023-04-04 02:09:14.969381: This epoch took 376.408966 s
+ 
+2023-04-04 02:09:14.971032: 
+epoch:  44 
+2023-04-04 02:15:07.229023: train loss : -0.1653 
+2023-04-04 02:15:21.446733: validation loss: -0.2085 
+2023-04-04 02:15:21.449831: Average global foreground Dice: [0.9185, 0.9443, 0.9349, 0.925, 0.9267, 0.9238, 0.9331, 0.9463, 0.9168, 0.8302, 0.766, 0.8221, 0.8494, 0.8353, 0.8306, 0.8376, 0.8646, 0.9107, 0.9331, 0.9349, 0.9086, 0.8991, 0.881, 0.8627] 
+2023-04-04 02:15:21.451557: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 02:15:23.604435: lr: 0.009975 
+2023-04-04 02:15:23.606679: This epoch took 368.634004 s
+ 
+2023-04-04 02:15:23.608358: 
+epoch:  45 
+2023-04-04 02:21:22.915727: train loss : -0.1647 
+2023-04-04 02:21:38.945121: validation loss: -0.2190 
+2023-04-04 02:21:38.948632: Average global foreground Dice: [0.9244, 0.9478, 0.9182, 0.9103, 0.9232, 0.924, 0.937, 0.9474, 0.9399, 0.9043, 0.8773, 0.8876, 0.8831, 0.8663, 0.8501, 0.8331, 0.853, 0.8847, 0.9199, 0.9283, 0.9055, 0.8888, 0.866, 0.86] 
+2023-04-04 02:21:38.950092: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 02:21:41.496298: lr: 0.009974 
+2023-04-04 02:21:41.498317: This epoch took 377.888634 s
+ 
+2023-04-04 02:21:41.499655: 
+epoch:  46 
+2023-04-04 02:27:55.466854: train loss : -0.1743 
+2023-04-04 02:28:11.172946: validation loss: -0.1951 
+2023-04-04 02:28:11.188863: Average global foreground Dice: [0.9315, 0.9557, 0.9377, 0.9304, 0.9269, 0.9187, 0.9292, 0.9517, 0.9445, 0.909, 0.8885, 0.8983, 0.8935, 0.8745, 0.8719, 0.8676, 0.8749, 0.8838, 0.8684, 0.8589, 0.8732, 0.8972, 0.9095, 0.8907] 
+2023-04-04 02:28:11.190464: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 02:28:12.773550: lr: 0.009974 
+2023-04-04 02:28:12.775457: This epoch took 391.274148 s
+ 
+2023-04-04 02:28:12.776873: 
+epoch:  47 
+2023-04-04 02:34:00.796794: train loss : -0.1688 
+2023-04-04 02:34:18.442479: validation loss: -0.1982 
+2023-04-04 02:34:18.446737: Average global foreground Dice: [0.9069, 0.9437, 0.9304, 0.9253, 0.9207, 0.9225, 0.9273, 0.9461, 0.934, 0.8784, 0.8485, 0.8731, 0.8512, 0.8072, 0.8078, 0.8335, 0.8621, 0.8818, 0.8994, 0.9014, 0.8526, 0.8497, 0.8571, 0.8096] 
+2023-04-04 02:34:18.459413: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 02:34:20.556386: lr: 0.009973 
+2023-04-04 02:34:20.558252: This epoch took 367.780012 s
+ 
+2023-04-04 02:34:20.559686: 
+epoch:  48 
+2023-04-04 02:40:16.655574: train loss : -0.1645 
+2023-04-04 02:40:31.500397: validation loss: -0.1977 
+2023-04-04 02:40:31.518147: Average global foreground Dice: [0.91, 0.9259, 0.9171, 0.9068, 0.9119, 0.9161, 0.9176, 0.939, 0.932, 0.9116, 0.8536, 0.8193, 0.8235, 0.8478, 0.8581, 0.8814, 0.8719, 0.8716, 0.9001, 0.9076, 0.8755, 0.8762, 0.8792, 0.8404] 
+2023-04-04 02:40:31.519770: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 02:40:33.704314: lr: 0.009972 
+2023-04-04 02:40:33.707672: This epoch took 373.146562 s
+ 
+2023-04-04 02:40:33.709146: 
+epoch:  49 
+2023-04-04 02:46:25.231871: train loss : -0.1738 
+2023-04-04 02:46:40.419795: validation loss: -0.2002 
+2023-04-04 02:46:40.430686: Average global foreground Dice: [0.9137, 0.9438, 0.8642, 0.8246, 0.8786, 0.9145, 0.9174, 0.9332, 0.9319, 0.9069, 0.8807, 0.8709, 0.8571, 0.8735, 0.8956, 0.8889, 0.8658, 0.8713, 0.8759, 0.8763, 0.8665, 0.8402, 0.8269, 0.8266] 
+2023-04-04 02:46:40.445289: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2023-04-04 02:46:43.151857: lr: 0.009972 
+2023-04-04 02:46:43.332165: saving scheduled checkpoint file... 
+2023-04-04 02:46:43.474666: saving checkpoint... 

fold_0/training_log_2023_4_4_14_31_51.txt

@@ -0,0 +1,9 @@
+Starting... 
+2023-04-04 14:31:51.682755: Using splits from existing split file: /dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/splits_final.pkl 
+2023-04-04 14:31:51.693639: The split file contains 5 splits. 
+2023-04-04 14:31:51.695981: Desired fold for training: 0 
+2023-04-04 14:31:51.697883: This split has 1262 training and 316 validation cases. 
+2023-04-04 14:31:56.132420: TRAINING KEYS:
+ odict_keys(['ts_0001', 'ts_0004', 'ts_0005', 'ts_0006', 'ts_0007', 'ts_0009', 'ts_0010', 'ts_0011', 'ts_0012', 'ts_0013', 'ts_0014', 'ts_0015', 'ts_0016', 'ts_0018', 'ts_0019', 'ts_0021', 'ts_0022', 'ts_0023', 'ts_0025', 'ts_0026', 'ts_0027', 'ts_0028', 'ts_0031', 'ts_0032', 'ts_0033', 'ts_0034', 'ts_0035', 'ts_0036', 'ts_0037', 'ts_0038', 'ts_0039', 'ts_0040', 'ts_0041', 'ts_0042', 'ts_0043', 'ts_0044', 'ts_0045', 'ts_0046', 'ts_0047', 'ts_0048', 'ts_0049', 'ts_0050', 'ts_0051', 'ts_0053', 'ts_0054', 'ts_0055', 'ts_0057', 'ts_0058', 'ts_0059', 'ts_0060', 'ts_0061', 'ts_0062', 'ts_0063', 'ts_0065', 'ts_0066', 'ts_0067', 'ts_0068', 'ts_0069', 'ts_0071', 'ts_0074', 'ts_0075', 'ts_0077', 'ts_0078', 'ts_0079', 'ts_0080', 'ts_0081', 'ts_0082', 'ts_0083', 'ts_0084', 'ts_0085', 'ts_0086', 'ts_0088', 'ts_0089', 'ts_0090', 'ts_0091', 'ts_0093', 'ts_0094', 'ts_0095', 'ts_0096', 'ts_0097', 'ts_0099', 'ts_0100', 'ts_0102', 'ts_0103', 'ts_0105', 'ts_0106', 'ts_0107', 'ts_0108', 'ts_0109', 'ts_0110', 'ts_0112', 'ts_0113', 'ts_0114', 'ts_0116', 'ts_0117', 'ts_0118', 'ts_0119', 'ts_0120', 'ts_0121', 'ts_0122', 'ts_0123', 'ts_0124', 'ts_0125', 'ts_0126', 'ts_0128', 'ts_0129', 'ts_0130', 'ts_0132', 'ts_0133', 'ts_0134', 'ts_0135', 'ts_0138', 'ts_0139', 'ts_0140', 'ts_0141', 'ts_0142', 'ts_0143', 'ts_0144', 'ts_0145', 'ts_0146', 'ts_0148', 'ts_0152', 'ts_0155', 'ts_0156', 'ts_0157', 'ts_0158', 'ts_0159', 'ts_0160', 'ts_0161', 'ts_0163', 'ts_0165', 'ts_0166', 'ts_0168', 'ts_0169', 'ts_0170', 'ts_0171', 'ts_0172', 'ts_0173', 'ts_0174', 'ts_0175', 'ts_0176', 'ts_0177', 'ts_0178', 'ts_0179', 'ts_0181', 'ts_0182', 'ts_0183', 'ts_0184', 'ts_0185', 'ts_0186', 'ts_0189', 'ts_0191', 'ts_0192', 'ts_0194', 'ts_0195', 'ts_0196', 'ts_0197', 'ts_0199', 'ts_0200', 'ts_0202', 'ts_0203', 'ts_0204', 'ts_0205', 'ts_0209', 'ts_0210', 'ts_0211', 'ts_0212', 'ts_0213', 'ts_0214', 'ts_0215', 'ts_0216', 'ts_0217', 'ts_0218', 'ts_0222', 'ts_0224', 'ts_0225', 'ts_0226', 'ts_0228', 'ts_0230', 'ts_0231', 'ts_0232', 'ts_0233', 'ts_0234', 'ts_0235', 'ts_0236', 'ts_0237', 'ts_0239', 'ts_0240', 'ts_0241', 'ts_0242', 'ts_0244', 'ts_0245', 'ts_0246', 'ts_0247', 'ts_0248', 'ts_0249', 'ts_0250', 'ts_0251', 'ts_0252', 'ts_0253', 'ts_0254', 'ts_0255', 'ts_0256', 'ts_0257', 'ts_0258', 'ts_0259', 'ts_0261', 'ts_0263', 'ts_0265', 'ts_0267', 'ts_0269', 'ts_0270', 'ts_0271', 'ts_0272', 'ts_0273', 'ts_0274', 'ts_0275', 'ts_0276', 'ts_0278', 'ts_0279', 'ts_0280', 'ts_0285', 'ts_0286', 'ts_0287', 'ts_0288', 'ts_0289', 'ts_0290', 'ts_0291', 'ts_0292', 'ts_0293', 'ts_0295', 'ts_0296', 'ts_0297', 'ts_0299', 'ts_0300', 'ts_0301', 'ts_0303', 'ts_0304', 'ts_0305', 'ts_0306', 'ts_0307', 'ts_0309', 'ts_0310', 'ts_0312', 'ts_0313', 'ts_0315', 'ts_0316', 'ts_0318', 'ts_0319', 'ts_0320', 'ts_0321', 'ts_0322', 'ts_0323', 'ts_0325', 'ts_0326', 'ts_0327', 'ts_0329', 'ts_0330', 'ts_0331', 'ts_0332', 'ts_0333', 'ts_0334', 'ts_0336', 'ts_0337', 'ts_0339', 'ts_0340', 'ts_0341', 'ts_0342', 'ts_0343', 'ts_0344', 'ts_0345', 'ts_0346', 'ts_0348', 'ts_0349', 'ts_0350', 'ts_0351', 'ts_0352', 'ts_0353', 'ts_0354', 'ts_0355', 'ts_0356', 'ts_0359', 'ts_0360', 'ts_0362', 'ts_0363', 'ts_0364', 'ts_0365', 'ts_0366', 'ts_0367', 'ts_0368', 'ts_0369', 'ts_0370', 'ts_0371', 'ts_0373', 'ts_0374', 'ts_0375', 'ts_0376', 'ts_0377', 'ts_0378', 'ts_0379', 'ts_0381', 'ts_0382', 'ts_0384', 'ts_0385', 'ts_0386', 'ts_0387', 'ts_0388', 'ts_0389', 'ts_0393', 'ts_0396', 'ts_0397', 'ts_0398', 'ts_0399', 'ts_0400', 'ts_0401', 'ts_0402', 'ts_0404', 'ts_0405', 'ts_0406', 'ts_0407', 'ts_0408', 'ts_0409', 'ts_0410', 'ts_0411', 'ts_0412', 'ts_0414', 'ts_0415', 'ts_0416', 'ts_0417', 'ts_0419', 'ts_0420', 'ts_0421', 'ts_0422', 'ts_0424', 'ts_0425', 'ts_0429', 'ts_0430', 'ts_0431', 'ts_0432', 'ts_0433', 'ts_0434', 'ts_0435', 'ts_0437', 'ts_0438', 'ts_0439', 'ts_0440', 'ts_0441', 'ts_0442', 'ts_0443', 'ts_0444', 'ts_0445', 'ts_0446', 'ts_0447', 'ts_0448', 'ts_0449', 'ts_0450', 'ts_0451', 'ts_0452', 'ts_0453', 'ts_0454', 'ts_0457', 'ts_0458', 'ts_0459', 'ts_0460', 'ts_0461', 'ts_0462', 'ts_0464', 'ts_0466', 'ts_0467', 'ts_0468', 'ts_0469', 'ts_0470', 'ts_0472', 'ts_0473', 'ts_0474', 'ts_0475', 'ts_0476', 'ts_0477', 'ts_0478', 'ts_0479', 'ts_0480', 'ts_0481', 'ts_0482', 'ts_0483', 'ts_0484', 'ts_0485', 'ts_0487', 'ts_0488', 'ts_0489', 'ts_0490', 'ts_0491', 'ts_0492', 'ts_0493', 'ts_0494', 'ts_0495', 'ts_0496', 'ts_0497', 'ts_0498', 'ts_0500', 'ts_0501', 'ts_0502', 'ts_0503', 'ts_0504', 'ts_0505', 'ts_0506', 'ts_0507', 'ts_0508', 'ts_0509', 'ts_0512', 'ts_0514', 'ts_0515', 'ts_0516', 'ts_0517', 'ts_0518', 'ts_0519', 'ts_0520', 'ts_0522', 'ts_0524', 'ts_0525', 'ts_0527', 'ts_0528', 'ts_0529', 'ts_0530', 'ts_0531', 'ts_0532', 'ts_0533', 'ts_0534', 'ts_0535', 'ts_0537', 'ts_0538', 'ts_0539', 'ts_0541', 'ts_0542', 'ts_0543', 'ts_0544', 'ts_0546', 'ts_0547', 'ts_0548', 'ts_0552', 'ts_0553', 'ts_0554', 'ts_0555', 'ts_0556', 'ts_0557', 'ts_0559', 'ts_0560', 'ts_0561', 'ts_0562', 'ts_0564', 'ts_0566', 'ts_0567', 'ts_0569', 'ts_0570', 'ts_0572', 'ts_0573', 'ts_0574', 'ts_0575', 'ts_0576', 'ts_0577', 'ts_0578', 'ts_0579', 'ts_0580', 'ts_0582', 'ts_0583', 'ts_0584', 'ts_0585', 'ts_0586', 'ts_0587', 'ts_0588', 'ts_0591', 'ts_0592', 'ts_0594', 'ts_0595', 'ts_0597', 'ts_0599', 'ts_0600', 'ts_0601', 'ts_0602', 'ts_0604', 'ts_0605', 'ts_0606', 'ts_0607', 'ts_0608', 'ts_0609', 'ts_0610', 'ts_0612', 'ts_0613', 'ts_0614', 'ts_0615', 'ts_0616', 'ts_0619', 'ts_0620', 'ts_0621', 'ts_0622', 'ts_0623', 'ts_0625', 'ts_0626', 'ts_0627', 'ts_0628', 'ts_0629', 'ts_0630', 'ts_0631', 'ts_0632', 'ts_0633', 'ts_0634', 'ts_0635', 'ts_0636', 'ts_0637', 'ts_0639', 'ts_0640', 'ts_0642', 'ts_0643', 'ts_0644', 'ts_0645', 'ts_0646', 'ts_0647', 'ts_0649', 'ts_0651', 'ts_0652', 'ts_0653', 'ts_0654', 'ts_0656', 'ts_0657', 'ts_0660', 'ts_0661', 'ts_0662', 'ts_0668', 'ts_0669', 'ts_0670', 'ts_0671', 'ts_0672', 'ts_0673', 'ts_0674', 'ts_0676', 'ts_0677', 'ts_0678', 'ts_0679', 'ts_0680', 'ts_0681', 'ts_0683', 'ts_0684', 'ts_0687', 'ts_0688', 'ts_0689', 'ts_0690', 'ts_0691', 'ts_0693', 'ts_0695', 'ts_0696', 'ts_0697', 'ts_0699', 'ts_0700', 'ts_0701', 'ts_0702', 'ts_0703', 'ts_0704', 'ts_0706', 'ts_0707', 'ts_0708', 'ts_0709', 'ts_0711', 'ts_0712', 'ts_0713', 'ts_0714', 'ts_0715', 'ts_0717', 'ts_0718', 'ts_0719', 'ts_0720', 'ts_0721', 'ts_0722', 'ts_0723', 'ts_0724', 'ts_0725', 'ts_0726', 'ts_0727', 'ts_0729', 'ts_0730', 'ts_0731', 'ts_0734', 'ts_0735', 'ts_0737', 'ts_0738', 'ts_0739', 'ts_0740', 'ts_0741', 'ts_0742', 'ts_0743', 'ts_0744', 'ts_0745', 'ts_0747', 'ts_0748', 'ts_0749', 'ts_0750', 'ts_0751', 'ts_0752', 'ts_0753', 'ts_0754', 'ts_0755', 'ts_0756', 'ts_0757', 'ts_0758', 'ts_0759', 'ts_0760', 'ts_0761', 'ts_0762', 'ts_0764', 'ts_0765', 'ts_0766', 'ts_0769', 'ts_0770', 'ts_0771', 'ts_0772', 'ts_0773', 'ts_0774', 'ts_0775', 'ts_0776', 'ts_0777', 'ts_0778', 'ts_0779', 'ts_0780', 'ts_0781', 'ts_0782', 'ts_0783', 'ts_0784', 'ts_0786', 'ts_0787', 'ts_0788', 'ts_0789', 'ts_0790', 'ts_0791', 'ts_0792', 'ts_0796', 'ts_0797', 'ts_0798', 'ts_0799', 'ts_0800', 'ts_0801', 'ts_0802', 'ts_0803', 'ts_0804', 'ts_0805', 'ts_0807', 'ts_0810', 'ts_0811', 'ts_0812', 'ts_0813', 'ts_0815', 'ts_0816', 'ts_0817', 'ts_0818', 'ts_0819', 'ts_0820', 'ts_0821', 'ts_0822', 'ts_0823', 'ts_0824', 'ts_0825', 'ts_0827', 'ts_0828', 'ts_0830', 'ts_0831', 'ts_0832', 'ts_0833', 'ts_0834', 'ts_0835', 'ts_0836', 'ts_0837', 'ts_0839', 'ts_0841', 'ts_0842', 'ts_0843', 'ts_0844', 'ts_0845', 'ts_0846', 'ts_0847', 'ts_0850', 'ts_0851', 'ts_0852', 'ts_0853', 'ts_0855', 'ts_0856', 'ts_0858', 'ts_0859', 'ts_0860', 'ts_0861', 'ts_0862', 'ts_0863', 'ts_0864', 'ts_0868', 'ts_0869', 'ts_0870', 'ts_0873', 'ts_0874', 'ts_0875', 'ts_0876', 'ts_0878', 'ts_0879', 'ts_0880', 'ts_0881', 'ts_0882', 'ts_0883', 'ts_0884', 'ts_0885', 'ts_0886', 'ts_0888', 'ts_0889', 'ts_0890', 'ts_0891', 'ts_0892', 'ts_0893', 'ts_0894', 'ts_0895', 'ts_0897', 'ts_0898', 'ts_0899', 'ts_0900', 'ts_0902', 'ts_0903', 'ts_0904', 'ts_0906', 'ts_0907', 'ts_0908', 'ts_0909', 'ts_0910', 'ts_0911', 'ts_0912', 'ts_0913', 'ts_0914', 'ts_0916', 'ts_0917', 'ts_0918', 'ts_0919', 'ts_0920', 'ts_0921', 'ts_0922', 'ts_0923', 'ts_0924', 'ts_0926', 'ts_0928', 'ts_0929', 'ts_0930', 'ts_0931', 'ts_0932', 'ts_0933', 'ts_0934', 'ts_0935', 'ts_0936', 'ts_0937', 'ts_0939', 'ts_0941', 'ts_0942', 'ts_0944', 'ts_0945', 'ts_0946', 'ts_0947', 'ts_0948', 'ts_0949', 'ts_0950', 'ts_0951', 'ts_0952', 'ts_0953', 'ts_0956', 'ts_0957', 'ts_0958', 'ts_0960', 'ts_0962', 'ts_0964', 'ts_0967', 'ts_0968', 'ts_0969', 'ts_0970', 'ts_0971', 'ts_0972', 'ts_0973', 'ts_0974', 'ts_0975', 'ts_0976', 'ts_0977', 'ts_0978', 'ts_0979', 'ts_0981', 'ts_0982', 'ts_0983', 'ts_0984', 'ts_0985', 'ts_0986', 'ts_0988', 'ts_0989', 'ts_0990', 'ts_0991', 'ts_0992', 'ts_0995', 'ts_0996', 'ts_0997', 'ts_0999', 'ts_1000', 'ts_1002', 'ts_1003', 'ts_1004', 'ts_1005', 'ts_1006', 'ts_1008', 'ts_1009', 'ts_1010', 'ts_1011', 'ts_1013', 'ts_1014', 'ts_1015', 'ts_1016', 'ts_1017', 'ts_1018', 'ts_1020', 'ts_1021', 'ts_1023', 'ts_1024', 'ts_1025', 'ts_1026', 'ts_1027', 'ts_1029', 'ts_1030', 'ts_1031', 'ts_1032', 'ts_1033', 'ts_1034', 'ts_1035', 'ts_1036', 'ts_1037', 'ts_1038', 'ts_1039', 'ts_1040', 'ts_1041', 'ts_1043', 'ts_1044', 'ts_1045', 'ts_1048', 'ts_1050', 'ts_1051', 'ts_1052', 'ts_1053', 'ts_1054', 'ts_1055', 'ts_1056', 'ts_1058', 'ts_1059', 'ts_1060', 'ts_1061', 'ts_1062', 'ts_1063', 'ts_1064', 'ts_1066', 'ts_1068', 'ts_1070', 'ts_1071', 'ts_1072', 'ts_1074', 'ts_1075', 'ts_1077', 'ts_1079', 'ts_1080', 'ts_1081', 'ts_1082', 'ts_1083', 'ts_1084', 'ts_1086', 'ts_1087', 'ts_1088', 'ts_1089', 'ts_1090', 'ts_1093', 'ts_1094', 'ts_1096', 'ts_1098', 'ts_1099', 'ts_1100', 'ts_1101', 'ts_1102', 'ts_1103', 'ts_1104', 'ts_1106', 'ts_1107', 'ts_1109', 'ts_1110', 'ts_1111', 'ts_1112', 'ts_1114', 'ts_1115', 'ts_1116', 'ts_1118', 'ts_1119', 'ts_1120', 'ts_1121', 'ts_1122', 'ts_1123', 'ts_1124', 'ts_1127', 'ts_1128', 'ts_1130', 'ts_1131', 'ts_1132', 'ts_1133', 'ts_1136', 'ts_1137', 'ts_1138', 'ts_1139', 'ts_1140', 'ts_1141', 'ts_1142', 'ts_1143', 'ts_1144', 'ts_1145', 'ts_1146', 'ts_1147', 'ts_1148', 'ts_1149', 'ts_1150', 'ts_1153', 'ts_1154', 'ts_1155', 'ts_1156', 'ts_1157', 'ts_1160', 'ts_1161', 'ts_1163', 'ts_1164', 'ts_1165', 'ts_1166', 'ts_1167', 'ts_1168', 'ts_1169', 'ts_1170', 'ts_1172', 'ts_1174', 'ts_1175', 'ts_1176', 'ts_1177', 'ts_1178', 'ts_1179', 'ts_1180', 'ts_1183', 'ts_1184', 'ts_1186', 'ts_1189', 'ts_1190', 'ts_1191', 'ts_1192', 'ts_1193', 'ts_1194', 'ts_1195', 'ts_1196', 'ts_1197', 'ts_1198', 'ts_1200', 'ts_1202', 'ts_1203', 'verse_0000', 'verse_0001', 'verse_0002', 'verse_0003', 'verse_0004', 'verse_0005', 'verse_0007', 'verse_0009', 'verse_0010', 'verse_0011', 'verse_0012', 'verse_0013', 'verse_0014', 'verse_0016', 'verse_0017', 'verse_0019', 'verse_0020', 'verse_0021', 'verse_0022', 'verse_0023', 'verse_0024', 'verse_0025', 'verse_0026', 'verse_0027', 'verse_0028', 'verse_0029', 'verse_0030', 'verse_0031', 'verse_0032', 'verse_0034', 'verse_0035', 'verse_0036', 'verse_0037', 'verse_0038', 'verse_0039', 'verse_0042', 'verse_0043', 'verse_0045', 'verse_0047', 'verse_0048', 'verse_0051', 'verse_0052', 'verse_0053', 'verse_0054', 'verse_0055', 'verse_0056', 'verse_0058', 'verse_0059', 'verse_0060', 'verse_0062', 'verse_0063', 'verse_0064', 'verse_0065', 'verse_0066', 'verse_0068', 'verse_0069', 'verse_0070', 'verse_0071', 'verse_0072', 'verse_0073', 'verse_0074', 'verse_0075', 'verse_0076', 'verse_0077', 'verse_0078', 'verse_0079', 'verse_0080', 'verse_0081', 'verse_0082', 'verse_0083', 'verse_0084', 'verse_0085', 'verse_0086', 'verse_0087', 'verse_0088', 'verse_0090', 'verse_0091', 'verse_0092', 'verse_0093', 'verse_0095', 'verse_0098', 'verse_0099', 'verse_0100', 'verse_0101', 'verse_0102', 'verse_0104', 'verse_0105', 'verse_0106', 'verse_0107', 'verse_0108', 'verse_0109', 'verse_0110', 'verse_0111', 'verse_0112', 'verse_0113', 'verse_0114', 'verse_0116', 'verse_0121', 'verse_0122', 'verse_0123', 'verse_0124', 'verse_0125', 'verse_0126', 'verse_0127', 'verse_0128', 'verse_0129', 'verse_0130', 'verse_0131', 'verse_0132', 'verse_0133', 'verse_0134', 'verse_0135', 'verse_0136', 'verse_0137', 'verse_0138', 'verse_0139', 'verse_0142', 'verse_0143', 'verse_0144', 'verse_0146', 'verse_0148', 'verse_0149', 'verse_0151', 'verse_0152', 'verse_0153', 'verse_0154', 'verse_0155', 'verse_0156', 'verse_0157', 'verse_0158', 'verse_0160', 'verse_0162', 'verse_0164', 'verse_0166', 'verse_0168', 'verse_0169', 'verse_0170', 'verse_0171', 'verse_0172', 'verse_0173', 'verse_0174', 'verse_0175', 'verse_0176', 'verse_0177', 'verse_0179', 'verse_0180', 'verse_0181', 'verse_0183', 'verse_0184', 'verse_0185', 'verse_0187', 'verse_0188', 'verse_0189', 'verse_0190', 'verse_0191', 'verse_0192', 'verse_0193', 'verse_0194', 'verse_0195', 'verse_0196', 'verse_0197', 'verse_0198', 'verse_0199', 'verse_0200', 'verse_0201', 'verse_0202', 'verse_0203', 'verse_0204', 'verse_0205', 'verse_0206', 'verse_0207', 'verse_0208', 'verse_0210', 'verse_0212', 'verse_0214', 'verse_0215', 'verse_0216', 'verse_0217', 'verse_0218', 'verse_0219', 'verse_0220', 'verse_0221', 'verse_0223', 'verse_0224', 'verse_0225', 'verse_0226', 'verse_0227', 'verse_0229', 'verse_0230', 'verse_0231', 'verse_0232', 'verse_0234', 'verse_0235', 'verse_0236', 'verse_0237', 'verse_0238', 'verse_0239', 'verse_0240', 'verse_0243', 'verse_0245', 'verse_0247', 'verse_0248', 'verse_0249', 'verse_0250', 'verse_0251', 'verse_0252', 'verse_0254', 'verse_0255', 'verse_0256', 'verse_0257', 'verse_0259', 'verse_0260', 'verse_0261', 'verse_0262', 'verse_0263', 'verse_0265', 'verse_0266', 'verse_0267', 'verse_0268', 'verse_0270', 'verse_0271', 'verse_0272', 'verse_0273', 'verse_0274', 'verse_0275', 'verse_0276', 'verse_0278', 'verse_0280', 'verse_0281', 'verse_0282', 'verse_0283', 'verse_0284', 'verse_0286', 'verse_0287', 'verse_0289', 'verse_0290', 'verse_0291', 'verse_0292', 'verse_0293', 'verse_0294', 'verse_0295', 'verse_0296', 'verse_0297', 'verse_0298', 'verse_0299', 'verse_0302', 'verse_0303', 'verse_0304', 'verse_0305', 'verse_0306', 'verse_0307', 'verse_0308', 'verse_0309', 'verse_0310', 'verse_0311', 'verse_0312', 'verse_0313', 'verse_0314', 'verse_0315', 'verse_0316', 'verse_0317', 'verse_0318', 'verse_0319', 'verse_0320', 'verse_0321', 'verse_0322', 'verse_0323', 'verse_0324', 'verse_0325', 'verse_0326', 'verse_0327', 'verse_0328', 'verse_0329', 'verse_0332', 'verse_0333', 'verse_0334', 'verse_0335', 'verse_0338', 'verse_0339', 'verse_0340', 'verse_0341', 'verse_0342', 'verse_0344', 'verse_0345', 'verse_0346', 'verse_0347', 'verse_0349', 'verse_0351', 'verse_0352', 'verse_0353', 'verse_0355', 'verse_0357', 'verse_0358', 'verse_0359', 'verse_0360', 'verse_0361', 'verse_0363', 'verse_0364', 'verse_0365', 'verse_0366', 'verse_0367', 'verse_0368', 'verse_0369', 'verse_0370', 'verse_0371', 'verse_0372', 'verse_0373']) 
+2023-04-04 14:31:56.134455: VALIDATION KEYS:
+ odict_keys(['ts_0000', 'ts_0002', 'ts_0003', 'ts_0008', 'ts_0017', 'ts_0020', 'ts_0024', 'ts_0029', 'ts_0030', 'ts_0052', 'ts_0056', 'ts_0064', 'ts_0070', 'ts_0072', 'ts_0073', 'ts_0076', 'ts_0087', 'ts_0092', 'ts_0098', 'ts_0101', 'ts_0104', 'ts_0111', 'ts_0115', 'ts_0127', 'ts_0131', 'ts_0136', 'ts_0137', 'ts_0147', 'ts_0149', 'ts_0150', 'ts_0151', 'ts_0153', 'ts_0154', 'ts_0162', 'ts_0164', 'ts_0167', 'ts_0180', 'ts_0187', 'ts_0188', 'ts_0190', 'ts_0193', 'ts_0198', 'ts_0201', 'ts_0206', 'ts_0207', 'ts_0208', 'ts_0219', 'ts_0220', 'ts_0221', 'ts_0223', 'ts_0227', 'ts_0229', 'ts_0238', 'ts_0243', 'ts_0260', 'ts_0262', 'ts_0264', 'ts_0266', 'ts_0268', 'ts_0277', 'ts_0281', 'ts_0282', 'ts_0283', 'ts_0284', 'ts_0294', 'ts_0298', 'ts_0302', 'ts_0308', 'ts_0311', 'ts_0314', 'ts_0317', 'ts_0324', 'ts_0328', 'ts_0335', 'ts_0338', 'ts_0347', 'ts_0357', 'ts_0358', 'ts_0361', 'ts_0372', 'ts_0380', 'ts_0383', 'ts_0390', 'ts_0391', 'ts_0392', 'ts_0394', 'ts_0395', 'ts_0403', 'ts_0413', 'ts_0418', 'ts_0423', 'ts_0426', 'ts_0427', 'ts_0428', 'ts_0436', 'ts_0455', 'ts_0456', 'ts_0463', 'ts_0465', 'ts_0471', 'ts_0486', 'ts_0499', 'ts_0510', 'ts_0511', 'ts_0513', 'ts_0521', 'ts_0523', 'ts_0526', 'ts_0536', 'ts_0540', 'ts_0545', 'ts_0549', 'ts_0550', 'ts_0551', 'ts_0558', 'ts_0563', 'ts_0565', 'ts_0568', 'ts_0571', 'ts_0581', 'ts_0589', 'ts_0590', 'ts_0593', 'ts_0596', 'ts_0598', 'ts_0603', 'ts_0611', 'ts_0617', 'ts_0618', 'ts_0624', 'ts_0638', 'ts_0641', 'ts_0648', 'ts_0650', 'ts_0655', 'ts_0658', 'ts_0659', 'ts_0663', 'ts_0664', 'ts_0665', 'ts_0666', 'ts_0667', 'ts_0675', 'ts_0682', 'ts_0685', 'ts_0686', 'ts_0692', 'ts_0694', 'ts_0698', 'ts_0705', 'ts_0710', 'ts_0716', 'ts_0728', 'ts_0732', 'ts_0733', 'ts_0736', 'ts_0746', 'ts_0763', 'ts_0767', 'ts_0768', 'ts_0785', 'ts_0793', 'ts_0794', 'ts_0795', 'ts_0806', 'ts_0808', 'ts_0809', 'ts_0814', 'ts_0826', 'ts_0829', 'ts_0838', 'ts_0840', 'ts_0848', 'ts_0849', 'ts_0854', 'ts_0857', 'ts_0865', 'ts_0866', 'ts_0867', 'ts_0871', 'ts_0872', 'ts_0877', 'ts_0887', 'ts_0896', 'ts_0901', 'ts_0905', 'ts_0915', 'ts_0925', 'ts_0927', 'ts_0938', 'ts_0940', 'ts_0943', 'ts_0954', 'ts_0955', 'ts_0959', 'ts_0961', 'ts_0963', 'ts_0965', 'ts_0966', 'ts_0980', 'ts_0987', 'ts_0993', 'ts_0994', 'ts_0998', 'ts_1001', 'ts_1007', 'ts_1012', 'ts_1019', 'ts_1022', 'ts_1028', 'ts_1042', 'ts_1046', 'ts_1047', 'ts_1049', 'ts_1057', 'ts_1065', 'ts_1067', 'ts_1069', 'ts_1073', 'ts_1076', 'ts_1078', 'ts_1085', 'ts_1091', 'ts_1092', 'ts_1095', 'ts_1097', 'ts_1105', 'ts_1108', 'ts_1113', 'ts_1117', 'ts_1125', 'ts_1126', 'ts_1129', 'ts_1134', 'ts_1135', 'ts_1151', 'ts_1152', 'ts_1158', 'ts_1159', 'ts_1162', 'ts_1171', 'ts_1173', 'ts_1181', 'ts_1182', 'ts_1185', 'ts_1187', 'ts_1188', 'ts_1199', 'ts_1201', 'verse_0006', 'verse_0008', 'verse_0015', 'verse_0018', 'verse_0033', 'verse_0040', 'verse_0041', 'verse_0044', 'verse_0046', 'verse_0049', 'verse_0050', 'verse_0057', 'verse_0061', 'verse_0067', 'verse_0089', 'verse_0094', 'verse_0096', 'verse_0097', 'verse_0103', 'verse_0115', 'verse_0117', 'verse_0118', 'verse_0119', 'verse_0120', 'verse_0140', 'verse_0141', 'verse_0145', 'verse_0147', 'verse_0150', 'verse_0159', 'verse_0161', 'verse_0163', 'verse_0165', 'verse_0167', 'verse_0178', 'verse_0182', 'verse_0186', 'verse_0209', 'verse_0211', 'verse_0213', 'verse_0222', 'verse_0228', 'verse_0233', 'verse_0241', 'verse_0242', 'verse_0244', 'verse_0246', 'verse_0253', 'verse_0258', 'verse_0264', 'verse_0269', 'verse_0277', 'verse_0279', 'verse_0285', 'verse_0288', 'verse_0300', 'verse_0301', 'verse_0330', 'verse_0331', 'verse_0336', 'verse_0337', 'verse_0343', 'verse_0348', 'verse_0350', 'verse_0354', 'verse_0356', 'verse_0362']) 

fold_0/training_log_2023_4_4_14_34_07.txt

@@ -0,0 +1,9 @@
+Starting... 
+2023-04-04 14:34:07.442763: Using splits from existing split file: /dataNAS/people/lblankem/nnunet_spine_data/nnUNet_preprocessed/Task002_SpineV2/splits_final.pkl 
+2023-04-04 14:34:07.449346: The split file contains 5 splits. 
+2023-04-04 14:34:07.450753: Desired fold for training: 0 
+2023-04-04 14:34:07.452082: This split has 1262 training and 316 validation cases. 
+2023-04-04 14:34:12.082440: TRAINING KEYS:
+ odict_keys(['ts_0001', 'ts_0004', 'ts_0005', 'ts_0006', 'ts_0007', 'ts_0009', 'ts_0010', 'ts_0011', 'ts_0012', 'ts_0013', 'ts_0014', 'ts_0015', 'ts_0016', 'ts_0018', 'ts_0019', 'ts_0021', 'ts_0022', 'ts_0023', 'ts_0025', 'ts_0026', 'ts_0027', 'ts_0028', 'ts_0031', 'ts_0032', 'ts_0033', 'ts_0034', 'ts_0035', 'ts_0036', 'ts_0037', 'ts_0038', 'ts_0039', 'ts_0040', 'ts_0041', 'ts_0042', 'ts_0043', 'ts_0044', 'ts_0045', 'ts_0046', 'ts_0047', 'ts_0048', 'ts_0049', 'ts_0050', 'ts_0051', 'ts_0053', 'ts_0054', 'ts_0055', 'ts_0057', 'ts_0058', 'ts_0059', 'ts_0060', 'ts_0061', 'ts_0062', 'ts_0063', 'ts_0065', 'ts_0066', 'ts_0067', 'ts_0068', 'ts_0069', 'ts_0071', 'ts_0074', 'ts_0075', 'ts_0077', 'ts_0078', 'ts_0079', 'ts_0080', 'ts_0081', 'ts_0082', 'ts_0083', 'ts_0084', 'ts_0085', 'ts_0086', 'ts_0088', 'ts_0089', 'ts_0090', 'ts_0091', 'ts_0093', 'ts_0094', 'ts_0095', 'ts_0096', 'ts_0097', 'ts_0099', 'ts_0100', 'ts_0102', 'ts_0103', 'ts_0105', 'ts_0106', 'ts_0107', 'ts_0108', 'ts_0109', 'ts_0110', 'ts_0112', 'ts_0113', 'ts_0114', 'ts_0116', 'ts_0117', 'ts_0118', 'ts_0119', 'ts_0120', 'ts_0121', 'ts_0122', 'ts_0123', 'ts_0124', 'ts_0125', 'ts_0126', 'ts_0128', 'ts_0129', 'ts_0130', 'ts_0132', 'ts_0133', 'ts_0134', 'ts_0135', 'ts_0138', 'ts_0139', 'ts_0140', 'ts_0141', 'ts_0142', 'ts_0143', 'ts_0144', 'ts_0145', 'ts_0146', 'ts_0148', 'ts_0152', 'ts_0155', 'ts_0156', 'ts_0157', 'ts_0158', 'ts_0159', 'ts_0160', 'ts_0161', 'ts_0163', 'ts_0165', 'ts_0166', 'ts_0168', 'ts_0169', 'ts_0170', 'ts_0171', 'ts_0172', 'ts_0173', 'ts_0174', 'ts_0175', 'ts_0176', 'ts_0177', 'ts_0178', 'ts_0179', 'ts_0181', 'ts_0182', 'ts_0183', 'ts_0184', 'ts_0185', 'ts_0186', 'ts_0189', 'ts_0191', 'ts_0192', 'ts_0194', 'ts_0195', 'ts_0196', 'ts_0197', 'ts_0199', 'ts_0200', 'ts_0202', 'ts_0203', 'ts_0204', 'ts_0205', 'ts_0209', 'ts_0210', 'ts_0211', 'ts_0212', 'ts_0213', 'ts_0214', 'ts_0215', 'ts_0216', 'ts_0217', 'ts_0218', 'ts_0222', 'ts_0224', 'ts_0225', 'ts_0226', 'ts_0228', 'ts_0230', 'ts_0231', 'ts_0232', 'ts_0233', 'ts_0234', 'ts_0235', 'ts_0236', 'ts_0237', 'ts_0239', 'ts_0240', 'ts_0241', 'ts_0242', 'ts_0244', 'ts_0245', 'ts_0246', 'ts_0247', 'ts_0248', 'ts_0249', 'ts_0250', 'ts_0251', 'ts_0252', 'ts_0253', 'ts_0254', 'ts_0255', 'ts_0256', 'ts_0257', 'ts_0258', 'ts_0259', 'ts_0261', 'ts_0263', 'ts_0265', 'ts_0267', 'ts_0269', 'ts_0270', 'ts_0271', 'ts_0272', 'ts_0273', 'ts_0274', 'ts_0275', 'ts_0276', 'ts_0278', 'ts_0279', 'ts_0280', 'ts_0285', 'ts_0286', 'ts_0287', 'ts_0288', 'ts_0289', 'ts_0290', 'ts_0291', 'ts_0292', 'ts_0293', 'ts_0295', 'ts_0296', 'ts_0297', 'ts_0299', 'ts_0300', 'ts_0301', 'ts_0303', 'ts_0304', 'ts_0305', 'ts_0306', 'ts_0307', 'ts_0309', 'ts_0310', 'ts_0312', 'ts_0313', 'ts_0315', 'ts_0316', 'ts_0318', 'ts_0319', 'ts_0320', 'ts_0321', 'ts_0322', 'ts_0323', 'ts_0325', 'ts_0326', 'ts_0327', 'ts_0329', 'ts_0330', 'ts_0331', 'ts_0332', 'ts_0333', 'ts_0334', 'ts_0336', 'ts_0337', 'ts_0339', 'ts_0340', 'ts_0341', 'ts_0342', 'ts_0343', 'ts_0344', 'ts_0345', 'ts_0346', 'ts_0348', 'ts_0349', 'ts_0350', 'ts_0351', 'ts_0352', 'ts_0353', 'ts_0354', 'ts_0355', 'ts_0356', 'ts_0359', 'ts_0360', 'ts_0362', 'ts_0363', 'ts_0364', 'ts_0365', 'ts_0366', 'ts_0367', 'ts_0368', 'ts_0369', 'ts_0370', 'ts_0371', 'ts_0373', 'ts_0374', 'ts_0375', 'ts_0376', 'ts_0377', 'ts_0378', 'ts_0379', 'ts_0381', 'ts_0382', 'ts_0384', 'ts_0385', 'ts_0386', 'ts_0387', 'ts_0388', 'ts_0389', 'ts_0393', 'ts_0396', 'ts_0397', 'ts_0398', 'ts_0399', 'ts_0400', 'ts_0401', 'ts_0402', 'ts_0404', 'ts_0405', 'ts_0406', 'ts_0407', 'ts_0408', 'ts_0409', 'ts_0410', 'ts_0411', 'ts_0412', 'ts_0414', 'ts_0415', 'ts_0416', 'ts_0417', 'ts_0419', 'ts_0420', 'ts_0421', 'ts_0422', 'ts_0424', 'ts_0425', 'ts_0429', 'ts_0430', 'ts_0431', 'ts_0432', 'ts_0433', 'ts_0434', 'ts_0435', 'ts_0437', 'ts_0438', 'ts_0439', 'ts_0440', 'ts_0441', 'ts_0442', 'ts_0443', 'ts_0444', 'ts_0445', 'ts_0446', 'ts_0447', 'ts_0448', 'ts_0449', 'ts_0450', 'ts_0451', 'ts_0452', 'ts_0453', 'ts_0454', 'ts_0457', 'ts_0458', 'ts_0459', 'ts_0460', 'ts_0461', 'ts_0462', 'ts_0464', 'ts_0466', 'ts_0467', 'ts_0468', 'ts_0469', 'ts_0470', 'ts_0472', 'ts_0473', 'ts_0474', 'ts_0475', 'ts_0476', 'ts_0477', 'ts_0478', 'ts_0479', 'ts_0480', 'ts_0481', 'ts_0482', 'ts_0483', 'ts_0484', 'ts_0485', 'ts_0487', 'ts_0488', 'ts_0489', 'ts_0490', 'ts_0491', 'ts_0492', 'ts_0493', 'ts_0494', 'ts_0495', 'ts_0496', 'ts_0497', 'ts_0498', 'ts_0500', 'ts_0501', 'ts_0502', 'ts_0503', 'ts_0504', 'ts_0505', 'ts_0506', 'ts_0507', 'ts_0508', 'ts_0509', 'ts_0512', 'ts_0514', 'ts_0515', 'ts_0516', 'ts_0517', 'ts_0518', 'ts_0519', 'ts_0520', 'ts_0522', 'ts_0524', 'ts_0525', 'ts_0527', 'ts_0528', 'ts_0529', 'ts_0530', 'ts_0531', 'ts_0532', 'ts_0533', 'ts_0534', 'ts_0535', 'ts_0537', 'ts_0538', 'ts_0539', 'ts_0541', 'ts_0542', 'ts_0543', 'ts_0544', 'ts_0546', 'ts_0547', 'ts_0548', 'ts_0552', 'ts_0553', 'ts_0554', 'ts_0555', 'ts_0556', 'ts_0557', 'ts_0559', 'ts_0560', 'ts_0561', 'ts_0562', 'ts_0564', 'ts_0566', 'ts_0567', 'ts_0569', 'ts_0570', 'ts_0572', 'ts_0573', 'ts_0574', 'ts_0575', 'ts_0576', 'ts_0577', 'ts_0578', 'ts_0579', 'ts_0580', 'ts_0582', 'ts_0583', 'ts_0584', 'ts_0585', 'ts_0586', 'ts_0587', 'ts_0588', 'ts_0591', 'ts_0592', 'ts_0594', 'ts_0595', 'ts_0597', 'ts_0599', 'ts_0600', 'ts_0601', 'ts_0602', 'ts_0604', 'ts_0605', 'ts_0606', 'ts_0607', 'ts_0608', 'ts_0609', 'ts_0610', 'ts_0612', 'ts_0613', 'ts_0614', 'ts_0615', 'ts_0616', 'ts_0619', 'ts_0620', 'ts_0621', 'ts_0622', 'ts_0623', 'ts_0625', 'ts_0626', 'ts_0627', 'ts_0628', 'ts_0629', 'ts_0630', 'ts_0631', 'ts_0632', 'ts_0633', 'ts_0634', 'ts_0635', 'ts_0636', 'ts_0637', 'ts_0639', 'ts_0640', 'ts_0642', 'ts_0643', 'ts_0644', 'ts_0645', 'ts_0646', 'ts_0647', 'ts_0649', 'ts_0651', 'ts_0652', 'ts_0653', 'ts_0654', 'ts_0656', 'ts_0657', 'ts_0660', 'ts_0661', 'ts_0662', 'ts_0668', 'ts_0669', 'ts_0670', 'ts_0671', 'ts_0672', 'ts_0673', 'ts_0674', 'ts_0676', 'ts_0677', 'ts_0678', 'ts_0679', 'ts_0680', 'ts_0681', 'ts_0683', 'ts_0684', 'ts_0687', 'ts_0688', 'ts_0689', 'ts_0690', 'ts_0691', 'ts_0693', 'ts_0695', 'ts_0696', 'ts_0697', 'ts_0699', 'ts_0700', 'ts_0701', 'ts_0702', 'ts_0703', 'ts_0704', 'ts_0706', 'ts_0707', 'ts_0708', 'ts_0709', 'ts_0711', 'ts_0712', 'ts_0713', 'ts_0714', 'ts_0715', 'ts_0717', 'ts_0718', 'ts_0719', 'ts_0720', 'ts_0721', 'ts_0722', 'ts_0723', 'ts_0724', 'ts_0725', 'ts_0726', 'ts_0727', 'ts_0729', 'ts_0730', 'ts_0731', 'ts_0734', 'ts_0735', 'ts_0737', 'ts_0738', 'ts_0739', 'ts_0740', 'ts_0741', 'ts_0742', 'ts_0743', 'ts_0744', 'ts_0745', 'ts_0747', 'ts_0748', 'ts_0749', 'ts_0750', 'ts_0751', 'ts_0752', 'ts_0753', 'ts_0754', 'ts_0755', 'ts_0756', 'ts_0757', 'ts_0758', 'ts_0759', 'ts_0760', 'ts_0761', 'ts_0762', 'ts_0764', 'ts_0765', 'ts_0766', 'ts_0769', 'ts_0770', 'ts_0771', 'ts_0772', 'ts_0773', 'ts_0774', 'ts_0775', 'ts_0776', 'ts_0777', 'ts_0778', 'ts_0779', 'ts_0780', 'ts_0781', 'ts_0782', 'ts_0783', 'ts_0784', 'ts_0786', 'ts_0787', 'ts_0788', 'ts_0789', 'ts_0790', 'ts_0791', 'ts_0792', 'ts_0796', 'ts_0797', 'ts_0798', 'ts_0799', 'ts_0800', 'ts_0801', 'ts_0802', 'ts_0803', 'ts_0804', 'ts_0805', 'ts_0807', 'ts_0810', 'ts_0811', 'ts_0812', 'ts_0813', 'ts_0815', 'ts_0816', 'ts_0817', 'ts_0818', 'ts_0819', 'ts_0820', 'ts_0821', 'ts_0822', 'ts_0823', 'ts_0824', 'ts_0825', 'ts_0827', 'ts_0828', 'ts_0830', 'ts_0831', 'ts_0832', 'ts_0833', 'ts_0834', 'ts_0835', 'ts_0836', 'ts_0837', 'ts_0839', 'ts_0841', 'ts_0842', 'ts_0843', 'ts_0844', 'ts_0845', 'ts_0846', 'ts_0847', 'ts_0850', 'ts_0851', 'ts_0852', 'ts_0853', 'ts_0855', 'ts_0856', 'ts_0858', 'ts_0859', 'ts_0860', 'ts_0861', 'ts_0862', 'ts_0863', 'ts_0864', 'ts_0868', 'ts_0869', 'ts_0870', 'ts_0873', 'ts_0874', 'ts_0875', 'ts_0876', 'ts_0878', 'ts_0879', 'ts_0880', 'ts_0881', 'ts_0882', 'ts_0883', 'ts_0884', 'ts_0885', 'ts_0886', 'ts_0888', 'ts_0889', 'ts_0890', 'ts_0891', 'ts_0892', 'ts_0893', 'ts_0894', 'ts_0895', 'ts_0897', 'ts_0898', 'ts_0899', 'ts_0900', 'ts_0902', 'ts_0903', 'ts_0904', 'ts_0906', 'ts_0907', 'ts_0908', 'ts_0909', 'ts_0910', 'ts_0911', 'ts_0912', 'ts_0913', 'ts_0914', 'ts_0916', 'ts_0917', 'ts_0918', 'ts_0919', 'ts_0920', 'ts_0921', 'ts_0922', 'ts_0923', 'ts_0924', 'ts_0926', 'ts_0928', 'ts_0929', 'ts_0930', 'ts_0931', 'ts_0932', 'ts_0933', 'ts_0934', 'ts_0935', 'ts_0936', 'ts_0937', 'ts_0939', 'ts_0941', 'ts_0942', 'ts_0944', 'ts_0945', 'ts_0946', 'ts_0947', 'ts_0948', 'ts_0949', 'ts_0950', 'ts_0951', 'ts_0952', 'ts_0953', 'ts_0956', 'ts_0957', 'ts_0958', 'ts_0960', 'ts_0962', 'ts_0964', 'ts_0967', 'ts_0968', 'ts_0969', 'ts_0970', 'ts_0971', 'ts_0972', 'ts_0973', 'ts_0974', 'ts_0975', 'ts_0976', 'ts_0977', 'ts_0978', 'ts_0979', 'ts_0981', 'ts_0982', 'ts_0983', 'ts_0984', 'ts_0985', 'ts_0986', 'ts_0988', 'ts_0989', 'ts_0990', 'ts_0991', 'ts_0992', 'ts_0995', 'ts_0996', 'ts_0997', 'ts_0999', 'ts_1000', 'ts_1002', 'ts_1003', 'ts_1004', 'ts_1005', 'ts_1006', 'ts_1008', 'ts_1009', 'ts_1010', 'ts_1011', 'ts_1013', 'ts_1014', 'ts_1015', 'ts_1016', 'ts_1017', 'ts_1018', 'ts_1020', 'ts_1021', 'ts_1023', 'ts_1024', 'ts_1025', 'ts_1026', 'ts_1027', 'ts_1029', 'ts_1030', 'ts_1031', 'ts_1032', 'ts_1033', 'ts_1034', 'ts_1035', 'ts_1036', 'ts_1037', 'ts_1038', 'ts_1039', 'ts_1040', 'ts_1041', 'ts_1043', 'ts_1044', 'ts_1045', 'ts_1048', 'ts_1050', 'ts_1051', 'ts_1052', 'ts_1053', 'ts_1054', 'ts_1055', 'ts_1056', 'ts_1058', 'ts_1059', 'ts_1060', 'ts_1061', 'ts_1062', 'ts_1063', 'ts_1064', 'ts_1066', 'ts_1068', 'ts_1070', 'ts_1071', 'ts_1072', 'ts_1074', 'ts_1075', 'ts_1077', 'ts_1079', 'ts_1080', 'ts_1081', 'ts_1082', 'ts_1083', 'ts_1084', 'ts_1086', 'ts_1087', 'ts_1088', 'ts_1089', 'ts_1090', 'ts_1093', 'ts_1094', 'ts_1096', 'ts_1098', 'ts_1099', 'ts_1100', 'ts_1101', 'ts_1102', 'ts_1103', 'ts_1104', 'ts_1106', 'ts_1107', 'ts_1109', 'ts_1110', 'ts_1111', 'ts_1112', 'ts_1114', 'ts_1115', 'ts_1116', 'ts_1118', 'ts_1119', 'ts_1120', 'ts_1121', 'ts_1122', 'ts_1123', 'ts_1124', 'ts_1127', 'ts_1128', 'ts_1130', 'ts_1131', 'ts_1132', 'ts_1133', 'ts_1136', 'ts_1137', 'ts_1138', 'ts_1139', 'ts_1140', 'ts_1141', 'ts_1142', 'ts_1143', 'ts_1144', 'ts_1145', 'ts_1146', 'ts_1147', 'ts_1148', 'ts_1149', 'ts_1150', 'ts_1153', 'ts_1154', 'ts_1155', 'ts_1156', 'ts_1157', 'ts_1160', 'ts_1161', 'ts_1163', 'ts_1164', 'ts_1165', 'ts_1166', 'ts_1167', 'ts_1168', 'ts_1169', 'ts_1170', 'ts_1172', 'ts_1174', 'ts_1175', 'ts_1176', 'ts_1177', 'ts_1178', 'ts_1179', 'ts_1180', 'ts_1183', 'ts_1184', 'ts_1186', 'ts_1189', 'ts_1190', 'ts_1191', 'ts_1192', 'ts_1193', 'ts_1194', 'ts_1195', 'ts_1196', 'ts_1197', 'ts_1198', 'ts_1200', 'ts_1202', 'ts_1203', 'verse_0000', 'verse_0001', 'verse_0002', 'verse_0003', 'verse_0004', 'verse_0005', 'verse_0007', 'verse_0009', 'verse_0010', 'verse_0011', 'verse_0012', 'verse_0013', 'verse_0014', 'verse_0016', 'verse_0017', 'verse_0019', 'verse_0020', 'verse_0021', 'verse_0022', 'verse_0023', 'verse_0024', 'verse_0025', 'verse_0026', 'verse_0027', 'verse_0028', 'verse_0029', 'verse_0030', 'verse_0031', 'verse_0032', 'verse_0034', 'verse_0035', 'verse_0036', 'verse_0037', 'verse_0038', 'verse_0039', 'verse_0042', 'verse_0043', 'verse_0045', 'verse_0047', 'verse_0048', 'verse_0051', 'verse_0052', 'verse_0053', 'verse_0054', 'verse_0055', 'verse_0056', 'verse_0058', 'verse_0059', 'verse_0060', 'verse_0062', 'verse_0063', 'verse_0064', 'verse_0065', 'verse_0066', 'verse_0068', 'verse_0069', 'verse_0070', 'verse_0071', 'verse_0072', 'verse_0073', 'verse_0074', 'verse_0075', 'verse_0076', 'verse_0077', 'verse_0078', 'verse_0079', 'verse_0080', 'verse_0081', 'verse_0082', 'verse_0083', 'verse_0084', 'verse_0085', 'verse_0086', 'verse_0087', 'verse_0088', 'verse_0090', 'verse_0091', 'verse_0092', 'verse_0093', 'verse_0095', 'verse_0098', 'verse_0099', 'verse_0100', 'verse_0101', 'verse_0102', 'verse_0104', 'verse_0105', 'verse_0106', 'verse_0107', 'verse_0108', 'verse_0109', 'verse_0110', 'verse_0111', 'verse_0112', 'verse_0113', 'verse_0114', 'verse_0116', 'verse_0121', 'verse_0122', 'verse_0123', 'verse_0124', 'verse_0125', 'verse_0126', 'verse_0127', 'verse_0128', 'verse_0129', 'verse_0130', 'verse_0131', 'verse_0132', 'verse_0133', 'verse_0134', 'verse_0135', 'verse_0136', 'verse_0137', 'verse_0138', 'verse_0139', 'verse_0142', 'verse_0143', 'verse_0144', 'verse_0146', 'verse_0148', 'verse_0149', 'verse_0151', 'verse_0152', 'verse_0153', 'verse_0154', 'verse_0155', 'verse_0156', 'verse_0157', 'verse_0158', 'verse_0160', 'verse_0162', 'verse_0164', 'verse_0166', 'verse_0168', 'verse_0169', 'verse_0170', 'verse_0171', 'verse_0172', 'verse_0173', 'verse_0174', 'verse_0175', 'verse_0176', 'verse_0177', 'verse_0179', 'verse_0180', 'verse_0181', 'verse_0183', 'verse_0184', 'verse_0185', 'verse_0187', 'verse_0188', 'verse_0189', 'verse_0190', 'verse_0191', 'verse_0192', 'verse_0193', 'verse_0194', 'verse_0195', 'verse_0196', 'verse_0197', 'verse_0198', 'verse_0199', 'verse_0200', 'verse_0201', 'verse_0202', 'verse_0203', 'verse_0204', 'verse_0205', 'verse_0206', 'verse_0207', 'verse_0208', 'verse_0210', 'verse_0212', 'verse_0214', 'verse_0215', 'verse_0216', 'verse_0217', 'verse_0218', 'verse_0219', 'verse_0220', 'verse_0221', 'verse_0223', 'verse_0224', 'verse_0225', 'verse_0226', 'verse_0227', 'verse_0229', 'verse_0230', 'verse_0231', 'verse_0232', 'verse_0234', 'verse_0235', 'verse_0236', 'verse_0237', 'verse_0238', 'verse_0239', 'verse_0240', 'verse_0243', 'verse_0245', 'verse_0247', 'verse_0248', 'verse_0249', 'verse_0250', 'verse_0251', 'verse_0252', 'verse_0254', 'verse_0255', 'verse_0256', 'verse_0257', 'verse_0259', 'verse_0260', 'verse_0261', 'verse_0262', 'verse_0263', 'verse_0265', 'verse_0266', 'verse_0267', 'verse_0268', 'verse_0270', 'verse_0271', 'verse_0272', 'verse_0273', 'verse_0274', 'verse_0275', 'verse_0276', 'verse_0278', 'verse_0280', 'verse_0281', 'verse_0282', 'verse_0283', 'verse_0284', 'verse_0286', 'verse_0287', 'verse_0289', 'verse_0290', 'verse_0291', 'verse_0292', 'verse_0293', 'verse_0294', 'verse_0295', 'verse_0296', 'verse_0297', 'verse_0298', 'verse_0299', 'verse_0302', 'verse_0303', 'verse_0304', 'verse_0305', 'verse_0306', 'verse_0307', 'verse_0308', 'verse_0309', 'verse_0310', 'verse_0311', 'verse_0312', 'verse_0313', 'verse_0314', 'verse_0315', 'verse_0316', 'verse_0317', 'verse_0318', 'verse_0319', 'verse_0320', 'verse_0321', 'verse_0322', 'verse_0323', 'verse_0324', 'verse_0325', 'verse_0326', 'verse_0327', 'verse_0328', 'verse_0329', 'verse_0332', 'verse_0333', 'verse_0334', 'verse_0335', 'verse_0338', 'verse_0339', 'verse_0340', 'verse_0341', 'verse_0342', 'verse_0344', 'verse_0345', 'verse_0346', 'verse_0347', 'verse_0349', 'verse_0351', 'verse_0352', 'verse_0353', 'verse_0355', 'verse_0357', 'verse_0358', 'verse_0359', 'verse_0360', 'verse_0361', 'verse_0363', 'verse_0364', 'verse_0365', 'verse_0366', 'verse_0367', 'verse_0368', 'verse_0369', 'verse_0370', 'verse_0371', 'verse_0372', 'verse_0373']) 
+2023-04-04 14:34:12.086366: VALIDATION KEYS:
+ odict_keys(['ts_0000', 'ts_0002', 'ts_0003', 'ts_0008', 'ts_0017', 'ts_0020', 'ts_0024', 'ts_0029', 'ts_0030', 'ts_0052', 'ts_0056', 'ts_0064', 'ts_0070', 'ts_0072', 'ts_0073', 'ts_0076', 'ts_0087', 'ts_0092', 'ts_0098', 'ts_0101', 'ts_0104', 'ts_0111', 'ts_0115', 'ts_0127', 'ts_0131', 'ts_0136', 'ts_0137', 'ts_0147', 'ts_0149', 'ts_0150', 'ts_0151', 'ts_0153', 'ts_0154', 'ts_0162', 'ts_0164', 'ts_0167', 'ts_0180', 'ts_0187', 'ts_0188', 'ts_0190', 'ts_0193', 'ts_0198', 'ts_0201', 'ts_0206', 'ts_0207', 'ts_0208', 'ts_0219', 'ts_0220', 'ts_0221', 'ts_0223', 'ts_0227', 'ts_0229', 'ts_0238', 'ts_0243', 'ts_0260', 'ts_0262', 'ts_0264', 'ts_0266', 'ts_0268', 'ts_0277', 'ts_0281', 'ts_0282', 'ts_0283', 'ts_0284', 'ts_0294', 'ts_0298', 'ts_0302', 'ts_0308', 'ts_0311', 'ts_0314', 'ts_0317', 'ts_0324', 'ts_0328', 'ts_0335', 'ts_0338', 'ts_0347', 'ts_0357', 'ts_0358', 'ts_0361', 'ts_0372', 'ts_0380', 'ts_0383', 'ts_0390', 'ts_0391', 'ts_0392', 'ts_0394', 'ts_0395', 'ts_0403', 'ts_0413', 'ts_0418', 'ts_0423', 'ts_0426', 'ts_0427', 'ts_0428', 'ts_0436', 'ts_0455', 'ts_0456', 'ts_0463', 'ts_0465', 'ts_0471', 'ts_0486', 'ts_0499', 'ts_0510', 'ts_0511', 'ts_0513', 'ts_0521', 'ts_0523', 'ts_0526', 'ts_0536', 'ts_0540', 'ts_0545', 'ts_0549', 'ts_0550', 'ts_0551', 'ts_0558', 'ts_0563', 'ts_0565', 'ts_0568', 'ts_0571', 'ts_0581', 'ts_0589', 'ts_0590', 'ts_0593', 'ts_0596', 'ts_0598', 'ts_0603', 'ts_0611', 'ts_0617', 'ts_0618', 'ts_0624', 'ts_0638', 'ts_0641', 'ts_0648', 'ts_0650', 'ts_0655', 'ts_0658', 'ts_0659', 'ts_0663', 'ts_0664', 'ts_0665', 'ts_0666', 'ts_0667', 'ts_0675', 'ts_0682', 'ts_0685', 'ts_0686', 'ts_0692', 'ts_0694', 'ts_0698', 'ts_0705', 'ts_0710', 'ts_0716', 'ts_0728', 'ts_0732', 'ts_0733', 'ts_0736', 'ts_0746', 'ts_0763', 'ts_0767', 'ts_0768', 'ts_0785', 'ts_0793', 'ts_0794', 'ts_0795', 'ts_0806', 'ts_0808', 'ts_0809', 'ts_0814', 'ts_0826', 'ts_0829', 'ts_0838', 'ts_0840', 'ts_0848', 'ts_0849', 'ts_0854', 'ts_0857', 'ts_0865', 'ts_0866', 'ts_0867', 'ts_0871', 'ts_0872', 'ts_0877', 'ts_0887', 'ts_0896', 'ts_0901', 'ts_0905', 'ts_0915', 'ts_0925', 'ts_0927', 'ts_0938', 'ts_0940', 'ts_0943', 'ts_0954', 'ts_0955', 'ts_0959', 'ts_0961', 'ts_0963', 'ts_0965', 'ts_0966', 'ts_0980', 'ts_0987', 'ts_0993', 'ts_0994', 'ts_0998', 'ts_1001', 'ts_1007', 'ts_1012', 'ts_1019', 'ts_1022', 'ts_1028', 'ts_1042', 'ts_1046', 'ts_1047', 'ts_1049', 'ts_1057', 'ts_1065', 'ts_1067', 'ts_1069', 'ts_1073', 'ts_1076', 'ts_1078', 'ts_1085', 'ts_1091', 'ts_1092', 'ts_1095', 'ts_1097', 'ts_1105', 'ts_1108', 'ts_1113', 'ts_1117', 'ts_1125', 'ts_1126', 'ts_1129', 'ts_1134', 'ts_1135', 'ts_1151', 'ts_1152', 'ts_1158', 'ts_1159', 'ts_1162', 'ts_1171', 'ts_1173', 'ts_1181', 'ts_1182', 'ts_1185', 'ts_1187', 'ts_1188', 'ts_1199', 'ts_1201', 'verse_0006', 'verse_0008', 'verse_0015', 'verse_0018', 'verse_0033', 'verse_0040', 'verse_0041', 'verse_0044', 'verse_0046', 'verse_0049', 'verse_0050', 'verse_0057', 'verse_0061', 'verse_0067', 'verse_0089', 'verse_0094', 'verse_0096', 'verse_0097', 'verse_0103', 'verse_0115', 'verse_0117', 'verse_0118', 'verse_0119', 'verse_0120', 'verse_0140', 'verse_0141', 'verse_0145', 'verse_0147', 'verse_0150', 'verse_0159', 'verse_0161', 'verse_0163', 'verse_0165', 'verse_0167', 'verse_0178', 'verse_0182', 'verse_0186', 'verse_0209', 'verse_0211', 'verse_0213', 'verse_0222', 'verse_0228', 'verse_0233', 'verse_0241', 'verse_0242', 'verse_0244', 'verse_0246', 'verse_0253', 'verse_0258', 'verse_0264', 'verse_0269', 'verse_0277', 'verse_0279', 'verse_0285', 'verse_0288', 'verse_0300', 'verse_0301', 'verse_0330', 'verse_0331', 'verse_0336', 'verse_0337', 'verse_0343', 'verse_0348', 'verse_0350', 'verse_0354', 'verse_0356', 'verse_0362']) 

plans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea64022b7a8e35ee9e698134db6b9b50a836f62d4ef540173e26f41ca5af21ef
+size 731464