ds_inference_config.json

{"type": "ds_model", "base_dir": "/science/lucadiliello/models/facebook/sharded/opt-30b-deepspeed-inference-fp16-shard-2", "checkpoints": {"non_tp": ["non-tp.pt"], "tp": ["tp_00_00.pt", "tp_01_00.pt", "tp_00_01.pt", "tp_01_01.pt", "tp_00_02.pt", "tp_01_02.pt", "tp_00_03.pt", "tp_01_03.pt", "tp_00_04.pt", "tp_01_04.pt", "tp_00_05.pt", "tp_01_05.pt", "tp_00_06.pt", "tp_01_06.pt", "tp_00_07.pt", "tp_01_07.pt"]}, "version": 1.0, "parallelization": "tp", "tp_size": 2, "dtype": "float16"}