From d144b4879c2969aa31e52a945798529da256bc8b Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Tue, 9 Jun 2026 12:20:27 +0000 Subject: [PATCH 1/8] [None][test] Add support for nemotron_3_ultra_550b_nvfp4 model in performance tests and configurations - Updated model paths to include nemotron_3_ultra_550b_nvfp4 in HF_MODEL_PATH. - Added configuration settings for nemotron_3_ultra_550b_nvfp4 in pytorch_model_config.py. - Included new performance test cases for nemotron_3_ultra_550b_nvfp4 in test_perf.py and updated llm_perf_core.yml. - Cleaned up legacy model name handling in test_perf.py. Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- tests/integration/defs/perf/_model_paths.py | 11 ++-------- .../defs/perf/pytorch_model_config.py | 22 +++++++++++++++++++ tests/integration/defs/perf/test_perf.py | 21 +++++++++--------- .../test_lists/qa/llm_perf_core.yml | 6 +++++ .../test_lists/test-db/l0_perf.yml | 15 ------------- 5 files changed, 41 insertions(+), 34 deletions(-) diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index c7c021cae7ba..e995ce257f8f 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -123,16 +123,9 @@ "glm_5_nvfp4": "GLM-5-NVFP4", } -# Model PATH of HuggingFace +# Models loaded directly by HuggingFace repo id (downloaded at runtime, not synced locally). HF_MODEL_PATH = { - "llama_v3.1_8b_hf": "meta-llama/Llama-3.1-8B", - "llama_v3.1_8b_instruct_hf": "nvidia/Llama-3.1-8B-Instruct-FP8", - "llama_v3.1_nemotron_nano_8b_hf": "nvidia/Llama-3.1-Nemotron-Nano-8B-v1", - "llama_v3.1_nemotron_nano_8b_fp8_hf": "nvidia/Llama-3.1-Nemotron-Nano-8B-v1-FP8", - "llama_v3.3_nemotron_super_49b_hf": "nvidia/Llama-3_3-Nemotron-Super-49B-v1", - "llama_v3.3_nemotron_super_49b_fp8_hf": "nvidia/Llama-3_3-Nemotron-Super-49B-v1-FP8", - "llama_v3.1_nemotron_ultra_253b_fp8_hf": "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1-FP8", - "phi_4_mini_instruct_hf": "microsoft/Phi-4-mini-instruct", + "nemotron_3_ultra_550b_nvfp4": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", } LORA_MODEL_PATH = { diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 0dbf1a91ee1d..66d1dd334002 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -586,6 +586,28 @@ def get_model_yaml_config(model_label: str, }, } }, + # Nemotron-3-Ultra-550B-NVFP4 throughput variant, aligned with curated yaml (served from HF). + { + 'patterns': ['nemotron_3_ultra_550b_nvfp4-serve-pytorch-'], + 'config': { + 'enable_attention_dp': True, + 'stream_interval': 10, + 'num_postprocess_workers': 4, + 'moe_config': { + 'backend': 'CUTEDSL', + }, + 'cuda_graph_config': { + 'enable_padding': True, + 'max_batch_size': 256, + }, + 'kv_cache_config': { + 'enable_block_reuse': False, + 'mamba_ssm_cache_dtype': 'float16', + 'mamba_ssm_stochastic_rounding': True, + 'mamba_ssm_philox_rounds': 5, + }, + } + }, ] # Apply pattern-based configurations on top of base config diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index c7b70638f700..855b1886f31d 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -49,6 +49,7 @@ NEMOTRON_SUPER_MODELS = { "nemotron_3_super_120b_nvfp4", "nemotron_3_super_120b_nvfp4_mtp", + "nemotron_3_ultra_550b_nvfp4", "nemotron_3_nano_omni_nvfp4", "nemotron_3_nano_omni_nvfp4_image", } @@ -61,6 +62,7 @@ "kimi_k2_nvfp4", "nemotron_3_super_120b_nvfp4", "nemotron_3_super_120b_nvfp4_mtp", + "nemotron_3_ultra_550b_nvfp4", "glm_5_fp8", "nemotron_3_nano_omni_nvfp4", "nemotron_3_nano_omni_nvfp4_image", @@ -106,13 +108,12 @@ def get_model_dir(model_name: str): - model_dir = "" + # HF models use the repo id verbatim (downloaded at runtime, no LLM_MODELS_ROOT prefix). + if model_name in HF_MODEL_PATH.keys(): + return HF_MODEL_PATH[model_name] if model_name in MODEL_PATH_DICT.keys(): - model_dir = os.path.join(llm_models_root(), MODEL_PATH_DICT[model_name]) - elif model_name in HF_MODEL_PATH.keys(): - model_dir = os.path.join(llm_models_root(), - MODEL_PATH_DICT[model_name.split('_hf')[0]]) - return model_dir + return os.path.join(llm_models_root(), MODEL_PATH_DICT[model_name]) + return "" def get_dataset_path(): @@ -1038,13 +1039,13 @@ def get_trtllm_bench_build_command(self, engine_dir) -> list: model_dir = self.get_trtllm_bench_model() if model_dir == "": pytest.skip("Model Name is not supported by trtllm-bench") + # Legacy "_hf" label; weights load from --model_path. model_name = self._config.model_name if not model_name.endswith("_hf"): model_name = model_name + "_hf" - hf_model_name = HF_MODEL_PATH.get(model_name, "") build_cmd = [ self._build_script, f"--log_level=info", - f"--workspace={engine_dir}", f"--model={hf_model_name}", + f"--workspace={engine_dir}", f"--model={model_name}", f"--model_path={model_dir}", "build", f"--tp_size={self._config.tp_size}", f"--pp_size={self._config.pp_size}" @@ -1170,11 +1171,11 @@ def get_trtllm_bench_command(self, engine_dir): model_name = self._config.model_name dataset_path = os.path.join(engine_dir, "synthetic_data.json") report_path = os.path.join(engine_dir, "report.json") + # Legacy "_hf" label; weights load from --model_path. if not model_name.endswith("_hf"): model_name = model_name + "_hf" - hf_model_name = HF_MODEL_PATH.get(model_name, "") tp_pp_str = f"tp_{self._config.tp_size}_pp_{self._config.pp_size}" - engine_dir = os.path.join(engine_dir, hf_model_name, tp_pp_str) + engine_dir = os.path.join(engine_dir, tp_pp_str) benchmark_cmd = [ self._benchmark_script, f"--model={model_name}", diff --git a/tests/integration/test_lists/qa/llm_perf_core.yml b/tests/integration/test_lists/qa/llm_perf_core.yml index 82ad7e80fc91..1af118aee3f9 100644 --- a/tests/integration/test_lists/qa/llm_perf_core.yml +++ b/tests/integration/test_lists/qa/llm_perf_core.yml @@ -214,9 +214,15 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:1000,1000-con:512-ep:4-tp:4-gpus:4] #max_throughput #nemotron_3_super_120b_nvfp4 (Hybrid MoE+SSM+Attn FP4 76G, 4-GPU ep=4 tp=4, throughput config) + #these test config come from docs/source/deployment-guide/deployment-guide-for-nemotron-3-on-trtllm.md - perf/test_perf.py::test_perf[nemotron_3_super_120b_nvfp4-serve-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024-reqs:5-con:1-ep:4-tp:4-gpus:4] #min_latency - perf/test_perf.py::test_perf[nemotron_3_super_120b_nvfp4-serve-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024-reqs:160-con:32-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[nemotron_3_super_120b_nvfp4-serve-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024-reqs:640-con:128-ep:4-tp:4-gpus:4] #max_throughput + #nemotron_3_ultra_550b_nvfp4 (Hybrid MoE FP4 ~275G, 4-GPU ep=4 tp=4, throughput config, HF download) + #these test config come from docs/source/deployment-guide/deployment-guide-for-nemotron-3-on-trtllm.md + - perf/test_perf.py::test_perf[nemotron_3_ultra_550b_nvfp4-serve-pytorch-float4-maxbs:256-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024-reqs:5-con:1-ep:4-tp:4-gpus:4] #min_latency + - perf/test_perf.py::test_perf[nemotron_3_ultra_550b_nvfp4-serve-pytorch-float4-maxbs:256-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024-reqs:160-con:32-ep:4-tp:4-gpus:4] + - perf/test_perf.py::test_perf[nemotron_3_ultra_550b_nvfp4-serve-pytorch-float4-maxbs:256-maxnt:2048-kv_frac:0.8-input_output_len:1024,1024-reqs:640-con:128-ep:4-tp:4-gpus:4] #max_throughput # 7: B200, B300 test cases diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml index 6842e5e4cbef..5bc5a05c45df 100644 --- a/tests/integration/test_lists/test-db/l0_perf.yml +++ b/tests/integration/test_lists/test-db/l0_perf.yml @@ -1,20 +1,5 @@ version: 0.0.1 l0_perf: - - condition: - ranges: - system_gpu_count: - gte: 1 - lte: 1 - wildcards: - gpu: - - '*h100*' - linux_distribution_name: ubuntu* - terms: - stage: pre_merge - backend: tensorrt - tests: - - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192] - - condition: ranges: system_gpu_count: From f782448eb5b3faf4697263084aaf486dacf3669b Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Tue, 9 Jun 2026 13:05:21 +0000 Subject: [PATCH 2/8] [test] Update llm_perf_core.yml to include additional performance tests for nemotron and llama models - Reintroduced performance tests for nemotron_nano_12b_v2 and qwen3.5_27b models with various configurations. - Added performance tests for llama_v3.3_nemotron_super_49b with multiple input/output lengths and GPU configurations. - Ensured comprehensive coverage of performance benchmarks in the llm_perf_core.yml file. Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- .../test_lists/qa/llm_perf_core.yml | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_lists/qa/llm_perf_core.yml b/tests/integration/test_lists/qa/llm_perf_core.yml index 1af118aee3f9..4fb008b7751d 100644 --- a/tests/integration/test_lists/qa/llm_perf_core.yml +++ b/tests/integration/test_lists/qa/llm_perf_core.yml @@ -52,24 +52,8 @@ llm_perf_core: tests: #nemotron_nano_12b_v2 - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-maxbs:1-input_output_len:128,128-reqs:10-con:1] #min_latency - - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-input_output_len:500,2000-con:250] #max_throughput - - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-input_output_len:128,128] #qwen3.5_9b (dense BF16 19G, 1-GPU) #qwen3.5_27b (dense BF16 52G, 2-GPU) - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:128,128-tp:2-gpus:2] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:2-gpus:2] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:2-gpus:2] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:2-gpus:2] #max_throughput - #llama_v3.3_nemotron_super_49b (nemotron-nas BF16 94G, 2-GPU) - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:128,128-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:2-gpus:2] #max_throughput - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-streaming-bfloat16-input_output_len:128,128-gpus:4] - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-bfloat16-input_output_len:128,128-gpus:4] - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp8-bench-pytorch-streaming-float8-input_output_len:2000,200-gpus:8] @@ -99,6 +83,23 @@ llm_perf_core: - perf/test_perf.py::test_perf[gpt_oss_20b_fp4-bench-pytorch-float4-maxbs:512-maxnt:8192-input_output_len:2000,200-con:64] - perf/test_perf.py::test_perf[gpt_oss_20b_fp4-bench-pytorch-float4-maxbs:512-maxnt:8192-input_output_len:128,128] - perf/test_perf.py::test_perf[gpt_oss_20b_fp4-bench-pytorch-float4-maxbs:512-maxnt:8192-input_output_len:2000,200-con:256] + - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-input_output_len:500,2000-con:250] #max_throughput + - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-input_output_len:128,128] + #qwen3.5_27b (dense BF16 52G, 2-GPU) + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:128,128-tp:2-gpus:2] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:2-gpus:2] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:2-gpus:2] #max_throughput + #llama_v3.3_nemotron_super_49b (nemotron-nas BF16 94G, 2-GPU) + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:128,128-tp:2-gpus:2] + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:2-gpus:2] + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:2-gpus:2] + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency + - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:2-gpus:2] #max_throughput # 4: H100, H20, H200, GB200, B200, B300, GB300, RTX6000-D, RTX6000-Server test cases From 3aa70ee420dacc33e1134437d518360ae9b54135 Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Tue, 9 Jun 2026 13:50:06 +0000 Subject: [PATCH 3/8] [test] Update llm_perf_core.yml to refine performance test cases - Removed redundant test cases for llama_v3.1_nemotron_ultra_253b and adjusted the configuration for qwen3.5_122b_a10b. - Added back performance tests for llama_v3.1_nemotron_ultra_253b with various input/output lengths and GPU configurations. - Updated comments for clarity on the test cases included. Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- .../test_lists/qa/llm_perf_core.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_lists/qa/llm_perf_core.yml b/tests/integration/test_lists/qa/llm_perf_core.yml index 4fb008b7751d..4d736cbe1f9d 100644 --- a/tests/integration/test_lists/qa/llm_perf_core.yml +++ b/tests/integration/test_lists/qa/llm_perf_core.yml @@ -11,7 +11,7 @@ llm_perf_core: # 6: B200, GB200, B300, GB300 test cases # 7: B200, B300 test cases # 8: H100, H20, H200, B200, B300, RTX6000D, RTX6000-Server test cases -# 9: H20, H200, B200, B300, RTX6000D, RTX6000-Server test cases +# 9: H20, H200, B200, B300, RTX6000-Server test cases # 10: RTX-6000D, RTX-6000 Server test cases # =============================================================================== @@ -143,7 +143,6 @@ llm_perf_core: - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:1000,1000-con:250-tp:2-gpus:2] #max_throughput #qwen3.5_122b_a10b (MoE BF16 234G, 4-GPU) - - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:128,128-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:500,2000-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:2000,500-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:1000,1000-ep:4-tp:4-gpus:4] @@ -291,15 +290,6 @@ llm_perf_core: - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-gpus:8] - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-input_output_len:1000,1000-con:512-ep:8-gpus:8] #max_throughput -# 9: H20, H200, B200, B300 test cases - #llama_v3.1_nemotron_ultra_253b (nemotron-nas BF16 474G, 8-GPU) - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:128,128-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:8-gpus:8] #max_throughput #llama_v3.1_nemotron_ultra_253b_fp8 (nemotron-nas FP8 241G, 8-GPU) - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:128,128-tp:8-gpus:8] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:8-gpus:8] @@ -316,8 +306,10 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-maxbs:512-input_output_len:1000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:128,128-ep:4-tp:4-gpus:4] -# 9: H20, H200, B200, B300, RTX6000D, RTX6000-Server test cases + +# 9: H20, H200, B200, B300, RTX6000-Server test cases - condition: ranges: system_gpu_count: @@ -343,6 +335,14 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:1000,2000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:1000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput + #llama_v3.1_nemotron_ultra_253b (nemotron-nas BF16 474G, 8-GPU) + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:128,128-tp:8-gpus:8] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:8-gpus:8] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:8-gpus:8] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,1000-tp:8-gpus:8] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:8-gpus:8] + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-maxbs:1-input_output_len:1000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,1000-con:250-tp:8-gpus:8] #max_throughput # 10: RTX-6000D, RTX-6000 Server test cases - condition: From 5c94d9be06c9e3e24da7555817dac48cbbe0125a Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Wed, 10 Jun 2026 06:16:52 +0000 Subject: [PATCH 4/8] [None][test] Remove unnecessary f-string prefix in build command Addresses CodeRabbit review: --log_level=info is a static literal and does not need an f-string prefix (ruff F541). Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- tests/integration/defs/perf/test_perf.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 855b1886f31d..1f01fe88b7fd 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -1044,9 +1044,8 @@ def get_trtllm_bench_build_command(self, engine_dir) -> list: if not model_name.endswith("_hf"): model_name = model_name + "_hf" build_cmd = [ - self._build_script, f"--log_level=info", - f"--workspace={engine_dir}", f"--model={model_name}", - f"--model_path={model_dir}", "build", + self._build_script, "--log_level=info", f"--workspace={engine_dir}", + f"--model={model_name}", f"--model_path={model_dir}", "build", f"--tp_size={self._config.tp_size}", f"--pp_size={self._config.pp_size}" ] From 0d3c81c84a5a0d35119499bad2efbe4008bbfc2a Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Wed, 10 Jun 2026 08:56:14 +0000 Subject: [PATCH 5/8] [None][fix] Enhance lock handling in config_file_lock Introduced a new helper function to identify lock-infrastructure errors, improving the robustness of the config_file_lock context manager. This change allows for better handling of temporary directory fallbacks during lock acquisition failures, ensuring that exceptions are properly propagated and logged. Signed-off-by: [Your Name] Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- tensorrt_llm/_torch/model_config.py | 60 +++++++++++++++-------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index fa448d8876c2..30e40634ebd6 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -66,6 +66,17 @@ def _unified_kv_pool_includes_mamba( return not (use_split_pool or use_spec) +def _is_lock_infra_error(exc: BaseException) -> bool: + """Whether exc is a lock-infrastructure failure eligible for tempdir fallback.""" + # filelock.Timeout subclasses OSError with errno=None, so match it explicitly. + if isinstance(exc, (filelock.Timeout, PermissionError)): + return True + if isinstance(exc, OSError): + return exc.errno in (errno.EACCES, errno.EPERM, errno.ENOLCK, + errno.ESTALE) + return False + + @contextlib.contextmanager def config_file_lock(timeout: int = 10): """ @@ -77,47 +88,40 @@ def config_file_lock(timeout: int = 10): Args: timeout: Maximum time to wait for lock acquisition in seconds """ - # Use a single global lock file in HF cache directory - # This serializes all model loading operations to prevent race conditions + # Use a single global lock file in HF cache directory to serialize loads. lock_path = Path(HF_MODULES_CACHE) / "_remote_code.lock" - - # Create and acquire the lock lock = filelock.FileLock(str(lock_path), timeout=timeout) + # Guard only acquisition so caller-body exceptions propagate (single-yield). try: - with lock: - yield + lock.acquire(timeout=timeout) except (PermissionError, OSError, filelock.Timeout) as e: - # Fallback to tempdir when primary lock path is unusable (e.g., - # NFS locking failures like ENOLCK/ESTALE, permission issues, - # or lock acquisition timeouts) - if isinstance(e, - OSError) and e.errno not in (errno.EACCES, errno.EPERM, - errno.ENOLCK, errno.ESTALE): + # Primary lock unusable (NFS ENOLCK/ESTALE, perms, or timeout); use tempdir. + if not _is_lock_infra_error(e): raise tmp_dir = Path(tempfile.gettempdir()) tmp_dir.mkdir(parents=True, exist_ok=True) - tmp_lock_path = tmp_dir / "_remote_code.lock" - tmp_lock = filelock.FileLock(str(tmp_lock_path), timeout=timeout) + tmp_lock = filelock.FileLock(str(tmp_dir / "_remote_code.lock"), + timeout=timeout) try: - with tmp_lock: - yield - except filelock.Timeout: - logger.warning( - f"failed to acquire tempdir config lock within {timeout} seconds, proceeding without lock" - ) - # proceed without lock - yield - except (PermissionError, OSError) as e: - if isinstance( - e, OSError) and e.errno not in (errno.EACCES, errno.EPERM, - errno.ENOLCK, errno.ESTALE): + tmp_lock.acquire(timeout=timeout) + except (PermissionError, OSError, filelock.Timeout) as tmp_e: + if not _is_lock_infra_error(tmp_e): raise logger.warning( - f"tempdir config lock unavailable due to OS/permission issue: {e}, proceeding without lock" + f"failed to acquire tempdir config lock within {timeout}s, proceeding without lock" ) - # proceed without lock yield + else: + try: + yield + finally: + tmp_lock.release() + else: + try: + yield + finally: + lock.release() @dataclass(kw_only=True) From 5b1a3da8d0efdfb8cb0d0847bf34a1729ae98dbc Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:25:22 +0000 Subject: [PATCH 6/8] [None][fix] Refine lock error handling in config_file_lock Updated the logic in the _is_lock_infra_error function to better differentiate between lock contention and broken infrastructure. Enhanced the config_file_lock context manager to log warnings appropriately when lock acquisition fails, ensuring clearer error handling and fallback behavior. Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- tensorrt_llm/_torch/model_config.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 30e40634ebd6..b0328160d98e 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -67,9 +67,11 @@ def _unified_kv_pool_includes_mamba( def _is_lock_infra_error(exc: BaseException) -> bool: - """Whether exc is a lock-infrastructure failure eligible for tempdir fallback.""" - # filelock.Timeout subclasses OSError with errno=None, so match it explicitly. - if isinstance(exc, (filelock.Timeout, PermissionError)): + """Whether exc indicates broken lock infrastructure (not mere contention).""" + # filelock.Timeout is contention, not broken infra, so it is not eligible. + if isinstance(exc, filelock.Timeout): + return False + if isinstance(exc, PermissionError): return True if isinstance(exc, OSError): return exc.errno in (errno.EACCES, errno.EPERM, errno.ENOLCK, @@ -95,8 +97,15 @@ def config_file_lock(timeout: int = 10): # Guard only acquisition so caller-body exceptions propagate (single-yield). try: lock.acquire(timeout=timeout) - except (PermissionError, OSError, filelock.Timeout) as e: - # Primary lock unusable (NFS ENOLCK/ESTALE, perms, or timeout); use tempdir. + except filelock.Timeout: + # Contention, not broken infra: a tempdir lock can't serialize against + # the holder, so degrade to no lock instead of crashing the process. + logger.warning( + f"could not acquire config lock within {timeout}s, proceeding without lock" + ) + yield + except (PermissionError, OSError) as e: + # Broken lock infra (perms / NFS ENOLCK/ESTALE): retry on a tempdir lock. if not _is_lock_infra_error(e): raise tmp_dir = Path(tempfile.gettempdir()) @@ -105,12 +114,9 @@ def config_file_lock(timeout: int = 10): timeout=timeout) try: tmp_lock.acquire(timeout=timeout) - except (PermissionError, OSError, filelock.Timeout) as tmp_e: - if not _is_lock_infra_error(tmp_e): - raise + except (PermissionError, OSError, filelock.Timeout): logger.warning( - f"failed to acquire tempdir config lock within {timeout}s, proceeding without lock" - ) + "tempdir config lock unavailable, proceeding without lock") yield else: try: From 7826b064a8a12f243b5b9033bb6ac6e227cead91 Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Mon, 13 Jul 2026 06:21:31 +0000 Subject: [PATCH 7/8] [None][fix] Update error handling in _is_lock_infra_error to clarify exception types Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- tensorrt_llm/_torch/model_config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 0ebba28e209f..1d965691fbbc 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -94,9 +94,7 @@ def _unified_kv_pool_includes_mamba( def _is_lock_infra_error(exc: BaseException) -> bool: """Whether exc indicates broken lock infrastructure (not mere contention).""" - # filelock.Timeout is contention, not broken infra, so it is not eligible. - if isinstance(exc, filelock.Timeout): - return False + # Callers pass only PermissionError/OSError; filelock.Timeout never reaches here. if isinstance(exc, PermissionError): return True if isinstance(exc, OSError): From 0d1c2ebd61df1e0806598d203483df2e52ae2343 Mon Sep 17 00:00:00 2001 From: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> Date: Mon, 13 Jul 2026 06:24:21 +0000 Subject: [PATCH 8/8] [None][fix] Remove outdated comment in _is_lock_infra_error for clarity Signed-off-by: yufeiwu-nv <230315618+yufeiwu-nv@users.noreply.github.com> --- tensorrt_llm/_torch/model_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 1d965691fbbc..92011e195b6b 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -94,7 +94,6 @@ def _unified_kv_pool_includes_mamba( def _is_lock_infra_error(exc: BaseException) -> bool: """Whether exc indicates broken lock infrastructure (not mere contention).""" - # Callers pass only PermissionError/OSError; filelock.Timeout never reaches here. if isinstance(exc, PermissionError): return True if isinstance(exc, OSError):