hrw4u: Allows bulk compilation of many files (faster)

Author: zwoop

doc/admin-guide/configuration/hrw4u.en.rst

@@ -80,15 +80,56 @@ follows to produce the help output:
 
    hrw4u --help
 
-Doing a compile is simply:
+Basic Usage
+^^^^^^^^^^^
+
+Compile a single file to stdout:
 
 .. code-block:: none
 
    hrw4u some_file.hrw4u
 
-in Addition to ``hrw4u``, you also have the reverse tool, converting existing ``header_rewrite``
-configurations to ``hrw4u``. This tool is named ``u4wrh``. For people using IDEs, the package also
-provides an LSP for this language, named ``hrw4u-lsp``.
+Compile from stdin:
+
+.. code-block:: none
+
+   cat some_file.hrw4u | hrw4u
+
+Compile multiple files to stdout (separated by ``# ---``):
+
+.. code-block:: none
+
+   hrw4u file1.hrw4u file2.hrw4u file3.hrw4u
+
+Bulk Compilation
+^^^^^^^^^^^^^^^^
+
+For bulk compilation, use the ``input:output`` format to compile multiple files
+to their respective output files in a single command:
+
+.. code-block:: none
+
+   hrw4u file1.hrw4u:file1.conf file2.hrw4u:file2.conf file3.hrw4u:file3.conf
+
+This is particularly useful for build systems or when processing many configuration
+files at once. All files are processed in a single invocation, improving performance
+for large batches of files.
+
+Reverse Tool (u4wrh)
+^^^^^^^^^^^^^^^^^^^^
+
+In addition to ``hrw4u``, you also have the reverse tool, converting existing ``header_rewrite``
+configurations to ``hrw4u``. This tool is named ``u4wrh`` and supports the same usage patterns:
+
+.. code-block:: none
+
+   # Convert single file to stdout
+   u4wrh existing_config.conf
+
+   # Bulk conversion
+   u4wrh file1.conf:file1.hrw4u file2.conf:file2.hrw4u
+
+For people using IDEs, the package also provides an LSP for this language, named ``hrw4u-lsp``.
 
 Syntax Differences
 ==================

tools/hrw4u/scripts/hrw4u

@@ -22,28 +22,19 @@ from __future__ import annotations
 from hrw4u.hrw4uLexer import hrw4uLexer
 from hrw4u.hrw4uParser import hrw4uParser
 from hrw4u.visitor import HRW4UVisitor
-from hrw4u.common import create_base_parser, create_parse_tree, generate_output, process_input
+from hrw4u.common import run_main
 
 
 def main() -> None:
     """Main entry point for the hrw4u script."""
-    parser, output_group = create_base_parser("Process HRW4U input and produce output (AST or HRW).")
-
-    # Argument parsing
-    output_group.add_argument("--hrw", action="store_true", help="Produce the HRW output (default)")
-    parser.add_argument("--no-comments", action="store_true", help="Skip comment preservation (ignore comments in output)")
-    args = parser.parse_args()
-
-    # Default to HRW output if neither AST nor HRW specified
-    if not (args.ast or args.hrw):
-        args.hrw = True
-
-    content, filename = process_input(args.input_file)
-    tree, parser_obj, error_collector = create_parse_tree(
-        content, filename, hrw4uLexer, hrw4uParser, "hrw4u", not args.stop_on_error)
-
-    # Generate output
-    generate_output(tree, parser_obj, HRW4UVisitor, filename, args, error_collector)
+    run_main(
+        description="Process HRW4U input and produce output (AST or HRW).",
+        lexer_class=hrw4uLexer,
+        parser_class=hrw4uParser,
+        visitor_class=HRW4UVisitor,
+        error_prefix="hrw4u",
+        output_flag_name="hrw",
+        output_flag_help="Produce the HRW output (default)")
 
 
 if __name__ == "__main__":

tools/hrw4u/scripts/u4wrh

@@ -19,31 +19,22 @@
 
 from __future__ import annotations
 
-from hrw4u.common import create_base_parser, create_parse_tree, generate_output, process_input
+from hrw4u.common import run_main
 from u4wrh.hrw_visitor import HRWInverseVisitor
 from u4wrh.u4wrhLexer import u4wrhLexer
 from u4wrh.u4wrhParser import u4wrhParser
 
 
 def main() -> None:
     """Main entry point for the u4wrh script."""
-    parser, output_group = create_base_parser("Process header_rewrite (HRW) lines and reconstruct hrw4u source.")
-
-    # Argument parsing
-    output_group.add_argument("--hrw4u", action="store_true", help="Produce reconstructed hrw4u output (default)")
-    parser.add_argument("--no-comments", action="store_true", help="Skip comment preservation (ignore comments in output)")
-    args = parser.parse_args()
-
-    # Default to hrw4u output if neither AST nor hrw4u specified
-    if not (args.ast or args.hrw4u):
-        args.hrw4u = True
-
-    content, filename = process_input(args.input_file)
-    tree, parser_obj, error_collector = create_parse_tree(
-        content, filename, u4wrhLexer, u4wrhParser, "u4wrh", not args.stop_on_error)
-
-    # Generate output
-    generate_output(tree, parser_obj, HRWInverseVisitor, filename, args, error_collector)
+    run_main(
+        description="Process header_rewrite (HRW) lines and reconstruct hrw4u source.",
+        lexer_class=u4wrhLexer,
+        parser_class=u4wrhParser,
+        visitor_class=HRWInverseVisitor,
+        error_prefix="u4wrh",
+        output_flag_name="hrw4u",
+        output_flag_help="Produce reconstructed hrw4u output (default)")
 
 
 if __name__ == "__main__":

tools/hrw4u/src/common.py

@@ -229,3 +229,110 @@ def generate_output(
         print(error_collector.get_error_summary(), file=sys.stderr)
         if not args.ast and tree is None:
             sys.exit(1)
+
+
+def run_main(
+        description: str, lexer_class: type[LexerProtocol], parser_class: type[ParserProtocol],
+        visitor_class: type[VisitorProtocol], error_prefix: str, output_flag_name: str, output_flag_help: str) -> None:
+    """
+    Generic main function for hrw4u and u4wrh scripts with bulk compilation support.
+
+    Args:
+        description: Description for argument parser
+        lexer_class: ANTLR lexer class to use
+        parser_class: ANTLR parser class to use
+        visitor_class: Visitor class to use
+        error_prefix: Error prefix for error messages
+        output_flag_name: Name of output flag (e.g., "hrw", "hrw4u")
+        output_flag_help: Help text for output flag
+    """
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description=description,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="For bulk compilation to files, use: input1.txt:output1.txt input2.txt:output2.txt ...")
+
+    parser.add_argument(
+        "files", help="Input file(s) to parse. Use input:output for bulk file output (default: stdin to stdout)", nargs="*")
+
+    output_group = parser.add_mutually_exclusive_group()
+    output_group.add_argument("--ast", action="store_true", help="Produce the ANTLR parse tree only")
+    output_group.add_argument(f"--{output_flag_name}", action="store_true", help=output_flag_help)
+
+    parser.add_argument("--no-comments", action="store_true", help="Skip comment preservation (ignore comments in output)")
+    parser.add_argument("--debug", action="store_true", help="Enable debug output")
+    parser.add_argument(
+        "--stop-on-error", action="store_true", help="Stop processing on first error (default: collect and report multiple errors)")
+
+    args = parser.parse_args()
+
+    if not hasattr(args, output_flag_name):
+        setattr(args, output_flag_name, False)
+
+    if not (args.ast or getattr(args, output_flag_name)):
+        setattr(args, output_flag_name, True)
+
+    if not args.files:
+        content, filename = process_input(sys.stdin)
+        tree, parser_obj, error_collector = create_parse_tree(
+            content, filename, lexer_class, parser_class, error_prefix, not args.stop_on_error)
+        generate_output(tree, parser_obj, visitor_class, filename, args, error_collector)
+        return
+
+    has_colons = any(':' in f for f in args.files)
+
+    if has_colons:
+        for pair in args.files:
+            if ':' not in pair:
+                print(
+                    f"Error: Mixed formats not allowed. All files must use 'input:output' format for bulk compilation.",
+                    file=sys.stderr)
+                sys.exit(1)
+
+            input_path, output_path = pair.split(':', 1)
+
+            try:
+                with open(input_path, 'r', encoding='utf-8') as input_file:
+                    content = input_file.read()
+                    filename = input_path
+            except FileNotFoundError:
+                print(f"Error: Input file '{input_path}' not found", file=sys.stderr)
+                sys.exit(1)
+            except Exception as e:
+                print(f"Error reading '{input_path}': {e}", file=sys.stderr)
+                sys.exit(1)
+
+            tree, parser_obj, error_collector = create_parse_tree(
+                content, filename, lexer_class, parser_class, error_prefix, not args.stop_on_error)
+
+            try:
+                original_stdout = sys.stdout
+                with open(output_path, 'w', encoding='utf-8') as output_file:
+                    sys.stdout = output_file
+                    generate_output(tree, parser_obj, visitor_class, filename, args, error_collector)
+                sys.stdout = original_stdout
+            except Exception as e:
+                sys.stdout = original_stdout
+                print(f"Error writing to '{output_path}': {e}", file=sys.stderr)
+                sys.exit(1)
+    else:
+        for i, input_path in enumerate(args.files):
+            if i > 0:
+                print("\n# ---")
+
+            try:
+                with open(input_path, 'r', encoding='utf-8') as input_file:
+                    content = input_file.read()
+                    filename = input_path
+            except FileNotFoundError:
+                print(f"Error: Input file '{input_path}' not found", file=sys.stderr)
+                sys.exit(1)
+            except Exception as e:
+                print(f"Error reading '{input_path}': {e}", file=sys.stderr)
+                sys.exit(1)
+
+            tree, parser_obj, error_collector = create_parse_tree(
+                content, filename, lexer_class, parser_class, error_prefix, not args.stop_on_error)
+
+            generate_output(tree, parser_obj, visitor_class, filename, args, error_collector)

tools/hrw4u/tests/utils.py

@@ -17,6 +17,8 @@
 from __future__ import annotations
 
 import re
+import subprocess
+import tempfile
 from pathlib import Path
 from typing import Final, Iterator
 
@@ -52,6 +54,7 @@ def __init__(self, filename: str, line: int, column: int, message: str, source_l
     "run_ast_test",
     "run_failing_test",
     "run_reverse_test",
+    "run_bulk_test",
 ]
 
 
@@ -315,3 +318,69 @@ def test_reverse_conversion(input_file: Path, output_file: Path) -> None:
         run_reverse_test(input_file, output_file)
 
     return test_reverse_conversion
+
+
+def run_bulk_test(group: str) -> None:
+    """
+    Run bulk compilation test for a specific test group.
+
+    Collects all .input.txt files in the group, runs hrw4u with bulk
+    input:output pairs, and compares each output with expected .output.txt.
+    """
+    base_dir = Path("tests/data") / group
+    exceptions = _read_exceptions(base_dir)
+
+    input_files = []
+    expected_outputs = []
+    file_pairs = []
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_path = Path(tmpdir)
+
+        for input_file in sorted(base_dir.glob("*.input.txt")):
+            if ".fail." in input_file.name:
+                continue
+
+            base = input_file.with_suffix('')
+            expected_output_file = base.with_suffix('.output.txt')
+            test_id = base.name
+
+            if test_id in exceptions:
+                test_direction = exceptions[test_id]
+                if test_direction != "hrw4u":
+                    continue
+
+            if not expected_output_file.exists():
+                continue
+
+            input_files.append(input_file)
+            expected_outputs.append(expected_output_file)
+
+            actual_output_file = tmp_path / f"{input_file.stem}.output.txt"
+            file_pairs.append(f"{input_file.resolve()}:{actual_output_file.resolve()}")
+
+        if not file_pairs:
+            pytest.skip(f"No valid test files found for bulk test in {group}")
+            return
+
+        hrw4u_script = Path("scripts/hrw4u").resolve()
+        cmd = ["python3", str(hrw4u_script)] + file_pairs
+
+        result = subprocess.run(cmd, capture_output=True, text=True, cwd=Path.cwd())
+
+        if result.returncode != 0:
+            pytest.fail(f"hrw4u bulk compilation failed:\nstdout: {result.stdout}\nstderr: {result.stderr}")
+
+        for input_file, expected_output_file in zip(input_files, expected_outputs):
+            actual_output_file = tmp_path / f"{input_file.stem}.output.txt"
+
+            if not actual_output_file.exists():
+                pytest.fail(f"Output file not created for {input_file.name}: {actual_output_file}")
+
+            actual_output = actual_output_file.read_text().strip()
+            expected_output = expected_output_file.read_text().strip()
+
+            assert actual_output == expected_output, (
+                f"Bulk output mismatch for {input_file.name}\n"
+                f"Expected:\n{expected_output}\n\n"
+                f"Actual:\n{actual_output}")