Merge pull request #177 from PDFMathTranslate/dev

feat: integrate BabelDOC v0.5.1 advanced processing options

Author: awwaawwa

docs/en/advanced/advanced.md

@@ -80,6 +80,11 @@ In the following table, we list all advanced options for reference:
 | `--only-include-translated-page`| Only include translated pages in the output PDF. Effective only when --pages is used. | `pdf2zh example.pdf --pages 1-5 --only-include-translated-page`                                                       |
 | `--glossaries`                  | Custom glossary for translation.                                                      | `pdf2zh example.pdf --glossaries "glossary1.csv,glossary2.csv,glossary3.csv"`                                         |
 | `--save-auto-extracted-glossary`| save automatically extracted glossary.                                                | `pdf2zh example.pdf --save-auto-extracted-glossary`                                                                   |
+| `--no-merge-alternating-line-numbers` | Disable merging of alternating line numbers and text paragraphs in documents with line numbers | `pdf2zh example.pdf --no-merge-alternating-line-numbers` |
+| `--no-remove-non-formula-lines` | Disable removal of non-formula lines within paragraph areas                          | `pdf2zh example.pdf --no-remove-non-formula-lines`                                                                    |
+| `--non-formula-line-iou-threshold` | Set IoU threshold for identifying non-formula lines (0.0-1.0)                     | `pdf2zh example.pdf --non-formula-line-iou-threshold 0.85`                                                            |
+| `--figure-table-protection-threshold` | Set protection threshold for figures and tables (0.0-1.0). Lines within figures/tables will not be processed | `pdf2zh example.pdf --figure-table-protection-threshold 0.95` |
+| `--skip-formula-offset-calculation` | Skip formula offset calculation during processing         | `pdf2zh example.pdf --skip-formula-offset-calculation`                                                                |
 
 
 ##### GUI Args

pdf2zh_next/__init__.py

@@ -30,7 +30,7 @@
 
 # from pdf2zh_next.high_level import translate, translate_stream
 
-__version__ = "2.4.2"
+__version__ = "2.5.0"
 __author__ = "Byaidu, awwaawwa"
 __license__ = "AGPL-3.0"
 __maintainer__ = "awwaawwa"

pdf2zh_next/config/model.py

@@ -178,6 +178,26 @@ class PDFSettings(BaseModel):
         default=False,
         description="Only include translated pages in the output PDF. Effective only when --pages is used.",
     )
+    no_merge_alternating_line_numbers: bool = Field(
+        default=False,
+        description="Handle alternating line numbers and text paragraphs in documents with line numbers",
+    )
+    no_remove_non_formula_lines: bool = Field(
+        default=False,
+        description="Remove non-formula lines within paragraph areas",
+    )
+    non_formula_line_iou_threshold: float = Field(
+        default=0.9,
+        description="IoU threshold for identifying non-formula lines",
+    )
+    figure_table_protection_threshold: float = Field(
+        default=0.9,
+        description="Protection threshold for figures and tables (lines within figures/tables will not be processed)",
+    )
+    skip_formula_offset_calculation: bool = Field(
+        default=False,
+        description="Skip formula offset calculation during processing",
+    )
 
 
 class SettingsModel(BaseModel):
@@ -307,6 +327,16 @@ def validate_settings(self) -> None:
                 f"Invalid primary font family: {self.translation.primary_font_family}"
             )
 
+        if not (0.0 <= self.pdf.non_formula_line_iou_threshold <= 1.0):
+            raise ValueError(
+                "non_formula_line_iou_threshold must be between 0.0 and 1.0"
+            )
+
+        if not (0.0 <= self.pdf.figure_table_protection_threshold <= 1.0):
+            raise ValueError(
+                "figure_table_protection_threshold must be between 0.0 and 1.0"
+            )
+
         if self.pdf.auto_enable_ocr_workaround and self.pdf.ocr_workaround:
             self.pdf.ocr_workaround = False
             log.warning(

pdf2zh_next/const.py

@@ -1,4 +1,4 @@
-__version__ = "2.4.2"
+__version__ = "2.5.0"
 __major_version__ = "2"
 __config_file_version__ = "3"
 

pdf2zh_next/gui.py

@@ -462,6 +462,15 @@ def _build_translate_settings(
     auto_enable_ocr_workaround = ui_inputs.get("auto_enable_ocr_workaround")
     only_include_translated_page = ui_inputs.get("only_include_translated_page")
 
+    # BabelDOC v0.5.1 new options
+    merge_alternating_line_numbers = ui_inputs.get("merge_alternating_line_numbers")
+    remove_non_formula_lines = ui_inputs.get("remove_non_formula_lines")
+    non_formula_line_iou_threshold = ui_inputs.get("non_formula_line_iou_threshold")
+    figure_table_protection_threshold = ui_inputs.get(
+        "figure_table_protection_threshold"
+    )
+    skip_formula_offset_calculation = ui_inputs.get("skip_formula_offset_calculation")
+
     # New input for custom_system_prompt
     custom_system_prompt_input = ui_inputs.get("custom_system_prompt_input")
     glossaries = ui_inputs.get("glossaries")
@@ -557,6 +566,23 @@ def _build_translate_settings(
     if formular_char_pattern:
         translate_settings.pdf.formular_char_pattern = formular_char_pattern
 
+    # Apply BabelDOC v0.5.1 new options
+    translate_settings.pdf.no_merge_alternating_line_numbers = (
+        not merge_alternating_line_numbers
+    )
+    translate_settings.pdf.no_remove_non_formula_lines = not remove_non_formula_lines
+    if non_formula_line_iou_threshold is not None:
+        translate_settings.pdf.non_formula_line_iou_threshold = float(
+            non_formula_line_iou_threshold
+        )
+    if figure_table_protection_threshold is not None:
+        translate_settings.pdf.figure_table_protection_threshold = float(
+            figure_table_protection_threshold
+        )
+    translate_settings.pdf.skip_formula_offset_calculation = (
+        skip_formula_offset_calculation
+    )
+
     assert service in TRANSLATION_ENGINE_METADATA_MAP, "UNKNOW TRANSLATION ENGINE!"
 
     for metadata in TRANSLATION_ENGINE_METADATA:
@@ -800,6 +826,12 @@ async def translate_file(
     ocr_workaround,
     auto_enable_ocr_workaround,
     only_include_translated_page,
+    # BabelDOC v0.5.1 new options
+    merge_alternating_line_numbers,
+    remove_non_formula_lines,
+    non_formula_line_iou_threshold,
+    figure_table_protection_threshold,
+    skip_formula_offset_calculation,
     *translation_engine_arg_inputs,
     progress=None,
 ):
@@ -889,6 +921,12 @@ async def translate_file(
         "ocr_workaround": ocr_workaround,
         "auto_enable_ocr_workaround": auto_enable_ocr_workaround,
         "only_include_translated_page": only_include_translated_page,
+        # BabelDOC v0.5.1 new options
+        "merge_alternating_line_numbers": merge_alternating_line_numbers,
+        "remove_non_formula_lines": remove_non_formula_lines,
+        "non_formula_line_iou_threshold": non_formula_line_iou_threshold,
+        "figure_table_protection_threshold": figure_table_protection_threshold,
+        "skip_formula_offset_calculation": skip_formula_offset_calculation,
     }
     for arg_name, arg_input in zip(
         __gui_service_arg_names, translation_engine_arg_inputs, strict=False
@@ -1434,6 +1472,50 @@ async def translate_file(
                     interactive=True,
                 )
 
+                # BabelDOC v0.5.1 new options
+                gr.Markdown("#### BabelDOC Advanced Options")
+
+                merge_alternating_line_numbers = gr.Checkbox(
+                    label="Merge alternating line numbers",
+                    info="Handle alternating line numbers and text paragraphs in documents with line numbers",
+                    value=not settings.pdf.no_merge_alternating_line_numbers,
+                    interactive=True,
+                )
+
+                remove_non_formula_lines = gr.Checkbox(
+                    label="Remove non-formula lines",
+                    info="Remove non-formula lines within paragraph areas",
+                    value=not settings.pdf.no_remove_non_formula_lines,
+                    interactive=True,
+                )
+
+                non_formula_line_iou_threshold = gr.Slider(
+                    label="Non-formula line IoU threshold",
+                    info="IoU threshold for identifying non-formula lines",
+                    value=settings.pdf.non_formula_line_iou_threshold,
+                    minimum=0.0,
+                    maximum=1.0,
+                    step=0.05,
+                    interactive=True,
+                )
+
+                figure_table_protection_threshold = gr.Slider(
+                    label="Figure/table protection threshold",
+                    info="Protection threshold for figures and tables (lines within figures/tables will not be processed)",
+                    value=settings.pdf.figure_table_protection_threshold,
+                    minimum=0.0,
+                    maximum=1.0,
+                    step=0.05,
+                    interactive=True,
+                )
+
+                skip_formula_offset_calculation = gr.Checkbox(
+                    label="Skip formula offset calculation",
+                    info="Skip formula offset calculation during processing",
+                    value=settings.pdf.skip_formula_offset_calculation,
+                    interactive=True,
+                )
+
             output_title = gr.Markdown("## Translated", visible=False)
             output_file_mono = gr.File(
                 label="Download Translation (Mono)", visible=False
@@ -1697,6 +1779,12 @@ def on_service_change_with_rate_limit(mode, service_name):
             ocr_workaround,
             auto_enable_ocr_workaround,
             only_include_translated_page,
+            # BabelDOC v0.5.1 new options
+            merge_alternating_line_numbers,
+            remove_non_formula_lines,
+            non_formula_line_iou_threshold,
+            figure_table_protection_threshold,
+            skip_formula_offset_calculation,
             *translation_engine_arg_inputs,
         ],
         outputs=[

pdf2zh_next/high_level.py

@@ -483,6 +483,12 @@ def create_babeldoc_config(settings: SettingsModel, file: Path) -> BabelDOCConfi
         auto_extract_glossary=not settings.translation.no_auto_extract_glossary,
         primary_font_family=settings.translation.primary_font_family,
         only_include_translated_page=settings.pdf.only_include_translated_page,
+        # BabelDOC v0.5.1 new options
+        merge_alternating_line_numbers=not settings.pdf.no_merge_alternating_line_numbers,
+        remove_non_formula_lines=not settings.pdf.no_remove_non_formula_lines,
+        non_formula_line_iou_threshold=settings.pdf.non_formula_line_iou_threshold,
+        figure_table_protection_threshold=settings.pdf.figure_table_protection_threshold,
+        skip_formula_offset_calculation=settings.pdf.skip_formula_offset_calculation,
     )
     return babeldoc_config
 

pdf2zh_next/main.py

@@ -14,7 +14,7 @@
 from pdf2zh_next.config import ConfigManager
 from pdf2zh_next.high_level import do_translate_file_async
 
-__version__ = "2.4.2"
+__version__ = "2.5.0"
 
 logger = logging.getLogger(__name__)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pdf2zh-next"
-version = "2.4.2"
+version = "2.5.0"
 description = "Latex PDF Translator"
 authors = [
     { name = "awwaawwa", email = "[email protected]" },
@@ -34,7 +34,7 @@ dependencies = [
     "gradio_pdf>=0.0.21",
     "peewee>=3.17.8",
     "fontTools",
-    "babeldoc>=0.4.11, <0.5.0",
+    "babeldoc>=0.5.1, <0.6.0",
     "rich",
     "pydantic-settings>=2.8.1",
     "pydantic>=2.10.6",
@@ -81,7 +81,7 @@ max-line-length = 88
 
 
 [bumpver]
-current_version = "2.4.2"
+current_version = "2.5.0"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]