update master (#465)

Co-authored-by: Alexander Golodkov <[email protected]>
Co-authored-by: Alexander Golodkov <[email protected]>
Co-authored-by: Andrew Perminov <[email protected]>
Co-authored-by: Oksana Belyaeva <[email protected]>
Co-authored-by: RichardScottOZ <[email protected]>
Co-authored-by: Andrey Mikhailov <[email protected]>

Author: 7 people

.flake8

@@ -7,6 +7,19 @@ inline-quotes = "
 application-import-names = dedoc, tests, scripts, train_dataset
 import-order-style = pycharm
 
+extend-immutable-calls = File, Depends
+
+banned-modules =
+    dedoc = Use full path
+    dedoc.data_structures = Use full path
+    dedoc.attachments_extractors = Use full path
+    dedoc.attachments_handler = Use full path
+    dedoc.converters = Use full path
+    dedoc.metadata_extractors = Use full path
+    dedoc.readers = Use full path
+    dedoc.structure_constructors = Use full path
+    dedoc.structure_extractors = Use full path
+
 exclude =
     .git,
     __pycache__,
@@ -28,9 +41,11 @@ exclude =
 # ANN202 - Missing return type annotation for protected function
 # ANN204 - Missing return type annotation for special method
 # N802 - function name should be lowercase
+# I251 - Banned import (Use full path)
 ignore =
     ANN101
 per-file-ignores =
     scripts/*:T201
     scripts/benchmark_pdf_performance*:JS101
     tests/custom_test_runner.py:ANN001,ANN201,ANN202,ANN204,N802
+    docs/source/_static/code_examples/*:I251

.github/check_version.py

@@ -23,7 +23,7 @@ def is_correct_version(version: str, tag: str, old_version: str, regexp: Pattern
     args = parser.parse_args()
 
     print(f"Old version: {args.old_version}, new version: {args.new_version}, "
-          f"branch: {args.branch}, tag: {args.tag}, pre_release: {args.pre_release}")  # noqa
+          f"branch: {args.branch}, tag: {args.tag}, pre_release: {args.pre_release}")
 
     master_version_pattern = re.compile(r"^\d+\.\d+(\.\d+)?$")
     develop_version_pattern = re.compile(r"^\d+\.\d+\.\d+rc\d+$")
@@ -43,4 +43,4 @@ def is_correct_version(version: str, tag: str, old_version: str, regexp: Pattern
         is_correct_version(args.new_version, args.tag, args.old_version, master_version_pattern)
         assert args.pre_release != "true", "Pre-releases are not allowed on master"
 
-    print("Version is correct")  # noqa
+    print("Version is correct")

.pre-commit-config.yaml

@@ -15,6 +15,7 @@ repos:
             flake8-import-order==0.18.2,
             flake8-multiline-containers==0.0.19,
             flake8-print==5.0.0,
+            flake8-tidy-imports==4.10.0,
             flake8-quotes==3.3.2,
             flake8-use-fstring==1.4,
             pycodestyle==2.9.0,

Dockerfile

@@ -1,5 +1,7 @@
 ARG REPOSITORY="docker.io"
 FROM dedocproject/dedoc_p3.9_base:version_2023_08_28
+ARG LANGUAGES=""
+RUN for lang in $LANGUAGES; do apt install -y tesseract-ocr-$lang; done
 
 ENV PYTHONPATH "${PYTHONPATH}:/dedoc_root"
 ENV RESOURCES_PATH "/dedoc_root/resources"

README.md

@@ -1,17 +1,18 @@
 # Dedoc
 
+[![GitHub release](https://img.shields.io/github/release/ispras/dedoc.svg)](https://github.com/ispras/dedoc/releases/)
 [![PyPI version](https://badge.fury.io/py/dedoc.svg)](https://badge.fury.io/py/dedoc)
+[![PyPI downloads](https://pepy.tech/badge/dedoc)](https://pepy.tech/project/dedoc)
+[![Docker Hub](https://img.shields.io/docker/pulls/dedocproject/dedoc.svg)](https://hub.docker.com/r/dedocproject/dedoc/ "Docker Pulls")
 [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)
-[![Documentation Status](https://readthedocs.org/projects/dedoc/badge/?version=latest)](https://dedoc.readthedocs.io/en/latest/?badge=latest)
-[![GitHub release](https://img.shields.io/github/release/ispras/dedoc.svg)](https://github.com/ispras/dedoc/releases/)
 [![Demo dedoc-readme.hf.space](https://img.shields.io/website-up-down-green-red/https/huggingface.co/spaces/dedoc/README.svg)](https://dedoc-readme.hf.space)
-[![Docker Hub](https://img.shields.io/docker/pulls/dedocproject/dedoc.svg)](https://hub.docker.com/r/dedocproject/dedoc/ "Docker Pulls")
+[![Documentation Status](https://readthedocs.org/projects/dedoc/badge/?version=latest)](https://dedoc.readthedocs.io/en/latest/?badge=latest)
 [![CI tests](https://github.com/ispras/dedoc/workflows/CI/badge.svg)](https://github.com/ispras/dedoc/actions)
 
 ![Dedoc](https://github.com/ispras/dedoc/raw/master/dedoc_logo.png)
 
 Dedoc is an open universal system for converting documents to a unified output format. 
-It extracts a document’s logical structure and content, its tables, text formatting and metadata. 
+It extracts a document’s logical structure and content: tables, text formatting and metadata. 
 The document’s content is represented as a tree storing headings and lists of any level. 
 Dedoc can be integrated in a document contents and structure analysis system as a separate module.
 
@@ -22,14 +23,14 @@ Dedoc can be integrated in a document contents and structure analysis system as
 Workflow description is given [`here`](https://dedoc.readthedocs.io/en/latest/?badge=latest#workflow)
 
 ## Features and advantages
-Dedoc is implemented in Python and works with semi-structured data formats (DOC/DOCX, ODT, XLS/XLSX, CSV, TXT, JSON) and none-structured data formats like images (PNG, JPG etc.), archives (ZIP, RAR etc.), PDF and HTML formats. 
+Dedoc is implemented in Python and works with semi-structured data formats (DOC/DOCX, ODT, XLS/XLSX, CSV, TXT, JSON) and unstructured data formats like images (PNG, JPG etc.), archives (ZIP, RAR etc.), PDF and HTML formats. 
 Document structure extraction is fully automatic regardless of input data type. 
 Metadata and text formatting are also extracted automatically. 
 
 In 2022, the system won a grant to support the development of promising AI projects from the [Innovation Assistance Foundation (Фонд содействия инновациям)](https://fasie.ru/).
 
 ## Dedoc provides:
-* Extensibility due to a flexible addition of new document formats and to an easy change of an output data format. 
+* Extensibility due to flexible addition of new document formats and easy change of an output data format. 
 * Support for extracting document structure out of nested documents having different formats. 
 * Extracting various text formatting features (indentation, font type, size, style etc.). 
 * Working with documents of various origin (statements of work, legal documents, technical reports, scientific papers) allowing flexible tuning for new domains. 
@@ -68,7 +69,7 @@ The system processes different document formats. The main formats are listed bel
 
 
 ## Impact
-This project may be useful as a first step of automatic document analysis pipeline (e.g. before the NLP part).
+This project may be useful as a first step of an automatic document analysis pipeline (e.g. before the NLP part).
 Dedoc is in demand for information analytic systems, information leak monitoring systems, as well as for natural language processing systems.
 The library is intended for application use by developers of systems for automatic analysis and structuring of electronic documents, including for further search in electronic documents. 
 
@@ -92,7 +93,7 @@ Relevant documentation of dedoc is available [here](https://dedoc.readthedocs.io
 
 # Installation instructions
 
-This project has REST Api and you can run it in Docker container.
+This project has a REST api and you can run it in Docker container.
 Also, dedoc can be installed as a library via `pip`.
 There are two ways to install and run dedoc as a web application or a library that are described below.
 
@@ -149,7 +150,7 @@ If you need to change some application settings, you may update `config.py` acco
 
 If you don't want to use docker for running the application, it's possible to run dedoc locally.
 However, it isn't suitable for any operating system (`Ubuntu 20+` is recommended) and
-there may be not enough machine's resources for its work.
+there may be not enough machine resources for its work.
 You should have `python` (`python3.8`, `python3.9` are recommended) and `pip` installed.
 Installation instructions via pip are available [here](https://dedoc.readthedocs.io/en/latest/getting_started/installation.html#install-dedoc-using-pip).
 

VERSION

@@ -1 +1 @@
-2.2.3
+2.2.4

dedoc/__init__.py

@@ -1,2 +1,2 @@
-from .dedoc_manager import DedocManager  # noqa
-from .version import __version__  # noqa
+from .dedoc_manager import DedocManager
+from .version import __version__

dedoc/api/api_args.py

@@ -28,7 +28,7 @@ class QueryParameters:
     # pdf handling
     pdf_with_text_layer: str = Form("auto_tabby", enum=["true", "false", "auto", "auto_tabby", "tabby"],
                                     description="Extract text from a text layer of PDF or using OCR methods for image-like documents")
-    language: str = Form("rus+eng", enum=["rus+eng", "rus", "eng", "fra", "spa"], description="Recognition language")
+    language: str = Form("rus+eng", description="Recognition language ('rus+eng', 'rus', 'eng', 'fra', 'spa')")
     pages: str = Form(":", description='Page numbers range for reading PDF or images, "left:right" means read pages from left to right')
     is_one_column_document: str = Form("auto", enum=["auto", "true", "false"],
                                        description='One or multiple column document, "auto" - predict number of page columns automatically')

dedoc/api/api_utils.py

@@ -120,12 +120,20 @@ def json2html(text: str,
               attachments: Optional[List[ParsedDocument]],
               tabs: int = 0,
               table2id: Dict[str, int] = None,
-              attach2id: Dict[str, int] = None) -> str:
+              attach2id: Dict[str, int] = None,
+              prev_page_id: Optional[List[int]] = None) -> str:
+    if prev_page_id is None:
+        prev_page_id = [0]
+
     tables = [] if tables is None else tables
     attachments = [] if attachments is None else attachments
     table2id = {table.metadata.uid: table_id for table_id, table in enumerate(tables)} if table2id is None else table2id
     attach2id = {attachment.metadata.uid: attachment_id for attachment_id, attachment in enumerate(attachments)} if attach2id is None else attach2id
 
+    if paragraph.metadata.page_id != prev_page_id[0]:
+        text += f"<center><small><b>Page {prev_page_id[0] + 1}</b></small></center><hr>"
+        prev_page_id[0] = paragraph.metadata.page_id
+
     ptext = __annotations2html(paragraph=paragraph, table2id=table2id, attach2id=attach2id, tabs=tabs)
 
     if paragraph.metadata.hierarchy_level.line_type in [HierarchyLevel.header, HierarchyLevel.root]:
@@ -141,7 +149,8 @@ def json2html(text: str,
     text += ptext
 
     for subparagraph in paragraph.subparagraphs:
-        text = json2html(text=text, paragraph=subparagraph, tables=None, attachments=None, tabs=tabs + 4, table2id=table2id, attach2id=attach2id)
+        text = json2html(text=text, paragraph=subparagraph, tables=None, attachments=None, tabs=tabs + 4, table2id=table2id, attach2id=attach2id,
+                         prev_page_id=prev_page_id)
 
     if tables is not None and len(tables) > 0:
         text += "<h3> Tables: </h3>"

dedoc/api/dedoc_api.py

@@ -7,13 +7,12 @@
 import traceback
 from typing import Optional
 
-import uvicorn
 from fastapi import Depends, FastAPI, File, Request, Response, UploadFile
 from fastapi.responses import ORJSONResponse, UJSONResponse
 from fastapi.staticfiles import StaticFiles
 from starlette.responses import FileResponse, HTMLResponse, JSONResponse, PlainTextResponse
 
-import dedoc
+import dedoc.version
 from dedoc.api.api_args import QueryParameters
 from dedoc.api.api_utils import json2collapsed_tree, json2html, json2tree, json2txt
 from dedoc.api.schema.parsed_document import ParsedDocument
@@ -53,7 +52,7 @@ def get_static_file(request: Request) -> Response:
 
 @app.get("/version")
 def get_version() -> Response:
-    return PlainTextResponse(dedoc.__version__)
+    return PlainTextResponse(dedoc.version.__version__)
 
 
 def _get_static_file_path(request: Request) -> str:
@@ -70,10 +69,10 @@ def __add_base64_info_to_attachments(document_tree: ParsedDocument, attachments_
 
 
 @app.post("/upload", response_model=ParsedDocument)
-async def upload(file: UploadFile = File(...), query_params: QueryParameters = Depends()) -> Response:  # noqa
+async def upload(file: UploadFile = File(...), query_params: QueryParameters = Depends()) -> Response:
     parameters = dataclasses.asdict(query_params)
     if not file or file.filename == "":
-        raise MissingFileError("Error: Missing content in request_post file parameter", version=dedoc.__version__)
+        raise MissingFileError("Error: Missing content in request_post file parameter", version=dedoc.version.__version__)
 
     return_format = str(parameters.get("return_format", "json")).lower()
 
@@ -152,4 +151,5 @@ def get_api() -> FastAPI:
 
 
 def run_api(app: FastAPI) -> None:
+    import uvicorn
     uvicorn.run(app=app, host="0.0.0.0", port=int(PORT))