Typed area support

.gitignore

@@ -21,3 +21,4 @@ doc/
 build/
 dist/
 grobid_client_python.egg-info/
+.venv

Readme.md

@@ -144,6 +144,9 @@ grobid_client [OPTIONS] SERVICE
 | `--flavor`                   | Processing flavor for fulltext extraction |
 | `--json`                     | Convert TEI output to JSON format         |
 | `--markdown`                 | Convert TEI output to Markdown format     |
+| `--typed_area`               | Enable sending typed-area layout JSON to GROBID |
+| `--typed_areas_dir`          | Directory of pre-computed JSON files      |
+| `--typed_area_server`        | URL of the PaddlePaddle server            |
 
 
 #### Examples
@@ -166,8 +169,22 @@ grobid_client --server https://grobid.example.com --input ~/citations.txt proces
 
 # Force reprocessing with sentence segmentation and JSON output
 grobid_client --input ~/docs --force --segment_sentences --json processFulltextDocument
+
+# Typed Area Processing (with PaddlePaddle server)
+# The client will fetch JSON from the paddle server and send it to Grobid
+grobid_client --input ~/pdfs --output ~/results --typed_area --typed_area_server h  processFulltextDocument
+
+# Typed Area Processing (with pre-computed offline JSON files)
+grobid_client --input ~/pdfs --output ~/results --typed_area --typed_areas_dir ~/precomputed_jsons processFulltextDocument
 ```
 
+###  Worker and Concurrency (Typed Areas)
+When using the `--typed_area_server` flag, the Grobid client makes requests to *both* the PaddlePaddle server and the Grobid server. 
+
+- **Grobid Client Threads (`--n`)**: Controls how many PDFs are processed concurrently.
+- **PaddlePaddle Server Workers (`--workers`)**: Controls how many concurrent layout requests the PaddlePaddle server can handle. 
+
+**Recommendation:** Set the PaddlePaddle server `--workers` to match the grobid-client `--n` threads (e.g., `--n 4` on the client and `--workers 4` on the server). 
 ### Python Library
 
 #### Basic Usage

grobid_client/grobid_client.py

@@ -366,7 +366,10 @@ def process(
             verbose=False,
             flavor=None,
             json_output=False,
-            markdown_output=False
+            markdown_output=False,
+            typed_area=False,
+            typed_areas_dir=None,
+            typed_area_server=None
     ):
         start_time = time.time()
         batch_size_pdf = self.config["batch_size"]
@@ -435,7 +438,10 @@ def process(
                     verbose,
                     flavor,
                     json_output,
-                    markdown_output
+                    markdown_output,
+                    typed_area,
+                    typed_areas_dir,
+                    typed_area_server
                 )
                 processed_files_count += batch_processed
                 errors_files_count += batch_errors
@@ -461,7 +467,10 @@ def process(
                 verbose,
                 flavor,
                 json_output,
-                markdown_output
+                markdown_output,
+                typed_area,
+                typed_areas_dir,
+                typed_area_server
             )
             processed_files_count += batch_processed
             errors_files_count += batch_errors
@@ -499,7 +508,10 @@ def process_batch(
             verbose=False,
             flavor=None,
             json_output=False,
-            markdown_output=False
+            markdown_output=False,
+            typed_area=False,
+            typed_areas_dir=None,
+            typed_area_server=None
     ):
         batch_start_time = time.time()
         if verbose:
@@ -584,7 +596,10 @@ def process_batch(
                     segment_sentences,
                     flavor,
                     -1,
-                    -1)
+                    -1,
+                    typed_area,
+                    typed_areas_dir,
+                    typed_area_server)
 
                 results.append(r)
 
@@ -668,6 +683,75 @@ def process_batch(
 
         return processed_count, error_count, skipped_count
 
+    def _resolve_typed_area(self, pdf_file, typed_areas_dir, typed_area_server):
+        """Resolve typed-area JSON for a PDF file.
+
+        Priority:
+            1. PaddlePaddle server (if typed_area_server is set)
+            2. Explicit directory (if typed_areas_dir is set)
+            3. Same directory as the PDF
+
+        Returns:
+            str or None: JSON string to attach as the typedAreas data field,
+                         or None if no typed-area data could be resolved.
+        """
+        stem = pathlib.Path(pdf_file).stem
+
+        #query the PaddlePaddle server
+        if typed_area_server:
+            try:
+                with open(pdf_file, "rb") as f:
+                    resp = requests.post(
+                        f"{typed_area_server.rstrip('/')}",
+                        files={"file": (os.path.basename(pdf_file), f, "application/pdf")},
+                        timeout=self.config["timeout"]
+                    )
+                if resp.status_code == 200:
+                    json_data = resp.json()
+
+                    # Save the JSON to disk
+                    save_dir = typed_areas_dir if typed_areas_dir else str(pathlib.Path(pdf_file).parent)
+                    os.makedirs(save_dir, exist_ok=True)
+                    json_path = os.path.join(save_dir, f"{stem}.json")
+                    try:
+                        with open(json_path, "w", encoding="utf-8") as f:
+                            json.dump(json_data, f, ensure_ascii=False, indent=2)
+                        self.logger.debug(f"Saved typed-area JSON to {json_path}")
+                    except Exception as e:
+                        self.logger.warning(f"Failed to save typed-area JSON to {json_path}: {e}")
+
+                    # Extract just the elements array if present, as GROBID expects a JSON Array
+                    payload = json_data.get("elements", json_data) if isinstance(json_data, dict) else json_data
+                    return json.dumps(payload)
+                else:
+                    self.logger.warning(
+                        f"Typed-area server returned {resp.status_code} for {pdf_file}"
+                    )
+            except Exception as e:
+                self.logger.warning(
+                    f"Typed-area server request failed for {pdf_file}: {e}"
+                )
+            return None
+
+        # explicit directory, or same directory as the PDF
+        search_dir = typed_areas_dir if typed_areas_dir else str(pathlib.Path(pdf_file).parent)
+        json_path = os.path.join(search_dir, f"{stem}.json")
+
+        if os.path.isfile(json_path):
+            try:
+                with open(json_path, "r", encoding="utf-8") as f:
+                    json_content = json.load(f)
+                    payload = json_content.get("elements", json_content) if isinstance(json_content, dict) else json_content
+                    return json.dumps(payload)
+            except Exception as e:
+                self.logger.warning(f"Failed to read typed-area JSON {json_path}: {e}")
+                return None
+
+        self.logger.warning(
+            f"No typed-area JSON found for {pdf_file} (looked in {search_dir})"
+        )
+        return None
+
     def process_pdf(
             self,
             service,
@@ -681,7 +765,10 @@ def process_pdf(
             segment_sentences,
             flavor=None,
             start=-1,
-            end=-1
+            end=-1,
+            typed_area=False,
+            typed_areas_dir=None,
+            typed_area_server=None
     ):
         pdf_handle = None
         try:
@@ -721,6 +808,14 @@ def process_pdf(
             if end and end > 0:
                 the_data["end"] = str(end)
 
+            # Resolve and attach typed-area JSON if enabled
+            if typed_area:
+                typed_area_json = self._resolve_typed_area(
+                    pdf_file, typed_areas_dir, typed_area_server
+                )
+                if typed_area_json:
+                    the_data["typedAreas"] = typed_area_json
+
             res, status = self.post(
                 url=the_url, files=files, data=the_data, headers={"Accept": "text/plain"},
                 timeout=self.config['timeout']
@@ -741,7 +836,10 @@ def process_pdf(
                     segment_sentences,
                     flavor,
                     start,
-                    end
+                    end,
+                    typed_area,
+                    typed_areas_dir,
+                    typed_area_server
                 )
 
             return (pdf_file, status, res.text)
@@ -941,6 +1039,21 @@ def main():
         action="store_true",
         help="Convert TEI output to Markdown format",
     )
+    parser.add_argument(
+        "--typed_area",
+        action="store_true",
+        help="Enable typed-area support: attach PaddlePaddle layout JSON to each Grobid request",
+    )
+    parser.add_argument(
+        "--typed_areas_dir",
+        default=None,
+        help="Directory containing pre-computed typed-area JSON files (default: same directory as the PDF)",
+    )
+    parser.add_argument(
+        "--typed_area_server",
+        default=None,
+        help="URL of the PaddlePaddle typed-area server",
+    )
 
     args = parser.parse_args()
 
@@ -950,6 +1063,9 @@ def main():
     flavor = args.flavor
     json_output = args.json
     markdown_output = args.markdown
+    typed_area = args.typed_area
+    typed_areas_dir = args.typed_areas_dir
+    typed_area_server = args.typed_area_server
 
     # Initialize n with default value
     n = 10
@@ -1020,7 +1136,10 @@ def main():
             verbose=verbose,
             flavor=flavor,
             json_output=json_output,
-            markdown_output=markdown_output
+            markdown_output=markdown_output,
+            typed_area=typed_area,
+            typed_areas_dir=typed_areas_dir,
+            typed_area_server=typed_area_server
         )
     except Exception as e:
         logger.error(f"Processing failed: {str(e)}")