diff --git a/docs/import/import-files/aggregate-xml.md b/docs/import/import-files/aggregate-xml.md index ae3f504f..7f9d0bb0 100644 --- a/docs/import/import-files/aggregate-xml.md +++ b/docs/import/import-files/aggregate-xml.md @@ -6,8 +6,8 @@ grand_parent: Importing Data nav_order: 5 --- -Flux can split large XML files - called "aggregate XML files" - in the same fashion as -[MarkLogic Content Pump](https://docs.marklogic.com/11.0/guide/mlcp-guide/en/importing-content-into-marklogic-server/splitting-large-xml-files-into-multiple-documents.html). +Flux can split large XML files - called "aggregate XML files" - in the same fashion as +[MarkLogic Content Pump](https://docs.marklogic.com/11.0/guide/mlcp-guide/en/importing-content-into-marklogic-server/splitting-large-xml-files-into-multiple-documents.html). ## Table of contents {: .no_toc .text-delta } @@ -19,7 +19,7 @@ Flux can split large XML files - called "aggregate XML files" - in the same fash The `import-aggregate-xml-files` command creates many XML documents based on a particular XML element name and optional namespace. The command requires an instance of the `--element` option to identify the name of the XML element -in each file where each occurrence of that XML element will result in a separate document being written to MarkLogic. +in each file where each occurrence of that XML element will result in a separate document being written to MarkLogic. The `--namespace` option must be used if that XML element has an associated namespace. For example, consider the following notional XML file located at `/data/people.xml`: @@ -68,9 +68,9 @@ bin\flux import-aggregate-xml-files ^ ## Controlling document URIs -In addition to the options for controlling URIs described in the [common import features guide](../common-import-features.md), +In addition to the options for controlling URIs described in the [common import features guide](../common-import-features.md), you can use the `--uri-element` and `--uri-namespace` options to identify an element in each XML document whose value should -be included in the URI. Using the example XML document in the above section, the following would construct a URI +be included in the URI. Using the example XML document in the above section, the following would construct a URI based on the value of each `id` element in the `org:example` namespace: {% tabs log %} @@ -107,14 +107,14 @@ example, adding the following would result in URIs of `/person/1.xml` and `/pers ## Compressed XML files -Flux supports gzip and ZIP aggregate XML files. Simply include the `--compression` option with a value of `GZIP` or +Flux supports gzip and ZIP aggregate XML files. Simply include the `--compression` option with a value of `GZIP` or `ZIP`. ## Specifying an encoding -MarkLogic stores all content +MarkLogic stores all content [in the UTF-8 encoding](https://docs.marklogic.com/guide/search-dev/encodings_collations#id_87576). -If your aggregate XML files use a different encoding, you must specify that via the `--encoding` option so that +If your aggregate XML files use a different encoding, you must specify that via the `--encoding` option so that the content can be correctly translated to UTF-8 when written to MarkLogic - e.g.: {% tabs log %} @@ -137,3 +137,23 @@ bin\flux import-aggregate-xml-files ^ ``` {% endtab %} {% endtabs %} + +## Zip bomb protection + +When importing compressed XML files from untrusted sources using `--compression ZIP`, you can protect against +[zip bombs](https://en.wikipedia.org/wiki/Zip_bomb) — archives crafted to expand to an enormous amount of data +and potentially exhaust the memory available to Flux. Both options below are disabled by default. + +To limit the maximum number of uncompressed bytes read from any single ZIP entry: + + --zip-max-uncompressed-entry-bytes 268435456 + +A value of `268435456` (256 MB) is a reasonable starting point for most use cases. If a single entry exceeds +this limit, an error is thrown. Set to `0` or any value less than `1` to disable. + +To limit the maximum number of entries processed from a single ZIP file: + + --zip-max-entry-count 100000 + +A value of `100000` is a reasonable starting point for most use cases. Set to `0` or any value less than `1` to +disable. diff --git a/docs/import/import-files/archives.md b/docs/import/import-files/archives.md index 6c057c39..f8e4c0a3 100644 --- a/docs/import/import-files/archives.md +++ b/docs/import/import-files/archives.md @@ -6,9 +6,9 @@ grand_parent: Importing Data nav_order: 7 --- -Flux can import archive files containing documents and their associated metadata. This includes archives written via -the [`export-archive-files` command](../../export/export-archives.md) as well as archives written by -[MarkLogic Content Pump](https://docs.marklogic.com/11.0/guide/mlcp-guide/en/exporting-content-from-marklogic-server/exporting-to-an-archive.html), +Flux can import archive files containing documents and their associated metadata. This includes archives written via +the [`export-archive-files` command](../../export/export-archives.md) as well as archives written by +[MarkLogic Content Pump](https://docs.marklogic.com/11.0/guide/mlcp-guide/en/exporting-content-from-marklogic-server/exporting-to-an-archive.html), which are hereafter referred to as "MLCP archives". ## Table of contents @@ -19,7 +19,7 @@ which are hereafter referred to as "MLCP archives". ## Usage -The `import-archive-files` command will import the documents and metadata files in a ZIP file produced by the +The `import-archive-files` command will import the documents and metadata files in a ZIP file produced by the `export-archive-files` command. You must specify at least one `--path` option along with connection information for the MarkLogic database you wish to write to: @@ -44,7 +44,7 @@ bin\flux import-archive-files ^ ## Importing MLCP archives -You can also import +You can also import [MLCP archives](https://docs.marklogic.com/11.0/guide/mlcp-guide/en/exporting-content-from-marklogic-server/exporting-to-an-archive.html) that were produced via the `EXPORT` command in MLCP. The `import-mlcp-archive-files` command is used instead, and it also requires at least one `--path` option along with connection information for the MarkLogic database you wish to write to: @@ -71,9 +71,9 @@ bin\flux import-mlcp-archive-files ^ ## Restricting metadata -By default, all metadata associated with a document in an archive will be included when the document is written to MarkLogic. -This is true for both the `import-archive-files` command and the `import-mlcp-archive-files` command. This is typically -desirable so that metadata like collections and permissions in the archive can be applied to the imported documents. +By default, all metadata associated with a document in an archive will be included when the document is written to MarkLogic. +This is true for both the `import-archive-files` command and the `import-mlcp-archive-files` command. This is typically +desirable so that metadata like collections and permissions in the archive can be applied to the imported documents. You can instead restrict which types of metadata are included via the `--categories` option. This option accepts a comma-delimited sequence of the following metadata types: @@ -84,7 +84,7 @@ sequence of the following metadata types: - `properties` - `metadatavalues` -For example, the following option will only include the collections and properties found in each metadata entry in an +For example, the following option will only include the collections and properties found in each metadata entry in an archive ZIP file or MLCP archive ZIP file: --categories collections,properties @@ -118,7 +118,7 @@ bin\flux import-archive-files ^ ## Importing large binary files in archives -When [exporting archives](../../export/export-archives.md), you can use the `--streaming` option introduced in Flux +When [exporting archives](../../export/export-archives.md), you can use the `--streaming` option introduced in Flux 1.1.0 to ensure that large binary documents in MarkLogic can be streamed to an archive file. When importing archives with large binary files, you should likewise use the `--streaming` option to ensure that each large binary can be read into MarkLogic without exhausting the memory available to Flux or MarkLogic. @@ -140,16 +140,16 @@ large binary document may exhaust the amount of memory available to MarkLogic. In addition, when streaming documents to MarkLogic, URIs will be encoded. For example, an entry named `/my file.json` will result in a URI of `/my%20file.json`. This is due to an [issue in the MarkLogic REST API endpoint](https://docs.marklogic.com/REST/PUT/v1/documents) that will be resolved in -a future server release. +a future server release. ### Applying transforms selectively when streaming -When streaming archive entries to MarkLogic, you may need to apply a REST transform to certain documents but not -others. One reason for this is when an archive contains files that appear to be JSON, XML, or text based on their URI +When streaming archive entries to MarkLogic, you may need to apply a REST transform to certain documents but not +others. One reason for this is when an archive contains files that appear to be JSON, XML, or text based on their URI extensions (such as `.json`, `.xml`, or `.txt`), but are stored in the archive with a binary format. As noted in the [MarkLogic REST API documentation](https://docs.progress.com/bundle/marklogic-server-develop-rest-api-12/page/topics/intro.html#id_53367), loading such documents as binaries requires the use of a REST transform that converts each document into a binary. This -approach prevents MarkLogic from automatically setting the document type based on the URI extension. The following +approach prevents MarkLogic from automatically setting the document type based on the URI extension. The following REST transform is one example of how a JSON or XML document can be converted to a binary: ``` @@ -166,10 +166,10 @@ declare function transform($context as map:map, $params as map:map, $content as }; ``` -The `--streaming-transform-binary-with-extension` option, introduced in Flux 2.1, allows you to specify which -documents should be sent to your transform during streaming. This allows you to stream as many documents as possible, -with only the documents that need to be converted into binaries being sent to the transform. The option accepts a -comma-delimited list of URI extensions. The transform specified via `--transform` will then only be applied if +The `--streaming-transform-binary-with-extension` option, introduced in Flux 2.1, allows you to specify which +documents should be sent to your transform during streaming. This allows you to stream as many documents as possible, +with only the documents that need to be converted into binaries being sent to the transform. The option accepts a +comma-delimited list of URI extensions. The transform specified via `--transform` will then only be applied if both of the following conditions are met for a document in the archive: 1. The document format in the archive is `BINARY` (requires an archive created with Flux 2.1 or later). @@ -185,14 +185,14 @@ For example, consider an archive containing: The following set of options will result in the transform only being applied to `report.json` and `document.xml`: ``` ---streaming +--streaming --transform my-transform --streaming-transform-binary-with-extension json,xml ``` This option only has an effect when `--streaming` and `--transform` are specified. Note that if `--streaming` -and `--transform` are specified without this option, then every document will be sent to the transform. This is -typically not desirable as it results in each document being read into memory in MarkLogic, which defeats some of the +and `--transform` are specified without this option, then every document will be sent to the transform. This is +typically not desirable as it results in each document being read into memory in MarkLogic, which defeats some of the purpose of streaming the data into MarkLogic. ## Common errors @@ -204,5 +204,26 @@ following message: com.marklogic.spark: Could not find metadata entry for entry ``` -To solve this, you should use Flux 1.1.0 or higher to import the archive. Flux 1.1.0 and higher can also import -archives created by Flux 1.0.x. +To solve this, you should use Flux 1.1.0 or higher to import the archive. Flux 1.1.0 and higher can also import +archives created by Flux 1.0.x. + +## Zip bomb protection + +When importing archive files from untrusted sources, you can protect against +[zip bombs](https://en.wikipedia.org/wiki/Zip_bomb) — archives crafted to expand to an enormous amount of data +and potentially exhaust the memory available to Flux. Both options below are disabled by default and apply to +both the `import-archive-files` and `import-mlcp-archive-files` commands. + +To limit the maximum number of uncompressed bytes read from any single archive entry: + + --zip-max-uncompressed-entry-bytes 268435456 + +A value of `268435456` (256 MB) is a reasonable starting point for most use cases. If a single entry exceeds +this limit, an error is thrown. Set to `0` or any value less than `1` to disable. + +To limit the maximum number of entries processed from a single archive file: + + --zip-max-entry-count 100000 + +A value of `100000` is a reasonable starting point for most use cases. Set to `0` or any value less than `1` to +disable. diff --git a/docs/import/import-files/generic-files.md b/docs/import/import-files/generic-files.md index df5a47dc..b5630b72 100644 --- a/docs/import/import-files/generic-files.md +++ b/docs/import/import-files/generic-files.md @@ -6,7 +6,7 @@ grand_parent: Importing Data nav_order: 2 --- -Flux can import any type of file as-is, with the contents of the file becoming a new document in MarkLogic. The term +Flux can import any type of file as-is, with the contents of the file becoming a new document in MarkLogic. The term "generic files" is used in this context to refer to files that do not require any special processing other than potentially decompressing the files. @@ -44,10 +44,10 @@ bin\flux import-files ^ ## Controlling document URIs -Each document will have an initial URI based on the absolute path of the associated file. See -[common import features](../common-import-features.md) for details on adjusting this URI. In particular, the +Each document will have an initial URI based on the absolute path of the associated file. See +[common import features](../common-import-features.md) for details on adjusting this URI. In particular, the `--uri-replace` option is often useful for removing most of the absolute path to produce a concise, self-describing -URI. +URI. ## Specifying a document type @@ -66,7 +66,7 @@ the content can be correctly translated to UTF-8 when written to MarkLogic: {% tab log Unix %} ``` ./bin/flux import-files \ - --path source \ + --path source \ --encoding ISO-8859-1 \ --connection-string "flux-example-user:password@localhost:8004" \ --permissions flux-example-role,read,flux-example-role,update @@ -89,12 +89,12 @@ bin\flux import-files ^ Flux can leverage MarkLogic's [support for large binary documents](https://docs.marklogic.com/guide/app-dev/binaries#id_93203) by importing binary files of any size. To ensure that binary files of any size can be loaded, consider using the `--streaming` option introduced in Flux 1.1.0. When this option is set, Flux will stream the contents of each file from -its source directly into MarkLogic, thereby avoiding reading the contents of a file into memory. +its source directly into MarkLogic, thereby avoiding reading the contents of a file into memory. As streaming a file requires Flux to only send one document at a time to MarkLogic, you should not use this option when importing smaller files that easily fit into the memory available to Flux. -When using `--streaming`, the following options will have no effect due to Flux not reading the file contents into +When using `--streaming`, the following options will have no effect due to Flux not reading the file contents into memory and always sending one file per request to MarkLogic: - `--batch-size` @@ -102,13 +102,13 @@ memory and always sending one file per request to MarkLogic: - `--failed-documents-path` - `--uri-template` -You typically will also not want to use the `--transform` option as applying a REST transform in MarkLogic to a very +You typically will also not want to use the `--transform` option as applying a REST transform in MarkLogic to a very large binary document may exhaust the amount of memory available to MarkLogic. -In addition, when streaming documents to MarkLogic, URIs will be encoded. For example, a file named `my file.json` -will result in a URI of `/my%20file.json`. This is due to an -[issue in the MarkLogic REST API endpoint](https://docs.marklogic.com/REST/PUT/v1/documents) that will be resolved in -a future server release. +In addition, when streaming documents to MarkLogic, URIs will be encoded. For example, a file named `my file.json` +will result in a URI of `/my%20file.json`. This is due to an +[issue in the MarkLogic REST API endpoint](https://docs.marklogic.com/REST/PUT/v1/documents) that will be resolved in +a future server release. ## Importing gzip files @@ -120,15 +120,15 @@ that may not fit into the memory available to Flux or to MarkLogic. ## Importing ZIP files To import each entry in a ZIP file as a separate document, include the `--compression` option with a value of `ZIP`. -Each document will have an initial URI based on both the absolute path of the ZIP file and the name of the ZIP entry. +Each document will have an initial URI based on both the absolute path of the ZIP file and the name of the ZIP entry. You can also use the `--document-type` option as described above to force a document type for any entry that has a file extension not recognized by MarkLogic. The `--streaming` option introduced in Flux 1.1.0 can also be used for ZIP files containing very large binary files that may not fit into the memory available to Flux or to MarkLogic. ## Extracting text -As of Flux 1.3.0, text can be extracted from files via [Apache Tika](https://tika.apache.org/) and written as separate -documents in MarkLogic. This is typically useful when importing binary content such as PDF and Word files, where both +As of Flux 1.3.0, text can be extracted from files via [Apache Tika](https://tika.apache.org/) and written as separate +documents in MarkLogic. This is typically useful when importing binary content such as PDF and Word files, where both the binary file and extracted text can be stored in MarkLogic. Text extraction is enabled by including the following option when executing the `import-files` command: @@ -141,12 +141,12 @@ behavior: 1. The document will be JSON with a URI equalling that of the binary document URI plus `-extracted-text.json`. 2. The document will inherit any permissions assigned to the binary document. 3. The document will not inherit any collections assigned to the binary document. -4. The document will have keys of `source-uri`, `content`, and `extracted-metadata`. +4. The document will have keys of `source-uri`, `content`, and `extracted-metadata`. The `content` in the document contains the extracted text. The `extracted-metadata` contains each metadata key and value -produced by Apache Tika. +produced by Apache Tika. -If you do not want Flux to write the file to a separate document but only want the extracted text document written, +If you do not want Flux to write the file to a separate document but only want the extracted text document written, include the following option: --extracted-text-drop-source @@ -158,18 +158,38 @@ Extracted text documents can be written as XML instead of JSON by including the --extracted-text-document-type XML When writing Tika metadata to the extracted text document, Flux will attempt to determine a well-known namespace based -on the name of the metadata key. For example, a Tika metadata key that begins with `pdf:` will result in an element +on the name of the metadata key. For example, a Tika metadata key that begins with `pdf:` will result in an element assigned to the `http://ns.adobe.com/pdf/1.3/` namespace. ### Assigning metadata to extracted text documents -Collections can be assigned to extracted text documents via the following option, which accepts a comma-delimited list +Collections can be assigned to extracted text documents via the following option, which accepts a comma-delimited list of collection names: --extracted-text-collections collection1,collection2 -Permissions can be assigned to extracted text documents via the following options, which accepts a comma-delimited +Permissions can be assigned to extracted text documents via the following options, which accepts a comma-delimited sequence of MarkLogic role names and capabilities: --extracted-text-permissions role1,read,role2,update +## Zip bomb protection + +When importing ZIP files from untrusted sources, you can protect against +[zip bombs](https://en.wikipedia.org/wiki/Zip_bomb) — archives crafted to expand to an enormous amount of data +and potentially exhaust the memory available to Flux. Both options below are disabled by default. + +To limit the maximum number of uncompressed bytes read from any single ZIP entry: + + --zip-max-uncompressed-entry-bytes 268435456 + +A value of `268435456` (256 MB) is a reasonable starting point for most use cases. If a single entry exceeds +this limit, an error is thrown. Set to `0` or any value less than `1` to disable. + +To limit the maximum number of entries processed from a single ZIP file: + + --zip-max-entry-count 100000 + +A value of `100000` is a reasonable starting point for most use cases. Set to `0` or any value less than `1` to +disable. + diff --git a/docs/import/import-files/rdf.md b/docs/import/import-files/rdf.md index 8c2b084b..f79874c5 100644 --- a/docs/import/import-files/rdf.md +++ b/docs/import/import-files/rdf.md @@ -6,7 +6,7 @@ grand_parent: Importing Data nav_order: 6 --- -Flux can import a variety of RDF files in a fashion similar to that of +Flux can import a variety of RDF files in a fashion similar to that of [MarkLogic Content Pump](https://docs.marklogic.com/11.0/guide/mlcp-guide/en/importing-content-into-marklogic-server/loading-triples.html). ## Table of contents @@ -17,11 +17,11 @@ Flux can import a variety of RDF files in a fashion similar to that of ## Usage -The `import-rdf-files` command reads RDF data from a variety of RDF file types and writes one or more -[managed triples documents](https://docs.marklogic.com/guide/semantics/loading) in MarkLogic. -Each managed triples document is an XML document containing up to 100 semantic triples. +The `import-rdf-files` command reads RDF data from a variety of RDF file types and writes one or more +[managed triples documents](https://docs.marklogic.com/guide/semantics/loading) in MarkLogic. +Each managed triples document is an XML document containing up to 100 semantic triples. -To import RDF files, you must specify at least one `--path` option along with connection information for the MarkLogic +To import RDF files, you must specify at least one `--path` option along with connection information for the MarkLogic database you wish to write to: {% tabs log %} @@ -49,33 +49,53 @@ bin\flux import-rdf-files ^ Flux supports the same [RDF data formats](https://docs.marklogic.com/guide/semantics/loading#id_70682) as MarkLogic server does, which are listed below: -| Format | File Extension | +| Format | File Extension | |---------|----------------| | RDF/XML | .rdf or .xml | | RDF/JSON | .json | -| N3 | .n3 | +| N3 | .n3 | | N-Quads | .nq | | N-Triples | .nt | -| TriG | .trig | +| TriG | .trig | | Turtle | .ttl | -Flux will attempt to load each file specified via `--path` by determining the format of the RDF data based on the +Flux will attempt to load each file specified via `--path` by determining the format of the RDF data based on the file extension. ## Specifying a graph -By default, every triple loaded into MarkLogic will be added to the default MarkLogic semantic graph of +By default, every triple loaded into MarkLogic will be added to the default MarkLogic semantic graph of `http://marklogic.com/semantics#default-graph`. Each RDF quad is loaded into the graph specified within the quad. -To specify a different graph for every triple, use the `--graph` option. This option only affects triples and not quads. +To specify a different graph for every triple, use the `--graph` option. This option only affects triples and not quads. -To specify a graph for both triples and quads - thus overriding the graph associated with each quad - use the -`--graph-override` option. +To specify a graph for both triples and quads - thus overriding the graph associated with each quad - use the +`--graph-override` option. -Note that the set of collections specified via the `--collections` option does not have any impact on the graph. You -are free to specify as many collections as you want in addition to the graph you choose via `--graph` or -`--graph-override`. +Note that the set of collections specified via the `--collections` option does not have any impact on the graph. You +are free to specify as many collections as you want in addition to the graph you choose via `--graph` or +`--graph-override`. ## Compressed files -Flux supports gzip and ZIP RDF files. Simply include the `--compression` option with a value of `GZIP` or `ZIP`. +Flux supports gzip and ZIP RDF files. Simply include the `--compression` option with a value of `GZIP` or `ZIP`. + +## Zip bomb protection + +When importing compressed RDF files from untrusted sources using `--compression ZIP`, you can protect against +[zip bombs](https://en.wikipedia.org/wiki/Zip_bomb) — archives crafted to expand to an enormous amount of data +and potentially exhaust the memory available to Flux. Both options below are disabled by default. + +To limit the maximum number of uncompressed bytes read from any single ZIP entry: + + --zip-max-uncompressed-entry-bytes 268435456 + +A value of `268435456` (256 MB) is a reasonable starting point for most use cases. If a single entry exceeds +this limit, an error is thrown. Set to `0` or any value less than `1` to disable. + +To limit the maximum number of entries processed from a single ZIP file: + + --zip-max-entry-count 100000 + +A value of `100000` is a reasonable starting point for most use cases. Set to `0` or any value less than `1` to +disable. diff --git a/flux-cli/src/main/java/com/marklogic/flux/api/AggregateXmlFilesImporter.java b/flux-cli/src/main/java/com/marklogic/flux/api/AggregateXmlFilesImporter.java index 7fcfe444..f2f0b5bf 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/api/AggregateXmlFilesImporter.java +++ b/flux-cli/src/main/java/com/marklogic/flux/api/AggregateXmlFilesImporter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.api; @@ -10,7 +10,7 @@ */ public interface AggregateXmlFilesImporter extends Executor { - interface ReadXmlFilesOptions extends ReadFilesOptions { + interface ReadXmlFilesOptions extends ReadCompressibleFilesOptions { ReadXmlFilesOptions element(String element); ReadXmlFilesOptions namespace(String namespace); @@ -22,8 +22,6 @@ interface ReadXmlFilesOptions extends ReadFilesOptions { ReadXmlFilesOptions compressionType(CompressionType compressionType); ReadXmlFilesOptions encoding(String encoding); - - ReadXmlFilesOptions partitions(int partitions); } AggregateXmlFilesImporter from(Consumer consumer); diff --git a/flux-cli/src/main/java/com/marklogic/flux/api/ArchiveFilesImporter.java b/flux-cli/src/main/java/com/marklogic/flux/api/ArchiveFilesImporter.java index c6521c0d..3eeec861 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/api/ArchiveFilesImporter.java +++ b/flux-cli/src/main/java/com/marklogic/flux/api/ArchiveFilesImporter.java @@ -10,11 +10,9 @@ */ public interface ArchiveFilesImporter extends Executor { - interface ReadArchiveFilesOptions extends ReadFilesOptions { + interface ReadArchiveFilesOptions extends ReadCompressibleFilesOptions { ReadArchiveFilesOptions categories(String... categories); - ReadArchiveFilesOptions partitions(int partitions); - ReadArchiveFilesOptions encoding(String encoding); } diff --git a/flux-cli/src/main/java/com/marklogic/flux/api/GenericFilesImporter.java b/flux-cli/src/main/java/com/marklogic/flux/api/GenericFilesImporter.java index 3e7ed4ae..ca6b4d83 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/api/GenericFilesImporter.java +++ b/flux-cli/src/main/java/com/marklogic/flux/api/GenericFilesImporter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.api; @@ -10,11 +10,9 @@ */ public interface GenericFilesImporter extends Executor { - interface ReadGenericFilesOptions extends ReadFilesOptions { + interface ReadGenericFilesOptions extends ReadCompressibleFilesOptions { ReadGenericFilesOptions compressionType(CompressionType compressionType); - ReadGenericFilesOptions partitions(int partitions); - ReadGenericFilesOptions encoding(String encoding); } diff --git a/flux-cli/src/main/java/com/marklogic/flux/api/MlcpArchiveFilesImporter.java b/flux-cli/src/main/java/com/marklogic/flux/api/MlcpArchiveFilesImporter.java index 8f3b180c..48a658e1 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/api/MlcpArchiveFilesImporter.java +++ b/flux-cli/src/main/java/com/marklogic/flux/api/MlcpArchiveFilesImporter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.api; @@ -10,10 +10,10 @@ */ public interface MlcpArchiveFilesImporter extends Executor { - interface ReadMlcpArchiveFilesOptions extends ReadFilesOptions { + interface ReadMlcpArchiveFilesOptions extends ReadCompressibleFilesOptions { ReadMlcpArchiveFilesOptions categories(String... categories); ReadMlcpArchiveFilesOptions encoding(String encoding); - ReadMlcpArchiveFilesOptions partitions(int partitions); + } MlcpArchiveFilesImporter from(Consumer consumer); diff --git a/flux-cli/src/main/java/com/marklogic/flux/api/RdfFilesImporter.java b/flux-cli/src/main/java/com/marklogic/flux/api/RdfFilesImporter.java index 6e52041c..5dbd8c54 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/api/RdfFilesImporter.java +++ b/flux-cli/src/main/java/com/marklogic/flux/api/RdfFilesImporter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.api; @@ -10,9 +10,9 @@ */ public interface RdfFilesImporter extends Executor { - interface ReadRdfFilesOptions extends ReadFilesOptions { + interface ReadRdfFilesOptions extends ReadCompressibleFilesOptions { ReadRdfFilesOptions compressionType(CompressionType compressionType); - ReadRdfFilesOptions partitions(int partitions); + } interface WriteTriplesDocumentsOptions extends WriteDocumentsOptions { diff --git a/flux-cli/src/main/java/com/marklogic/flux/api/ReadCompressibleFilesOptions.java b/flux-cli/src/main/java/com/marklogic/flux/api/ReadCompressibleFilesOptions.java new file mode 100644 index 00000000..22d5a761 --- /dev/null +++ b/flux-cli/src/main/java/com/marklogic/flux/api/ReadCompressibleFilesOptions.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.flux.api; + +/** + * Options shared by all import commands that support reading zip-compressed files. + * Provides the {@code partitions}, {@code zipMaxUncompressedEntryBytes}, and + * {@code zipMaxEntryCount} options, which are common to all five zip-capable import commands. + * + * @since 2.1.2 + */ +@SuppressWarnings("unchecked") +public interface ReadCompressibleFilesOptions extends ReadFilesOptions { + + /** + * Specifies the number of Spark partitions used for reading files. More partitions increase parallelism + * but also increase overhead. When not set, the connector determines the number of partitions automatically + * based on the number of files found. + */ + T partitions(int partitions); + + /** + * Set to a positive integer to enable zip bomb protection. Any value less than 1 (including 0) + * disables the limit. When this option is not set, the connector default applies. + * + * @since 2.1.2 + */ + T zipMaxUncompressedEntryBytes(long bytes); + + /** + * Set to a positive integer to enable zip bomb protection. Any value less than 1 (including 0) + * disables the limit. When this option is not set, the connector default applies. + * + * @since 2.1.2 + */ + T zipMaxEntryCount(int count); +} diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportAggregateXmlFilesCommand.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportAggregateXmlFilesCommand.java index d85a4579..949fcecc 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportAggregateXmlFilesCommand.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportAggregateXmlFilesCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.impl.importdata; @@ -42,7 +42,7 @@ protected Supplier> getWriteParams() { return writeParams; } - public static class ReadXmlFilesParams extends ReadFilesParams implements ReadXmlFilesOptions { + public static class ReadXmlFilesParams extends ReadCompressibleFilesParams implements ReadXmlFilesOptions { @CommandLine.Option(required = true, names = "--element", description = "Specifies the local name of the element to use as the root of each document." @@ -72,15 +72,11 @@ public static class ReadXmlFilesParams extends ReadFilesParams makeOptions() { return OptionsUtil.addOptions( super.makeOptions(), Options.READ_FILES_ENCODING, encoding, - Options.READ_NUM_PARTITIONS, OptionsUtil.intOption(partitions), Options.READ_AGGREGATES_XML_ELEMENT, element, Options.READ_AGGREGATES_XML_NAMESPACE, namespace, Options.READ_AGGREGATES_XML_URI_ELEMENT, uriElement, @@ -124,12 +120,6 @@ public ReadXmlFilesOptions compressionType(CompressionType compressionType) { this.compressionType = compressionType; return this; } - - @Override - public ReadXmlFilesOptions partitions(int partitions) { - this.partitions = partitions; - return this; - } } @Override diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportArchiveFilesCommand.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportArchiveFilesCommand.java index 96f0a7d0..41d08fd2 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportArchiveFilesCommand.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportArchiveFilesCommand.java @@ -90,16 +90,13 @@ public WriteArchiveDocumentsOptions streamingTransformBinaryWithExtension(String } } - public static class ReadArchiveFilesParams extends ReadFilesParams implements ReadArchiveFilesOptions { + public static class ReadArchiveFilesParams extends ReadCompressibleFilesParams implements ReadArchiveFilesOptions { @CommandLine.Option(names = "--categories", description = "Comma-delimited sequence of categories of metadata to include. " + "If not specified, all types of metadata are included. " + "Valid choices are: collections, permissions, quality, properties, and metadatavalues.") private String categories; - @CommandLine.Option(names = "--partitions", description = "Specifies the number of partitions used for reading files.") - private int partitions; - @CommandLine.Option(names = "--encoding", description = "Specify an encoding when reading files.") private String encoding; @@ -108,8 +105,7 @@ public Map makeOptions() { return OptionsUtil.addOptions(super.makeOptions(), Options.READ_FILES_TYPE, "archive", Options.READ_FILES_ENCODING, encoding, - Options.READ_ARCHIVES_CATEGORIES, categories, - Options.READ_NUM_PARTITIONS, OptionsUtil.intOption(partitions) + Options.READ_ARCHIVES_CATEGORIES, categories ); } @@ -124,12 +120,6 @@ public ReadArchiveFilesOptions categories(String... categories) { this.categories = Stream.of(categories).collect(Collectors.joining(",")); return this; } - - @Override - public ReadArchiveFilesOptions partitions(int partitions) { - this.partitions = partitions; - return this; - } } @Override diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportFilesCommand.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportFilesCommand.java index 84651b0f..e1bc33f8 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportFilesCommand.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportFilesCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.impl.importdata; @@ -74,7 +74,7 @@ public GenericFilesImporter to(Consumer consumer) return this; } - public static class ReadGenericFilesParams extends ReadFilesParams implements ReadGenericFilesOptions { + public static class ReadGenericFilesParams extends ReadCompressibleFilesParams implements ReadGenericFilesOptions { @CommandLine.Option(names = "--compression", description = "When importing compressed files, specify the type of compression used. " + OptionsUtil.VALID_VALUES_DESCRIPTION) @@ -83,9 +83,6 @@ public static class ReadGenericFilesParams extends ReadFilesParams makeOptions() { return OptionsUtil.addOptions(super.makeOptions(), - Options.READ_NUM_PARTITIONS, OptionsUtil.intOption(partitions), Options.READ_FILES_COMPRESSION, compressionType != null ? compressionType.name() : null, Options.READ_FILES_ENCODING, encoding ); } - - @Override - public ReadGenericFilesOptions partitions(int partitions) { - this.partitions = partitions; - return this; - } } public static class WriteGenericDocumentsParams extends WriteDocumentParams implements WriteGenericDocumentsOptions { diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportMlcpArchiveFilesCommand.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportMlcpArchiveFilesCommand.java index ee19ba84..5cfa2c15 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportMlcpArchiveFilesCommand.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportMlcpArchiveFilesCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.impl.importdata; @@ -41,7 +41,7 @@ protected String getReadFormat() { return MARKLOGIC_CONNECTOR; } - public static class ReadMlcpArchiveFilesParams extends ReadFilesParams implements ReadMlcpArchiveFilesOptions { + public static class ReadMlcpArchiveFilesParams extends ReadCompressibleFilesParams implements ReadMlcpArchiveFilesOptions { @CommandLine.Option(names = "--categories", description = "Comma-delimited sequence of categories of metadata to include. " + "If not specified, all types of metadata are included. " + @@ -51,16 +51,12 @@ public static class ReadMlcpArchiveFilesParams extends ReadFilesParams makeOptions() { return OptionsUtil.addOptions(super.makeOptions(), Options.READ_FILES_TYPE, "mlcp_archive", Options.READ_ARCHIVES_CATEGORIES, categories, - Options.READ_FILES_ENCODING, encoding, - Options.READ_NUM_PARTITIONS, OptionsUtil.intOption(partitions) + Options.READ_FILES_ENCODING, encoding ); } @@ -75,12 +71,6 @@ public ReadMlcpArchiveFilesOptions categories(String... categories) { this.categories = Stream.of(categories).collect(Collectors.joining(",")); return this; } - - @Override - public ReadMlcpArchiveFilesOptions partitions(int partitions) { - this.partitions = partitions; - return this; - } } @Override diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportRdfFilesCommand.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportRdfFilesCommand.java index 97c5f360..1bf6f4de 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportRdfFilesCommand.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ImportRdfFilesCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.impl.importdata; @@ -41,21 +41,17 @@ protected Supplier> getWriteParams() { return writeParams; } - public static class ReadRdfFilesParams extends ReadFilesParams implements ReadRdfFilesOptions { + public static class ReadRdfFilesParams extends ReadCompressibleFilesParams implements ReadRdfFilesOptions { @CommandLine.Option(names = "--compression", description = "When importing compressed files, specify the type of compression used. " + OptionsUtil.VALID_VALUES_DESCRIPTION) private CompressionType compressionType; - @CommandLine.Option(names = "--partitions", description = "Specifies the number of partitions used for reading files.") - private int partitions; - @Override public Map makeOptions() { return OptionsUtil.addOptions(super.makeOptions(), Options.READ_FILES_TYPE, "rdf", - Options.READ_FILES_COMPRESSION, compressionType != null ? compressionType.name() : null, - Options.READ_NUM_PARTITIONS, OptionsUtil.intOption(partitions) + Options.READ_FILES_COMPRESSION, compressionType != null ? compressionType.name() : null ); } @@ -64,12 +60,6 @@ public ReadRdfFilesOptions compressionType(CompressionType compressionType) { this.compressionType = compressionType; return this; } - - @Override - public ReadRdfFilesOptions partitions(int partitions) { - this.partitions = partitions; - return this; - } } public static class WriteTriplesDocumentsParams extends WriteDocumentParams implements WriteTriplesDocumentsOptions { diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ReadCompressibleFilesParams.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ReadCompressibleFilesParams.java new file mode 100644 index 00000000..2784de2f --- /dev/null +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/ReadCompressibleFilesParams.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.flux.impl.importdata; + +import com.marklogic.flux.api.ReadCompressibleFilesOptions; +import com.marklogic.flux.impl.OptionsUtil; +import com.marklogic.spark.Options; +import picocli.CommandLine; + +import java.util.Map; + +/** + * Base params class for all import commands that support reading zip-compressed files. + * Declares the shared {@code --partitions}, {@code --zip-max-uncompressed-entry-bytes}, and + * {@code --zip-max-entry-count} CLI options and handles them in {@code makeOptions()}. + */ +@SuppressWarnings("unchecked") +public abstract class ReadCompressibleFilesParams + extends ReadFilesParams + implements ReadCompressibleFilesOptions { + + @CommandLine.Option(names = "--partitions", description = "Specifies the number of partitions used for reading files.") + private int partitions; + + @CommandLine.Option( + names = "--zip-max-uncompressed-entry-bytes", + description = "Maximum number of uncompressed bytes to read from a single zip entry. " + + "Set to a positive integer to enable protection. Any value less than 1 (including 0) disables the limit." + ) + private Long zipMaxEntryBytes; + + @CommandLine.Option( + names = "--zip-max-entry-count", + description = "Maximum number of entries to process from a single zip archive. " + + "Set to a positive integer to enable protection. Any value less than 1 (including 0) disables the limit." + ) + private Integer zipMaxEntryCount; + + @Override + public Map makeOptions() { + Map options = super.makeOptions(); + OptionsUtil.addOptions(options, Options.READ_NUM_PARTITIONS, OptionsUtil.intOption(partitions)); + // Forward any explicitly set value, including 0 and negatives. + // The connector treats any value < 1 as "disabled", so forwarding these allows operators + // to explicitly override future connector defaults rather than relying on omission. + if (zipMaxEntryBytes != null) { + options.put(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES, String.valueOf(zipMaxEntryBytes)); + } + if (zipMaxEntryCount != null) { + options.put(Options.READ_ZIP_MAX_ENTRY_COUNT, String.valueOf(zipMaxEntryCount)); + } + return options; + } + + @Override + public T partitions(int partitions) { + this.partitions = partitions; + return (T) this; + } + + @Override + public T zipMaxUncompressedEntryBytes(long bytes) { + this.zipMaxEntryBytes = bytes; + return (T) this; + } + + @Override + public T zipMaxEntryCount(int count) { + this.zipMaxEntryCount = count; + return (T) this; + } +} diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportAggregateXmlFilesOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportAggregateXmlFilesOptionsTest.java index e96b02c7..a3e199c4 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportAggregateXmlFilesOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportAggregateXmlFilesOptionsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.impl.importdata; @@ -7,6 +7,10 @@ import com.marklogic.spark.Options; import org.junit.jupiter.api.Test; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + class ImportAggregateXmlFilesOptionsTest extends AbstractOptionsTest { @Test @@ -25,4 +29,52 @@ void numPartitions() { Options.READ_NUM_PARTITIONS, "3" ); } + + @Test + void zipProtectionOptions() { + ImportAggregateXmlFilesCommand command = (ImportAggregateXmlFilesCommand) getCommand( + "import-aggregate-xml-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources", + "--element", "anything", + "--zip-max-uncompressed-entry-bytes", "268435456", + "--zip-max-entry-count", "100000" + ); + + assertOptions(command.getReadParams().makeOptions(), + Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES, "268435456", + Options.READ_ZIP_MAX_ENTRY_COUNT, "100000" + ); + } + + @Test + void zipProtectionOptionsNotSetByDefault() { + ImportAggregateXmlFilesCommand command = (ImportAggregateXmlFilesCommand) getCommand( + "import-aggregate-xml-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources", + "--element", "anything" + ); + + Map options = command.getReadParams().makeOptions(); + assertFalse(options.containsKey(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES), + "Zip byte limit should not be set when the flag is omitted."); + assertFalse(options.containsKey(Options.READ_ZIP_MAX_ENTRY_COUNT), + "Zip entry count limit should not be set when the flag is omitted."); + } + + @Test + void zipZeroAndNegativeValuesAreForwarded() { + ImportAggregateXmlFilesCommand command = (ImportAggregateXmlFilesCommand) getCommand( + "import-aggregate-xml-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources", + "--element", "anything", + "--zip-max-uncompressed-entry-bytes", "0", + "--zip-max-entry-count", "-1" + ); + Map options = command.getReadParams().makeOptions(); + assertEquals("0", options.get(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES)); + assertEquals("-1", options.get(Options.READ_ZIP_MAX_ENTRY_COUNT)); + } } diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportArchiveFilesOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportArchiveFilesOptionsTest.java index 7f6195d1..befc4d27 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportArchiveFilesOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportArchiveFilesOptionsTest.java @@ -7,6 +7,10 @@ import com.marklogic.spark.Options; import org.junit.jupiter.api.Test; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + class ImportArchiveFilesOptionsTest extends AbstractOptionsTest { @Test @@ -48,4 +52,50 @@ void streaming() { Options.WRITE_DOCUMENT_TYPE, "XML" ); } + + @Test + void zipProtectionOptions() { + ImportArchiveFilesCommand command = (ImportArchiveFilesCommand) getCommand( + "import-archive-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources/archive-files", + "--zip-max-uncompressed-entry-bytes", "268435456", + "--zip-max-entry-count", "100000" + ); + + assertOptions(command.getReadParams().makeOptions(), + Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES, "268435456", + Options.READ_ZIP_MAX_ENTRY_COUNT, "100000" + ); + } + + @Test + void zipProtectionOptionsNotSetByDefault() { + ImportArchiveFilesCommand command = (ImportArchiveFilesCommand) getCommand( + "import-archive-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources/archive-files" + ); + + Map options = command.getReadParams().makeOptions(); + assertFalse(options.containsKey(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES), + "Zip byte limit should not be set when the flag is omitted."); + assertFalse(options.containsKey(Options.READ_ZIP_MAX_ENTRY_COUNT), + "Zip entry count limit should not be set when the flag is omitted."); + } + + @Test + void zipZeroAndNegativeValuesAreForwarded() { + // Any value < 1 (including 0, -1, -5) is forwarded to the connector so it can explicitly disable protection. + ImportArchiveFilesCommand command = (ImportArchiveFilesCommand) getCommand( + "import-archive-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources/archive-files", + "--zip-max-uncompressed-entry-bytes", "0", + "--zip-max-entry-count", "-5" + ); + Map options = command.getReadParams().makeOptions(); + assertEquals("0", options.get(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES), "A value of 0 is forwarded so the connector can explicitly disable."); + assertEquals("-5", options.get(Options.READ_ZIP_MAX_ENTRY_COUNT), "A negative value is forwarded so the connector can explicitly disable."); + } } diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java index a12572f9..3988f94c 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java @@ -53,7 +53,9 @@ void test() { "--doc-prop", "prop1=value1", "--doc-prop", "prop2=value2", "--write-prop", "spark.someprop=somevalue", - "--write-prop", "other.prop=othervalue" + "--write-prop", "other.prop=othervalue", + "--zip-max-uncompressed-entry-bytes", "268435456", + "--zip-max-entry-count", "100000" ); assertOptions(command.getConnectionParams().makeOptions(), @@ -66,7 +68,9 @@ void test() { assertOptions(command.getReadParams().makeOptions(), Options.READ_NUM_PARTITIONS, "6", Options.READ_FILES_ENCODING, "UTF-16", - Options.STREAM_FILES, "true" + Options.STREAM_FILES, "true", + Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES, "268435456", + Options.READ_ZIP_MAX_ENTRY_COUNT, "100000" ); assertOptions(command.getWriteParams().makeOptions(), @@ -279,4 +283,19 @@ void incrementalWriteSchemaNoView() { assertEquals("Error: Missing required argument(s): --incremental-write-view=", ex.getMessage()); } + + @Test + void zipZeroAndNegativeValuesAreForwarded() { + // Any value < 1 disables the limit; the option should not appear in the map. + ImportFilesCommand command = (ImportFilesCommand) getCommand( + "import-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources/mixed-files", + "--zip-max-uncompressed-entry-bytes", "0", + "--zip-max-entry-count", "-1" + ); + Map options = command.getReadParams().makeOptions(); + assertEquals("0", options.get(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES), "A value of 0 is forwarded so the connector can explicitly disable."); + assertEquals("-1", options.get(Options.READ_ZIP_MAX_ENTRY_COUNT), "A negative value is forwarded so the connector can explicitly disable."); + } } diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportMlcpArchiveFilesOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportMlcpArchiveFilesOptionsTest.java index 34b2dcab..0e73288f 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportMlcpArchiveFilesOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportMlcpArchiveFilesOptionsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.impl.importdata; @@ -7,6 +7,10 @@ import com.marklogic.spark.Options; import org.junit.jupiter.api.Test; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + class ImportMlcpArchiveFilesOptionsTest extends AbstractOptionsTest { @Test @@ -25,4 +29,49 @@ void test() { Options.READ_FILES_ENCODING, "UTF-16" ); } + + @Test + void zipProtectionOptions() { + ImportMlcpArchiveFilesCommand command = (ImportMlcpArchiveFilesCommand) getCommand( + "import-mlcp-archive-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources", + "--zip-max-uncompressed-entry-bytes", "268435456", + "--zip-max-entry-count", "100000" + ); + + assertOptions(command.getReadParams().makeOptions(), + Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES, "268435456", + Options.READ_ZIP_MAX_ENTRY_COUNT, "100000" + ); + } + + @Test + void zipProtectionOptionsNotSetByDefault() { + ImportMlcpArchiveFilesCommand command = (ImportMlcpArchiveFilesCommand) getCommand( + "import-mlcp-archive-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources" + ); + + Map options = command.getReadParams().makeOptions(); + assertFalse(options.containsKey(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES), + "Zip byte limit should not be set when the flag is omitted."); + assertFalse(options.containsKey(Options.READ_ZIP_MAX_ENTRY_COUNT), + "Zip entry count limit should not be set when the flag is omitted."); + } + + @Test + void zipZeroAndNegativeValuesAreForwarded() { + ImportMlcpArchiveFilesCommand command = (ImportMlcpArchiveFilesCommand) getCommand( + "import-mlcp-archive-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources", + "--zip-max-uncompressed-entry-bytes", "0", + "--zip-max-entry-count", "-1" + ); + Map options = command.getReadParams().makeOptions(); + assertEquals("0", options.get(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES)); + assertEquals("-1", options.get(Options.READ_ZIP_MAX_ENTRY_COUNT)); + } } diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportRdfFilesOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportRdfFilesOptionsTest.java index 5189d549..84eceb0f 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportRdfFilesOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportRdfFilesOptionsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.impl.importdata; @@ -7,6 +7,10 @@ import com.marklogic.spark.Options; import org.junit.jupiter.api.Test; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + class ImportRdfFilesOptionsTest extends AbstractOptionsTest { @Test @@ -23,4 +27,49 @@ void numPartitions() { Options.READ_NUM_PARTITIONS, "4" ); } + + @Test + void zipProtectionOptions() { + ImportRdfFilesCommand command = (ImportRdfFilesCommand) getCommand( + "import-rdf-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources/rdf", + "--zip-max-uncompressed-entry-bytes", "268435456", + "--zip-max-entry-count", "100000" + ); + + assertOptions(command.getReadParams().makeOptions(), + Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES, "268435456", + Options.READ_ZIP_MAX_ENTRY_COUNT, "100000" + ); + } + + @Test + void zipProtectionOptionsNotSetByDefault() { + ImportRdfFilesCommand command = (ImportRdfFilesCommand) getCommand( + "import-rdf-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources/rdf" + ); + + Map options = command.getReadParams().makeOptions(); + assertFalse(options.containsKey(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES), + "Zip byte limit should not be set when the flag is omitted."); + assertFalse(options.containsKey(Options.READ_ZIP_MAX_ENTRY_COUNT), + "Zip entry count limit should not be set when the flag is omitted."); + } + + @Test + void zipZeroAndNegativeValuesAreForwarded() { + ImportRdfFilesCommand command = (ImportRdfFilesCommand) getCommand( + "import-rdf-files", + "--connection-string", makeConnectionString(), + "--path", "src/test/resources/rdf", + "--zip-max-uncompressed-entry-bytes", "0", + "--zip-max-entry-count", "-1" + ); + Map options = command.getReadParams().makeOptions(); + assertEquals("0", options.get(Options.READ_ZIP_MAX_UNCOMPRESSED_ENTRY_BYTES)); + assertEquals("-1", options.get(Options.READ_ZIP_MAX_ENTRY_COUNT)); + } }