diff --git a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/DocUtils.java b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/DocUtils.java index 0ad2d2aa0..3be039105 100644 --- a/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/DocUtils.java +++ b/beans/src/main/java/fr/pilato/elasticsearch/crawler/fs/beans/DocUtils.java @@ -20,10 +20,6 @@ */ package fr.pilato.elasticsearch.crawler.fs.beans; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerIllegalConfigurationException; import fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil; import java.io.IOException; @@ -31,9 +27,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Iterator; +import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.node.ObjectNode; public class DocUtils { @@ -123,10 +123,8 @@ public static Doc getMergedDoc(Doc doc, InputStream tags, ObjectMapper mapper) * @return the merged nodes */ private static JsonNode merge(JsonNode mainNode, JsonNode updateNode) { - Iterator fieldNames = updateNode.fieldNames(); - - while (fieldNames.hasNext()) { - String fieldName = fieldNames.next(); + for (Map.Entry entry : updateNode.properties()) { + String fieldName = entry.getKey(); JsonNode jsonNode = mainNode.get(fieldName); if (jsonNode != null) { @@ -152,7 +150,7 @@ private static JsonNode merge(JsonNode mainNode, JsonNode updateNode) { public static String prettyPrint(Doc doc) { try { return JsonUtil.prettyMapper.writeValueAsString(doc); - } catch (JsonProcessingException e) { + } catch (JacksonException e) { logger.warn("Can not pretty print the document as json", e); return null; } diff --git a/core/pom.xml b/core/pom.xml index ba1d72ce0..ebe375b5f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -52,10 +52,6 @@ org.glassfish.jersey.containers jersey-container-grizzly2-http - - org.glassfish.jersey.media - jersey-media-json-jackson - org.glassfish.jersey.media jersey-media-multipart diff --git a/docs/source/admin/fs/elasticsearch.rst b/docs/source/admin/fs/elasticsearch.rst index 69d31f51d..abd04e82c 100644 --- a/docs/source/admin/fs/elasticsearch.rst +++ b/docs/source/admin/fs/elasticsearch.rst @@ -730,33 +730,33 @@ Here is a typical JSON document generated by the crawler: { "content":"This is a sample text available in page 1\n\nThis second part of the text is in Page 2\n\n", "content_semantic":"This is a sample text available in page 1\n\nThis second part of the text is in Page 2\n\n", + "file":{ + "content_type":"application/vnd.oasis.opendocument.text", + "created":"2018-07-30T11:35:08.000+0000", + "extension":"odt", + "filename":"test.odt", + "filesize":6236, + "indexing_date":"2018-07-30T11:35:19.781+0000", + "last_accessed":"2018-07-30T11:35:08.000+0000", + "last_modified":"2018-07-30T11:35:08.000+0000", + "url":"file:///tmp/test.odt" + }, "meta":{ "author":"David Pilato", - "title":"Test Tika title", + "created":"2016-07-07T16:37:00.000+0000", "date":"2016-07-07T16:37:00.000+0000", + "description":"Comments", "keywords":[ "keyword1", " keyword2" ], "language":"en", - "description":"Comments", - "created":"2016-07-07T16:37:00.000+0000" - }, - "file":{ - "extension":"odt", - "content_type":"application/vnd.oasis.opendocument.text", - "created":"2018-07-30T11:35:08.000+0000", - "last_modified":"2018-07-30T11:35:08.000+0000", - "last_accessed":"2018-07-30T11:35:08.000+0000", - "indexing_date":"2018-07-30T11:35:19.781+0000", - "filesize":6236, - "filename":"test.odt", - "url":"file:///tmp/test.odt" + "title":"Test Tika title" }, "path":{ + "real":"/tmp/test.odt", "root":"7537e4fb47e553f110a1ec312c2537c0", - "virtual":"/test.odt", - "real":"/tmp/test.odt" + "virtual":"/test.odt" } } diff --git a/docs/source/admin/fs/rest.rst b/docs/source/admin/fs/rest.rst index 4f43647c3..4ecc1c210 100644 --- a/docs/source/admin/fs/rest.rst +++ b/docs/source/admin/fs/rest.rst @@ -45,63 +45,63 @@ It will give you a response similar to: :substitutions: { - "ok" : true, - "version" : "|FSCrawler_version|", "elasticsearch" : "|ES_stack_version|", + "ok" : true, "settings" : { - "name" : "fscrawler", + "elasticsearch" : { + "bulk_size" : 100, + "byte_size" : "10mb", + "flush_interval" : "5s", + "index" : "fscrawler_docs", + "index_folder" : "fscrawler_folder", + "push_templates" : true, + "semantic_search" : true, + "ssl_verification" : true, + "urls" : [ "http://es-fscrawler:9200" ], + "username" : "elastic" + }, "fs" : { - "url" : "/tmp/es", - "update_rate" : "15m", - "excludes" : [ "*/~*" ], - "json_support" : false, + "acl_support" : false, "add_as_inner_object" : false, - "xml_support" : false, - "follow_symlinks" : false, - "remove_deleted" : true, - "continue_on_error" : false, - "filename_as_id" : false, "add_filesize" : true, "attributes_support" : false, - "store_source" : false, + "continue_on_error" : false, + "excludes" : [ "*/~*" ], + "filename_as_id" : false, + "follow_symlinks" : false, "index_content" : true, - "acl_support" : false, - "raw_metadata" : true, "index_folders" : true, + "json_support" : false, "lang_detect" : false, "ocr" : { "enabled" : true, "language" : "eng", "output_type" : "txt", - "pdf_strategy" : "ocr_and_text", "page_seg_mode" : 1, + "pdf_strategy" : "ocr_and_text", "preserve_interword_spacing" : false - } + }, + "raw_metadata" : true, + "remove_deleted" : true, + "store_source" : false, + "update_rate" : "15m", + "url" : "/tmp/es", + "xml_support" : false + }, + "name" : "fscrawler", + "rest" : { + "enable_cors" : false, + "url" : "http://127.0.0.1:8080" }, "server" : { "port" : 0, "protocol" : "local" }, - "elasticsearch" : { - "urls" : [ "http://es-fscrawler:9200" ], - "index" : "fscrawler_docs", - "index_folder" : "fscrawler_folder", - "bulk_size" : 100, - "flush_interval" : "5s", - "byte_size" : "10mb", - "username" : "elastic", - "ssl_verification" : true, - "push_templates" : true, - "semantic_search" : true - }, - "rest" : { - "url" : "http://127.0.0.1:8080", - "enable_cors" : false - }, "tags" : { "meta_filename" : ".meta.yml" } - } + }, + "version" : "|FSCrawler_version|" } Uploading a binary document @@ -119,8 +119,8 @@ It will give you a response similar to: .. code:: json { - "ok" : true, "filename" : "test.txt", + "ok" : true, "url" : "http://127.0.0.1:9200/fscrawler-rest-tests_doc/_doc/dd18bf3a8ea2a3e53e2661c7fb53534" } @@ -143,6 +143,12 @@ You will get back your document as it has been stored by elasticsearch: "found" : true, "_source" : { "content" : "This file contains some words.\n", + "file" : { + "content_type" : "text/plain; charset=ISO-8859-1", + "extension" : "txt", + "filename" : "test.txt", + "indexing_date" : "2017-01-04T21:01:08.043" + }, "meta" : { "raw" : { "X-Parsed-By" : "org.apache.tika.parser.DefaultParser", @@ -150,15 +156,9 @@ You will get back your document as it has been stored by elasticsearch: "Content-Type" : "text/plain; charset=ISO-8859-1" } }, - "file" : { - "extension" : "txt", - "content_type" : "text/plain; charset=ISO-8859-1", - "indexing_date" : "2017-01-04T21:01:08.043", - "filename" : "test.txt" - }, "path" : { - "virtual" : "test.txt", - "real" : "test.txt" + "real" : "test.txt", + "virtual" : "test.txt" } } } @@ -177,11 +177,14 @@ will give .. code:: json { - "ok" : true, - "filename" : "test.txt", - "url" : "http://127.0.0.1:9200/fscrawler-rest-tests_doc/_doc/dd18bf3a8ea2a3e53e2661c7fb53534", "doc" : { "content" : "This file contains some words.\n", + "file" : { + "content_type" : "text/plain; charset=ISO-8859-1", + "extension" : "txt", + "filename" : "test.txt", + "indexing_date" : "2017-01-04T14:05:10.325" + }, "meta" : { "raw" : { "X-Parsed-By" : "org.apache.tika.parser.DefaultParser", @@ -189,17 +192,14 @@ will give "Content-Type" : "text/plain; charset=ISO-8859-1" } }, - "file" : { - "extension" : "txt", - "content_type" : "text/plain; charset=ISO-8859-1", - "indexing_date" : "2017-01-04T14:05:10.325", - "filename" : "test.txt" - }, "path" : { - "virtual" : "test.txt", - "real" : "test.txt" + "real" : "test.txt", + "virtual" : "test.txt" } - } + }, + "filename" : "test.txt", + "ok" : true, + "url" : "http://127.0.0.1:9200/fscrawler-rest-tests_doc/_doc/dd18bf3a8ea2a3e53e2661c7fb53534" } Uploading a binary document from a 3rd party service @@ -541,10 +541,10 @@ It will give you a response similar to: .. code:: json { - "ok": true, "filename": "test.txt", + "id": "dd18bf3a8ea2a3e53e2661c7fb53534", "index": "rest", - "id": "dd18bf3a8ea2a3e53e2661c7fb53534" + "ok": true } If you know the document id, you can pass it to FSCrawler within the url: @@ -558,11 +558,11 @@ If the document does not exist, you will get the following response: .. code:: json { - "ok": false, - "message": "Can not remove document [rest/test.txt]: Can not remove document rest/dd18bf3a8ea2a3e53e2661c7fb53534 cause: NOT_FOUND", "filename": "test.txt", + "id": "dd18bf3a8ea2a3e53e2661c7fb53534", "index": "rest", - "id": "dd18bf3a8ea2a3e53e2661c7fb53534" + "message": "Can not remove document [rest/test.txt]: Can not remove document rest/dd18bf3a8ea2a3e53e2661c7fb53534 cause: NOT_FOUND", + "ok": false } Specifying an elasticsearch index @@ -664,19 +664,19 @@ It will give you a response similar to: .. code:: json { - "state" : "RUNNING", - "scan_id" : "abc123-def456", - "current_path" : "/data/documents/subfolder", - "pending_directories" : 42, "completed_directories" : 158, - "files_processed" : 1523, + "current_path" : "/data/documents/subfolder", + "elapsed_time" : "15m 32s", "files_deleted" : 12, - "scan_start_time" : "2024-01-15T10:30:00", - "scan_end_time" : null, + "files_processed" : 1523, + "last_error" : null, "next_check" : null, - "elapsed_time" : "15m 32s", + "pending_directories" : 42, "retry_count" : 0, - "last_error" : null + "scan_end_time" : null, + "scan_id" : "abc123-def456", + "scan_start_time" : "2024-01-15T10:30:00", + "state" : "RUNNING" } When a scan is completed, the response will also include the ``scan_end_time`` and ``next_check`` fields: @@ -684,13 +684,13 @@ When a scan is completed, the response will also include the ``scan_end_time`` a .. code:: json { - "state" : "COMPLETED", - "files_processed" : 2500, + "elapsed_time" : "1h 15m", "files_deleted" : 25, - "scan_start_time" : "2024-01-15T10:30:00", - "scan_end_time" : "2024-01-15T11:45:00", + "files_processed" : 2500, "next_check" : "2024-01-15T12:00:00", - "elapsed_time" : "1h 15m" + "scan_end_time" : "2024-01-15T11:45:00", + "scan_start_time" : "2024-01-15T10:30:00", + "state" : "COMPLETED" } The possible states are: @@ -738,8 +738,8 @@ Success response (200): .. code:: json { - "ok" : true, - "message" : "Crawler paused. Checkpoint saved." + "message" : "Crawler paused. Checkpoint saved.", + "ok" : true } If the crawler is already paused, you get 200 with: @@ -747,8 +747,8 @@ If the crawler is already paused, you get 200 with: .. code:: json { - "ok" : true, - "message" : "Crawler is already paused." + "message" : "Crawler is already paused.", + "ok" : true } Error response (400) when the crawler is not running: @@ -756,8 +756,8 @@ Error response (400) when the crawler is not running: .. code:: json { - "ok" : false, - "message" : "Crawler is not running" + "message" : "Crawler is not running", + "ok" : false } Resuming the crawler @@ -774,8 +774,8 @@ Success response (200) when resuming from pause: .. code:: json { - "ok" : true, - "message" : "Crawler resumed." + "message" : "Crawler resumed.", + "ok" : true } If the crawler is not paused, you get 200 with no action taken: @@ -783,8 +783,8 @@ If the crawler is not paused, you get 200 with no action taken: .. code:: json { - "ok" : true, - "message" : "Crawler is not paused. No action needed." + "message" : "Crawler is not paused. No action needed.", + "ok" : true } Error response (400) when the crawler is closed: @@ -792,8 +792,8 @@ Error response (400) when the crawler is closed: .. code:: json { - "ok" : false, - "message" : "Crawler is closed. Cannot resume." + "message" : "Crawler is closed. Cannot resume.", + "ok" : false } Clearing the checkpoint @@ -811,8 +811,8 @@ Success response (200): .. code:: json { - "ok" : true, - "message" : "Checkpoint cleared" + "message" : "Checkpoint cleared", + "ok" : true } Error response (400) when the crawler is running and not paused: @@ -820,8 +820,8 @@ Error response (400) when the crawler is running and not paused: .. code:: json { - "ok" : false, - "message" : "Cannot clear checkpoint while crawler is running. Pause or stop it first." + "message" : "Cannot clear checkpoint while crawler is running. Pause or stop it first.", + "ok" : false } Error response (404) when there is no active crawler (e.g. started with ``--loop 0``): @@ -829,8 +829,8 @@ Error response (404) when there is no active crawler (e.g. started with ``--loop .. code:: json { - "ok" : false, - "message" : "Failed to clear checkpoint as we don't have a checkpoint handler. This probably means there's no active crawler. Did you start with --loop 0?" + "message" : "Failed to clear checkpoint as we don't have a checkpoint handler. This probably means there's no active crawler. Did you start with --loop 0?", + "ok" : false } On I/O error, the server returns 500 with a message starting with ``Failed to clear checkpoint:``. diff --git a/docs/source/admin/status.rst b/docs/source/admin/status.rst index b52a4e839..01673cc28 100644 --- a/docs/source/admin/status.rst +++ b/docs/source/admin/status.rst @@ -34,11 +34,11 @@ For example, a checkpoint for a completed scan: .. code-block:: json { - "state": "COMPLETED", - "scan_end_time": "2025-07-01T12:00:00", - "next_check": "2025-07-01T12:15:00", + "files_deleted": 0, "files_processed": 100, - "files_deleted": 0 + "next_check": "2025-07-01T12:15:00", + "scan_end_time": "2025-07-01T12:00:00", + "state": "COMPLETED" } A checkpoint for a running scan: @@ -46,15 +46,15 @@ A checkpoint for a running scan: .. code-block:: json { - "scan_id": "abc123-def456", - "state": "RUNNING", - "scan_start_time": "2025-07-01T12:00:00", - "current_path": "/data/documents/subfolder", - "pending_paths": ["/data/documents/other"], "completed_paths": ["/data/documents", "/data/documents/processed"], - "files_processed": 50, + "current_path": "/data/documents/subfolder", "files_deleted": 0, - "retry_count": 0 + "files_processed": 50, + "pending_paths": ["/data/documents/other"], + "retry_count": 0, + "scan_id": "abc123-def456", + "scan_start_time": "2025-07-01T12:00:00", + "state": "RUNNING" } Forcing a new scan diff --git a/docs/source/release/2.10.rst b/docs/source/release/2.10.rst index a69ff7e9f..5022c6387 100644 --- a/docs/source/release/2.10.rst +++ b/docs/source/release/2.10.rst @@ -79,6 +79,11 @@ Updated * Update to Tika |Tika_version|_. Thanks to dadoonet. * The default alias name is now the job name and not forced to ``fscrawler`` anymore. Thanks to dadoonet. * The default REST endpoint is now running at ``/`` instead of ``/fscrawler/``. Thanks to dadoonet. +* Upgrade to Jackson 3.x. Closes `#2419 `_. Thanks to dadoonet. +* As a consequence of the Jackson 3 upgrade, the JSON and YAML documents produced by FSCrawler now serialize their + fields in alphabetical order. This affects the indexed documents, the generated ``_settings.yaml`` and + ``_checkpoint.json`` files, and the REST API responses. This is purely cosmetic (field order is not significant in + JSON), but users who version their configuration files may notice a one-time reordering. Thanks to dadoonet. Removed ------- diff --git a/docs/source/user/rest.rst b/docs/source/user/rest.rst index d318e6649..7c97c44e7 100644 --- a/docs/source/user/rest.rst +++ b/docs/source/user/rest.rst @@ -39,8 +39,8 @@ elasticsearch URL for the created document, like: .. code:: json { - "ok" : true, "filename" : "test.txt", + "ok" : true, "url" : "http://127.0.0.1:9200/fscrawler-rest-tests_doc/doc/dd18bf3a8ea2a3e53e2661c7fb53534" } diff --git a/elasticsearch-client/pom.xml b/elasticsearch-client/pom.xml index 1d5af9a20..e11167296 100644 --- a/elasticsearch-client/pom.xml +++ b/elasticsearch-client/pom.xml @@ -38,21 +38,11 @@ org.glassfish.jersey.inject jersey-hk2 - - org.glassfish.jersey.media - jersey-media-json-jackson - org.glassfish.jersey.connectors jersey-apache5-connector - - - com.fasterxml.jackson.module - jackson-module-jaxb-annotations - - com.jayway.jsonpath diff --git a/framework/pom.xml b/framework/pom.xml index b091befa1..ff3c53f5e 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -38,25 +38,21 @@ pf4j - + - com.fasterxml.jackson.core + tools.jackson.core jackson-core - com.fasterxml.jackson.core + tools.jackson.core jackson-databind - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - com.fasterxml.jackson.dataformat + tools.jackson.dataformat jackson-dataformat-xml - com.fasterxml.jackson.dataformat + tools.jackson.dataformat jackson-dataformat-yaml diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueDeserializer.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueDeserializer.java index bb662017b..57fa009b6 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueDeserializer.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueDeserializer.java @@ -20,10 +20,9 @@ */ package fr.pilato.elasticsearch.crawler.fs.framework; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import java.io.IOException; +import tools.jackson.core.JsonParser; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.deser.std.StdDeserializer; /** Jackson Deserializer for ByteSizeValue object */ public class ByteSizeValueDeserializer extends StdDeserializer { @@ -32,7 +31,7 @@ public ByteSizeValueDeserializer() { } @Override - public ByteSizeValue deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return ByteSizeValue.parseBytesSizeValue(p.getText()); + public ByteSizeValue deserialize(JsonParser p, DeserializationContext ctxt) { + return ByteSizeValue.parseBytesSizeValue(p.getString()); } } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueSerializer.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueSerializer.java index 828a89bb9..5f30c3005 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueSerializer.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/ByteSizeValueSerializer.java @@ -20,10 +20,9 @@ */ package fr.pilato.elasticsearch.crawler.fs.framework; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import java.io.IOException; +import tools.jackson.core.JsonGenerator; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ser.std.StdSerializer; /** Jackson Serializer for ByteSizeValue object */ public class ByteSizeValueSerializer extends StdSerializer { @@ -32,7 +31,7 @@ public ByteSizeValueSerializer() { } @Override - public void serialize(ByteSizeValue value, JsonGenerator gen, SerializerProvider provider) throws IOException { + public void serialize(ByteSizeValue value, JsonGenerator gen, SerializationContext ctxt) { gen.writeString(value.toString()); } } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtil.java index cdcef3641..87ae51f2c 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtil.java @@ -21,25 +21,25 @@ package fr.pilato.elasticsearch.crawler.fs.framework; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.Version; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.Predicate; import com.jayway.jsonpath.spi.json.JsonSmartJsonProvider; import com.jayway.jsonpath.spi.mapper.JsonSmartMappingProvider; -import java.io.IOException; import java.io.InputStream; import java.util.Map; +import tools.jackson.core.JacksonException; +import tools.jackson.core.Version; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.PropertyNamingStrategies; +import tools.jackson.databind.SerializationFeature; +import tools.jackson.databind.cfg.DateTimeFeature; +import tools.jackson.databind.json.JsonMapper; +import tools.jackson.databind.module.SimpleModule; +import tools.jackson.dataformat.yaml.YAMLMapper; public class JsonUtil { @@ -47,8 +47,8 @@ private JsonUtil() { // Utility class, do not instantiate } - public static final ObjectMapper prettyMapper; - public static final ObjectMapper mapper; + public static final JsonMapper prettyMapper; + public static final JsonMapper mapper; public static final ObjectMapper ymlMapper; public static final Configuration configuration = new Configuration.ConfigurationBuilder() .jsonProvider(new JsonSmartJsonProvider()) @@ -65,44 +65,54 @@ private JsonUtil() { fscrawler.addSerializer(new ByteSizeValueSerializer()); fscrawler.addDeserializer(ByteSizeValue.class, new ByteSizeValueDeserializer()); - mapper = new ObjectMapper(); - mapper.registerModule(new JavaTimeModule()); - mapper.registerModule(fscrawler); - mapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); - mapper.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false); - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - mapper.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - mapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_EMPTY); + // Jackson 3 mappers are immutable: they are configured through their builder. + // The java.time support is now built into jackson-databind, so no JavaTimeModule registration is needed. + mapper = JsonMapper.builder() + .addModule(fscrawler) + .propertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) + .disable(DateTimeFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) + .disable(DateTimeFeature.WRITE_DATES_AS_TIMESTAMPS) + .disable(DateTimeFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + // Jackson 2's setDefaultPropertyInclusion(NON_EMPTY) applied to both value and content + // (map/collection entries); reproduce it so empty map values are omitted too. + .changeDefaultPropertyInclusion(incl -> incl.withValueInclusion(JsonInclude.Include.NON_EMPTY) + .withContentInclusion(JsonInclude.Include.NON_EMPTY)) + .build(); - prettyMapper = new ObjectMapper(); - prettyMapper.registerModule(new JavaTimeModule()); - prettyMapper.registerModule(fscrawler); - prettyMapper.configure(SerializationFeature.INDENT_OUTPUT, true); - prettyMapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); - prettyMapper.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false); - prettyMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - prettyMapper.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false); - prettyMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - prettyMapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_EMPTY); + prettyMapper = JsonMapper.builder() + .addModule(fscrawler) + .enable(SerializationFeature.INDENT_OUTPUT) + .propertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) + .disable(DateTimeFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) + .disable(DateTimeFeature.WRITE_DATES_AS_TIMESTAMPS) + .disable(DateTimeFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + // Jackson 2's setDefaultPropertyInclusion(NON_EMPTY) applied to both value and content + // (map/collection entries); reproduce it so empty map values are omitted too. + .changeDefaultPropertyInclusion(incl -> incl.withValueInclusion(JsonInclude.Include.NON_EMPTY) + .withContentInclusion(JsonInclude.Include.NON_EMPTY)) + .build(); - YAMLFactory yamlFactory = new YAMLFactory(); - ymlMapper = new ObjectMapper(yamlFactory); - ymlMapper.registerModule(new JavaTimeModule()); - ymlMapper.registerModule(fscrawler); - ymlMapper.configure(SerializationFeature.INDENT_OUTPUT, true); - ymlMapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); - ymlMapper.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false); - ymlMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - ymlMapper.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false); - ymlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - ymlMapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_EMPTY); + ymlMapper = YAMLMapper.builder() + .addModule(fscrawler) + .enable(SerializationFeature.INDENT_OUTPUT) + .propertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) + .disable(DateTimeFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) + .disable(DateTimeFeature.WRITE_DATES_AS_TIMESTAMPS) + .disable(DateTimeFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + // Jackson 2's setDefaultPropertyInclusion(NON_EMPTY) applied to both value and content + // (map/collection entries); reproduce it so empty map values are omitted too. + .changeDefaultPropertyInclusion(incl -> incl.withValueInclusion(JsonInclude.Include.NON_EMPTY) + .withContentInclusion(JsonInclude.Include.NON_EMPTY)) + .build(); } public static String serialize(Object object) { try { return mapper.writeValueAsString(object); - } catch (JsonProcessingException e) { + } catch (JacksonException e) { throw new RuntimeException(e); } } @@ -110,7 +120,7 @@ public static String serialize(Object object) { public static T deserialize(String json, Class clazz) { try { return mapper.readValue(json, clazz); - } catch (IOException e) { + } catch (JacksonException e) { throw new RuntimeException(e); } } @@ -118,7 +128,7 @@ public static T deserialize(String json, Class clazz) { public static Map asMap(InputStream stream) { try { return mapper.readValue(stream, new TypeReference<>() {}); - } catch (IOException e) { + } catch (JacksonException e) { throw new RuntimeException(e); } } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/PercentageDeserializer.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/PercentageDeserializer.java index df915070a..a255312bf 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/PercentageDeserializer.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/PercentageDeserializer.java @@ -18,29 +18,11 @@ * * Made from 🇫🇷🇪🇺 with ❤️ - 2011-2026 */ -package fr.pilato.elasticsearch.crawler.fs.framework; /* - * Licensed to David Pilato (the "Author") under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Author licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +package fr.pilato.elasticsearch.crawler.fs.framework; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import java.io.IOException; +import tools.jackson.core.JsonParser; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.deser.std.StdDeserializer; /** Jackson Deserializer for Percentage object */ public class PercentageDeserializer extends StdDeserializer { @@ -49,7 +31,7 @@ public PercentageDeserializer() { } @Override - public Percentage deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return Percentage.parse(p.getText()); + public Percentage deserialize(JsonParser p, DeserializationContext ctxt) { + return Percentage.parse(p.getString()); } } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/PercentageSerializer.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/PercentageSerializer.java index 49a2320b1..fcd427016 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/PercentageSerializer.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/PercentageSerializer.java @@ -18,29 +18,11 @@ * * Made from 🇫🇷🇪🇺 with ❤️ - 2011-2026 */ -package fr.pilato.elasticsearch.crawler.fs.framework; /* - * Licensed to David Pilato (the "Author") under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Author licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +package fr.pilato.elasticsearch.crawler.fs.framework; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import java.io.IOException; +import tools.jackson.core.JsonGenerator; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ser.std.StdSerializer; /** Jackson Serializer for Percentage object */ public class PercentageSerializer extends StdSerializer { @@ -49,7 +31,7 @@ public PercentageSerializer() { } @Override - public void serialize(Percentage value, JsonGenerator gen, SerializerProvider provider) throws IOException { + public void serialize(Percentage value, JsonGenerator gen, SerializationContext ctxt) { gen.writeString(value.format()); } } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/TimeValueDeserializer.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/TimeValueDeserializer.java index cfacceb9d..eff41c02c 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/TimeValueDeserializer.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/TimeValueDeserializer.java @@ -18,29 +18,11 @@ * * Made from 🇫🇷🇪🇺 with ❤️ - 2011-2026 */ -package fr.pilato.elasticsearch.crawler.fs.framework; /* - * Licensed to David Pilato (the "Author") under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Author licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +package fr.pilato.elasticsearch.crawler.fs.framework; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import java.io.IOException; +import tools.jackson.core.JsonParser; +import tools.jackson.databind.DeserializationContext; +import tools.jackson.databind.deser.std.StdDeserializer; /** Jackson Deserializer for TimeValue object */ public class TimeValueDeserializer extends StdDeserializer { @@ -49,7 +31,7 @@ public TimeValueDeserializer() { } @Override - public TimeValue deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return TimeValue.parseTimeValue(p.getText()); + public TimeValue deserialize(JsonParser p, DeserializationContext ctxt) { + return TimeValue.parseTimeValue(p.getString()); } } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/TimeValueSerializer.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/TimeValueSerializer.java index aca1c1ee4..a51d46c51 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/TimeValueSerializer.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/TimeValueSerializer.java @@ -18,29 +18,11 @@ * * Made from 🇫🇷🇪🇺 with ❤️ - 2011-2026 */ -package fr.pilato.elasticsearch.crawler.fs.framework; /* - * Licensed to David Pilato (the "Author") under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Author licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +package fr.pilato.elasticsearch.crawler.fs.framework; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import java.io.IOException; +import tools.jackson.core.JsonGenerator; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ser.std.StdSerializer; /** Jackson Serializer for TimeValue object */ public class TimeValueSerializer extends StdSerializer { @@ -49,7 +31,7 @@ public TimeValueSerializer() { } @Override - public void serialize(TimeValue value, JsonGenerator gen, SerializerProvider provider) throws IOException { + public void serialize(TimeValue value, JsonGenerator gen, SerializationContext ctxt) { gen.writeString(value.toString()); } } diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtilTest.java index 2cc992a04..b019d1008 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/JsonUtilTest.java @@ -20,15 +20,15 @@ */ package fr.pilato.elasticsearch.crawler.fs.framework; -import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.DocumentContext; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; -import java.io.IOException; import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.ObjectMapper; class JsonUtilTest extends AbstractFSCrawlerTestCase { @@ -101,56 +101,57 @@ public void setCities(List cities) { } @Test - void mappersWithStringsArray() throws IOException { - // We try with multiple elements in the cities field as an array + void mappersWithStringsArray() { + // We try with multiple elements in the cities field as an array. + // Jackson 3 serializes properties in alphabetical order by default, so "cities" comes before "name". mapperTester( JsonUtil.mapper, - "{\"name\":\"Netherlands\",\"cities\":[\"Amsterdam\",\"Tamassint\"]}", + "{\"cities\":[\"Amsterdam\",\"Tamassint\"],\"name\":\"Netherlands\"}", List.of("Amsterdam", "Tamassint")); mapperTester(JsonUtil.prettyMapper, """ { - "name" : "Netherlands", - "cities" : [ "Amsterdam", "Tamassint" ] + "cities" : [ "Amsterdam", "Tamassint" ], + "name" : "Netherlands" }""", List.of("Amsterdam", "Tamassint")); mapperTester(JsonUtil.ymlMapper, """ --- - name: "Netherlands" cities: - "Amsterdam" - "Tamassint" + name: "Netherlands" """, List.of("Amsterdam", "Tamassint")); // We try with one single element in the cities field as an array - mapperTester(JsonUtil.mapper, "{\"name\":\"Netherlands\",\"cities\":[\"Amsterdam\"]}", List.of("Amsterdam")); + mapperTester(JsonUtil.mapper, "{\"cities\":[\"Amsterdam\"],\"name\":\"Netherlands\"}", List.of("Amsterdam")); mapperTester(JsonUtil.prettyMapper, """ { - "name" : "Netherlands", - "cities" : [ "Amsterdam" ] + "cities" : [ "Amsterdam" ], + "name" : "Netherlands" }""", List.of("Amsterdam")); mapperTester(JsonUtil.ymlMapper, """ --- - name: "Netherlands" cities: - "Amsterdam" + name: "Netherlands" """, List.of("Amsterdam")); // We try with one single element in the cities field as a string and this should fail Assertions.assertThatThrownBy(() -> JsonUtil.mapper.readValue("{\"name\":\"Netherlands\",\"cities\":\"Amsterdam\"}", Country.class)) - .isInstanceOf(IOException.class) + .isInstanceOf(JacksonException.class) .hasMessageContaining( - "Cannot construct instance of `java.util.ArrayList` (although at least one Creator exists)"); + "Cannot deserialize value of type `java.util.ArrayList` from String value"); Assertions.assertThatThrownBy(() -> JsonUtil.prettyMapper.readValue( "{\"name\":\"Netherlands\",\"cities\":\"Amsterdam\"}", Country.class)) - .isInstanceOf(IOException.class) + .isInstanceOf(JacksonException.class) .hasMessageContaining( - "Cannot construct instance of `java.util.ArrayList` (although at least one Creator exists)"); + "Cannot deserialize value of type `java.util.ArrayList` from String value"); Assertions.assertThatThrownBy(() -> JsonUtil.ymlMapper.readValue(""" --- name: "Netherlands" cities: "Amsterdam" """, Country.class)) - .isInstanceOf(IOException.class) + .isInstanceOf(JacksonException.class) .hasMessageContaining( - "Cannot construct instance of `java.util.ArrayList` (although at least one Creator exists)"); + "Cannot deserialize value of type `java.util.ArrayList` from String value"); } /** @@ -160,9 +161,8 @@ void mappersWithStringsArray() throws IOException { * @param mapper the mapper to test * @param input the input to use * @param expectedCities the expected cities list - * @throws IOException in case of error */ - private void mapperTester(ObjectMapper mapper, String input, List expectedCities) throws IOException { + private void mapperTester(ObjectMapper mapper, String input, List expectedCities) { logger.debug("Testing mapper: {} with {}", mapper.version().toFullString(), input); Country country = mapper.readValue(input, Country.class); Assertions.assertThat(country.name).isEqualTo("Netherlands"); diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java index b34f212f4..86e70680e 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractRestITCase.java @@ -56,7 +56,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.assertj.core.api.Assertions; -import org.glassfish.jersey.jackson.JacksonFeature; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; @@ -65,6 +64,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.parallel.ResourceLock; +import tools.jackson.jakarta.rs.json.JacksonJsonProvider; @SuppressWarnings("ALL") @ResourceLock("rest-server") @@ -208,7 +208,7 @@ static void startRestClient() throws IOException { httpClient = ClientBuilder.newBuilder() .register(MultiPartFeature.class) .register(RestJsonProvider.class) - .register(JacksonFeature.class) + .register(JacksonJsonProvider.class) .build(); target = httpClient.target("http://127.0.0.1:" + getRestPort() + "/fscrawler"); diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestCrawlerControlIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestCrawlerControlIT.java index f664ee839..af68cf6c4 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestCrawlerControlIT.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestCrawlerControlIT.java @@ -51,10 +51,10 @@ import org.apache.logging.log4j.Logger; import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; -import org.glassfish.jersey.jackson.JacksonFeature; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import tools.jackson.jakarta.rs.json.JacksonJsonProvider; /** * Integration tests for the REST API crawler control endpoints (pause, resume, checkpoint). These tests use a real @@ -79,7 +79,7 @@ void startRestServer() throws Exception { // Create HTTP client httpClient = ClientBuilder.newBuilder() .register(RestJsonProvider.class) - .register(JacksonFeature.class) + .register(JacksonJsonProvider.class) .build(); target = httpClient.target("http://127.0.0.1:" + restPort + "/fscrawler"); } diff --git a/pom.xml b/pom.xml index c9b2a326a..aa5b142c0 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,13 @@ 2.6 3.3.1 - 2.22.0 + 3.1.4 + + 2.22.0 3.0.0 2.26.1 4.0.2 @@ -273,12 +279,20 @@ - com.fasterxml.jackson + tools.jackson jackson-bom ${jackson.version} pom import + + + com.fasterxml.jackson + jackson-bom + ${jackson2.version} + pom + import + - - com.fasterxml.jackson.module - jackson-module-jaxb-annotations - - fr.pilato.elasticsearch.crawler diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestJsonProvider.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestJsonProvider.java index 1dd5b3f58..6731a79ec 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestJsonProvider.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestJsonProvider.java @@ -20,21 +20,26 @@ */ package fr.pilato.elasticsearch.crawler.fs.rest; -import com.fasterxml.jackson.databind.ObjectMapper; import fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil; import jakarta.ws.rs.ext.ContextResolver; import jakarta.ws.rs.ext.Provider; import org.apache.logging.log4j.LogManager; +import tools.jackson.databind.json.JsonMapper; +/** + * Supplies the FSCrawler {@link JsonMapper} to the Jackson 3 Jakarta-RS provider. The provider looks up a + * {@code ContextResolver} (not {@code ObjectMapper}), so this must be typed on {@link JsonMapper} to be + * picked up; otherwise the provider falls back to a default mapper and our custom (de)serializers are ignored. + */ @Provider -public class RestJsonProvider implements ContextResolver { +public class RestJsonProvider implements ContextResolver { // We initialize the object mapper depending on debug mode - private static final ObjectMapper jsonMapper = + private static final JsonMapper jsonMapper = LogManager.getLogger(RestJsonProvider.class).isDebugEnabled() ? JsonUtil.prettyMapper : JsonUtil.mapper; @Override - public ObjectMapper getContext(Class type) { + public JsonMapper getContext(Class type) { return jsonMapper; } } diff --git a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java index 015218dd8..2e0faad0d 100644 --- a/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java +++ b/rest/src/main/java/fr/pilato/elasticsearch/crawler/fs/rest/RestServer.java @@ -30,9 +30,9 @@ import org.apache.logging.log4j.Logger; import org.glassfish.grizzly.http.server.HttpServer; import org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpServerFactory; -import org.glassfish.jersey.jackson.JacksonFeature; import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.glassfish.jersey.server.ResourceConfig; +import tools.jackson.jakarta.rs.json.JacksonJsonProvider; public class RestServer implements AutoCloseable { private static final Logger logger = LogManager.getLogger(); @@ -78,7 +78,7 @@ public void start() { new CrawlerApi(fsParser, settings.getName())) .register(MultiPartFeature.class) .register(RestJsonProvider.class) - .register(JacksonFeature.class) + .register(JacksonJsonProvider.class) .register(new CORSFilter(settings.getRest())) .packages("fr.pilato.elasticsearch.crawler.fs.rest"); diff --git a/settings/pom.xml b/settings/pom.xml index b64fa5b16..77168f5bb 100644 --- a/settings/pom.xml +++ b/settings/pom.xml @@ -23,11 +23,11 @@ com.github.gestalt-config - gestalt-yaml + gestalt-yaml-jackson3 com.github.gestalt-config - gestalt-json + gestalt-json-jackson3 jakarta.annotation diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsParser.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsParser.java index 4139dc9bc..4c53629a0 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsParser.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsParser.java @@ -20,7 +20,6 @@ */ package fr.pilato.elasticsearch.crawler.fs.settings; -import com.fasterxml.jackson.core.JsonProcessingException; import fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil; public class FsSettingsParser { @@ -29,7 +28,7 @@ private FsSettingsParser() { // Utility class, do not instantiate } - public static String toYaml(FsSettings settings) throws JsonProcessingException { + public static String toYaml(FsSettings settings) { return JsonUtil.ymlMapper.writeValueAsString(settings); } } diff --git a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParser.java b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParser.java index a73755797..bc0b5a67a 100644 --- a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParser.java +++ b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParser.java @@ -20,15 +20,14 @@ */ package fr.pilato.elasticsearch.crawler.fs.tika; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; import fr.pilato.elasticsearch.crawler.fs.framework.JsonUtil; -import java.io.IOException; import java.io.InputStream; import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.dataformat.xml.XmlMapper; /** Parse a XML document and generate a FSCrawler Doc */ @SuppressWarnings("unchecked") @@ -42,10 +41,10 @@ private XmlDocParser() { private static final ObjectMapper xmlMapper; static { - xmlMapper = XmlMapper.xmlBuilder().nameForTextElement("$").build(); + xmlMapper = XmlMapper.builder().nameForTextElement("$").build(); } - public static String generate(InputStream inputStream) throws JsonProcessingException { + public static String generate(InputStream inputStream) { logger.trace("Converting XML document"); // Extracting XML content // See #185: https://github.com/dadoonet/fscrawler/issues/185 @@ -76,7 +75,7 @@ public static Map generateMap(InputStream inputStream) { private static Map asMap(InputStream stream) { try { return (Map) xmlMapper.readValue(stream, Object.class); - } catch (IOException e) { + } catch (JacksonException e) { throw new RuntimeException(e); } } diff --git a/tika/src/test/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParserTest.java b/tika/src/test/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParserTest.java index 394b23da8..e3ae3c77c 100644 --- a/tika/src/test/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParserTest.java +++ b/tika/src/test/java/fr/pilato/elasticsearch/crawler/fs/tika/XmlDocParserTest.java @@ -20,7 +20,6 @@ */ package fr.pilato.elasticsearch.crawler.fs.tika; -import com.fasterxml.jackson.core.JsonProcessingException; import java.io.InputStream; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -28,7 +27,7 @@ class XmlDocParserTest extends DocParserTestCase { @Test - void xml() throws JsonProcessingException { + void xml() { String doc = extractFromFile("issue-163.xml"); Assertions.assertThat(doc) .isEqualTo("{\"version\":\"1.0\",\"subscription-update\":{\"subscriptionid\":\"0\",\"requestid\":\"0\"," @@ -37,24 +36,24 @@ void xml() throws JsonProcessingException { } @Test - void xmlNestedObjects() throws JsonProcessingException { + void xmlNestedObjects() { String doc = extractFromFile("issue-592.xml"); Assertions.assertThat(doc) .isEqualTo("{\"object\":[{\"id\":\"1\",\"name\":\"foo\"},{\"id\":\"2\",\"name\":\"bar\"}]}"); } @Test - void xmlNotReadable() throws JsonProcessingException { + void xmlNotReadable() { String doc = extractFromFile(null, "issue-1753.xml"); Assertions.assertThat(doc).isEqualTo("{\"Tag\":{\"attr\":\"false\",\"$\":\"Content\"}}"); } - private String extractFromFile(String root, String filename) throws JsonProcessingException { + private String extractFromFile(String root, String filename) { InputStream data = getBinaryContent(root, filename); return XmlDocParser.generate(data); } - private String extractFromFile(String filename) throws JsonProcessingException { + private String extractFromFile(String filename) { InputStream data = getBinaryContent(filename); return XmlDocParser.generate(data); }