From 7a6e77725003dc5a6a82f681d771d082779a510f Mon Sep 17 00:00:00 2001 From: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:28:06 +0000 Subject: [PATCH] updated versioned plugin docs --- .../elastic_integration-index.asciidoc | 8 + .../elastic_integration-v8.19.9.asciidoc | 655 ++++++++++++++++ .../elastic_integration-v9.3.8.asciidoc | 655 ++++++++++++++++ .../elastic_integration-v9.4.6.asciidoc | 655 ++++++++++++++++ .../elastic_integration-v9.5.2.asciidoc | 655 ++++++++++++++++ .../filters/jdbc_static-index.asciidoc | 2 + .../filters/jdbc_static-v5.6.4.asciidoc | 629 +++++++++++++++ .../filters/jdbc_streaming-index.asciidoc | 2 + .../filters/jdbc_streaming-v5.6.4.asciidoc | 318 ++++++++ .../inputs/jdbc-index.asciidoc | 2 + .../inputs/jdbc-v5.6.4.asciidoc | 724 ++++++++++++++++++ .../integrations/jdbc-index.asciidoc | 2 + .../integrations/jdbc-v5.6.4.asciidoc | 32 + 13 files changed, 4339 insertions(+) create mode 100644 docs/versioned-plugins/filters/elastic_integration-v8.19.9.asciidoc create mode 100644 docs/versioned-plugins/filters/elastic_integration-v9.3.8.asciidoc create mode 100644 docs/versioned-plugins/filters/elastic_integration-v9.4.6.asciidoc create mode 100644 docs/versioned-plugins/filters/elastic_integration-v9.5.2.asciidoc create mode 100644 docs/versioned-plugins/filters/jdbc_static-v5.6.4.asciidoc create mode 100644 docs/versioned-plugins/filters/jdbc_streaming-v5.6.4.asciidoc create mode 100644 docs/versioned-plugins/inputs/jdbc-v5.6.4.asciidoc create mode 100644 docs/versioned-plugins/integrations/jdbc-v5.6.4.asciidoc diff --git a/docs/versioned-plugins/filters/elastic_integration-index.asciidoc b/docs/versioned-plugins/filters/elastic_integration-index.asciidoc index a6e09008..8aca71c8 100644 --- a/docs/versioned-plugins/filters/elastic_integration-index.asciidoc +++ b/docs/versioned-plugins/filters/elastic_integration-index.asciidoc @@ -5,12 +5,15 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2026-07-21 | <> | 2026-07-13 | <> | 2026-07-09 +| <> | 2026-07-21 | <> | 2026-07-14 | <> | 2026-06-30 | <> | 2026-05-15 | <> | 2026-05-11 +| <> | 2026-07-21 | <> | 2026-07-14 | <> | 2026-06-30 | <> | 2026-05-15 @@ -22,6 +25,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2025-07-16 | <> | 2025-06-28 | <> | 2025-04-28 +| <> | 2026-07-21 | <> | 2026-07-14 | <> | 2026-06-30 | <> | 2026-05-15 @@ -58,12 +62,15 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2023-04-14 |======================================================================= +include::elastic_integration-v9.5.2.asciidoc[] include::elastic_integration-v9.5.1.asciidoc[] include::elastic_integration-v9.5.0.asciidoc[] +include::elastic_integration-v9.4.6.asciidoc[] include::elastic_integration-v9.4.5.asciidoc[] include::elastic_integration-v9.4.4.asciidoc[] include::elastic_integration-v9.4.3.asciidoc[] include::elastic_integration-v9.4.2.asciidoc[] +include::elastic_integration-v9.3.8.asciidoc[] include::elastic_integration-v9.3.7.asciidoc[] include::elastic_integration-v9.3.6.asciidoc[] include::elastic_integration-v9.3.5.asciidoc[] @@ -75,6 +82,7 @@ include::elastic_integration-v9.1.0.asciidoc[] include::elastic_integration-v9.0.2.asciidoc[] include::elastic_integration-v9.0.1.asciidoc[] include::elastic_integration-v9.0.0.asciidoc[] +include::elastic_integration-v8.19.9.asciidoc[] include::elastic_integration-v8.19.8.asciidoc[] include::elastic_integration-v8.19.7.asciidoc[] include::elastic_integration-v8.19.6.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.19.9.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.19.9.asciidoc new file mode 100644 index 00000000..f249e3c5 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.19.9.asciidoc @@ -0,0 +1,655 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.19.9 +:release_date: 2026-07-21 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.19.9/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +- unsupported processor + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +- version compatibility + +Since {plugin} plugin embeds {es} ingest node components, there are situations where {es} introduces breaking changes that affect older plugin versions. +A notable example is {es} 9.2, which added new fields (`created_date_millis`, `modified_date_millis`) to the ingest pipeline model. +When a plugin version built for {es} <9.2 (e.g. v8.19.1 or v9.1.0) connects to {es} 9.2 or later, pipelines fetched from {es} may include these new parameters that the embedded ingest components do not recognize, causing pipeline creation to fail. + + [source] + ---- + [2026-02-25T07:56:20,091][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main][es_integ_filter] failed to create ingest pipeline `logs-panw.panos-5.4.1` from pipeline configuration + org.elasticsearch.ElasticsearchParseException: pipeline [logs-panw.panos-5.4.1] doesn't support one or more provided configuration parameters [created_date_millis, modified_date_millis] + at org.elasticsearch.ingest.Pipeline.create(Pipeline.java:111) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory.create(IngestPipelineFactory.java:49) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at co.elastic.logstash.filters.elasticintegration.SimpleIngestPipelineResolver.lambda$resolve$0(SimpleIngestPipelineResolver.java:60) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at java.util.Optional.flatMap(Optional.java:289) ~[?:?] + ---- + +*Resolution:* When connecting to {es} 9.2 or newer, update {plugin} plugin to at least v9.2 to ensure compatibility with the updated ingest pipeline model. +For more details, see https://github.com/elastic/logstash-filter-elastic_integration/issues/409[GitHub issue #409] and the related {es} change in https://github.com/elastic/elasticsearch/pull/130847[elastic/elasticsearch#130847]. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.3.8.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.3.8.asciidoc new file mode 100644 index 00000000..b427efd5 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.3.8.asciidoc @@ -0,0 +1,655 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.3.8 +:release_date: 2026-07-21 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.3.8/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +- unsupported processor + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +- version compatibility + +Since {plugin} plugin embeds {es} ingest node components, there are situations where {es} introduces breaking changes that affect older plugin versions. +A notable example is {es} 9.2, which added new fields (`created_date_millis`, `modified_date_millis`) to the ingest pipeline model. +When a plugin version built for {es} <9.2 (e.g. v8.19.1 or v9.1.0) connects to {es} 9.2 or later, pipelines fetched from {es} may include these new parameters that the embedded ingest components do not recognize, causing pipeline creation to fail. + + [source] + ---- + [2026-02-25T07:56:20,091][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main][es_integ_filter] failed to create ingest pipeline `logs-panw.panos-5.4.1` from pipeline configuration + org.elasticsearch.ElasticsearchParseException: pipeline [logs-panw.panos-5.4.1] doesn't support one or more provided configuration parameters [created_date_millis, modified_date_millis] + at org.elasticsearch.ingest.Pipeline.create(Pipeline.java:111) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory.create(IngestPipelineFactory.java:49) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at co.elastic.logstash.filters.elasticintegration.SimpleIngestPipelineResolver.lambda$resolve$0(SimpleIngestPipelineResolver.java:60) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at java.util.Optional.flatMap(Optional.java:289) ~[?:?] + ---- + +*Resolution:* When connecting to {es} 9.2 or newer, update {plugin} plugin to at least v9.2 to ensure compatibility with the updated ingest pipeline model. +For more details, see https://github.com/elastic/logstash-filter-elastic_integration/issues/409[GitHub issue #409] and the related {es} change in https://github.com/elastic/elasticsearch/pull/130847[elastic/elasticsearch#130847]. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.4.6.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.4.6.asciidoc new file mode 100644 index 00000000..75c1398b --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.4.6.asciidoc @@ -0,0 +1,655 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.4.6 +:release_date: 2026-07-21 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.4.6/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +- unsupported processor + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +- version compatibility + +Since {plugin} plugin embeds {es} ingest node components, there are situations where {es} introduces breaking changes that affect older plugin versions. +A notable example is {es} 9.2, which added new fields (`created_date_millis`, `modified_date_millis`) to the ingest pipeline model. +When a plugin version built for {es} <9.2 (e.g. v8.19.1 or v9.1.0) connects to {es} 9.2 or later, pipelines fetched from {es} may include these new parameters that the embedded ingest components do not recognize, causing pipeline creation to fail. + + [source] + ---- + [2026-02-25T07:56:20,091][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main][es_integ_filter] failed to create ingest pipeline `logs-panw.panos-5.4.1` from pipeline configuration + org.elasticsearch.ElasticsearchParseException: pipeline [logs-panw.panos-5.4.1] doesn't support one or more provided configuration parameters [created_date_millis, modified_date_millis] + at org.elasticsearch.ingest.Pipeline.create(Pipeline.java:111) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory.create(IngestPipelineFactory.java:49) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at co.elastic.logstash.filters.elasticintegration.SimpleIngestPipelineResolver.lambda$resolve$0(SimpleIngestPipelineResolver.java:60) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at java.util.Optional.flatMap(Optional.java:289) ~[?:?] + ---- + +*Resolution:* When connecting to {es} 9.2 or newer, update {plugin} plugin to at least v9.2 to ensure compatibility with the updated ingest pipeline model. +For more details, see https://github.com/elastic/logstash-filter-elastic_integration/issues/409[GitHub issue #409] and the related {es} change in https://github.com/elastic/elasticsearch/pull/130847[elastic/elasticsearch#130847]. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.5.2.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.5.2.asciidoc new file mode 100644 index 00000000..0f663478 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.5.2.asciidoc @@ -0,0 +1,655 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.5.2 +:release_date: 2026-07-21 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.5.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +- unsupported processor + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +- version compatibility + +Since {plugin} plugin embeds {es} ingest node components, there are situations where {es} introduces breaking changes that affect older plugin versions. +A notable example is {es} 9.2, which added new fields (`created_date_millis`, `modified_date_millis`) to the ingest pipeline model. +When a plugin version built for {es} <9.2 (e.g. v8.19.1 or v9.1.0) connects to {es} 9.2 or later, pipelines fetched from {es} may include these new parameters that the embedded ingest components do not recognize, causing pipeline creation to fail. + + [source] + ---- + [2026-02-25T07:56:20,091][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main][es_integ_filter] failed to create ingest pipeline `logs-panw.panos-5.4.1` from pipeline configuration + org.elasticsearch.ElasticsearchParseException: pipeline [logs-panw.panos-5.4.1] doesn't support one or more provided configuration parameters [created_date_millis, modified_date_millis] + at org.elasticsearch.ingest.Pipeline.create(Pipeline.java:111) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory.create(IngestPipelineFactory.java:49) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at co.elastic.logstash.filters.elasticintegration.SimpleIngestPipelineResolver.lambda$resolve$0(SimpleIngestPipelineResolver.java:60) ~[logstash-filter-elastic_integration-8.19.1.jar:?] + at java.util.Optional.flatMap(Optional.java:289) ~[?:?] + ---- + +*Resolution:* When connecting to {es} 9.2 or newer, update {plugin} plugin to at least v9.2 to ensure compatibility with the updated ingest pipeline model. +For more details, see https://github.com/elastic/logstash-filter-elastic_integration/issues/409[GitHub issue #409] and the related {es} change in https://github.com/elastic/elasticsearch/pull/130847[elastic/elasticsearch#130847]. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-index.asciidoc b/docs/versioned-plugins/filters/jdbc_static-index.asciidoc index 8a55b0ee..41f0d246 100644 --- a/docs/versioned-plugins/filters/jdbc_static-index.asciidoc +++ b/docs/versioned-plugins/filters/jdbc_static-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2026-07-21 | <> | 2026-02-11 | <> | 2026-01-31 | <> | 2025-09-30 @@ -62,6 +63,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2018-01-11 |======================================================================= +include::jdbc_static-v5.6.4.asciidoc[] include::jdbc_static-v5.6.3.asciidoc[] include::jdbc_static-v5.6.2.asciidoc[] include::jdbc_static-v5.6.1.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_static-v5.6.4.asciidoc b/docs/versioned-plugins/filters/jdbc_static-v5.6.4.asciidoc new file mode 100644 index 00000000..186c6f9a --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_static-v5.6.4.asciidoc @@ -0,0 +1,629 @@ +:integration: jdbc +:plugin: jdbc_static +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.4 +:release_date: 2026-07-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.6.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_static filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter enriches events with data pre-loaded from a remote database. + +This filter is best suited for enriching events with reference data that is +static or does not change very often, such as environments, users, and products. + +This filter works by fetching data from a remote database, caching it in a +local, in-memory https://db.apache.org/derby/manuals/#docs_10.14[Apache Derby] +database, and using lookups to enrich events with data cached in the local +database. You can set up the filter to load the remote data once (for static +data), or you can schedule remote loading to run periodically (for data that +needs to be refreshed). + +To define the filter, you specify three main sections: local_db_objects, loaders, +and lookups. + +*local_db_objects*:: + +Define the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +Define as many of these objects as needed to build the local database +structure. + +*loaders*:: + +Query the external database to fetch the dataset that will be cached locally. +Define as many loaders as needed to fetch the remote data. Each +loader should fill a table defined by `local_db_objects`. Make sure +the column names and datatypes in the loader SQL statement match the +columns defined under `local_db_objects`. Each loader has an independent remote +database connection. + +*lookups*:: + +Perform lookup queries on the local database to enrich the events. +Define as many lookups as needed to enrich the event from all +lookup tables in one pass. Ideally the SQL statement should only +return one row. Any rows are converted to Hash objects and are +stored in a target field that is an Array. ++ +The following example config fetches data from a remote database, caches it in a +local database, and uses lookups to enrich events with data cached in the local +database. ++ +["source","json",subs="callouts"] +----- +filter { + jdbc_static { + loaders => [ <1> + { + id => "remote-servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + }, + { + id => "remote-users" + query => "select firstname, lastname, userid from ref.local_users order by userid" + local_table => "users" + } + ] + local_db_objects => [ <2> + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + }, + { + name => "users" + index_columns => ["userid"] + columns => [ + ["firstname", "varchar(255)"], + ["lastname", "varchar(255)"], + ["userid", "int"] + ] + } + ] + local_lookups => [ <3> + { + id => "local-servers" + query => "SELECT descr as description FROM servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + }, + { + id => "local-users" + query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?" + prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4> + target => "user" <5> + default_hash => { <6> + firstname => nil + lastname => nil + } + } + ] + # using add_field here to add & rename values to the event root + add_field => { server_name => "%{[server][0][description]}" } <7> + add_field => { user_firstname => "%{[user][0][firstname]}" } + add_field => { user_lastname => "%{[user][0][lastname]}" } + remove_field => ["server", "user"] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "* */2 * * *" <8> + jdbc_user => "logstash" + jdbc_password => "example" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/tmp/logstash/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2" + } +} + +output { + if "_jdbcstaticdefaultsused" in [tags] { + # Print all the not found users + stdout { } + } +} +----- +<1> Queries an external database to fetch the dataset that will be cached +locally. +<2> Defines the columns, types, and indexes used to build the local database +structure. The column names and types should match the external database. +The order of table definitions is significant and should match that of the loader queries. +See <<{version}-plugins-{type}s-{plugin}-object_order>>. +<3> Performs lookup queries on the local database to enrich the events. +<4> Local lookup queries can also use prepared statements where the parameters +follow the positional ordering. +<5> Specifies the event field that will store the looked-up data. If the lookup +returns multiple columns, the data is stored as a JSON object within the field. +<6> When the user is not found in the database, an event is created using data from the <<{version}-plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>>. +<7> Takes data from the JSON object and stores it in top-level event fields for +easier analysis in Kibana. +<8> Runs loaders every 2 hours. + +Here's a full example: + +[source,json] +----- +input { + generator { + lines => [ + '{"from_ip": "10.2.3.20", "app": "foobar", "amount": 32.95}', + '{"from_ip": "10.2.3.30", "app": "barfoo", "amount": 82.95}', + '{"from_ip": "10.2.3.40", "app": "bazfoo", "amount": 22.95}' + ] + count => 200 + } +} + +filter { + json { + source => "message" + } + + jdbc_static { + loaders => [ + { + id => "servers" + query => "select ip, descr from ref.local_ips order by ip" + local_table => "servers" + } + ] + local_db_objects => [ + { + name => "servers" + index_columns => ["ip"] + columns => [ + ["ip", "varchar(15)"], + ["descr", "varchar(255)"] + ] + } + ] + local_lookups => [ + { + query => "select descr as description from servers WHERE ip = :ip" + parameters => {ip => "[from_ip]"} + target => "server" + } + ] + staging_directory => "/tmp/logstash/jdbc_static/import_data" + loader_schedule => "*/30 * * * *" + jdbc_user => "logstash" + jdbc_password => "logstash??" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/Users/guy/tmp/logstash-6.0.0/vendor/postgresql-42.1.4.jar" + jdbc_connection_string => "jdbc:postgresql://localhost:5432/ls_test_2" + } +} + +output { + stdout { + codec => rubydebug {metadata => true} + } +} +----- + +Assuming the loader fetches the following data from a Postgres database: + +[source,shell] +select * from ref.local_ips order by ip; + ip | descr +-----------+----------------------- + 10.2.3.10 | Authentication Server + 10.2.3.20 | Payments Server + 10.2.3.30 | Events Server + 10.2.3.40 | Payroll Server + 10.2.3.50 | Uploads Server + + +The events are enriched with a description of the server based on the value of +the IP: + +[source,shell] +{ + "app" => "bazfoo", + "sequence" => 0, + "server" => [ + [0] { + "description" => "Payroll Server" + } + ], + "amount" => 22.95, + "@timestamp" => 2017-11-30T18:08:15.694Z, + "@version" => "1", + "host" => "Elastics-MacBook-Pro.local", + "message" => "{\"from_ip\": \"10.2.3.40\", \"app\": \"bazfoo\", \"amount\": 22.95}", + "from_ip" => "10.2.3.40" +} + + +==== Using this plugin with multiple pipelines + +[IMPORTANT] +=============================== +Logstash uses a single, in-memory Apache Derby instance as the lookup database +engine for the entire JVM. Because each plugin instance uses a unique database +inside the shared Derby engine, there should be no conflicts with plugins +attempting to create and populate the same tables. This is true regardless of +whether the plugins are defined in a single pipeline, or multiple pipelines. +However, after setting up the filter, you should watch the lookup results and +view the logs to verify correct operation. +=============================== + +[id="{version}-plugins-{type}s-{plugin}-object_order"] +==== Loader column and local_db_object order dependency + +[IMPORTANT] +=============================== +For loader performance reasons, the loading mechanism uses a CSV style file with an inbuilt +Derby file import procedure to add the remote data to the local db. The retrieved columns +are written to the CSV file as is and the file import procedure expects a 1 to 1 correspondence +to the order of the columns specified in the local_db_object settings. Please ensure that this +order is in place. +=============================== + + +[id="{version}-plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin is compatible with the {ecs-ref}[Elastic Common Schema (ECS)]. +It behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_static filter configuration options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-staging_directory>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loader_schedule>>|{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-loaders>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_db_objects>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-local_lookups>>|{logstash-ref}/configuration-file-structure.html#array[array]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver". + +NOTE: According to https://github.com/logstash-plugins/logstash-input-jdbc/issues/43[Issue 43], +if you are using the Oracle JDBC driver (ojdbc6.jar), the correct +`jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third-party driver library. Use comma separated paths +in one string if you need multiple libraries. + +If the driver class is not provided, the plugin looks for it in the Logstash +Java classpath. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstaticfailure"]` + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-staging_directory"] +===== `staging_directory` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is derived from the Ruby temp directory + plugin_name + "import_data" + * e.g. `"/tmp/logstash/jdbc_static/import_data"` + +The directory used stage the data for bulk loading, there should be sufficient +disk space to handle the data you wish to use to enrich events. +Previous versions of this plugin did not handle loading datasets of more than +several thousand rows well due to an open bug in Apache Derby. This setting +introduces an alternative way of loading large recordsets. As each row is +received it is spooled to file and then that file is imported using a +system 'import table' system call. + +Append values to the `tags` field if a SQL error occurred. + +[id="{version}-plugins-{type}s-{plugin}-loader_schedule"] +===== `loader_schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +You can schedule remote loading to run periodically according to a +specific schedule. This scheduling syntax is powered by +https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. The +syntax is cron-like with some extensions specific to Rufus +(for example, timezone support). For more about this syntax, see +https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[parsing cronlines and time strings]. + +Examples: + +|========================================================== +| `*/30 * * * *` | will execute on the 0th and 30th minute of every hour every day. +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Debugging using the Logstash interactive shell: +[source,shell] +bin/logstash -i irb +irb(main):001:0> require 'rufus-scheduler' +=> true +irb(main):002:0> Rufus::Scheduler.parse('*/10 * * * *') +=> # +irb(main):003:0> exit + + +The object returned by the above call, an instance of `Rufus::Scheduler::CronLine` shows the seconds, minutes etc. of execution. + +[id="{version}-plugins-{type}s-{plugin}-loaders"] +===== `loaders` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| local_table|string|Yes +| query|string|Yes +| max_rows|number|No +| jdbc_connection_string|string|No +| jdbc_driver_class|string|No +| jdbc_driver_library|a valid filesystem path|No +| jdbc_password|password|No +| jdbc_user|string|No +|======================================================================= + +*Loader Field Descriptions:* + +id:: +An optional identifier. This is used to identify the loader that is +generating error messages and log lines. + +local_table:: +The destination table in the local lookup database that the loader will fill. + +query:: +The SQL statement that is executed to fetch the remote records. Use SQL +aliases and casts to ensure that the record's columns and datatype match the +table structure in the local database as defined in the `local_db_objects`. + +max_rows:: +The default for this setting is 1 million. Because the lookup database is +in-memory, it will take up JVM heap space. If the query returns many millions +of rows, you should increase the JVM memory given to Logstash or limit the +number of rows returned, perhaps to those most frequently found in the +event data. + +jdbc_connection_string:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_connection_string` setting. + +jdbc_driver_class:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_class` setting. + +jdbc_driver_library:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_driver_library` setting. + +jdbc_password:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_password` setting. + +jdbc_user:: +If not set in a loader, this setting defaults to the plugin-level +`jdbc_user` setting. + +[id="{version}-plugins-{type}s-{plugin}-local_db_objects"] +===== `local_db_objects` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a table +schema for the local lookups database. Each Hash is validated +according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| name|string|Yes +| columns|array|Yes +| index_columns|number|No +| preserve_existing|boolean|No +|======================================================================= + +*Local_db_objects Field Descriptions:* + +name:: +The name of the table to be created in the database. + +columns:: +An array of column specifications. Each column specification is an array +of exactly two elements, for example `["ip", "varchar(15)"]`. The first +element is the column name string. The second element is a string that +is an +https://db.apache.org/derby/docs/10.14/ref/crefsqlj31068.html[Apache Derby SQL type]. +The string content is checked when the local lookup tables are built, not when +the settings are validated. Therefore, any misspelled SQL type strings result in +errors. + +index_columns:: +An array of strings. Each string must be defined in the `columns` setting. The +index name will be generated internally. Unique or sorted indexes are not +supported. + +preserve_existing:: +This setting, when `true`, checks whether the table already exists in the local +lookup database. If you have multiple pipelines running in the same +instance of Logstash, and more than one pipeline is using this plugin, then you +must read the important multiple pipeline notice at the top of the page. + +[id="{version}-plugins-{type}s-{plugin}-local_lookups"] +===== `local_lookups` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +The array should contain one or more Hashes. Each Hash represents a lookup +enrichment. Each Hash is validated according to the table below. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| id|string|No +| query|string|Yes +| parameters|hash|Yes +| target|string|No +| default_hash|hash|No +| tag_on_failure|string|No +| tag_on_default_use|string|No +|======================================================================= + +*Local_lookups Field Descriptions:* + +id:: +An optional identifier. This is used to identify the lookup that is +generating error messages and log lines. If you omit this setting then a +default id is used instead. + +query:: +A SQL SELECT statement that is executed to achieve the lookup. To use +parameters, use named parameter syntax, for example +`"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign +can be used as a prepared statement parameter, in which case +the `prepared_parameters` array is used to populate the values + +parameters:: +A key/value Hash or dictionary. The key (LHS) is the text that is +substituted for in the SQL statement +`SELECT * FROM sensors WHERE reference = :p1`. The value (RHS) +is the field name in your event. The plugin reads the value from +this key out of the event and substitutes that value into the +statement, for example, `parameters => { "p1" => "ref" }`. Quoting is +automatic - you do not need to put quotes in the statement. +Only use the field interpolation syntax on the RHS if you need to +add a prefix/suffix or join two event field values together to build +the substitution value. For example, imagine an IOT message that has +an id and a location, and you have a table of sensors that have a +column of `id-loc_id`. In this case your parameter hash would look +like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`. + +prepared_parameters:: +An Array, where the position is related to the position of the `?` in +the query syntax. The values of array follow the same semantic of `parameters`. +If `prepared_parameters` is valorized the filter is forced to use JDBC's +prepared statement to query the local database. +Prepared statements provides two benefits: one on the performance side, because +avoid the DBMS to parse and compile the SQL expression for every call; +the other benefit is on the security side, using prepared statements +avoid SQL-injection attacks based on query string concatenation. + +target:: +An optional name for the field that will receive the looked-up data. +If you omit this setting then the `id` setting (or the default id) is +used. The looked-up data, an array of results converted to Hashes, is +never added to the root of the event. If you want to do this, you +should use the `add_field` setting. This means that +you are in full control of how the fields/values are put in the root +of the event, for example, +`add_field => { user_firstname => "%{[user][0][firstname]}" }` - +where `[user]` is the target field, `[0]` is the first result in the +array, and `[firstname]` is the key in the result hash. + +default_hash:: +An optional hash that will be put in the target field array when the +lookup returns no results. Use this setting if you need to ensure that later +references in other parts of the config actually refer to something. + +tag_on_failure:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +tag_on_default_use:: +An optional string that overrides the plugin-level setting. This is +useful when defining multiple lookups. + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-index.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-index.asciidoc index ee6c7a5a..a1f8775e 100644 --- a/docs/versioned-plugins/filters/jdbc_streaming-index.asciidoc +++ b/docs/versioned-plugins/filters/jdbc_streaming-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2026-07-21 | <> | 2026-02-11 | <> | 2026-01-31 | <> | 2025-09-30 @@ -62,6 +63,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::jdbc_streaming-v5.6.4.asciidoc[] include::jdbc_streaming-v5.6.3.asciidoc[] include::jdbc_streaming-v5.6.2.asciidoc[] include::jdbc_streaming-v5.6.1.asciidoc[] diff --git a/docs/versioned-plugins/filters/jdbc_streaming-v5.6.4.asciidoc b/docs/versioned-plugins/filters/jdbc_streaming-v5.6.4.asciidoc new file mode 100644 index 00000000..04b8c8c6 --- /dev/null +++ b/docs/versioned-plugins/filters/jdbc_streaming-v5.6.4.asciidoc @@ -0,0 +1,318 @@ +:integration: jdbc +:plugin: jdbc_streaming +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.4 +:release_date: 2026-07-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.6.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc_streaming filter plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This filter executes a SQL query and stores the result set in the field +specified as `target`. +It will cache the results locally in an LRU cache with expiry. + +For example, you can load a row based on an id in the event. + +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = :code" + parameters => { "code" => "country_code"} + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-prepared_statements"] +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. + +Use the boolean setting `use_prepared_statements` to enable this execution mode. + +Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. + +Use the `prepared_statement_bind_values` array setting to specify the bind values. Typically, these values are indirectly extracted from your event, i.e. the string in the array refers to a field name in your event. You can also use constant values like numbers or strings but ensure that any string constants (e.g. a locale constant of "en" or "de") is not also an event field name. It is a good idea to use the bracketed field reference syntax for fields and normal strings for constants, e.g. `prepared_statement_bind_values => ["[src_ip]", "tokyo"],`. + +There are 3 possible parameter schemes. Interpolated, field references and constants. Use interpolation when you are prefixing, suffixing or concatenating field values to create a value that exists in your database, e.g. "%{username}@%{domain}" -> "alice@example.org", "%{distance}km" -> "42km". Use field references for exact field values e.g. "[srcip]" -> "192.168.1.2". Use constants when a database column holds values that slice or categorise a number of similar records e.g. language translations. + +A boolean setting `prepared_statement_warn_on_constant_usage`, defaulting to true, controls whether you will see a WARN message logged that warns when constants could be missing the bracketed field reference syntax. If you have set your field references and constants correctly you should set `prepared_statement_warn_on_constant_usage` to false. This setting and code checks should be deprecated in a future major Logstash release. + +The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. +Some technologies may require connection string properties to be set, see MySQL example below. + +Example: +[source,ruby] +----- +filter { + jdbc_streaming { + jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydatabase?cachePrepStmts=true&prepStmtCacheSize=250&prepStmtCacheSqlLimit=2048&useServerPrepStmts=true" + jdbc_user => "me" + jdbc_password => "secret" + statement => "select * from WORLD.COUNTRY WHERE Code = ?" + use_prepared_statements => true + prepared_statement_name => "lookup_country_info" + prepared_statement_bind_values => ["[country_code]"] + target => "country_details" + } +} +----- + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc_streaming Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-cache_expiration>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-cache_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-default_hash>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-tag_on_default_use>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-target>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-use_cache>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-cache_expiration"] +===== `cache_expiration` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5.0` + +The minimum number of seconds any entry should remain in the cache. Defaults to 5 seconds. + +A numeric value. You can use decimals for example: `cache_expiration => 0.25`. +If there are transient jdbc errors, the cache will store empty results for a +given parameter set and bypass the jbdc lookup. This will merge the default_hash +into the event until the cache entry expires. Then the jdbc lookup will be tried +again for the same parameters. Conversely, while the cache contains valid results, +any external problem that would cause jdbc errors will not be noticed for the +cache_expiration period. + +[id="{version}-plugins-{type}s-{plugin}-cache_size"] +===== `cache_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `500` + +The maximum number of cache entries that will be stored. Defaults to 500 entries. +The least recently used entry will be evicted. + +[id="{version}-plugins-{type}s-{plugin}-default_hash"] +===== `default_hash` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Define a default object to use when lookup fails to return a matching row. +Ensure that the key names of this object match the columns from the statement. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver" + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds). + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "id" => "id_field" }`. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. Use field references and constants. See the section on <<{version}-plugins-{type}s-{plugin}-prepared_statements,prepared_statements>> for more info. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database. +You need to supply this if `use_prepared_statements` is true. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_warn_on_constant_usage"] +===== `prepared_statement_warn_on_constant_usage` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +A flag that controls whether a warning is logged if, in `prepared_statement_bind_values`, +a String constant is detected that might be intended as a field reference. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this documentation page: +https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Statement to execute. +To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id". + +[id="{version}-plugins-{type}s-{plugin}-tag_on_default_use"] +===== `tag_on_default_use` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingdefaultsused"]` + +Append values to the `tags` field if no record was found and default values were used. + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_jdbcstreamingfailure"]` + +Append values to the `tags` field if sql error occurred. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Define the target field to store the extracted result(s). +Field is overwritten if exists. + +[id="{version}-plugins-{type}s-{plugin}-use_cache"] +===== `use_cache` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Enable or disable caching, boolean true or false. Defaults to true. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/inputs/jdbc-index.asciidoc b/docs/versioned-plugins/inputs/jdbc-index.asciidoc index 993dab68..35b87fb7 100644 --- a/docs/versioned-plugins/inputs/jdbc-index.asciidoc +++ b/docs/versioned-plugins/inputs/jdbc-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2026-07-21 | <> | 2026-02-11 | <> | 2026-01-31 | <> | 2025-09-30 @@ -75,6 +76,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-06-23 |======================================================================= +include::jdbc-v5.6.4.asciidoc[] include::jdbc-v5.6.3.asciidoc[] include::jdbc-v5.6.2.asciidoc[] include::jdbc-v5.6.1.asciidoc[] diff --git a/docs/versioned-plugins/inputs/jdbc-v5.6.4.asciidoc b/docs/versioned-plugins/inputs/jdbc-v5.6.4.asciidoc new file mode 100644 index 00000000..6f6c30cb --- /dev/null +++ b/docs/versioned-plugins/inputs/jdbc-v5.6.4.asciidoc @@ -0,0 +1,724 @@ +:integration: jdbc +:plugin: jdbc +:type: input +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.4 +:release_date: 2026-07-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.6.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Jdbc input plugin {version} + +include::{include_path}/plugin_header-integration.asciidoc[] + +==== Description + +This plugin was created as a way to ingest data in any database +with a JDBC interface into Logstash. You can periodically schedule ingestion +using a cron syntax (see `schedule` setting) or run the query one time to load +data into Logstash. Each row in the resultset becomes a single event. +Columns in the resultset are converted into fields in the event. + +==== Drivers + +This plugin does not come packaged with JDBC driver libraries. The desired +jdbc driver library must be explicitly passed in to the plugin using the +`jdbc_driver_library` configuration option. + +See the <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> and <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> +options for more info. + +==== Scheduling + +Input from this plugin can be scheduled to run periodically according to a specific +schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. +The syntax is either cron-like with some extensions specific to Rufus (e.g. timezone support ) if using the `schedule` option or periodic when using `period` or `interval` option. + +Examples for `schedule`: + +|========================================================== +| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. +| `0 * * * *` | will execute on the 0th minute of every hour every day. +| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. +|========================================================== + +Examples for `period` or `interval`: + +|========================================================== +| `1m` | will execute every minute +| `3h10m` | will execute every three hours and 10 minutes +|========================================================== + +Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. + +`interval` jobs trigger, execute and then trigger again after the interval elapsed. + +`period` jobs try to trigger following the frequency they were scheduled with. + +You can only use one of `interval`, `period` or `schedule` at the same time. + +==== State + +The plugin will persist the `sql_last_value` parameter in the form of a +metadata file stored in the configured `last_run_metadata_path`. Upon query execution, +this file will be updated with the current value of `sql_last_value`. Next time +the pipeline starts up, this value will be updated by reading from the file. If +`clean_run` is set to true, this value will be ignored and `sql_last_value` will be +set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed. + +==== Dealing With Large Result-sets + +Many JDBC drivers use the `fetch_size` parameter to limit how many +results are pre-fetched at a time from the cursor into the client's cache +before retrieving more results from the result-set. This is configured in +this plugin using the `jdbc_fetch_size` configuration option. No fetch size +is set by default in this plugin, so the specific driver's default size will +be used. + +==== Usage: + +Here is an example of setting up the plugin to fetch data from a MySQL database. +First, we place the appropriate JDBC driver library in our current +path (this can be placed anywhere on your filesystem). In this example, we connect to +the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs' +table that match a specific artist. The following examples demonstrates a possible +Logstash configuration for this. The `schedule` option in this example will +instruct the plugin to execute this input statement on the minute, every minute. + +[source,ruby] +------------------------------------------------------------------------------ +input { + jdbc { + jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb" + jdbc_user => "mysql" + parameters => { "favorite_artist" => "Beethoven" } + schedule => "* * * * *" + statement => "SELECT * from songs where artist = :favorite_artist" + } +} +------------------------------------------------------------------------------ + +==== Configuring SQL statement + +A sql statement is required for this input. This can be passed-in via a +statement option in the form of a string, or read from a file (`statement_filepath`). File +option is typically used when the SQL statement is large or cumbersome to supply in the config. +The file option only supports one SQL statement. The plugin will only accept one of the options. +It cannot read a statement from a file as well as from the `statement` configuration parameter. + +==== Configuring multiple SQL statements + +Configuring multiple SQL statements is useful when there is a need to query and ingest data +from different database tables or views. It is possible to define separate Logstash +configuration files for each statement or to define multiple statements in a single configuration +file. When using multiple statements in a single Logstash configuration file, each statement +has to be defined as a separate jdbc input (including jdbc driver, connection string and other +required parameters). + +Please note that if any of the statements use the `sql_last_value` parameter (e.g. for +ingesting only data changed since last run), each input should define its own +`last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as +all inputs will store their state to the same (default) metadata file, effectively +overwriting each other's `sql_last_value`. + +==== Predefined Parameters + +Some parameters are built-in and can be used from within your queries. +Here is the list: + +|========================================================== +|sql_last_value | The value used to calculate which rows to query. Before any query is run, +this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +`tracking_column` is set. It is updated accordingly after subsequent queries are run. +|offset, size| Values used with manual paging mode to explicitly implement the paging. +Supported only if <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and +<<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value. +|========================================================== + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value" + use_column_value => true + tracking_column => "id" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Prepared Statements + +Using server side prepared statements can speed up execution times as the server optimises the query plan and execution. + +NOTE: Not all JDBC accessible technologies will support prepared statements. + +With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on. +Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array. + +NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements. + +Example: +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC" + prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4] + prepared_statement_name => "foobar" + use_prepared_statements => true + use_column_value => true + tracking_column_type => "numeric" + tracking_column => "_sequence_key" + last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- + +==== Database-specific considerations + +The JDBC input plugin leverages the https://github.com/jeremyevans/sequel[sequel] library to query databases through their JDBC drivers. +The implementation of drivers will vary, however, potentially leading to unexpected behavior. + +===== Unable to reuse connections + +Some databases - such as Sybase or SQL Anywhere - may have issues with stale connections, timing out between scheduled runs and never reconnecting. + +To ensure connections are valid before queries are executed, enable <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> and set <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> to a shorter interval than the <<{version}-plugins-{type}s-{plugin}-schedule>>. + +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + schedule => "* * * * *" # run every minute + jdbc_validate_connection => true + jdbc_validation_timeout => 50 # 50 seconds + } +} +--------------------------------------------------------------------------------------------------- + + + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Jdbc Input Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-clean_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-columns_charset>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-interval>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_connection_string>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_default_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_class>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_driver_library>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_fetch_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_page_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_paging_mode>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["auto", "explicit"]`|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_user>> |{logstash-ref}/configuration-file-structure.html#string[string]|Yes +| <<{version}-plugins-{type}s-{plugin}-jdbc_validate_connection>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-lowercase_column_names>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-parameters>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-period>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-plugin_timezone>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["local", "utc"]`|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-prepared_statement_name>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-record_last_run>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-schedule>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-sequel_opts>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-sql_log_level>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["fatal", "error", "warn", "info", "debug"]`|No +| <<{version}-plugins-{type}s-{plugin}-statement>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No +| <<{version}-plugins-{type}s-{plugin}-tracking_column>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-tracking_column_type>> |{logstash-ref}/configuration-file-structure.html#string[string], one of `["numeric", "timestamp"]`|No +| <<{version}-plugins-{type}s-{plugin}-use_column_value>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-use_prepared_statements>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +input plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-clean_run"] +===== `clean_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Whether the previous run state should be preserved + +[id="{version}-plugins-{type}s-{plugin}-columns_charset"] +===== `columns_charset` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +The character encoding for specific columns. This option will override the `:charset` option +for the specified columns. + +Example: +[source,ruby] +------------------------------------------------------- +input { + jdbc { + ... + columns_charset => { "column0" => "ISO-8859-1" } + ... + } +} +------------------------------------------------------- +this will only convert column0 that has ISO-8859-1 as an original encoding. + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts"] +===== `connection_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try connecting to database + +[id="{version}-plugins-{type}s-{plugin}-connection_retry_attempts_wait_time"] +===== `connection_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between connection attempts + +[id="{version}-plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This takes a string in the form of `1h`, `1m`, to denote a time interval. `interval` jobs trigger, execute and trigger again after the provided time interval has elapsed. + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_connection_string"] +===== `jdbc_connection_string` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC connection string + +[id="{version}-plugins-{type}s-{plugin}-jdbc_default_timezone"] +===== `jdbc_default_timezone` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3` + ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]` + * There is no default value for this setting. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_timezone_conv"] +===== Timezone conversion + +Logstash and Elasticsearch expect timestamps to be expressed in UTC terms. +If your database has recorded timestamps that are relative to another timezone, +the database timezone if you will, then set this setting to be the timezone that +the database is using. However, as SQL does not allow for timezone data in +timestamp fields we can't figure this out on a record by record basis. This plugin +will automatically convert your SQL timestamp fields to Logstash timestamps, +in relative UTC time in ISO8601 format. + +Using this setting will manually assign a specified timezone offset, instead +of using the timezone setting of the local machine. You must use a canonical +timezone, `America/Denver`, for example. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"] +===== Ambiguous timestamps + +While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline. +For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous. + +When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`). + +[id="{version}-plugins-{type}s-{plugin}-plugin_timezone"] +===== `plugin_timezone` + + * Value can be any of: `utc`, `local` + * Default value is `"utc"` + +If you want this plugin to offset timestamps to a timezone other than UTC, you +can set this setting to `local` and the plugin will use the OS timezone for offset +adjustments. + +Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset +adjustments are made in two places, if `sql_last_value` is a timestamp and it +is used as a parameter in the statement then offset adjustment is done from the +plugin timezone into the data timezone and while records are processed, timestamps +are offset adjusted from the database timezone to the plugin timezone. If your +database timezone is UTC then you do not need to set either of these settings. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_class"] +===== `jdbc_driver_class` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver" + +NOTE: Per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43, prepending `Java::` to the driver class + may be required if it appears that the driver is not being loaded correctly despite relevant jar(s) being provided by + either via the `jdbc_driver_library` setting or being placed in the Logstash Java classpath. This is known to be the + case for the Oracle JDBC driver (ojdbc6.jar), where the correct `jdbc_driver_class` is + `"Java::oracle.jdbc.driver.OracleDriver"`, and may also be the case for other JDBC drivers. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_driver_library"] +===== `jdbc_driver_library` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC driver library path to third party driver library. In case of multiple libraries being +required you can pass them separated by a comma. + +NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library + does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java + classpath rather than via this setting may help. + Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service). + +[id="{version}-plugins-{type}s-{plugin}-jdbc_fetch_size"] +===== `jdbc_fetch_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * There is no default value for this setting. + +JDBC fetch size. if not provided, respective driver's default will be used + +[id="{version}-plugins-{type}s-{plugin}-jdbc_page_size"] +===== `jdbc_page_size` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `100000` + +JDBC page size + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_enabled"] +===== `jdbc_paging_enabled` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +JDBC enable paging + +This will cause a sql statement to be broken up into multiple queries. +Each query will use limits and offsets to collectively retrieve the full +result-set. The limit size is set with `jdbc_page_size`. + +Be aware that ordering is not guaranteed between queries. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_paging_mode"] +===== `jdbc_paging_mode` + + * Value can be any of: `auto`, `explicit` + * Default value is `"auto"` + +Whether to use `explicit` or `auto` mode during the JDBC paging + +If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.). + +If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved. +You have to write your own paging conditions in your statement configuration. +The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query). +When the number of rows returned by the query is not equal to `size`, SQL paging will be ended. +Example: + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +[source, ruby] +------------------------------------------------------ +input { + jdbc { + statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)", + jdbc_paging_enabled => true, + jdbc_paging_mode => "explicit", + jdbc_page_size => 100000 + } +} +------------------------------------------------------ + +This mode can be considered in the following situations: + +. Performance issues encountered in default paging mode. +. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want. +. Your statement is a stored procedure, and the actual paging statement is inside it. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password"] +===== `jdbc_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +JDBC password + +[id="{version}-plugins-{type}s-{plugin}-jdbc_password_filepath"] +===== `jdbc_password_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +JDBC password filename + +[id="{version}-plugins-{type}s-{plugin}-jdbc_pool_timeout"] +===== `jdbc_pool_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `5` + +Connection pool configuration. +The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5) + +[id="{version}-plugins-{type}s-{plugin}-jdbc_user"] +===== `jdbc_user` + + * This is a required setting. + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +JDBC user + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validate_connection"] +===== `jdbc_validate_connection` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +Connection pool configuration. +Validate connection before use. + +[id="{version}-plugins-{type}s-{plugin}-jdbc_validation_timeout"] +===== `jdbc_validation_timeout` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `3600` + +Connection pool configuration. +How often to validate a connection (in seconds) + +[id="{version}-plugins-{type}s-{plugin}-last_run_metadata_path"] +===== `last_run_metadata_path` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"/plugins/inputs/jdbc/logstash_jdbc_last_run"` + +Path to file with last run time + +In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed. + +[id="{version}-plugins-{type}s-{plugin}-lowercase_column_names"] +===== `lowercase_column_names` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to force the lowercasing of identifier fields + +[id="{version}-plugins-{type}s-{plugin}-parameters"] +===== `parameters` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of query parameter, for example `{ "target_id" => "321" }` + +[id="{version}-plugins-{type}s-{plugin}-period"] +===== `period` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +This takes a string in the form of `1h`, `1m`, to denote a time interval. `period` jobs try hard to trigger following the frequency they were scheduled with. + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_bind_values"] +===== `prepared_statement_bind_values` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `[]` + +Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string + +[id="{version}-plugins-{type}s-{plugin}-prepared_statement_name"] +===== `prepared_statement_name` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Name given to the prepared statement. It must be unique in your config and in the database + +[id="{version}-plugins-{type}s-{plugin}-record_last_run"] +===== `record_last_run` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether to save state or not in <<{version}-plugins-{type}s-{plugin}-last_run_metadata_path>> + +[id="{version}-plugins-{type}s-{plugin}-schedule"] +===== `schedule` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Schedule of when to periodically run statement, in Cron format for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. + +[id="{version}-plugins-{type}s-{plugin}-sequel_opts"] +===== `sequel_opts` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +General/Vendor-specific Sequel configuration options. + +An example of an optional connection pool configuration + max_connections - The maximum number of connections the connection pool + +examples of vendor-specific options can be found in this +documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc + +[id="{version}-plugins-{type}s-{plugin}-sql_log_level"] +===== `sql_log_level` + + * Value can be any of: `fatal`, `error`, `warn`, `info`, `debug` + * Default value is `"info"` + +Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn, +info and debug. The default value is info. + +[id="{version}-plugins-{type}s-{plugin}-statement"] +===== `statement` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +If undefined, Logstash will complain, even if codec is unused. +Statement to execute + +To use parameters, use named parameter syntax. +For example: + +[source, ruby] +----------------------------------------------- +"SELECT * FROM MYTABLE WHERE id = :target_id" +----------------------------------------------- + +here, ":target_id" is a named parameter. You can configure named parameters +with the `parameters` setting. + +[id="{version}-plugins-{type}s-{plugin}-statement_filepath"] +===== `statement_filepath` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +Path of file containing statement to execute + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts"] +===== `statement_retry_attempts` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `1` + +Maximum number of times to try executing a statement. + +[id="{version}-plugins-{type}s-{plugin}-statement_retry_attempts_wait_time"] +===== `statement_retry_attempts_wait_time` + + * Value type is {logstash-ref}/configuration-file-structure.html#number[number] + * Default value is `0.5` + +Number of seconds to sleep between statement execution attempts. + +[id="{version}-plugins-{type}s-{plugin}-target"] +===== `target` + +* Value type is {logstash-ref}/field-references-deepdive.html[field reference] +* There is no default value for this setting. + +Without a `target`, events are created from each row column at the root level. +When the `target` is set to a field reference, the column of each row is placed in the target field instead. + +This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced. + +[id="{version}-plugins-{type}s-{plugin}-tracking_column"] +===== `tracking_column` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +The column whose value is to be tracked if `use_column_value` is set to `true` + +[id="{version}-plugins-{type}s-{plugin}-tracking_column_type"] +===== `tracking_column_type` + + * Value can be any of: `numeric`, `timestamp` + * Default value is `"numeric"` + +Type of tracking column. Currently only "numeric" and "timestamp" + +[id="{version}-plugins-{type}s-{plugin}-use_column_value"] +===== `use_column_value` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, uses the defined +<<{version}-plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set +to `false`, `:sql_last_value` reflects the last time the query was executed. + +[id="{version}-plugins-{type}s-{plugin}-use_prepared_statements"] +===== `use_prepared_statements` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +When set to `true`, enables prepare statement usage + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/docs/versioned-plugins/integrations/jdbc-index.asciidoc b/docs/versioned-plugins/integrations/jdbc-index.asciidoc index a70280f4..627edba8 100644 --- a/docs/versioned-plugins/integrations/jdbc-index.asciidoc +++ b/docs/versioned-plugins/integrations/jdbc-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2026-07-21 | <> | 2026-02-11 | <> | 2026-01-31 | <> | 2025-09-30 @@ -53,6 +54,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2020-01-09 |======================================================================= +include::jdbc-v5.6.4.asciidoc[] include::jdbc-v5.6.3.asciidoc[] include::jdbc-v5.6.2.asciidoc[] include::jdbc-v5.6.1.asciidoc[] diff --git a/docs/versioned-plugins/integrations/jdbc-v5.6.4.asciidoc b/docs/versioned-plugins/integrations/jdbc-v5.6.4.asciidoc new file mode 100644 index 00000000..6b86dc51 --- /dev/null +++ b/docs/versioned-plugins/integrations/jdbc-v5.6.4.asciidoc @@ -0,0 +1,32 @@ +:plugin: jdbc +:type: integration +:no_codec: + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v5.6.4 +:release_date: 2026-07-21 +:changelog_url: https://github.com/logstash-plugins/logstash-integration-jdbc/blob/v5.6.4/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== JDBC Integration Plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +The JDBC Integration Plugin provides integrated plugins for working with databases that provide JDBC drivers: + + - {logstash-ref}/plugins-inputs-jdbc.html[JDBC Input Plugin] + - {logstash-ref}/plugins-filters-jdbc_static.html[JDBC Static Filter Plugin] + - {logstash-ref}/plugins-filters-jdbc_streaming.html[JDBC Streaming Filter Plugin] + +:no_codec!: + +