Simplify codebase

Author: skjolber

README.md

@@ -22,11 +22,11 @@ Features:
 
  * Mask single values or whole subtrees
  * Remove single values or whole subtrees
- * Truncate String values
- * Truncate document size (max total output size)
- * Remove whitespace (for pretty-printed documents)
+ * Limit String value size
+ * Limit document size (skip end of document when max size is reached)
+ * Remove whitespace
 
-The library contains multiple filter implementations as to accommodate combinations of the above features with as little overhead as possible. 
+The library contains multiple filter implementations as to accommodate combinations of the above features with as little overhead as possible.
 
 The equivalent filters are also implemented using [Jackson]:
 
@@ -176,15 +176,15 @@ to output like
 ### Path syntax
 A simple syntax is supported, where each path segment corresponds to a `field name`. Expressions are case-sensitive. Supported syntax:
 
-    /my/field/name
+    $.my.field.name
 
 with support for wildcards; 
 
-    /my/field/*
+    $.my.field.*
 
 or a simple any-level field name search 
 
-    //myFieldName
+    ..myFieldName
 
 The filters within this library support using multiple expressions at once. Note that path expressions are see through arrays.
 
@@ -212,14 +212,14 @@ The resulting metrics could be logged as metadata alongside the JSON payload or
 ## Performance
 The `core` processors within this project are faster than the `Jackson`-based processors. This is expected as parser/serializer features have been traded for performance:
 
- * `core` is something like 3x-9x as fast as `Jackson` processors, where
+ * `core` processors are multiple times as fast as (streaming) `Jackson` processors, where
  * skipping large parts of JSON documents (prune) decreases the difference, and
  * small documents increase the difference, as `Jackson` is more expensive to initialize.
  * working directly on bytes is faster than working on characters for the `core` processors.
 
 For a typical, light-weight web service, the overall system performance improvement for using the `core` filters over the `Jackson`-based filters will most likely be a few percent.
 
-Memory use will be at 2-8 times the raw JSON byte size; depending on the invoked `JsonFilter` method (some accept string, other raw bytes or chars).
+Memory use will be at most 2-8 times the raw JSON byte size; depending on the invoked `JsonFilter` method (some accept string, other raw bytes or chars).
 
 See the benchmark results ([JDK 17](https://jmh.morethan.io/?source=https://raw.githubusercontent.com/skjolber/json-log-filter/master/benchmark/jmh/results/jmh-results-4.1.2.jdk17.json&topBar=off)) and the [JMH] module for running detailed benchmarks.
 

benchmark/jmh/src/main/java/com/github/skjolber/jsonfilter/jmh/AbstractSingleFullPathMaxStringLengthFilterBenchmark.java

@@ -31,16 +31,12 @@
 import com.github.skjolber.jsonfilter.core.MultiPathJsonFilter;
 import com.github.skjolber.jsonfilter.core.MultiPathMaxSizeMaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.core.MultiPathMaxStringLengthJsonFilter;
-import com.github.skjolber.jsonfilter.core.SingleAnyPathMaxStringLengthJsonFilter;
-import com.github.skjolber.jsonfilter.core.SingleFullPathJsonFilter;
-import com.github.skjolber.jsonfilter.core.SingleFullPathMaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.jackson.JacksonJsonFilter;
 import com.github.skjolber.jsonfilter.jackson.JacksonMaxSizeJsonFilter;
 import com.github.skjolber.jsonfilter.jackson.JacksonMaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.jackson.JacksonMultiAnyPathMaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.jackson.JacksonMultiPathMaxSizeMaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.jackson.JacksonMultiPathMaxStringLengthJsonFilter;
-import com.github.skjolber.jsonfilter.jackson.JacksonSingleFullPathMaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.jmh.filter.PrimitiveJsonPropertyBodyFilter;
 import com.github.skjolber.jsonfilter.jmh.utils.JsonMaskerJsonFilter;
 
@@ -51,17 +47,15 @@
 @State(Scope.Thread)
 @BenchmarkMode(Mode.Throughput)
 @OutputTimeUnit(TimeUnit.SECONDS)
-@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
-@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
-
+@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
 @Fork(1)
 public class AllFilterBenchmark {
 
 	public static final String DEFAULT_XPATH = "/address";
 	public static final String DEFAULT_ANY_XPATH = "//address";
 
 	private JacksonBenchmarkRunner jacksonMultiPathMaxSizeMaxStringLengthJsonFilter;
-	private BenchmarkRunner<JacksonJsonFilter> singleFullPathAnonymizeMaxStringLengthJacksonJsonFilter;
 	private BenchmarkRunner<JsonFilter> anyPathJsonMaskerJsonFilter;
 	private BenchmarkRunner<JsonFilter> singlePathJsonMaskerJsonFilter;
 
@@ -70,12 +64,9 @@ public class AllFilterBenchmark {
 
 	private BenchmarkRunner<JsonFilter> defaultJsonFilter;
 
-	private BenchmarkRunner<JsonFilter> singleFullPathAnonymizeJsonFilter;
 	private BenchmarkRunner<JsonFilter> maxStringLengthJsonFilter;
 	private BenchmarkRunner<JsonFilter> maxSizeJsonFilter;
 
-	private BenchmarkRunner<JsonFilter> singleFullPathMaxStringLengthAnonymizeJsonFilter;
-
 	private BenchmarkRunner<JsonFilter> multiPathAnonymizeJsonFilter;
 	private BenchmarkRunner<JsonFilter> multiPathAnonymizeFullPathJsonFilter;
 	private BenchmarkRunner<JsonFilter> multiPathMaxStringLengthAnonymizeJsonFilter;
@@ -84,7 +75,6 @@ public class AllFilterBenchmark {
 	private BenchmarkRunner<JacksonJsonFilter> multiAnyPathAnonymizeMaxStringLengthJacksonJsonFilter;
 	private BenchmarkRunner<JsonFilter> multiAnyPathLogbookJsonFilter;
 	private BenchmarkRunner<JsonFilter> anyPathAnonymizeJsonFilter;
-	private BenchmarkRunner<JsonFilter> singleAnyPathAnonymizeMaxStringLengthJsonFilter;
 
 	private BenchmarkRunner<JsonFilter> multiPathAnonymizeMaxSizeMaxStringLengthJsonFilter;
 
@@ -103,7 +93,6 @@ public void init() throws Exception {
 		// generic filters
 		// jackson
 		jacksonMultiPathMaxSizeMaxStringLengthJsonFilter = new JacksonBenchmarkRunner(file, true, new JacksonMultiPathMaxSizeMaxStringLengthJsonFilter(20, -1, new String[] {xpath}, null), prettyPrinted);
-		singleFullPathAnonymizeMaxStringLengthJacksonJsonFilter = new JacksonBenchmarkRunner(file, true, new JacksonSingleFullPathMaxStringLengthJsonFilter(20, xpath, FilterType.ANON), prettyPrinted);
 		multiPathAnonymizeMaxStringLengthJacksonJsonFilter = new JacksonBenchmarkRunner(file, true, new JacksonMultiPathMaxStringLengthJsonFilter(20, new String[] {xpath}, null), prettyPrinted);
 		multiAnyPathAnonymizeMaxStringLengthJacksonJsonFilter = new JacksonBenchmarkRunner(file, true, new JacksonMultiAnyPathMaxStringLengthJsonFilter(20, new String[] {DEFAULT_ANY_XPATH}, null), prettyPrinted);
 		maxStringLengthJacksonJsonFilter = new JacksonBenchmarkRunner(file, true, new JacksonMaxStringLengthJsonFilter(20), prettyPrinted);
@@ -114,10 +103,7 @@ public void init() throws Exception {
 		maxSizeJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new MaxSizeJsonFilter(128), prettyPrinted);
 
 		// path - single
-		singleFullPathAnonymizeJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new SingleFullPathJsonFilter(-1, xpath, FilterType.ANON), prettyPrinted);
-		singleFullPathMaxStringLengthAnonymizeJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new SingleFullPathMaxStringLengthJsonFilter(20, -1, xpath, FilterType.ANON), prettyPrinted);
 		anyPathAnonymizeJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new AnyPathJsonFilter(-1, new String[]{DEFAULT_ANY_XPATH}, null), prettyPrinted);
-		singleAnyPathAnonymizeMaxStringLengthJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new SingleAnyPathMaxStringLengthJsonFilter(20, -1, DEFAULT_ANY_XPATH, FilterType.ANON), prettyPrinted);
 
 		// path - multiple
 		multiPathAnonymizeJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new MultiPathJsonFilter(-1, new String[]{xpath}, null), prettyPrinted);
@@ -153,11 +139,6 @@ public long noop_passthrough() throws IOException {
 	public long multiAnyPathLogbook() throws IOException {
 		return multiAnyPathLogbookJsonFilter.benchmarkBytes();
 	}
-
-	@Benchmark
-	public long anonSingleMaxStringLengthJackson() throws IOException {
-		return singleFullPathAnonymizeMaxStringLengthJacksonJsonFilter.benchmarkBytes();
-	}
 
 	@Benchmark
 	public long anonMultiMaxStringLengthJackson() throws IOException {
@@ -184,11 +165,6 @@ public long maxSizeJackson() throws IOException {
 		return maxSizeJacksonJsonFilter.benchmarkBytes();
 	}
 
-	@Benchmark
-	public long anonSingleFullPath() throws IOException {
-		return singleFullPathAnonymizeJsonFilter.benchmarkBytes();
-	}
-
 	@Benchmark
 	public long anonAnyPath() throws IOException {
 		return anyPathAnonymizeJsonFilter.benchmarkBytes();
@@ -204,16 +180,6 @@ public long anonSingleFullPathJsonMask() throws IOException {
 		return singlePathJsonMaskerJsonFilter.benchmarkBytesAsArray();
 	}
 
-	@Benchmark
-	public long anonSingleAnyPathMaxStringLength() throws IOException {
-		return singleAnyPathAnonymizeMaxStringLengthJsonFilter.benchmarkBytes();
-	}
-	
-	@Benchmark
-	public long anonSingleFullPathMaxStringLength() throws IOException {
-		return singleFullPathMaxStringLengthAnonymizeJsonFilter.benchmarkBytes();
-	}
-
 	@Benchmark
 	public long anonMultiPathMaxStringLength() throws IOException {
 		return multiPathMaxStringLengthAnonymizeJsonFilter.benchmarkBytes();

benchmark/jmh/src/main/java/com/github/skjolber/jsonfilter/jmh/AllFilterBenchmark.java

@@ -1,6 +1,8 @@
 package com.github.skjolber.jsonfilter.jmh;
 import java.io.File;
 import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import org.openjdk.jmh.annotations.Benchmark;
@@ -22,12 +24,12 @@
 
 import com.github.skjolber.jsonfilter.JsonFilter;
 import com.github.skjolber.jsonfilter.base.AbstractPathJsonFilter.FilterType;
+import com.github.skjolber.jsonfilter.core.AnyPathJsonFilter;
 import com.github.skjolber.jsonfilter.core.DefaultJsonLogFilterBuilder;
 import com.github.skjolber.jsonfilter.core.MaxSizeJsonFilter;
 import com.github.skjolber.jsonfilter.core.MaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.core.MaxStringLengthMaxSizeJsonFilter;
 import com.github.skjolber.jsonfilter.core.MultiPathMaxSizeMaxStringLengthJsonFilter;
-import com.github.skjolber.jsonfilter.core.SingleFullPathMaxSizeMaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.core.ws.MaxSizeRemoveWhitespaceJsonFilter;
 import com.github.skjolber.jsonfilter.core.ws.MaxStringLengthMaxSizeRemoveWhitespaceJsonFilter;
 import com.github.skjolber.jsonfilter.core.ws.MaxStringLengthRemoveWhitespaceJsonFilter;
@@ -37,17 +39,20 @@
 import com.github.skjolber.jsonfilter.jackson.JacksonMaxSizeJsonFilter;
 import com.github.skjolber.jsonfilter.jackson.JacksonMaxSizeMaxStringLengthJsonFilter;
 import com.github.skjolber.jsonfilter.jackson.JacksonMultiPathMaxSizeMaxStringLengthJsonFilter;
-import com.github.skjolber.jsonfilter.jackson.JacksonSingleFullPathMaxSizeMaxStringLengthJsonFilter;
+import com.github.skjolber.jsonfilter.jmh.utils.JsonMaskerJsonFilter;
+
+import dev.blaauwendraad.masker.json.JsonMasker;
+import dev.blaauwendraad.masker.json.config.JsonMaskingConfig;
 
 
 @State(Scope.Thread)
 @BenchmarkMode(Mode.Throughput)
 @OutputTimeUnit(TimeUnit.SECONDS)
-@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
-@Measurement(iterations = 15, time = 1, timeUnit = TimeUnit.SECONDS)
+@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
 
 @Fork(1)
-public abstract class CveFilterBenchmark {
+public class CveFilterBenchmark {
 
 	private final int maxStringLength = 64;
 	private static String[] anon = new String[] {"/CVE_Items/cve/affects/vendor/vendor_data/vendor_name"};
@@ -61,7 +66,9 @@ public abstract class CveFilterBenchmark {
 
 	private BenchmarkRunner<JsonFilter> multiPathMaxStringLengthJsonFilter;
 	private BenchmarkRunner<JacksonJsonFilter> multiPathMaxStringLengthJacksonJsonFilter;
-	
+
+	private BenchmarkRunner<JsonFilter> anyPathJsonFilter; 
+
 	private BenchmarkRunner<JsonFilter> maxStringLengthJsonFilter;
 	private BenchmarkRunner<JacksonJsonFilter> maxStringLengthJacksonJsonFilter;
 	private BenchmarkRunner<JsonFilter> maxStringLengthRemoveWhitespaceJsonFilter;
@@ -77,13 +84,13 @@ public abstract class CveFilterBenchmark {
 	private BenchmarkRunner<JsonFilter> maxSizeJsonFilter; 
 	private BenchmarkRunner<JsonFilter> maxSizeRemoveWhitespaceJsonFilter; 
 
-	private BenchmarkRunner<JsonFilter> singleFullPathMaxSizeMaxStringLengthJsonFilter;
-	private JacksonBenchmarkRunner singleFullPathMaxSizeMaxStringLengthJacksonJsonFilter; 
+	private BenchmarkRunner<JsonFilter> anyPathJsonMaskerJsonFilter;
+	private BenchmarkRunner<JsonFilter> singlePathJsonMaskerJsonFilter;
 
 	private BenchmarkRunner<JsonFilter> removeWhitespaceJsonFilter;
 
-	@Param(value={"2KB","8KB","14KB","22KB","30KB","50KB","70KB","100KB","200KB"})
-	//@Param(value={"2KB"})
+	//@Param(value={"2KB","8KB","14KB","22KB","30KB","50KB","70KB","100KB","200KB"})
+	@Param(value={"8KB"})
 	private String fileName;
 	private final boolean prettyPrinted = false;
 
@@ -115,10 +122,19 @@ public void init() throws Exception {
 		maxStringLengthMaxSizeRemoveWhitespaceJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new MaxStringLengthMaxSizeRemoveWhitespaceJsonFilter(maxStringLength, size), prettyPrinted);
 		maxStringLengthMaxSizeJacksonJsonFilter = new BenchmarkRunner<JacksonJsonFilter>(file, true, new JacksonMaxSizeMaxStringLengthJsonFilter(maxStringLength, size), prettyPrinted);
 
-		singleFullPathMaxSizeMaxStringLengthJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new SingleFullPathMaxSizeMaxStringLengthJsonFilter(maxStringLength, size, -1, anon[0], FilterType.ANON), prettyPrinted);
-		singleFullPathMaxSizeMaxStringLengthJacksonJsonFilter = new JacksonBenchmarkRunner(file, true, new JacksonSingleFullPathMaxSizeMaxStringLengthJsonFilter(maxStringLength, size, -1, anon[0], FilterType.ANON), prettyPrinted);
-
 		removeWhitespaceJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new RemoveWhitespaceJsonFilter(), prettyPrinted);
+		
+		anyPathJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new AnyPathJsonFilter(size, new String[] {"//version"}, null), prettyPrinted);
+		
+		// other filters
+		var singlePathJsonMasker = JsonMasker.getMasker(
+	        JsonMaskingConfig.builder()
+	                .maskJsonPaths(Set.of("$.CVE_Items.cve.affects.vendor.vendor_data.vendor_name"))
+	                .build()
+		);
+		
+		anyPathJsonMaskerJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new JsonMaskerJsonFilter(JsonMasker.getMasker(Set.of("version"))), false);
+		singlePathJsonMaskerJsonFilter = new BenchmarkRunner<JsonFilter>(file, true, new JsonMaskerJsonFilter(singlePathJsonMasker), false);
 	}
 
 	@Benchmark
@@ -185,16 +201,6 @@ public long maxSizeJackson() {
 		return maxSizeJacksonJsonFilter.benchmarkBytes();
 	}
 
-	@Benchmark
-	public long all_single() throws IOException {
-		return singleFullPathMaxSizeMaxStringLengthJsonFilter.benchmarkCharacters();
-	}
-
-	@Benchmark
-	public long all_single_jackson() throws IOException {
-		return singleFullPathMaxSizeMaxStringLengthJacksonJsonFilter.benchmarkCharacters();
-	}
-
 	@Benchmark
 	public long maxStringLengthMaxSize_core() throws IOException {
 		return maxStringLengthMaxSizeJsonFilter.benchmarkCharacters();
@@ -210,6 +216,22 @@ public long maxStringLengthMaxSize_jackson() throws IOException {
 		return maxStringLengthMaxSizeJacksonJsonFilter.benchmarkCharacters();
 	}
 
+	@Benchmark
+	public long anon_core() throws IOException {
+		return anyPathJsonFilter.benchmarkBytes();
+	}
+
+	@Benchmark
+	public long anon_jsonMask() throws IOException {
+		return anyPathJsonMaskerJsonFilter.benchmarkBytesAsArray();
+	}
+
+	@Benchmark
+	public long anon_single_jsonMask() throws IOException {
+		return singlePathJsonMaskerJsonFilter.benchmarkBytesAsArray();
+	}
+
+
 	public static void main(String[] args) throws RunnerException {
 		Options opt = new OptionsBuilder()
 				.include(CveFilterBenchmark.class.getSimpleName())