From 7f0cf463d4136d30ad73165f7add7c8a406a8b99 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Mon, 2 Nov 2026 12:03:34 +0100 Subject: [PATCH] #2299 SelfURLFilter: compare the path and query case-sensitively --- .../filtering/basic/SelfURLFilter.java | 25 +++++- .../filtering/SelfURLFilterTest.java | 86 +++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/filtering/SelfURLFilterTest.java diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/SelfURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/SelfURLFilter.java index 9a975ba0e..8c714c376 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/SelfURLFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/SelfURLFilter.java @@ -18,10 +18,13 @@ package org.apache.stormcrawler.filtering.basic; import com.fasterxml.jackson.databind.JsonNode; +import java.net.MalformedURLException; import java.net.URL; import java.util.Map; +import java.util.Objects; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.filtering.URLFilter; +import org.apache.stormcrawler.util.URLUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -38,13 +41,33 @@ public class SelfURLFilter extends URLFilter { return urlToFilter; } - if (sourceUrl.toExternalForm().equalsIgnoreCase(urlToFilter)) { + final URL target; + try { + target = URLUtil.toURL(urlToFilter); + } catch (MalformedURLException e) { + // not for this filter to judge + return urlToFilter; + } + + if (isSameURL(sourceUrl, target)) { return null; } return urlToFilter; } + /** + * Only the scheme and the host are case-insensitive (RFC 3986, section 6.2.2.1), the path and + * the query are not: a redirect from /a.html to /A.html points to another resource. + */ + private static boolean isSameURL(URL source, URL target) { + return source.getProtocol().equalsIgnoreCase(target.getProtocol()) + && source.getHost().equalsIgnoreCase(target.getHost()) + && source.getPort() == target.getPort() + && source.getFile().equals(target.getFile()) + && Objects.equals(source.getRef(), target.getRef()); + } + @Override public void configure(@NotNull Map stormConf, @NotNull JsonNode paramNode) {} } diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/SelfURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/SelfURLFilterTest.java new file mode 100644 index 000000000..6e9b39ed6 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/filtering/SelfURLFilterTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.filtering; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.filtering.basic.SelfURLFilter; +import org.apache.stormcrawler.util.URLUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class SelfURLFilterTest { + + private static String filter(String source, String target) throws MalformedURLException { + SelfURLFilter filter = new SelfURLFilter(); + filter.configure(new HashMap<>(), JsonNodeFactory.instance.objectNode()); + URL sourceURL = URLUtil.toURL(source); + return filter.filter(sourceURL, new Metadata(), target); + } + + @Test + void testSelfIsRemoved() throws MalformedURLException { + Assertions.assertNull( + filter("https://www.example.com/a.html", "https://www.example.com/a.html")); + } + + @Test + void testSchemeAndHostAreCaseInsensitive() throws MalformedURLException { + Assertions.assertNull( + filter("https://www.example.com/a.html", "HTTPS://WWW.Example.COM/a.html")); + } + + @Test + void testPathIsCaseSensitive() throws MalformedURLException { + String target = "https://whimsy.apache.org/board/minutes/ACE.html"; + Assertions.assertEquals( + target, filter("https://whimsy.apache.org/board/minutes/ace.html", target)); + } + + @Test + void testQueryIsCaseSensitive() throws MalformedURLException { + String target = "https://www.example.com/search?q=ACE"; + Assertions.assertEquals(target, filter("https://www.example.com/search?q=ace", target)); + } + + @Test + void testOtherURLsAreKept() throws MalformedURLException { + String source = "https://www.example.com/a.html"; + for (String target : + new String[] { + "http://www.example.com/a.html", + "https://example.com/a.html", + "https://www.example.com:8443/a.html", + "https://www.example.com/b.html", + "https://www.example.com/a.html?x=1", + "https://www.example.com/a.html#top" + }) { + Assertions.assertEquals(target, filter(source, target)); + } + } + + @Test + void testNoSource() { + SelfURLFilter filter = new SelfURLFilter(); + String target = "https://www.example.com/a.html"; + Assertions.assertEquals(target, filter.filter(null, new Metadata(), target)); + } +}