#2299 SelfURLFilter: compare the path and query case-sensitively

core/src/main/java/org/apache/stormcrawler/filtering/basic/SelfURLFilter.java

@@ -18,10 +18,13 @@
 package org.apache.stormcrawler.filtering.basic;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Map;
+import java.util.Objects;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.filtering.URLFilter;
+import org.apache.stormcrawler.util.URLUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -38,13 +41,33 @@ public class SelfURLFilter extends URLFilter {
             return urlToFilter;
         }
 
-        if (sourceUrl.toExternalForm().equalsIgnoreCase(urlToFilter)) {
+        final URL target;
+        try {
+            target = URLUtil.toURL(urlToFilter);
+        } catch (MalformedURLException e) {
+            // not for this filter to judge
+            return urlToFilter;
+        }
+
+        if (isSameURL(sourceUrl, target)) {
             return null;
         }
 
         return urlToFilter;
     }
 
+    /**
+     * Only the scheme and the host are case-insensitive (RFC 3986, section 6.2.2.1), the path and
+     * the query are not: a redirect from /a.html to /A.html points to another resource.
+     */
+    private static boolean isSameURL(URL source, URL target) {
+        return source.getProtocol().equalsIgnoreCase(target.getProtocol())
+                && source.getHost().equalsIgnoreCase(target.getHost())
+                && source.getPort() == target.getPort()
+                && source.getFile().equals(target.getFile())
+                && Objects.equals(source.getRef(), target.getRef());
+    }
+
     @Override
     public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode paramNode) {}
 }

core/src/test/java/org/apache/stormcrawler/filtering/SelfURLFilterTest.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.filtering;
+
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.basic.SelfURLFilter;
+import org.apache.stormcrawler.util.URLUtil;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class SelfURLFilterTest {
+
+    private static String filter(String source, String target) throws MalformedURLException {
+        SelfURLFilter filter = new SelfURLFilter();
+        filter.configure(new HashMap<>(), JsonNodeFactory.instance.objectNode());
+        URL sourceURL = URLUtil.toURL(source);
+        return filter.filter(sourceURL, new Metadata(), target);
+    }
+
+    @Test
+    void testSelfIsRemoved() throws MalformedURLException {
+        Assertions.assertNull(
+                filter("https://www.example.com/a.html", "https://www.example.com/a.html"));
+    }
+
+    @Test
+    void testSchemeAndHostAreCaseInsensitive() throws MalformedURLException {
+        Assertions.assertNull(
+                filter("https://www.example.com/a.html", "HTTPS://WWW.Example.COM/a.html"));
+    }
+
+    @Test
+    void testPathIsCaseSensitive() throws MalformedURLException {
+        String target = "https://whimsy.apache.org/board/minutes/ACE.html";
+        Assertions.assertEquals(
+                target, filter("https://whimsy.apache.org/board/minutes/ace.html", target));
+    }
+
+    @Test
+    void testQueryIsCaseSensitive() throws MalformedURLException {
+        String target = "https://www.example.com/search?q=ACE";
+        Assertions.assertEquals(target, filter("https://www.example.com/search?q=ace", target));
+    }
+
+    @Test
+    void testOtherURLsAreKept() throws MalformedURLException {
+        String source = "https://www.example.com/a.html";
+        for (String target :
+                new String[] {
+                    "http://www.example.com/a.html",
+                    "https://example.com/a.html",
+                    "https://www.example.com:8443/a.html",
+                    "https://www.example.com/b.html",
+                    "https://www.example.com/a.html?x=1",
+                    "https://www.example.com/a.html#top"
+                }) {
+            Assertions.assertEquals(target, filter(source, target));
+        }
+    }
+
+    @Test
+    void testNoSource() {
+        SelfURLFilter filter = new SelfURLFilter();
+        String target = "https://www.example.com/a.html";
+        Assertions.assertEquals(target, filter.filter(null, new Metadata(), target));
+    }
+}