Add unicode normalization and apply Formatter to every message

Author: skbeh

api/src/main/java/io/github/_4drian3d/chatregulator/api/utils/Replacer.java

@@ -1,5 +1,6 @@
 package io.github._4drian3d.chatregulator.api.utils;
 
+import java.text.Normalizer;
 import java.util.Objects;
 
 import org.jetbrains.annotations.NotNull;
@@ -38,20 +39,26 @@ private Replacer() {
      */
     public static @NotNull String addFinalDot(@NotNull final String string) {
         Objects.requireNonNull(string);
-        if (string.length() <= 1 || string.charAt(string.length() - 1) == '.') {
+        if (string.length() <= 1 || string.length() >= 256 || string.charAt(string.length() - 1) == '.') {
             return string;
         }
 
         return string + ".";
     }
 
+    public static @NotNull String unicodeNormalize(@NotNull final String string, @NotNull final Normalizer.Form form) {
+        Objects.requireNonNull(string);
+        final String normalized = Normalizer.normalize(string, form);
+        return normalized.length() > 256 ? normalized.substring(0, 256) : normalized;
+    }
+
     /**
      * Applies a trailing dot and a leading capital letter to the specified string
      *
      * @param string the string
      * @return the string converted
      */
     public static @NotNull String applyFormat(final @NotNull String string) {
-        return firstLetterUppercase(addFinalDot(string));
+        return firstLetterUppercase(addFinalDot(unicodeNormalize(string, Normalizer.Form.NFC)));
     }
 }

common/src/main/java/io/github/_4drian3d/chatregulator/common/configuration/Configuration.java

@@ -1,11 +1,13 @@
 package io.github._4drian3d.chatregulator.common.configuration;
 
+import java.text.Normalizer;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import io.github._4drian3d.chatregulator.api.enums.*;
 import io.github._4drian3d.chatregulator.api.utils.Commands;
 import net.kyori.adventure.audience.Audience;
+import org.jetbrains.annotations.NotNull;
 import org.spongepowered.configurate.objectmapping.ConfigSerializable;
 import org.spongepowered.configurate.objectmapping.meta.Comment;
 import org.spongepowered.configurate.objectmapping.meta.Setting;
@@ -96,6 +98,14 @@ public static class Formatter {
         @Setting(value = "final-dot")
         private boolean finalDot = true;
 
+        @Comment("Apply unicode normalization to each sentence")
+        @Setting(value = "unicode-normalize")
+        private boolean unicodeNormalize = true;
+
+        @Comment("The unicode normalization form to apply")
+        @Setting(value = "unicode-normalization-form")
+        private @NotNull Normalizer.Form unicodeNormalizationForm = Normalizer.Form.NFC;
+
         public boolean enabled(){
             return this.enabled;
         }
@@ -107,6 +117,14 @@ public boolean setFirstLetterUppercase(){
         public boolean setFinalDot(){
             return this.finalDot;
         }
+
+        public boolean setUnicodeNormalize(){
+            return this.unicodeNormalize;
+        }
+
+            public @NotNull Normalizer.Form setUnicodeNormalizationForm() {
+            return this.unicodeNormalizationForm;
+        }
     }
 
     @ConfigSerializable

plugin/src/main/java/io/github/_4drian3d/chatregulator/plugin/listener/chat/ChatListener.java

@@ -71,19 +71,26 @@ public final class ChatListener implements RegulatorExecutor<PlayerChatEvent> {
                         this.eventManager.fireAndForget(new ChatInfractionEvent(player, deniedResult.infractionType(), checkResult, event.getMessage()));
                         player.onDetection(deniedResult, event.getMessage());
                         event.setResult(ChatResult.denied());
-                    } else if (checkResult instanceof final CheckResult.ReplaceCheckResult replaceResult) {
-                        String finalMessage = replaceResult.replaced();
+                    } else {
+                        String finalMessage;
+                        if (checkResult instanceof final CheckResult.ReplaceCheckResult replaceResult) {
+                            finalMessage = replaceResult.replaced();
+
+                            this.eventManager.fireAndForget(new ChatInfractionEvent(player, replaceResult.infractionType(), checkResult, event.getMessage()));
+                            player.onDetection(replaceResult, event.getMessage());
+                        } else {
+                            finalMessage = event.getMessage();
+                        }
 
                         final Configuration.Formatter configuration = configurationContainer.get().getFormatterConfig();
                         if (configuration.enabled()) {
                             finalMessage = applyFormat(finalMessage, configuration);
                         }
+
                         player.getChain(SourceType.CHAT).executed(event.getMessage());
-                        this.eventManager.fireAndForget(new ChatInfractionEvent(player, replaceResult.infractionType(), checkResult, event.getMessage()));
-                        player.onDetection(replaceResult, event.getMessage());
-                        event.setResult(ChatResult.message(finalMessage));
-                    } else {
-                        player.getChain(SourceType.CHAT).executed(event.getMessage());
+                        if (!finalMessage.equals(event.getMessage())) {
+                            event.setResult(ChatResult.message(finalMessage));
+                        }
                     }
                 }).exceptionally(ex -> {
                     logger.error("An error occurred while setting chat result", ex);
@@ -103,7 +110,7 @@ public PostOrder postOrder() {
     }
 
     public static @NotNull String applyFormat(final @NotNull String string, Configuration.Formatter config) {
-        return firstLetterUppercase(addFinalDot(string, config), config);
+        return firstLetterUppercase(addFinalDot(unicodeNormalize(string, config), config), config);
     }
     public static @NotNull String firstLetterUppercase(@NotNull final String string, Configuration.Formatter config) {
         if (!config.setFirstLetterUppercase()) return string;
@@ -114,4 +121,8 @@ public static String addFinalDot(final String string, Configuration.Formatter co
                 ? Replacer.addFinalDot(string)
                 : string;
     }
+
+    public static @NotNull String unicodeNormalize(@NotNull final String string, Configuration.Formatter config) {
+        return config.setUnicodeNormalize() ? Replacer.unicodeNormalize(string, config.setUnicodeNormalizationForm()) : string;
+    }
 }

plugin/src/test/java/io/github/_4drian3d/chatregulator/modules/ReplacerTest.java

@@ -4,6 +4,8 @@
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 
+import java.text.Normalizer;
+
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 class ReplacerTest {
@@ -28,6 +30,17 @@ void finalDot(){
         assertEquals(replaced, expected);
     }
 
+    @Test
+    @DisplayName("Unicode Normalize")
+    void unicodeNormalize() {
+        String original = "\u1100\uAC00\u11A8";
+        String expected = "\u1100\uAC01";
+
+        String replaced = Replacer.unicodeNormalize(original, Normalizer.Form.NFC);
+
+        assertEquals(expected, replaced);
+    }
+
     @Test
     @DisplayName("Full Format")
     void applyFullFormat(){