add analysis of exponential distribution and some analysis of error against centroid sizes

joe-sfx · joe-sfx · commit 01bdcccdf19b · 2020-10-06T14:15:41.000-07:00
diff --git a/core/src/main/java/com/tdunning/math/stats/ScaleFunction.java b/core/src/main/java/com/tdunning/math/stats/ScaleFunction.java
@@ -123,7 +123,7 @@ public double normalizer(double compression, double n) {
     },
 
     /**
-     * Generates cluster sizes proportional to sqrt(1-q) for q >= 1/2, and uniform cluster sizes for q < 1/2 by gluing
+     * Generates cluster sizes proportional to sqrt(1-q) for q geq 1/2, and uniform cluster sizes for q lt 1/2 by gluing
      * the graph of the K_1 function to its tangent line at q=1/2. Changing the split point is possible.
      */
     K_1_GLUED {
@@ -327,7 +327,7 @@ private double Z(double compression, double n) {
     },
 
     /**
-     * Generates cluster sizes proportional to 1-q for q >= 1/2, and uniform cluster sizes for q < 1/2 by gluing
+     * Generates cluster sizes proportional to 1-q for q geq 1/2, and uniform cluster sizes for q lt 1/2 by gluing
      * the graph of the K_2 function to its tangent line at q=1/2. Changing the split point is possible.
      */
     K_2_GLUED {
@@ -494,7 +494,7 @@ private double Z(double compression, double n) {
     },
 
     /**
-     * Generates cluster sizes proportional to 1-q for q >= 1/2, and uniform cluster sizes for q < 1/2 by gluing
+     * Generates cluster sizes proportional to 1-q for q geq 1/2, and uniform cluster sizes for q lt 1/2 by gluing
      * the graph of the K_3 function to its tangent line at q=1/2.
      */
     K_3_GLUED {
diff --git a/core/src/test/java/com/tdunning/math/stats/MergingDigestTest.java b/core/src/test/java/com/tdunning/math/stats/MergingDigestTest.java
@@ -17,19 +17,26 @@
 
 package com.tdunning.math.stats;
 
-import com.carrotsearch.randomizedtesting.annotations.Seed;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
 import org.apache.commons.math3.util.Pair;
 import org.apache.mahout.common.RandomUtils;
 import org.apache.mahout.math.jet.random.AbstractContinousDistribution;
+import org.apache.mahout.math.jet.random.Exponential;
 import org.apache.mahout.math.jet.random.Uniform;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
+import com.carrotsearch.randomizedtesting.annotations.Seed;
 
 //to freeze the tests with a particular seed, put the seed on the next line
 //@Seed("84527677CF03B566:A6FF596BDDB2D59D")
@@ -59,10 +66,25 @@ protected TDigest fromBytes(ByteBuffer bytes) {
         return MergingDigest.fromBytes(bytes);
     }
 
-
+    @Test
+    public void writeUniformAsymmetricScaleFunctionResults() {
+        try {
+            writeAsymmetricScaleFunctionResults(Distribution.UNIFORM);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
 
     @Test
-    public void writeAsymmetricScaleFunctionResults() {
+    public void writeExponentialAsymmetricScaleFunctionResults() {
+        try {
+            writeAsymmetricScaleFunctionResults(Distribution.EXPONENTIAL);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void writeAsymmetricScaleFunctionResults(Distribution distribution) throws Exception {
 
         List<ScaleFunction> scaleFcns = Arrays.asList(ScaleFunction.K_0, ScaleFunction.K_1,
                 ScaleFunction.K_2, ScaleFunction.K_3, ScaleFunction.K_1_GLUED,
@@ -79,12 +101,12 @@ public void writeAsymmetricScaleFunctionResults() {
                 digestParams.put(fcn.toString() + "_USUAL", new Pair<>(fcn, false));
             }
         }
-        writeSeveralDigestUniformResults(digestParams, numTrials, "../docs/asymmetric/data/merging/");
-
+        writeSeveralDigestUniformResults(digestParams, numTrials, distribution,
+            "../docs/asymmetric/data/merging/" + distribution.name() + "/");
     }
 
-    public void writeSeveralDigestUniformResults(Map<String, Pair<ScaleFunction, Boolean>> digestParams, int numTrials,
-                                                 String writeLocation) {
+    private void writeSeveralDigestUniformResults(Map<String, Pair<ScaleFunction, Boolean>> digestParams,
+        int numTrials, Distribution distribution, String writeLocation) throws Exception {
 
         int trialSize = 1_000_000;
         double compression = 100;
@@ -93,8 +115,12 @@ public void writeSeveralDigestUniformResults(Map<String, Pair<ScaleFunction, Boo
 
         Map<String, List<Integer>> centroidCounts= new HashMap<>();
 
+        Map<String, List<List<Integer>>> centroidSequences= new HashMap<>();
+
+
         for (Map.Entry<String, Pair<ScaleFunction, Boolean>> entry : digestParams.entrySet()) {
             centroidCounts.put(entry.getKey(), new ArrayList<Integer>());
+            centroidSequences.put(entry.getKey(), new ArrayList<List<Integer>>());
             try {
                 Map<Double, List<String>> records = new HashMap<>();
                 for (double q : quants) {
@@ -105,7 +131,12 @@ public void writeSeveralDigestUniformResults(Map<String, Pair<ScaleFunction, Boo
                         digest.setScaleFunction(entry.getValue().getFirst());
                         digest.setUseAlternatingSort(entry.getValue().getSecond());
                         Random rand = new Random();
-                        AbstractContinousDistribution gen = new Uniform(50, 51, rand);
+                        AbstractContinousDistribution gen;
+                        if (distribution.equals(Distribution.UNIFORM)) {
+                            gen = new Uniform(50, 51, rand);
+                        } else if (distribution.equals(Distribution.EXPONENTIAL)) {
+                            gen = new Exponential(5, rand);
+                        } else throw new Exception("distribution not specified");
                         double[] data = new double[trialSize];
                         for (int i = 0; i < trialSize; i++) {
                             data[i] = gen.nextDouble();
@@ -121,6 +152,12 @@ public void writeSeveralDigestUniformResults(Map<String, Pair<ScaleFunction, Boo
                                     String.valueOf(Math.abs(q1 - q2) / Math.min(q, 1 - q)) + "\n");
                         }
                         centroidCounts.get(entry.getKey()).add(digest.centroids().size());
+
+                        List<Integer> seq = new ArrayList<>();
+                        for (Centroid c : digest.centroids()) {
+                            seq.add(c.count());
+                        }
+                        centroidSequences.get(entry.getKey()).add(seq);
                 }
                 for (double q : quants) {
                     FileWriter csvWriter = new FileWriter(writeLocation + entry.getKey() + "_" + String.valueOf(q) + ".csv");
@@ -140,6 +177,17 @@ public void writeSeveralDigestUniformResults(Map<String, Pair<ScaleFunction, Boo
                 csvWriter.flush();
                 csvWriter.close();
 
+
+                FileWriter csvWriter2 = new FileWriter(writeLocation + entry.getKey()  + "_centroid_sizes.csv");
+                for (List<Integer> ct : centroidSequences.get(entry.getKey())) {
+                    for (Integer c : ct) {
+                        csvWriter2.append(c.toString()).append(",");
+                    }
+                    csvWriter2.append("\n");
+                }
+                csvWriter2.flush();
+                csvWriter2.close();
+
             } catch (IOException e) {
                 System.out.println(e.toString());
                 return;
diff --git a/core/src/test/java/com/tdunning/math/stats/TDigestTest.java b/core/src/test/java/com/tdunning/math/stats/TDigestTest.java
@@ -18,8 +18,10 @@
 package com.tdunning.math.stats;
 
 import com.google.common.collect.Lists;
+
 import org.apache.mahout.common.RandomUtils;
 import org.apache.mahout.math.jet.random.AbstractContinousDistribution;
+import org.apache.mahout.math.jet.random.Exponential;
 import org.apache.mahout.math.jet.random.Gamma;
 import org.apache.mahout.math.jet.random.Normal;
 import org.apache.mahout.math.jet.random.Uniform;
@@ -31,7 +33,6 @@
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
-
 /**
  * Base test case for TDigests, just extend this class and implement the abstract methods.
  */
@@ -44,6 +45,8 @@ public abstract class TDigestTest extends AbstractTest {
 
     private static String digestName;
 
+    protected enum Distribution {UNIFORM, EXPONENTIAL};
+
     @BeforeClass
     public static void freezeSeed() {
         RandomUtils.useTestSeed();
@@ -129,6 +132,23 @@ public void offsetUniform() {
 
     @Test
     public void writeUniformResultsWithCompression() {
+        try {
+            writeResultsWithCompression(Distribution.UNIFORM);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Test
+    public void writeExponentialResultsWithCompression() {
+        try {
+            writeResultsWithCompression(Distribution.EXPONENTIAL);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void writeResultsWithCompression(Distribution distribution) throws Exception {
 
         List<ScaleFunction> scaleFcns = Arrays.asList(ScaleFunction.K_0, ScaleFunction.K_1,
                 ScaleFunction.K_2, ScaleFunction.K_3, ScaleFunction.K_1_GLUED,
@@ -138,8 +158,12 @@ public void writeUniformResultsWithCompression() {
 
         Map<ScaleFunction, List<Integer>> centroidCounts= new HashMap<>();
 
+        Map<ScaleFunction, List<List<Integer>>> centroidSequences= new HashMap<>();
+
         for (ScaleFunction scaleFcn : scaleFcns) {
             centroidCounts.put(scaleFcn, new ArrayList<Integer>());
+            centroidSequences.put(scaleFcn, new ArrayList<List<Integer>>());
+
             try {
                 Map<Double, List<String>> records = new HashMap<>();
                 double[] quants = new double[]{0.00001, 0.0001, 0.001, 0.01, 0.1,
@@ -152,7 +176,12 @@ public void writeUniformResultsWithCompression() {
                         TDigest digest = factory(compression).create();
                         digest.setScaleFunction(scaleFcn);
                         Random rand = new Random();
-                        AbstractContinousDistribution gen = new Uniform(50, 51, rand);
+                        AbstractContinousDistribution gen;
+                        if (distribution.equals(Distribution.UNIFORM)) {
+                            gen = new Uniform(50, 51, rand);
+                        } else if (distribution.equals(Distribution.EXPONENTIAL)) {
+                            gen = new Exponential(5, rand);
+                        } else throw new Exception("distribution not specified");
                         double[] data = new double[trialSize];
                         for (int i = 0; i < trialSize; i++) {
                             data[i] = gen.nextDouble();
@@ -168,6 +197,12 @@ public void writeUniformResultsWithCompression() {
                                     String.valueOf(Math.abs(q1 - q2) / Math.min(q, 1 - q)) + "\n");
                         }
                         centroidCounts.get(scaleFcn).add(digest.centroids().size());
+
+                        List<Integer> seq = new ArrayList<>();
+                        for (Centroid c : digest.centroids()) {
+                            seq.add(c.count());
+                        }
+                        centroidSequences.get(scaleFcn).add(seq);
                     }
                 }
 
@@ -180,7 +215,7 @@ public void writeUniformResultsWithCompression() {
                 }
 
                 for (double q : quants) {
-                    FileWriter csvWriter = new FileWriter("../docs/asymmetric/data/tree/" + fcnName + "_" + String.valueOf(q) + ".csv");
+                    FileWriter csvWriter = new FileWriter("../docs/asymmetric/data/tree/" + distribution.name() + "/" + fcnName + "_" + String.valueOf(q) + ".csv");
                     csvWriter.append("error_q,norm_error_q\n");
                     for (String obs : records.get(q)) {
                         csvWriter.append(obs);
@@ -189,14 +224,25 @@ public void writeUniformResultsWithCompression() {
                     csvWriter.close();
                 }
 
-                FileWriter csvWriter = new FileWriter("../docs/asymmetric/data/tree/" + fcnName + "_centroid_counts.csv");
+                FileWriter csvWriter = new FileWriter("../docs/asymmetric/data/tree/" + distribution.name() + "/" + fcnName + "_centroid_counts.csv");
                 csvWriter.append("centroid_count\n");
                 for (Integer ct : centroidCounts.get(scaleFcn)) {
                     csvWriter.append(ct.toString()).append("\n");
                 }
                 csvWriter.flush();
                 csvWriter.close();
 
+
+                FileWriter csvWriter2 = new FileWriter("../docs/asymmetric/data/tree/" + distribution.name() + "/" + fcnName + "_centroid_sizes.csv");
+                for (List<Integer> ct : centroidSequences.get(scaleFcn)) {
+                    for (Integer c : ct) {
+                        csvWriter2.append(c.toString()).append(",");
+                    }
+                    csvWriter2.append("\n");
+                }
+                csvWriter2.flush();
+                csvWriter2.close();
+
                 } catch (IOException e) {
                     return;
                 }
diff --git a/docs/asymmetric/generate_plots.py b/docs/asymmetric/generate_plots.py
