[SYSTEMDS-2850]Generalize autoencoder training

scripts/builtin/autoencoderGeneralized.dml

@@ -0,0 +1,57 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+source("scripts/builtin/autoencoder_2layer.dml") as ae
+
+X = read($X)
+
+hidden_layers = list(as.integer($H1))
+if(as.integer($H2) > 0)
+  hidden_layers = append(hidden_layers, list(as.integer($H2)))
+if(as.integer($H3) > 0)
+  hidden_layers = append(hidden_layers, list(as.integer($H3)))
+
+[model, hidden] = ae::m_autoencoder(
+  X=X,
+  hidden_layers=hidden_layers,
+  max_epochs=as.integer($EPOCH),
+  batch_size=as.integer($BATCH),
+  step=as.double($STEP),
+  decay=as.double($DECAY),
+  mu=as.double($MOMENTUM),
+  method=$METHOD,
+  mode=$MODE,
+  utype=$UTYPE,
+  freq=$FREQ,
+  k=as.integer($WORKERS),
+  scheme=$SCHEME,
+  nbatches=as.integer($NBATCHES),
+  modelAvg=as.boolean($MODELAVG)
+)
+
+layer_count = as.integer(length(model) %/% 4)
+
+W1_out = as.matrix(model[1])
+Wlast_out = as.matrix(model[layer_count])
+hidden_out = hidden
+
+write(W1_out, $W1_out)
+write(Wlast_out, $Wlast_out)
+write(hidden_out, $hidden_out)

src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/ParamservUtils.java

@@ -118,6 +118,10 @@ public static void cleanupListObject(ExecutionContext ec, ListObject lo) {
 	}
 
 	public static void cleanupListObject(ExecutionContext ec, ListObject lo, boolean[] status) {
+		if (status != null && status.length != lo.getLength()){
+			lo.deriveAndSetStatusFromData();
+			status = lo.getStatus();
+		}
 		for (int i = 0; i < lo.getLength(); i++) {
 			if (status != null && !status[i])
 				continue; // data ref by other object must not be cleaned up
@@ -387,6 +391,9 @@ public static ListObject accrueGradients(ListObject accGradients, ListObject gra
 	public static ListObject accrueGradients(ListObject accGradients, ListObject gradients, boolean par, boolean cleanup) {
 		if (accGradients == null)
 			return ParamservUtils.copyList(gradients, cleanup);
+		if (accGradients.getLength() != gradients.getLength()) {
+			throw new DMLRuntimeException("Gradient list length mismatch: accGradients=" + accGradients.getLength() + ", gradients=" + gradients.getLength());
+		}
 		IntStream range = IntStream.range(0, accGradients.getLength());
 		(par ? range.parallel() : range).forEach(i -> {
 			MatrixBlock mb1 = ((MatrixObject) accGradients.getData().get(i)).acquireReadAndRelease();
@@ -422,6 +429,8 @@ public static ListObject accrueModels(ListObject accModels, ListObject model, bo
 	public static ListObject accrueModels(ListObject accModels, ListObject model, boolean par, boolean cleanup) {
 		if (accModels == null)
 			return ParamservUtils.copyList(model, cleanup);
+		if (accModels.getLength() != model.getLength())
+			throw new DMLRuntimeException("Model list length mismatch: acc=" + accModels.getLength() + ", model=" + model.getLength());
 		IntStream range = IntStream.range(0, accModels.getLength());
 		(par ? range.parallel() : range).forEach(i -> {
 			MatrixBlock mb1 = ((MatrixObject) accModels.getData().get(i)).acquireReadAndRelease();

src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinAutoencoderGeneralizedBasicTest.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.builtin.part1;
+
+import org.apache.sysds.runtime.meta.MatrixCharacteristics;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BuiltinAutoencoderGeneralizedBasicTest extends AutomatedTestBase {
+    private static final String TEST_NAME = "autoencoderGeneralized";
+    private static final String TEST_DIR = "functions/builtin/";
+    private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinAutoencoderGeneralizedBasicTest.class.getSimpleName() + "/";
+
+    private static final int ROWS = 128;
+    private static final int COLS = 64;
+    private static final double DENSE = 0.9;
+
+    @Override
+    public void setUp() {
+        addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME,
+                new String[] { "W1_out", "Wlast_out", "hidden_out" }));
+    }
+
+    @Test
+    public void testAutoencoderThreeLayerOutputs() {
+        runAutoencoderTest(16, 8, 4, 2, DENSE);
+    }
+
+    private void runAutoencoderTest(int h1, int h2, int h3, int maxEpochs, double sparsity) {
+        loadTestConfiguration(getTestConfiguration(TEST_NAME));
+
+        String home = SCRIPT_DIR + TEST_DIR;
+        fullDMLScriptName = home + TEST_NAME + ".dml";
+        programArgs = new String[] { "-nvargs",
+                "X=" + input("X"),
+                "H1=" + h1, "H2=" + h2, "H3=" + h3,
+                "EPOCH=" + maxEpochs, "BATCH=" + 32,
+                "STEP=" + 1e-4, "DECAY=" + 0.95, "MOMENTUM=" + 0.9,
+                "METHOD=DEFAULTSERVER", "MODE=LOCAL", "UTYPE=BSP",
+                "FREQ=BATCH", "WORKERS=1", "SCHEME=DISJOINT_RANDOM",
+                "NBATCHES=0", "MODELAVG=FALSE",
+                "W1_out=" + output("W1_out"),
+                "Wlast_out=" + output("Wlast_out"),
+                "hidden_out=" + output("hidden_out")
+        };
+
+        double[][] X = getRandomMatrix(ROWS, COLS, 0, 1, sparsity, 42);
+        writeInputMatrixWithMTD("X", X, true);
+
+        runTest(true, false, null, -1);
+
+        MatrixCharacteristics w1Meta = readDMLMetaDataFile("W1_out");
+        MatrixCharacteristics wlastMeta = readDMLMetaDataFile("Wlast_out");
+        MatrixCharacteristics hiddenMeta = readDMLMetaDataFile("hidden_out");
+
+        Assert.assertEquals(h1, w1Meta.getRows());
+        Assert.assertEquals(COLS, w1Meta.getCols());
+        Assert.assertEquals(COLS, wlastMeta.getRows());
+        Assert.assertEquals(h1, wlastMeta.getCols());
+        Assert.assertEquals(ROWS, hiddenMeta.getRows());
+        Assert.assertEquals(h3, hiddenMeta.getCols());
+    }
+}

src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinAutoencoderGeneralizedTest.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sysds.test.functions.builtin.part1;
+
+import org.apache.sysds.runtime.meta.MatrixCharacteristics;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class BuiltinAutoencoderGeneralizedTest extends AutomatedTestBase {
+    private static final String TEST_NAME = "autoencoderGeneralized";
+    private static final String TEST_DIR = "functions/builtin/";
+    private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinAutoencoderGeneralizedTest.class.getSimpleName() + "/";
+
+    private static final int ROWS = 128;
+    private static final int COLS = 64;
+    private static final double DENSE = 0.9;
+
+    @Override
+    public void setUp() {
+        addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME,
+                new String[] { "W1_out", "Wlast_out", "hidden_out" }));
+    }
+
+    @Test
+    public void testAutoencoderThreeLayerOutputs() {
+        runAutoencoderTest("DEFAULTSERVER", 16, 8, 4, 2, DENSE);
+    }
+
+    @Test
+    public void testAutoencoderTwoLayerOutputs() {
+        runAutoencoderTest("DEFAULTSERVER", 16, 8, 0, 0, DENSE);
+    }
+
+    @Test
+    public void testAutoencoderSingleLayerOutputs() {
+        runAutoencoderTest("DEFAULTSERVER", 16, 0, 0, 1, DENSE);
+    }
+
+    @Test
+    public void testAutoencoderSparseInputOutputs() {
+        runAutoencoderTest("DEFAULTSERVER", 32, 16, 8, 2, 0.2);
+    }
+
+    @Test
+    public void testAutoencoderParamservOutputs() {
+        runAutoencoderTest("PARAMSERVER", 16, 8, 4, 2, DENSE);
+    }
+    private void runAutoencoderTest(String method, int h1, int h2, int h3, int maxEpochs, double sparsity) {
+        int expectedHidden = h3 > 0 ? h3 : (h2 > 0 ? h2 : h1);
+        loadTestConfiguration(getTestConfiguration(TEST_NAME));
+
+        String home = SCRIPT_DIR + TEST_DIR;
+        fullDMLScriptName = home + TEST_NAME + ".dml";
+        programArgs = new String[] { "-nvargs",
+                "X=" + input("X"),
+                "H1=" + h1, "H2=" + h2, "H3=" + h3,
+                "EPOCH=" + maxEpochs, "BATCH=" + 32,
+                "STEP=" + 1e-4, "DECAY=" + 0.95, "MOMENTUM=" + 0.9,
+                "METHOD=" + method, "MODE=LOCAL", "UTYPE=BSP",
+                "FREQ=BATCH", "WORKERS=1", "SCHEME=DISJOINT_RANDOM",
+                "NBATCHES=0", "MODELAVG=FALSE",
+                "W1_out=" + output("W1_out"),
+                "Wlast_out=" + output("Wlast_out"),
+                "hidden_out=" + output("hidden_out")
+        };
+
+        double[][] X = getRandomMatrix(ROWS, COLS, 0, 1, sparsity, 42);
+        writeInputMatrixWithMTD("X", X, true);
+
+        runTest(true, false, null, -1);
+
+        MatrixCharacteristics w1Meta = readDMLMetaDataFile("W1_out");
+        MatrixCharacteristics wlastMeta = readDMLMetaDataFile("Wlast_out");
+        MatrixCharacteristics hiddenMeta = readDMLMetaDataFile("hidden_out");
+
+        Assert.assertEquals(h1, w1Meta.getRows());
+        Assert.assertEquals(COLS, w1Meta.getCols());
+        Assert.assertEquals(COLS, wlastMeta.getRows());
+        Assert.assertEquals(h1, wlastMeta.getCols());
+        Assert.assertEquals(ROWS, hiddenMeta.getRows());
+        Assert.assertEquals(expectedHidden, hiddenMeta.getCols());
+    }
+}

src/test/scripts/functions/builtin/autoencoderGeneralized.dml

@@ -0,0 +1,57 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+source("scripts/builtin/autoencoder_2layer.dml") as ae
+
+X = read($X)
+hidden_layers = list(as.integer($H1))
+if(as.integer($H2) > 0)
+  hidden_layers = append(hidden_layers, list(as.integer($H2)))
+if(as.integer($H3) > 0)
+  hidden_layers = append(hidden_layers, list(as.integer($H3)))
+
+[model, hidden] = ae::m_autoencoder(
+  X=X,
+  hidden_layers=hidden_layers,
+  max_epochs=as.integer($EPOCH),
+  batch_size=as.integer($BATCH),
+  step=as.double($STEP),
+  decay=as.double($DECAY),
+  mu=as.double($MOMENTUM),
+  method=$METHOD,
+  mode=$MODE,
+  utype=$UTYPE,
+  freq=$FREQ,
+  k=as.integer($WORKERS),
+  scheme=$SCHEME,
+  nbatches=as.integer($NBATCHES),
+  modelAvg=as.boolean($MODELAVG)
+)
+
+encoder_layers = length(hidden_layers)
+layer_count = 2 * encoder_layers
+
+W1_out = as.matrix(model[1])
+Wlast_out = as.matrix(model[layer_count])
+hidden_out = hidden
+
+write(W1_out, $W1_out)
+write(Wlast_out, $Wlast_out)
+write(hidden_out, $hidden_out)

src/test/scripts/functions/builtin/run_ae2layer.dml

@@ -0,0 +1,21 @@
+source("scripts/builtin/autoencoder_2layer.dml") as ae
+
+X = read($X)
+
+[W1, b1, W2, b2, W3, b3, W4, b4, hidden] =
+  ae::m_autoencoder_2layer(
+    X, as.integer($num_hidden1), as.integer($num_hidden2), as.integer($max_epochs),
+    FALSE, as.integer($batch_size), as.double($step), as.double($decay), as.double($mu),
+    $method, $mode, $utype, $freq, as.integer($k), $scheme, as.integer($nbatches), as.boolean($modelAvg)
+  )
+
+write(W1, $W1_out)
+write(b1, $b1_out)
+write(W2, $W2_out)
+write(b2, $b2_out)
+write(W3, $W3_out)
+write(b3, $b3_out)
+write(W4, $W4_out)
+write(b4, $b4_out)
+write(hidden, $HIDDEN)
+EOF