Skip to content

Commit 85e073d

Browse files
committed
Add parquet methods to release tests
1 parent 91c0f62 commit 85e073d

File tree

1 file changed

+21
-7
lines changed

1 file changed

+21
-7
lines changed

test-release.scala

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,26 @@
1-
import uk.co.gresearch.spark.diff._
1+
import org.apache.spark.sql.DataFrame
2+
3+
def assertSize(df: DataFrame, size: Long): Unit = {
4+
df.show()
5+
assert(df.collect().size == size)
6+
}
27

3-
val left = Seq((1, "one"), (2, "two"), (3, "three")).toDF("id", "value")
4-
val right = Seq((1, "one"), (2, "Two"), (4, "four")).toDF("id", "value")
58

6-
val diff = left.diff(right)
7-
diff.show()
9+
import uk.co.gresearch.spark.diff._
10+
import uk.co.gresearch.spark.parquet._
11+
12+
try {
13+
val left = Seq((1, "one"), (2, "two"), (3, "three")).toDF("id", "value")
14+
val right = Seq((1, "one"), (2, "Two"), (4, "four")).toDF("id", "value")
15+
assertSize(left.diff(right), 5)
816

9-
if (diff.collect().size == 5) { sys.exit(0) }
17+
assertSize(spark.read.parquetMetadata("src/test/files/test.parquet"), 5)
18+
assertSize(spark.read.parquetMetadata("src/test/files/nested.parquet"), 5)
19+
assertSize(spark.read.parquetMetadata("src/test/files/encrypted1.parquet"), 5)
20+
assertSize(spark.read.parquetMetadata("src/test/files/encrypted2.parquet"), 5)
21+
} catch {
22+
case e: Throwable => sys.exit(1)
23+
}
1024

11-
sys.exit(1)
25+
sys.exit(0)
1226

0 commit comments

Comments
 (0)