Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
630e219
6091: New pair class to remove facebook.collections usage
rtjd6554 Nov 27, 2025
f6bb550
6091 Remove Pair
patchwork01 Nov 28, 2025
0f9c02e
Merge branch 'develop' into 6091-Remove-Facebook-Collections-Library
rtjd6554 Dec 2, 2025
f09aec3
Merge branch 'develop' into 6091-Remove-Facebook-Collections-Library
rtjd6554 Dec 2, 2025
da1f344
6091: Move records out to seperate class files.
rtjd6554 Dec 2, 2025
972cafa
6091: New Class for ByteArray
rtjd6554 Dec 2, 2025
4b26524
6091: Adjust imports to replace facebook collections with core versio…
rtjd6554 Dec 2, 2025
0bae30e
6091: Update test class
rtjd6554 Dec 2, 2025
0fab329
6091: Update pom declarations
rtjd6554 Dec 2, 2025
35524cf
Merge branch 'develop' into 6091-Remove-Facebook-Collections-Library
rtjd6554 Dec 2, 2025
e2612aa
6091: Update NOTICES
rtjd6554 Dec 2, 2025
95d060b
6091: Improve comparator
rtjd6554 Dec 2, 2025
d9f2036
6091: Access modifiers correction
rtjd6554 Dec 2, 2025
73d4c64
6091: Update compare logic
rtjd6554 Dec 2, 2025
46b7450
6091: Correct class name typo
rtjd6554 Dec 3, 2025
d3cd782
6091: Correct java doc typo
rtjd6554 Dec 3, 2025
6fa837a
6091: Additional unit test
rtjd6554 Dec 3, 2025
d4fa64f
Merge branch 'develop' into 6091-Remove-Facebook-Collections-Library
rtjd6554 Dec 3, 2025
1e019bb
6091: Update copyright declaration
rtjd6554 Dec 3, 2025
1dc5db4
6091: Typos and Javadoc corrections
rtjd6554 Dec 3, 2025
1e9e865
6091: Rename class and Javadoc
rtjd6554 Dec 3, 2025
2fb1a8f
6091: MIssing Javadoc
rtjd6554 Dec 3, 2025
764cbbf
6091: Javadoc improvements
rtjd6554 Dec 3, 2025
8729321
6091: Update unit tests
rtjd6554 Dec 3, 2025
837a218
6091: Checkstyle updates
rtjd6554 Dec 3, 2025
87aa5c2
6091: Update copyright
rtjd6554 Dec 3, 2025
e1cc495
6091: Update copyright
rtjd6554 Dec 3, 2025
5908008
6091: Update copyright
rtjd6554 Dec 3, 2025
4048f0c
6091: Simplfy copyright
rtjd6554 Dec 3, 2025
fc6e5d0
Merge branch 'develop' into 6091-Remove-Facebook-Collections-Library
rtjd6554 Dec 3, 2025
90047a1
6091: Refactor nested classes
rtjd6554 Dec 3, 2025
5a4faee
6091: Checkstyle updates
rtjd6554 Dec 3, 2025
615dabf
6091: Spotbugs adjustments
rtjd6554 Dec 3, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions NOTICES
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,6 @@ Logback (ch.qos.logback:logback-*:1.*)
- Eclipse Public License 1.0
- GNU Lesser General Public License, version 2.1

Facebook Jcommon collections (com.facebook.jcommon:collections:0.*)

- Apache License, Version 2.0
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is no longer a Maven dependency, but we've copied code directly from there so I think we need to state that in this file. I think we still need this notice, but in its own section explaining we copied the ByteArray class.


Guava: Google Core Libraries For Java (com.google.guava:guava:33.*)

- Apache License, Version 2.0
Expand Down
25 changes: 25 additions & 0 deletions java/athena/src/main/java/sleeper/athena/record/FieldAsString.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Copyright 2022-2025 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sleeper.athena.record;

/**
* Storage of key pairing for field and value.
*
* @param fieldName field name
* @param value value stored
*/
public record FieldAsString(String fieldName, String value) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This class is missing Javadoc.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Copyright 2022-2025 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sleeper.athena.record;

/**
* Storage of key pairing of dimension and value.
*
* @param dimension dimension key
* @param value value stored
*/
public record FieldAtDimension(int dimension, Object value) {
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import com.amazonaws.athena.connector.lambda.domain.predicate.SortedRangeSet;
import com.amazonaws.athena.connector.lambda.domain.predicate.ValueSet;
import com.amazonaws.athena.connector.lambda.records.ReadRecordsRequest;
import com.facebook.collections.Pair;
import com.google.gson.Gson;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.pojo.ArrowType;
Expand Down Expand Up @@ -55,7 +54,6 @@
import sleeper.query.core.rowretrieval.RowRetrievalException;
import sleeper.query.runner.rowretrieval.LeafPartitionRowRetrieverImpl;

import java.io.Serializable;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -128,9 +126,9 @@ protected CloseableIterator<Row> createRowIterator(ReadRecordsRequest recordsReq
Set<String> relevantFiles = new HashSet<>(new Gson().fromJson(split.getProperty(RELEVANT_FILES_FIELD), List.class));
List<Field> rowKeyFields = schema.getRowKeyFields();

List<Pair<String, String>> rowKeys = split.getProperties().entrySet().stream()
List<FieldAsString> rowKeys = split.getProperties().entrySet().stream()
.filter(entry -> ROW_KEY_PREFIX_TEST.test(entry.getKey()))
.map(entry -> Pair.of(entry.getKey(), entry.getValue()))
.map(entry -> new FieldAsString(entry.getKey(), entry.getValue()))
.collect(Collectors.toList());

List<Object> minRowKeys = getRowKey(rowKeys, rowKeyFields, "Min");
Expand All @@ -139,33 +137,33 @@ protected CloseableIterator<Row> createRowIterator(ReadRecordsRequest recordsReq
return createIterator(relevantFiles, minRowKeys, maxRowKeys, schema, tableProperties, recordsRequest.getConstraints().getSummary());
}

private List<Object> getRowKey(List<Pair<String, String>> rowKeyStream, List<Field> rowKeyFields, String indicator) {
private List<Object> getRowKey(List<FieldAsString> rowKeyStream, List<Field> rowKeyFields, String indicator) {
List<Object> rowKey = new ArrayList<>();
for (int i = 0; i < rowKeyFields.size(); i++) {
rowKey.add(null);
}

rowKeyStream.stream()
.filter(entry -> entry.getFirst().contains(indicator))
.map(entry -> getIndexToObjectPair(rowKeyFields, entry))
.forEach(pair -> rowKey.set(pair.getFirst(), pair.getSecond()));
.filter(entry -> entry.fieldName().contains(indicator))
.map(entry -> getFieldAtDimension(rowKeyFields, entry))
.forEach(valueWithIndex -> rowKey.set(valueWithIndex.dimension(), valueWithIndex.value()));

return rowKey;
}

private Pair<Integer, ? extends Serializable> getIndexToObjectPair(List<Field> rowKeyFields, Pair<String, String> entry) {
String key = entry.getFirst();
private FieldAtDimension getFieldAtDimension(List<Field> rowKeyFields, FieldAsString entry) {
String key = entry.fieldName();
Integer index = Integer.valueOf(key.substring(key.lastIndexOf("RowKey") + 6));
String stringValue = entry.getSecond();
String stringValue = entry.value();
Type type = rowKeyFields.get(index).getType();
if (type instanceof StringType) {
return Pair.of(index, stringValue);
return new FieldAtDimension(index, stringValue);
} else if (type instanceof ByteArrayType) {
return Pair.of(index, BinaryUtils.fromBase64(stringValue));
return new FieldAtDimension(index, BinaryUtils.fromBase64(stringValue));
} else if (type instanceof IntType) {
return Pair.of(index, Integer.parseInt(stringValue));
return new FieldAtDimension(index, Integer.parseInt(stringValue));
} else if (type instanceof LongType) {
return Pair.of(index, Long.parseLong(stringValue));
return new FieldAtDimension(index, Long.parseLong(stringValue));
} else {
throw new RuntimeException("Unexpected Primitive type: " + type);
}
Expand Down Expand Up @@ -258,4 +256,5 @@ private CloseableIterator<Row> applyCompactionIterators(CloseableIterator<Row> m
.getIterator(IteratorConfig.from(tableProperties), schema)
.applyTransform(mergingIterator);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@
*/
package sleeper.clients.table.partition;

import com.facebook.collections.ByteArray;
import org.apache.datasketches.quantiles.ItemsSketch;

import sleeper.core.row.Row;
import sleeper.core.schema.Field;
import sleeper.core.schema.Schema;
import sleeper.core.schema.type.ByteArrayType;
import sleeper.core.util.ByteArray;
import sleeper.sketches.Sketches;

import java.util.Arrays;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/
package sleeper.clients.table.partition;

import com.facebook.collections.ByteArray;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.s3.S3Client;

Expand All @@ -33,6 +32,7 @@
import sleeper.core.schema.type.StringType;
import sleeper.core.schema.type.Type;
import sleeper.core.statestore.StateStore;
import sleeper.core.util.ByteArray;
import sleeper.statestore.StateStoreFactory;

import java.io.BufferedWriter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/
package sleeper.compaction.job.execution.testutils;

import com.facebook.collections.ByteArray;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.ParquetWriter;

Expand All @@ -24,6 +23,7 @@
import sleeper.core.statestore.FileReference;
import sleeper.core.statestore.FileReferenceFactory;
import sleeper.core.statestore.StateStore;
import sleeper.core.util.ByteArray;
import sleeper.parquet.row.ParquetReaderIterator;
import sleeper.parquet.row.ParquetRowReaderFactory;
import sleeper.parquet.row.ParquetRowWriterFactory;
Expand Down
4 changes: 0 additions & 4 deletions java/core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,6 @@
<artifactId>core</artifactId>

<dependencies>
<dependency>
<groupId>com.facebook.jcommon</groupId>
<artifactId>collections</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion java/core/src/main/java/sleeper/core/key/Key.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
*/
package sleeper.core.key;

import com.facebook.collections.ByteArray;
import sleeper.core.util.ByteArray;

import java.util.ArrayList;
import java.util.Collections;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/
package sleeper.core.partition;

import com.facebook.collections.ByteArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -30,6 +29,7 @@
import sleeper.core.schema.type.PrimitiveType;
import sleeper.core.schema.type.StringType;
import sleeper.core.schema.type.Type;
import sleeper.core.util.ByteArray;

import java.util.ArrayList;
import java.util.Collections;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@
*/
package sleeper.core.properties.local;

import com.facebook.collections.ByteArray;
import org.apache.commons.codec.binary.Base64;

import sleeper.core.util.ByteArray;

import java.io.IOException;
import java.io.StringWriter;
import java.io.UncheckedIOException;
Expand Down
3 changes: 1 addition & 2 deletions java/core/src/main/java/sleeper/core/range/Range.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,12 @@
*/
package sleeper.core.range;

import com.facebook.collections.ByteArray;

import sleeper.core.schema.Field;
import sleeper.core.schema.Schema;
import sleeper.core.schema.type.ByteArrayType;
import sleeper.core.schema.type.PrimitiveType;
import sleeper.core.schema.type.Type;
import sleeper.core.util.ByteArray;

import java.util.HashMap;
import java.util.HashSet;
Expand Down
3 changes: 1 addition & 2 deletions java/core/src/main/java/sleeper/core/row/Row.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,11 @@
*/
package sleeper.core.row;

import com.facebook.collections.ByteArray;

import sleeper.core.key.Key;
import sleeper.core.schema.Field;
import sleeper.core.schema.Schema;
import sleeper.core.schema.type.ByteArrayType;
import sleeper.core.util.ByteArray;

import java.util.ArrayList;
import java.util.Collections;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
*/
package sleeper.core.schema.type;

import com.facebook.collections.ByteArray;
import sleeper.core.util.ByteArray;

/**
* A primitive type that represents a byte array.
Expand Down
Loading
Loading