From 4aa6e829e876fc2a92a8c5ad07aff99a72b2133b Mon Sep 17 00:00:00 2001 From: qbialota Date: Wed, 5 Mar 2025 16:27:11 +0100 Subject: [PATCH 1/4] Update S3 path / file system services with possibility to request data stored on custom S3 storage (not only AWS) --- .../sis/cloud/aws/s3/ClientFileSystem.java | 69 +++++- .../sis/cloud/aws/s3/ClientFileSystemKey.java | 111 +++++++++ .../apache/sis/cloud/aws/s3/FileService.java | 214 ++++++++++++++++-- .../org/apache/sis/cloud/aws/s3/KeyPath.java | 18 ++ 4 files changed, 388 insertions(+), 24 deletions(-) create mode 100644 endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java index b8aa19d3eb3..fcc3100c780 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java @@ -16,6 +16,7 @@ */ package org.apache.sis.cloud.aws.s3; +import java.net.URI; import java.util.Set; import java.util.Collections; import java.util.regex.PatternSyntaxException; @@ -37,6 +38,7 @@ import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import org.apache.sis.util.ArgumentChecks; +import software.amazon.awssdk.services.s3.S3Configuration; import org.apache.sis.util.collection.Containers; import org.apache.sis.util.internal.shared.Strings; @@ -46,6 +48,7 @@ * which is kept ready-to-use until the file system is {@linkplain #close closed}. * * @author Martin Desruisseaux (Geomatys) + * @author Quentin Bialota (Geomatys) */ final class ClientFileSystem extends FileSystem { /** @@ -59,6 +62,21 @@ final class ClientFileSystem extends FileSystem { */ final String accessKey; + /** + * The S3 host (if not stored on Amazon Infrastructure), or {@code null} if none. + */ + final String host; + + /** + * The S3 port (if not stored on Amazon Infrastructure), or {@code null} if none. + */ + final Integer port; + + /** + * Is the S3 HTTP Protocol secure (if not stored on Amazon Infrastructure) (default {@code true)}. + */ + boolean isHttps = true; + /** * The provider of this file system. */ @@ -88,20 +106,58 @@ final class ClientFileSystem extends FileSystem { this.provider = provider; this.client = client; this.accessKey = null; + this.host = null; + this.port = null; this.separator = DEFAULT_SEPARATOR; duplicatedSeparator = DEFAULT_SEPARATOR + DEFAULT_SEPARATOR; } + /** + * Creates a file system with default hostname and default separator. + */ + ClientFileSystem(final FileService provider, final S3Client client, String accessKey) { + this.provider = provider; + this.client = client; + this.accessKey = accessKey; + this.host = null; + this.port = null; + this.separator = DEFAULT_SEPARATOR; + duplicatedSeparator = DEFAULT_SEPARATOR + DEFAULT_SEPARATOR; + } + + /** + * Creates a file system with default credential and default separator. + */ + ClientFileSystem(final FileService provider, String accessKey, String host, Integer port, boolean isHttps) { + this.provider = provider; + this.accessKey = accessKey; + this.host = host; + this.port = port; + this.isHttps = isHttps; + this.separator = DEFAULT_SEPARATOR; + duplicatedSeparator = DEFAULT_SEPARATOR + DEFAULT_SEPARATOR; + + String hostname = (port != null ? host+":"+port : host); + String protocol = (isHttps ? "https" : "http"); + this.client = S3Client.builder().endpointOverride(URI.create(protocol + "://" + hostname)) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + } + /** * Creates a new file system with the specified credential. * * @param provider the provider creating this file system. * @param region the AWS region, or {@code null} for default. + * @param host the host or {@code null} for aws request + * @param port the port or {@code null} for aws request + * @param isHttps the protocol is secure or not or {@code null} for aws request * @param accessKey the AWS S3 access key for this file system. * @param secret the password. * @param separator the separator in paths, or {@code null} for the default value. */ - ClientFileSystem(final FileService provider, final Region region, final String accessKey, final String secret, + ClientFileSystem(final FileService provider, final Region region, final String host, final Integer port, + final Boolean isHttps, final String accessKey, final String secret, String separator) { if (separator == null) { @@ -115,6 +171,15 @@ final class ClientFileSystem extends FileSystem { if (region != null) { builder = builder.region(region); } + this.host = host; + this.port = port; + this.isHttps = (isHttps != null ? isHttps : true); + if (host != null) { + String hostname = (port != null ? host+":"+port : host); + String protocol = (this.isHttps ? "https" : "http"); + builder = builder.endpointOverride(URI.create(protocol + "://" + hostname)) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()); + } client = builder.build(); this.separator = separator; duplicatedSeparator = separator.concat(separator); @@ -148,7 +213,7 @@ public synchronized void close() throws IOException { final S3Client c = client; client = null; if (c != null) try { - provider.dispose(accessKey); + provider.dispose(new ClientFileSystemKey(accessKey, host, port, isHttps)); c.close(); } catch (SdkException e) { throw new IOException(e); diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java new file mode 100644 index 00000000000..13a28efa0fc --- /dev/null +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java @@ -0,0 +1,111 @@ +package org.apache.sis.cloud.aws.s3; + +import java.util.Objects; + +/** + * File System Key stored in {@link FileService#fileSystems}, + * + * @author Quentin Bialota (Geomatys) + */ +public class ClientFileSystemKey { + + /** + * The S3 access key + */ + private final String accessKey; + + /** + * The S3 host (if not stored on Amazon AWS Infrastructure) + */ + private final String host; + + /** + * The S3 port (if not stored on Amazon AWS Infrastructure) + */ + private final Integer port; + + /** + * Is the S3 HTTP Protocol secure (if not stored on Amazon AWS Infrastructure) + */ + private final boolean isHttps; + + /** + * Creates a new file system key for the {@link FileService} with access key, host, port and protocol (secure or not secure) + * + * @param accessKey the S3 access key for this file system. + * @param host the host or {@code null} for aws request + * @param port the port or {@code -1} for aws request + * @param isHttps the protocol is secure or not + */ + public ClientFileSystemKey(String accessKey, String host, Integer port, boolean isHttps) { + this.accessKey = accessKey; + this.host = host; + this.port = port; + this.isHttps = isHttps; + } + + /** + * Creates a new file system key for the {@link FileService} with access key, host and port + * (protocol used defined as secure (HTTPS)) + * + * @param accessKey the S3 access key for this file system. + * @param host the host or {@code null} for aws request + * @param port the port or {@code -1} for aws request + */ + public ClientFileSystemKey(String accessKey, String host, Integer port) { + this.accessKey = accessKey; + this.host = host; + this.port = port; + this.isHttps = true; + } + + /** + * Returns the access key as a string + */ + public String getAccessKey() { + return accessKey; + } + + /** + * Returns the host as a string + */ + public String getHost() { + return host; + } + + /** + * Returns the port as an integer + */ + public Integer getPort() { + return port; + } + + /** + * Returns if true the protocol is secure + */ + public boolean isHttps() { return isHttps; } + + /** + * Indicates whether some other object is "equal to" this one. + * + * @param o The reference object with which to compare. + * @return {@code true} if this object is the same as the o argument; {@code false} otherwise. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClientFileSystemKey that = (ClientFileSystemKey) o; + return Objects.equals(accessKey, that.accessKey) && Objects.equals(host, that.host) && Objects.equals(port, that.port); + } + + /** + * Returns a hash code value for the object. + * + * @return A hash code value for this object. + */ + @Override + public int hashCode() { + return Objects.hash(accessKey, host, port); + } +} diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java index 635dc545552..11547c663d9 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java @@ -64,8 +64,10 @@ * This provider accepts URIs of the following forms: * * * * "Files" are S3 keys interpreted as paths with components separated by the {@code '/'} separator. @@ -75,6 +77,7 @@ * instead of the data to access, and can be a global configuration for the server. * * @author Martin Desruisseaux (Geomatys) + * @author Quentin Bialota (Geomatys) * @version 1.5 * @since 1.2 */ @@ -93,6 +96,42 @@ public class FileService extends FileSystemProvider { */ private static final String DEFAULT_ACCESS_KEY = ""; + /** + * An arbitrary string used as part of the key in the {@link #fileSystems} map + * when the user did not specified explicitly a host. + * In such case, the default host is the amazon host and is defined with the region. + * + * Host can also be defined with : + * + */ + private static final String DEFAULT_HOST_KEY = null; + + /** + * An arbitrary string used as part of the key in the {@link #fileSystems} map + * when the user did not specified explicitly a port. + * In such case, no port is assigned, the default port is used + * + * Port can also be defined with : + * + */ + private static final Integer DEFAULT_PORT_KEY = null; + + /** + * A boolean used as part of the key in the {@link #fileSystems} map + * when the user did not specified explicitly a protocol. + * In such case, the default protocol is HTTPS + * + * Port can also be defined with : + * + */ + private static final boolean DEFAULT_IS_HTTPS = true; + /** * The property for the secret access key (password). * Values shall be instances of {@link String}. @@ -107,6 +146,31 @@ public class FileService extends FileSystemProvider { */ public static final String AWS_SECRET_ACCESS_KEY = "aws.secretAccessKey"; + /** + * The property for the host (mandatory if you do not want to use AWS S3). + * Values shall be instances of {@link String}. + * + * @see #newFileSystem(URI, Map) + */ + public static final String S3_HOST_URL = "s3.hostUrl"; + + /** + * The property for the port (mandatory if you do not want to use AWS S3). + * Values shall be instances of {@link Integer}. + * + * @see #newFileSystem(URI, Map) + */ + public static final String S3_PORT = "s3.port"; + + /** + * The property for the protocol (optional even if you do not want to use AWS S3). + * Values shall be instances of {@link Boolean}. + * Default value : True (HTTPS) + * + * @see #newFileSystem(URI, Map) + */ + public static final String S3_IS_HTTPS = "s3.isHttps"; + /** * The property for the secret access key (password). * Values shall should be instances of {@link Region} or @@ -134,7 +198,7 @@ public class FileService extends FileSystemProvider { /** * All file systems created by this provider. Keys are AWS S3 access keys. */ - private final ConcurrentMap fileSystems; + private final ConcurrentMap fileSystems; /** * Creates a new provider of file systems for Amazon S3. @@ -193,7 +257,8 @@ private String getAccessKey(final URI uri) { * {@linkplain Region#of(String) convertible} to region. * * - * @param uri a URI of the form {@code "s3://accessKey@bucket/file"}. + * @param uri a URI of the form {@code "s3://accessKey@bucket/file"} + * or {@code s3://accessKey@host:port/bucket/file} (in this second case, host AND port are mandatory). * @param properties properties to configure the file system, or {@code null} if none. * @return the new file system. * @throws IllegalArgumentException if the URI or the map contains invalid values. @@ -204,6 +269,8 @@ private String getAccessKey(final URI uri) { @Override public FileSystem newFileSystem(final URI uri, final Map properties) throws IOException { final String accessKey = getAccessKey(uri); + String host = uri.getHost(); + Integer port = uri.getPort(); final String secret; if (accessKey == null || (secret = Containers.property(properties, AWS_SECRET_ACCESS_KEY, String.class)) == null) { throw new IllegalArgumentException(Resources.format(Resources.Keys.MissingAccessKey_2, (accessKey == null) ? 0 : 1, uri)); @@ -216,17 +283,44 @@ public FileSystem newFileSystem(final URI uri, final Map properties) t } else { region = Containers.property(properties, AWS_REGION, Region.class); } - final class Creator implements Function { + final class Creator implements Function { /** Identifies if a new file system is created. */ boolean created; /** Invoked if the map does not already contains the file system. */ - @Override public ClientFileSystem apply(final String key) { + @Override public ClientFileSystem apply(final ClientFileSystemKey key) { created = true; - return new ClientFileSystem(FileService.this, region, key, secret, separator); + return new ClientFileSystem(FileService.this, region, key.getHost(), key.getPort(), key.isHttps(), key.getAccessKey(), secret, separator); } } + + Boolean isHttps; + if ((isHttps = Containers.property(properties, S3_IS_HTTPS, Boolean.class)) == null) { + isHttps = DEFAULT_IS_HTTPS; + } + + // In case of Self-Hosted S3, if host and port are not found in the URI + // We check in java properties + // Else we use Default values (=> use AWS S3) + if (port == -1) { + if ((host = Containers.property(properties, S3_HOST_URL, String.class)) != null) { + + // In case of Self-Hosted S3, if port is not found in the URI, but a host is defined + if ((port = Containers.property(properties, S3_PORT, Integer.class)) == null) { + if (isHttps) { + port = 443; // Default HTTPS port + } else { + port = 80; // Default HTTP port + } + } + + } else { + host = DEFAULT_HOST_KEY; + port = DEFAULT_PORT_KEY; + } + } + final Creator c = new Creator(); - final ClientFileSystem fs = fileSystems.computeIfAbsent(accessKey, c); + final ClientFileSystem fs = fileSystems.computeIfAbsent(new ClientFileSystemKey(accessKey, host, port, isHttps), c); if (c.created) { return fs; } @@ -237,20 +331,56 @@ final class Creator implements Function { * Removes the given file system from the cache. * This method is invoked after the file system has been closed. */ - final void dispose(String identifier) { + final void dispose(ClientFileSystemKey identifier) { if (identifier == null) { - identifier = DEFAULT_ACCESS_KEY; + identifier = new ClientFileSystemKey(DEFAULT_ACCESS_KEY, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS); } fileSystems.remove(identifier); } /** - * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}. + * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}, {@link #DEFAULT_HOST_KEY}, {@link #DEFAULT_PORT_KEY} and {@link #DEFAULT_IS_HTTPS}. * * @throws SdkException if the file system cannot be created. */ private ClientFileSystem getDefaultFileSystem() { - return fileSystems.computeIfAbsent(DEFAULT_ACCESS_KEY, (key) -> new ClientFileSystem(this, S3Client.create())); + return fileSystems.computeIfAbsent(new ClientFileSystemKey(DEFAULT_ACCESS_KEY, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS), (key) -> new ClientFileSystem(this, S3Client.create())); + } + + /** + * Returns the file system associated to the {@link #DEFAULT_HOST_KEY} and {@link #DEFAULT_PORT_KEY}. + * @param accessKey the access key + * + * @throws SdkException if the file system cannot be created. + */ + private ClientFileSystem getDefaultFileSystem(String accessKey) { + return fileSystems.computeIfAbsent( + new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY), (key) -> new ClientFileSystem(this, S3Client.create(), key.getAccessKey())); + } + + /** + * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}. + * @param host the host + * @param port the port + * + * @throws SdkException if the file system cannot be created. + */ + private ClientFileSystem getDefaultFileSystem(String host, Integer port) { + return fileSystems.computeIfAbsent( + new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port), (key) -> new ClientFileSystem(this, key.getAccessKey(), key.getHost(), key.getPort(), key.isHttps())); + } + + /** + * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}. + * @param host the host + * @param port the port + * @param isHttps the protocol + * + * @throws SdkException if the file system cannot be created. + */ + private ClientFileSystem getDefaultFileSystem(String host, Integer port, boolean isHttps) { + return fileSystems.computeIfAbsent( + new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, isHttps), (key) -> new ClientFileSystem(this, key.getAccessKey(), key.getHost(), key.getPort(), key.isHttps())); } /** @@ -258,7 +388,7 @@ private ClientFileSystem getDefaultFileSystem() { * If the file system has not been created or has been closed, * then this method throws {@link FileSystemNotFoundException}. * - * @param uri a URI of the form {@code "s3://accessKey@bucket/file"}. + * @param uri a URI of the form {@code "s3://accessKey@bucket/file"} or {@code "s3://accessKey@host:port/bucket/key"}. * @return the file system previously created by {@link #newFileSystem(URI, Map)}. * @throws IllegalArgumentException if the URI is not supported by this provider. * @throws FileSystemNotFoundException if the file system does not exist or has been closed. @@ -266,10 +396,23 @@ private ClientFileSystem getDefaultFileSystem() { @Override public FileSystem getFileSystem(final URI uri) { final String accessKey = getAccessKey(uri); - if (accessKey == null) { + final String host = uri.getHost(); + final int port = uri.getPort(); + + //No access key, but host and port are defined => Self Hosted + if (accessKey == null && port > -1) { + return getDefaultFileSystem(host, port); + + //No access key, no host, no port => AWS S3 + } else if (accessKey == null && port == -1) { return getDefaultFileSystem(); + + //Access key, no host, no port => AWS S3 + } else if(accessKey != null && port == -1) { + return getDefaultFileSystem(accessKey); } - final ClientFileSystem fs = fileSystems.get(accessKey); + + final ClientFileSystem fs = fileSystems.get(new ClientFileSystemKey(accessKey, host, port)); if (fs != null) { return fs; } @@ -277,7 +420,7 @@ public FileSystem getFileSystem(final URI uri) { } /** - * Return a {@code Path} object by converting the given {@code URI}. + * Return a {@code Path} object by converting the given {@code URI} or {@code "s3://accessKey@host:port/bucket/key"}. * The resulting {@code Path} is associated with a {@link FileSystem} * that already exists or is constructed automatically. * @@ -289,14 +432,9 @@ public FileSystem getFileSystem(final URI uri) { @Override public Path getPath(final URI uri) { final String accessKey = getAccessKey(uri); - final ClientFileSystem fs; - if (accessKey == null) { - fs = getDefaultFileSystem(); - } else { - // TODO: we may need a way to get password here. - fs = fileSystems.computeIfAbsent(accessKey, (key) -> new ClientFileSystem(FileService.this, null, key, null, null)); - } String host = uri.getHost(); + final int port = uri.getPort(); + if (host == null) { /* * The host is null if the authority contains characters that are invalid for a host name. @@ -308,7 +446,39 @@ public Path getPath(final URI uri) { if (host == null) host = uri.toString(); throw new IllegalArgumentException(Resources.format(Resources.Keys.InvalidBucketName_1, host)); } - final String path = uri.getPath(); + + final ClientFileSystem fs; + if (accessKey == null && port == -1) { + fs = getDefaultFileSystem(); + } else if (accessKey == null && port > -1) { + fs = getDefaultFileSystem(host, port); + } else { + // TODO: we may need a way to get password here. + ClientFileSystemKey fsKey; + + if (port == -1) { + fsKey = new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY); + } else { + fsKey = new ClientFileSystemKey(accessKey, host, port); + } + fs = fileSystems.computeIfAbsent(fsKey, (key) -> + new ClientFileSystem(FileService.this, null, key.getHost(), key.getPort(), key.isHttps(), key.getAccessKey(), null, null)); + } + + String path = uri.getPath(); + + if (fs.host != null) { + if (!(fs.host.equalsIgnoreCase(DEFAULT_HOST_KEY))) { + path = path.substring(1); + String[] parts = path.split("/", 2); + if (parts.length >= 2) { + host = parts[0]; + path = "/" + parts[1]; + } + } + } + + //host in this part is the S3 bucket name return new KeyPath(fs, host, (path != null) ? new String[] {path} : CharSequences.EMPTY_ARRAY, true); } diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java index 50171eb101e..f3e242034ff 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java @@ -703,11 +703,19 @@ public URI toUri() { if (isDirectory) sb.append('/'); path = sb.toString(); } + try { + //Case : s3://accessKey@host:port/bucket/key (self-hosted path) + if (fs != null && fs.host != null && fs.port != null) { + return new URI(SCHEME, fs.accessKey, fs.host, fs.port, "/"+bucket+path, null, null); + } + + //Case : s3://accessKey@bucket/key (aws path) return new URI(SCHEME, fs.accessKey, bucket, -1, path, null, null); } catch (URISyntaxException e) { throw new IllegalStateException(e.getMessage(), e); } + } /** @@ -724,6 +732,15 @@ public String toString() { if (fs.accessKey != null) { sb.append(fs.accessKey).append('@'); } + + if(fs.host != null) { + if(fs.port != null) { + sb.append(fs.host).append(":").append(fs.port).append("/"); + } else { + sb.append(fs.host).append("/"); + } + } + sb.append(bucket); } if (key != null) { @@ -733,6 +750,7 @@ public String toString() { sb.append(key); } return sb.toString(); + } /** From 6effa81c578350d30626e7e37e98a1fe5f3c0deb Mon Sep 17 00:00:00 2001 From: qbialota Date: Fri, 7 Mar 2025 08:58:13 +0100 Subject: [PATCH 2/4] Fix issues with S3 path containing ./ --- .../org/apache/sis/cloud/aws/s3/KeyPath.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java index f3e242034ff..2789a14935d 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java @@ -30,6 +30,9 @@ import java.util.Iterator; import java.util.Objects; import java.util.NoSuchElementException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import software.amazon.awssdk.services.s3.model.Bucket; import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; @@ -158,7 +161,7 @@ final class KeyPath implements Path { bucketMetadata = root.bucketMetadata; this.bucket = root.bucket; this.fs = root.fs; - this.key = key; + this.key = reformatPath(key); this.isDirectory = isDirectory; assert key == null || !key.isEmpty(); // Do not copy `objectMetadata` because it is not for the same object. @@ -175,7 +178,7 @@ final class KeyPath implements Path { KeyPath(final ClientFileSystem fs, final String key, final boolean isDirectory) { this.fs = fs; this.bucket = null; - this.key = key; + this.key = reformatPath(key); this.isDirectory = isDirectory; assert !key.isEmpty(); } @@ -199,7 +202,13 @@ final class KeyPath implements Path { * If {@code false}, will be determined automatically. * @throws InvalidPathException if the path uses a protocol other than {@value #SCHEME}. */ - KeyPath(final ClientFileSystem fs, final String first, final String[] more, boolean isAbsolute) { + KeyPath(final ClientFileSystem fs, String first, final String[] more, boolean isAbsolute) { + Pattern pattern = Pattern.compile("^S3://[^@]+@([^/]+)(/.*)?$"); + Matcher matcher = pattern.matcher(first); + if (matcher.matches()) { + first = "S3:/" + matcher.group(2); + } + /* * Verify if the path start with "S3://" or "/" prefix. In both cases the path is considered absolute * and the prefix is skipped. The `start` variable is the index of the first character after prefix, @@ -265,7 +274,8 @@ final class KeyPath implements Path { throw emptyPath(first, 0); } isDirectory = path.endsWith(fs.separator); - key = isDirectory ? path.substring(0, end) : path; + String key = isDirectory ? path.substring(0, end) : path; + this.key = key.replace("./","/"); return; } } @@ -313,7 +323,8 @@ final class KeyPath implements Path { } buffer.setLength(i); } - key = buffer.toString(); + String key = buffer.toString(); + this.key = reformatPath(key); return; } } @@ -324,6 +335,16 @@ final class KeyPath implements Path { key = null; } + /** + * Reformat path, replacing "./" by "/" + */ + private static String reformatPath(String path) { + if(path != null) { + path = path.replace("./","/"); + } + return path; + } + /** * Returns the exception to throw for an empty path. */ From 28b317094e6664f2eb432b0077ccda3cd1e4f13c Mon Sep 17 00:00:00 2001 From: qbialota Date: Fri, 13 Feb 2026 11:42:50 +0100 Subject: [PATCH 3/4] feat/CustomHostS3 : refactor code, fix issues, fix doc --- .../sis/cloud/aws/s3/ClientFileSystem.java | 65 ++++------- .../sis/cloud/aws/s3/ClientFileSystemKey.java | 88 ++++++--------- .../apache/sis/cloud/aws/s3/FileService.java | 101 +++++++++++------- .../org/apache/sis/cloud/aws/s3/KeyPath.java | 49 ++++----- .../cloud/aws/s3/ClientFileSystemTest.java | 2 +- 5 files changed, 133 insertions(+), 172 deletions(-) diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java index fcc3100c780..e9de6a45b99 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java @@ -34,11 +34,11 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import org.apache.sis.util.ArgumentChecks; -import software.amazon.awssdk.services.s3.S3Configuration; import org.apache.sis.util.collection.Containers; import org.apache.sis.util.internal.shared.Strings; @@ -70,12 +70,12 @@ final class ClientFileSystem extends FileSystem { /** * The S3 port (if not stored on Amazon Infrastructure), or {@code null} if none. */ - final Integer port; + final int port; /** * Is the S3 HTTP Protocol secure (if not stored on Amazon Infrastructure) (default {@code true)}. */ - boolean isHttps = true; + final boolean isHttps; /** * The provider of this file system. @@ -99,19 +99,6 @@ final class ClientFileSystem extends FileSystem { */ final String duplicatedSeparator; - /** - * Creates a file system with default credential and default separator. - */ - ClientFileSystem(final FileService provider, final S3Client client) { - this.provider = provider; - this.client = client; - this.accessKey = null; - this.host = null; - this.port = null; - this.separator = DEFAULT_SEPARATOR; - duplicatedSeparator = DEFAULT_SEPARATOR + DEFAULT_SEPARATOR; - } - /** * Creates a file system with default hostname and default separator. */ @@ -119,29 +106,11 @@ final class ClientFileSystem extends FileSystem { this.provider = provider; this.client = client; this.accessKey = accessKey; - this.host = null; - this.port = null; + this.host = null; + this.port = -1; + this.isHttps = true; this.separator = DEFAULT_SEPARATOR; - duplicatedSeparator = DEFAULT_SEPARATOR + DEFAULT_SEPARATOR; - } - - /** - * Creates a file system with default credential and default separator. - */ - ClientFileSystem(final FileService provider, String accessKey, String host, Integer port, boolean isHttps) { - this.provider = provider; - this.accessKey = accessKey; - this.host = host; - this.port = port; - this.isHttps = isHttps; - this.separator = DEFAULT_SEPARATOR; - duplicatedSeparator = DEFAULT_SEPARATOR + DEFAULT_SEPARATOR; - - String hostname = (port != null ? host+":"+port : host); - String protocol = (isHttps ? "https" : "http"); - this.client = S3Client.builder().endpointOverride(URI.create(protocol + "://" + hostname)) - .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) - .build(); + duplicatedSeparator = separator.concat(separator); } /** @@ -149,15 +118,15 @@ final class ClientFileSystem extends FileSystem { * * @param provider the provider creating this file system. * @param region the AWS region, or {@code null} for default. - * @param host the host or {@code null} for aws request - * @param port the port or {@code null} for aws request - * @param isHttps the protocol is secure or not or {@code null} for aws request + * @param host the host or {@code null} for AWS request. + * @param port the port or {@code -1} for AWS request. + * @param isHttps the protocol is secure or not or {@code null} for AWS request. * @param accessKey the AWS S3 access key for this file system. * @param secret the password. * @param separator the separator in paths, or {@code null} for the default value. */ - ClientFileSystem(final FileService provider, final Region region, final String host, final Integer port, - final Boolean isHttps, final String accessKey, final String secret, + ClientFileSystem(final FileService provider, final Region region, final String host, final int port, + final boolean isHttps, final String accessKey, final String secret, String separator) { if (separator == null) { @@ -166,16 +135,18 @@ final class ClientFileSystem extends FileSystem { ArgumentChecks.ensureNonEmpty("separator", separator); this.provider = provider; this.accessKey = accessKey; - S3ClientBuilder builder = S3Client.builder().credentialsProvider( - StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secret))); + S3ClientBuilder builder = S3Client.builder(); + if (secret != null) { + builder = builder.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secret))); + } if (region != null) { builder = builder.region(region); } this.host = host; this.port = port; - this.isHttps = (isHttps != null ? isHttps : true); + this.isHttps = isHttps; if (host != null) { - String hostname = (port != null ? host+":"+port : host); + String hostname = (port < 0 ? host + ':' + port : host); String protocol = (this.isHttps ? "https" : "http"); builder = builder.endpointOverride(URI.create(protocol + "://" + hostname)) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()); diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java index 13a28efa0fc..c5a9a522989 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java @@ -1,90 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.sis.cloud.aws.s3; import java.util.Objects; /** - * File System Key stored in {@link FileService#fileSystems}, + * File System Key stored in {@link FileService#fileSystems}. * * @author Quentin Bialota (Geomatys) */ -public class ClientFileSystemKey { - +final class ClientFileSystemKey { /** - * The S3 access key + * The S3 access key. */ - private final String accessKey; + final String accessKey; /** - * The S3 host (if not stored on Amazon AWS Infrastructure) + * The S3 host (if not stored on Amazon AWS Infrastructure). */ - private final String host; + final String host; /** - * The S3 port (if not stored on Amazon AWS Infrastructure) + * The S3 port (if not stored on Amazon AWS Infrastructure). */ - private final Integer port; + final int port; /** - * Is the S3 HTTP Protocol secure (if not stored on Amazon AWS Infrastructure) + * Is the S3 HTTP Protocol secure (if not stored on Amazon AWS Infrastructure). */ - private final boolean isHttps; + final boolean isHttps; /** - * Creates a new file system key for the {@link FileService} with access key, host, port and protocol (secure or not secure) + * Creates a new file system key for the {@code FileService} with access key, host, port and protocol (secure or not secure). * * @param accessKey the S3 access key for this file system. - * @param host the host or {@code null} for aws request - * @param port the port or {@code -1} for aws request + * @param host the host or {@code null} for AWS request. + * @param port the port or {@code -1} for AWS request. * @param isHttps the protocol is secure or not */ - public ClientFileSystemKey(String accessKey, String host, Integer port, boolean isHttps) { + public ClientFileSystemKey(String accessKey, String host, int port, boolean isHttps) { this.accessKey = accessKey; this.host = host; this.port = port; this.isHttps = isHttps; } - /** - * Creates a new file system key for the {@link FileService} with access key, host and port - * (protocol used defined as secure (HTTPS)) - * - * @param accessKey the S3 access key for this file system. - * @param host the host or {@code null} for aws request - * @param port the port or {@code -1} for aws request - */ - public ClientFileSystemKey(String accessKey, String host, Integer port) { - this.accessKey = accessKey; - this.host = host; - this.port = port; - this.isHttps = true; - } - - /** - * Returns the access key as a string - */ - public String getAccessKey() { - return accessKey; - } - - /** - * Returns the host as a string - */ - public String getHost() { - return host; - } - - /** - * Returns the port as an integer - */ - public Integer getPort() { - return port; - } - - /** - * Returns if true the protocol is secure - */ - public boolean isHttps() { return isHttps; } - /** * Indicates whether some other object is "equal to" this one. * @@ -94,7 +68,7 @@ public Integer getPort() { @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (!(o instanceof ClientFileSystemKey)) return false; ClientFileSystemKey that = (ClientFileSystemKey) o; return Objects.equals(accessKey, that.accessKey) && Objects.equals(host, that.host) && Objects.equals(port, that.port); } diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java index 11547c663d9..6a9554ad71d 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java @@ -64,8 +64,8 @@ * This provider accepts URIs of the following forms: * *
    - *
  • {@code S3://host:port/bucket/file}
    • *
  • {@code S3://bucket/file}
    • + *
  • {@code S3://host:port/bucket/file}
    • *
  • {@code S3://accessKey@bucket/file} (password not allowed)
    • *
  • {@code S3://accessKey@host:port/bucket/key} (password not allowed)
    • *
@@ -77,7 +77,7 @@ * instead of the data to access, and can be a global configuration for the server. * * @author Martin Desruisseaux (Geomatys) - * @author Quentin Bialota (Geomatys) + * @author Quentin Bialota (Geomatys) * @version 1.5 * @since 1.2 */ @@ -101,7 +101,7 @@ public class FileService extends FileSystemProvider { * when the user did not specified explicitly a host. * In such case, the default host is the amazon host and is defined with the region. * - * Host can also be defined with : + * Host can also be defined with: *
    *
  • {@code s3.hostUrl} Java system properties.
    • *
@@ -118,7 +118,7 @@ public class FileService extends FileSystemProvider { *
  • {@code s3.port} Java system properties.
    • * */ - private static final Integer DEFAULT_PORT_KEY = null; + private static final int DEFAULT_PORT_KEY = -1; /** * A boolean used as part of the key in the {@link #fileSystems} map @@ -147,7 +147,7 @@ public class FileService extends FileSystemProvider { public static final String AWS_SECRET_ACCESS_KEY = "aws.secretAccessKey"; /** - * The property for the host (mandatory if you do not want to use AWS S3). + * The property for the host (mandatory if not using AWS S3). * Values shall be instances of {@link String}. * * @see #newFileSystem(URI, Map) @@ -155,7 +155,7 @@ public class FileService extends FileSystemProvider { public static final String S3_HOST_URL = "s3.hostUrl"; /** - * The property for the port (mandatory if you do not want to use AWS S3). + * The property for the port (mandatory if not using AWS S3). * Values shall be instances of {@link Integer}. * * @see #newFileSystem(URI, Map) @@ -163,7 +163,7 @@ public class FileService extends FileSystemProvider { public static final String S3_PORT = "s3.port"; /** - * The property for the protocol (optional even if you do not want to use AWS S3). + * The property for the protocol (optional). * Values shall be instances of {@link Boolean}. * Default value : True (HTTPS) * @@ -270,7 +270,7 @@ private String getAccessKey(final URI uri) { public FileSystem newFileSystem(final URI uri, final Map properties) throws IOException { final String accessKey = getAccessKey(uri); String host = uri.getHost(); - Integer port = uri.getPort(); + int port = uri.getPort(); final String secret; if (accessKey == null || (secret = Containers.property(properties, AWS_SECRET_ACCESS_KEY, String.class)) == null) { throw new IllegalArgumentException(Resources.format(Resources.Keys.MissingAccessKey_2, (accessKey == null) ? 0 : 1, uri)); @@ -289,7 +289,7 @@ final class Creator implements Function { /** Invoked if the map does not already contains the file system. */ @Override public ClientFileSystem apply(final ClientFileSystemKey key) { created = true; - return new ClientFileSystem(FileService.this, region, key.getHost(), key.getPort(), key.isHttps(), key.getAccessKey(), secret, separator); + return new ClientFileSystem(FileService.this, region, key.host, key.port, key.isHttps, key.accessKey, secret, separator); } } @@ -298,14 +298,16 @@ final class Creator implements Function { isHttps = DEFAULT_IS_HTTPS; } - // In case of Self-Hosted S3, if host and port are not found in the URI - // We check in java properties - // Else we use Default values (=> use AWS S3) - if (port == -1) { + /* + * In case of Self-Hosted S3, if host and port are not found in the URI + * We check in java properties + * Else we use Default values (=> use AWS S3) + */ + if (port < 0) { if ((host = Containers.property(properties, S3_HOST_URL, String.class)) != null) { // In case of Self-Hosted S3, if port is not found in the URI, but a host is defined - if ((port = Containers.property(properties, S3_PORT, Integer.class)) == null) { + if ((port = Containers.property(properties, S3_PORT, int.class)) < -1) { if (isHttps) { port = 443; // Default HTTPS port } else { @@ -344,43 +346,46 @@ final void dispose(ClientFileSystemKey identifier) { * @throws SdkException if the file system cannot be created. */ private ClientFileSystem getDefaultFileSystem() { - return fileSystems.computeIfAbsent(new ClientFileSystemKey(DEFAULT_ACCESS_KEY, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS), (key) -> new ClientFileSystem(this, S3Client.create())); + return fileSystems.computeIfAbsent(new ClientFileSystemKey(DEFAULT_ACCESS_KEY, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS), (key) -> + new ClientFileSystem(this, S3Client.create(), null)); } /** * Returns the file system associated to the {@link #DEFAULT_HOST_KEY} and {@link #DEFAULT_PORT_KEY}. - * @param accessKey the access key * + * @param accessKey the access key * @throws SdkException if the file system cannot be created. */ private ClientFileSystem getDefaultFileSystem(String accessKey) { return fileSystems.computeIfAbsent( - new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY), (key) -> new ClientFileSystem(this, S3Client.create(), key.getAccessKey())); + new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS), (key) -> new ClientFileSystem(this, S3Client.create(), key.accessKey)); } /** * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}. + * * @param host the host * @param port the port - * * @throws SdkException if the file system cannot be created. */ private ClientFileSystem getDefaultFileSystem(String host, Integer port) { return fileSystems.computeIfAbsent( - new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port), (key) -> new ClientFileSystem(this, key.getAccessKey(), key.getHost(), key.getPort(), key.isHttps())); + new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, DEFAULT_IS_HTTPS), (key) -> + new ClientFileSystem(this, null, key.host, key.port, key.isHttps, key.accessKey, null, null)); } /** * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}. + * * @param host the host * @param port the port * @param isHttps the protocol - * * @throws SdkException if the file system cannot be created. */ private ClientFileSystem getDefaultFileSystem(String host, Integer port, boolean isHttps) { return fileSystems.computeIfAbsent( - new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, isHttps), (key) -> new ClientFileSystem(this, key.getAccessKey(), key.getHost(), key.getPort(), key.isHttps())); + new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, isHttps), (key) -> + new ClientFileSystem(this, null, key.host, key.port, key.isHttps, key.accessKey, null, null)); } /** @@ -399,20 +404,18 @@ public FileSystem getFileSystem(final URI uri) { final String host = uri.getHost(); final int port = uri.getPort(); - //No access key, but host and port are defined => Self Hosted if (accessKey == null && port > -1) { + // No access key, but host and port are defined => Self Hosted return getDefaultFileSystem(host, port); - - //No access key, no host, no port => AWS S3 - } else if (accessKey == null && port == -1) { + } else if (accessKey == null && port < 0) { + // No access key, no host, no port => AWS S3 return getDefaultFileSystem(); - - //Access key, no host, no port => AWS S3 - } else if(accessKey != null && port == -1) { + } else if(accessKey != null && port < 0) { + // Access key, no host, no port => AWS S3 return getDefaultFileSystem(accessKey); } - final ClientFileSystem fs = fileSystems.get(new ClientFileSystemKey(accessKey, host, port)); + final ClientFileSystem fs = fileSystems.get(new ClientFileSystemKey(accessKey, host, port, DEFAULT_IS_HTTPS)); if (fs != null) { return fs; } @@ -424,7 +427,8 @@ public FileSystem getFileSystem(final URI uri) { * The resulting {@code Path} is associated with a {@link FileSystem} * that already exists or is constructed automatically. * - * @param uri a URI of the form {@code "s3://accessKey@bucket/file"}. + * @param uri a URI of the form {@code "s3://accessKey@bucket/file"} (AWS S3) + * or {@code s3://accessKey@host:port/bucket/file} (Self-hosted S3) (in this second case, host AND port are mandatory). * @return the resulting {@code Path}. * @throws IllegalArgumentException if the URI is not supported by this provider. * @throws FileSystemNotFoundException if the file system does not exist and cannot be created automatically. @@ -448,37 +452,54 @@ public Path getPath(final URI uri) { } final ClientFileSystem fs; - if (accessKey == null && port == -1) { - fs = getDefaultFileSystem(); - } else if (accessKey == null && port > -1) { - fs = getDefaultFileSystem(host, port); + if (accessKey == null) { + if (port < 0) { + fs = getDefaultFileSystem(); + } else { + fs = getDefaultFileSystem(host, port); + } } else { // TODO: we may need a way to get password here. + // TODO: we may need a way to get SSL status here (is HTTPS or not) ClientFileSystemKey fsKey; - if (port == -1) { - fsKey = new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY); + if (port < 0) { + fsKey = new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS); } else { - fsKey = new ClientFileSystemKey(accessKey, host, port); + fsKey = new ClientFileSystemKey(accessKey, host, port, DEFAULT_IS_HTTPS); } fs = fileSystems.computeIfAbsent(fsKey, (key) -> - new ClientFileSystem(FileService.this, null, key.getHost(), key.getPort(), key.isHttps(), key.getAccessKey(), null, null)); + new ClientFileSystem(FileService.this, null, key.host, key.port, key.isHttps, key.accessKey, null, null)); } String path = uri.getPath(); + /* + * In case of custom host, bucket name will be the first element of the "uri.getPath()" + * We need to get the first element of this path (this will be the bucket), and the second part will be + */ if (fs.host != null) { if (!(fs.host.equalsIgnoreCase(DEFAULT_HOST_KEY))) { - path = path.substring(1); + if (path.startsWith("/")) { + path = path.substring(1); + } String[] parts = path.split("/", 2); if (parts.length >= 2) { + // Bucket + Path specified (=> /bucket/path/to/folder) host = parts[0]; path = "/" + parts[1]; + } else if (parts.length == 1) { + // Bucket specified (no path) (=> /bucket) + host = parts[0]; + path = null; } } } - //host in this part is the S3 bucket name + /* + * - "host" in this part is the S3 bucket name + * - "path" is the path in this bucket + */ return new KeyPath(fs, host, (path != null) ? new String[] {path} : CharSequences.EMPTY_ARRAY, true); } diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java index 2789a14935d..4dea4e9f1ca 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java @@ -113,6 +113,13 @@ final class KeyPath implements Path { */ private S3Object objectMetadata; + /** + * Pattern used by KeyPath constructor to retrieve elements from S3 URI structure + * + * @see KeyPath(ClientFileSystem, String, String[], boolean) + */ + private static final Pattern pattern = Pattern.compile("^S3://[^@]+@([^/]+)(/.*)?$"); + /** * Creates an absolute path for a root defined by a S3 bucket. * This is used when listing the roots in a file system. @@ -161,7 +168,7 @@ final class KeyPath implements Path { bucketMetadata = root.bucketMetadata; this.bucket = root.bucket; this.fs = root.fs; - this.key = reformatPath(key); + this.key = key; this.isDirectory = isDirectory; assert key == null || !key.isEmpty(); // Do not copy `objectMetadata` because it is not for the same object. @@ -178,7 +185,7 @@ final class KeyPath implements Path { KeyPath(final ClientFileSystem fs, final String key, final boolean isDirectory) { this.fs = fs; this.bucket = null; - this.key = reformatPath(key); + this.key = key; this.isDirectory = isDirectory; assert !key.isEmpty(); } @@ -203,7 +210,6 @@ final class KeyPath implements Path { * @throws InvalidPathException if the path uses a protocol other than {@value #SCHEME}. */ KeyPath(final ClientFileSystem fs, String first, final String[] more, boolean isAbsolute) { - Pattern pattern = Pattern.compile("^S3://[^@]+@([^/]+)(/.*)?$"); Matcher matcher = pattern.matcher(first); if (matcher.matches()) { first = "S3:/" + matcher.group(2); @@ -274,8 +280,7 @@ final class KeyPath implements Path { throw emptyPath(first, 0); } isDirectory = path.endsWith(fs.separator); - String key = isDirectory ? path.substring(0, end) : path; - this.key = key.replace("./","/"); + this.key = isDirectory ? path.substring(0, end) : path; return; } } @@ -323,8 +328,7 @@ final class KeyPath implements Path { } buffer.setLength(i); } - String key = buffer.toString(); - this.key = reformatPath(key); + this.key = buffer.toString(); return; } } @@ -335,16 +339,6 @@ final class KeyPath implements Path { key = null; } - /** - * Reformat path, replacing "./" by "/" - */ - private static String reformatPath(String path) { - if(path != null) { - path = path.replace("./","/"); - } - return path; - } - /** * Returns the exception to throw for an empty path. */ @@ -726,17 +720,19 @@ public URI toUri() { } try { - //Case : s3://accessKey@host:port/bucket/key (self-hosted path) - if (fs != null && fs.host != null && fs.port != null) { - return new URI(SCHEME, fs.accessKey, fs.host, fs.port, "/"+bucket+path, null, null); - } + if (fs != null) { + // Case : s3://accessKey@host:port/bucket/key (self-hosted path) + if (fs.host != null && fs.port < 0) { + return new URI(SCHEME, fs.accessKey, fs.host, fs.port, "/" + bucket + (path != null ? path : ""), null, null); + } - //Case : s3://accessKey@bucket/key (aws path) - return new URI(SCHEME, fs.accessKey, bucket, -1, path, null, null); + // Case : s3://accessKey@bucket/key (AWS path) + return new URI(SCHEME, fs.accessKey, bucket, -1, (path != null ? path : ""), null, null); + } + throw new IllegalStateException("No filesystem associated with this path."); } catch (URISyntaxException e) { throw new IllegalStateException(e.getMessage(), e); } - } /** @@ -754,8 +750,8 @@ public String toString() { sb.append(fs.accessKey).append('@'); } - if(fs.host != null) { - if(fs.port != null) { + if (fs.host != null) { + if(fs.port < 0) { sb.append(fs.host).append(":").append(fs.port).append("/"); } else { sb.append(fs.host).append("/"); @@ -771,7 +767,6 @@ public String toString() { sb.append(key); } return sb.toString(); - } /** diff --git a/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/ClientFileSystemTest.java b/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/ClientFileSystemTest.java index bad2ad252e4..bb0374e5b9a 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/ClientFileSystemTest.java +++ b/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/ClientFileSystemTest.java @@ -37,7 +37,7 @@ public final class ClientFileSystemTest extends TestCase { * Returns the file system to use for testing purpose. */ static ClientFileSystem create() { - return new ClientFileSystem(new FileService(), null); + return new ClientFileSystem(new FileService(), null, null); } /** From cadbada0193d604306fa7777d2a495a811dfd3a2 Mon Sep 17 00:00:00 2001 From: qbialota Date: Fri, 20 Feb 2026 15:51:47 +0100 Subject: [PATCH 4/4] feat/CustomHostS3 : remove unused case, update doc, add URISyntaxException management --- .../main/module-info.java | 3 +- .../sis/cloud/aws/s3/ClientFileSystem.java | 10 +- .../sis/cloud/aws/s3/ClientFileSystemKey.java | 2 +- .../apache/sis/cloud/aws/s3/FileService.java | 102 +++++++++++++----- .../org/apache/sis/cloud/aws/s3/KeyPath.java | 22 +--- .../apache/sis/cloud/aws/s3/package-info.java | 3 +- 6 files changed, 92 insertions(+), 50 deletions(-) diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/module-info.java b/endorsed/src/org.apache.sis.cloud.aws/main/module-info.java index 5ab60c1c7a7..7fc1f857a0b 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/module-info.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/module-info.java @@ -19,7 +19,8 @@ * Java NIO wrappers for Amazon Simple Storage Service (S3). * * @author Martin Desruisseaux (Geomatys) - * @version 1.4 + * @author Quentin Bialota (Geomatys) + * @version 1.7 * @since 1.2 */ module org.apache.sis.cloud.aws { diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java index e9de6a45b99..28cc3e8c6c5 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java @@ -17,6 +17,7 @@ package org.apache.sis.cloud.aws.s3; import java.net.URI; +import java.net.URISyntaxException; import java.util.Set; import java.util.Collections; import java.util.regex.PatternSyntaxException; @@ -68,12 +69,13 @@ final class ClientFileSystem extends FileSystem { final String host; /** - * The S3 port (if not stored on Amazon Infrastructure), or {@code null} if none. + * The S3 port (if not stored on Amazon Infrastructure), or {@code -1} if none. */ final int port; /** - * Is the S3 HTTP Protocol secure (if not stored on Amazon Infrastructure) (default {@code true)}. + * Whether the S3 HTTP Protocol is secure (if this information is not stored on Amazon Infrastructure). + * Default is {@code true}. */ final boolean isHttps; @@ -127,7 +129,7 @@ final class ClientFileSystem extends FileSystem { */ ClientFileSystem(final FileService provider, final Region region, final String host, final int port, final boolean isHttps, final String accessKey, final String secret, - String separator) + String separator) throws URISyntaxException { if (separator == null) { separator = DEFAULT_SEPARATOR; @@ -148,7 +150,7 @@ final class ClientFileSystem extends FileSystem { if (host != null) { String hostname = (port < 0 ? host + ':' + port : host); String protocol = (this.isHttps ? "https" : "http"); - builder = builder.endpointOverride(URI.create(protocol + "://" + hostname)) + builder = builder.endpointOverride(new URI(protocol + "://" + hostname)) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()); } client = builder.build(); diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java index c5a9a522989..46207025853 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java @@ -70,7 +70,7 @@ public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof ClientFileSystemKey)) return false; ClientFileSystemKey that = (ClientFileSystemKey) o; - return Objects.equals(accessKey, that.accessKey) && Objects.equals(host, that.host) && Objects.equals(port, that.port); + return Objects.equals(accessKey, that.accessKey) && Objects.equals(host, that.host) && port == that.port && isHttps == that.isHttps; } /** diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java index 6a9554ad71d..4d137ea6595 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java @@ -16,6 +16,7 @@ */ package org.apache.sis.cloud.aws.s3; +import java.net.URISyntaxException; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -78,7 +79,7 @@ * * @author Martin Desruisseaux (Geomatys) * @author Quentin Bialota (Geomatys) - * @version 1.5 + * @version 1.7 * @since 1.2 */ public class FileService extends FileSystemProvider { @@ -99,11 +100,11 @@ public class FileService extends FileSystemProvider { /** * An arbitrary string used as part of the key in the {@link #fileSystems} map * when the user did not specified explicitly a host. - * In such case, the default host is the amazon host and is defined with the region. + * In such case, the default host is the Amazon host and is defined with the region. * * Host can also be defined with: *
      - *
    • {@code s3.hostUrl} Java system properties.
      • + *
    • {@code #S3_HOST_URL} Java system properties.
      • *
    */ private static final String DEFAULT_HOST_KEY = null; @@ -115,7 +116,7 @@ public class FileService extends FileSystemProvider { * * Port can also be defined with : *
      - *
    • {@code s3.port} Java system properties.
      • + *
    • {@code #S3_PORT} Java system properties.
      • *
    */ private static final int DEFAULT_PORT_KEY = -1; @@ -127,7 +128,7 @@ public class FileService extends FileSystemProvider { * * Port can also be defined with : *
      - *
    • {@code s3.isHttps} Java system properties.
      • + *
    • {@code #S3_IS_HTTPS} Java system properties.
      • *
    */ private static final boolean DEFAULT_IS_HTTPS = true; @@ -151,16 +152,18 @@ public class FileService extends FileSystemProvider { * Values shall be instances of {@link String}. * * @see #newFileSystem(URI, Map) + * @since 1.7 */ - public static final String S3_HOST_URL = "s3.hostUrl"; + public static final String S3_HOST_URL = "org.apache.sis.s3.hostURL"; /** * The property for the port (mandatory if not using AWS S3). * Values shall be instances of {@link Integer}. * * @see #newFileSystem(URI, Map) + * @since 1.7 */ - public static final String S3_PORT = "s3.port"; + public static final String S3_PORT = "org.apache.sis.s3.port"; /** * The property for the protocol (optional). @@ -168,8 +171,9 @@ public class FileService extends FileSystemProvider { * Default value : True (HTTPS) * * @see #newFileSystem(URI, Map) + * @since 1.7 */ - public static final String S3_IS_HTTPS = "s3.isHttps"; + public static final String S3_IS_HTTPS = "org.apache.sis.s3.isHttps"; /** * The property for the secret access key (password). @@ -283,13 +287,21 @@ public FileSystem newFileSystem(final URI uri, final Map properties) t } else { region = Containers.property(properties, AWS_REGION, Region.class); } + final class Creator implements Function { /** Identifies if a new file system is created. */ boolean created; + /** Store URI Syntax exception. */ URISyntaxException exception; - /** Invoked if the map does not already contains the file system. */ - @Override public ClientFileSystem apply(final ClientFileSystemKey key) { + @Override + public ClientFileSystem apply(final ClientFileSystemKey key) { created = true; - return new ClientFileSystem(FileService.this, region, key.host, key.port, key.isHttps, key.accessKey, secret, separator); + try { + return new ClientFileSystem(FileService.this, region, key.host, key.port, key.isHttps, key.accessKey, secret, separator); + } catch (URISyntaxException ex) { + created = false; + exception = ex; + return null; // Nothing added to the map + } } } @@ -325,6 +337,8 @@ final class Creator implements Function { final ClientFileSystem fs = fileSystems.computeIfAbsent(new ClientFileSystemKey(accessKey, host, port, isHttps), c); if (c.created) { return fs; + } else if (c.exception != null) { + throw new IllegalArgumentException("Invalid URI: " + uri, c.exception); } throw new FileSystemAlreadyExistsException(Resources.format(Resources.Keys.FileSystemInitialized_2, 1, accessKey)); } @@ -358,7 +372,9 @@ private ClientFileSystem getDefaultFileSystem() { */ private ClientFileSystem getDefaultFileSystem(String accessKey) { return fileSystems.computeIfAbsent( - new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS), (key) -> new ClientFileSystem(this, S3Client.create(), key.accessKey)); + new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS), + (key) -> + new ClientFileSystem(this, S3Client.create(), key.accessKey)); } /** @@ -367,11 +383,18 @@ private ClientFileSystem getDefaultFileSystem(String accessKey) { * @param host the host * @param port the port * @throws SdkException if the file system cannot be created. + * @throws URISyntaxException if the URI is not valid. */ - private ClientFileSystem getDefaultFileSystem(String host, Integer port) { - return fileSystems.computeIfAbsent( - new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, DEFAULT_IS_HTTPS), (key) -> - new ClientFileSystem(this, null, key.host, key.port, key.isHttps, key.accessKey, null, null)); + private ClientFileSystem getDefaultFileSystem(String host, Integer port) throws URISyntaxException { + ClientFileSystemKey key = new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, DEFAULT_IS_HTTPS); + + synchronized (fileSystems) { + ClientFileSystem fs = fileSystems.get(key); + if (fs != null) return fs; + fs = new ClientFileSystem(this, null, key.host, key.port, key.isHttps, key.accessKey, null, null); + fileSystems.put(key, fs); + return fs; + } } /** @@ -381,11 +404,19 @@ private ClientFileSystem getDefaultFileSystem(String host, Integer port) { * @param port the port * @param isHttps the protocol * @throws SdkException if the file system cannot be created. + * @throws URISyntaxException if the URI is not valid. */ - private ClientFileSystem getDefaultFileSystem(String host, Integer port, boolean isHttps) { - return fileSystems.computeIfAbsent( - new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, isHttps), (key) -> - new ClientFileSystem(this, null, key.host, key.port, key.isHttps, key.accessKey, null, null)); + private ClientFileSystem getDefaultFileSystem(String host, Integer port, boolean isHttps) throws URISyntaxException { + ClientFileSystemKey key = new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, isHttps); + + // 1. Try to get the existing one first (no locking) + synchronized (fileSystems) { + ClientFileSystem fs = fileSystems.get(key); + if (fs != null) return fs; + fs = new ClientFileSystem(this, null, key.host, key.port, key.isHttps, key.accessKey, null, null); + fileSystems.put(key, fs); + return fs; + } } /** @@ -406,11 +437,15 @@ public FileSystem getFileSystem(final URI uri) { if (accessKey == null && port > -1) { // No access key, but host and port are defined => Self Hosted - return getDefaultFileSystem(host, port); + try { + return getDefaultFileSystem(host, port); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("Invalid URI: " + uri, ex); + } } else if (accessKey == null && port < 0) { // No access key, no host, no port => AWS S3 return getDefaultFileSystem(); - } else if(accessKey != null && port < 0) { + } else if (accessKey != null && port < 0) { // Access key, no host, no port => AWS S3 return getDefaultFileSystem(accessKey); } @@ -451,12 +486,16 @@ public Path getPath(final URI uri) { throw new IllegalArgumentException(Resources.format(Resources.Keys.InvalidBucketName_1, host)); } - final ClientFileSystem fs; + ClientFileSystem fs; if (accessKey == null) { if (port < 0) { fs = getDefaultFileSystem(); } else { - fs = getDefaultFileSystem(host, port); + try { + fs = getDefaultFileSystem(host, port); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("Invalid URI: " + uri, ex); + } } } else { // TODO: we may need a way to get password here. @@ -468,8 +507,19 @@ public Path getPath(final URI uri) { } else { fsKey = new ClientFileSystemKey(accessKey, host, port, DEFAULT_IS_HTTPS); } - fs = fileSystems.computeIfAbsent(fsKey, (key) -> - new ClientFileSystem(FileService.this, null, key.host, key.port, key.isHttps, key.accessKey, null, null)); + + // Compute if absent + try { + synchronized (fileSystems) { + fs = fileSystems.get(fsKey); + if (fs == null) { + fs = new ClientFileSystem(this, null, fsKey.host, fsKey.port, fsKey.isHttps, fsKey.accessKey, null, null); + fileSystems.put(fsKey, fs); + } + } + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("Invalid URI: " + uri, ex); + } } String path = uri.getPath(); diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java index 4dea4e9f1ca..608c04618ff 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java @@ -113,13 +113,6 @@ final class KeyPath implements Path { */ private S3Object objectMetadata; - /** - * Pattern used by KeyPath constructor to retrieve elements from S3 URI structure - * - * @see KeyPath(ClientFileSystem, String, String[], boolean) - */ - private static final Pattern pattern = Pattern.compile("^S3://[^@]+@([^/]+)(/.*)?$"); - /** * Creates an absolute path for a root defined by a S3 bucket. * This is used when listing the roots in a file system. @@ -210,11 +203,6 @@ final class KeyPath implements Path { * @throws InvalidPathException if the path uses a protocol other than {@value #SCHEME}. */ KeyPath(final ClientFileSystem fs, String first, final String[] more, boolean isAbsolute) { - Matcher matcher = pattern.matcher(first); - if (matcher.matches()) { - first = "S3:/" + matcher.group(2); - } - /* * Verify if the path start with "S3://" or "/" prefix. In both cases the path is considered absolute * and the prefix is skipped. The `start` variable is the index of the first character after prefix, @@ -280,7 +268,7 @@ final class KeyPath implements Path { throw emptyPath(first, 0); } isDirectory = path.endsWith(fs.separator); - this.key = isDirectory ? path.substring(0, end) : path; + key = isDirectory ? path.substring(0, end) : path; return; } } @@ -751,11 +739,11 @@ public String toString() { } if (fs.host != null) { - if(fs.port < 0) { - sb.append(fs.host).append(":").append(fs.port).append("/"); - } else { - sb.append(fs.host).append("/"); + sb.append(fs.host); + if (fs.port < 0) { + sb.append(':').append(fs.port); } + sb.append('/'); } sb.append(bucket); diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/package-info.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/package-info.java index caf3fc4632e..695d6fe1442 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/package-info.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/package-info.java @@ -54,7 +54,8 @@ * All classes provided by this package are safe of usage in multi-threading environment. * * @author Martin Desruisseaux (Geomatys) - * @version 1.5 + * @author Quentin Bialota (Geomatys) + * @version 1.7 * * @see AWS SDK for Java *