Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/release-notes/12925-s3-proxy-fix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A bug in S3 support for proxying (via the dataverse.files.{id}.proxy-url setting) was broken when also used with dataverse.files.{id}.path-style-access.
57 changes: 25 additions & 32 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,28 @@ public S3AccessIO(T dvObject, DataAccessRequest req, @NotNull S3AsyncClient s3cl
private String endpoint = null;
private String proxy = null;

/**
* Centralized helper to rewrite a presigned S3 URL to route through a proxy.
* Replaces the URL's host with the proxy's host and uses the proxy's port
* (if specified), while preserving the path and query parameters (which contain the signature).
*
* @param urlString the presigned URL string to rewrite
* @param endpoint the configured S3 endpoint (may include scheme and port, e.g. "https://s3.example.com:9000" or "s3.example.com")
* @param proxy the proxy URL to route through (e.g. "https://proxy.example.com")
* @return the rewritten URL string, or the original if no rewrite is needed
*/
static String rewriteUrlForProxy(String urlString, String endpoint, String proxy) {
if (StringUtil.isEmpty(proxy) || StringUtil.isEmpty(endpoint)) {
return urlString;
}
int endpointEnd = urlString.indexOf(endpoint);
if (endpointEnd != -1) {
return proxy + urlString.substring(endpointEnd + endpoint.length());
} else {
return urlString;
}
}

@Override
public void open(DataAccessOption... options) throws IOException {
if (s3ReadClient == null || s3WriteClient == null) {
Expand Down Expand Up @@ -1082,20 +1104,7 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary

if (presignedRequest != null) {
String urlString = presignedRequest.url().toString();
if (!StringUtil.isEmpty(proxy)) {
String endpointServer = endpoint;
int protocolEnd = endpoint.indexOf(DataAccess.SEPARATOR);
if (protocolEnd >= 0) {
endpointServer = endpoint.substring(protocolEnd + DataAccess.SEPARATOR.length());
}
logger.fine("Endpoint: " + endpointServer);
logger.fine("Original Url: " + urlString);
String finalUrl = urlString.replaceFirst("http[s]*:\\/\\/([^\\/]+\\.)" + endpointServer, proxy);
logger.fine("ProxiedURL: " + finalUrl);
return finalUrl;
} else {
return urlString;
}
return rewriteUrlForProxy(urlString, endpoint, proxy);
}

return null;
Expand Down Expand Up @@ -1137,21 +1146,7 @@ private String generateTemporaryS3UploadUrl(String key, Date expiration) throws
}

String urlString = presignedRequest.url().toString();

if (!StringUtil.isEmpty(proxy)) {
String endpointServer = endpoint;
int protocolEnd = endpoint.indexOf(DataAccess.SEPARATOR);
if (protocolEnd >= 0) {
endpointServer = endpoint.substring(protocolEnd + DataAccess.SEPARATOR.length());
}
logger.fine("Endpoint: " + endpointServer);
// We're then replacing
// http or https followed by :// and an optional <bucketname>. before the
// normalized endpoint url
// with the proxy info (which is protocol + machine name and optional port)
urlString = urlString.replaceFirst("http[s]*:\\/\\/([^\\/]+\\.)" + endpointServer, proxy);
logger.fine("ProxiedURL: " + urlString);
}
urlString = rewriteUrlForProxy(urlString, endpoint, proxy);

return urlString;
}
Expand Down Expand Up @@ -1183,9 +1178,7 @@ public JsonObjectBuilder generateTemporaryS3UploadUrls(String globalId, String s
.build());

String urlString = presignedRequest.url().toString();
if (!StringUtil.isEmpty(proxy)) {
urlString = urlString.replace(endpoint, proxy);
}
urlString = rewriteUrlForProxy(urlString, endpoint, proxy);
urls.add(Integer.toString(i), urlString);
}

Expand Down
159 changes: 159 additions & 0 deletions src/test/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIOTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -129,4 +129,163 @@ void testS3IdentifierFormats() throws IOException {
//bad bucket
assertFalse(DataAccess.isValidDirectStorageIdentifier("s3test://bucket:" + FileUtil.generateStorageIdentifier()));
}

// ---- Tests for rewriteUrlForProxy ----

@Test
void rewriteUrlForProxy_virtualHostedStyle_rewritesToProxy() {
String url = "https://mybucket.s3.example.com/path/to/key?signature=abc";
String endpoint = "s3.example.com";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals("https://proxy.example.com/path/to/key?signature=abc", result);
}

@Test
void rewriteUrlForProxy_pathStyle_rewritesToProxy() {
String url = "https://s3.example.com/mybucket/path/to/key?signature=abc";
String endpoint = "s3.example.com";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals("https://proxy.example.com/mybucket/path/to/key?signature=abc", result);
}

@Test
void rewriteUrlForProxy_endpointWithScheme_rewritesToProxy() {
String url = "https://s3.example.com/mybucket/path/to/key?signature=abc";
String endpoint = "https://s3.example.com";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals("https://proxy.example.com/mybucket/path/to/key?signature=abc", result);
}

@Test
void rewriteUrlForProxy_endpointWithPort_pathStyle_rewritesToProxy() {
String url = "https://s3.example.com:9000/mybucket/path/to/key?signature=abc";
String endpoint = "s3.example.com:9000";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals("https://proxy.example.com/mybucket/path/to/key?signature=abc", result);
}

@Test
void rewriteUrlForProxy_endpointWithPort_virtualHostedStyle_rewritesToProxy() {
String url = "https://mybucket.s3.example.com:9000/path/to/key?signature=abc";
String endpoint = "s3.example.com:9000";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals("https://proxy.example.com/path/to/key?signature=abc", result);
}

@Test
void rewriteUrlForProxy_endpointWithSchemeAndPort_rewritesToProxy() {
String url = "https://s3.example.com:9000/mybucket/path/to/key?signature=abc";
String endpoint = "https://s3.example.com:9000";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals("https://proxy.example.com/mybucket/path/to/key?signature=abc", result);
}

@Test
void rewriteUrlForProxy_noProxy_returnsOriginal() {
String url = "https://s3.example.com/mybucket/path/to/key?signature=abc";
String endpoint = "s3.example.com";
String proxy = "";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals(url, result);
}

@Test
void rewriteUrlForProxy_noEndpoint_returnsOriginal() {
String url = "https://s3.example.com/mybucket/path/to/key?signature=abc";
String endpoint = "";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals(url, result);
}

@Test
void rewriteUrlForProxy_unmatchedHost_returnsOriginal() {
String url = "https://other.example.com/mybucket/path/to/key?signature=abc";
String endpoint = "s3.example.com";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals(url, result);
}

@Test
void rewriteUrlForProxy_proxyWithPort_carriesProxyPort() {
// When the proxy has a port, it should appear in the result
String url = "https://s3.example.com/mybucket/path/to/key?signature=abc";
String endpoint = "s3.example.com";
String proxy = "https://proxy.example.com:8443";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals("https://proxy.example.com:8443/mybucket/path/to/key?signature=abc", result);
}

@Test
void rewriteUrlForProxy_proxyPortOverridesOriginalPort() {
// The proxy's port is used regardless of the original URL's port
String url = "https://s3.example.com:9000/mybucket/path/to/key?signature=abc";
String endpoint = "s3.example.com:9000";
String proxy = "https://proxy.example.com:8443";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertEquals("https://proxy.example.com:8443/mybucket/path/to/key?signature=abc", result);
}

@Test
void rewriteUrlForProxy_preservesQueryParameters() {
String url = "https://s3.example.com/bucket/key?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=test&X-Amz-Signature=abc123";
String endpoint = "s3.example.com";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

assertTrue(result.startsWith("https://proxy.example.com/bucket/key?"));
assertTrue(result.contains("X-Amz-Algorithm=AWS4-HMAC-SHA256"));
assertTrue(result.contains("X-Amz-Signature=abc123"));
}

@Test
void rewriteUrlForProxy_preservesPercentEncodedQueryExactly() {
// URI.getQuery() decodes percent-encoded sequences, which could alter
// cryptographic signatures. getRawQuery() must be used to preserve them exactly.
// The /%2F sequence should remain as %2F, not be decoded to /.
String url = "https://s3.example.com/bucket/path%2Fto%2Fkey?X-Amz-Credential=test%2F20240101%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=abc%2Bdef";
String endpoint = "s3.example.com";
String proxy = "https://proxy.example.com";

String result = S3AccessIO.rewriteUrlForProxy(url, endpoint, proxy);

// Verify the raw percent-encoded sequences are preserved exactly
assertTrue(result.contains("/path%2Fto%2Fkey"), "Path %2F should not be decoded");
assertTrue(result.contains("test%2F20240101%2Fus-east-1%2Fs3%2Faws4_request"), "Credential %2F should not be decoded");
assertTrue(result.contains("abc%2Bdef"), "Signature %2B should not be decoded");
// Verify the full query string is identical to the original
int originalQueryStart = url.indexOf('?');
int resultQueryStart = result.indexOf('?');
assertEquals(url.substring(originalQueryStart), result.substring(resultQueryStart), "Query string must be preserved exactly");
}
}
Loading