S3 fixes for non AWS instances

doc/release-notes/12454-S3Fixes.md

@@ -0,0 +1,5 @@
+### Support for Backblaze B2 as an S3 store, improved support for storJ
+
+An improvement to the .disable-tagging=true support for S3 stores now allows use of BackBlaze B2 as an S3 implementation (and may help other stores that do not handle tagging).
+
+The /api/datasets/<id>/cleanStorage endpoint will now work for datasets with more than 1000 files when storJ is used as the S3 store.

doc/sphinx-guides/source/installation/config.rst

@@ -1479,6 +1479,12 @@ You may provide the values for these via any `supported MicroProfile Config API
 Reported Working S3-Compatible Storage
 ######################################
 
+
+`BackBlaze B2 <https://www.backblaze.com/cloud-storage>`_
+  (As of 2026-06-11)
+  Set ``dataverse.files.<id>.disable-tagging=true``, as B2 does not support tagging (and will fail without this setting).
+  Tested with ``.path-style-access=true``, ``.download-redirect=true``, and ``.upload-redirect=true``.
+
 `StorJ Object Store <https://www.storj.io>`_
  StorJ is a distributed object store that can be configured with an S3 gateway. Per the S3 Storage instructions above, you'll first set up the StorJ S3 store by defining the id, type, and label. After following the general installation, set the following configuration to use a StorJ object store: ``dataverse.files.<id>.chunked-encoding=false``. For step-by-step instructions see https://docs.storj.io/dcs/how-tos/dataverse-integration-guide/
 

src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java

@@ -77,23 +77,17 @@ public void copyFromS3(Dataset dataset, String s3ImportPath) throws IOException
         ListObjectsV2Request listReq = ListObjectsV2Request.builder()
                 .bucket(dcmBucketName)
                 .prefix(dcmDatasetKey)
+                .maxKeys(1000)
                 .build();
 
-        ListObjectsV2Response listRes;
+        List<S3Object> storedDcmDatasetFilesSummary = new ArrayList<>();
         try {
-            listRes = s3.listObjectsV2(listReq);
+            s3.listObjectsV2Paginator(listReq).stream()
+                .flatMap(r -> r.contents().stream())
+                .forEach(storedDcmDatasetFilesSummary::add);
         } catch (S3Exception se) {
             logger.info("Caught an S3Exception in s3ImportUtil: " + se.getMessage());
-            throw new IOException("S3 listAuxObjects: failed to get a listing for " + dcmDatasetKey);
-        }
-
-        List<S3Object> storedDcmDatasetFilesSummary = new ArrayList<>(listRes.contents());
-
-        while (listRes.isTruncated()) {
-            logger.fine("S3 listAuxObjects: going to next page of list");
-            listReq = listReq.toBuilder().continuationToken(listRes.nextContinuationToken()).build();
-            listRes = s3.listObjectsV2(listReq);
-            storedDcmDatasetFilesSummary.addAll(listRes.contents());
+            throw new IOException("S3 listObjects: failed to get a listing for " + dcmDatasetKey);
         }
 
         for (S3Object item : storedDcmDatasetFilesSummary) {

src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java

@@ -697,56 +697,53 @@ private File createTempFile(Path path, InputStream inputStream) throws IOExcepti
         return targetFile;
     }
 
-    @Override
-    public List<String> listAuxObjects() throws IOException {
-        if (!this.canWrite()) {
-            open();
-        }
-        String prefix = getDestinationKey("");
-
-        List<String> ret = new ArrayList<>();
-        ListObjectsV2Request listObjectsReqManual = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
+    private List<S3Object> listObjects(String prefix, String methodName) throws IOException {
+        List<S3Object> objects = new ArrayList<>();
+        ListObjectsV2Request listRequest = ListObjectsV2Request.builder()
+                .bucket(bucketName)
+                .prefix(prefix)
+                .maxKeys(1000) // Required for storJ
                 .build();
 
-        ListObjectsV2Response listObjectsResponse = null;
         try {
-            listObjectsResponse = s3.listObjectsV2(listObjectsReqManual).get();
-        } catch (InterruptedException | ExecutionException e) {
-            throw new IOException("S3 listAuxObjects: failed to get a listing for " + prefix, e);
-        }
-
-        if (listObjectsResponse == null) {
-            return ret;
-        }
-
-        List<S3Object> storedAuxFilesSummary = new ArrayList<>(listObjectsResponse.contents());
-
-        try {
-            String nextContinuationToken = listObjectsResponse.nextContinuationToken();
-            while (nextContinuationToken != null) {
-                logger.fine("S3 listAuxObjects: going to next page of list");
-                ListObjectsV2Request nextReq = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
-                        .continuationToken(nextContinuationToken).build();
-
-                ListObjectsV2Response nextResponse = s3.listObjectsV2(nextReq).get();
-                if (nextResponse != null) {
-                    storedAuxFilesSummary.addAll(nextResponse.contents());
-                    nextContinuationToken = nextResponse.nextContinuationToken();
-                } else {
-                    nextContinuationToken = null;
+            ListObjectsV2Response listResponse;
+            String nextToken = null;
+            do {
+                ListObjectsV2Request.Builder reqBuilder = listRequest.toBuilder();
+                if (nextToken != null) {
+                    reqBuilder = reqBuilder.continuationToken(nextToken);
                 }
-            }
+                ListObjectsV2Request req = reqBuilder.build();
+                listResponse = s3.listObjectsV2(req).get();
+                objects.addAll(listResponse.contents());
+                nextToken = listResponse.nextContinuationToken();
+                if (listResponse.isTruncated() && nextToken == null) {
+                    logger.warning("S3 " + methodName + ": list is truncated but nextContinuationToken is null; stopping to avoid infinite loop");
+                    break;
+                }
+            } while (listResponse.isTruncated());
         } catch (InterruptedException | ExecutionException e) {
-            throw new IOException("S3AccessIO: Failed to get aux objects for listing.", e);
+            throw new IOException("S3AccessIO: Failed to get objects for listing in " + methodName + ".", e);
         }
+        return objects;
+    }
 
-        for (S3Object item : storedAuxFilesSummary) {
-            String destinationKey = item.key();
-            String fileName = destinationKey.substring(destinationKey.lastIndexOf(".") + 1);
-            logger.fine("S3 cached aux object fileName: " + fileName);
-            ret.add(fileName);
+    @Override
+    public List<String> listAuxObjects() throws IOException {
+        if (!this.canWrite()) {
+            open();
         }
-        return ret;
+        String prefix = getDestinationKey("");
+        List<S3Object> contents = listObjects(prefix, "listAuxObjects");
+
+        return contents.stream()
+                .map(item -> {
+                    String destinationKey = item.key();
+                    String fileName = destinationKey.substring(destinationKey.lastIndexOf(".") + 1);
+                    logger.fine("S3 cached aux object fileName: " + fileName);
+                    return fileName;
+                })
+                .collect(Collectors.toList());
     }
 
     @Override
@@ -774,22 +771,7 @@ public void deleteAllAuxObjects() throws IOException {
         }
 
         String prefix = getDestinationKey("");
-
-        List<S3Object> storedAuxFilesSummary = new ArrayList<>();
-        try {
-            ListObjectsV2Request listRequest = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix).build();
-
-            ListObjectsV2Response listResponse;
-            do {
-                listResponse = s3.listObjectsV2(listRequest).get();
-                storedAuxFilesSummary.addAll(listResponse.contents());
-
-                listRequest = listRequest.toBuilder().continuationToken(listResponse.nextContinuationToken()).build();
-            } while (listResponse.isTruncated());
-
-        } catch (InterruptedException | ExecutionException e) {
-            throw new IOException("S3AccessIO: Failed to get aux objects for listing to delete.", e);
-        }
+        List<S3Object> storedAuxFilesSummary = listObjects(prefix, "deleteAllAuxObjects");
 
         if (storedAuxFilesSummary.isEmpty()) {
             logger.fine("S3AccessIO: No auxiliary objects to delete.");
@@ -987,7 +969,7 @@ public boolean downloadRedirectEnabled(String auxObjectTag) {
      * @param auxiliaryFileName (optional) - file name, if different from the main
      *                          file label.
      * @return redirect url
-     * @throws IOException.
+     * @throws IOException
      */
     public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliaryType, String auxiliaryFileName)
             throws IOException {
@@ -1315,6 +1297,12 @@ private static AwsCredentialsProvider getCredentialsProvider(String driverId) {
     }
 
     public void removeTempTag() throws IOException {
+        final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId)
+                .orElse(false);
+        if (taggingDisabled) {
+            logger.fine("S3 tagging disabled for storage driver " + driverId + "; skipping temp tag removal.");
+            return;
+        }
         if (!(dvObject instanceof DataFile)) {
             logger.warning("Attempt to remove tag from non-file DVObject id: " + dvObject.getId());
             throw new IOException("Attempt to remove temp tag from non-file S3 Object");
@@ -1454,47 +1442,11 @@ private List<String> listAllFiles() throws IOException {
         }
         String prefix = dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/";
 
-        List<String> ret = new ArrayList<>();
-        ListObjectsV2Request listObjectsReqManual = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
-                .build();
-
-        ListObjectsV2Response listObjectsResponse = null;
-        try {
-            listObjectsResponse = s3.listObjectsV2(listObjectsReqManual).get();
-        } catch (InterruptedException | ExecutionException e) {
-            throw new IOException("S3 listObjects: failed to get a listing for " + prefix, e);
-        }
-
-        if (listObjectsResponse == null) {
-            return ret;
-        }
-
-        List<S3Object> storedFilesSummary = new ArrayList<>(listObjectsResponse.contents());
-
-        try {
-            String nextContinuationToken = listObjectsResponse.nextContinuationToken();
-            while (nextContinuationToken != null) {
-                logger.fine("S3 listObjects: going to next page of list");
-                ListObjectsV2Request nextReq = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix)
-                        .continuationToken(nextContinuationToken).build();
-
-                ListObjectsV2Response nextResponse = s3.listObjectsV2(nextReq).get();
-                if (nextResponse != null) {
-                    storedFilesSummary.addAll(nextResponse.contents());
-                    nextContinuationToken = nextResponse.nextContinuationToken();
-                } else {
-                    nextContinuationToken = null;
-                }
-            }
-        } catch (InterruptedException | ExecutionException e) {
-            throw new IOException("S3AccessIO: Failed to get objects for listing.", e);
-        }
+        List<S3Object> contents = listObjects(prefix, "listAllFiles");
 
-        for (S3Object item : storedFilesSummary) {
-            String fileName = item.key().substring(prefix.length());
-            ret.add(fileName);
-        }
-        return ret;
+        return contents.stream()
+                .map(item -> item.key().substring(prefix.length()))
+                .collect(Collectors.toList());
     }
 
     private void deleteFile(String fileName) throws IOException {