From 31b81b564bdec4f56cd5a63bff6648b2c7199b8a Mon Sep 17 00:00:00 2001 From: sbeimin Date: Mon, 26 Feb 2018 15:32:26 +0100 Subject: [PATCH 1/2] Restrict use of AmazonS3.listBuckets() #101 --- .../com/upplication/s3fs/S3FileChannel.java | 2 +- .../com/upplication/s3fs/S3FileStore.java | 35 ++++++++++++++----- .../s3fs/S3FileSystemProvider.java | 7 ++-- .../s3fs/S3SeekableByteChannel.java | 2 +- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/upplication/s3fs/S3FileChannel.java b/src/main/java/com/upplication/s3fs/S3FileChannel.java index 95d916d..0681ec7 100644 --- a/src/main/java/com/upplication/s3fs/S3FileChannel.java +++ b/src/main/java/com/upplication/s3fs/S3FileChannel.java @@ -44,7 +44,7 @@ else if (!exists && !this.options.contains(StandardOpenOption.CREATE_NEW) && if (exists) { try (S3Object object = path.getFileSystem() .getClient() - .getObject(path.getFileStore().getBucket().getName(), key)) { + .getObject(path.getFileStore().name(), key)) { Files.copy(object.getObjectContent(), tempFile, StandardCopyOption.REPLACE_EXISTING); } } diff --git a/src/main/java/com/upplication/s3fs/S3FileStore.java b/src/main/java/com/upplication/s3fs/S3FileStore.java index 245e7f9..9824711 100644 --- a/src/main/java/com/upplication/s3fs/S3FileStore.java +++ b/src/main/java/com/upplication/s3fs/S3FileStore.java @@ -4,11 +4,13 @@ import java.nio.file.FileStore; import java.nio.file.attribute.FileAttributeView; import java.nio.file.attribute.FileStoreAttributeView; +import java.util.Date; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.Owner; -import com.google.common.collect.ImmutableList; public class S3FileStore extends FileStore implements Comparable { @@ -65,9 +67,18 @@ public boolean supportsFileAttributeView(String attributeViewName) { public V getFileStoreAttributeView(Class type) { if (type != S3FileStoreAttributeView.class) throw new IllegalArgumentException("FileStoreAttributeView of type '" + type.getName() + "' is not supported."); - Bucket buck = getBucket(); - Owner owner = buck.getOwner(); - return (V) new S3FileStoreAttributeView(buck.getCreationDate(), buck.getName(), owner.getId(), owner.getDisplayName()); + Date creationDate = null; + String bucketName = name; + try { + Bucket bucket = getBucket(); + creationDate = bucket.getCreationDate(); + bucketName = bucket.getName(); + } catch (@SuppressWarnings("unused") AmazonS3Exception e) { + // This is probably caused by not being authorized to call ListAll Buckets. + // Lets return what we can get our hands on instead of crashing at this point. + } + Owner owner = getOwner(); + return (V) new S3FileStoreAttributeView(creationDate, bucketName, owner.getId(), owner.getDisplayName()); } @Override @@ -83,6 +94,10 @@ public Bucket getBucket() { return getBucket(name); } + public boolean doesBucketExist(String bucketName) { + return getClient().doesBucketExistV2(bucketName); + } + private Bucket getBucket(String bucketName) { for (Bucket buck : getClient().listBuckets()) if (buck.getName().equals(bucketName)) @@ -99,10 +114,14 @@ private AmazonS3 getClient() { } public Owner getOwner() { - Bucket buck = getBucket(); - if (buck != null) - return buck.getOwner(); - return fileSystem.getClient().getS3AccountOwner(); + try { + AccessControlList acl = getClient().getBucketAcl(name); + return acl.getOwner(); + } catch (@SuppressWarnings("unused") AmazonS3Exception e) { + // Client might not be authorized to request this info? + // User S3AccountOwner as fallback. + } + return fileSystem.getClient().getS3AccountOwner(); } @Override diff --git a/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java b/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java index 77aca37..318a84d 100644 --- a/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java +++ b/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java @@ -3,7 +3,6 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.internal.Constants; import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; import com.google.common.base.Preconditions; @@ -364,9 +363,9 @@ public void createDirectory(Path dir, FileAttribute... attrs) throws IOExcept if (exists(s3Path)) throw new FileAlreadyExistsException(format("target already exists: %s", s3Path)); // create bucket if necesary - Bucket bucket = s3Path.getFileStore().getBucket(); - String bucketName = s3Path.getFileStore().name(); - if (bucket == null) { + S3FileStore fileStore = s3Path.getFileStore(); + String bucketName = fileStore.name(); + if (!fileStore.doesBucketExist(bucketName)) { s3Path.getFileSystem().getClient().createBucket(bucketName); } // create the object as directory diff --git a/src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java b/src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java index 3cb9dbf..ff5ed38 100644 --- a/src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java +++ b/src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java @@ -49,7 +49,7 @@ else if (!exists && !this.options.contains(StandardOpenOption.CREATE_NEW) && if (exists) { try (S3Object object = path.getFileSystem() .getClient() - .getObject(path.getFileStore().getBucket().getName(), key)) { + .getObject(path.getFileStore().name(), key)) { Files.copy(object.getObjectContent(), tempFile, StandardCopyOption.REPLACE_EXISTING); } } From 0fb064e3fb634a29c65ed34852e67ed6ff4ec225 Mon Sep 17 00:00:00 2001 From: sbeimin Date: Mon, 26 Feb 2018 15:42:22 +0100 Subject: [PATCH 2/2] Mock extended for use of new methods --- .../s3fs/util/AmazonS3ClientMock.java | 119 +++++++++++++++++- 1 file changed, 113 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/upplication/s3fs/util/AmazonS3ClientMock.java b/src/test/java/com/upplication/s3fs/util/AmazonS3ClientMock.java index 792c886..5369de1 100644 --- a/src/test/java/com/upplication/s3fs/util/AmazonS3ClientMock.java +++ b/src/test/java/com/upplication/s3fs/util/AmazonS3ClientMock.java @@ -28,22 +28,124 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.*; -import java.util.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.HttpMethod; +import com.amazonaws.SdkClientException; import com.amazonaws.regions.Region; import com.amazonaws.services.s3.AbstractAmazonS3; -import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.S3ClientOptions; import com.amazonaws.services.s3.S3ResponseMetadata; -import com.amazonaws.services.s3.model.*; +import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.BucketAccelerateConfiguration; +import com.amazonaws.services.s3.model.BucketCrossOriginConfiguration; +import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; +import com.amazonaws.services.s3.model.BucketLoggingConfiguration; +import com.amazonaws.services.s3.model.BucketNotificationConfiguration; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.amazonaws.services.s3.model.BucketReplicationConfiguration; +import com.amazonaws.services.s3.model.BucketTaggingConfiguration; +import com.amazonaws.services.s3.model.BucketVersioningConfiguration; +import com.amazonaws.services.s3.model.BucketWebsiteConfiguration; +import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.CopyObjectResult; +import com.amazonaws.services.s3.model.CopyPartRequest; +import com.amazonaws.services.s3.model.CopyPartResult; +import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.amazonaws.services.s3.model.DeleteBucketCrossOriginConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketLifecycleConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest; +import com.amazonaws.services.s3.model.DeleteBucketReplicationConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketRequest; +import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketWebsiteConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteObjectRequest; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.DeleteVersionRequest; +import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; +import com.amazonaws.services.s3.model.GetBucketAccelerateConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketAclRequest; +import com.amazonaws.services.s3.model.GetBucketCrossOriginConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketLifecycleConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketLocationRequest; +import com.amazonaws.services.s3.model.GetBucketLoggingConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketNotificationConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketPolicyRequest; +import com.amazonaws.services.s3.model.GetBucketReplicationConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketVersioningConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketWebsiteConfigurationRequest; +import com.amazonaws.services.s3.model.GetObjectAclRequest; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.GetS3AccountOwnerRequest; +import com.amazonaws.services.s3.model.Grantee; +import com.amazonaws.services.s3.model.HeadBucketRequest; +import com.amazonaws.services.s3.model.HeadBucketResult; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ListBucketsRequest; +import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; +import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest; +import com.amazonaws.services.s3.model.ListNextBatchOfVersionsRequest; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ListPartsRequest; +import com.amazonaws.services.s3.model.ListVersionsRequest; +import com.amazonaws.services.s3.model.MultipartUploadListing; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.Owner; +import com.amazonaws.services.s3.model.PartListing; +import com.amazonaws.services.s3.model.Permission; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.RestoreObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.SetBucketAccelerateConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketAclRequest; +import com.amazonaws.services.s3.model.SetBucketCrossOriginConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketLifecycleConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketLoggingConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketNotificationConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketPolicyRequest; +import com.amazonaws.services.s3.model.SetBucketReplicationConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketTaggingConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketWebsiteConfigurationRequest; +import com.amazonaws.services.s3.model.SetObjectAclRequest; +import com.amazonaws.services.s3.model.StorageClass; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; +import com.amazonaws.services.s3.model.VersionListing; import com.amazonaws.services.s3.waiters.AmazonS3Waiters; import com.amazonaws.util.StringUtils; -import org.apache.http.MethodNotSupportedException; public class AmazonS3ClientMock extends AbstractAmazonS3 { /** @@ -537,7 +639,7 @@ public String getIdentifier() { } } }catch (IOException e) { - throw new RuntimeException(e); + throw new AmazonS3Exception("Problem creating Acl", e); } return res; @@ -1346,4 +1448,9 @@ public URL getUrl(String bucketName, String key) { public AmazonS3Waiters waiters() { throw new UnsupportedOperationException(); } + + @Override + public boolean doesBucketExistV2(String bucketName) throws SdkClientException, AmazonServiceException { + return Files.exists(base.resolve(bucketName)); + } } \ No newline at end of file