Skip to content

Add ability to fetch restored glacier object #113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add ability to fetch restored glacier object #113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@
package org.embulk.input.s3.explorer;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.embulk.input.s3.DefaultRetryable;
import org.embulk.util.retryhelper.RetryExecutor;
Expand Down Expand Up @@ -53,6 +55,17 @@ public ObjectListing call()
return ol.getObjectSummaries();
}

@Override
protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) {
final GetObjectMetadataRequest req = new GetObjectMetadataRequest(obj.getBucketName(), obj.getKey());

return new DefaultRetryable<ObjectMetadata>("Get object metadata")
{
@Override
public ObjectMetadata call() { return s3Client.getObjectMetadata(req); }
}.executeWith(retryExecutor);
}

@Override
protected boolean hasNext()
{
Expand Down
11 changes: 10 additions & 1 deletion src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.embulk.input.s3.explorer;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.StorageClass;
import org.embulk.config.ConfigException;
Expand Down Expand Up @@ -54,7 +55,13 @@ public void addToBuilder(final FileList.Builder builder)
LOGGER.warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey());
continue;
}
throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");

ObjectMetadata objectMetadata = fetchObjectMetadata(s);
if (objectMetadata != null && objectMetadata.getRestoreExpirationTime() != null) {
LOGGER.info("Restored Glacier object \"s3://{}/{}\" found", bucketName, s.getKey());
} else {
throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
}
}
if (s.getSize() > 0) {
builder.add(s.getKey(), s.getSize());
Expand All @@ -69,5 +76,7 @@ public void addToBuilder(final FileList.Builder builder)

protected abstract List<S3ObjectSummary> fetch();

protected abstract ObjectMetadata fetchObjectMetadata(final S3ObjectSummary obj);

protected abstract boolean hasNext();
}
13 changes: 13 additions & 0 deletions src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@
package org.embulk.input.s3.explorer;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.embulk.input.s3.DefaultRetryable;
import org.embulk.util.retryhelper.RetryExecutor;
Expand Down Expand Up @@ -72,6 +74,17 @@ public ObjectListing call()
.collect(Collectors.toList());
}

@Override
protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) {
final GetObjectMetadataRequest req = new GetObjectMetadataRequest(obj.getBucketName(), obj.getKey());

return new DefaultRetryable<ObjectMetadata>("Get object metadata")
{
@Override
public ObjectMetadata call() { return s3Client.getObjectMetadata(req); }
}.executeWith(retryExecutor);
}

@Override
public boolean hasNext()
{
Expand Down
19 changes: 19 additions & 0 deletions src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.embulk.input.s3.explorer;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.StorageClass;
import org.embulk.EmbulkTestRuntime;
Expand All @@ -31,6 +32,7 @@
import org.mockito.runners.MockitoJUnitRunner;

import java.util.Collections;
import java.util.Date;
import java.util.List;

import static org.mockito.Mockito.doReturn;
Expand Down Expand Up @@ -75,6 +77,20 @@ public void addToBuilder_should_throw_exception_if_notskipped_glacier_storage()
s3PrefixFileExplorer.addToBuilder(builder);
}

@Test
public void addToBuilder_should_not_throw_exception_if_restored_glacier_object()
{
when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
when(s3ObjectSummary.getKey()).thenReturn(PATH_PREFIX + OBJECT_KEY);
when(s3ObjectSummary.getSize()).thenReturn(1L);
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setRestoreExpirationTime(new Date());
doReturn(objectMetadata).when(s3PrefixFileExplorer).fetchObjectMetadata(s3ObjectSummary);
doReturn(true).when(s3PrefixFileExplorer).hasNext();
s3PrefixFileExplorer.addToBuilder(builder);
verify(builder).add(PATH_PREFIX + OBJECT_KEY, 1);
}

@Test
public void addToBuilder_should_skip_glacier_storage_if_allowed()
{
Expand Down Expand Up @@ -132,6 +148,9 @@ protected List<S3ObjectSummary> fetch()
return null;
}

@Override
protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) { return null; }

@Override
protected boolean hasNext()
{
Expand Down