Skip to content

Commit 8f6443b

Browse files
medbmedb
committed
Eagerly fetch GoogleCloudStorageReadChannel metadata if 'fs.gs.inputstream.fast.fail.on.not.found.enable' is true
1 parent f81d823 commit 8f6443b

5 files changed

Lines changed: 84 additions & 5 deletions

File tree

‎gcs/CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
1. Fix bug when GCS connector lists all files in directory instead of
44
specified limit.
55

6+
2. Eagerly initialize `GoogleCloudStorageReadChannel` metadata if
7+
`fs.gs.inputstream.fast.fail.on.not.found.enable` set to true.
8+
69

710
1.9.15 - 2019-02-21
811

‎gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageReadChannel.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.google.cloud.hadoop.gcsio;
1818

19+
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.createItemInfoForStorageObject;
1920
import static com.google.cloud.hadoop.gcsio.StorageResourceId.createReadableString;
2021
import static com.google.common.base.Preconditions.checkArgument;
2122
import static com.google.common.base.Preconditions.checkNotNull;
@@ -36,6 +37,8 @@
3637
import com.google.cloud.hadoop.gcsio.GoogleCloudStorageReadOptions.GenerationReadConsistency;
3738
import com.google.cloud.hadoop.util.ApiErrorExtractor;
3839
import com.google.cloud.hadoop.util.ClientRequestHelper;
40+
import com.google.cloud.hadoop.util.ResilientOperation;
41+
import com.google.cloud.hadoop.util.RetryDeterminer;
3942
import com.google.common.annotations.VisibleForTesting;
4043
import com.google.common.base.Preconditions;
4144
import com.google.common.base.Supplier;
@@ -289,8 +292,28 @@ ExponentialBackOff createBackOff() {
289292
* read.
290293
*/
291294
@Nullable
292-
protected GoogleCloudStorageItemInfo getInitialMetadata() {
293-
return null;
295+
protected GoogleCloudStorageItemInfo getInitialMetadata() throws IOException {
296+
if (!readOptions.getFastFailOnNotFound()) {
297+
return null;
298+
}
299+
Storage.Objects.Get getObject = gcs.objects().get(bucketName, objectName);
300+
StorageObject object;
301+
try {
302+
object =
303+
ResilientOperation.retry(
304+
ResilientOperation.getGoogleRequestCallable(getObject),
305+
readBackOff.get(),
306+
RetryDeterminer.SOCKET_ERRORS,
307+
IOException.class,
308+
sleeper);
309+
} catch (IOException e) {
310+
throw errorExtractor.itemNotFound(e)
311+
? GoogleCloudStorageExceptions.getFileNotFoundException(bucketName, objectName)
312+
: new IOException("Error reading " + resourceIdString, e);
313+
} catch (InterruptedException e) { // From the sleep
314+
throw new IOException("Thread interrupt received.", e);
315+
}
316+
return createItemInfoForStorageObject(new StorageResourceId(bucketName, objectName), object);
294317
}
295318

296319
/**

‎gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/InMemoryObjectEntry.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ public synchronized SeekableByteChannel getReadChannel(GoogleCloudStorageReadOpt
259259
return new InMemoryObjectReadChannel(completedContents, readOptions) {
260260
@Nullable
261261
@Override
262-
protected GoogleCloudStorageItemInfo getInitialMetadata() {
262+
protected GoogleCloudStorageItemInfo getInitialMetadata() throws IOException {
263263
return readOptions.getFastFailOnNotFound() ? getInfo() : super.getInitialMetadata();
264264
}
265265
};

‎gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageReadChannelTest.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,28 @@
1414

1515
package com.google.cloud.hadoop.gcsio;
1616

17+
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.BUCKET_NAME;
1718
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.JSON_FACTORY;
19+
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.OBJECT_NAME;
1820
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.createReadChannel;
1921
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.dataRangeResponse;
2022
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.dataResponse;
23+
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.metadataResponse;
2124
import static com.google.common.truth.Truth.assertThat;
2225
import static java.util.stream.Collectors.toList;
2326

2427
import com.google.api.client.http.HttpRequest;
2528
import com.google.api.client.testing.http.MockHttpTransport;
29+
import com.google.api.client.util.DateTime;
2630
import com.google.api.services.storage.Storage;
31+
import com.google.api.services.storage.model.StorageObject;
2732
import com.google.cloud.hadoop.gcsio.GoogleCloudStorageReadOptions.Fadvise;
2833
import java.io.IOException;
34+
import java.math.BigInteger;
2935
import java.nio.ByteBuffer;
3036
import java.util.ArrayList;
3137
import java.util.Arrays;
38+
import java.util.Date;
3239
import java.util.List;
3340
import org.junit.Test;
3441
import org.junit.runner.RunWith;
@@ -38,6 +45,49 @@
3845
@RunWith(JUnit4.class)
3946
public class GoogleCloudStorageReadChannelTest {
4047

48+
@Test
49+
public void metadataInitialization_eager() throws IOException {
50+
MockHttpTransport transport =
51+
GoogleCloudStorageTestUtils.mockTransport(
52+
metadataResponse(
53+
new StorageObject()
54+
.setBucket(BUCKET_NAME)
55+
.setName(OBJECT_NAME)
56+
.setSize(new BigInteger("123"))
57+
.setGeneration(1L)
58+
.setMetageneration(1L)
59+
.setUpdated(new DateTime(new Date()))));
60+
61+
List<HttpRequest> requests = new ArrayList<>();
62+
63+
Storage storage = new Storage(transport, JSON_FACTORY, requests::add);
64+
65+
GoogleCloudStorageReadOptions options =
66+
GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(true).build();
67+
68+
GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);
69+
70+
assertThat(readChannel.size()).isEqualTo(123);
71+
assertThat(requests).hasSize(1);
72+
}
73+
74+
@Test
75+
public void metadataInitialization_lazy() throws IOException {
76+
MockHttpTransport transport = GoogleCloudStorageTestUtils.mockTransport();
77+
78+
List<HttpRequest> requests = new ArrayList<>();
79+
80+
Storage storage = new Storage(transport, JSON_FACTORY, requests::add);
81+
82+
GoogleCloudStorageReadOptions options =
83+
GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(false).build();
84+
85+
GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);
86+
87+
assertThat(readChannel.size()).isEqualTo(-1);
88+
assertThat(requests).isEmpty();
89+
}
90+
4191
@Test
4292
public void fadviseAuto_onForwardRead_switchesToRandom() throws IOException {
4393
int seekPosition = 5;
@@ -57,6 +107,7 @@ public void fadviseAuto_onForwardRead_switchesToRandom() throws IOException {
57107

58108
GoogleCloudStorageReadOptions options =
59109
GoogleCloudStorageReadOptions.builder()
110+
.setFastFailOnNotFound(false)
60111
.setFadvise(Fadvise.AUTO)
61112
.setMinRangeRequestSize(1)
62113
.setInplaceSeekLimit(2)
@@ -102,6 +153,7 @@ public void fadviseAuto_onBackwardRead_switchesToRandom() throws IOException {
102153

103154
GoogleCloudStorageReadOptions options =
104155
GoogleCloudStorageReadOptions.builder()
156+
.setFastFailOnNotFound(false)
105157
.setFadvise(Fadvise.AUTO)
106158
.setMinRangeRequestSize(1)
107159
.build();
@@ -148,6 +200,7 @@ public void footerPrefetch_reused() throws IOException {
148200

149201
GoogleCloudStorageReadOptions options =
150202
GoogleCloudStorageReadOptions.builder()
203+
.setFastFailOnNotFound(false)
151204
.setFadvise(Fadvise.RANDOM)
152205
.setMinRangeRequestSize(footeSize)
153206
.build();

‎gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageTestUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ public final class GoogleCloudStorageTestUtils {
3838

3939
public static final JacksonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
4040

41-
private static final String BUCKET_NAME = "foo-bucket";
42-
private static final String OBJECT_NAME = "bar-object";
41+
static final String BUCKET_NAME = "foo-bucket";
42+
static final String OBJECT_NAME = "bar-object";
4343

4444
private static final ApiErrorExtractor ERROR_EXTRACTOR = ApiErrorExtractor.INSTANCE;
4545
private static final ClientRequestHelper<StorageObject> REQUEST_HELPER =

0 commit comments

Comments
 (0)