diff --git a/Storage/src/Connection/Rest.php b/Storage/src/Connection/Rest.php index 9388839e8bb4..7bd1d66ea059 100644 --- a/Storage/src/Connection/Rest.php +++ b/Storage/src/Connection/Rest.php @@ -19,7 +19,6 @@ use Google\Auth\GetUniverseDomainInterface; use Google\Cloud\Core\RequestBuilder; -use Google\Cloud\Core\RequestWrapper; use Google\Cloud\Core\RestTrait; use Google\Cloud\Core\Retry; use Google\Cloud\Core\Upload\AbstractUploader; @@ -141,7 +140,7 @@ public function __construct(array $config = []) $this->apiEndpoint = $this->getApiEndpoint(null, $config, self::DEFAULT_API_ENDPOINT_TEMPLATE); - $this->setRequestWrapper(new RequestWrapper($config)); + $this->setRequestWrapper(new StorageRequestWrapper($config)); $this->setRequestBuilder(new RequestBuilder( $config['serviceDefinitionPath'], $this->apiEndpoint diff --git a/Storage/src/Connection/StorageRequestWrapper.php b/Storage/src/Connection/StorageRequestWrapper.php new file mode 100644 index 000000000000..79dffa8e9f02 --- /dev/null +++ b/Storage/src/Connection/StorageRequestWrapper.php @@ -0,0 +1,95 @@ +addToken($request, $options); + return parent::send($request, $options); + } + + /** + * @param RequestInterface $request A PSR-7 request. + * @param array $options [optional] + * @return mixed + */ + public function sendAsync(RequestInterface $request, array $options = []) + { + $options = $this->addToken($request, $options); + return parent::sendAsync($request, $options); + } + + /** + * Helper to inject the token. + * + * @param RequestInterface $request + * @param array $options + * @return array + */ + private function addToken(RequestInterface $request, array $options) + { + $method = strtoupper($request->getMethod()); + if ($method === 'GET' || $method === 'HEAD' || $method === 'OPTIONS') { + return $options; + } + + $hasTokenInOptions = false; + if (isset($options['restOptions']['headers'])) { + foreach ($options['restOptions']['headers'] as $key => $value) { + if (strtolower($key) === 'x-goog-gcs-idempotency-token') { + $hasTokenInOptions = true; + break; + } + } + } + + if (!$hasTokenInOptions && !$request->hasHeader('x-goog-gcs-idempotency-token')) { + $token = Uuid::uuid4()->toString(); + if (isset($options['retryHeaders'])) { + foreach ($options['retryHeaders'] as $header) { + if (strpos($header, 'gccl-invocation-id/') === 0) { + $extractedToken = substr($header, 19); + if ($extractedToken !== false && $extractedToken !== '') { + $token = $extractedToken; + } + break; + } + } + } + $options['restOptions']['headers']['x-goog-gcs-idempotency-token'] = $token; + } + return $options; + } +} diff --git a/Storage/tests/System/ManageObjectsTest.php b/Storage/tests/System/ManageObjectsTest.php index de8bde92d072..d3410bffe73c 100644 --- a/Storage/tests/System/ManageObjectsTest.php +++ b/Storage/tests/System/ManageObjectsTest.php @@ -59,7 +59,7 @@ public function testListsObjects() public function testListsObjectsWithMatchGlob() { $bucket = self::createBucket(self::$client, uniqid('matchglob-')); - $objectsToCreate = ["foo/bar", "foo/baz", "foo/foobar", "foobar"]; + $objectsToCreate = ['foo/bar', 'foo/baz', 'foo/foobar', 'foobar']; $matchGlobCases = [ 'foo*bar' => ['foobar'], 'foo**bar' => ['foo/bar', 'foo/foobar', 'foobar'], @@ -92,8 +92,8 @@ public function testObjectRetentionLockedMode() ]); // Test create object with object retention enabled - $objectName = "object-retention-lock"; - $time = (new \DateTime)->add( + $objectName = 'object-retention-lock'; + $time = (new \DateTime())->add( \DateInterval::createFromDateString('+2 hours') ); $object = $bucket->upload(self::DATA, [ @@ -105,7 +105,7 @@ public function testObjectRetentionLockedMode() ]); $this->assertEquals('Locked', $object->info()['retention']['mode']); - $laterTime = (new \DateTime)->add( + $laterTime = (new \DateTime())->add( \DateInterval::createFromDateString('+4 hours') ); @@ -174,8 +174,8 @@ public function testObjectRetentionUnlockedMode() $this->assertEquals('Enabled', $bucket->info()['objectRetention']['mode']); // Test create object with object retention enabled - $objectName = "object-retention-lock"; - $expires = (new \DateTime)->add( + $objectName = 'object-retention-lock'; + $expires = (new \DateTime())->add( \DateInterval::createFromDateString('+2 hours') ); $uploader = $bucket->getStreamableUploader('initial contents', [ @@ -320,7 +320,7 @@ public function testGetContextsWithServerTime() $object = $this->createObjectWithContexts($initialContexts); $info = $object->info(); $this->assertArrayHasKey('contexts', $info); - + $context = $info['contexts']['custom']; $this->assertEquals('temp', $context['temp-key']['value']); $this->assertArrayHasKey( @@ -348,7 +348,7 @@ public function testClearAllExistingContexts() $this->assertArrayHasKey('contexts', $info); $this->assertEquals('temp', $info['contexts']['custom']['temp-key']['value']); $this->assertEquals('to-be-cleared', $info['contexts']['custom']['status']['value']); - + $object->update([ 'contexts' => null ]); @@ -377,13 +377,13 @@ public function testRewriteObjectWithContexts() 'name' => 'override-' . uniqid(), 'contexts' => ['custom' => [$overrideKey => ['value' => $overrideVal]]] ]); - + $info = $overridden->info(); $this->assertEquals($overrideVal, $info['contexts']['custom'][$overrideKey]['value']); $this->assertArrayNotHasKey('tag', $info['contexts']['custom']); $object->delete(); } - + public function testOverrideContextsDuringCopy() { $initialContexts = [ @@ -414,7 +414,7 @@ public function testComposeObjectWithOverrideAndInheritContexts() 'tag' => ['value' => 'file1'], ], ]; - + $source1 = $this->createObjectWithContexts($initialContexts); $bucket = self::$client->bucket($source1->info()['bucket']); $s2Key = 's2-key'; @@ -464,7 +464,7 @@ public function testListObjectsWithContextFilters() $noneFile = $bucket->upload('content', [ 'name' => 'test-none.txt' ]); - + // Should list all objects matching a prefix $objects = iterator_to_array($bucket->objects()); $this->assertCount(3, $objects); @@ -542,6 +542,81 @@ public function testUploadAsync() $this->assertInstanceOf(StorageObject::class, $resp); } + public function testIdempotencyTokenRetries() + { + $name = uniqid(self::TESTING_PREFIX); + $object = self::$bucket->upload('test data', [ + 'name' => $name + ]); + + $uuid = \Ramsey\Uuid\Uuid::uuid4()->toString(); + + // First delete will succeed + $object->delete([ + 'restOptions' => [ + 'headers' => [ + 'x-goog-gcs-idempotency-token' => $uuid + ] + ] + ]); + + // Second delete uses the exact same UUID, simulating a network retry. + // It should NOT throw a NotFoundException because the GCS backend + // will recognize the token and return the cached success response. + $object->delete([ + 'restOptions' => [ + 'headers' => [ + 'x-goog-gcs-idempotency-token' => $uuid + ] + ] + ]); + + $this->assertFalse($object->exists()); + } + + public function testIdempotencyTokenUpdateRetriesWithPrecondition() + { + $name = uniqid(self::TESTING_PREFIX); + $object = self::$bucket->upload('test data', [ + 'name' => $name + ]); + + $info = $object->info(); + $metageneration = $info['metageneration']; + + $uuid = \Ramsey\Uuid\Uuid::uuid4()->toString(); + + $metadata = [ + 'metadata' => [ + 'location' => 'test' + ] + ]; + + // First update will succeed and increment the metageneration + $object->update($metadata, [ + 'ifMetagenerationMatch' => $metageneration, + 'restOptions' => [ + 'headers' => [ + 'x-goog-gcs-idempotency-token' => $uuid + ] + ] + ]); + + // Second update uses the exact same UUID, simulating a network retry. + // Even though the metageneration has changed, the backend recognizes + // the idempotency token and returns 200 OK instead of 412 Precondition Failed. + $object->update($metadata, [ + 'ifMetagenerationMatch' => $metageneration, + 'restOptions' => [ + 'headers' => [ + 'x-goog-gcs-idempotency-token' => $uuid + ] + ] + ]); + + $this->assertEquals('test', $object->info()['metadata']['location']); + } + public function testUpdateObject() { $metadata = [ @@ -677,7 +752,7 @@ public function testComposeObjectsWithDeleteSourceObjectsNull() public function testSoftDeleteObject() { - $softDeleteBucketName = "soft-delete-bucket-" . uniqid(); + $softDeleteBucketName = 'soft-delete-bucket-' . uniqid(); $softDeleteBucket = self::createBucket( self::$client, $softDeleteBucketName, @@ -706,14 +781,14 @@ public function testSoftDeleteObject() public function testSoftDeleteHNSObject() { - $softDeleteBucketName = "soft-delete-hns-bucket-" . uniqid(); + $softDeleteBucketName = 'soft-delete-hns-bucket-' . uniqid(); $softDeleteHNSBucket = self::createBucket( self::$client, $softDeleteBucketName, [ 'location' => 'us-west1', 'softDeletePolicy' => ['retentionDurationSeconds' => 8 * 24 * 60 * 60], - 'hierarchicalNamespace' => ['enabled' => true,], + 'hierarchicalNamespace' => ['enabled' => true, ], 'iamConfiguration' => ['uniformBucketLevelAccess' => ['enabled' => true]] ] ); @@ -750,7 +825,7 @@ public function testSoftDeleteHNSObject() */ public function testMoveObject(bool $hnEnabled) { - $name = "move-object-bucket-" . uniqid(); + $name = 'move-object-bucket-' . uniqid(); $sourceObjectName = uniqid(self::TESTING_PREFIX); $destinationObjectName = uniqid(self::TESTING_PREFIX); $sourceBucket = self::createBucket( @@ -898,8 +973,8 @@ public function testStringNormalization() $bucket = self::$client->bucket(self::NORMALIZATION_TEST_BUCKET); $cases = [ - ["Caf\xC3\xA9", "Normalization Form C"], - ["Cafe\xCC\x81", "Normalization Form D"], + ["Caf\xC3\xA9", 'Normalization Form C'], + ["Cafe\xCC\x81", 'Normalization Form D'], ]; foreach ($cases as list($name, $expectedContent)) { @@ -913,7 +988,7 @@ public function testStringNormalization() public function testDownloadsWithDefaultCrc32cValidationSuccess() { $object = self::$bucket->upload('system-test-data', ['name' => uniqid(self::TESTING_PREFIX)]); - + // Automatic CRC32C validation runs under the hood $content = $object->downloadAsString(); $this->assertEquals('system-test-data', $content); @@ -924,7 +999,7 @@ public function testDownloadsWithDefaultCrc32cValidationSuccess() public function testDownloadsWithExplicitMd5ValidationSuccess() { $object = self::$bucket->upload('system-test-data', ['name' => uniqid(self::TESTING_PREFIX)]); - + // Explicitly opt-in to MD5 validation $content = $object->downloadAsString(['validate' => 'md5']); $this->assertEquals('system-test-data', $content); @@ -935,7 +1010,7 @@ public function testDownloadsWithExplicitMd5ValidationSuccess() public function testDownloadsWithValidationDisabledSuccess() { $object = self::$bucket->upload('system-test-data', ['name' => uniqid(self::TESTING_PREFIX)]); - + // Explicitly disable validation $content = $object->downloadAsString(['validate' => false]); $this->assertEquals('system-test-data', $content); @@ -947,7 +1022,7 @@ public function testDownloadsWithRangeBypassesValidation() { $data = 'system-test-range-data'; $object = self::$bucket->upload($data, ['name' => uniqid(self::TESTING_PREFIX)]); - + // Default validate is 'crc32', but we pass a Range header inside restOptions. // This should successfully download the slice 'system' without throwing a mismatch exception. $content = $object->downloadAsString([ @@ -966,12 +1041,12 @@ public function testDownloadToFileWithDefaultValidationSuccess() { $data = 'system-test-to-file-data'; $object = self::$bucket->upload($data, ['name' => uniqid(self::TESTING_PREFIX)]); - + $tempFile = tempnam(sys_get_temp_dir(), 'gcs-test'); $object->downloadToFile($tempFile); - + $this->assertEquals($data, file_get_contents($tempFile)); - + unlink($tempFile); $object->delete(); } diff --git a/Storage/tests/Unit/Connection/RestTest.php b/Storage/tests/Unit/Connection/RestTest.php index d2afc3ddf590..81d19574824a 100644 --- a/Storage/tests/Unit/Connection/RestTest.php +++ b/Storage/tests/Unit/Connection/RestTest.php @@ -17,18 +17,22 @@ namespace Google\Cloud\Storage\Tests\Unit\Connection; -use Google\Auth\HttpHandler\Guzzle7HttpHandler; +use Google\Auth\HttpHandler\HttpHandlerFactory; use Google\Cloud\Core\RequestBuilder; use Google\Cloud\Core\RequestWrapper; use Google\Cloud\Core\Retry; -use Google\Cloud\Core\Testing\TestHelpers; use Google\Cloud\Core\Upload\MultipartUploader; use Google\Cloud\Core\Upload\ResumableUploader; use Google\Cloud\Core\Upload\StreamableUploader; use Google\Cloud\Storage\Connection\Rest; use Google\Cloud\Storage\Connection\RetryTrait; +use Google\Cloud\Storage\Connection\StorageRequestWrapper; use GuzzleHttp\Client; use GuzzleHttp\Exception\BadResponseException; +use GuzzleHttp\Exception\RequestException; +use GuzzleHttp\Handler\MockHandler; +use GuzzleHttp\HandlerStack; +use GuzzleHttp\Middleware; use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Psr7\Request; @@ -517,7 +521,7 @@ function ($args) use ( } ); $requestWrapper = new RequestWrapper([ - 'httpHandler' => new Guzzle7HttpHandler($mockClient->reveal()), + 'httpHandler' => HttpHandlerFactory::build($mockClient->reveal()), 'accessToken' => 'Fake token', 'retries' => 3, ]); @@ -964,7 +968,7 @@ public function validationMethod() true, true, false - ],[ + ], [ ['validate' => null], true, true, @@ -1030,6 +1034,195 @@ public function provideRetryHeaders() ]; } + public function testIdempotencyTokenHeaderAdded() + { + $mockClient = $this->prophesize(Client::class); + $mockClient->send( + Argument::type(RequestInterface::class), + Argument::that(function ($options) { + if (!isset($options['headers']['x-goog-gcs-idempotency-token'])) { + return false; + } + $token = $options['headers']['x-goog-gcs-idempotency-token']; + return preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/', $token) === 1; + }) + )->willReturn(new Response(200, [], '{}'))->shouldBeCalled(); + + $rest = new Rest(); + $rest->setRequestWrapper(new StorageRequestWrapper([ + 'httpHandler' => HttpHandlerFactory::build($mockClient->reveal()), + 'accessToken' => 'Fake token', + ])); + + $rest->insertBucket(); + } + + public function testIdempotencyTokenNotOverwrittenIfProvided() + { + $customToken = 'my-custom-uuid-1234'; + + $mockClient = $this->prophesize(Client::class); + $mockClient->send( + Argument::type(RequestInterface::class), + Argument::that(function ($options) use ($customToken) { + if (!isset($options['headers']['x-goog-gcs-idempotency-token'])) { + return false; + } + return $options['headers']['x-goog-gcs-idempotency-token'] === $customToken; + }) + )->willReturn(new Response(200, [], '{}'))->shouldBeCalled(); + + $rest = new Rest(); + $rest->setRequestWrapper(new StorageRequestWrapper([ + 'httpHandler' => HttpHandlerFactory::build($mockClient->reveal()), + 'accessToken' => 'Fake token', + ])); + + $rest->insertBucket([ + 'restOptions' => [ + 'headers' => [ + 'x-goog-gcs-idempotency-token' => $customToken + ] + ] + ]); + } + + public function testIdempotencyTokenNotOverwrittenIfProvidedWithMixedCase() + { + $customToken = 'my-custom-uuid-1234'; + + $mockClient = $this->prophesize(Client::class); + $mockClient->send( + Argument::type(RequestInterface::class), + Argument::that(function ($options) use ($customToken) { + if (isset($options['headers']['x-goog-gcs-idempotency-token'])) { + return false; + } + if (!isset($options['headers']['X-Goog-Gcs-Idempotency-Token'])) { + return false; + } + return $options['headers']['X-Goog-Gcs-Idempotency-Token'] === $customToken; + }) + )->willReturn(new Response(200, [], '{}'))->shouldBeCalled(); + + $rest = new Rest(); + $rest->setRequestWrapper(new StorageRequestWrapper([ + 'httpHandler' => HttpHandlerFactory::build($mockClient->reveal()), + 'accessToken' => 'Fake token', + ])); + + $rest->insertBucket([ + 'restOptions' => [ + 'headers' => [ + 'X-Goog-Gcs-Idempotency-Token' => $customToken + ] + ] + ]); + } + + public function testIdempotencyTokenGeneratedIfGcclInvocationIdMalformed() + { + $mockClient = $this->prophesize(Client::class); + $mockClient->send( + Argument::type(RequestInterface::class), + Argument::that(function ($options) { + if (!isset($options['headers']['x-goog-gcs-idempotency-token'])) { + return false; + } + $token = $options['headers']['x-goog-gcs-idempotency-token']; + return preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/', $token) === 1; + }) + )->willReturn(new Response(200, [], '{}'))->shouldBeCalled(); + + $rest = new Rest(); + $rest->setRequestWrapper(new StorageRequestWrapper([ + 'httpHandler' => HttpHandlerFactory::build($mockClient->reveal()), + 'accessToken' => 'Fake token', + ])); + + $rest->insertBucket([ + 'retryHeaders' => [ + 'gccl-invocation-id/' + ] + ]); + } + + /** + * Test idempotency token and custom headers are preserved across retries. + */ + public function testIdempotencyTokenResusedOnRetry() + { + $container = []; + $history = Middleware::history($container); + + $mockHandler = new MockHandler([ + new Response(500, [], 'Internal Server Error'), + new Response(200, [], '{}') + ]); + + $handlerStack = HandlerStack::create($mockHandler); + $handlerStack->push($history); + + $client = new Client(['handler' => $handlerStack]); + + $customToken = 'my-custom-idempotency-token-12345'; + $contentType = 'application/json'; + $customHeader = 'my-custom-header-value'; + + $rest = new Rest([ + 'restDelayFunction' => function () { + }, + ]); + $rest->setRequestWrapper(new StorageRequestWrapper([ + 'httpHandler' => HttpHandlerFactory::build($client), + 'accessToken' => 'Fake token', + 'restDelayFunction' => function () { + }, + ])); + + $rest->insertBucket([ + 'restOptions' => [ + 'headers' => [ + 'x-goog-gcs-idempotency-token' => $customToken, + 'Content-Type' => $contentType, + 'X-Custom-Header' => $customHeader, + ] + ] + ]); + + $this->assertCount(2, $container); + + $firstRequest = $container[0]['request']; + $secondRequest = $container[1]['request']; + + $this->assertEquals( + $customToken, + $firstRequest->getHeaderLine('x-goog-gcs-idempotency-token') + ); + $this->assertEquals( + $firstRequest->getHeaderLine('x-goog-gcs-idempotency-token'), + $secondRequest->getHeaderLine('x-goog-gcs-idempotency-token') + ); + + $this->assertEquals( + $contentType, + $firstRequest->getHeaderLine('Content-Type') + ); + $this->assertEquals( + $firstRequest->getHeaderLine('Content-Type'), + $secondRequest->getHeaderLine('Content-Type') + ); + + $this->assertEquals( + $customHeader, + $firstRequest->getHeaderLine('X-Custom-Header') + ); + $this->assertEquals( + $firstRequest->getHeaderLine('X-Custom-Header'), + $secondRequest->getHeaderLine('X-Custom-Header') + ); + } + private function getContentTypeAndMetadata(RequestInterface $request) { // Resumable upload request diff --git a/dev/tests/Snippet/ProductNeutralGuides/AuthenticationTest.php b/dev/tests/Snippet/ProductNeutralGuides/AuthenticationTest.php index 6c02a31a2d76..e1821281a5f1 100644 --- a/dev/tests/Snippet/ProductNeutralGuides/AuthenticationTest.php +++ b/dev/tests/Snippet/ProductNeutralGuides/AuthenticationTest.php @@ -82,7 +82,10 @@ public function testAuthenticationCredentialsFetcherOption() $this->assertInstanceOf(StorageClient::class, $client); $connection = (new ReflectionClass($client))->getProperty('connection')->getValue($client); $requestWrapper = (new ReflectionClass($connection))->getProperty('requestWrapper')->getValue($connection); - $creds = (new ReflectionClass($requestWrapper))->getProperty('credentialsFetcher')->getValue($requestWrapper); + $requestWrapperReflection = new ReflectionClass(\Google\Cloud\Core\RequestWrapper::class); + $credentialsFetcherProperty = $requestWrapperReflection->getProperty('credentialsFetcher'); + $credentialsFetcherProperty->setAccessible(true); + $creds = $credentialsFetcherProperty->getValue($requestWrapper); $this->assertInstanceOf(ServiceAccountCredentials::class, $creds); $this->assertEquals($clientEmail, $creds->getClientName()); }