diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index f35f4721f..7f0d9f2f0 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -39,7 +39,7 @@ branchProtectionRules: - 'Kokoro - Test: Java GraalVM Native Image' - 'Kokoro - Test: Java 17 GraalVM Native Image' - javadoc - + - unmanaged_dependency_check # Identifies the protection rule pattern. Name of the branch to be protected. # Defaults to `main` - pattern: 1.21.x diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd86ef0b..74bfd538c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [2.25.0](https://github.com/googleapis/java-spanner-jdbc/compare/v2.24.2...v2.25.0) (2024-12-04) + + +### Features + +* Add fallback to PDML mode ([#1841](https://github.com/googleapis/java-spanner-jdbc/issues/1841)) ([1e81863](https://github.com/googleapis/java-spanner-jdbc/commit/1e818634d1f4845ef96c206de26388e6c3c80bf7)) + + +### Dependencies + +* Update dependency com.fasterxml.jackson.core:jackson-databind to v2.18.2 ([#1846](https://github.com/googleapis/java-spanner-jdbc/issues/1846)) ([1a010a1](https://github.com/googleapis/java-spanner-jdbc/commit/1a010a1ecb4e5f3c83c8fca26c64e607095f1351)) +* Update dependency com.google.cloud:google-cloud-spanner-bom to v6.81.2 ([#1837](https://github.com/googleapis/java-spanner-jdbc/issues/1837)) ([52180d9](https://github.com/googleapis/java-spanner-jdbc/commit/52180d9ad8ff9ae1beda42af4c16c0796948e5a0)) +* Update dependency com.google.cloud:google-cloud-spanner-bom to v6.82.0 ([#1847](https://github.com/googleapis/java-spanner-jdbc/issues/1847)) ([b4ea413](https://github.com/googleapis/java-spanner-jdbc/commit/b4ea4130e667f417d249edbeb560720f58a3c1aa)) +* Update dependency org.mybatis.spring.boot:mybatis-spring-boot-starter to v3.0.4 ([#1844](https://github.com/googleapis/java-spanner-jdbc/issues/1844)) ([3cd9cd6](https://github.com/googleapis/java-spanner-jdbc/commit/3cd9cd6d3d2e7be023e1ff80019bf35bfedc07f9)) +* Update dependency org.springframework.boot:spring-boot to v3.4.0 ([#1838](https://github.com/googleapis/java-spanner-jdbc/issues/1838)) ([fb20987](https://github.com/googleapis/java-spanner-jdbc/commit/fb2098723ad193bf7331578113c0ed0f2e734101)) +* Update dependency org.springframework.boot:spring-boot-starter-data-jdbc to v3.4.0 ([#1840](https://github.com/googleapis/java-spanner-jdbc/issues/1840)) ([3f9dbf1](https://github.com/googleapis/java-spanner-jdbc/commit/3f9dbf18415315db204d679c28d6f226b3edd7f1)) +* Update dependency org.springframework.boot:spring-boot-starter-parent to v3.4.0 ([#1839](https://github.com/googleapis/java-spanner-jdbc/issues/1839)) ([d681cea](https://github.com/googleapis/java-spanner-jdbc/commit/d681cea1d03f1b57d86f362a5bd2f5089ffcde4c)) +* Update dependency org.testcontainers:testcontainers to v1.20.4 ([#1835](https://github.com/googleapis/java-spanner-jdbc/issues/1835)) ([78aa4bf](https://github.com/googleapis/java-spanner-jdbc/commit/78aa4bf90e8a5339f5179fe1b95d6ed6e1b9ebbc)) +* Update dependency org.testcontainers:testcontainers-bom to v1.20.4 ([#1836](https://github.com/googleapis/java-spanner-jdbc/issues/1836)) ([c01ab98](https://github.com/googleapis/java-spanner-jdbc/commit/c01ab9800483db3eec5da0bd35acda5fb00de663)) + ## [2.24.2](https://github.com/googleapis/java-spanner-jdbc/compare/v2.24.1...v2.24.2) (2024-11-20) diff --git a/README.md b/README.md index 70bace0de..4ad9bf804 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ If you are using Maven, add this to your pom.xml file: com.google.cloud google-cloud-spanner-jdbc - 2.24.2 + 2.25.0 ``` @@ -30,7 +30,7 @@ If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-spanner-jdbc:2.24.2' +implementation 'com.google.cloud:google-cloud-spanner-jdbc:2.25.0' ``` @@ -38,7 +38,7 @@ If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-spanner-jdbc" % "2.24.2" +libraryDependencies += "com.google.cloud" % "google-cloud-spanner-jdbc" % "2.25.0" ``` @@ -118,6 +118,15 @@ these can also be supplied in a Properties instance that is passed to the - maxSessions (int): Sets the maximum number of sessions in the backing session pool. Defaults to 400. - numChannels (int): Sets the number of gRPC channels to use. Defaults to 4. - retryAbortsInternally (boolean): The JDBC driver will by default automatically retry aborted transactions internally. This is done by keeping track of all statements and the results of these during a transaction, and if the transaction is aborted by Cloud Spanner, it will replay the statements on a new transaction and compare the results with the initial attempt. Disable this option if you want to handle aborted transactions in your own application. +- autocommit_dml_mode (string): Determines the transaction type that is used to execute + [DML statements](https://cloud.google.com/spanner/docs/dml-tasks#using-dml) when the connection is + in auto-commit mode. The following values are supported: + - TRANSACTIONAL (default): Uses atomic read/write transactions. + - PARTITIONED_NON_ATOMIC: Use Partitioned DML for DML statements in auto-commit mode. Use this mode + to execute DML statements that exceed the transaction mutation limit in Spanner. + - TRANSACTIONAL_WITH_FALLBACK_TO_PARTITIONED_NON_ATOMIC: Execute DML statements using atomic read/write + transactions. If this fails because the mutation limit on Spanner has been exceeded, the DML statement + is retried using a Partitioned DML transaction. - auto_batch_dml (boolean): Automatically buffer DML statements and execute them as one batch, instead of executing them on Spanner directly. The buffered DML statements are executed on Spanner in one batch when a query is executed, or when the transaction is committed. This option can for diff --git a/pom.xml b/pom.xml index 420f3315e..3ea029965 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 google-cloud-spanner-jdbc - 2.24.2 + 2.25.0 jar Google Cloud Spanner JDBC https://github.com/googleapis/java-spanner-jdbc @@ -61,7 +61,7 @@ com.google.cloud google-cloud-spanner-bom - 6.81.2 + 6.82.0 pom import @@ -166,7 +166,7 @@ org.testcontainers testcontainers - 1.20.3 + 1.20.4 test diff --git a/samples/quickperf/pom.xml b/samples/quickperf/pom.xml index ef6e6e9b6..6a09a0aab 100644 --- a/samples/quickperf/pom.xml +++ b/samples/quickperf/pom.xml @@ -67,19 +67,19 @@ com.fasterxml.jackson.core jackson-databind - 2.18.1 + 2.18.2 org.testcontainers testcontainers - 1.20.3 + 1.20.4 test org.springframework.boot spring-boot - 3.3.5 + 3.4.0 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 3b4703db1..d197033bf 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-spanner-jdbc - 2.24.2 + 2.25.0 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 114c0f8e8..36fbfb9b0 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -54,7 +54,7 @@ org.testcontainers testcontainers - 1.20.3 + 1.20.4 test diff --git a/samples/spring-data-jdbc/pom.xml b/samples/spring-data-jdbc/pom.xml index 5882816eb..50e4c1226 100644 --- a/samples/spring-data-jdbc/pom.xml +++ b/samples/spring-data-jdbc/pom.xml @@ -30,7 +30,7 @@ com.google.cloud google-cloud-spanner-bom - 6.81.1 + 6.82.0 import pom @@ -55,7 +55,7 @@ org.springframework.boot spring-boot-starter-data-jdbc - 3.3.5 + 3.4.0 @@ -119,7 +119,7 @@ org.testcontainers testcontainers - 1.20.3 + 1.20.4 test diff --git a/samples/spring-data-mybatis/pom.xml b/samples/spring-data-mybatis/pom.xml index df6f597f7..469207ca3 100644 --- a/samples/spring-data-mybatis/pom.xml +++ b/samples/spring-data-mybatis/pom.xml @@ -13,7 +13,7 @@ org.springframework.boot spring-boot-starter-parent - 3.3.5 + 3.4.0 @@ -42,7 +42,7 @@ org.testcontainers testcontainers-bom - 1.20.3 + 1.20.4 import pom @@ -53,7 +53,7 @@ org.mybatis.spring.boot mybatis-spring-boot-starter - 3.0.3 + 3.0.4 org.mybatis.dynamic-sql diff --git a/src/test/java/com/google/cloud/spanner/jdbc/FallbackToPartitionedDMLMockServerTest.java b/src/test/java/com/google/cloud/spanner/jdbc/FallbackToPartitionedDMLMockServerTest.java new file mode 100644 index 000000000..587143a86 --- /dev/null +++ b/src/test/java/com/google/cloud/spanner/jdbc/FallbackToPartitionedDMLMockServerTest.java @@ -0,0 +1,259 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.jdbc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.spanner.Dialect; +import com.google.cloud.spanner.ErrorCode; +import com.google.cloud.spanner.MockSpannerServiceImpl; +import com.google.cloud.spanner.MockSpannerServiceImpl.SimulatedExecutionTime; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.TransactionMutationLimitExceededException; +import com.google.cloud.spanner.connection.AbstractMockServerTest; +import com.google.cloud.spanner.connection.AutocommitDmlMode; +import com.google.cloud.spanner.connection.SpannerPool; +import com.google.protobuf.Any; +import com.google.rpc.Help; +import com.google.rpc.Help.Link; +import com.google.spanner.v1.BeginTransactionRequest; +import com.google.spanner.v1.CommitRequest; +import com.google.spanner.v1.ExecuteSqlRequest; +import io.grpc.Metadata; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Properties; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class FallbackToPartitionedDMLMockServerTest extends AbstractMockServerTest { + + static StatusRuntimeException createTransactionMutationLimitExceededException() { + Metadata.Key key = + Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER); + Help help = + Help.newBuilder() + .addLinks( + Link.newBuilder() + .setDescription("Cloud Spanner limits documentation.") + .setUrl("https://cloud.google.com/spanner/docs/limits") + .build()) + .build(); + com.google.rpc.Status status = + com.google.rpc.Status.newBuilder().addDetails(Any.pack(help)).build(); + + Metadata trailers = new Metadata(); + trailers.put(key, status.toByteArray()); + + return Status.INVALID_ARGUMENT + .withDescription("The transaction contains too many mutations.") + .asRuntimeException(trailers); + } + + @Test + public void testConnectionProperty() throws SQLException { + for (AutocommitDmlMode mode : AutocommitDmlMode.values()) { + Properties properties = new Properties(); + properties.put("autocommit_dml_mode", mode.name()); + try (Connection connection = + DriverManager.getConnection("jdbc:" + getBaseUrl(), properties)) { + assertEquals( + mode, connection.unwrap(CloudSpannerJdbcConnection.class).getAutocommitDmlMode()); + } + } + } + + @Test + public void testTransactionMutationLimitExceeded_isNotRetriedByDefault() throws SQLException { + mockSpanner.setExecuteSqlExecutionTime( + SimulatedExecutionTime.ofException(createTransactionMutationLimitExceededException())); + + try (Connection connection = createJdbcConnection()) { + connection.setAutoCommit(true); + assertEquals( + AutocommitDmlMode.TRANSACTIONAL, + connection.unwrap(CloudSpannerJdbcConnection.class).getAutocommitDmlMode()); + + SQLException exception = + assertThrows( + SQLException.class, + () -> + connection.createStatement().executeUpdate("update test set value=1 where true")); + assertNotNull(exception.getCause()); + assertEquals( + TransactionMutationLimitExceededException.class, exception.getCause().getClass()); + TransactionMutationLimitExceededException transactionMutationLimitExceededException = + (TransactionMutationLimitExceededException) exception.getCause(); + assertEquals(0, transactionMutationLimitExceededException.getSuppressed().length); + } + assertEquals(1, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + assertEquals(0, mockSpanner.countRequestsOfType(CommitRequest.class)); + } + + @Test + public void testTransactionMutationLimitExceeded_canBeRetriedAsPDML() throws SQLException { + String sql = "update test set value=1 where true"; + com.google.cloud.spanner.Statement statement = com.google.cloud.spanner.Statement.of(sql); + mockSpanner.setExecuteSqlExecutionTime( + SimulatedExecutionTime.ofException(createTransactionMutationLimitExceededException())); + mockSpanner.putStatementResult( + MockSpannerServiceImpl.StatementResult.update(statement, 100000L)); + + try (Connection connection = createJdbcConnection()) { + connection.setAutoCommit(true); + connection + .unwrap(CloudSpannerJdbcConnection.class) + .setAutocommitDmlMode( + AutocommitDmlMode.TRANSACTIONAL_WITH_FALLBACK_TO_PARTITIONED_NON_ATOMIC); + + long updateCount = connection.createStatement().executeUpdate(sql); + assertEquals(100000L, updateCount); + } + // Verify that the request is retried as Partitioned DML. + assertEquals(2, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + // The transactional request uses inline-begin. + ExecuteSqlRequest transactionalRequest = + mockSpanner.getRequestsOfType(ExecuteSqlRequest.class).get(0); + assertTrue(transactionalRequest.getTransaction().getBegin().hasReadWrite()); + + // Partitioned DML uses an explicit BeginTransaction RPC. + assertEquals(1, mockSpanner.countRequestsOfType(BeginTransactionRequest.class)); + BeginTransactionRequest beginRequest = + mockSpanner.getRequestsOfType(BeginTransactionRequest.class).get(0); + assertTrue(beginRequest.getOptions().hasPartitionedDml()); + ExecuteSqlRequest partitionedDmlRequest = + mockSpanner.getRequestsOfType(ExecuteSqlRequest.class).get(1); + assertTrue(partitionedDmlRequest.getTransaction().hasId()); + + // Partitioned DML transactions are not committed. + assertEquals(0, mockSpanner.countRequestsOfType(CommitRequest.class)); + } + + @Test + public void testTransactionMutationLimitExceeded_retryAsPDMLFails() throws SQLException { + String sql = "insert into test (id, value) select -id, value from test"; + com.google.cloud.spanner.Statement statement = com.google.cloud.spanner.Statement.of(sql); + // The transactional update statement uses ExecuteSql(..). + mockSpanner.setExecuteSqlExecutionTime( + SimulatedExecutionTime.ofException(createTransactionMutationLimitExceededException())); + mockSpanner.putStatementResult( + MockSpannerServiceImpl.StatementResult.exception( + statement, + Status.INVALID_ARGUMENT + .withDescription("This statement is not supported with Partitioned DML") + .asRuntimeException())); + + try (Connection connection = createJdbcConnection()) { + connection.setAutoCommit(true); + connection + .unwrap(CloudSpannerJdbcConnection.class) + .setAutocommitDmlMode( + AutocommitDmlMode.TRANSACTIONAL_WITH_FALLBACK_TO_PARTITIONED_NON_ATOMIC); + + // The connection throws TransactionMutationLimitExceededException if the retry using + // partitioned DML fails. The exception from the failed retry is returned as a suppressed + // exception of the TransactionMutationLimitExceededException. + SQLException exception = + assertThrows(SQLException.class, () -> connection.createStatement().executeUpdate(sql)); + assertNotNull(exception.getCause()); + assertEquals( + TransactionMutationLimitExceededException.class, exception.getCause().getClass()); + TransactionMutationLimitExceededException transactionMutationLimitExceededException = + (TransactionMutationLimitExceededException) exception.getCause(); + assertEquals(1, transactionMutationLimitExceededException.getSuppressed().length); + assertEquals( + SpannerException.class, + transactionMutationLimitExceededException.getSuppressed()[0].getClass()); + SpannerException spannerException = + (SpannerException) transactionMutationLimitExceededException.getSuppressed()[0]; + assertEquals(ErrorCode.INVALID_ARGUMENT, spannerException.getErrorCode()); + assertTrue( + spannerException.getMessage(), + spannerException + .getMessage() + .contains("This statement is not supported with Partitioned DML")); + } + // Verify that the request was retried as Partitioned DML. + assertEquals(2, mockSpanner.countRequestsOfType(ExecuteSqlRequest.class)); + // The transactional request uses inline-begin. + ExecuteSqlRequest transactionalRequest = + mockSpanner.getRequestsOfType(ExecuteSqlRequest.class).get(0); + assertTrue(transactionalRequest.getTransaction().getBegin().hasReadWrite()); + + // Partitioned DML uses an explicit BeginTransaction RPC. + assertEquals(1, mockSpanner.countRequestsOfType(BeginTransactionRequest.class)); + BeginTransactionRequest beginRequest = + mockSpanner.getRequestsOfType(BeginTransactionRequest.class).get(0); + assertTrue(beginRequest.getOptions().hasPartitionedDml()); + ExecuteSqlRequest partitionedDmlRequest = + mockSpanner.getRequestsOfType(ExecuteSqlRequest.class).get(1); + assertTrue(partitionedDmlRequest.getTransaction().hasId()); + + // Partitioned DML transactions are not committed. + assertEquals(0, mockSpanner.countRequestsOfType(CommitRequest.class)); + } + + @Test + public void testSqlStatements() throws SQLException { + for (Dialect dialect : Dialect.values()) { + SpannerPool.closeSpannerPool(); + mockSpanner.putStatementResult( + MockSpannerServiceImpl.StatementResult.detectDialectResult(dialect)); + String prefix = dialect == Dialect.POSTGRESQL ? "SPANNER." : ""; + + try (Connection connection = createJdbcConnection()) { + connection.setAutoCommit(true); + try (ResultSet resultSet = + connection + .createStatement() + .executeQuery(String.format("show variable %sautocommit_dml_mode", prefix))) { + assertTrue(resultSet.next()); + assertEquals( + AutocommitDmlMode.TRANSACTIONAL.name(), + resultSet.getString(String.format("%sAUTOCOMMIT_DML_MODE", prefix))); + assertFalse(resultSet.next()); + } + connection + .createStatement() + .execute( + String.format( + "set %sautocommit_dml_mode = 'transactional_with_fallback_to_partitioned_non_atomic'", + prefix)); + try (ResultSet resultSet = + connection + .createStatement() + .executeQuery(String.format("show variable %sautocommit_dml_mode", prefix))) { + assertTrue(resultSet.next()); + assertEquals( + AutocommitDmlMode.TRANSACTIONAL_WITH_FALLBACK_TO_PARTITIONED_NON_ATOMIC.name(), + resultSet.getString(String.format("%sAUTOCOMMIT_DML_MODE", prefix))); + assertFalse(resultSet.next()); + } + } + } + } +} diff --git a/versions.txt b/versions.txt index 45c90fc28..04768795a 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-spanner-jdbc:2.24.2:2.24.2 +google-cloud-spanner-jdbc:2.25.0:2.25.0