feat: support Spanner PostgreSQL dialect (#485)

* chore: add getDialect() method (#465)

* add support postgesql Dialect

* add support postgesql Dialect

* add Liquibase system tables for PostgreSQL dialect

* fix tests

* fix formating

* resolve comments

* Add PostgreSQL dialect support for test configuration (#468)

* Add PostgreSQL dialect support for test configuration

* Add PostgreSQL dialect support for test configuration

* add changelog pg-sql file

* remove temporarily solution

* feat: PostgreSQL data type mappings (#478)

* Implement Cloud Spanner PostgreSQL data type mappings

* Implement Cloud Spanner PostgreSQL data type mappings

* resolve comments

* resolve comments

* fix timestamp test

* chore: re-throw original exception

---------

Co-authored-by: Knut Olav Løite <[email protected]>

* Refactor Spanner Data Types and Add PostgreSQL Dialect Support for Data Type Handling and SQL Generation (#480)

* add sql generation all type, default value and etc.

* add comments

* reformat

* add session management commands (PostgreSQL) and fix: INSERT OR UPDATE instead of INSERT OR IGNORE.

* feat: add sequence min/max validation and reformat

* fix: add explicit type casting for literals in PostgreSQL insert generation

* Enhance Mock Server Support for PostgreSQL Dialect in Liquibase-Spanner Tests (#513)

* Implement Posgresql dialect for unit tests

* resolve comments

* Add Liquibase Harness Tests for PostgreSQL Dialect (#519)

* add liquibase harness test for posgresql dialect

* resolve comments

* chore: try to downgrade JDBC version

---------

Co-authored-by: Vladimir Evdokimov <[email protected]>

Author: olavloite

build.gradle

@@ -78,8 +78,8 @@ dependencies {
         }
     }
     // Cloud Spanner related
+    implementation("com.google.cloud:google-cloud-spanner-jdbc:2.33.3")
     implementation platform('com.google.cloud:libraries-bom:26.71.0')
-    implementation("com.google.cloud:google-cloud-spanner-jdbc")
 
     // Liquibase Core - needed for testing and docker container
     implementation("org.liquibase:liquibase-core:4.33.0")
@@ -88,6 +88,7 @@ dependencies {
 
     // Testing
     testImplementation("org.junit.jupiter:junit-jupiter-api:5.13.4")
+    testImplementation("org.junit.jupiter:junit-jupiter-params:5.13.4")
     testImplementation("org.testcontainers:testcontainers:1.21.3")
     testImplementation("net.java.dev.jna:jna:5.17.0")
     testImplementation("com.google.truth:truth:1.4.4") {
@@ -96,7 +97,7 @@ dependencies {
 
     // For using the Liquibase test harness
     testImplementation 'junit:junit:4.13.2'
-    testImplementation ('org.liquibase:liquibase-test-harness:1.0.10'){
+    testImplementation ('org.liquibase:liquibase-test-harness:1.0.11'){
         exclude group: 'org.firebirdsql.jdbc', module: 'jaybird'
     }
     testImplementation('org.apache.groovy:groovy-all:4.0.28') {
@@ -127,6 +128,12 @@ test {
     excludeTags "integration"
     exclude '**/CloudSpannerBaseHarnessSuiteTest*'
     exclude '**/CloudSpannerAdvancedHarnessSuiteTest*'
+    exclude '**/CloudSpannerFoundationalHarnessSuiteTest*'
+  }
+
+  testLogging {
+    events "failed"
+    exceptionFormat "full"
   }
 }
 
@@ -157,6 +164,7 @@ tasks.register('integrationTest', Test) {
     includeTags "integration"
     exclude '**/CloudSpannerBaseHarnessSuiteTest*'
     exclude '**/CloudSpannerAdvancedHarnessSuiteTest*'
+    exclude '**/CloudSpannerFoundationalHarnessSuiteTest*'
   }
 }
 

pom.xml

@@ -84,6 +84,7 @@
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-spanner-jdbc</artifactId>
+      <version>2.33.3</version>
     </dependency>
 
     <!-- Liquibase test dependencies -->
@@ -150,6 +151,12 @@
       <version>${jupiter.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <version>${jupiter.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>testcontainers</artifactId>
@@ -393,7 +400,9 @@
             <configuration>
               <excludedGroups>integration</excludedGroups>
               <excludes>
-                <exclude>**/CloudSpannerHarnessTest.java</exclude>
+                <exclude>**/CloudSpannerAdvancedHarnessSuiteTest*</exclude>
+                <exclude>**/CloudSpannerBaseHarnessSuiteTest*</exclude>
+                <exclude>**/CloudSpannerFoundationalHarnessSuiteTest*</exclude>
               </excludes>
             </configuration>
           </plugin>
@@ -439,6 +448,18 @@
               </execution>
             </executions>
           </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <excludedGroups>integration</excludedGroups>
+              <excludes>
+                <exclude>**/CloudSpannerAdvancedHarnessSuiteTest*</exclude>
+                <exclude>**/CloudSpannerBaseHarnessSuiteTest*</exclude>
+                <exclude>**/CloudSpannerFoundationalHarnessSuiteTest*</exclude>
+              </excludes>
+            </configuration>
+          </plugin>
         </plugins>
       </build>
     </profile>

src/main/java/liquibase/ext/spanner/CloudSpanner.java

@@ -16,15 +16,18 @@
 import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE;
 import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME;
 
+import com.google.cloud.spanner.Dialect;
 import com.google.cloud.spanner.Type;
 import com.google.cloud.spanner.jdbc.CloudSpannerJdbcConnection;
+import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.text.ParseException;
 import java.time.Instant;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.Date;
+import java.util.Objects;
 import liquibase.Scope;
 import liquibase.database.AbstractJdbcDatabase;
 import liquibase.database.DatabaseConnection;
@@ -59,24 +62,37 @@ public boolean dataTypeIsNotModifiable(final String typeName) {
   @Override
   public String getDateLiteral(final String isoDate) {
     // Construct the literal based on whether it is a DATE or TIMESTAMP
-    if (isDateTime(isoDate)) {
-      try {
+    if (isoDate == null || isoDate.trim().equalsIgnoreCase("null")) {
+      return this.getDialect() == Dialect.POSTGRESQL ? "CAST(NULL AS timestamptz)" : "NULL";
+    }
+    try {
+      if (isDateTime(isoDate)) {
         Date date = new ISODateFormat().parse(isoDate);
         Instant instant = date.toInstant();
         OffsetDateTime utcDateTime = instant.atOffset(ZoneOffset.UTC);
         String formattedDate = utcDateTime.format(ISO_LOCAL_DATE);
         String formattedTime = utcDateTime.format(ISO_LOCAL_TIME);
-        return "TIMESTAMP '" + formattedDate + "T" + formattedTime + "Z'";
-      } catch (ParseException e) {
-        return "BAD_DATE_FORMAT:" + isoDate;
+        return this.getDialect() == Dialect.POSTGRESQL
+            ? "'" + formattedDate + "T" + formattedTime + "Z'" + "::timestamptz"
+            : "TIMESTAMP '" + formattedDate + "T" + formattedTime + "Z'";
+
+      } else {
+        String formattedDate = super.getDateLiteral(isoDate);
+        return this.getDialect() == Dialect.POSTGRESQL
+            ? formattedDate + "::date"
+            : "DATE " + formattedDate;
       }
-    } else {
-      return "DATE " + super.getDateLiteral(isoDate);
+    } catch (ParseException e) {
+      return "BAD_DATE_FORMAT:" + isoDate;
     }
   }
 
   @Override
   public String getCurrentDateTimeFunction() {
+    Dialect dialect = this.getDialect();
+    if (Objects.requireNonNull(dialect) == Dialect.POSTGRESQL) {
+      return "CURRENT_TIMESTAMP";
+    }
     return "CURRENT_TIMESTAMP()";
   }
 
@@ -243,21 +259,46 @@ public boolean supportsPrimaryKeyNames() {
 
   @Override
   protected String getQuotingStartCharacter() {
-    return "`";
+    Dialect dialect = this.getDialect();
+    return dialect == Dialect.POSTGRESQL ? "\"" : "`";
   }
 
   @Override
   protected String getQuotingEndCharacter() {
-    return "`";
+    Dialect dialect = this.getDialect();
+    return dialect == Dialect.POSTGRESQL ? "\"" : "`";
   }
 
   @Override
   protected String getQuotingEndReplacement() {
-    return "\\`";
+    Dialect dialect = this.getDialect();
+    return dialect == Dialect.POSTGRESQL ? "\"" : "\\`";
   }
 
   @Override
   public String escapeStringForDatabase(String string) {
+    Dialect dialect = this.getDialect();
+    if (dialect == Dialect.POSTGRESQL) {
+      return string == null ? null : string.replace("'", "''");
+    }
     return string == null ? null : string.replace("'", "\\'");
   }
+
+  @Override
+  public Dialect getDialect() {
+    try {
+      DatabaseConnection conn = getConnection();
+
+      if (conn instanceof JdbcConnection) {
+        Connection underlying = ((JdbcConnection) conn).getUnderlyingConnection();
+        if (underlying.isWrapperFor(CloudSpannerJdbcConnection.class)) {
+          return underlying.unwrap(CloudSpannerJdbcConnection.class).getDialect();
+        }
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException("Failed to get dialect from connection", e);
+    }
+
+    return null;
+  }
 }

src/main/java/liquibase/ext/spanner/ICloudSpanner.java

@@ -1,5 +1,8 @@
 package liquibase.ext.spanner;
 
+import com.google.cloud.spanner.Dialect;
 import liquibase.database.Database;
 
-public interface ICloudSpanner extends Database {}
+public interface ICloudSpanner extends Database {
+  Dialect getDialect();
+}

src/main/java/liquibase/ext/spanner/change/AddLookupTableChangeSpanner.java

@@ -13,6 +13,7 @@
  */
 package liquibase.ext.spanner.change;
 
+import com.google.cloud.spanner.Dialect;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -21,6 +22,8 @@
 import liquibase.change.core.AddForeignKeyConstraintChange;
 import liquibase.change.core.AddLookupTableChange;
 import liquibase.database.Database;
+import liquibase.datatype.DataTypeFactory;
+import liquibase.datatype.LiquibaseDataType;
 import liquibase.ext.spanner.ICloudSpanner;
 import liquibase.statement.SqlStatement;
 import liquibase.statement.core.RawSqlStatement;
@@ -48,6 +51,11 @@ public SqlStatement[] generateStatements(Database database) {
     String existingTableCatalogName = getExistingTableCatalogName();
     String existingTableSchemaName = getExistingTableSchemaName();
 
+    Dialect dialect = ((ICloudSpanner) database).getDialect();
+    String rawType = getNewColumnDataType();
+    LiquibaseDataType liquibaseType =
+        DataTypeFactory.getInstance().fromDescription(rawType, database);
+    String actualType = liquibaseType.toDatabaseDataType(database).toString();
     SqlStatement[] createTablesSQL =
         new SqlStatement[] {
           new RawSqlStatement(
@@ -57,10 +65,12 @@ public SqlStatement[] generateStatements(Database database) {
                   + " ("
                   + database.escapeObjectName(getNewColumnName(), Column.class)
                   + " "
-                  + getNewColumnDataType()
-                  + " NOT NULL) PRIMARY KEY ("
+                  + actualType
+                  + (dialect == Dialect.POSTGRESQL
+                      ? " NOT NULL, PRIMARY KEY ("
+                      : " NOT NULL) PRIMARY KEY (")
                   + database.escapeObjectName(getNewColumnName(), Column.class)
-                  + ")"),
+                  + (dialect == Dialect.POSTGRESQL ? "))" : ")")),
           new RawSqlStatement(
               "INSERT INTO "
                   + database.escapeTableName(

src/main/java/liquibase/ext/spanner/change/DropAllForeignKeyConstraintsChangeSpanner.java

@@ -37,14 +37,18 @@ public SqlStatement[] generateStatements(Database database) {
     try (PreparedStatement ps =
         connection.prepareStatement(
             "SELECT CONSTRAINT_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE TABLE_CATALOG=? AND TABLE_SCHEMA=? AND TABLE_NAME=? AND CONSTRAINT_TYPE='FOREIGN KEY'")) {
-      ps.setString(1, MoreObjects.firstNonNull(getBaseTableCatalogName(), ""));
-      ps.setString(2, MoreObjects.firstNonNull(getBaseTableSchemaName(), ""));
+      String catalogName =
+          MoreObjects.firstNonNull(getBaseTableCatalogName(), database.getDefaultCatalogName());
+      String schemaName =
+          MoreObjects.firstNonNull(getBaseTableSchemaName(), database.getDefaultSchemaName());
+      ps.setString(1, catalogName);
+      ps.setString(2, schemaName);
       ps.setString(3, getBaseTableName());
       try (ResultSet rs = ps.executeQuery()) {
         while (rs.next()) {
           DropForeignKeyConstraintChange drop = new DropForeignKeyConstraintChange();
-          drop.setBaseTableCatalogName(getBaseTableCatalogName());
-          drop.setBaseTableSchemaName(getBaseTableSchemaName());
+          drop.setBaseTableCatalogName(catalogName);
+          drop.setBaseTableSchemaName(schemaName);
           drop.setBaseTableName(getBaseTableName());
           drop.setConstraintName(rs.getString(1));
           sqlStatements.addAll(Arrays.asList(drop.generateStatements(database)));

src/main/java/liquibase/ext/spanner/change/MergeColumnsChangeSpanner.java

@@ -13,6 +13,7 @@
  */
 package liquibase.ext.spanner.change;
 
+import com.google.cloud.spanner.Dialect;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -23,6 +24,7 @@
 import liquibase.change.core.DropColumnChange;
 import liquibase.change.core.MergeColumnChange;
 import liquibase.database.Database;
+import liquibase.ext.spanner.ICloudSpanner;
 import liquibase.statement.SqlStatement;
 import liquibase.statement.core.RawSqlStatement;
 import liquibase.structure.core.Column;
@@ -32,7 +34,7 @@
  * UPDATE and DELETE statements to include a WHERE clause, even when all rows should be
  * updated/deleted. This feature is a safety precaution against accidental updates/deletes.
  *
- * <p>{@link SpannerMergeColumnsChange} will use a Partitioned DML statement to fill the data in the
+ * <p>{@link MergeColumnsChangeSpanner} will use a Partitioned DML statement to fill the data in the
  * new column.
  */
 @DatabaseChange(
@@ -44,6 +46,7 @@ public class MergeColumnsChangeSpanner extends MergeColumnChange {
 
   @Override
   public SqlStatement[] generateStatements(final Database database) {
+    Dialect dialect = ((ICloudSpanner) database).getDialect();
     List<SqlStatement> statements = new ArrayList<>();
 
     AddColumnChange addNewColumnChange = new AddColumnChange();
@@ -55,9 +58,14 @@ public SqlStatement[] generateStatements(final Database database) {
     columnConfig.setType(getFinalColumnType());
     addNewColumnChange.addColumn(columnConfig);
     statements.addAll(Arrays.asList(addNewColumnChange.generateStatements(database)));
-
-    statements.add(new RawSqlStatement("SET AUTOCOMMIT=TRUE"));
-    statements.add(new RawSqlStatement("SET AUTOCOMMIT_DML_MODE='PARTITIONED_NON_ATOMIC'"));
+    if (dialect == Dialect.POSTGRESQL) {
+      statements.add(new RawSqlStatement("set autocommit=true"));
+      statements.add(
+          new RawSqlStatement("set spanner.autocommit_dml_mode='partitioned_non_atomic'"));
+    } else {
+      statements.add(new RawSqlStatement("SET AUTOCOMMIT=TRUE"));
+      statements.add(new RawSqlStatement("SET AUTOCOMMIT_DML_MODE='PARTITIONED_NON_ATOMIC'"));
+    }
     String updateStatement =
         "UPDATE "
             + database.escapeTableName(getCatalogName(), getSchemaName(), getTableName())
@@ -70,8 +78,11 @@ public SqlStatement[] generateStatements(final Database database) {
                 database.escapeObjectName(getColumn2Name(), Column.class))
             + " WHERE TRUE";
     statements.add(new RawSqlStatement(updateStatement));
-    statements.add(new RawSqlStatement("SET AUTOCOMMIT_DML_MODE='TRANSACTIONAL'"));
-
+    if (dialect == Dialect.POSTGRESQL) {
+      statements.add(new RawSqlStatement("set spanner.autocommit_dml_mode='transactional'"));
+    } else {
+      statements.add(new RawSqlStatement("SET AUTOCOMMIT_DML_MODE='TRANSACTIONAL'"));
+    }
     DropColumnChange dropColumn1Change = new DropColumnChange();
     dropColumn1Change.setCatalogName(getCatalogName());
     dropColumn1Change.setSchemaName(getSchemaName());