feat(db-postgres): support read replicas (#12728)

Adds support for read replicas
https://orm.drizzle.team/docs/read-replicas that can be used to offload
read-heavy traffic.

To use (both `db-postgres` and `db-vercel-postgres` are supported):
```ts
import { postgresAdapter } from '@payloadcms/db-postgres'

database: postgresAdapter({
  pool: {
    connectionString: process.env.POSTGRES_URL,
  },
  readReplicas: [process.env.POSTGRES_REPLICA_URL],
})
```

---------

Co-authored-by: Dan Ribbens <[email protected]>

Author: r1tsuu

docs/database/postgres.mdx

@@ -80,6 +80,7 @@ export default buildConfig({
 | `afterSchemaInit`           | Drizzle schema hook. Runs after the schema is built. [More Details](#afterschemainit)                                                                                            |
 | `generateSchemaOutputFile`  | Override generated schema from `payload generate:db-schema` file path. Defaults to `{CWD}/src/payload-generated.schema.ts`                                                       |
 | `allowIDOnCreate`           | Set to `true` to use the `id` passed in data on the create API operations without using a custom ID field.                                                                       |
+| `readReplicas`              | An array of DB read replicas connection strings, can be used to offload read-heavy traffic.                                                                                      |
 
 ## Access to Drizzle
 

packages/db-postgres/src/connect.ts

@@ -1,31 +1,32 @@
 import type { DrizzleAdapter } from '@payloadcms/drizzle/types'
-import type { Connect, Migration, Payload } from 'payload'
+import type { Connect, Migration } from 'payload'
 
 import { pushDevSchema } from '@payloadcms/drizzle'
 import { drizzle } from 'drizzle-orm/node-postgres'
+import { withReplicas } from 'drizzle-orm/pg-core'
 
 import type { PostgresAdapter } from './types.js'
 
 const connectWithReconnect = async function ({
   adapter,
-  payload,
+  pool,
   reconnect = false,
 }: {
   adapter: PostgresAdapter
-  payload: Payload
+  pool: PostgresAdapter['pool']
   reconnect?: boolean
 }) {
   let result
 
   if (!reconnect) {
-    result = await adapter.pool.connect()
+    result = await pool.connect()
   } else {
     try {
-      result = await adapter.pool.connect()
+      result = await pool.connect()
     } catch (ignore) {
       setTimeout(() => {
-        payload.logger.info('Reconnecting to postgres')
-        void connectWithReconnect({ adapter, payload, reconnect: true })
+        adapter.payload.logger.info('Reconnecting to postgres')
+        void connectWithReconnect({ adapter, pool, reconnect: true })
       }, 1000)
     }
   }
@@ -35,7 +36,7 @@ const connectWithReconnect = async function ({
   result.prependListener('error', (err) => {
     try {
       if (err.code === 'ECONNRESET') {
-        void connectWithReconnect({ adapter, payload, reconnect: true })
+        void connectWithReconnect({ adapter, pool, reconnect: true })
       }
     } catch (ignore) {
       // swallow error
@@ -54,12 +55,29 @@ export const connect: Connect = async function connect(
   try {
     if (!this.pool) {
       this.pool = new this.pg.Pool(this.poolOptions)
-      await connectWithReconnect({ adapter: this, payload: this.payload })
+      await connectWithReconnect({ adapter: this, pool: this.pool })
     }
 
     const logger = this.logger || false
     this.drizzle = drizzle({ client: this.pool, logger, schema: this.schema })
 
+    if (this.readReplicaOptions) {
+      const readReplicas = this.readReplicaOptions.map((connectionString) => {
+        const options = {
+          ...this.poolOptions,
+          connectionString,
+        }
+        const pool = new this.pg.Pool(options)
+        void connectWithReconnect({
+          adapter: this,
+          pool,
+        })
+        return drizzle({ client: pool, logger, schema: this.schema })
+      })
+      const myReplicas = withReplicas(this.drizzle, readReplicas as any)
+      this.drizzle = myReplicas
+    }
+
     if (!hotReload) {
       if (process.env.PAYLOAD_DROP_DATABASE === 'true') {
         this.payload.logger.info(`---- DROPPING TABLES SCHEMA(${this.schemaName || 'public'}) ----`)

packages/db-postgres/src/index.ts

@@ -139,6 +139,7 @@ export function postgresAdapter(args: Args): DatabaseAdapterObj<PostgresAdapter>
       prodMigrations: args.prodMigrations,
       // @ts-expect-error - vestiges of when tsconfig was not strict. Feel free to improve
       push: args.push,
+      readReplicaOptions: args.readReplicas,
       relations: {},
       relationshipsSuffix: args.relationshipsSuffix || '_rels',
       schema: {},

packages/db-postgres/src/types.ts

@@ -3,13 +3,19 @@ import type {
   GenericEnum,
   MigrateDownArgs,
   MigrateUpArgs,
-  PostgresDB,
   PostgresSchemaHook,
 } from '@payloadcms/drizzle/postgres'
 import type { DrizzleAdapter } from '@payloadcms/drizzle/types'
-import type { DrizzleConfig } from 'drizzle-orm'
+import type { DrizzleConfig, ExtractTablesWithRelations } from 'drizzle-orm'
 import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
-import type { PgSchema, PgTableFn, PgTransactionConfig } from 'drizzle-orm/pg-core'
+import type {
+  PgDatabase,
+  PgQueryResultHKT,
+  PgSchema,
+  PgTableFn,
+  PgTransactionConfig,
+  PgWithReplicas,
+} from 'drizzle-orm/pg-core'
 import type { Pool, PoolConfig } from 'pg'
 
 type PgDependency = typeof import('pg')
@@ -55,6 +61,7 @@ export type Args = {
     up: (args: MigrateUpArgs) => Promise<void>
   }[]
   push?: boolean
+  readReplicas?: string[]
   relationshipsSuffix?: string
   /**
    * The schema name to use for the database
@@ -74,7 +81,16 @@ type ResolveSchemaType<T> = 'schema' extends keyof T
   ? T['schema']
   : GeneratedDatabaseSchema['schemaUntyped']
 
-type Drizzle = NodePgDatabase<ResolveSchemaType<GeneratedDatabaseSchema>>
+type Drizzle =
+  | NodePgDatabase<ResolveSchemaType<GeneratedDatabaseSchema>>
+  | PgWithReplicas<
+      PgDatabase<
+        PgQueryResultHKT,
+        Record<string, unknown>,
+        ExtractTablesWithRelations<Record<string, unknown>>
+      >
+    >
+
 export type PostgresAdapter = {
   drizzle: Drizzle
   pg: PgDependency

packages/db-vercel-postgres/src/connect.ts

@@ -4,6 +4,7 @@ import type { Connect, Migration } from 'payload'
 import { pushDevSchema } from '@payloadcms/drizzle'
 import { sql, VercelPool } from '@vercel/postgres'
 import { drizzle } from 'drizzle-orm/node-postgres'
+import { withReplicas } from 'drizzle-orm/pg-core'
 import pg from 'pg'
 
 import type { VercelPostgresAdapter } from './types.js'
@@ -46,6 +47,19 @@ export const connect: Connect = async function connect(
       schema: this.schema,
     })
 
+    if (this.readReplicaOptions) {
+      const readReplicas = this.readReplicaOptions.map((connectionString) => {
+        const options = {
+          ...this.poolOptions,
+          connectionString,
+        }
+        const pool = new VercelPool(options)
+        return drizzle({ client: pool, logger, schema: this.schema })
+      })
+      const myReplicas = withReplicas(this.drizzle, readReplicas as any)
+      this.drizzle = myReplicas
+    }
+
     if (!hotReload) {
       if (process.env.PAYLOAD_DROP_DATABASE === 'true') {
         this.payload.logger.info(`---- DROPPING TABLES SCHEMA(${this.schemaName || 'public'}) ----`)

packages/db-vercel-postgres/src/index.ts

@@ -174,6 +174,7 @@ export function vercelPostgresAdapter(args: Args = {}): DatabaseAdapterObj<Verce
       find,
       findGlobal,
       findGlobalVersions,
+      readReplicaOptions: args.readReplicas,
       // @ts-expect-error - vestiges of when tsconfig was not strict. Feel free to improve
       findOne,
       findVersions,

packages/db-vercel-postgres/src/types.ts

@@ -64,6 +64,7 @@ export type Args = {
     up: (args: MigrateUpArgs) => Promise<void>
   }[]
   push?: boolean
+  readReplicas?: string[]
   relationshipsSuffix?: string
   /**
    * The schema name to use for the database

packages/drizzle/src/postgres/types.ts

@@ -159,6 +159,7 @@ export type BasePostgresAdapter = {
     up: (args: MigrateUpArgs) => Promise<void>
   }[]
   push: boolean
+  readReplicaOptions?: string[]
   rejectInitializing: () => void
   relations: Record<string, GenericRelation>
   relationshipsSuffix?: string

test/database/pg-replica/docker-compose.yaml

@@ -0,0 +1,36 @@
+# Copyright Broadcom, Inc. All Rights Reserved.
+# SPDX-License-Identifier: APACHE-2.0
+
+services:
+  postgresql-master:
+    image: docker.io/bitnami/postgresql:17
+    ports:
+      - '5433:5432'
+    volumes:
+      - 'postgresql_master_data:/bitnami/postgresql'
+    environment:
+      - POSTGRESQL_REPLICATION_MODE=master
+      - POSTGRESQL_REPLICATION_USER=repl_user
+      - POSTGRESQL_REPLICATION_PASSWORD=repl_password
+      - POSTGRESQL_USERNAME=postgres
+      - POSTGRESQL_PASSWORD=my_password
+      - POSTGRESQL_DATABASE=my_database
+      - ALLOW_EMPTY_PASSWORD=yes
+  postgresql-slave:
+    image: docker.io/bitnami/postgresql:17
+    ports:
+      - '5434:5432'
+    depends_on:
+      - postgresql-master
+    environment:
+      - POSTGRESQL_REPLICATION_MODE=slave
+      - POSTGRESQL_REPLICATION_USER=repl_user
+      - POSTGRESQL_REPLICATION_PASSWORD=repl_password
+      - POSTGRESQL_MASTER_HOST=postgresql-master
+      - POSTGRESQL_PASSWORD=my_password
+      - POSTGRESQL_MASTER_PORT_NUMBER=5432
+      - ALLOW_EMPTY_PASSWORD=yes
+
+volumes:
+  postgresql_master_data:
+    driver: local

test/generateDatabaseAdapter.ts

@@ -47,6 +47,26 @@ export const allDatabaseAdapters = {
       connectionString: process.env.POSTGRES_URL || 'postgres://127.0.0.1:5432/payloadtests',
     },
   })`,
+  'postgres-read-replica': `
+  import { postgresAdapter } from '@payloadcms/db-postgres'
+
+  export const databaseAdapter = postgresAdapter({
+    pool: {
+      connectionString: process.env.POSTGRES_URL,
+    },
+    readReplicas: [process.env.POSTGRES_REPLICA_URL],
+  })
+  `,
+  'vercel-postgres-read-replica': `
+  import { vercelPostgresAdapter } from '@payloadcms/db-vercel-postgres'
+
+  export const databaseAdapter = vercelPostgresAdapter({
+    pool: {
+      connectionString: process.env.POSTGRES_URL,
+    },
+    readReplicas: [process.env.POSTGRES_REPLICA_URL],
+  })
+  `,
   sqlite: `
   import { sqliteAdapter } from '@payloadcms/db-sqlite'