Code refactoring

.changeset/blue-impalas-cheer.md

@@ -0,0 +1,5 @@
+---
+'@electric-sql/pglite': patch
+---
+
+Added dict_int, dict_xsyn, file_fdw, intarray, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility, pg_walinspect, unaccent contrib extensions

docs/docs/videos.md

@@ -3,3 +3,11 @@
 Compiling Postgres to WASM with PGlite (Sam Willis at [pgconf.dev](https://pgconf.dev)).
 
 <iframe width="640" height="480" src="https://www.youtube.com/embed/hlWWG5WZHOA" frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>
+
+Compiling Postgres to WASM with PGlite (Sam Willis at [San Francisco Bay Area PostgreSQL Users Group](https://www.youtube.com/@sanfranciscobayareapostgre4592))
+
+<iframe width="640" height="480" src="https://www.youtube.com/embed/cPMi1GbICmE" frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>
+
+Vibe coding with a database in the sandbox - [Bolt](https://bolt.new) + PGlite - Demo video
+
+<iframe width="640" height="480" src="https://www.youtube.com/embed/1XI2WPbSleo" frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>

docs/extensions/extensions.data.ts

@@ -179,6 +179,183 @@ const baseExtensions: Extension[] = [
     core: true,
     size: 21380,
   },
+  {
+    name: 'intarray',
+    description: `
+      The intarray module provides a number of useful functions and operators for 
+      manipulating null-free arrays of integers. There is also support for indexed 
+      searches using some of the operators.
+    `,
+    shortDescription: 'Operators for manipulating null-free arrays of integers',
+    docs: 'https://www.postgresql.org/docs/9.1/intarray.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/intarray',
+    importName: 'intarray',
+    core: true,
+    size: 14712,
+  },
+  {
+    name: 'dict_xsyn',
+    description: `
+      dict_xsyn (Extended Synonym Dictionary) is an example of an add-on dictionary 
+      template for full-text search. This dictionary type replaces words with groups 
+      of their synonyms, and so makes it possible to search for a word using any of 
+      its synonyms.
+    `,
+    shortDescription: 'Example synonym full-text search dictionary',
+    docs: 'https://www.postgresql.org/docs/18/dict-xsyn.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/dict_xsyn',
+    importName: 'dict_xsyn',
+    core: true,
+    size: 1948,
+  },
+  {
+    name: 'pageinspect',
+    description: `
+      The pageinspect module provides functions that allow you to inspect the contents 
+      of database pages at a low level, which is useful for debugging purposes. All of 
+      these functions may be used only by superusers.
+    `,
+    shortDescription: 'Low-level inspection of database pages ',
+    docs: 'https://www.postgresql.org/docs/18/pageinspect.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/pageinspect',
+    importName: 'pageinspect',
+    core: true,
+    size: 15923,
+  },
+  {
+    name: 'dict_int',
+    description: `
+      dict_int is an example of an add-on dictionary template for full-text search. 
+      The motivation for this example dictionary is to control the indexing of integers 
+      (signed and unsigned), allowing such numbers to be indexed while preventing 
+      excessive growth in the number of unique words, which greatly affects the 
+      performance of searching.
+    `,
+    shortDescription: 'Example full-text search dictionary for integers',
+    docs: 'https://www.postgresql.org/docs/18/dict-int.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/dict_int',
+    importName: 'dict_int',
+    core: true,
+    size: 1361,
+  },
+  {
+    name: 'unaccent',
+    description: `
+      unaccent is a text search dictionary that removes accents (diacritic signs) 
+      from lexemes. It's a filtering dictionary, which means its output is always 
+      passed to the next dictionary (if any), unlike the normal behavior of 
+      dictionaries. This allows accent-insensitive processing for full text search.
+    `,
+    shortDescription: 'A text search dictionary which removes diacritics',
+    docs: 'https://www.postgresql.org/docs/current/unaccent.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/unaccent',
+    importName: 'unaccent',
+    core: true,
+    size: 9323,
+  },
+  {
+    name: 'pg_surgery',
+    description: `
+      The pg_surgery module provides various functions to perform surgery on a damaged 
+      relation. These functions are unsafe by design and using them may corrupt 
+      (or further corrupt) your database. For example, these functions can easily be 
+      used to make a table inconsistent with its own indexes, to cause UNIQUE or 
+      FOREIGN KEY constraint violations, or even to make tuples visible which, when read, 
+      will cause a database server crash. They should be used with great caution and 
+      only as a last resort.
+    `,
+    shortDescription: 'Perform low-level surgery on relation data',
+    docs: 'https://www.postgresql.org/docs/current/pgsurgery.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/pg_surgery',
+    importName: 'pg_surgery',
+    core: true,
+    size: 2635,
+  },
+  {
+    name: 'pg_walinspect',
+    description: `
+      The pg_walinspect module provides SQL functions that allow you to inspect the 
+      contents of write-ahead log of a running PostgreSQL database cluster at a low level, 
+      which is useful for debugging, analytical, reporting or educational purposes. 
+      It is similar to pg_waldump, but accessible through SQL rather than a separate utility.
+    `,
+    shortDescription: 'Low-level WAL inspection',
+    docs: 'https://www.postgresql.org/docs/current/pgwalinspect.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/pg_walinspect',
+    importName: 'pg_walinspect',
+    core: true,
+    size: 4689,
+  },
+  {
+    name: 'pg_visibility',
+    description: `
+      The pg_visibility module provides a means for examining the visibility map (VM) 
+      and page-level visibility information of a table. It also provides functions to 
+      check the integrity of a visibility map and to force it to be rebuilt.
+    `,
+    shortDescription: 'Visibility map information and utilities',
+    docs: 'https://www.postgresql.org/docs/current/pgvisibility.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/pg_visibility',
+    importName: 'pg_visibility',
+    core: true,
+    size: 4159,
+  },
+  {
+    name: 'pg_freespacemap',
+    description: `
+      The pg_freespacemap module provides a means for examining the free space map (FSM). 
+      It provides a function called pg_freespace, or two overloaded functions, to be precise. 
+      The functions show the value recorded in the free space map for a given page, or 
+      for all pages in the relation.
+    `,
+    shortDescription: 'Examine the free space map',
+    docs: 'https://www.postgresql.org/docs/current/pgfreespacemap.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/pg_freespacemap',
+    importName: 'pg_freespacemap',
+    core: true,
+    size: 1485,
+  },
+  {
+    name: 'pg_buffercache',
+    description: `
+      The pg_buffercache module provides a means for examining what's happening in the 
+      shared buffer cache in real time. It also offers a low-level way to evict data 
+      from it, for testing purposes.
+    `,
+    shortDescription: 'Inspect PostgreSQL buffer cache state',
+    docs: 'https://www.postgresql.org/docs/current/pgbuffercache.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/pg_buffercache',
+    importName: 'pg_buffercache',
+    core: true,
+    size: 3133,
+  },
+  {
+    name: 'file_fdw',
+    description: `
+      The file_fdw module provides the foreign-data wrapper file_fdw, which can be 
+      used to access data files in the server's file system, or to execute programs 
+      on the server and read their output. The data file or program output must be 
+      in a format that can be read by COPY FROM. Access to data files is currently 
+      read-only.
+    `,
+    shortDescription: "Acess data files in the server's file system",
+    docs: 'https://www.postgresql.org/docs/18/file-fdw.html',
+    tags: ['postgres extension', 'postgres/contrib'],
+    importPath: '@electric-sql/pglite/contrib/file_fdw',
+    importName: 'file_fdw',
+    core: true,
+    size: 4467,
+  },
   {
     name: 'isn',
     description: `

docs/repl/allExtensions.ts

@@ -1,21 +1,32 @@
-export { vector } from '@electric-sql/pglite/vector'
 export { amcheck } from '@electric-sql/pglite/contrib/amcheck'
 export { auto_explain } from '@electric-sql/pglite/contrib/auto_explain'
 export { bloom } from '@electric-sql/pglite/contrib/bloom'
 export { btree_gin } from '@electric-sql/pglite/contrib/btree_gin'
 export { btree_gist } from '@electric-sql/pglite/contrib/btree_gist'
 export { citext } from '@electric-sql/pglite/contrib/citext'
 export { cube } from '@electric-sql/pglite/contrib/cube'
+export { dict_int } from '@electric-sql/pglite/contrib/dict_int'
+export { dict_xsyn } from '@electric-sql/pglite/contrib/dict_xsyn'
 export { earthdistance } from '@electric-sql/pglite/contrib/earthdistance'
+export { file_fdw } from '@electric-sql/pglite/contrib/file_fdw'
 export { fuzzystrmatch } from '@electric-sql/pglite/contrib/fuzzystrmatch'
 export { hstore } from '@electric-sql/pglite/contrib/hstore'
+export { intarray } from '@electric-sql/pglite/contrib/intarray'
 export { isn } from '@electric-sql/pglite/contrib/isn'
 export { lo } from '@electric-sql/pglite/contrib/lo'
 export { ltree } from '@electric-sql/pglite/contrib/ltree'
+export { pageinspect } from '@electric-sql/pglite/contrib/pageinspect'
+export { pg_buffercache } from '@electric-sql/pglite/contrib/pg_buffercache'
+export { pg_freespacemap } from '@electric-sql/pglite/contrib/pg_freespacemap'
+export { pg_surgery } from '@electric-sql/pglite/contrib/pg_surgery'
 export { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'
+export { pg_visibility } from '@electric-sql/pglite/contrib/pg_visibility'
+export { pg_walinspect } from '@electric-sql/pglite/contrib/pg_walinspect'
 export { seg } from '@electric-sql/pglite/contrib/seg'
 export { tablefunc } from '@electric-sql/pglite/contrib/tablefunc'
 export { tcn } from '@electric-sql/pglite/contrib/tcn'
 export { tsm_system_rows } from '@electric-sql/pglite/contrib/tsm_system_rows'
 export { tsm_system_time } from '@electric-sql/pglite/contrib/tsm_system_time'
+export { unaccent } from '@electric-sql/pglite/contrib/unaccent'
 export { uuid_ossp } from '@electric-sql/pglite/contrib/uuid_ossp'
+export { vector } from '@electric-sql/pglite/vector'

package.json

@@ -12,7 +12,7 @@
     "ci:publish": "pnpm changeset publish",
     "ts:build": "pnpm -r --filter \"./packages/**\" build",
     "ts:build:debug": "DEBUG=true pnpm ts:build",
-    "wasm:build": "cd postgres-pglite && ./build-with-docker.sh; cd .. && mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/",
+    "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/",
     "wasm:build:debug": "DEBUG=true pnpm wasm:build",
     "build:all": "pnpm wasm:build && pnpm ts:build",
     "build:all:debug": "DEBUG=true pnpm build:all"

packages/benchmark/src/rtt.js

@@ -9,13 +9,11 @@ const CONFIGURATIONS = new Map(
       label: 'PGlite Memory<br> (CMA Transport <em>default</em>)',
       db: 'pglite',
       dataDir: '',
-      options: { defaultDataTransferContainer: 'cma' },
     },
     {
       label: 'PGlite Memory<br> (File Transport)',
       db: 'pglite',
       dataDir: '',
-      options: { defaultDataTransferContainer: 'file' },
     },
     {
       label: 'PGlite IDB',

packages/pglite/src/base.ts

@@ -65,7 +65,7 @@ export abstract class BasePGlite
    */
   abstract execProtocolRaw(
     message: Uint8Array,
-    { syncToFs, dataTransferContainer }: ExecProtocolOptions,
+    { syncToFs }: ExecProtocolOptions,
   ): Promise<Uint8Array>
 
   /**
@@ -228,7 +228,7 @@ export abstract class BasePGlite
       this.#log('runQuery', query, params, options)
       await this._handleBlob(options?.blob)
 
-      let results
+      let results = []
 
       try {
         const { messages: parseResults } = await this.#execProtocolNoSync(
@@ -288,7 +288,10 @@ export abstract class BasePGlite
         }
         throw e
       } finally {
-        await this.#execProtocolNoSync(serializeProtocol.sync(), options)
+        results.push(
+          ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options))
+            .messages,
+        )
       }
 
       await this._cleanupBlob()
@@ -315,7 +318,7 @@ export abstract class BasePGlite
       // No params so we can just send the query
       this.#log('runExec', query, options)
       await this._handleBlob(options?.blob)
-      let results
+      let results = []
       try {
         results = (
           await this.#execProtocolNoSync(
@@ -335,7 +338,10 @@ export abstract class BasePGlite
         }
         throw e
       } finally {
-        await this.#execProtocolNoSync(serializeProtocol.sync(), options)
+        results.push(
+          ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options))
+            .messages,
+        )
       }
       this._cleanupBlob()
       if (!this.#inTransaction) {
@@ -360,38 +366,19 @@ export abstract class BasePGlite
     query: string,
     options?: QueryOptions,
   ): Promise<DescribeQueryResult> {
+    let messages = []
     try {
       await this.#execProtocolNoSync(
         serializeProtocol.parse({ text: query, types: options?.paramTypes }),
         options,
       )
 
-      const describeResults = await this.#execProtocolNoSync(
-        serializeProtocol.describe({ type: 'S' }),
-        options,
-      )
-      const paramDescription = describeResults.messages.find(
-        (msg): msg is ParameterDescriptionMessage =>
-          msg.name === 'parameterDescription',
-      )
-      const resultDescription = describeResults.messages.find(
-        (msg): msg is RowDescriptionMessage => msg.name === 'rowDescription',
-      )
-
-      const queryParams =
-        paramDescription?.dataTypeIDs.map((dataTypeID) => ({
-          dataTypeID,
-          serializer: this.serializers[dataTypeID],
-        })) ?? []
-
-      const resultFields =
-        resultDescription?.fields.map((field) => ({
-          name: field.name,
-          dataTypeID: field.dataTypeID,
-          parser: this.parsers[field.dataTypeID],
-        })) ?? []
-
-      return { queryParams, resultFields }
+      messages = (
+        await this.#execProtocolNoSync(
+          serializeProtocol.describe({ type: 'S' }),
+          options,
+        )
+      ).messages
     } catch (e) {
       if (e instanceof DatabaseError) {
         const pgError = makePGliteError({
@@ -404,8 +391,34 @@ export abstract class BasePGlite
       }
       throw e
     } finally {
-      await this.#execProtocolNoSync(serializeProtocol.sync(), options)
+      messages.push(
+        ...(await this.#execProtocolNoSync(serializeProtocol.sync(), options))
+          .messages,
+      )
     }
+
+    const paramDescription = messages.find(
+      (msg): msg is ParameterDescriptionMessage =>
+        msg.name === 'parameterDescription',
+    )
+    const resultDescription = messages.find(
+      (msg): msg is RowDescriptionMessage => msg.name === 'rowDescription',
+    )
+
+    const queryParams =
+      paramDescription?.dataTypeIDs.map((dataTypeID) => ({
+        dataTypeID,
+        serializer: this.serializers[dataTypeID],
+      })) ?? []
+
+    const resultFields =
+      resultDescription?.fields.map((field) => ({
+        name: field.name,
+        dataTypeID: field.dataTypeID,
+        parser: this.parsers[field.dataTypeID],
+      })) ?? []
+
+    return { queryParams, resultFields }
   }
 
   /**