From 7ccaf3dc86003b46337fbcd79f63c8d03410b375 Mon Sep 17 00:00:00 2001 From: Philipp Omenitsch Date: Tue, 2 Sep 2025 19:11:43 +0100 Subject: [PATCH 1/6] feat(firestore-bigquery-export): add collection group query support - Add --is-collection-group-query flag to gen-schema-view script - Support both regular collection and collection group queries in Gemini schema generation - Update sampleFirestoreDocuments function with isCollectionGroupQuery parameter - Add interactive prompt for collection group query selection - Update documentation with examples and explanations - Add comprehensive test coverage for new functionality - Maintain backward compatibility with existing functionality This allows users to generate schemas for subcollections that appear across multiple parent documents by using collection group queries instead of regular collection queries. --- .../guides/GENERATE_SCHEMA_VIEWS.md | 42 +++++++++ .../src/__tests__/config/index.test.ts | 3 + .../src/__tests__/config/interactive.test.ts | 2 +- .../__tests__/config/non-interactive.test.ts | 8 +- .../genkit/sampleFirestoreDocuments.test.ts | 94 +++++++++++++++++++ .../gen-schema-view/src/config/index.ts | 4 + .../gen-schema-view/src/config/interactive.ts | 8 ++ .../src/config/non-interactive.ts | 5 + .../gen-schema-view/src/schema/genkit.ts | 24 +++-- package-lock.json | 8 ++ package.json | 3 + 11 files changed, 192 insertions(+), 9 deletions(-) diff --git a/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md b/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md index 8c28bef7a..c4cd138e9 100644 --- a/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md +++ b/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md @@ -60,6 +60,7 @@ You'll be prompted for: - BigQuery dataset ID - Table Prefix - Firestore collection path to sample +- Whether to use collection group query - Google AI API key - Directory and filename for the schema @@ -78,6 +79,34 @@ npx @firebaseextensions/fs-bq-schema-views \ --gemini-schema-file-name=user_schema ``` +For collection group queries (to query all collections with the same name across your database): + +```bash +npx @firebaseextensions/fs-bq-schema-views \ + --non-interactive \ + --project=my-firebase-project \ + --big-query-project=my-bq-project \ + --dataset=firestore_changelog \ + --table-name-prefix=user_profiles \ + --use-gemini=secure \ + --is-collection-group-query \ + --google-ai-key=$GOOGLE_API_KEY \ + --schema-directory=./schemas \ + --gemini-schema-file-name=user_schema +``` + +#### Understanding Collection vs Collection Group Queries + +- **Collection Query** (default): Queries documents from a specific collection path + + - Example: `users/123/orders` - queries orders for a specific user + - Use when you have a specific collection path + +- **Collection Group Query** (`--is-collection-group-query`): Queries all collections with the same name across your entire database + - Example: `orders` - queries all order collections regardless of their parent path + - Use when you have collections with the same name under different documents + - Useful for subcollections that appear in multiple places + ⚠️ **Important**: Always review generated schemas before using them in production. ### Option 2: Create a Schema File Manually @@ -133,6 +162,19 @@ npx @firebaseextensions/fs-bq-schema-views \ --schema-files=./test_schema.json ``` +For collection group queries with manual schemas: + +```bash +npx @firebaseextensions/fs-bq-schema-views \ + --non-interactive \ + --project=YOUR_PROJECT_ID \ + --big-query-project=YOUR_BIGQUERY_PROJECT_ID \ + --dataset=YOUR_DATASET_ID \ + --table-name-prefix=YOUR_TABLE_PREFIX \ + --schema-files=./test_schema.json \ + --is-collection-group-query +``` + For multiple schema files, use comma separation: ``` diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts index b7597bca9..8d7287eab 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts @@ -68,6 +68,7 @@ describe("parseConfig", () => { googleAiKey: undefined, schemaDirectory: undefined, useGemini: false, + isCollectionGroupQuery: undefined, }); }); @@ -107,6 +108,7 @@ describe("parseConfig", () => { googleAiKey: "test-key", geminiAnalyzeCollectionPath: "test-collection", schemaDirectory: "test-directory", + isCollectionGroupQuery: true, outputHelp: jest.fn(), }; @@ -120,6 +122,7 @@ describe("parseConfig", () => { expect(result.geminiAnalyzeCollectionPath).toBe("test-collection"); expect(result.schemaDirectory).toBe("test-directory"); expect(result.agentSampleSize).toBe(100); + expect(result.isCollectionGroupQuery).toBe(true); }); it("should exit if required parameters are missing", async () => { diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/interactive.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/interactive.test.ts index 46fa7672a..3a5b7aba3 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/interactive.test.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/interactive.test.ts @@ -24,7 +24,7 @@ describe("Interactive Prompts", () => { describe("questions array", () => { it("should have the correct number of questions", () => { - expect(questions).toHaveLength(10); + expect(questions).toHaveLength(11); }); it("should have properly formatted questions with required properties", () => { diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/non-interactive.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/non-interactive.test.ts index 8a4bf0942..b1be7b34e 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/non-interactive.test.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/non-interactive.test.ts @@ -69,7 +69,7 @@ describe("Command Line Parser", () => { expect(commander.version).toHaveBeenCalledWith("1.0.0"); // Check that all options are configured - expect(commander.option).toHaveBeenCalledTimes(10); + expect(commander.option).toHaveBeenCalledTimes(11); // Check specific options - just a sample to ensure we're setting up correctly expect(commander.option).toHaveBeenCalledWith( @@ -89,6 +89,12 @@ describe("Command Line Parser", () => { collect, [] ); + + expect(commander.option).toHaveBeenCalledWith( + "--is-collection-group-query", + "Use collection group query instead of regular collection query", + false + ); }); it("should return the configured program", () => { diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts index f57ab0834..2b5efcd76 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts @@ -17,6 +17,7 @@ interface FirestoreModule { (): { collection: jest.Mock; + collectionGroup: jest.Mock; where: jest.Mock; limit: jest.Mock; get: jest.Mock; @@ -31,6 +32,7 @@ interface FirestoreModule { jest.mock("firebase-admin", () => { const mockFirestore = { collection: jest.fn().mockReturnThis(), + collectionGroup: jest.fn().mockReturnThis(), where: jest.fn().mockReturnThis(), limit: jest.fn().mockReturnThis(), get: jest.fn().mockResolvedValue({ @@ -136,4 +138,96 @@ describe("sampleFirestoreDocuments", () => { sampleFirestoreDocuments(collectionPath, sampleSize) ).rejects.toThrow("Firestore error"); }); + + describe("collection group queries", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("should sample documents from Firestore collection group", async () => { + const collectionPath = "orders"; + const sampleSize = 2; + const isCollectionGroupQuery = true; + + // Mock collection group data (subcollections from different parents) + const firebase = require("firebase-admin"); + const mockFirestore = firebase.firestore(); + + mockFirestore.get.mockResolvedValueOnce({ + docs: [ + { + data: () => ({ orderId: "order1", amount: 50, userId: "user1" }), + id: "order1", + }, + { + data: () => ({ orderId: "order2", amount: 75, userId: "user2" }), + id: "order2", + }, + ], + }); + + const result = await sampleFirestoreDocuments( + collectionPath, + sampleSize, + isCollectionGroupQuery + ); + + expect(mockFirestore.collectionGroup).toHaveBeenCalledWith( + collectionPath + ); + expect(mockFirestore.collection).not.toHaveBeenCalled(); + expect(mockFirestore.where).toHaveBeenCalledWith( + "__name__", + ">=", + expect.any(String) + ); + expect(mockFirestore.limit).toHaveBeenCalledWith(sampleSize); + expect(mockFirestore.get).toHaveBeenCalled(); + + expect(result).toHaveLength(2); + expect(result[0]).toHaveProperty("orderId", "order1"); + expect(result[0]).toHaveProperty("amount", 50); + expect(result[0]).toHaveProperty("userId", "user1"); + }); + + it("should default to regular collection query when isCollectionGroupQuery is false", async () => { + const collectionPath = "test-collection"; + const sampleSize = 2; + const isCollectionGroupQuery = false; + + const result = await sampleFirestoreDocuments( + collectionPath, + sampleSize, + isCollectionGroupQuery + ); + + const firebase = require("firebase-admin"); + const mockFirestore = firebase.firestore(); + + expect(mockFirestore.collection).toHaveBeenCalledWith(collectionPath); + expect(mockFirestore.collectionGroup).not.toHaveBeenCalled(); + expect(result).toHaveLength(2); + }); + + it("should handle errors properly for collection group queries", async () => { + const firebase = require("firebase-admin"); + const mockFirestore = firebase.firestore(); + + mockFirestore.get.mockRejectedValueOnce( + new Error("Collection group error") + ); + + const collectionPath = "orders"; + const sampleSize = 2; + const isCollectionGroupQuery = true; + + await expect( + sampleFirestoreDocuments( + collectionPath, + sampleSize, + isCollectionGroupQuery + ) + ).rejects.toThrow("Collection group error"); + }); + }); }); diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/config/index.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/config/index.ts index 25fbfaa82..9f213b8f3 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/config/index.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/config/index.ts @@ -33,6 +33,7 @@ export interface CliConfig { googleAiKey?: string; schemaDirectory?: string; geminiSchemaFileName?: string; + isCollectionGroupQuery?: boolean; } export async function parseConfig(): Promise { @@ -55,6 +56,7 @@ export async function parseConfig(): Promise { googleAiKey: program.googleAiKey, schemaDirectory: program.schemaDirectory, geminiSchemaFileName: program.geminiSchemaFileName, + isCollectionGroupQuery: program.isCollectionGroupQuery, }; } const { @@ -68,6 +70,7 @@ export async function parseConfig(): Promise { googleAiKey, schemaDirectory, geminiSchemaFileName, + isCollectionGroupQuery, } = await promptInquirer(); return { @@ -82,5 +85,6 @@ export async function parseConfig(): Promise { googleAiKey, schemaDirectory, geminiSchemaFileName, + isCollectionGroupQuery, }; } diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/config/interactive.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/config/interactive.ts index 24104ebb5..ac9c8d718 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/config/interactive.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/config/interactive.ts @@ -112,6 +112,14 @@ export const questions = [ when: (answers) => answers.useGemini, default: "schema", }, + { + message: + "Do you want to use a collection group query instead of a regular collection query?", + name: "isCollectionGroupQuery", + type: "confirm", + when: (answers) => answers.useGemini, + default: false, + }, ]; export const promptInquirer = () => { diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/config/non-interactive.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/config/non-interactive.ts index c6e659638..6d02a876d 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/config/non-interactive.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/config/non-interactive.ts @@ -74,6 +74,11 @@ export const configureProgram = () => { "--gemini-schema-file-name ", "Name of schema json file generated by Gemini (without .json extension)", "schema" + ) + .option( + "--is-collection-group-query", + "Use collection group query instead of regular collection query", + false ); return program; diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/schema/genkit.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/schema/genkit.ts index 31259bb9f..8119196cc 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/schema/genkit.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/schema/genkit.ts @@ -24,16 +24,25 @@ import inquirer from "inquirer"; export async function sampleFirestoreDocuments( collectionPath: string, - sampleSize: number + sampleSize: number, + isCollectionGroupQuery: boolean = false ): Promise { const db = firebase.firestore(); try { - const snapshot = await db - .collection(collectionPath) - .where("__name__", ">=", Math.random().toString()) - .limit(sampleSize) - .get(); + const query = isCollectionGroupQuery + ? db.collectionGroup(collectionPath) + : db.collection(collectionPath); + + let snapshot = null; + if (isCollectionGroupQuery) { + snapshot = await query.limit(sampleSize).get(); + } else { + snapshot = await query + .where("__name__", ">=", Math.random().toString()) + .limit(sampleSize) + .get(); + } const documents = snapshot.docs.map((doc) => { const data = doc.data(); @@ -197,7 +206,8 @@ export const generateSchemaFilesWithGemini = async (config: CliConfig) => { // get sample data from Firestore const sampleData = await sampleFirestoreDocuments( config.geminiAnalyzeCollectionPath!, - config.agentSampleSize! + config.agentSampleSize!, + config.isCollectionGroupQuery || false ); if (sampleData.length === 0) { diff --git a/package-lock.json b/package-lock.json index 5f37ec7cf..bdd733034 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,9 @@ "version": "1.0.0", "hasInstallScript": true, "license": "Apache-2.0", + "dependencies": { + "@firebaseextensions/firestore-bigquery-change-tracker": "file:./firestore-bigquery-change-tracker" + }, "devDependencies": { "@types/jest": "29.5.0", "codecov": "^3.8.1", @@ -22,6 +25,7 @@ "typescript": "^4.8.4" } }, + "firestore-bigquery-change-tracker": {}, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -694,6 +698,10 @@ "semver": "bin/semver" } }, + "node_modules/@firebaseextensions/firestore-bigquery-change-tracker": { + "resolved": "firestore-bigquery-change-tracker", + "link": true + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", diff --git a/package.json b/package.json index 4088af372..45c0c4b4f 100644 --- a/package.json +++ b/package.json @@ -39,5 +39,8 @@ }, "lint-staged": { "*.{js,md,yml,ts,json,yaml}": "prettier --write" + }, + "dependencies": { + "@firebaseextensions/firestore-bigquery-change-tracker": "file:./firestore-bigquery-change-tracker" } } From 3d8281c66c9874c5e5a3418b9549c23f5e53eac8 Mon Sep 17 00:00:00 2001 From: Philipp Omenitsch Date: Tue, 2 Sep 2025 19:19:14 +0100 Subject: [PATCH 2/6] refactor: change flag name to --query-collection-group - Update command line flag from --is-collection-group-query to --query-collection-group - Update all documentation and examples with new flag name - Update tests to use new flag name - Maintain same functionality with cleaner flag naming This provides a more concise and intuitive flag name for collection group queries. --- .../guides/GENERATE_SCHEMA_VIEWS.md | 6 +++--- .../src/__tests__/config/index.test.ts | 2 +- .../__tests__/config/non-interactive.test.ts | 2 +- .../genkit/sampleFirestoreDocuments.test.ts | 17 ++++++++++------- .../scripts/gen-schema-view/src/config/index.ts | 2 +- .../src/config/non-interactive.ts | 2 +- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md b/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md index c4cd138e9..8d8f230b6 100644 --- a/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md +++ b/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md @@ -89,7 +89,7 @@ npx @firebaseextensions/fs-bq-schema-views \ --dataset=firestore_changelog \ --table-name-prefix=user_profiles \ --use-gemini=secure \ - --is-collection-group-query \ + --query-collection-group \ --google-ai-key=$GOOGLE_API_KEY \ --schema-directory=./schemas \ --gemini-schema-file-name=user_schema @@ -102,7 +102,7 @@ npx @firebaseextensions/fs-bq-schema-views \ - Example: `users/123/orders` - queries orders for a specific user - Use when you have a specific collection path -- **Collection Group Query** (`--is-collection-group-query`): Queries all collections with the same name across your entire database +- **Collection Group Query** (`--query-collection-group`): Queries all collections with the same name across your entire database - Example: `orders` - queries all order collections regardless of their parent path - Use when you have collections with the same name under different documents - Useful for subcollections that appear in multiple places @@ -172,7 +172,7 @@ npx @firebaseextensions/fs-bq-schema-views \ --dataset=YOUR_DATASET_ID \ --table-name-prefix=YOUR_TABLE_PREFIX \ --schema-files=./test_schema.json \ - --is-collection-group-query + --query-collection-group ``` For multiple schema files, use comma separation: diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts index 8d7287eab..e84df471b 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts @@ -108,7 +108,7 @@ describe("parseConfig", () => { googleAiKey: "test-key", geminiAnalyzeCollectionPath: "test-collection", schemaDirectory: "test-directory", - isCollectionGroupQuery: true, + queryCollectionGroup: true, outputHelp: jest.fn(), }; diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/non-interactive.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/non-interactive.test.ts index b1be7b34e..8764736cf 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/non-interactive.test.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/non-interactive.test.ts @@ -91,7 +91,7 @@ describe("Command Line Parser", () => { ); expect(commander.option).toHaveBeenCalledWith( - "--is-collection-group-query", + "--query-collection-group", "Use collection group query instead of regular collection query", false ); diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts index 2b5efcd76..33e65560a 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts @@ -140,10 +140,6 @@ describe("sampleFirestoreDocuments", () => { }); describe("collection group queries", () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - it("should sample documents from Firestore collection group", async () => { const collectionPath = "orders"; const sampleSize = 2; @@ -153,6 +149,8 @@ describe("sampleFirestoreDocuments", () => { const firebase = require("firebase-admin"); const mockFirestore = firebase.firestore(); + // Clear mocks and set up specific mock for this test + jest.clearAllMocks(); mockFirestore.get.mockResolvedValueOnce({ docs: [ { @@ -195,15 +193,18 @@ describe("sampleFirestoreDocuments", () => { const sampleSize = 2; const isCollectionGroupQuery = false; + const firebase = require("firebase-admin"); + const mockFirestore = firebase.firestore(); + + // Clear mocks for this test + jest.clearAllMocks(); + const result = await sampleFirestoreDocuments( collectionPath, sampleSize, isCollectionGroupQuery ); - const firebase = require("firebase-admin"); - const mockFirestore = firebase.firestore(); - expect(mockFirestore.collection).toHaveBeenCalledWith(collectionPath); expect(mockFirestore.collectionGroup).not.toHaveBeenCalled(); expect(result).toHaveLength(2); @@ -213,6 +214,8 @@ describe("sampleFirestoreDocuments", () => { const firebase = require("firebase-admin"); const mockFirestore = firebase.firestore(); + // Clear mocks and set up error for this test + jest.clearAllMocks(); mockFirestore.get.mockRejectedValueOnce( new Error("Collection group error") ); diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/config/index.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/config/index.ts index 9f213b8f3..92f01ef09 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/config/index.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/config/index.ts @@ -56,7 +56,7 @@ export async function parseConfig(): Promise { googleAiKey: program.googleAiKey, schemaDirectory: program.schemaDirectory, geminiSchemaFileName: program.geminiSchemaFileName, - isCollectionGroupQuery: program.isCollectionGroupQuery, + isCollectionGroupQuery: program.queryCollectionGroup, }; } const { diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/config/non-interactive.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/config/non-interactive.ts index 6d02a876d..8e69e6681 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/config/non-interactive.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/config/non-interactive.ts @@ -76,7 +76,7 @@ export const configureProgram = () => { "schema" ) .option( - "--is-collection-group-query", + "--query-collection-group", "Use collection group query instead of regular collection query", false ); From 53b87bd9106e871d4ca2d62351d74261558f260d Mon Sep 17 00:00:00 2001 From: Philipp Omenitsch Date: Tue, 9 Sep 2025 17:49:48 +0100 Subject: [PATCH 3/6] Remove Firebase extensions dependency from package.json --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index 45c0c4b4f..4088af372 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,5 @@ }, "lint-staged": { "*.{js,md,yml,ts,json,yaml}": "prettier --write" - }, - "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "file:./firestore-bigquery-change-tracker" } } From 0f4a40f0b4f2b488a2d99c6b61594dc77bdfc406 Mon Sep 17 00:00:00 2001 From: Philipp Omenitsch Date: Tue, 9 Sep 2025 17:51:23 +0100 Subject: [PATCH 4/6] Remove firestore-bigquery-change-tracker dependency --- package-lock.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index bdd733034..5f37ec7cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,6 @@ "version": "1.0.0", "hasInstallScript": true, "license": "Apache-2.0", - "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "file:./firestore-bigquery-change-tracker" - }, "devDependencies": { "@types/jest": "29.5.0", "codecov": "^3.8.1", @@ -25,7 +22,6 @@ "typescript": "^4.8.4" } }, - "firestore-bigquery-change-tracker": {}, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -698,10 +694,6 @@ "semver": "bin/semver" } }, - "node_modules/@firebaseextensions/firestore-bigquery-change-tracker": { - "resolved": "firestore-bigquery-change-tracker", - "link": true - }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", From 3093c2eb6bf4464452fc50bc049f7fe7a694cf77 Mon Sep 17 00:00:00 2001 From: Philipp Omenitsch Date: Tue, 23 Sep 2025 08:46:02 +0100 Subject: [PATCH 5/6] fix test for collection group query --- .../src/__tests__/genkit/sampleFirestoreDocuments.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts index 33e65560a..12b1a72b0 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/genkit/sampleFirestoreDocuments.test.ts @@ -174,11 +174,7 @@ describe("sampleFirestoreDocuments", () => { collectionPath ); expect(mockFirestore.collection).not.toHaveBeenCalled(); - expect(mockFirestore.where).toHaveBeenCalledWith( - "__name__", - ">=", - expect.any(String) - ); + expect(mockFirestore.where).not.toHaveBeenCalled(); expect(mockFirestore.limit).toHaveBeenCalledWith(sampleSize); expect(mockFirestore.get).toHaveBeenCalled(); From 273f5db6aa98e8d238461b772efd885137e6fdcf Mon Sep 17 00:00:00 2001 From: Philipp Omenitsch Date: Thu, 25 Sep 2025 12:50:57 +0100 Subject: [PATCH 6/6] - Modify the prompt to clarify the option for using a collection group query. - Update the message for the collection path to include both collection and collection group options. --- .../gen-schema-view/src/config/interactive.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/config/interactive.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/config/interactive.ts index ac9c8d718..8329a6a38 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/src/config/interactive.ts +++ b/firestore-bigquery-export/scripts/gen-schema-view/src/config/interactive.ts @@ -90,7 +90,15 @@ export const questions = [ }, { message: - "What is the Firestore collection path you want Gemini to analyze?", + "Do you want to use a collection group query instead of a regular collection query?", + name: "isCollectionGroupQuery", + type: "confirm", + when: (answers) => answers.useGemini, + default: false, + }, + { + message: + "What is the Firestore collection or collection group path you want Gemini to analyze?", name: "geminiAnalyzeCollectionPath", type: "input", when: (answers) => answers.useGemini, @@ -112,14 +120,6 @@ export const questions = [ when: (answers) => answers.useGemini, default: "schema", }, - { - message: - "Do you want to use a collection group query instead of a regular collection query?", - name: "isCollectionGroupQuery", - type: "confirm", - when: (answers) => answers.useGemini, - default: false, - }, ]; export const promptInquirer = () => {