From 629c206532cae2ea18168df3a253365e6ef52cde Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Thu, 4 Sep 2025 19:21:16 +0530 Subject: [PATCH 01/26] VertexAI Initial draft- prompt response logging --- .github/CODEOWNERS | 1 + .../metricbeat/exported-fields-gcp.md | 90 ++++++++ .../metricbeat/metricbeat-module-gcp.md | 9 + x-pack/metricbeat/include/list.go | 1 + x-pack/metricbeat/metricbeat.reference.yml | 9 + x-pack/metricbeat/module/gcp/_meta/config.yml | 11 + x-pack/metricbeat/module/gcp/fields.go | 2 +- .../module/gcp/vertexai_logs/_meta/data.json | 95 ++++++++ .../module/gcp/vertexai_logs/_meta/fields.yml | 50 +++++ .../module/gcp/vertexai_logs/data.go | 129 +++++++++++ .../module/gcp/vertexai_logs/vertexai_logs.go | 206 ++++++++++++++++++ x-pack/metricbeat/modules.d/gcp.yml.disabled | 9 + 12 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 x-pack/metricbeat/module/gcp/vertexai_logs/_meta/data.json create mode 100644 x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/gcp/vertexai_logs/data.go create mode 100644 x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6afd968e80a8..c2cc631c8474 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -236,6 +236,7 @@ CHANGELOG* /x-pack/metricbeat/module/coredns @elastic/obs-infraobs-integrations /x-pack/metricbeat/module/enterprisesearch @elastic/app-search-team /x-pack/metricbeat/module/gcp @elastic/obs-ds-hosted-services @elastic/obs-infraobs-integrations +/x-pack/metricbeat/module/gcp/vertexai_logs @elastic/obs-infraobs-integrations /x-pack/metricbeat/module/gcp/billing @elastic/obs-infraobs-integrations /x-pack/metricbeat/module/gcp/cloudrun_metrics @elastic/obs-infraobs-integrations /x-pack/metricbeat/module/gcp/cloudsql_mysql @elastic/obs-infraobs-integrations diff --git a/docs/reference/metricbeat/exported-fields-gcp.md b/docs/reference/metricbeat/exported-fields-gcp.md index 71485a2fa389..65675a5e55df 100644 --- a/docs/reference/metricbeat/exported-fields-gcp.md +++ b/docs/reference/metricbeat/exported-fields-gcp.md @@ -1201,3 +1201,93 @@ Google Cloud Storage metrics type: long +## vertexai_logs [_vertexai_logs] + +```{applies_to} +stack: beta +``` + +Google Cloud Vertex AI Prompt Response Logs metrics + +**`gcp.vertexai_logs.endpoint`** +: The Vertex AI API endpoint URL used for the request. + + type: keyword + + +**`gcp.vertexai_logs.deployed_model_id`** +: The ID of the deployed model that processed the request. + + type: keyword + + +**`gcp.vertexai_logs.logging_time`** +: Timestamp when the AI interaction was logged. + + type: date + + +**`gcp.vertexai_logs.request_id`** +: Unique identifier for the AI request. + + type: double + + +**`gcp.vertexai_logs.request_payload`** +: Array of request payload strings containing user prompts and inputs. + + type: text + +Field is not indexed. + + +**`gcp.vertexai_logs.response_payload`** +: Array of response payload strings containing AI model outputs. + + type: text + +Field is not indexed. + + +**`gcp.vertexai_logs.model`** +: Name of the AI model used (e.g., gemini-2.5-pro). + + type: keyword + + +**`gcp.vertexai_logs.model_version`** +: Version of the AI model used. + + type: keyword + + +**`gcp.vertexai_logs.api_method`** +: The API method called (e.g., generateContent, predict). + + type: keyword + + +**`gcp.vertexai_logs.full_request`** +: Complete request object containing all request details in JSON format. + + type: object + + +**`gcp.vertexai_logs.full_response`** +: Complete response object containing all response details in JSON format. + + type: object + + +**`gcp.vertexai_logs.metadata`** +: Additional metadata associated with the AI interaction in JSON format. + + type: object + + +**`gcp.vertexai_logs.otel_log`** +: OpenTelemetry log data associated with the AI interaction in JSON format. + + type: object + + diff --git a/docs/reference/metricbeat/metricbeat-module-gcp.md b/docs/reference/metricbeat/metricbeat-module-gcp.md index 51e7478caa17..33dc0081c57c 100644 --- a/docs/reference/metricbeat/metricbeat-module-gcp.md +++ b/docs/reference/metricbeat/metricbeat-module-gcp.md @@ -361,6 +361,15 @@ metricbeat.modules: exclude_labels: false period: 1m collect_dataproc_user_labels: true + +- module: gcp + metricsets: + - vertexai_logs + period: 300s # 5 minutes + project_id: "your-project-id" + table_id: "your-project-id.dataset.id.table_name" + credentials_file_path: "/path/to/service-account.json" + # credentials_json: '{"type": "service_account", ...}' ``` diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 055ecb30bac3..af78ad3a9427 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -48,6 +48,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/billing" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/carbon" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/vertexai_logs" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/ibmmq" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/iis" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/iis/application_pool" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 16d017bc2957..9c1e2fd0d4c4 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -697,6 +697,15 @@ metricbeat.modules: period: 1m collect_dataproc_user_labels: true +- module: gcp + metricsets: + - vertexai_logs + period: 300s # 5 minutes + project_id: "your-project-id" + table_id: "your-project-id.dataset.id.table_name" + credentials_file_path: "/path/to/service-account.json" + # credentials_json: '{"type": "service_account", ...}' + #-------------------------------- Golang Module -------------------------------- - module: golang #metricsets: diff --git a/x-pack/metricbeat/module/gcp/_meta/config.yml b/x-pack/metricbeat/module/gcp/_meta/config.yml index 33339e9ecfc0..2285ab6385f2 100644 --- a/x-pack/metricbeat/module/gcp/_meta/config.yml +++ b/x-pack/metricbeat/module/gcp/_meta/config.yml @@ -82,3 +82,14 @@ exclude_labels: false period: 1m collect_dataproc_user_labels: true + +- module: gcp + metricsets: + - vertexai_logs + period: 300s # 5 minutes + project_id: "your-project-id" + table_id: "your-project-id.dataset.id.table_name" + credentials_file_path: "/path/to/service-account.json" + # credentials_json: '{"type": "service_account", ...}' + + diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index 04b919fd0a6c..5292259a2175 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzsXduP27aaf89fQZyXNIupu00X+xAsDjCdnPQETXoG9aTAPulQ5GebMUWqJDUT969f8CZRtmzLuniaYpOnsSXy91343fiR/hZtYfcGrUn5AiHDDIc36OVPUq45oDsuK4ruOTYrqYqXLxBSwAFreIPW+AVCFDRRrDRMijfo7y8QQuinu3tUSFpxeIHQigGn+o374lskcAFxIvvP7Er7t5JV/CR9Pn2H4xy4rj+Or8r8MxCTfNyBJ/7zuAQzUjGxRgUYxYg+HHkfQgqj0qAW/9H66igU+89/mPkntrB7kop2DlyAwRQbPNfgltRZxtY7baCYZWgFWlaKwGSDx4H/VjPE///beb1qjUtllTvt7vg2K3BZMrEOj/6tNfgJ7fwY1NFssEEKTKUEULRSskCtpXh7/x79XoHaLQ7IyhnnTKyPzdca5kf/bFSN5J32+m6zJV2pqHOpRCxEas+PF4eC65J5C+md1MY9qxEThFcUkIJ1xbG6QQZ/uUGYfq60KUCYG4QFRUpWglqmg1JSLTrwMPEoGYGskMJshmCKDFNQSmWQG6drolJJpwuMDpnl3r+N3r9FcoXMBqJQ47w5cCnWGhnZNbmRBvOOeVdcYnN81gf7Wj0TLmQlTNfwelsNpOthAwlNcWFbc0pRvnMfalCPjMCxeZPhhgC4Tf88wGEH6QSD3kmF4AsuSg43CO+9sZIqLKelkQqvATGNlgYLilXz2adlJ01+hkn4GcbytsN+UOkABmstCcMGKHpi3QobgYxksEXUsjAtUF57gTp0bQvkvF6XLuO17gAiQBs4KWgiOQcS5byF3bePmFeASsxUsK+lko+MAsKUMvsg5o0Dbg3dFQs0ELew2/vmFLua9xye3m/Gt2C1smQ9QlYqRroM67llvgFENlitgSI3hFNgrytBk1oSXP78STvzuvz5EzIMlF6gX2FlmasRkcIoTIwbycqRrRAuS84Izu1akWYD6olpuEHMvNRudM60fx4OnRfBKm9p3nHfdeceRe+kNKViwpxyYjmYMW5sMmtuyfckolWNu5dNjxDsX2NA2PdHwBhnr5bBGLx/63SuE4ZTxBktVMSQOoLLwShYD5z/V/dmx5RnZqxBLTSRJXx/+cJf2vfQ9wdU9pjt9YJLgo+wvNe0r7uYaw1GHPjbHFvPW4DZSCq5XO964Sqw2oKZHJUfdhCmH4aC+eFCycjVSoPp8o69Ar0wWRilwxTLoqwM9LPF/tnZ8ogVU/CEOV9QJcsS6CLfGeii3Nqu44S/F0QWVrzudRQGi9FenKTH/FmJyRaMzogLkg89+UVowmAX4GFCGywILEhZLRRY0wg0I1KBPgrmIF3dg/NLVeSgbAjgxkFxWCSFg7Ox6VgIEOL856A5a5YZVsBCAxkA6pMLS6yxxJwHYEwgDUQKqntNvyhJl304N7MNk1Y2tEkyBcydtQKK7u4/+RiSaUQqpUAYvrPIKg2RYX2YRJneLhTgoRp9Z/XPwvMabUfy1QI7cK+JM1mOU+Magh3SI3j/LyRLUM6unxSSQ/GkmIFp6LdDGRDIyH4McFNPzAE3Zn8WFFBItVvkVrekWChcZJr9AQOhWK116X/I3y0qP4NVTquVv31coIcN08FWWwWWgu8QfsSM26DdrbbfPoYcyYeDlqH2ZXiNVrhgvNMPniLJZtUDSfro4TerzGXoz0aNfsJlxsRAfbXyOZCMWzNMBFTrCrTxi5gZjeSTQHZOpEtM4FmolZWZkty4SB2JDcVGPge9AsyTVNsFE2sFWk9lhgmwx1i4tWDCNJcgCVHBwlmm4YhicDEKE0zIHA32r0dQA0FMzJeL4VTliXjm9ORNiBUCmGDD0AZrlAMIpCohmFifVFkPIHNmfhCMf3BcWhtqh0GaCQIRxxPWSBusDNCbJM5aoKWrfVIEj6B26L//s/nmdmVAIW2/Z2J94wp5dqEKadAj0yyu0qq0C/P7182rBxmHfbVUkvRKOd6GhyfPOZIMiFfagFps6EovLDghKXQp3lG2H0T8lNn40ctd1Nrwz7fvlo6gX+wE3phhBVEbrOhdnbJGdB6u9mXnjOASE2Z2Hc73RCR8FHccrkbtN/5qpFJEGFZ9fvrxAqSVYZz94QKmUWCtzylBERAmlDQ90FCHb8cRPfBVYgOYm80uy7kk2znkX0+B/BRR4K5A1Efkn2W+WGHGgc6A7rPMg05u8CMgP4+VdE99tOCiXZsTXbpiLkOnq7ywMckVuOesfD2fNYk9cdbZxHyCbhKWMeJugM4n9H2kw0TfIJ1TATq5OlgNdliJRV2HyHxInTX2bmLT+b+3v/5SJ5C6KYD0AVmWc9hK7Da/PLCw0+SY2wMRkcJgJkDNgcsBSmY4DyfI7ki2P8Q7W0SpxOzQffywg2ODmwILvJ6RPza++Rjm2I9u+qt/Cb7NYyoGuh1RHxY0KfnvNim98dy7qWt9YWq7YHNAq4qvGOdJwwDZAK14LyoemTIV5qFqOz3Dw/hN0dRK4LwvJNJG+nZoX7c9ztjznVcbpo1cK1yc477LRLynlHJruRtgQJOuumd8W4NNOJzt1Ajb12LK7x6pEwO/oakR81XaMOAZ02VZQCt1LgydhXbnGXRp1ZBZJ7Zmj9ZLGGygryN7Pumlbu5iGdYvTyjJhinPIM+EG72kmm402Uyl37bbu/j0jEkwlaQqQJgFBcv0EYbqoWWedEUIaL2qeD0F8lMc8Z01ELddMicMO4H2Svt7BYqBRlIhLuW2Ks+B85sZc6JzM3RUT9bbo0rz8t/rLfw7RifeURRpp+mTdV0Kl3Wv6d09WhpMtlSxR1Cu3TS87XbHD/unV1K5t376+R8vJ9PCZis6hFVuV886tGz0zuJdVVQcu1jy7v5T7H8SyT5j2glYI+hXFTtLA2cFMyM3bC1sj9SNFjcpm4lG40tLMtPtolrcHnHfHdSwC+Ab+AgWQhoEXwgARd8jrIPw2l/4TrOCmemKl6//6wIOhghysk35Rtph5A7lnLlOe47UedSlIfeSTfd9lbHh+loBNq7Qj8We4qRaEyZ8Br2BcgMFKMyzUK3063DgvssHSTBH9Zh1BdSvPSb8fsxQK3GINfJtWrRh1BnwuvrvtGBDx/VIqGEPdozsw6Z5t2OYFOBMq75ICfjK/ERv4xn4WNpQZoUrbgZuajZeonTtUnYofYNyJbcgEJVPwrmKXQk3qMCfpXL93AUT3adkDgCOW9kfWyWVQdo4swCu5cCCUv+lfVhg6QjjGvRlGltqE2Ssxi8sm2noPcXdYNcl6ffLn0F7T3YinNVUVkCSzDSnhtr0tboSLpKBkNR3YYZNA5xzmCwaTsZMIuOwftzM47HOYw32kfc3BldayjUvrppnJ6IbnGLXyF1rzEhd803yoiP/Gqdlh3FoohETh6O+4SrVuKmxM9cSk60UDO1YfacgrXj5AS1O3k3UlLB9C9XATts9/bgOcI94YDOmQ3xMS6bWjBmyKw80LcsNRBtilPELLzFuIbabEeMVHNLIALVPvtUx3XWj/5S7Y5ZTE4yEBVQ3OE9ncMcH0zNqaOzLjZ3FmZfjwL7cBqfYY2zduZzAPejYHYFfgzBzYTcKCx2afCaGXzKa+cLF8KMiBf6C7v0NAv9ajtRVi2fEMY/2xlfsDSmVJKB1PO4xFmRGMRRSPNs2kpM9h0fgsXHWAxoV8kaiJs68Y0vLPs4B6Xgp6YG1GG/PztmJUtKR6yzFba3E9Jg77MM0sB8lr4op4sUGsTvdFlKK+gBO6NewU17LgSfkjdD1X7roStfqaZLO4JolUvODd0RnOVhbmYI/FaGdriLulcbDYcJa3gFDOJx1lWMhXGKaY44F6Xut1QeJKfoxvjLbqfSNMaVe5JhsQdBsXJG87f6Sk1K43vANLSn/fHi4/27puII8W6wgJQo4OlWzG+mwaGfPUUds4bB4vquBuKMXHWD7ANSlFHpoVnaal37owMwa6zdSIYLJBl5ZXsIXA0pg7vB/s3zVl4Br6QDhDITRFuplHJ5Z9JeCuZaYj62ZwMcuiPyHRdSCcccfj8J0yGpNe7i7/+7T2/vo8fewBj1tMEefsOLyyd9P9nB37/7S/qYlJ+Hmrgsbkjbd8kgbBbhwp2n7EZ+NO3bZZkLr7OV0bDhDyLgjvqfFGNCkpqMXJfOKLlA8s+wG096Nnon5V937Dz926NI3qxGyeNWPnJll0U3Y+UVSo7zOIklhzsv1ay+BhLLO6wxJmWnNs1LJL7sF4VK7e3uE8FcVHt+u6pvMJGPF1l4FyIAqmHA317jU0q7P5fID8jDO4hy1EvfrpA3XfvuY6Ki/d6snoHFaehxRI8ffPl6GSMDTFeRIXN42TIiyBDEBxLtwUCo54FgZm2y6c1Ft2EpW640zPWexthMAjg0Iwk5s33Yc2OhxXGP/MlimjWJ5lSbZfuodIpiTijtuO9V42oBIW2v8XQmWE7FeYEmL9ZBo3KwZ48nXh7dw1Nbaza2N00kfXofo+Ci3aj/vbyPE/P/5N5J/vnHga+HeQcHTsWifZWtp0O3dzy0DJ4XnVeSRY9pxRq2UFMbqlbUoyhzfcZiNL78+PKACsK6U5YhUCDDZJNYG5WCeAEQkEAt6ztbUIcLXvGgiEd0Z7mxLCp3n6te9lPrwdYaF9nUzrVU1uwbT0lxXGfOcjHO/PYCMYqU/ABoYedNYrRiztWKkeA1wzbrOjPlckvOMtLcM81ES6+UUCwNt1XCZ3/kI9vndkC6kNBugjuxvmECFftWQn7rhl9oxQhtMtjfeWxVMVAZamSzHO1AhCymxDtXK2mz79ZDuiHRvXLgf7jnc5ejezxhyi++x3+Mpq1xXea/B76t8WeWz7cVogUu9kcZ5cy7Xo/Y8nb9zl2fUp1G0xmu3x+x6o6k/yl1P2gOQ7zDJ8l3mo82rAjw4C+JlEpCcQk+kWLF1VpUUT9IgQ+L9e37gcDIfkQ0Wa9A3XuJ+LSWXAuxK8L/wArrip9ktqiKL3BidhY+RewokEfvMkIZLWnIK2kTIGV4Pu+Pwdg3OMobt3FdRQ/3wEf4Qhh4CTNh6daiXM7rK67kXmGwjIROuqVo3MNkK+cSBrv1Sum3+rrftWmuNAmduD99OfRb9LDa2Ei3UNS3f4MV2gRcoTFp/8SrIIwV2FvjOQEbk4B3aFtP9Wbbmpo8k1mvapFz7BQudJUbGR9DvlTQYpX0i57Bf1xI3SXYKYoR9TkmhgGnGwRhQc66Csso50xvPeDsn8nMiI0tGWrT0BF5ImtmlawfjTMCc6J82UgOKM7mEy8veAf4oKVvtbsn2bXxggnV9jLwsvWBoKkIPKYgGN11SE8hoXM9DP+idzO8J2AYKSQ19huglaEG8aDCd/aVdJnqDQNBSMmHdWmVcl9YOTMuP9KKjEvVc09FxDc8Qgovo660GNSTMFQx1kbUfbkxKxOg4dFqCxsVQgVqnHfPLqlsJj+ngCBXsoGdusR2hbWJ5lRXnWRL4zuJVEkJ6+hPXgYMRhRUTLBZ7ul7VkNQyvmuKGS0qvzvvQ8NvjfRm1/T+C59LB3p6LgdxTmnaCf4KYpxehI4z42SnN7Pg0huEjYGi7MaFPgnOtuAI0De+YGrfcU2kCrGi5FCAMD6zoBJ8z3iODdn4ayPruAItpU9R6qtFBN81t9xJAa0XFv5+42QyZQXv+4vcTwZb7ZD1FY3pu+5+DVyWgBUqKm5Yyf1Nj51l6xaj29542py5Oy4aEUHshQ6z1iwvBT/O+2iAOUypHfakvp/HJcycaWTcLW6bPCMjChtjhzboEZmjb8mz2ctVvPwyTndvzeBfx+d3sHEGlY2TtJ2I+2loJKT41uryrsVVRofpdpucOTVij6hrqcH/EEnh74OU4VLmXbM2015fh+WOcRWac5Q9m8ofEuo3HCcgsm6dmZGocORlCNqYdM7q8I/VccZ7elfQHV/dH1fST4r5rnruSvzdbQ0O7teyterRRnNjBXq88XqCzou3e30XoYpf22UHwFUzHIhXRxF/DdWwsBMxSN3/nPWiMRT92bKPMbRoEHTm2kh7XXw1EW/CnznqD54rF5ZGAqY/u8/2MJ/fWQ9bGAcdEFfpWdz3J5hsUT2xJadgnLP64oIHbxvqTlimmx8kiU3mrhu2vUi0O0WO6yWVbrQ1KbbztB1vh05ajbDo6tUIbaK47km5qLr3PGx2dbaIv+G3a5pkRMnoWk5qfk2bvwHuxZG5W42Gy3Bb3FydhrhkI4/RvwVucGMT3I9rYFcNTVhhlc5+U4DZSOrmjnGab4q2mWeXHuDKbP5YYMKzHGugWRAkdr8kMg3iOn9yOumF5gyDCGoTbiRZKywMUOTnRlpy4DtEK7c8wpO3dx86Y+SGjMY7DUT/SYdfs7u9+5D+Mk/H/djHkQQ26hIIWzGSWWRFZcZ49T2uxs6bAtOUQXHGo5ya6GalPTR7dyrt30g0ja5OcLlSJ+zOH3WeBnK839JLZ+h54r2rlfxgGpWgUF6RLZgW2Pq3aznWunWlj/tt9Nh9L4j7+UZE8e7GEeMuDYrPKSj94VFsQmNXuJzHd+c/Yh5PbMrK3wRNcefxqdZdpC4IyaL3/Ho0b7oLVWPQhDmv5Riuq/uTifL/AgAA//+rygKM" + return "eJzsXUuT27aW3vtXoO7G9lRbmTgzs3BN3apO+ya3J7bTZXW7ala8EHEkIQIBBgC7rfz6KbxIUIIkig91nJpk5RaJ850HzgM4AN+gDWzfoVVevkBIU83gHXr5sxArBuiGiYqgO4b1Usji5QuEJDDACt6hFX6BEAGVS1pqKvg79PcXCCH0880dKgSpGLxAaEmBEfXO/vAGcVxAIGT+09vS/FuKKvwlfj5+h+EFMFX/ObwqFr9BrqM/J/CE/xwuTrWQlK9QAVrSXO2PvAshhlEpkLN/a/10EIr5z/0xc09sYPskJEkOXIDGBGs81eCG1UnGVluloZhkaAlKVDKH0QYPA/+tFoj7/2+n7ao1LhHVwlp34teswGVJ+co/+rfW4Ees86M3R73GGknQleRA0FKKArWm4vXdLfq9Armd7bG1oIxRvjpErzXMj+7ZYBrRO+353RZLPFNRcqoELLlQTh4v9hWX0nkL6Y1Q2j6rEOU5qwggCauKYXmFNP56hTD5rVK6AK6vEOYESVFxYoQOUgo5S+Ch/FHQHLJCcL3ugykITEIppEZ2nBShUgprC5T0oXLn3ka375FYIr2GoNRAdwFM8JVCWqSIa6ExS9BdMoH1Yar35rWaEi5ExXVqeLWpevJ1v4aIpzCxjTslaLG1f1QgH2kOh+hGw/UBcB3/cw+HGSQJBv0kJIKvuCgZXCG888ZSSD+d5lpIvAJEFZprzAmWzd8e5kmeHIVR5OnHcr7D/KFSHgxWSuQUayDoiaYNNgAZKGCDqOVhWqCc9QKx6NoeyEa9lC3jlUoA4aA0HFV0LhiDPOh5A9s3j5hVgEpMpfevpRSPlADChFDzIGZNAG4NncoFGogb2O78ckxczXsWT+c3w1uwXBq2HiErJc1TjvXUNF8DytdYroAgO4Q1YGcr3pJaGpz/8qCse53/8oA0Balm6DMsjXAVygXXEufajmT0SJcIlyWjOV6YuSL0GuQTVXCFqH6p7OiMKvc87AevHMtFy/IOx64b+yj6SQhdSsr1sSC2AD0kjI3mzQ37jkW0rHF38ukBgvnXEBDm/QEwhvmruXcGt++tzSVhWEOc0EMFDHEgOB+MhFVP+p/tmwmSJyjWoGYqFyV8f/7En5v30Pd7XHag9nbGRI4PiLwT2bcp4RqHEQZ+s8Am8hag14IIJlbbTrgKLDegR0flhu2F6Ye+YH44UzNiuVSgU9GxU6LniflREq5YFGWloZsvds9OVkcsqYQnzNiMSFGWQGaLrYYU58Z3HWb8lueiMOq1ryM/WMj2ApEO9LMS5xvQKsttkrwfyc9C4wc7Aw/lSmOewywvq5kE4xqBZLmQoA6C2StXd+B8qooFSJMC2HFQGBYJbuGsTTnmE4RA/xQ0680yTQuYKch7gHqwaYlxlpgxD4xypCAXnKhO5GdlnvIPpyibNGlpUpuoUsDMeisg6ObuweWQVKG8khK4ZluDrFIQBNZFSISqzUwC7mvRN8b+DDxn0WYkt1pgBu5EOBPlMDOuIZghHYLbX5EoQVq/flRJFsWTpBrG4d8MpYEjLboJwJIeWQJ2zO4iKKAQcjtbGNsSfCZxkSn6B/SEYqzWlv++fjeoHAVjnMYqv3ycofs1Vd5XGwMWnG0RfsSUmaTdzrYvH32N5NJBI1DzMrxFS1xQloyDx1gyVXVPlj46+M0ssxX6s3GjnnCZUd7TXo1+9jRj5wzlHtWqAqXdJKZaIfHEkaGJVIlzeBZuRaXHZDdMUstiw7EWz8EvB/0k5GZG+UqCUmO54RzoY1i4NWA8mXOQ+KxgZj1Tf0QhuRiECUYUjgLzr0eQPUGMLJez4VTlkXzmOPEmxfIJjPdhaI0VWgBwJCvOKV8dNVkHILNuvheMfzBcGh9qhkGK8hwCjieskNJYaiBXUZ41Q3O79kkQPILcov/69+aX66UGiZT5nfLVlV3IMxOVC40eqaJhllalmZjfv21e3as4zKulFHmnkuO9f3j0miOqgFilNMjZmizVzIDjgkDK8A6KfS/jJ9Tkj07vvLaGf77/aW4Z+mQIOGeGJQRrMKq365Q1otNwlVt2znJc4pzqbSL4HsmED+IOw9Wo3cZfjVTwAMOYz88/noG00pTRP2zCNAisiTklyBy49kuaDqhfh2/nER3wVXwNmOn1NlswkW+m0H9NAjkSQeF2gaiLyn8Ti9kSUwZkAnS/iYW3yTV+BOToGE13tEcDLvi1KdHFM+Y8dKpaFCYnuYD0rJev6RmX2BFnXU1Mp+imYBmi7gbodErfRdpP9Q3SKQ0gKdXeZrDFks/qdYjMpdRZ4+9Gdp3/e/35U11AqmYBpAvIspzCV2K7+eWA+Z0mK9wOiHLBNaYc5BS4LKCIwmk4XncHqv0+0dkgijVmhu4Shy0ck9wUmOPVhPIx+c1HT2M3u+lu/iW4No+xBGh3RF1a0JTkv5ui9MpJ76pe6/OkzYRdAFpWbEkZixoG8jWQinXi4pFKXWHmV23HF7gfv1k0NRo4HQtzYTJ9M7Rbtz0s2NOdV2uqtFhJXJySvq1EXKQUYmOk62FAU67aZ1xbgyk4rO9UCJvXQslvH6kLA7ehqRB1q7R+wBOuy4iAVPJUGjoJ7zYyqNKYITVBbEUfTZTQWEPXQPZ82ovD3Nk6rF8eUZONUJ5Bn5E0Omk13mgylUq3bbefwtMTFsFE5FUBXM8IGKEPcFT3LfekqjwHpZYVq0kgR+JA7KyB2O2SKWEYAsoZ7e8VSAoKCYmYEJuqPAXObWZMic5SSKyerDYHjeblv1Yb+FfITlygKOJO0ycTuiQu617Tmzs01zjfEEkfQdp2U/+23R3f759eCmnf+vmXf7wczQqbrWifVtldPRPQssE7izdVUTFsc8mbu4fQ/8Sjfca4E7BG0G1V7CQPjBZUD9ywNbAdUjta2KRsCA3GFy/JjLeLanA7xF13UP0ugGvgyzHnQiP4mgMQ9D3Cyiuv/YPrNCuoHm/x8u1/nCFBn0GOtinfaNuPnDDOiddpT7E6jbk07J6z6b5rMiZdX0nA2i70Y75jOLHVeILPYDdQrqEAiVnmVyvdPOy57/JB5Jihesx6BdTNPcrdfkxfL7GPNchtXLR+1Anw2vXfccH6juuBUP0e7BDd+03zdGAYFeBEs76IGfjG4kRn5+nlWJpUZokrpntuajZRorTtUmYodYUWUmyAIyKeuA0V2xKuUIF/E9L2cxeUp0/J7AEcNrM/tpZUelnjxAq4VADzRv2XjmFepAOcq7eXcXypKZCxHD6xTKWhdgx3jW2XpNsvfwbrPdqJcNJSaQFRMdOcGmrz1+pKOEsHXBDXhek3DfCCwWjZcDRmlBn7+WMpD8c6jTfYRd7dGVxoKteyuGidHamud4ldI7etMQNtzTXJ80T9NczK9vPQyCJGTkddw1VscWNjp7YlJltK6Nux+pOEeMXLDWhwsjRTY8J2LVQ9O2137OMywB3ins2YFvEhKxnbMiaorhzQeFmuJ1qfowyfeJFz87ndhBgvEJAGJqhd6q0Euctm/7F0h0ynJhnxE6hucB7P4Q5Ppie00NCXGzqLM6fHnn25DU6+I9i6czmCu9exOwC/Aq6nwq4l5so3+YwMv6QkcwsX/Y+KFPgrunM3CPw6H2irBs+AYx7tja/QG1JKkYNS4bjHUJAZwVAI/mzbSFb3DB6BhcZZB2hQyhuYGrnyDi0tuzh7lOOlIHveYrg/O+UnSkEGzrMYt/ES42NO+IdxYD8KVhVj5IsNYnu6zZcU9QEc369hSF4qgEfsDbD1Tym+4rl6nKUTuCbJ1NzgiexsAcZXxuCPZWjHVxF3lsb9YcJa3x6DP5x1kWMhTGCywAzzvOu1Vh8EJujH8Mpkp9LXWpdqtsD5BjjJhi2St8NfdFIK1xu+viXln/f3d9/NrVSQE4tRpEAeR9I000j7ZTs7gTpg84fFF9saiD16kQDbBaAqBVd9q7LjsnRDe2HWWF8JiXKcr+G1kSV81SA5Zhb/q/nrrgxcygZyRoFrZaCeJ+GJVX8umEup+dCc8XJMQWQ/zIIVDDv+eBCmRVZb2v3N3XcP7+9CxN/B6u20wRxiwpKJJ3c/2f3Nnf2XcjctWQ03d12YlLTplkdKS8CFPU3bjfls2LHLthBaZy/HE8MJRoYd8T2uRo8mdh2dOJlWdZ7jiXXXm/c0esqnn3W3H35M2NKr5QBdvO7GzsS6SDN2epLUKC8zSWKY00r90lMg4ix5nWFeZkqxrJTi63aWM6HsvT2cu6sKD29XdS1morFCa68EpEEWlNuba2xpaebnfP4BORgncQ6aibvrpI3UvnyMbNTdu9UR0DArPYyo0eOXj+ch4vB0AT3mtm7rp0RRAh8B4o0/KBUdcKy0KTbtuag2bCmq1dq6npNY2wUAwxp4To9s3yYObHQ4rrF7GSxVWtJFFRfZjvQW5ZjlFbPStqbxtAYet9a4uxKMJMJ6gWEtrIcE52bcGIt+3r+Fo/bWlrbS1iZdeu2z44PSquO8u40Qs/+X30D5ucaBb0V6ewueVkS7IlsJja5vfmk5OMGdrIKMrNAOC2opBdfGroxHkfrwjsNkcvl8f48KwKqSRiJCIsD5OvI2aAH6CYAHBjEnp3xNnSJ8y5MmMJGucCebUui0VL/tqdRFrhNMtG9baK1Vs0sILa51pdbPKTj77QGkJS3dAVAvyKvGa4WcrZUjhWuAa9ElK+ZTRc4z8t5yzAdZrKdTWBhom4at/E5nsM8fhlQhhF4DsWy/ohwV6nXDfhyGXyorCKVxvrly0aqgvNLQqmQZ3oL0VUiJlV+trN22mw/xjkh648J+uGd/lyO9n9HnFt9D3+Mpq4WqFp0Gv6sW82ox2V6M4rhUa6FtNGdiNWjP08Y7e3lGfRpFKbyye8y2N5q4o9w10Q6AXIdJtthmLtu8KMC9syBOJx7JMfS54Eu6yqqS4FEaZPJw/54b2J/MR/ka8xWoK6dxN5eiSwG2JbgvvICq2HFx86rIgjQGV+FD9B4DidQ+MaT+mhaMgNIBcoZX/e44vF6B9Yx+O/d1sFA3fIDfR6D7ACOxXhzq+YKuFjXtGc43gZER51RtGzjfcPHEgKzcVLpu/l1v27XmGgFG7R6+IX0S/SQ+tuIt1DUvr/BsM8Mz5InWP7z2+oiBnQS+1ZDlovcObUvo7ixbc9NHlOs1bVK2/YL6zhItwiPo90pojOI+kVPYL+uJmyI7BjHAP8esEMAkY6A1yClnQVktGFVrJ3hDEzmaSIuS5i1eOgIvBMnM1DWDMcphSvRPa6EABUq24HK6t4A/CkKX2+t88z48MMK8PsReFl8wNBaj+xwEhxtPqRF0NKznoRv0pPA7AjaJQrSGPkH24q0gXDQYU39ppolaI+CkFJSbsFZp26W1Bd2KI534qHhNazw+LhEZfHIRYr2xoIaFqZKhFFu76caoTAzOQ8dlaFgO5bm11jG9rtJGeMgGB5hggp+p1XaAt5H1VVaMZVHiO0lUiRjpGE9sBw5GBJaU07DYk3pVQbSW8V2zmNHi8rvTMdR/a6SzuMaPX/hUOdAxclmIU2rTEPgrqHF8FVrJDNOdWk+CS60R1hqKMo0LPXBGN2AZUFduwdS8Y5tIJaJFyaAArl1lQQS4nvEF1vnaXRtZ5xVoLlyJUl8twtm2ueVOcGi9MHP3G0fEpFG86y+ynww21iHqKxrjd+39GrgsAUtUVEzTkrmbHpPL1i1Bt6PxuDVzOi8akEHspA6TrlmeC35Y9FEAU7hSM+xRez+Ni+spy8iwW9x2eVoEFCbH9m3QAypH15JnqpeLRPl5IHdn3OBfJ+YnxDiByQYi7SBiPw2NuOBvjC1vW1KlpJ9tt9mZ0iJ2mLqUGfx3Lgj8vZcxnCu8S67NtOfX/nLHsBWaU5w9m8nvM+o2HEdgsm6dmZApf+SlD9pQdE4a8A+t4wyP9HZBd/jq/rAl/Wgx366e2yX+dFuDhfutbK06tMHdGIUebrweofPi/U7fhV/Fr/2yBWBXMyyI1wcRfwurYX4nope5/znXi4Zw9GerPobwooCTiddG2vPim8l4I/lMsf7gpHLm0ojH9GeP2Q7m8wfrfhNjrwPiIj2Lu/EE5xtUEzbsFJQxWl9ccO98Q90JS1XzQZLQZG67YduTRNlT5LieUvFGW1Ni20ibeNt30iqEeapXw7eJ4ron5azVvecRs11nC/gbedumSZpLEULLUcuveXM3wL04QLvVaDj3t8VN1WmISzrwGP17YBo3PsF+XAPb1dBIFMbozC8F6LUglnbI01xTtKk8U3aAK73+Y4Zzli2wApJ5RWL7JZFxENf1k7VJpzTrGLg3G38jyUpiroEgRxspwYBtEans9PBPXt98SObIDRtNdOqJ/kH5r9ld33yIv8yTuB/7MBIvRlVCTpc0zwyyotJDovqOVEPnTYFJLKBA8aCkRrpZaQfNzp1KuzcSjWOrI1yulISd/KjzOJDD/ZZOO33PE+9creQGU6gEiRZVvgHdAlt/u5ZhpVpX+thvo4fue57bzzcigrdXlhl7aVB4TkLpDo9i7Ru7/OU8rjv/EbNwYlNU7iZogpPHp1p3kdokJAvR89uxvPEuVA1JE2as1qO/ru5Pqsogg0eQGr5imjGxUp1i6xf7Brq+RXdSFKVGn4OcP4iVOhZyF6CHBN3QnpTQ0Qa2T0KSI2paQwTcWEsYDT18/uCWccI5nBDYExAIlExsTaEtCLCMkr5Y3H2LVit+SGSHdNHU33i4c49/Ag8Tq5UpVUxmmoBCsD5x+7vSuCibk13Xt856/J1jT1hZEunGr5BaJqVw6m6zB05/N4ZMgGu6pCBr8V/fHuM4EC3xlgmcoqzha9tEKCfw9R1aYqaOALqWEm+jxAZ5CqZco3yl4m+f2V3h0lq/sjOT8rJKXyEUnNCkgP38O4L4+tbbl6j0Iaj2gT4G/cl4XG/ONSE7pV7BbDW7QisoKKdv3s7+800pRfK+DjehHkEqKngfEF/cq0kcyYyupJmLGH3ncBRzTOoes8tNfgk39qiavkKlBEJznWR8GfWGdKvKgOMFA/IOaVkdu5U0fHmz7oxwuWRkFbjZB0EENKbMhq3/mf/6yUzHAien4DJeF5kKsjfpQ5j9z91BF6Axwa34MxzvNSF2HQ2zenyElRI5tZHZ7iwl3OpptEIDMxF5VLS/lsDvgYGJ0Fvj11FfwP8XAAD//ykgdJA=" } diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/data.json b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/data.json new file mode 100644 index 000000000000..1bb2cee29a3d --- /dev/null +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/data.json @@ -0,0 +1,95 @@ +{ + "@timestamp": "2025-09-02T10:14:50.313Z", + "agent": { + "hostname": "metricbeat-host", + "name": "metricbeat-host" + }, + "cloud": { + "account": { + "id": "elastic-beats" + }, + "provider": "gcp", + "project": { + "id": "elastic-beats" + } + }, + "event": { + "dataset": "gcp.vertexai_logs", + "duration": 123456789, + "module": "gcp" + }, + "gcp": { + "vertexai_logs": { + "endpoint": "https://us-central1-aiplatform.googleapis.com/v1/projects/elastic-beats/locations/us-central1/endpoints/123456789", + "deployed_model_id": "model-deployment-123", + "logging_time": "2025-09-02T10:14:50.313Z", + "request_id": 98765432101234567, + "model": "gemini-2.5-pro", + "model_version": "001", + "api_method": "generateContent", + "request_payload": [ + "What is the weather like today?" + ], + "response_payload": [ + "I don't have access to real-time weather information. Please check a weather service or app for current conditions." + ], + "full_request": { + "contents": [ + { + "parts": [ + { + "text": "What is the weather like today?" + } + ], + "role": "user" + } + ], + "generationConfig": { + "temperature": 0.7, + "maxOutputTokens": 1024 + } + }, + "full_response": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I don't have access to real-time weather information. Please check a weather service or app for current conditions." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "safetyRatings": [ + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "probability": "NEGLIGIBLE" + } + ] + } + ], + "usageMetadata": { + "promptTokenCount": 8, + "candidatesTokenCount": 24, + "totalTokenCount": 32 + } + }, + "metadata": { + "region": "us-central1", + "zone": "us-central1-a" + }, + "otel_log": { + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "span_id": "00f067aa0ba902b7" + } + } + }, + "metricset": { + "name": "vertexai_logs", + "period": 300000 + }, + "service": { + "type": "gcp" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml new file mode 100644 index 000000000000..be8202fb123a --- /dev/null +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml @@ -0,0 +1,50 @@ +- name: vertexai_logs + description: Google Cloud Vertex AI Prompt Response Logs metrics + release: beta + type: group + fields: + - name: endpoint + type: keyword + description: The Vertex AI API endpoint URL used for the request. + - name: deployed_model_id + type: keyword + description: The ID of the deployed model that processed the request. + - name: logging_time + type: date + description: Timestamp when the AI interaction was logged. + - name: request_id + type: double + description: Unique identifier for the AI request. + - name: request_payload + type: text + index: false + description: Array of request payload strings containing user prompts and inputs. + - name: response_payload + type: text + index: false + description: Array of response payload strings containing AI model outputs. + - name: model + type: keyword + description: Name of the AI model used (e.g., gemini-2.5-pro). + - name: model_version + type: keyword + description: Version of the AI model used. + - name: api_method + type: keyword + description: The API method called (e.g., generateContent, predict). + - name: full_request + type: object + enabled: true + description: Complete request object containing all request details in JSON format. + - name: full_response + type: object + enabled: true + description: Complete response object containing all response details in JSON format. + - name: metadata + type: object + enabled: true + description: Additional metadata associated with the AI interaction in JSON format. + - name: otel_log + type: object + enabled: true + description: OpenTelemetry log data associated with the AI interaction in JSON format. \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/data.go b/x-pack/metricbeat/module/gcp/vertexai_logs/data.go new file mode 100644 index 000000000000..f027bffdb306 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/data.go @@ -0,0 +1,129 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package vertexai_logs + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// VertexAILogRow represents a single row from BigQuery vertex AI logs table +type VertexAILogRow struct { + Endpoint string `bigquery:"endpoint"` + DeployedModelID string `bigquery:"deployed_model_id"` + LoggingTime time.Time `bigquery:"logging_time"` + RequestID float64 `bigquery:"request_id"` + RequestPayload []string `bigquery:"request_payload"` + ResponsePayload []string `bigquery:"response_payload"` + Model string `bigquery:"model"` + ModelVersion string `bigquery:"model_version"` + APIMethod string `bigquery:"api_method"` + FullRequest string `bigquery:"full_request"` + FullResponse string `bigquery:"full_response"` + Metadata string `bigquery:"metadata"` + OtelLog string `bigquery:"otel_log"` +} + +// CreateEvent creates a single mb.Event from a VertexAILogRow +func CreateEvent(row VertexAILogRow, projectID string, logger *logp.Logger) (mb.Event, error) { + event := mb.Event{ + Timestamp: row.LoggingTime, + } + + // Build the main metricset fields + fields := mapstr.M{ + "endpoint": row.Endpoint, + "deployed_model_id": row.DeployedModelID, + "logging_time": row.LoggingTime, + "request_id": row.RequestID, + "request_payload": row.RequestPayload, + "response_payload": row.ResponsePayload, + "model": row.Model, + "model_version": row.ModelVersion, + "api_method": row.APIMethod, + } + + // Process JSON fields with error handling + if err := processJSONField(row.FullRequest, "full_request", fields, logger); err != nil { + logger.Warnf("failed to process full_request: %v", err) + } + + if err := processJSONField(row.FullResponse, "full_response", fields, logger); err != nil { + logger.Warnf("failed to process full_response: %v", err) + } + + if err := processJSONField(row.Metadata, "metadata", fields, logger); err != nil { + logger.Warnf("failed to process metadata: %v", err) + } + + if err := processJSONField(row.OtelLog, "otel_log", fields, logger); err != nil { + logger.Warnf("failed to process otel_log: %v", err) + } + + event.MetricSetFields = fields + + // Set cloud provider information + event.RootFields = mapstr.M{ + "cloud.provider": "gcp", + "cloud.project.id": projectID, + } + + // Generate unique event ID + event.ID = generateEventID(row) + + return event, nil +} + +// processJSONField processes a JSON string field and adds it to the fields map +func processJSONField(jsonStr, fieldName string, fields mapstr.M, logger *logp.Logger) error { + if jsonStr == "" || jsonStr == "{}" { + return nil + } + + var parsedJSON interface{} + if err := json.Unmarshal([]byte(jsonStr), &parsedJSON); err != nil { + // If JSON parsing fails, store the raw string in a structured way + fields[fieldName] = mapstr.M{"raw": jsonStr} + return fmt.Errorf("failed to parse %s JSON: %w", fieldName, err) + } + + fields[fieldName] = parsedJSON + return nil +} + +// generateEventID creates a unique event ID based on row data +func generateEventID(row VertexAILogRow) string { + eventData := fmt.Sprintf("%d_%.0f_%d", + row.LoggingTime.Unix(), + row.RequestID, + len(row.RequestPayload)) + + h := sha256.New() + h.Write([]byte(eventData)) + return hex.EncodeToString(h.Sum(nil))[:20] +} + +// EventsMapping processes multiple VertexAILogRow items and creates events +func EventsMapping(rows []VertexAILogRow, projectID string, logger *logp.Logger) []mb.Event { + var events []mb.Event + + for _, row := range rows { + event, err := CreateEvent(row, projectID, logger) + if err != nil { + logger.Warnf("failed to create event from row: %v", err) + continue + } + events = append(events, event) + } + + return events +} diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go new file mode 100644 index 000000000000..9f18276575de --- /dev/null +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -0,0 +1,206 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package vertexai_logs + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "cloud.google.com/go/bigquery" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" + "github.com/elastic/elastic-agent-libs/logp" +) + +const ( + metricsetName = "vertexai_logs" +) + +func init() { + mb.Registry.MustAddMetricSet(gcp.ModuleName, metricsetName, New) +} + +type MetricSet struct { + mb.BaseMetricSet + config config + logger *logp.Logger +} + +type config struct { + Period time.Duration `config:"period" validate:"required"` + ProjectID string `config:"project_id" validate:"required"` + TableID string `config:"table_id" validate:"required"` + CredentialsFilePath string `config:"credentials_file_path"` + CredentialsJSON string `config:"credentials_json"` +} + +func (c config) Validate() error { + if c.CredentialsFilePath == "" && c.CredentialsJSON == "" { + return errors.New("no credentials_file_path or credentials_json specified") + } + + if c.ProjectID == "" { + return errors.New("project_id is required") + } + + if c.TableID == "" { + return errors.New("table_id is required") + } + + parts := strings.Split(c.TableID, ".") + if len(parts) != 3 { + return fmt.Errorf("table_id must be in format 'project_id.dataset_id.table_name', got: %s", c.TableID) + } + + return nil +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + m := &MetricSet{ + BaseMetricSet: base, + logger: base.Logger().Named(metricsetName), + } + + if err := base.Module().UnpackConfig(&m.config); err != nil { + return nil, fmt.Errorf("unpack vertexai_logs config failed: %w", err) + } + + m.logger.Debugf("metricset config: project_id=%s, dataset_id=%s, table_name=%s", + m.config.ProjectID, getDatasetID(m.config.TableID), getTableName(m.config.TableID)) + return m, nil +} + +func getDatasetID(tableID string) string { + parts := strings.Split(tableID, ".") + if len(parts) >= 3 { + return parts[1] + } + return "" +} + +func getTableName(tableID string) string { + parts := strings.Split(tableID, ".") + if len(parts) >= 3 { + return parts[2] + } + return "" +} + +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { + var opt []option.ClientOption + if m.config.CredentialsFilePath != "" && m.config.CredentialsJSON != "" { + return errors.New("both credentials_file_path and credentials_json specified, you must use only one of them") + } else if m.config.CredentialsFilePath != "" { + opt = []option.ClientOption{option.WithCredentialsFile(m.config.CredentialsFilePath)} + } else if m.config.CredentialsJSON != "" { + opt = []option.ClientOption{option.WithCredentialsJSON([]byte(m.config.CredentialsJSON))} + } else { + return errors.New("no credentials_file_path or credentials_json specified") + } + + client, err := bigquery.NewClient(ctx, m.config.ProjectID, opt...) + if err != nil { + return fmt.Errorf("error creating bigquery client: %w", err) + } + defer client.Close() + + datasetID := getDatasetID(m.config.TableID) + dataset := client.Dataset(datasetID) + meta, err := dataset.Metadata(ctx) + if err != nil { + return fmt.Errorf("error getting dataset metadata: %w", err) + } + + events, err := m.queryVertexAILogs(ctx, client, meta.Location) + if err != nil { + return fmt.Errorf("queryVertexAILogs failed: %w", err) + } + + m.logger.Debugf("Total %d events created for vertexai_logs", len(events)) + for _, event := range events { + reporter.Event(event) + } + + return nil +} + +func (m *MetricSet) queryVertexAILogs(ctx context.Context, client *bigquery.Client, location string) ([]mb.Event, error) { + query := m.generateQuery() + m.logger.Debug("bigquery query = ", query) + + q := client.Query(query) + q.Location = location + + job, err := q.Run(ctx) + if err != nil { + return nil, fmt.Errorf("bigquery Run failed: %w", err) + } + + status, err := job.Wait(ctx) + if err != nil { + return nil, fmt.Errorf("bigquery Wait failed: %w", err) + } + + if err := status.Err(); err != nil { + return nil, fmt.Errorf("bigquery status error: %w", err) + } + + it, err := job.Read(ctx) + if err != nil { + return nil, fmt.Errorf("reading from bigquery job failed: %w", err) + } + + var rows []VertexAILogRow + for { + var row VertexAILogRow + err := it.Next(&row) + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return nil, fmt.Errorf("bigquery RowIterator Next failed: %w", err) + } + rows = append(rows, row) + } + + events := EventsMapping(rows, m.config.ProjectID, m.logger) + return events, nil +} + +func (m *MetricSet) generateQuery() string { + escapedTableID := fmt.Sprintf("`%s`", m.config.TableID) + + query := fmt.Sprintf(` +SELECT + IFNULL(endpoint, '') AS endpoint, + IFNULL(deployed_model_id, '') AS deployed_model_id, + logging_time, + CAST(IFNULL(request_id, 0) AS FLOAT64) AS request_id, + IFNULL(request_payload, []) AS request_payload, + IFNULL(response_payload, []) AS response_payload, + IFNULL(model, '') AS model, + IFNULL(model_version, '') AS model_version, + IFNULL(api_method, '') AS api_method, + IFNULL(TO_JSON_STRING(full_request), '{}') AS full_request, + IFNULL(TO_JSON_STRING(full_response), '{}') AS full_response, + IFNULL(TO_JSON_STRING(metadata), '{}') AS metadata, + IFNULL(TO_JSON_STRING(otel_log), '{}') AS otel_log +FROM + %s +WHERE + logging_time IS NOT NULL +ORDER BY + logging_time DESC +LIMIT 10000;`, + escapedTableID) + + return query +} diff --git a/x-pack/metricbeat/modules.d/gcp.yml.disabled b/x-pack/metricbeat/modules.d/gcp.yml.disabled index e6c4c8205923..a96d32763d5d 100644 --- a/x-pack/metricbeat/modules.d/gcp.yml.disabled +++ b/x-pack/metricbeat/modules.d/gcp.yml.disabled @@ -85,3 +85,12 @@ exclude_labels: false period: 1m collect_dataproc_user_labels: true + +- module: gcp + metricsets: + - vertexai_logs + period: 300s # 5 minutes + project_id: "your-project-id" + table_id: "your-project-id.dataset.id.table_name" + credentials_file_path: "/path/to/service-account.json" + # credentials_json: '{"type": "service_account", ...}' From 53f0461cb7a104d96df98b37455fdfcae0757d7f Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Thu, 4 Sep 2025 19:38:21 +0530 Subject: [PATCH 02/26] Add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 674288e05e67..1c791971bd76 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -596,6 +596,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - Add SSL support for sql module: drivers mysql, postgres, and mssql. {pull}44748[44748] - Add support for Kafka 4.0 in the Kafka module. {pull}44723[44723] - Add NTP response validation for system/ntp module. {pull}46184[46184] +- Add vertexai_logs metricset to GCP for prompt response collection from VertexAI service. {pull}46383[46383] *Metricbeat* From 21366f711d9aa58d5c60eaab5ffda4a18eb3d186 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Thu, 4 Sep 2025 20:25:59 +0530 Subject: [PATCH 03/26] make update --- x-pack/metricbeat/metricbeat.reference.yml | 2 ++ x-pack/metricbeat/modules.d/gcp.yml.disabled | 2 ++ 2 files changed, 4 insertions(+) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 9c1e2fd0d4c4..fce10c39d1c4 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -706,6 +706,8 @@ metricbeat.modules: credentials_file_path: "/path/to/service-account.json" # credentials_json: '{"type": "service_account", ...}' + + #-------------------------------- Golang Module -------------------------------- - module: golang #metricsets: diff --git a/x-pack/metricbeat/modules.d/gcp.yml.disabled b/x-pack/metricbeat/modules.d/gcp.yml.disabled index a96d32763d5d..c218344ab0dc 100644 --- a/x-pack/metricbeat/modules.d/gcp.yml.disabled +++ b/x-pack/metricbeat/modules.d/gcp.yml.disabled @@ -94,3 +94,5 @@ table_id: "your-project-id.dataset.id.table_name" credentials_file_path: "/path/to/service-account.json" # credentials_json: '{"type": "service_account", ...}' + + From 0e8f5c82321b1c21da660f397038027b719d4e8a Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Thu, 4 Sep 2025 22:41:15 +0530 Subject: [PATCH 04/26] Add test file --- .../gcp/vertexai_logs/vertexai_logs_test.go | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go new file mode 100644 index 000000000000..98ed8d845dda --- /dev/null +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -0,0 +1,198 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package vertexai_logs + +import ( + "testing" + "time" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/stretchr/testify/assert" +) + +func TestGenerateQuery(t *testing.T) { + m := &MetricSet{ + config: config{ + TableID: "project-123.dataset.table_name", + }, + } + + query := m.generateQuery() + + // verify that table name quoting is in effect + assert.Contains(t, query, "`project-123.dataset.table_name`") + // verify WHERE clause is present + assert.Contains(t, query, "WHERE") + assert.Contains(t, query, "logging_time IS NOT NULL") + // verify ORDER BY is present + assert.Contains(t, query, "ORDER BY") + assert.Contains(t, query, "logging_time DESC") + // verify LIMIT is present + assert.Contains(t, query, "LIMIT 10000") + // verify CAST for request_id + assert.Contains(t, query, "CAST(IFNULL(request_id, 0) AS FLOAT64)") +} + +func TestCreateEvent(t *testing.T) { + assert := assert.New(t) + + testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) + row := VertexAILogRow{ + Endpoint: "https://us-central1-aiplatform.googleapis.com", + DeployedModelID: "model-123456", + LoggingTime: testTime, + RequestID: 12345.67, + RequestPayload: []string{"prompt1", "prompt2"}, + ResponsePayload: []string{"response1", "response2"}, + Model: "gemini-2.5-pro", + ModelVersion: "1.0", + APIMethod: "generateContent", + FullRequest: `{"inputs": ["test"]}`, + FullResponse: `{"outputs": ["result"]}`, + Metadata: `{"user_id": "user123"}`, + OtelLog: `{"trace_id": "abc123"}`, + } + + projectID := "test-project" + logger := logp.NewLogger("test") + + event, err := CreateEvent(row, projectID, logger) + + assert.NoError(err) + assert.Equal(testTime, event.Timestamp) + + // Check MetricSetFields + expectedFields := mapstr.M{ + "endpoint": "https://us-central1-aiplatform.googleapis.com", + "deployed_model_id": "model-123456", + "logging_time": testTime, + "request_id": 12345.67, + "request_payload": []string{"prompt1", "prompt2"}, + "response_payload": []string{"response1", "response2"}, + "model": "gemini-2.5-pro", + "model_version": "1.0", + "api_method": "generateContent", + "full_request": map[string]interface{}{"inputs": []interface{}{"test"}}, + "full_response": map[string]interface{}{"outputs": []interface{}{"result"}}, + "metadata": map[string]interface{}{"user_id": "user123"}, + "otel_log": map[string]interface{}{"trace_id": "abc123"}, + } + + assert.Equal(expectedFields, event.MetricSetFields) + + // Check RootFields + expectedRootFields := mapstr.M{ + "cloud.provider": "gcp", + "cloud.project.id": projectID, + } + assert.Equal(expectedRootFields, event.RootFields) + + // Check that ID is generated + assert.NotEmpty(event.ID) + assert.Len(event.ID, 20) // generateEventID returns 20 character hash +} + +func TestCreateEventWithInvalidJSON(t *testing.T) { + assert := assert.New(t) + + testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) + row := VertexAILogRow{ + Endpoint: "https://us-central1-aiplatform.googleapis.com", + DeployedModelID: "model-123456", + LoggingTime: testTime, + RequestID: 12345.67, + RequestPayload: []string{"prompt1"}, + ResponsePayload: []string{"response1"}, + Model: "gemini-2.5-pro", + ModelVersion: "1.0", + APIMethod: "generateContent", + FullRequest: `{"invalid": json}`, // Invalid JSON + FullResponse: `{}`, + Metadata: `{}`, + OtelLog: `{}`, + } + + projectID := "test-project" + logger := logp.NewLogger("test") + + event, err := CreateEvent(row, projectID, logger) + + assert.NoError(err) // Should not error, but log warning + + // Invalid JSON should be stored as raw string + fullRequestField, err := event.MetricSetFields.GetValue("full_request.raw") + assert.NoError(err) + assert.Equal(`{"invalid": json}`, fullRequestField) +} + +func TestGenerateEventID(t *testing.T) { + testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) + row := VertexAILogRow{ + LoggingTime: testTime, + RequestID: 12345.67, + RequestPayload: []string{"prompt1", "prompt2"}, + } + + id1 := generateEventID(row) + id2 := generateEventID(row) + + // Same input should produce same ID + assert.Equal(t, id1, id2) + assert.Len(t, id1, 20) + + // Different input should produce different ID + row.RequestID = 98765.43 + id3 := generateEventID(row) + assert.NotEqual(t, id1, id3) +} + +func TestEventsMapping(t *testing.T) { + assert := assert.New(t) + + testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) + rows := []VertexAILogRow{ + { + Endpoint: "https://us-central1-aiplatform.googleapis.com", + DeployedModelID: "model-123456", + LoggingTime: testTime, + RequestID: 12345.67, + RequestPayload: []string{"prompt1"}, + ResponsePayload: []string{"response1"}, + Model: "gemini-2.5-pro", + ModelVersion: "1.0", + APIMethod: "generateContent", + FullRequest: `{}`, + FullResponse: `{}`, + Metadata: `{}`, + OtelLog: `{}`, + }, + { + Endpoint: "https://us-west1-aiplatform.googleapis.com", + DeployedModelID: "model-789012", + LoggingTime: testTime.Add(time.Hour), + RequestID: 67890.12, + RequestPayload: []string{"prompt2"}, + ResponsePayload: []string{"response2"}, + Model: "gemini-1.5-pro", + ModelVersion: "2.0", + APIMethod: "predict", + FullRequest: `{}`, + FullResponse: `{}`, + Metadata: `{}`, + OtelLog: `{}`, + }, + } + + projectID := "test-project" + logger := logp.NewLogger("test") + + events := EventsMapping(rows, projectID, logger) + + assert.Len(events, 2) + assert.Equal("model-123456", events[0].MetricSetFields["deployed_model_id"]) + assert.Equal("model-789012", events[1].MetricSetFields["deployed_model_id"]) +} + From ff778fdf30aa3590e687370058e59d516b8a8731 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Thu, 4 Sep 2025 23:13:19 +0530 Subject: [PATCH 05/26] Add documentation --- .../metricbeat-metricset-gcp-vertexai_logs.md | 144 ++++++++++++++++++ .../metricbeat/metricbeat-module-gcp.md | 1 + .../metricbeat/metricbeat-modules.md | 2 +- .../module/gcp/vertexai_logs/_meta/docs.md | 28 ++++ 4 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md create mode 100644 x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md diff --git a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md new file mode 100644 index 000000000000..fb7b97a54cb1 --- /dev/null +++ b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md @@ -0,0 +1,144 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-metricset-gcp-vertexai_logs.html + +applies_to: + stack: beta +--- + +% This file is generated! See scripts/docs_collector.py + +# Google Cloud Platform vertexai_logs metricset [metricbeat-metricset-gcp-vertexai_logs] + +The `vertexai_logs` metricset is designed to collect Vertex AI prompt-response logs from GCP BigQuery. BigQuery is a fully-managed, serverless data warehouse that stores detailed logs of interactions with Vertex AI models. + +Vertex AI logs export to BigQuery enables you to export detailed Google Cloud Vertex AI interaction data (such as prompts, responses, model usage, and metadata) automatically to a BigQuery dataset that you specify. Then you can access your Vertex AI logs from BigQuery for detailed analysis and monitoring using Metricbeat. This enables comprehensive tracking of AI model usage, performance monitoring, and cost analysis. + +The logs include detailed information about: +- API endpoints and deployed models +- Request and response payloads +- Model versions and API methods used +- OpenTelemetry trace data +- Request metadata and timing information + + +## Metricset-specific configuration notes [_metricset_specific_configuration_notes_14] + +* **table_id**: (Required) Full table identifier in the format `project_id.dataset_id.table_name` that contains the Vertex AI logs data. You can copy this from "Details" tab under + + +## Configuration example [_configuration_example_22] + +```yaml +- module: gcp + metricsets: + - vertexai_logs + period: 10m + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + table_id: "your_project.your_dataset.your_vertex_ai_logs_table" +``` + +## Fields [_fields] + +For a description of each field in the metricset, see the [exported fields](/reference/metricbeat/exported-fields-gcp.md) section. + +Here is an example document generated by this metricset: + +```json +{ + "@timestamp": "2025-09-02T10:14:50.313Z", + "agent": { + "hostname": "metricbeat-host", + "name": "metricbeat-host" + }, + "cloud": { + "account": { + "id": "elastic-beats" + }, + "provider": "gcp", + "project": { + "id": "elastic-beats" + } + }, + "event": { + "dataset": "gcp.vertexai_logs", + "duration": 123456789, + "module": "gcp" + }, + "gcp": { + "vertexai_logs": { + "endpoint": "https://us-central1-aiplatform.googleapis.com/v1/projects/elastic-beats/locations/us-central1/endpoints/123456789", + "deployed_model_id": "model-deployment-123", + "logging_time": "2025-09-02T10:14:50.313Z", + "request_id": 98765432101234567, + "model": "gemini-2.5-pro", + "model_version": "001", + "api_method": "generateContent", + "request_payload": [ + "What is the weather like today?" + ], + "response_payload": [ + "I don't have access to real-time weather information. Please check a weather service or app for current conditions." + ], + "full_request": { + "contents": [ + { + "parts": [ + { + "text": "What is the weather like today?" + } + ], + "role": "user" + } + ], + "generationConfig": { + "temperature": 0.7, + "maxOutputTokens": 1024 + } + }, + "full_response": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I don't have access to real-time weather information. Please check a weather service or app for current conditions." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "safetyRatings": [ + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "probability": "NEGLIGIBLE" + } + ] + } + ], + "usageMetadata": { + "promptTokenCount": 8, + "candidatesTokenCount": 24, + "totalTokenCount": 32 + } + }, + "metadata": { + "region": "us-central1", + "zone": "us-central1-a" + }, + "otel_log": { + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "span_id": "00f067aa0ba902b7" + } + } + }, + "metricset": { + "name": "vertexai_logs", + "period": 300000 + }, + "service": { + "type": "gcp" + } +} +``` diff --git a/docs/reference/metricbeat/metricbeat-module-gcp.md b/docs/reference/metricbeat/metricbeat-module-gcp.md index 33dc0081c57c..68e394f37adf 100644 --- a/docs/reference/metricbeat/metricbeat-module-gcp.md +++ b/docs/reference/metricbeat/metricbeat-module-gcp.md @@ -387,3 +387,4 @@ The following metricsets are available: * [metrics](/reference/metricbeat/metricbeat-metricset-gcp-metrics.md) * [pubsub](/reference/metricbeat/metricbeat-metricset-gcp-pubsub.md) * [storage](/reference/metricbeat/metricbeat-metricset-gcp-storage.md) +* [vertexai_logs](/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md) {applies_to}`stack: beta` diff --git a/docs/reference/metricbeat/metricbeat-modules.md b/docs/reference/metricbeat/metricbeat-modules.md index 3fb7b8dd3b70..e309c61fca4e 100644 --- a/docs/reference/metricbeat/metricbeat-modules.md +++ b/docs/reference/metricbeat/metricbeat-modules.md @@ -33,7 +33,7 @@ This section contains detailed information about the metric collecting modules c | [Elasticsearch](/reference/metricbeat/metricbeat-module-elasticsearch.md) | ![No prebuilt dashboards](images/icon-no.png "") | [ccr](/reference/metricbeat/metricbeat-metricset-elasticsearch-ccr.md)
[cluster_stats](/reference/metricbeat/metricbeat-metricset-elasticsearch-cluster_stats.md)
[enrich](/reference/metricbeat/metricbeat-metricset-elasticsearch-enrich.md)
[index](/reference/metricbeat/metricbeat-metricset-elasticsearch-index.md)
[index_recovery](/reference/metricbeat/metricbeat-metricset-elasticsearch-index_recovery.md)
[index_summary](/reference/metricbeat/metricbeat-metricset-elasticsearch-index_summary.md)
[ingest_pipeline](/reference/metricbeat/metricbeat-metricset-elasticsearch-ingest_pipeline.md) {applies_to}`stack: beta`
[ml_job](/reference/metricbeat/metricbeat-metricset-elasticsearch-ml_job.md)
[node](/reference/metricbeat/metricbeat-metricset-elasticsearch-node.md)
[node_stats](/reference/metricbeat/metricbeat-metricset-elasticsearch-node_stats.md)
[pending_tasks](/reference/metricbeat/metricbeat-metricset-elasticsearch-pending_tasks.md)
[shard](/reference/metricbeat/metricbeat-metricset-elasticsearch-shard.md) | | [Envoyproxy](/reference/metricbeat/metricbeat-module-envoyproxy.md) | ![No prebuilt dashboards](images/icon-no.png "") | [server](/reference/metricbeat/metricbeat-metricset-envoyproxy-server.md) | | [Etcd](/reference/metricbeat/metricbeat-module-etcd.md) | ![No prebuilt dashboards](images/icon-no.png "") | [leader](/reference/metricbeat/metricbeat-metricset-etcd-leader.md)
[metrics](/reference/metricbeat/metricbeat-metricset-etcd-metrics.md) {applies_to}`stack: beta`
[self](/reference/metricbeat/metricbeat-metricset-etcd-self.md)
[store](/reference/metricbeat/metricbeat-metricset-etcd-store.md) | -| [Google Cloud Platform](/reference/metricbeat/metricbeat-module-gcp.md) | ![Prebuilt dashboards are available](images/icon-yes.png "") | [billing](/reference/metricbeat/metricbeat-metricset-gcp-billing.md)
[carbon](/reference/metricbeat/metricbeat-metricset-gcp-carbon.md) {applies_to}`stack: beta`
[compute](/reference/metricbeat/metricbeat-metricset-gcp-compute.md)
[dataproc](/reference/metricbeat/metricbeat-metricset-gcp-dataproc.md)
[firestore](/reference/metricbeat/metricbeat-metricset-gcp-firestore.md)
[gke](/reference/metricbeat/metricbeat-metricset-gcp-gke.md)
[loadbalancing](/reference/metricbeat/metricbeat-metricset-gcp-loadbalancing.md)
[metrics](/reference/metricbeat/metricbeat-metricset-gcp-metrics.md)
[pubsub](/reference/metricbeat/metricbeat-metricset-gcp-pubsub.md)
[storage](/reference/metricbeat/metricbeat-metricset-gcp-storage.md) | +| [Google Cloud Platform](/reference/metricbeat/metricbeat-module-gcp.md) | ![Prebuilt dashboards are available](images/icon-yes.png "") | [billing](/reference/metricbeat/metricbeat-metricset-gcp-billing.md)
[carbon](/reference/metricbeat/metricbeat-metricset-gcp-carbon.md) {applies_to}`stack: beta`
[compute](/reference/metricbeat/metricbeat-metricset-gcp-compute.md)
[dataproc](/reference/metricbeat/metricbeat-metricset-gcp-dataproc.md)
[firestore](/reference/metricbeat/metricbeat-metricset-gcp-firestore.md)
[gke](/reference/metricbeat/metricbeat-metricset-gcp-gke.md)
[loadbalancing](/reference/metricbeat/metricbeat-metricset-gcp-loadbalancing.md)
[metrics](/reference/metricbeat/metricbeat-metricset-gcp-metrics.md)
[pubsub](/reference/metricbeat/metricbeat-metricset-gcp-pubsub.md)
[storage](/reference/metricbeat/metricbeat-metricset-gcp-storage.md)
[vertexai_logs](/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md) {applies_to}`stack: beta` | | [Golang](/reference/metricbeat/metricbeat-module-golang.md) | ![Prebuilt dashboards are available](images/icon-yes.png "") | [expvar](/reference/metricbeat/metricbeat-metricset-golang-expvar.md)
[heap](/reference/metricbeat/metricbeat-metricset-golang-heap.md) | | [Graphite](/reference/metricbeat/metricbeat-module-graphite.md) | ![No prebuilt dashboards](images/icon-no.png "") | [server](/reference/metricbeat/metricbeat-metricset-graphite-server.md) | | [HAProxy](/reference/metricbeat/metricbeat-module-haproxy.md) | ![Prebuilt dashboards are available](images/icon-yes.png "") | [info](/reference/metricbeat/metricbeat-metricset-haproxy-info.md)
[stat](/reference/metricbeat/metricbeat-metricset-haproxy-stat.md) | diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md new file mode 100644 index 000000000000..2c9843532fc9 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md @@ -0,0 +1,28 @@ +The `vertexai_logs` metricset is designed to collect Vertex AI prompt-response logs from GCP BigQuery. BigQuery is a fully-managed, serverless data warehouse that stores detailed logs of interactions with Vertex AI models. + +Vertex AI logs export to BigQuery enables you to export detailed Google Cloud Vertex AI interaction data (such as prompts, responses, model usage, and metadata) automatically to a BigQuery dataset that you specify. Then you can access your Vertex AI logs from BigQuery for detailed analysis and monitoring using Metricbeat. This enables comprehensive tracking of AI model usage, performance monitoring, and cost analysis. + +The logs include detailed information about: +- API endpoints and deployed models +- Request and response payloads +- Model versions and API methods used +- OpenTelemetry trace data +- Request metadata and timing information + + +## Metricset-specific configuration notes [_metricset_specific_configuration_notes_14] + +* **table_id**: (Required) Full table identifier in the format `project_id.dataset_id.table_name` that contains the Vertex AI logs data. You can copy this from "Details" tab under + + +## Configuration example [_configuration_example_22] + +```yaml +- module: gcp + metricsets: + - vertexai_logs + period: 10m + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + table_id: "your_project.your_dataset.your_vertex_ai_logs_table" +``` From 124e2847bac8950ac8043b573248d47de2524d9c Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Thu, 4 Sep 2025 23:46:42 +0530 Subject: [PATCH 06/26] Update docs --- .../metricbeat-metricset-gcp-vertexai_logs.md | 50 +++++++++++++++++++ .../module/gcp/vertexai_logs/_meta/docs.md | 50 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md index fb7b97a54cb1..61e12dbd14ab 100644 --- a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md +++ b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md @@ -39,6 +39,56 @@ The logs include detailed information about: table_id: "your_project.your_dataset.your_vertex_ai_logs_table" ``` +## Sample Event + +Here is a sample event for `vertexai_logs`: + +```json +{ + "@timestamp": "2023-12-01T10:30:45.000Z", + "cloud": { + "provider": "gcp", + "project": { + "id": "my-gcp-project" + } + }, + "gcp": { + "vertexai_logs": { + "endpoint": "https://us-central1-aiplatform.googleapis.com", + "deployed_model_id": "1234567890123456789", + "logging_time": "2023-12-01T10:30:45.000Z", + "request_id": 98765432101234567, + "request_payload": ["What is machine learning?"], + "response_payload": ["Machine learning is a subset of artificial intelligence..."], + "model": "gemini-2.5-pro", + "model_version": "1.0", + "api_method": "generateContent", + "full_request": { + "inputs": ["What is machine learning?"], + "parameters": { + "temperature": 0.7 + } + }, + "full_response": { + "outputs": ["Machine learning is a subset of artificial intelligence..."], + "usage": { + "input_tokens": 5, + "output_tokens": 50 + } + }, + "metadata": { + "user_id": "user123", + "session_id": "session456" + }, + "otel_log": { + "trace_id": "abc123def456", + "span_id": "789ghi012jkl" + } + } + } +} +``` + ## Fields [_fields] For a description of each field in the metricset, see the [exported fields](/reference/metricbeat/exported-fields-gcp.md) section. diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md index 2c9843532fc9..f0d66c9bf1f4 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md @@ -26,3 +26,53 @@ The logs include detailed information about: credentials_file_path: "your JSON credentials file path" table_id: "your_project.your_dataset.your_vertex_ai_logs_table" ``` + +## Sample Event + +Here is a sample event for `vertexai_logs`: + +```json +{ + "@timestamp": "2023-12-01T10:30:45.000Z", + "cloud": { + "provider": "gcp", + "project": { + "id": "my-gcp-project" + } + }, + "gcp": { + "vertexai_logs": { + "endpoint": "https://us-central1-aiplatform.googleapis.com", + "deployed_model_id": "1234567890123456789", + "logging_time": "2023-12-01T10:30:45.000Z", + "request_id": 98765432101234567, + "request_payload": ["What is machine learning?"], + "response_payload": ["Machine learning is a subset of artificial intelligence..."], + "model": "gemini-2.5-pro", + "model_version": "1.0", + "api_method": "generateContent", + "full_request": { + "inputs": ["What is machine learning?"], + "parameters": { + "temperature": 0.7 + } + }, + "full_response": { + "outputs": ["Machine learning is a subset of artificial intelligence..."], + "usage": { + "input_tokens": 5, + "output_tokens": 50 + } + }, + "metadata": { + "user_id": "user123", + "session_id": "session456" + }, + "otel_log": { + "trace_id": "abc123def456", + "span_id": "789ghi012jkl" + } + } + } +} +``` \ No newline at end of file From e7a3c9a486f72e3092f036b77f9117ea7a3833fe Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Fri, 5 Sep 2025 09:53:43 +0530 Subject: [PATCH 07/26] Update documentaion and make update --- docs/reference/metricbeat/exported-fields-gcp.md | 2 +- .../metricbeat/metricbeat-metricset-gcp-vertexai_logs.md | 2 +- docs/reference/metricbeat/metricbeat-module-gcp.md | 2 +- docs/reference/metricbeat/metricbeat-modules.md | 2 +- docs/reference/toc.yml | 1 + x-pack/metricbeat/module/gcp/fields.go | 2 +- x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml | 2 ++ 7 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/reference/metricbeat/exported-fields-gcp.md b/docs/reference/metricbeat/exported-fields-gcp.md index 65675a5e55df..bc2fff92cbb3 100644 --- a/docs/reference/metricbeat/exported-fields-gcp.md +++ b/docs/reference/metricbeat/exported-fields-gcp.md @@ -1204,7 +1204,7 @@ Google Cloud Storage metrics ## vertexai_logs [_vertexai_logs] ```{applies_to} -stack: beta +stack: beta 9.2.0 ``` Google Cloud Vertex AI Prompt Response Logs metrics diff --git a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md index 61e12dbd14ab..d1cd86f9634f 100644 --- a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md +++ b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md @@ -3,7 +3,7 @@ mapped_pages: - https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-metricset-gcp-vertexai_logs.html applies_to: - stack: beta + stack: beta 9.2.0 --- % This file is generated! See scripts/docs_collector.py diff --git a/docs/reference/metricbeat/metricbeat-module-gcp.md b/docs/reference/metricbeat/metricbeat-module-gcp.md index 68e394f37adf..245dfbbd2402 100644 --- a/docs/reference/metricbeat/metricbeat-module-gcp.md +++ b/docs/reference/metricbeat/metricbeat-module-gcp.md @@ -387,4 +387,4 @@ The following metricsets are available: * [metrics](/reference/metricbeat/metricbeat-metricset-gcp-metrics.md) * [pubsub](/reference/metricbeat/metricbeat-metricset-gcp-pubsub.md) * [storage](/reference/metricbeat/metricbeat-metricset-gcp-storage.md) -* [vertexai_logs](/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md) {applies_to}`stack: beta` +* [vertexai_logs](/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md) {applies_to}`stack: beta 9.2.0` diff --git a/docs/reference/metricbeat/metricbeat-modules.md b/docs/reference/metricbeat/metricbeat-modules.md index e309c61fca4e..073af657abd2 100644 --- a/docs/reference/metricbeat/metricbeat-modules.md +++ b/docs/reference/metricbeat/metricbeat-modules.md @@ -33,7 +33,7 @@ This section contains detailed information about the metric collecting modules c | [Elasticsearch](/reference/metricbeat/metricbeat-module-elasticsearch.md) | ![No prebuilt dashboards](images/icon-no.png "") | [ccr](/reference/metricbeat/metricbeat-metricset-elasticsearch-ccr.md)
[cluster_stats](/reference/metricbeat/metricbeat-metricset-elasticsearch-cluster_stats.md)
[enrich](/reference/metricbeat/metricbeat-metricset-elasticsearch-enrich.md)
[index](/reference/metricbeat/metricbeat-metricset-elasticsearch-index.md)
[index_recovery](/reference/metricbeat/metricbeat-metricset-elasticsearch-index_recovery.md)
[index_summary](/reference/metricbeat/metricbeat-metricset-elasticsearch-index_summary.md)
[ingest_pipeline](/reference/metricbeat/metricbeat-metricset-elasticsearch-ingest_pipeline.md) {applies_to}`stack: beta`
[ml_job](/reference/metricbeat/metricbeat-metricset-elasticsearch-ml_job.md)
[node](/reference/metricbeat/metricbeat-metricset-elasticsearch-node.md)
[node_stats](/reference/metricbeat/metricbeat-metricset-elasticsearch-node_stats.md)
[pending_tasks](/reference/metricbeat/metricbeat-metricset-elasticsearch-pending_tasks.md)
[shard](/reference/metricbeat/metricbeat-metricset-elasticsearch-shard.md) | | [Envoyproxy](/reference/metricbeat/metricbeat-module-envoyproxy.md) | ![No prebuilt dashboards](images/icon-no.png "") | [server](/reference/metricbeat/metricbeat-metricset-envoyproxy-server.md) | | [Etcd](/reference/metricbeat/metricbeat-module-etcd.md) | ![No prebuilt dashboards](images/icon-no.png "") | [leader](/reference/metricbeat/metricbeat-metricset-etcd-leader.md)
[metrics](/reference/metricbeat/metricbeat-metricset-etcd-metrics.md) {applies_to}`stack: beta`
[self](/reference/metricbeat/metricbeat-metricset-etcd-self.md)
[store](/reference/metricbeat/metricbeat-metricset-etcd-store.md) | -| [Google Cloud Platform](/reference/metricbeat/metricbeat-module-gcp.md) | ![Prebuilt dashboards are available](images/icon-yes.png "") | [billing](/reference/metricbeat/metricbeat-metricset-gcp-billing.md)
[carbon](/reference/metricbeat/metricbeat-metricset-gcp-carbon.md) {applies_to}`stack: beta`
[compute](/reference/metricbeat/metricbeat-metricset-gcp-compute.md)
[dataproc](/reference/metricbeat/metricbeat-metricset-gcp-dataproc.md)
[firestore](/reference/metricbeat/metricbeat-metricset-gcp-firestore.md)
[gke](/reference/metricbeat/metricbeat-metricset-gcp-gke.md)
[loadbalancing](/reference/metricbeat/metricbeat-metricset-gcp-loadbalancing.md)
[metrics](/reference/metricbeat/metricbeat-metricset-gcp-metrics.md)
[pubsub](/reference/metricbeat/metricbeat-metricset-gcp-pubsub.md)
[storage](/reference/metricbeat/metricbeat-metricset-gcp-storage.md)
[vertexai_logs](/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md) {applies_to}`stack: beta` | +| [Google Cloud Platform](/reference/metricbeat/metricbeat-module-gcp.md) | ![Prebuilt dashboards are available](images/icon-yes.png "") | [billing](/reference/metricbeat/metricbeat-metricset-gcp-billing.md)
[carbon](/reference/metricbeat/metricbeat-metricset-gcp-carbon.md) {applies_to}`stack: beta`
[compute](/reference/metricbeat/metricbeat-metricset-gcp-compute.md)
[dataproc](/reference/metricbeat/metricbeat-metricset-gcp-dataproc.md)
[firestore](/reference/metricbeat/metricbeat-metricset-gcp-firestore.md)
[gke](/reference/metricbeat/metricbeat-metricset-gcp-gke.md)
[loadbalancing](/reference/metricbeat/metricbeat-metricset-gcp-loadbalancing.md)
[metrics](/reference/metricbeat/metricbeat-metricset-gcp-metrics.md)
[pubsub](/reference/metricbeat/metricbeat-metricset-gcp-pubsub.md)
[storage](/reference/metricbeat/metricbeat-metricset-gcp-storage.md)
[vertexai_logs](/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md) {applies_to}`stack: beta 9.2.0` | | [Golang](/reference/metricbeat/metricbeat-module-golang.md) | ![Prebuilt dashboards are available](images/icon-yes.png "") | [expvar](/reference/metricbeat/metricbeat-metricset-golang-expvar.md)
[heap](/reference/metricbeat/metricbeat-metricset-golang-heap.md) | | [Graphite](/reference/metricbeat/metricbeat-module-graphite.md) | ![No prebuilt dashboards](images/icon-no.png "") | [server](/reference/metricbeat/metricbeat-metricset-graphite-server.md) | | [HAProxy](/reference/metricbeat/metricbeat-module-haproxy.md) | ![Prebuilt dashboards are available](images/icon-yes.png "") | [info](/reference/metricbeat/metricbeat-metricset-haproxy-info.md)
[stat](/reference/metricbeat/metricbeat-metricset-haproxy-stat.md) | diff --git a/docs/reference/toc.yml b/docs/reference/toc.yml index 238ab1d82ba8..59bf3020676d 100644 --- a/docs/reference/toc.yml +++ b/docs/reference/toc.yml @@ -955,6 +955,7 @@ toc: - file: metricbeat/metricbeat-metricset-gcp-metrics.md - file: metricbeat/metricbeat-metricset-gcp-pubsub.md - file: metricbeat/metricbeat-metricset-gcp-storage.md + - file: metricbeat/metricbeat-metricset-gcp-vertexai_logs.md - file: metricbeat/metricbeat-module-golang.md children: - file: metricbeat/metricbeat-metricset-golang-expvar.md diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index 5292259a2175..ece39e58da4f 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzsXUuT27aW3vtXoO7G9lRbmTgzs3BN3apO+ya3J7bTZXW7ala8EHEkIQIBBgC7rfz6KbxIUIIkig91nJpk5RaJ850HzgM4AN+gDWzfoVVevkBIU83gHXr5sxArBuiGiYqgO4b1Usji5QuEJDDACt6hFX6BEAGVS1pqKvg79PcXCCH0880dKgSpGLxAaEmBEfXO/vAGcVxAIGT+09vS/FuKKvwlfj5+h+EFMFX/ObwqFr9BrqM/J/CE/xwuTrWQlK9QAVrSXO2PvAshhlEpkLN/a/10EIr5z/0xc09sYPskJEkOXIDGBGs81eCG1UnGVluloZhkaAlKVDKH0QYPA/+tFoj7/2+n7ao1LhHVwlp34teswGVJ+co/+rfW4Ees86M3R73GGknQleRA0FKKArWm4vXdLfq9Armd7bG1oIxRvjpErzXMj+7ZYBrRO+353RZLPFNRcqoELLlQTh4v9hWX0nkL6Y1Q2j6rEOU5qwggCauKYXmFNP56hTD5rVK6AK6vEOYESVFxYoQOUgo5S+Ch/FHQHLJCcL3ugykITEIppEZ2nBShUgprC5T0oXLn3ka375FYIr2GoNRAdwFM8JVCWqSIa6ExS9BdMoH1Yar35rWaEi5ExXVqeLWpevJ1v4aIpzCxjTslaLG1f1QgH2kOh+hGw/UBcB3/cw+HGSQJBv0kJIKvuCgZXCG888ZSSD+d5lpIvAJEFZprzAmWzd8e5kmeHIVR5OnHcr7D/KFSHgxWSuQUayDoiaYNNgAZKGCDqOVhWqCc9QKx6NoeyEa9lC3jlUoA4aA0HFV0LhiDPOh5A9s3j5hVgEpMpfevpRSPlADChFDzIGZNAG4NncoFGogb2O78ckxczXsWT+c3w1uwXBq2HiErJc1TjvXUNF8DytdYroAgO4Q1YGcr3pJaGpz/8qCse53/8oA0Balm6DMsjXAVygXXEufajmT0SJcIlyWjOV6YuSL0GuQTVXCFqH6p7OiMKvc87AevHMtFy/IOx64b+yj6SQhdSsr1sSC2AD0kjI3mzQ37jkW0rHF38ukBgvnXEBDm/QEwhvmruXcGt++tzSVhWEOc0EMFDHEgOB+MhFVP+p/tmwmSJyjWoGYqFyV8f/7En5v30Pd7XHag9nbGRI4PiLwT2bcp4RqHEQZ+s8Am8hag14IIJlbbTrgKLDegR0flhu2F6Ye+YH44UzNiuVSgU9GxU6LniflREq5YFGWloZsvds9OVkcsqYQnzNiMSFGWQGaLrYYU58Z3HWb8lueiMOq1ryM/WMj2ApEO9LMS5xvQKsttkrwfyc9C4wc7Aw/lSmOewywvq5kE4xqBZLmQoA6C2StXd+B8qooFSJMC2HFQGBYJbuGsTTnmE4RA/xQ0680yTQuYKch7gHqwaYlxlpgxD4xypCAXnKhO5GdlnvIPpyibNGlpUpuoUsDMeisg6ObuweWQVKG8khK4ZluDrFIQBNZFSISqzUwC7mvRN8b+DDxn0WYkt1pgBu5EOBPlMDOuIZghHYLbX5EoQVq/flRJFsWTpBrG4d8MpYEjLboJwJIeWQJ2zO4iKKAQcjtbGNsSfCZxkSn6B/SEYqzWlv++fjeoHAVjnMYqv3ycofs1Vd5XGwMWnG0RfsSUmaTdzrYvH32N5NJBI1DzMrxFS1xQloyDx1gyVXVPlj46+M0ssxX6s3GjnnCZUd7TXo1+9jRj5wzlHtWqAqXdJKZaIfHEkaGJVIlzeBZuRaXHZDdMUstiw7EWz8EvB/0k5GZG+UqCUmO54RzoY1i4NWA8mXOQ+KxgZj1Tf0QhuRiECUYUjgLzr0eQPUGMLJez4VTlkXzmOPEmxfIJjPdhaI0VWgBwJCvOKV8dNVkHILNuvheMfzBcGh9qhkGK8hwCjieskNJYaiBXUZ41Q3O79kkQPILcov/69+aX66UGiZT5nfLVlV3IMxOVC40eqaJhllalmZjfv21e3as4zKulFHmnkuO9f3j0miOqgFilNMjZmizVzIDjgkDK8A6KfS/jJ9Tkj07vvLaGf77/aW4Z+mQIOGeGJQRrMKq365Q1otNwlVt2znJc4pzqbSL4HsmED+IOw9Wo3cZfjVTwAMOYz88/noG00pTRP2zCNAisiTklyBy49kuaDqhfh2/nER3wVXwNmOn1NlswkW+m0H9NAjkSQeF2gaiLyn8Ti9kSUwZkAnS/iYW3yTV+BOToGE13tEcDLvi1KdHFM+Y8dKpaFCYnuYD0rJev6RmX2BFnXU1Mp+imYBmi7gbodErfRdpP9Q3SKQ0gKdXeZrDFks/qdYjMpdRZ4+9Gdp3/e/35U11AqmYBpAvIspzCV2K7+eWA+Z0mK9wOiHLBNaYc5BS4LKCIwmk4XncHqv0+0dkgijVmhu4Shy0ck9wUmOPVhPIx+c1HT2M3u+lu/iW4No+xBGh3RF1a0JTkv5ui9MpJ76pe6/OkzYRdAFpWbEkZixoG8jWQinXi4pFKXWHmV23HF7gfv1k0NRo4HQtzYTJ9M7Rbtz0s2NOdV2uqtFhJXJySvq1EXKQUYmOk62FAU67aZ1xbgyk4rO9UCJvXQslvH6kLA7ehqRB1q7R+wBOuy4iAVPJUGjoJ7zYyqNKYITVBbEUfTZTQWEPXQPZ82ovD3Nk6rF8eUZONUJ5Bn5E0Omk13mgylUq3bbefwtMTFsFE5FUBXM8IGKEPcFT3LfekqjwHpZYVq0kgR+JA7KyB2O2SKWEYAsoZ7e8VSAoKCYmYEJuqPAXObWZMic5SSKyerDYHjeblv1Yb+FfITlygKOJO0ycTuiQu617Tmzs01zjfEEkfQdp2U/+23R3f759eCmnf+vmXf7wczQqbrWifVtldPRPQssE7izdVUTFsc8mbu4fQ/8Sjfca4E7BG0G1V7CQPjBZUD9ywNbAdUjta2KRsCA3GFy/JjLeLanA7xF13UP0ugGvgyzHnQiP4mgMQ9D3Cyiuv/YPrNCuoHm/x8u1/nCFBn0GOtinfaNuPnDDOiddpT7E6jbk07J6z6b5rMiZdX0nA2i70Y75jOLHVeILPYDdQrqEAiVnmVyvdPOy57/JB5Jihesx6BdTNPcrdfkxfL7GPNchtXLR+1Anw2vXfccH6juuBUP0e7BDd+03zdGAYFeBEs76IGfjG4kRn5+nlWJpUZokrpntuajZRorTtUmYodYUWUmyAIyKeuA0V2xKuUIF/E9L2cxeUp0/J7AEcNrM/tpZUelnjxAq4VADzRv2XjmFepAOcq7eXcXypKZCxHD6xTKWhdgx3jW2XpNsvfwbrPdqJcNJSaQFRMdOcGmrz1+pKOEsHXBDXhek3DfCCwWjZcDRmlBn7+WMpD8c6jTfYRd7dGVxoKteyuGidHamud4ldI7etMQNtzTXJ80T9NczK9vPQyCJGTkddw1VscWNjp7YlJltK6Nux+pOEeMXLDWhwsjRTY8J2LVQ9O2137OMywB3ins2YFvEhKxnbMiaorhzQeFmuJ1qfowyfeJFz87ndhBgvEJAGJqhd6q0Euctm/7F0h0ynJhnxE6hucB7P4Q5Ppie00NCXGzqLM6fHnn25DU6+I9i6czmCu9exOwC/Aq6nwq4l5so3+YwMv6QkcwsX/Y+KFPgrunM3CPw6H2irBs+AYx7tja/QG1JKkYNS4bjHUJAZwVAI/mzbSFb3DB6BhcZZB2hQyhuYGrnyDi0tuzh7lOOlIHveYrg/O+UnSkEGzrMYt/ES42NO+IdxYD8KVhVj5IsNYnu6zZcU9QEc369hSF4qgEfsDbD1Tym+4rl6nKUTuCbJ1NzgiexsAcZXxuCPZWjHVxF3lsb9YcJa3x6DP5x1kWMhTGCywAzzvOu1Vh8EJujH8Mpkp9LXWpdqtsD5BjjJhi2St8NfdFIK1xu+viXln/f3d9/NrVSQE4tRpEAeR9I000j7ZTs7gTpg84fFF9saiD16kQDbBaAqBVd9q7LjsnRDe2HWWF8JiXKcr+G1kSV81SA5Zhb/q/nrrgxcygZyRoFrZaCeJ+GJVX8umEup+dCc8XJMQWQ/zIIVDDv+eBCmRVZb2v3N3XcP7+9CxN/B6u20wRxiwpKJJ3c/2f3Nnf2XcjctWQ03d12YlLTplkdKS8CFPU3bjfls2LHLthBaZy/HE8MJRoYd8T2uRo8mdh2dOJlWdZ7jiXXXm/c0esqnn3W3H35M2NKr5QBdvO7GzsS6SDN2epLUKC8zSWKY00r90lMg4ix5nWFeZkqxrJTi63aWM6HsvT2cu6sKD29XdS1morFCa68EpEEWlNuba2xpaebnfP4BORgncQ6aibvrpI3UvnyMbNTdu9UR0DArPYyo0eOXj+ch4vB0AT3mtm7rp0RRAh8B4o0/KBUdcKy0KTbtuag2bCmq1dq6npNY2wUAwxp4To9s3yYObHQ4rrF7GSxVWtJFFRfZjvQW5ZjlFbPStqbxtAYet9a4uxKMJMJ6gWEtrIcE52bcGIt+3r+Fo/bWlrbS1iZdeu2z44PSquO8u40Qs/+X30D5ucaBb0V6ewueVkS7IlsJja5vfmk5OMGdrIKMrNAOC2opBdfGroxHkfrwjsNkcvl8f48KwKqSRiJCIsD5OvI2aAH6CYAHBjEnp3xNnSJ8y5MmMJGucCebUui0VL/tqdRFrhNMtG9baK1Vs0sILa51pdbPKTj77QGkJS3dAVAvyKvGa4WcrZUjhWuAa9ElK+ZTRc4z8t5yzAdZrKdTWBhom4at/E5nsM8fhlQhhF4DsWy/ohwV6nXDfhyGXyorCKVxvrly0aqgvNLQqmQZ3oL0VUiJlV+trN22mw/xjkh648J+uGd/lyO9n9HnFt9D3+Mpq4WqFp0Gv6sW82ox2V6M4rhUa6FtNGdiNWjP08Y7e3lGfRpFKbyye8y2N5q4o9w10Q6AXIdJtthmLtu8KMC9syBOJx7JMfS54Eu6yqqS4FEaZPJw/54b2J/MR/ka8xWoK6dxN5eiSwG2JbgvvICq2HFx86rIgjQGV+FD9B4DidQ+MaT+mhaMgNIBcoZX/e44vF6B9Yx+O/d1sFA3fIDfR6D7ACOxXhzq+YKuFjXtGc43gZER51RtGzjfcPHEgKzcVLpu/l1v27XmGgFG7R6+IX0S/SQ+tuIt1DUvr/BsM8Mz5InWP7z2+oiBnQS+1ZDlovcObUvo7ixbc9NHlOs1bVK2/YL6zhItwiPo90pojOI+kVPYL+uJmyI7BjHAP8esEMAkY6A1yClnQVktGFVrJ3hDEzmaSIuS5i1eOgIvBMnM1DWDMcphSvRPa6EABUq24HK6t4A/CkKX2+t88z48MMK8PsReFl8wNBaj+xwEhxtPqRF0NKznoRv0pPA7AjaJQrSGPkH24q0gXDQYU39ppolaI+CkFJSbsFZp26W1Bd2KI534qHhNazw+LhEZfHIRYr2xoIaFqZKhFFu76caoTAzOQ8dlaFgO5bm11jG9rtJGeMgGB5hggp+p1XaAt5H1VVaMZVHiO0lUiRjpGE9sBw5GBJaU07DYk3pVQbSW8V2zmNHi8rvTMdR/a6SzuMaPX/hUOdAxclmIU2rTEPgrqHF8FVrJDNOdWk+CS60R1hqKMo0LPXBGN2AZUFduwdS8Y5tIJaJFyaAArl1lQQS4nvEF1vnaXRtZ5xVoLlyJUl8twtm2ueVOcGi9MHP3G0fEpFG86y+ynww21iHqKxrjd+39GrgsAUtUVEzTkrmbHpPL1i1Bt6PxuDVzOi8akEHspA6TrlmeC35Y9FEAU7hSM+xRez+Ni+spy8iwW9x2eVoEFCbH9m3QAypH15JnqpeLRPl5IHdn3OBfJ+YnxDiByQYi7SBiPw2NuOBvjC1vW1KlpJ9tt9mZ0iJ2mLqUGfx3Lgj8vZcxnCu8S67NtOfX/nLHsBWaU5w9m8nvM+o2HEdgsm6dmZApf+SlD9pQdE4a8A+t4wyP9HZBd/jq/rAl/Wgx366e2yX+dFuDhfutbK06tMHdGIUebrweofPi/U7fhV/Fr/2yBWBXMyyI1wcRfwurYX4nope5/znXi4Zw9GerPobwooCTiddG2vPim8l4I/lMsf7gpHLm0ojH9GeP2Q7m8wfrfhNjrwPiIj2Lu/EE5xtUEzbsFJQxWl9ccO98Q90JS1XzQZLQZG67YduTRNlT5LieUvFGW1Ni20ibeNt30iqEeapXw7eJ4ron5azVvecRs11nC/gbedumSZpLEULLUcuveXM3wL04QLvVaDj3t8VN1WmISzrwGP17YBo3PsF+XAPb1dBIFMbozC8F6LUglnbI01xTtKk8U3aAK73+Y4Zzli2wApJ5RWL7JZFxENf1k7VJpzTrGLg3G38jyUpiroEgRxspwYBtEans9PBPXt98SObIDRtNdOqJ/kH5r9ld33yIv8yTuB/7MBIvRlVCTpc0zwyyotJDovqOVEPnTYFJLKBA8aCkRrpZaQfNzp1KuzcSjWOrI1yulISd/KjzOJDD/ZZOO33PE+9creQGU6gEiRZVvgHdAlt/u5ZhpVpX+thvo4fue57bzzcigrdXlhl7aVB4TkLpDo9i7Ru7/OU8rjv/EbNwYlNU7iZogpPHp1p3kdokJAvR89uxvPEuVA1JE2as1qO/ru5Pqsogg0eQGr5imjGxUp1i6xf7Brq+RXdSFKVGn4OcP4iVOhZyF6CHBN3QnpTQ0Qa2T0KSI2paQwTcWEsYDT18/uCWccI5nBDYExAIlExsTaEtCLCMkr5Y3H2LVit+SGSHdNHU33i4c49/Ag8Tq5UpVUxmmoBCsD5x+7vSuCibk13Xt856/J1jT1hZEunGr5BaJqVw6m6zB05/N4ZMgGu6pCBr8V/fHuM4EC3xlgmcoqzha9tEKCfw9R1aYqaOALqWEm+jxAZ5CqZco3yl4m+f2V3h0lq/sjOT8rJKXyEUnNCkgP38O4L4+tbbl6j0Iaj2gT4G/cl4XG/ONSE7pV7BbDW7QisoKKdv3s7+800pRfK+DjehHkEqKngfEF/cq0kcyYyupJmLGH3ncBRzTOoes8tNfgk39qiavkKlBEJznWR8GfWGdKvKgOMFA/IOaVkdu5U0fHmz7oxwuWRkFbjZB0EENKbMhq3/mf/6yUzHAien4DJeF5kKsjfpQ5j9z91BF6Axwa34MxzvNSF2HQ2zenyElRI5tZHZ7iwl3OpptEIDMxF5VLS/lsDvgYGJ0Fvj11FfwP8XAAD//ykgdJA=" + return "eJzsXd2P27aWf89fQfQlyWLi3qa7C2ywuMB0ctubbZIO4pkA+6RLi8c2a4pUSGom7l+/4JdE2bQt68PTFNs+ZSzx/M4Hzwd5SL1CG9i+Qau8fIaQpprBG/T8FyFWDNANExVBtwzrpZDF82cISWCAFbxBK/wMIQIql7TUVPA36O/PEELol5tbVAhSMXiG0JICI+qN/eEV4riAQMj8p7el+bcUVfhL/Hz8DsMLYKr+c3hVLH6HXEd/TuAJ/zlcnGohKV+hArSkudofeRdCDKNSIGf/1vrpIBTzn/tj5p7YwPZRSJIcuACNCdZ4qsENq5OMrbZKQzHJ0BKUqGQOow0eBv6uFoj7/7vTdtUal4hqYa078WtW4LKkfOUf/a41+BHr/ODNUa+xRhJ0JTkQtJSiQK2peH37Dn2pQG5ne2wtKGOUrw7Raw3zk3s2mEb0Tnt+t8USz1SUnCoBSy6Uk8ezfcWldN5CeiOUts8qRHnOKgJIwqpiWF4hjb9eIUx+r5QugOsrhDlBUlScGKGDlELOEngofxA0h6wQXK/7YAoCk1AKqZEdJ0WolMLaAiV9qNy6t9G7t0gskV5DUGqguwAm+EohLVLEtdCYJegumcD6MNU781pNCRei4jo1vNpUPfm6W0PEU5jYxp0StNjaPyqQDzSHQ3Sj4foAuI7/uYfDDJIEg34WEsFXXJQMrhDeeWMppJ9Ocy0kXgGiCs015gTL5m/38yRPjsIo8vRjOd9h/lApDwYrJXKKNRD0SNMGG4AMFLBB1PIwLVDOeoFYdG0PZKNeypbxSiWAcFAajio6F4xBHvS8ge2rB8wqQCWm0vvXUooHSgBhQqh5ELMmALeGTuUCDcQNbHd+OSau5j2Lp/Ob4S1YLg1bD5CVkuYpx3pqmq8B5WssV0CQHcIasLMVb0ktDc5/vVfWvc5/vUeaglQz9AmWRrgK5YJriXNtRzJ6pEuEy5LRHC/MXBF6DfKRKrhCVD9XdnRGlXse9oNXjuWiZXmHY9eNfRT9LIQuJeX6WBBbgB4Sxkbz5oZ9xyJa1rg7+fQAwfxrCAjz/gAYw/zV3DuDd2+tzSVhWEOc0EMFDHEgOB+MhFVP+p/smwmSJyjWoGYqFyX8cP7En5v30A97XHag9nrGRI4PiLwT2dcp4RqHEQZ+tcAm8hag14IIJlbbTrgKLDegR0flhu2F6ce+YH48UzNiuVSgU9GxU6LniflREq5YFGWloZsvds9OVkcsqYRHzNiMSFGWQGaLrYYU58Z3HWb8Hc9FYdRrX0d+sJDtBSId6GclzjegVZbbJHk/kp+Fxg92Bh7KlcY8h1leVjMJxjUCyXIhQR0Es1eu7sD5WBULkCYFsOOgMCwS3MJZm3LMJwiB/ilo1ptlmhYwU5D3AHVv0xLjLDFjHhjlSEEuOFGdyM/KPOUfTlE2adLSpDZRpYCZ9VZA0M3tvcshqUJ5JSVwzbYGWaUgCKyLkAhVm5kE3Neib4z9GXjOos1IbrXADNyJcCbKYWZcQzBDOgTvfkOiBGn9+lElWRSPkmoYh38zlAaOtOgmAEt6ZAnYMbuLoIBCyO1sYWxL8JnERaboH9ATirFaW/77+t2gchSMcRqr/Pxhhu7WVHlfbQxYcLZF+AFTZpJ2O9s+f/A1kksHjUDNy/AaLXFBWTIOHmPJVNU9Wfrg4DezzFboT8aNesRlRnlPezX62dOMnTOUe1SrCpR2k5hqhcQjR4YmUiXO4Um4FZUek90wSS2LDcdaPAW/HPSjkJsZ5SsJSo3lhnOgD2Hh1oDxZM5B4rOCmfVM/RGF5GIQJhhROArMvx5A9gQxslzOhlOVR/KZ48SbFMsnMN6HoTVWaAHAkaw4p3x11GQdgMy6+V4w/sFwaXyoGQYpynMIOB6xQkpjqYFcRXnWDM3t2idB8AByi/7zb80v10sNEinzO+WrK7uQZyYqFxo9UEXDLK1KMzF/eN28uldxmFdLKfJOJcdb//DoNUdUAbFKaZCzNVmqmQHHBYGU4R0U+17GT6jJH53eeW0N/3z789wy9NEQcM4MSwjWYFRv1ylrRKfhKrfsnOW4xDnV20TwPZIJH8QdhqtRu42/GqngAYYxn19+OgNppSmjf9iEaRBYE3NKkDlw7Zc0HVC/Dt/OIzrgq/gaMNPrbbZgIt9Mof+aBHIkgsLtAlEXlf8uFrMlpgzIBOh+Fwtvk2v8AMjRMZruaI8GXPBrU6KLZ8x56FS1KExOcgHpWS9f0zMusSPOupqYTtFNwTJE3Q3Q6ZS+i7Sf6hukUxpAUqq9zWCLJZ/V6xCZS6mzxt+N7Dr/9/rTx7qAVM0CSBeQZTmFr8R288sB8ztNVrgdEOWCa0w5yClwWUARhdNwvO4OVPt9orNBFGvMDN0lDls4JrkpMMerCeVj8psPnsZudtPd/EtwbR5jCdDuiLq0oCnJv5ii9MpJ76pe6/OkzYRdAFpWbEkZixoG8jWQinXi4oFKXWHmV23HF7gfv1k0NRo4HQtzYTJ9M7Rbtz0s2NOdV2uqtFhJXJySvq1EXKQUYmOk62FAU67aZ1xbgyk4rO9UCJvXQslvH6kLA7ehqRB1q7R+wBOuy4iAVPJUGjoJ7zYyqNKYITVBbEUfTJTQWEPXQPZ02ovD3Nk6rF8eUZONUJ5An5E0Omk13mgylUq3bbefw9MTFsFE5FUBXM8IGKEPcFR3LfekqjwHpZYVq0kgR+JA7KyB2O2SKWEYAsoZ7ZcKJAWFhERMiE1VngLnNjOmRGcpJFZPVpuDRvP8X6sN/CtkJy5QFHGn6aMJXRKXda/pzS2aa5xviKQPIG27qX/b7o7v908vhbRv/fLrP56PZoXNVrRPq+yunglo2eCdxZuqqBi2ueTN7X3of+LRPmPcCVgj6LYqdpIHRguqB27YGtgOqR0tbFI2hAbji5dkxttFNbgd4q47qH4XwDXw5ZhzoRF8zQEI+gFh5ZXX/sF1mhVUj7d4+frfz5CgzyBH25RvtO1HThjnxOu0p1idxlwads/ZdN81GZOuryRgbRf6Md8xnNhqPMEnsBso11CAxCzzq5VuHvbcd3kvcsxQPWa9AurmHuVuP6avl9jHGuQ2Llo/6gR47frvuGB9x/VAqH4Pdoju/aZ5OjCMCnCiWV/EDHxjcaKz8/RyLE0qs8QV0z03NZsoUdp2KTOUukILKTbAERGP3IaKbQlXqMC/C2n7uQvK06dk9gAOm9kfWksqvaxxYgVcKoB5o/5LxzAv0gHO1dvLOL7UFMhYDp9YptJQO4a7xrZL0u2XP4H1Hu1EOGmptIComGlODbX5a3UlnKUDLojrwvSbBnjBYLRsOBozyoz9/LGUh2OdxhvsIu/uDC40lWtZXLTOjlTXu8SukdvWmIG25prkeaL+GmZl+3loZBEjp6Ou4Sq2uLGxU9sSky0l9O1Y/VlCvOLlBjQ4WZqpMWG7FqqenbY79nEZ4A5xz2ZMi/iQlYxtGRNUVw5ovCzXE63PUYZPvMi5+dxuQowXCEgDE9Qu9VaC3GWz/1i6Q6ZTk4z4CVQ3OI/ncIcn0xNaaOjLDZ3FmdNjz77cBiffEWzduRzB3evYHYBfAddTYdcSc+WbfEaGX1KSuYWL/kdFCvwV3bobBH6bD7RVg2fAMY/2xlfoDSmlyEGpcNxjKMiMYCgEf7JtJKt7Bg/AQuOsAzQo5Q1MjVx5h5aWXZw9yvFSkD1vMdyfnfITpSAD51mM23iJ8TEn/MM4sB8Eq4ox8sUGsT3d5kuK+gCO79cwJC8VwCP2Btj6xxRf8Vw9ztIJXJNkam7wRHa2AOMrY/DHMrTjq4g7S+P+MGGtb4/BH866yLEQJjBZYIZ53vVaq/cCE/RTeGWyU+lrrUs1W+B8A5xkwxbJ2+EvOimF6w1f35Lyz7u72+/nVirIicUoUiCPI2maaaT9sp2dQB2w+cPii20NxB69SIDtAlCVgqu+VdlxWbqhvTBrrC+ERDnO1/DSyBK+apAcM4v/xfxlVwYuZQM5o8C1MlDPk/DEqj8XzKXUfGjOeDmmILIfZ8EKhh1/PAjTIqst7e7m9vv7t7ch4u9g9XbaYA4xYcnEo7uf7O7m1v5LuZuWrIabuy5MStp0yyOlJeDCnqbtxnw27NhlWwits5fjieEEI8OO+B5Xo0cTu45OnEyrOs/xxLrrzXsaPeXTz7p3739K2NKL5QBdvOzGzsS6SDN2epLUKC8zSWKY00r90lMg4ix5nWFeZkqxrJTi63aWM6HsvT2cu6sKD29XdS1morFCa68EpEEWlNuba2xpaebnfP4eORgncQ6aibvrpI3UPn+IbNTdu9UR0DArPYyo0ePnD+ch4vB4AT3mtm7rp0RRAh8B4o0/KBUdcKy0KTbtuag2bCmq1dq6npNY2wUAwxp4To9s3yYObHQ4rrF7GSxVWtJFFRfZjvQW5ZjlFbPStqbxuAYet9a4uxKMJMJ6gWEtrIcE52bcGIt+3r+Fo/bWlrbS1iZdeu2z44PSquO8u40Qs/+X30D5ucaBb0V6ewueVkS7IlsJja5vfm05OMGdrIKMrNAOC2opBdfGroxHkfrwjsNkcvl0d4cKwKqSRiJCIsD5OvI2aAH6EYAHBjEnp3xNnSJ8y5MmMJGucCebUui0VL/tqdRFrhNMtG9baK1Vs0sILa51pdZPKTj77QGkJS3dAVAvyKvGa4WcrZUjhWuAa9ElK+ZTRc4T8t5yzAdZrKdTWBhom4at/E5nsE8fhlQhhF4DsWy/oBwV6mXDfhyGnysrCKVxvrly0aqgvNLQqmQZ3oL0VUiJlV+trN22mw/xjkh648J+uGd/lyO9n9HnFt9D3+Mpq4WqFp0Gv60W82ox2V6M4rhUa6FtNGdiNWjP08Y7e3lGfRpFKbyye8y2N5q4o9w10Q6AXIdJtthmLtu8KMC9syBOJx7JMfS54Eu6yqqS4FEaZPJw/54b2J/MR/ka8xWoK6dxN5eiSwG2JbgvvICq2HFx86rIgjQGV+FD9B4DidQ+MaT+mhaMgNIBcoZX/e44vF6B9Yx+O/dlsFA3fIDfR6D7ACOxXhzq+YKuFjXtGc43gZER51RtGzjfcPHIgKzcVLpu/l1v27XmGgFG7R6+IX0S/SQ+tuIt1DUvL/BsM8Mz5InWP7z0+oiBnQS+1ZDlovcObUvo7ixbc9NHlOs1bVK2/YL6zhItwiPoSyU0RnGfyCnsl/XETZEdgxjgn2NWCGCSMdAa5JSzoKwWjKq1E7yhiRxNpEVJ8xYvHYEXgmRm6prBGOUwJfrHtVCAAiVbcDndW8AfBKHL7XW+eRseGGFeH2Iviy8YGovRfQ6Cw42n1Ag6Gtbz0A16UvgdAZtEIVpDnyB78VYQLhqMqT8300StEXBSCspNWKu07dLagm7FkU58VLymNR4fl4gMPrkIsd5YUMPCVMlQiq3ddGNUJgbnoeMyNCyH8txa65heV2kjPGSDA0wwwc/UajvA28j6KivGsijxnSSqRIx0jCe2AwcjAkvKaVjsSb2qIFrL+L5ZzGhx+f3pGOq/NdJZXOPHL3yqHOgYuSzEKbVpCPwV1Di+Cq1khulOrSfBpdYIaw1FmcaF7jmjG7AMqCu3YGresU2kEtGiZFAA166yIAJcz/gC63ztro2s8wo0F65Eqa8W4Wzb3HInOLRemLn7jSNi0ije9RfZTwYb6xD1FY3xu/Z+DVyWgCUqKqZpydxNj8ll65ag29F43Jo5nRcNyCB2UodJ1yzPBT8s+iiAKVypGfaovZ/GxfWUZWTYLW67PC0CCpNj+zboAZWja8kz1ctFovw8kLs1bvCvE/MTYpzAZAORdhCxn4ZGXPBXxpa3LalS0s+22+xMaRE7TF3KDP47FwT+3ssYzhXeJddm2vNrf7lj2ArNKc6ezOT3GXUbjiMwWbfOTMiUP/LSB20oOicN+IfWcYZHerugO3x1f9iSfrSYb1fP7RJ/uq3Bwv1WtlYd2uBujEIPN16P0Hnxdqfvwq/i137ZArCrGRbEy4OIv4XVML8T0cvc/5zrRUM4+rNVH0N4UcDJxGsj7XnxzWS8kXymWH9wUjlzacRj+rPHbAfz6YN1v4mx1wFxkZ7F3XiC8w2qCRt2CsoYrS8uuHO+oe6Epar5IEloMrfdsO1JouwpclxPqXijrSmxbaRNvO07aRXCPNWr4dtEcd2Tctbq3tOI2a6zBfyNvG3TJM2lCKHlqOXXvLkb4J4doN1qNJz72+Km6jTEJR14jP4tMI0bn2A/roHtamgkCmN05pcC9FoQSzvkaa4p2lSeKTvAlV7/McM5yxZYAcm8IrH9ksg4iOv6ydqkU5p1DNybjb+RZCUx10CQo42UYMC2iFR2evgnr2/eJ3Pkho0mOvVEf6/81+yub97HX+ZJ3I99GIkXoyohp0uaZwZZUekhUX1HqqHzpsAkFlCgeFBSI92stINm506l3RuJxrHVES5XSsJOftR5HMjhfkunnb7niXeuVnKDKVSCRIsq34Buga2/XcuwUq0rfey30UP3Pc/t5xsRwdsry4y9NCg8J6F0h0ex9o1d/nIe153/gFk4sSkqdxM0wcnjU627SG0SkoXo+e1Y3ngXqoakCTNW69FfV/cnVWWQwQNIDV8xzZhYqU6x9bN9A12/Q7dSFKVGn4Kc34uVOhZyF6DjoPsAUtnhUUu85qk36L9mr2d/a/1ybowO3UwJlW5g+ygkOaLVNUR8GuMKo6H7T+/dqk84thPygAQEAiUTW1OXCwIso6QvFnc9o1WiHxLZIV3w9Rck7lz7n8DDxGplKhuTyCagEKxPXBavNC7K5iDY9TtnbP6KskesLIl0n1jIRJNSOHUV2j2nX4zdE+CaLinIWvzX745xHIiWeMsETlHW8LVtIpQT+PoGLTFTRwBdS4m3UR6EPAVT3VG+UvGn0uwmcmkni7ITmfKySt84FHzWpID9dD2C+Pqdty9R6UNQ7QN9DPqjcdDenGtCdkq9gNlqdoVWUFBOX72e/cerUork9R5uQnkP0gfEZ/dqEkcyASxp5gJM3zkchSiT6cfscpOOwo092aavUCmB0FwnGV9GrSTdijjgeMGAvEFaVscuMQ0f6qwbKVzqGVkFbrZNEAGNKbNR7n/mv30007HAySm4jJdRpoLsTfoQZv9zd9AFaExwK1wNx3tNiF12w6weH2GlRE5tILcbUQm3ehqt0MBMAB8V7W8l8DtgYAL61vh11Bfw/wUAAP//qkt+EQ==" } diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml index be8202fb123a..2cf4c9d713c7 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml @@ -1,6 +1,8 @@ - name: vertexai_logs description: Google Cloud Vertex AI Prompt Response Logs metrics release: beta + version: + beta: 9.2.0 type: group fields: - name: endpoint From cf6b3615f83d2d319cdb5777ea008e24706cb6ca Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Fri, 5 Sep 2025 11:01:49 +0530 Subject: [PATCH 08/26] Resolve lint errors --- .../metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index 98ed8d845dda..88c57996ce92 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -121,7 +121,7 @@ func TestCreateEventWithInvalidJSON(t *testing.T) { event, err := CreateEvent(row, projectID, logger) assert.NoError(err) // Should not error, but log warning - + // Invalid JSON should be stored as raw string fullRequestField, err := event.MetricSetFields.GetValue("full_request.raw") assert.NoError(err) @@ -195,4 +195,3 @@ func TestEventsMapping(t *testing.T) { assert.Equal("model-123456", events[0].MetricSetFields["deployed_model_id"]) assert.Equal("model-789012", events[1].MetricSetFields["deployed_model_id"]) } - From 11c901081c9fb5be5a031f9f3af341caabf04b97 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Fri, 5 Sep 2025 12:15:46 +0530 Subject: [PATCH 09/26] Updated test file --- .../metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index 88c57996ce92..79dc072fe356 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -16,14 +16,14 @@ import ( func TestGenerateQuery(t *testing.T) { m := &MetricSet{ config: config{ - TableID: "project-123.dataset.table_name", + TableID: "project-1233.dataset.table_name", }, } query := m.generateQuery() // verify that table name quoting is in effect - assert.Contains(t, query, "`project-123.dataset.table_name`") + assert.Contains(t, query, "`project-1233.dataset.table_name`") // verify WHERE clause is present assert.Contains(t, query, "WHERE") assert.Contains(t, query, "logging_time IS NOT NULL") From c06a75db0597fd26a24786b709b4d36398a8e1e4 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Fri, 5 Sep 2025 19:13:15 +0530 Subject: [PATCH 10/26] Remove test file --- .../gcp/vertexai_logs/vertexai_logs_test.go | 197 ------------------ 1 file changed, 197 deletions(-) delete mode 100644 x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go deleted file mode 100644 index 79dc072fe356..000000000000 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package vertexai_logs - -import ( - "testing" - "time" - - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/stretchr/testify/assert" -) - -func TestGenerateQuery(t *testing.T) { - m := &MetricSet{ - config: config{ - TableID: "project-1233.dataset.table_name", - }, - } - - query := m.generateQuery() - - // verify that table name quoting is in effect - assert.Contains(t, query, "`project-1233.dataset.table_name`") - // verify WHERE clause is present - assert.Contains(t, query, "WHERE") - assert.Contains(t, query, "logging_time IS NOT NULL") - // verify ORDER BY is present - assert.Contains(t, query, "ORDER BY") - assert.Contains(t, query, "logging_time DESC") - // verify LIMIT is present - assert.Contains(t, query, "LIMIT 10000") - // verify CAST for request_id - assert.Contains(t, query, "CAST(IFNULL(request_id, 0) AS FLOAT64)") -} - -func TestCreateEvent(t *testing.T) { - assert := assert.New(t) - - testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) - row := VertexAILogRow{ - Endpoint: "https://us-central1-aiplatform.googleapis.com", - DeployedModelID: "model-123456", - LoggingTime: testTime, - RequestID: 12345.67, - RequestPayload: []string{"prompt1", "prompt2"}, - ResponsePayload: []string{"response1", "response2"}, - Model: "gemini-2.5-pro", - ModelVersion: "1.0", - APIMethod: "generateContent", - FullRequest: `{"inputs": ["test"]}`, - FullResponse: `{"outputs": ["result"]}`, - Metadata: `{"user_id": "user123"}`, - OtelLog: `{"trace_id": "abc123"}`, - } - - projectID := "test-project" - logger := logp.NewLogger("test") - - event, err := CreateEvent(row, projectID, logger) - - assert.NoError(err) - assert.Equal(testTime, event.Timestamp) - - // Check MetricSetFields - expectedFields := mapstr.M{ - "endpoint": "https://us-central1-aiplatform.googleapis.com", - "deployed_model_id": "model-123456", - "logging_time": testTime, - "request_id": 12345.67, - "request_payload": []string{"prompt1", "prompt2"}, - "response_payload": []string{"response1", "response2"}, - "model": "gemini-2.5-pro", - "model_version": "1.0", - "api_method": "generateContent", - "full_request": map[string]interface{}{"inputs": []interface{}{"test"}}, - "full_response": map[string]interface{}{"outputs": []interface{}{"result"}}, - "metadata": map[string]interface{}{"user_id": "user123"}, - "otel_log": map[string]interface{}{"trace_id": "abc123"}, - } - - assert.Equal(expectedFields, event.MetricSetFields) - - // Check RootFields - expectedRootFields := mapstr.M{ - "cloud.provider": "gcp", - "cloud.project.id": projectID, - } - assert.Equal(expectedRootFields, event.RootFields) - - // Check that ID is generated - assert.NotEmpty(event.ID) - assert.Len(event.ID, 20) // generateEventID returns 20 character hash -} - -func TestCreateEventWithInvalidJSON(t *testing.T) { - assert := assert.New(t) - - testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) - row := VertexAILogRow{ - Endpoint: "https://us-central1-aiplatform.googleapis.com", - DeployedModelID: "model-123456", - LoggingTime: testTime, - RequestID: 12345.67, - RequestPayload: []string{"prompt1"}, - ResponsePayload: []string{"response1"}, - Model: "gemini-2.5-pro", - ModelVersion: "1.0", - APIMethod: "generateContent", - FullRequest: `{"invalid": json}`, // Invalid JSON - FullResponse: `{}`, - Metadata: `{}`, - OtelLog: `{}`, - } - - projectID := "test-project" - logger := logp.NewLogger("test") - - event, err := CreateEvent(row, projectID, logger) - - assert.NoError(err) // Should not error, but log warning - - // Invalid JSON should be stored as raw string - fullRequestField, err := event.MetricSetFields.GetValue("full_request.raw") - assert.NoError(err) - assert.Equal(`{"invalid": json}`, fullRequestField) -} - -func TestGenerateEventID(t *testing.T) { - testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) - row := VertexAILogRow{ - LoggingTime: testTime, - RequestID: 12345.67, - RequestPayload: []string{"prompt1", "prompt2"}, - } - - id1 := generateEventID(row) - id2 := generateEventID(row) - - // Same input should produce same ID - assert.Equal(t, id1, id2) - assert.Len(t, id1, 20) - - // Different input should produce different ID - row.RequestID = 98765.43 - id3 := generateEventID(row) - assert.NotEqual(t, id1, id3) -} - -func TestEventsMapping(t *testing.T) { - assert := assert.New(t) - - testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) - rows := []VertexAILogRow{ - { - Endpoint: "https://us-central1-aiplatform.googleapis.com", - DeployedModelID: "model-123456", - LoggingTime: testTime, - RequestID: 12345.67, - RequestPayload: []string{"prompt1"}, - ResponsePayload: []string{"response1"}, - Model: "gemini-2.5-pro", - ModelVersion: "1.0", - APIMethod: "generateContent", - FullRequest: `{}`, - FullResponse: `{}`, - Metadata: `{}`, - OtelLog: `{}`, - }, - { - Endpoint: "https://us-west1-aiplatform.googleapis.com", - DeployedModelID: "model-789012", - LoggingTime: testTime.Add(time.Hour), - RequestID: 67890.12, - RequestPayload: []string{"prompt2"}, - ResponsePayload: []string{"response2"}, - Model: "gemini-1.5-pro", - ModelVersion: "2.0", - APIMethod: "predict", - FullRequest: `{}`, - FullResponse: `{}`, - Metadata: `{}`, - OtelLog: `{}`, - }, - } - - projectID := "test-project" - logger := logp.NewLogger("test") - - events := EventsMapping(rows, projectID, logger) - - assert.Len(events, 2) - assert.Equal("model-123456", events[0].MetricSetFields["deployed_model_id"]) - assert.Equal("model-789012", events[1].MetricSetFields["deployed_model_id"]) -} From 6f3d5d02b07ef519151e3d69b3ed363afe06d3a1 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Sat, 6 Sep 2025 18:21:06 +0530 Subject: [PATCH 11/26] Add test file --- .../gcp/vertexai_logs/vertexai_logs_test.go | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go new file mode 100644 index 000000000000..c9377ddd5def --- /dev/null +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -0,0 +1,170 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. +package vertexai_logs + +import ( + "testing" + "time" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/stretchr/testify/assert" +) + +func TestGenerateQuery(t *testing.T) { + m := &MetricSet{ + config: config{ + TableID: "project-1233.dataset.table_name", + }, + } + query := m.generateQuery() + // verify that table name quoting is in effect + assert.Contains(t, query, "`project-1233.dataset.table_name`") + // verify WHERE clause is present + assert.Contains(t, query, "WHERE") + assert.Contains(t, query, "logging_time IS NOT NULL") + // verify ORDER BY is present + assert.Contains(t, query, "ORDER BY") + assert.Contains(t, query, "logging_time DESC") + // verify LIMIT is present + assert.Contains(t, query, "LIMIT 10000") + // verify CAST for request_id + assert.Contains(t, query, "CAST(IFNULL(request_id, 0) AS FLOAT64)") +} +func TestCreateEvent(t *testing.T) { + assert := assert.New(t) + testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) + row := VertexAILogRow{ + Endpoint: "https://us-central1-aiplatform.googleapis.com", + DeployedModelID: "model-123456", + LoggingTime: testTime, + RequestID: 12345.67, + RequestPayload: []string{"prompt1", "prompt2"}, + ResponsePayload: []string{"response1", "response2"}, + Model: "gemini-2.5-pro", + ModelVersion: "1.0", + APIMethod: "generateContent", + FullRequest: `{"inputs": ["test"]}`, + FullResponse: `{"outputs": ["result"]}`, + Metadata: `{"user_id": "user123"}`, + OtelLog: `{"trace_id": "abc123"}`, + } + projectID := "test-project" + logger := logp.NewLogger("test") + event, err := CreateEvent(row, projectID, logger) + assert.NoError(err) + assert.Equal(testTime, event.Timestamp) + // Check MetricSetFields + expectedFields := mapstr.M{ + "endpoint": "https://us-central1-aiplatform.googleapis.com", + "deployed_model_id": "model-123456", + "logging_time": testTime, + "request_id": 12345.67, + "request_payload": []string{"prompt1", "prompt2"}, + "response_payload": []string{"response1", "response2"}, + "model": "gemini-2.5-pro", + "model_version": "1.0", + "api_method": "generateContent", + "full_request": map[string]interface{}{"inputs": []interface{}{"test"}}, + "full_response": map[string]interface{}{"outputs": []interface{}{"result"}}, + "metadata": map[string]interface{}{"user_id": "user123"}, + "otel_log": map[string]interface{}{"trace_id": "abc123"}, + } + assert.Equal(expectedFields, event.MetricSetFields) + // Check RootFields + expectedRootFields := mapstr.M{ + "cloud.provider": "gcp", + "cloud.project.id": projectID, + } + assert.Equal(expectedRootFields, event.RootFields) + // Check that ID is generated + assert.NotEmpty(event.ID) + assert.Len(event.ID, 20) // generateEventID returns 20 character hash +} +func TestCreateEventWithInvalidJSON(t *testing.T) { + assert := assert.New(t) + testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) + row := VertexAILogRow{ + Endpoint: "https://us-central1-aiplatform.googleapis.com", + DeployedModelID: "model-123456", + LoggingTime: testTime, + RequestID: 12345.67, + RequestPayload: []string{"prompt1"}, + ResponsePayload: []string{"response1"}, + Model: "gemini-2.5-pro", + ModelVersion: "1.0", + APIMethod: "generateContent", + FullRequest: `{"invalid": json}`, // Invalid JSON + FullResponse: `{}`, + Metadata: `{}`, + OtelLog: `{}`, + } + projectID := "test-project" + logger := logp.NewLogger("test") + event, err := CreateEvent(row, projectID, logger) + assert.NoError(err) // Should not error, but log warning + // Invalid JSON should be stored as raw string + fullRequestField, err := event.MetricSetFields.GetValue("full_request.raw") + assert.NoError(err) + assert.Equal(`{"invalid": json}`, fullRequestField) +} +func TestGenerateEventID(t *testing.T) { + testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) + row := VertexAILogRow{ + LoggingTime: testTime, + RequestID: 12345.67, + RequestPayload: []string{"prompt1", "prompt2"}, + } + id1 := generateEventID(row) + id2 := generateEventID(row) + // Same input should produce same ID + assert.Equal(t, id1, id2) + assert.Len(t, id1, 20) + // Different input should produce different ID + row.RequestID = 98765.43 + id3 := generateEventID(row) + assert.NotEqual(t, id1, id3) +} +func TestEventsMapping(t *testing.T) { + assert := assert.New(t) + testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) + rows := []VertexAILogRow{ + { + Endpoint: "https://us-central1-aiplatform.googleapis.com", + DeployedModelID: "model-123456", + LoggingTime: testTime, + RequestID: 12345.67, + RequestPayload: []string{"prompt1"}, + ResponsePayload: []string{"response1"}, + Model: "gemini-2.5-pro", + ModelVersion: "1.0", + APIMethod: "generateContent", + FullRequest: `{}`, + FullResponse: `{}`, + Metadata: `{}`, + OtelLog: `{}`, + }, + { + Endpoint: "https://us-west1-aiplatform.googleapis.com", + DeployedModelID: "model-789012", + LoggingTime: testTime.Add(time.Hour), + RequestID: 67890.12, + RequestPayload: []string{"prompt2"}, + ResponsePayload: []string{"response2"}, + Model: "gemini-1.5-pro", + ModelVersion: "2.0", + APIMethod: "predict", + FullRequest: `{}`, + FullResponse: `{}`, + Metadata: `{}`, + OtelLog: `{}`, + }, + } + projectID := "test-project" + logger := logp.NewLogger("test") + events := EventsMapping(rows, projectID, logger) + assert.Len(events, 2) + assert.Equal("model-123456", events[0].MetricSetFields["deployed_model_id"]) + assert.Equal("model-789012", events[1].MetricSetFields["deployed_model_id"]) +} From 09bc67619e8fa1abe0097fe0f899dc01e66c2baf Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Sat, 6 Sep 2025 18:37:29 +0530 Subject: [PATCH 12/26] update test file --- .../metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index c9377ddd5def..e1b56edd87cd 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -32,6 +32,7 @@ func TestGenerateQuery(t *testing.T) { // verify CAST for request_id assert.Contains(t, query, "CAST(IFNULL(request_id, 0) AS FLOAT64)") } + func TestCreateEvent(t *testing.T) { assert := assert.New(t) testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) @@ -82,6 +83,7 @@ func TestCreateEvent(t *testing.T) { assert.NotEmpty(event.ID) assert.Len(event.ID, 20) // generateEventID returns 20 character hash } + func TestCreateEventWithInvalidJSON(t *testing.T) { assert := assert.New(t) testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) @@ -109,6 +111,7 @@ func TestCreateEventWithInvalidJSON(t *testing.T) { assert.NoError(err) assert.Equal(`{"invalid": json}`, fullRequestField) } + func TestGenerateEventID(t *testing.T) { testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) row := VertexAILogRow{ @@ -126,6 +129,7 @@ func TestGenerateEventID(t *testing.T) { id3 := generateEventID(row) assert.NotEqual(t, id1, id3) } + func TestEventsMapping(t *testing.T) { assert := assert.New(t) testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) From d050a1fab7766e3401940ae746cb7b16e9e169b8 Mon Sep 17 00:00:00 2001 From: Ishleen Kaur <102962586+ishleenk17@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:36:44 +0530 Subject: [PATCH 13/26] Update x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md index f0d66c9bf1f4..50614879d094 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md @@ -12,7 +12,7 @@ The logs include detailed information about: ## Metricset-specific configuration notes [_metricset_specific_configuration_notes_14] -* **table_id**: (Required) Full table identifier in the format `project_id.dataset_id.table_name` that contains the Vertex AI logs data. You can copy this from "Details" tab under +* **table_id**: (Required) Full table identifier in the format `project_id.dataset_id.table_name` that contains the Vertex AI logs data. You can copy this from the "Details" tab when viewing your table in the BigQuery web console, under the "Table ID" field. ## Configuration example [_configuration_example_22] From f005382e5fe1860994fcaff0c285aaf657dcc306 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Wed, 10 Sep 2025 11:59:30 +0530 Subject: [PATCH 14/26] Update the partiotin logic in BigQuery --- .../metricbeat-metricset-gcp-vertexai_logs.md | 2 +- .../module/gcp/vertexai_logs/vertexai_logs.go | 7 ++++--- .../module/gcp/vertexai_logs/vertexai_logs_test.go | 12 ++++++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md index d1cd86f9634f..0183c7ddd52e 100644 --- a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md +++ b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md @@ -24,7 +24,7 @@ The logs include detailed information about: ## Metricset-specific configuration notes [_metricset_specific_configuration_notes_14] -* **table_id**: (Required) Full table identifier in the format `project_id.dataset_id.table_name` that contains the Vertex AI logs data. You can copy this from "Details" tab under +* **table_id**: (Required) Full table identifier in the format `project_id.dataset_id.table_name` that contains the Vertex AI logs data. You can copy this from the "Details" tab when viewing your table in the BigQuery web console, under the "Table ID" field. ## Configuration example [_configuration_example_22] diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go index 9f18276575de..6116cf249724 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -196,10 +196,11 @@ SELECT FROM %s WHERE - logging_time IS NOT NULL + _PARTITIONDATE >= DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) + AND logging_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) + AND logging_time IS NOT NULL ORDER BY - logging_time DESC -LIMIT 10000;`, + logging_time ASC`, escapedTableID) return query diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index e1b56edd87cd..d9480d36fd8d 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -24,11 +24,15 @@ func TestGenerateQuery(t *testing.T) { // verify WHERE clause is present assert.Contains(t, query, "WHERE") assert.Contains(t, query, "logging_time IS NOT NULL") - // verify ORDER BY is present + // verify partition filter is present + assert.Contains(t, query, "_PARTITIONDATE >= DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY)") + // verify timestamp filter is present + assert.Contains(t, query, "logging_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR)") + // verify ORDER BY is present (should be ASC now) assert.Contains(t, query, "ORDER BY") - assert.Contains(t, query, "logging_time DESC") - // verify LIMIT is present - assert.Contains(t, query, "LIMIT 10000") + assert.Contains(t, query, "logging_time ASC") + // verify LIMIT is NOT present (removed for no data loss) + assert.NotContains(t, query, "LIMIT") // verify CAST for request_id assert.Contains(t, query, "CAST(IFNULL(request_id, 0) AS FLOAT64)") } From d72b5833e6ea23eb13754fe30cb98751d24fe29c Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Thu, 11 Sep 2025 11:37:35 +0530 Subject: [PATCH 15/26] Add support of logging time filtering --- .../metricbeat/metricbeat-module-gcp.md | 1 + x-pack/metricbeat/metricbeat.reference.yml | 1 + x-pack/metricbeat/module/gcp/_meta/config.yml | 1 + .../module/gcp/vertexai_logs/vertexai_logs.go | 68 +++++++++++++++++-- .../gcp/vertexai_logs/vertexai_logs_test.go | 66 +++++++++++++++--- x-pack/metricbeat/modules.d/gcp.yml.disabled | 1 + 6 files changed, 123 insertions(+), 15 deletions(-) diff --git a/docs/reference/metricbeat/metricbeat-module-gcp.md b/docs/reference/metricbeat/metricbeat-module-gcp.md index 245dfbbd2402..ff01115fe6fa 100644 --- a/docs/reference/metricbeat/metricbeat-module-gcp.md +++ b/docs/reference/metricbeat/metricbeat-module-gcp.md @@ -370,6 +370,7 @@ metricbeat.modules: table_id: "your-project-id.dataset.id.table_name" credentials_file_path: "/path/to/service-account.json" # credentials_json: '{"type": "service_account", ...}' + time_lookback_hours: 1 # How many hours back to look for initial data fetch ``` diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index fce10c39d1c4..bdb2c496236c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -705,6 +705,7 @@ metricbeat.modules: table_id: "your-project-id.dataset.id.table_name" credentials_file_path: "/path/to/service-account.json" # credentials_json: '{"type": "service_account", ...}' + time_lookback_hours: 1 # How many hours back to look for initial data fetch diff --git a/x-pack/metricbeat/module/gcp/_meta/config.yml b/x-pack/metricbeat/module/gcp/_meta/config.yml index 2285ab6385f2..98f9efc75f20 100644 --- a/x-pack/metricbeat/module/gcp/_meta/config.yml +++ b/x-pack/metricbeat/module/gcp/_meta/config.yml @@ -91,5 +91,6 @@ table_id: "your-project-id.dataset.id.table_name" credentials_file_path: "/path/to/service-account.json" # credentials_json: '{"type": "service_account", ...}' + time_lookback_hours: 1 # How many hours back to look for initial data fetch diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go index 6116cf249724..6e982d8f33f6 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -30,8 +30,9 @@ func init() { type MetricSet struct { mb.BaseMetricSet - config config - logger *logp.Logger + config config + logger *logp.Logger + lastLoggingTime *time.Time } type config struct { @@ -40,6 +41,7 @@ type config struct { TableID string `config:"table_id" validate:"required"` CredentialsFilePath string `config:"credentials_file_path"` CredentialsJSON string `config:"credentials_json"` + TimeLookbackHours int `config:"time_lookback_hours"` } func (c config) Validate() error { @@ -60,6 +62,10 @@ func (c config) Validate() error { return fmt.Errorf("table_id must be in format 'project_id.dataset_id.table_name', got: %s", c.TableID) } + if c.TimeLookbackHours < 0 { + return errors.New("time_lookback_hours must be non-negative") + } + return nil } @@ -73,8 +79,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, fmt.Errorf("unpack vertexai_logs config failed: %w", err) } - m.logger.Debugf("metricset config: project_id=%s, dataset_id=%s, table_name=%s", - m.config.ProjectID, getDatasetID(m.config.TableID), getTableName(m.config.TableID)) + // Set defaults + if m.config.TimeLookbackHours == 0 { + m.config.TimeLookbackHours = 1 // Default: 1 hour + } + + m.logger.Debugf("metricset config: project_id=%s, dataset_id=%s, table_name=%s, time_lookback=%dh", + m.config.ProjectID, getDatasetID(m.config.TableID), getTableName(m.config.TableID), + m.config.TimeLookbackHours) return m, nil } @@ -129,6 +141,9 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { reporter.Event(event) } + // Update watermark with latest logging_time from events + m.updateLastLoggingTime(events) + return nil } @@ -178,6 +193,19 @@ func (m *MetricSet) queryVertexAILogs(ctx context.Context, client *bigquery.Clie func (m *MetricSet) generateQuery() string { escapedTableID := fmt.Sprintf("`%s`", m.config.TableID) + var whereClause string + if m.lastLoggingTime != nil { + // Incremental query: get records after last processed time + whereClause = fmt.Sprintf("logging_time > TIMESTAMP('%s')", + m.lastLoggingTime.Format("2006-01-02 15:04:05.000000")) + m.logger.Debugf("Using incremental query from logging_time: %s", m.lastLoggingTime.Format(time.RFC3339)) + } else { + // First run: use timestamp filter with lookback + whereClause = fmt.Sprintf("logging_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL %d HOUR)", + m.config.TimeLookbackHours) + m.logger.Debugf("Using initial query with timestamp filter: %d hours", m.config.TimeLookbackHours) + } + query := fmt.Sprintf(` SELECT IFNULL(endpoint, '') AS endpoint, @@ -196,12 +224,38 @@ SELECT FROM %s WHERE - _PARTITIONDATE >= DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) - AND logging_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) + %s AND logging_time IS NOT NULL ORDER BY logging_time ASC`, - escapedTableID) + escapedTableID, whereClause) return query } + +// updateLastLoggingTime updates the watermark with the latest logging_time from events + +func (m *MetricSet) updateLastLoggingTime(events []mb.Event) { + if len(events) == 0 { + return + } + + var latestTime *time.Time + for _, event := range events { + // Get logging_time from the event's MetricSetFields + if loggingTimeField, exists := event.MetricSetFields["logging_time"]; exists { + if loggingTime, ok := loggingTimeField.(time.Time); ok { + if !loggingTime.IsZero() && (latestTime == nil || loggingTime.After(*latestTime)) { + latestTime = &loggingTime + } + } + } + } + + if latestTime != nil { + // Store in UTC for consistency + utcTime := latestTime.UTC() + m.lastLoggingTime = &utcTime + m.logger.Debugf("Updated last logging time to: %s", latestTime.Format(time.RFC3339)) + } +} diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index d9480d36fd8d..a96a5c74d993 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -7,34 +7,42 @@ import ( "testing" "time" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/stretchr/testify/assert" ) func TestGenerateQuery(t *testing.T) { + // Test initial query (no watermark) m := &MetricSet{ config: config{ - TableID: "project-1233.dataset.table_name", + TableID: "project-1233.dataset.table_name", + TimeLookbackHours: 2, }, + logger: logp.NewLogger("test"), } + query := m.generateQuery() // verify that table name quoting is in effect assert.Contains(t, query, "`project-1233.dataset.table_name`") // verify WHERE clause is present assert.Contains(t, query, "WHERE") assert.Contains(t, query, "logging_time IS NOT NULL") - // verify partition filter is present - assert.Contains(t, query, "_PARTITIONDATE >= DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY)") - // verify timestamp filter is present - assert.Contains(t, query, "logging_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR)") - // verify ORDER BY is present (should be ASC now) + // verify timestamp filter is present for initial query + assert.Contains(t, query, "logging_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 2 HOUR)") + // verify ORDER BY is present (should be ASC for incremental) assert.Contains(t, query, "ORDER BY") assert.Contains(t, query, "logging_time ASC") - // verify LIMIT is NOT present (removed for no data loss) - assert.NotContains(t, query, "LIMIT") // verify CAST for request_id assert.Contains(t, query, "CAST(IFNULL(request_id, 0) AS FLOAT64)") + // Test incremental query (with watermark) + lastTime := time.Date(2023, 12, 1, 10, 0, 0, 0, time.UTC) + m.lastLoggingTime = &lastTime + + queryIncremental := m.generateQuery() + // verify incremental query uses logging_time filter + assert.Contains(t, queryIncremental, "logging_time > TIMESTAMP('2023-12-01 10:00:00.000000')") } func TestCreateEvent(t *testing.T) { @@ -176,3 +184,45 @@ func TestEventsMapping(t *testing.T) { assert.Equal("model-123456", events[0].MetricSetFields["deployed_model_id"]) assert.Equal("model-789012", events[1].MetricSetFields["deployed_model_id"]) } + +func TestUpdateLastLoggingTime(t *testing.T) { + logger := logp.NewLogger("test") + m := &MetricSet{ + logger: logger, + } + + testTime1 := time.Date(2023, 12, 1, 10, 0, 0, 0, time.UTC) + testTime2 := time.Date(2023, 12, 1, 11, 0, 0, 0, time.UTC) + testTime3 := time.Date(2023, 12, 1, 9, 0, 0, 0, time.UTC) + + // Test with empty events + m.updateLastLoggingTime([]mb.Event{}) + assert.Nil(t, m.lastLoggingTime) + + // Test with single event + events1 := []mb.Event{{ + Timestamp: testTime1, + MetricSetFields: mapstr.M{"logging_time": testTime1}, + }} + m.updateLastLoggingTime(events1) + assert.NotNil(t, m.lastLoggingTime) + assert.Equal(t, testTime1, *m.lastLoggingTime) + + // Test with multiple events - should pick the latest + events2 := []mb.Event{ + {Timestamp: testTime1, MetricSetFields: mapstr.M{"logging_time": testTime1}}, + {Timestamp: testTime2, MetricSetFields: mapstr.M{"logging_time": testTime2}}, + {Timestamp: testTime3, MetricSetFields: mapstr.M{"logging_time": testTime3}}, + } + m.updateLastLoggingTime(events2) + assert.Equal(t, testTime2, *m.lastLoggingTime) + + // Test with zero timestamps (should be skipped) + events3 := []mb.Event{ + {Timestamp: time.Time{}, MetricSetFields: mapstr.M{"logging_time": time.Time{}}}, + {Timestamp: testTime1, MetricSetFields: mapstr.M{"logging_time": testTime1}}, + } + m.lastLoggingTime = nil + m.updateLastLoggingTime(events3) + assert.Equal(t, testTime1, *m.lastLoggingTime) +} diff --git a/x-pack/metricbeat/modules.d/gcp.yml.disabled b/x-pack/metricbeat/modules.d/gcp.yml.disabled index c218344ab0dc..4f032bd19284 100644 --- a/x-pack/metricbeat/modules.d/gcp.yml.disabled +++ b/x-pack/metricbeat/modules.d/gcp.yml.disabled @@ -94,5 +94,6 @@ table_id: "your-project-id.dataset.id.table_name" credentials_file_path: "/path/to/service-account.json" # credentials_json: '{"type": "service_account", ...}' + time_lookback_hours: 1 # How many hours back to look for initial data fetch From 475cf8957995afbd858a0f91e69b81d80a1fb1af Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 11 Sep 2025 14:01:17 +0300 Subject: [PATCH 16/26] reshuffle imports in vertexai_logs_test.go --- .../metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index a96a5c74d993..3571875642d4 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -7,10 +7,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/stretchr/testify/assert" ) func TestGenerateQuery(t *testing.T) { From 44a151a3768e0a6cf3d106b8bbd27870bc3ac582 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Thu, 11 Sep 2025 19:50:59 +0530 Subject: [PATCH 17/26] Add string datatype for request id --- .../metricbeat/module/gcp/vertexai_logs/data.go | 4 ++-- .../module/gcp/vertexai_logs/vertexai_logs.go | 2 +- .../gcp/vertexai_logs/vertexai_logs_test.go | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/data.go b/x-pack/metricbeat/module/gcp/vertexai_logs/data.go index f027bffdb306..854ec9d55777 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/data.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/data.go @@ -21,7 +21,7 @@ type VertexAILogRow struct { Endpoint string `bigquery:"endpoint"` DeployedModelID string `bigquery:"deployed_model_id"` LoggingTime time.Time `bigquery:"logging_time"` - RequestID float64 `bigquery:"request_id"` + RequestID string `bigquery:"request_id"` RequestPayload []string `bigquery:"request_payload"` ResponsePayload []string `bigquery:"response_payload"` Model string `bigquery:"model"` @@ -102,7 +102,7 @@ func processJSONField(jsonStr, fieldName string, fields mapstr.M, logger *logp.L // generateEventID creates a unique event ID based on row data func generateEventID(row VertexAILogRow) string { - eventData := fmt.Sprintf("%d_%.0f_%d", + eventData := fmt.Sprintf("%d_%s_%d", row.LoggingTime.Unix(), row.RequestID, len(row.RequestPayload)) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go index 6e982d8f33f6..3e9cb4b4d060 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -211,7 +211,7 @@ SELECT IFNULL(endpoint, '') AS endpoint, IFNULL(deployed_model_id, '') AS deployed_model_id, logging_time, - CAST(IFNULL(request_id, 0) AS FLOAT64) AS request_id, + IFNULL(CAST(request_id AS STRING), '') AS request_id, IFNULL(request_payload, []) AS request_payload, IFNULL(response_payload, []) AS response_payload, IFNULL(model, '') AS model, diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index 3571875642d4..b783c33f5e77 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -36,7 +36,7 @@ func TestGenerateQuery(t *testing.T) { assert.Contains(t, query, "ORDER BY") assert.Contains(t, query, "logging_time ASC") // verify CAST for request_id - assert.Contains(t, query, "CAST(IFNULL(request_id, 0) AS FLOAT64)") + assert.Contains(t, query, "IFNULL(CAST(request_id AS STRING), '')") // Test incremental query (with watermark) lastTime := time.Date(2023, 12, 1, 10, 0, 0, 0, time.UTC) m.lastLoggingTime = &lastTime @@ -53,7 +53,7 @@ func TestCreateEvent(t *testing.T) { Endpoint: "https://us-central1-aiplatform.googleapis.com", DeployedModelID: "model-123456", LoggingTime: testTime, - RequestID: 12345.67, + RequestID: "12345.67", RequestPayload: []string{"prompt1", "prompt2"}, ResponsePayload: []string{"response1", "response2"}, Model: "gemini-2.5-pro", @@ -74,7 +74,7 @@ func TestCreateEvent(t *testing.T) { "endpoint": "https://us-central1-aiplatform.googleapis.com", "deployed_model_id": "model-123456", "logging_time": testTime, - "request_id": 12345.67, + "request_id": "12345.67", "request_payload": []string{"prompt1", "prompt2"}, "response_payload": []string{"response1", "response2"}, "model": "gemini-2.5-pro", @@ -104,7 +104,7 @@ func TestCreateEventWithInvalidJSON(t *testing.T) { Endpoint: "https://us-central1-aiplatform.googleapis.com", DeployedModelID: "model-123456", LoggingTime: testTime, - RequestID: 12345.67, + RequestID: "12345.67", RequestPayload: []string{"prompt1"}, ResponsePayload: []string{"response1"}, Model: "gemini-2.5-pro", @@ -129,7 +129,7 @@ func TestGenerateEventID(t *testing.T) { testTime := time.Date(2023, 12, 1, 10, 30, 45, 0, time.UTC) row := VertexAILogRow{ LoggingTime: testTime, - RequestID: 12345.67, + RequestID: "12345.67", RequestPayload: []string{"prompt1", "prompt2"}, } id1 := generateEventID(row) @@ -138,7 +138,7 @@ func TestGenerateEventID(t *testing.T) { assert.Equal(t, id1, id2) assert.Len(t, id1, 20) // Different input should produce different ID - row.RequestID = 98765.43 + row.RequestID = "98765.43" id3 := generateEventID(row) assert.NotEqual(t, id1, id3) } @@ -151,7 +151,7 @@ func TestEventsMapping(t *testing.T) { Endpoint: "https://us-central1-aiplatform.googleapis.com", DeployedModelID: "model-123456", LoggingTime: testTime, - RequestID: 12345.67, + RequestID: "12345.67", RequestPayload: []string{"prompt1"}, ResponsePayload: []string{"response1"}, Model: "gemini-2.5-pro", @@ -166,7 +166,7 @@ func TestEventsMapping(t *testing.T) { Endpoint: "https://us-west1-aiplatform.googleapis.com", DeployedModelID: "model-789012", LoggingTime: testTime.Add(time.Hour), - RequestID: 67890.12, + RequestID: "67890.12", RequestPayload: []string{"prompt2"}, ResponsePayload: []string{"response2"}, Model: "gemini-1.5-pro", From f9b152bc1d8658ee191fba1f225a73b1f98098f5 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Tue, 16 Sep 2025 11:48:58 +0530 Subject: [PATCH 18/26] Address comments --- .../module/gcp/vertexai_logs/vertexai_logs.go | 26 +++++++------------ .../gcp/vertexai_logs/vertexai_logs_test.go | 6 ++--- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go index 3e9cb4b4d060..86942b90d7f0 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -196,7 +196,7 @@ func (m *MetricSet) generateQuery() string { var whereClause string if m.lastLoggingTime != nil { // Incremental query: get records after last processed time - whereClause = fmt.Sprintf("logging_time > TIMESTAMP('%s')", + whereClause = fmt.Sprintf("logging_time >= TIMESTAMP('%s')", m.lastLoggingTime.Format("2006-01-02 15:04:05.000000")) m.logger.Debugf("Using incremental query from logging_time: %s", m.lastLoggingTime.Format(time.RFC3339)) } else { @@ -240,22 +240,14 @@ func (m *MetricSet) updateLastLoggingTime(events []mb.Event) { return } - var latestTime *time.Time - for _, event := range events { - // Get logging_time from the event's MetricSetFields - if loggingTimeField, exists := event.MetricSetFields["logging_time"]; exists { - if loggingTime, ok := loggingTimeField.(time.Time); ok { - if !loggingTime.IsZero() && (latestTime == nil || loggingTime.After(*latestTime)) { - latestTime = &loggingTime - } - } + // Since query is sorted by logging_time ASC, the last event has the latest time + lastEvent := events[len(events)-1] + if loggingTimeField, exists := lastEvent.MetricSetFields["logging_time"]; exists { + if loggingTime, ok := loggingTimeField.(time.Time); ok && !loggingTime.IsZero() { + // Store in UTC for consistency + utcTime := loggingTime.UTC() + m.lastLoggingTime = &utcTime + m.logger.Debugf("Updated last logging time to: %s", loggingTime.Format(time.RFC3339)) } } - - if latestTime != nil { - // Store in UTC for consistency - utcTime := latestTime.UTC() - m.lastLoggingTime = &utcTime - m.logger.Debugf("Updated last logging time to: %s", latestTime.Format(time.RFC3339)) - } } diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index b783c33f5e77..62e844315e21 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -43,7 +43,7 @@ func TestGenerateQuery(t *testing.T) { queryIncremental := m.generateQuery() // verify incremental query uses logging_time filter - assert.Contains(t, queryIncremental, "logging_time > TIMESTAMP('2023-12-01 10:00:00.000000')") + assert.Contains(t, queryIncremental, "logging_time >= TIMESTAMP('2023-12-01 10:00:00.000000')") } func TestCreateEvent(t *testing.T) { @@ -209,11 +209,11 @@ func TestUpdateLastLoggingTime(t *testing.T) { assert.NotNil(t, m.lastLoggingTime) assert.Equal(t, testTime1, *m.lastLoggingTime) - // Test with multiple events - should pick the latest + // Test with multiple events - should pick the last one (assumes sorted by logging_time ASC) events2 := []mb.Event{ + {Timestamp: testTime3, MetricSetFields: mapstr.M{"logging_time": testTime3}}, {Timestamp: testTime1, MetricSetFields: mapstr.M{"logging_time": testTime1}}, {Timestamp: testTime2, MetricSetFields: mapstr.M{"logging_time": testTime2}}, - {Timestamp: testTime3, MetricSetFields: mapstr.M{"logging_time": testTime3}}, } m.updateLastLoggingTime(events2) assert.Equal(t, testTime2, *m.lastLoggingTime) From 336920e8f18b35ef5a54d7a92c21fa22335b3e05 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Tue, 16 Sep 2025 16:37:30 +0530 Subject: [PATCH 19/26] Remove the otel_log field --- docs/reference/metricbeat/exported-fields-gcp.md | 6 ------ .../metricbeat/metricbeat-metricset-gcp-vertexai_logs.md | 9 --------- x-pack/metricbeat/module/gcp/fields.go | 2 +- .../metricbeat/module/gcp/vertexai_logs/_meta/data.json | 4 ---- x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md | 5 ----- .../metricbeat/module/gcp/vertexai_logs/_meta/fields.yml | 6 +----- x-pack/metricbeat/module/gcp/vertexai_logs/data.go | 5 ----- .../metricbeat/module/gcp/vertexai_logs/vertexai_logs.go | 3 +-- .../module/gcp/vertexai_logs/vertexai_logs_test.go | 9 ++------- 9 files changed, 5 insertions(+), 44 deletions(-) diff --git a/docs/reference/metricbeat/exported-fields-gcp.md b/docs/reference/metricbeat/exported-fields-gcp.md index bc2fff92cbb3..a77a3b53b9f7 100644 --- a/docs/reference/metricbeat/exported-fields-gcp.md +++ b/docs/reference/metricbeat/exported-fields-gcp.md @@ -1285,9 +1285,3 @@ Field is not indexed. type: object -**`gcp.vertexai_logs.otel_log`** -: OpenTelemetry log data associated with the AI interaction in JSON format. - - type: object - - diff --git a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md index 0183c7ddd52e..157605739d95 100644 --- a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md +++ b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md @@ -18,7 +18,6 @@ The logs include detailed information about: - API endpoints and deployed models - Request and response payloads - Model versions and API methods used -- OpenTelemetry trace data - Request metadata and timing information @@ -79,10 +78,6 @@ Here is a sample event for `vertexai_logs`: "metadata": { "user_id": "user123", "session_id": "session456" - }, - "otel_log": { - "trace_id": "abc123def456", - "span_id": "789ghi012jkl" } } } @@ -176,10 +171,6 @@ Here is an example document generated by this metricset: "metadata": { "region": "us-central1", "zone": "us-central1-a" - }, - "otel_log": { - "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", - "span_id": "00f067aa0ba902b7" } } }, diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index ece39e58da4f..90674b46e89a 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzsXd2P27aWf89fQfQlyWLi3qa7C2ywuMB0ctubbZIO4pkA+6RLi8c2a4pUSGom7l+/4JdE2bQt68PTFNs+ZSzx/M4Hzwd5SL1CG9i+Qau8fIaQpprBG/T8FyFWDNANExVBtwzrpZDF82cISWCAFbxBK/wMIQIql7TUVPA36O/PEELol5tbVAhSMXiG0JICI+qN/eEV4riAQMj8p7el+bcUVfhL/Hz8DsMLYKr+c3hVLH6HXEd/TuAJ/zlcnGohKV+hArSkudofeRdCDKNSIGf/1vrpIBTzn/tj5p7YwPZRSJIcuACNCdZ4qsENq5OMrbZKQzHJ0BKUqGQOow0eBv6uFoj7/7vTdtUal4hqYa078WtW4LKkfOUf/a41+BHr/ODNUa+xRhJ0JTkQtJSiQK2peH37Dn2pQG5ne2wtKGOUrw7Raw3zk3s2mEb0Tnt+t8USz1SUnCoBSy6Uk8ezfcWldN5CeiOUts8qRHnOKgJIwqpiWF4hjb9eIUx+r5QugOsrhDlBUlScGKGDlELOEngofxA0h6wQXK/7YAoCk1AKqZEdJ0WolMLaAiV9qNy6t9G7t0gskV5DUGqguwAm+EohLVLEtdCYJegumcD6MNU781pNCRei4jo1vNpUPfm6W0PEU5jYxp0StNjaPyqQDzSHQ3Sj4foAuI7/uYfDDJIEg34WEsFXXJQMrhDeeWMppJ9Ocy0kXgGiCs015gTL5m/38yRPjsIo8vRjOd9h/lApDwYrJXKKNRD0SNMGG4AMFLBB1PIwLVDOeoFYdG0PZKNeypbxSiWAcFAajio6F4xBHvS8ge2rB8wqQCWm0vvXUooHSgBhQqh5ELMmALeGTuUCDcQNbHd+OSau5j2Lp/Ob4S1YLg1bD5CVkuYpx3pqmq8B5WssV0CQHcIasLMVb0ktDc5/vVfWvc5/vUeaglQz9AmWRrgK5YJriXNtRzJ6pEuEy5LRHC/MXBF6DfKRKrhCVD9XdnRGlXse9oNXjuWiZXmHY9eNfRT9LIQuJeX6WBBbgB4Sxkbz5oZ9xyJa1rg7+fQAwfxrCAjz/gAYw/zV3DuDd2+tzSVhWEOc0EMFDHEgOB+MhFVP+p/smwmSJyjWoGYqFyX8cP7En5v30A97XHag9nrGRI4PiLwT2dcp4RqHEQZ+tcAm8hag14IIJlbbTrgKLDegR0flhu2F6ce+YH48UzNiuVSgU9GxU6LniflREq5YFGWloZsvds9OVkcsqYRHzNiMSFGWQGaLrYYU58Z3HWb8Hc9FYdRrX0d+sJDtBSId6GclzjegVZbbJHk/kp+Fxg92Bh7KlcY8h1leVjMJxjUCyXIhQR0Es1eu7sD5WBULkCYFsOOgMCwS3MJZm3LMJwiB/ilo1ptlmhYwU5D3AHVv0xLjLDFjHhjlSEEuOFGdyM/KPOUfTlE2adLSpDZRpYCZ9VZA0M3tvcshqUJ5JSVwzbYGWaUgCKyLkAhVm5kE3Neib4z9GXjOos1IbrXADNyJcCbKYWZcQzBDOgTvfkOiBGn9+lElWRSPkmoYh38zlAaOtOgmAEt6ZAnYMbuLoIBCyO1sYWxL8JnERaboH9ATirFaW/77+t2gchSMcRqr/Pxhhu7WVHlfbQxYcLZF+AFTZpJ2O9s+f/A1kksHjUDNy/AaLXFBWTIOHmPJVNU9Wfrg4DezzFboT8aNesRlRnlPezX62dOMnTOUe1SrCpR2k5hqhcQjR4YmUiXO4Um4FZUek90wSS2LDcdaPAW/HPSjkJsZ5SsJSo3lhnOgD2Hh1oDxZM5B4rOCmfVM/RGF5GIQJhhROArMvx5A9gQxslzOhlOVR/KZ48SbFMsnMN6HoTVWaAHAkaw4p3x11GQdgMy6+V4w/sFwaXyoGQYpynMIOB6xQkpjqYFcRXnWDM3t2idB8AByi/7zb80v10sNEinzO+WrK7uQZyYqFxo9UEXDLK1KMzF/eN28uldxmFdLKfJOJcdb//DoNUdUAbFKaZCzNVmqmQHHBYGU4R0U+17GT6jJH53eeW0N/3z789wy9NEQcM4MSwjWYFRv1ylrRKfhKrfsnOW4xDnV20TwPZIJH8QdhqtRu42/GqngAYYxn19+OgNppSmjf9iEaRBYE3NKkDlw7Zc0HVC/Dt/OIzrgq/gaMNPrbbZgIt9Mof+aBHIkgsLtAlEXlf8uFrMlpgzIBOh+Fwtvk2v8AMjRMZruaI8GXPBrU6KLZ8x56FS1KExOcgHpWS9f0zMusSPOupqYTtFNwTJE3Q3Q6ZS+i7Sf6hukUxpAUqq9zWCLJZ/V6xCZS6mzxt+N7Dr/9/rTx7qAVM0CSBeQZTmFr8R288sB8ztNVrgdEOWCa0w5yClwWUARhdNwvO4OVPt9orNBFGvMDN0lDls4JrkpMMerCeVj8psPnsZudtPd/EtwbR5jCdDuiLq0oCnJv5ii9MpJ76pe6/OkzYRdAFpWbEkZixoG8jWQinXi4oFKXWHmV23HF7gfv1k0NRo4HQtzYTJ9M7Rbtz0s2NOdV2uqtFhJXJySvq1EXKQUYmOk62FAU67aZ1xbgyk4rO9UCJvXQslvH6kLA7ehqRB1q7R+wBOuy4iAVPJUGjoJ7zYyqNKYITVBbEUfTJTQWEPXQPZ02ovD3Nk6rF8eUZONUJ5An5E0Omk13mgylUq3bbefw9MTFsFE5FUBXM8IGKEPcFR3LfekqjwHpZYVq0kgR+JA7KyB2O2SKWEYAsoZ7ZcKJAWFhERMiE1VngLnNjOmRGcpJFZPVpuDRvP8X6sN/CtkJy5QFHGn6aMJXRKXda/pzS2aa5xviKQPIG27qX/b7o7v908vhbRv/fLrP56PZoXNVrRPq+yunglo2eCdxZuqqBi2ueTN7X3of+LRPmPcCVgj6LYqdpIHRguqB27YGtgOqR0tbFI2hAbji5dkxttFNbgd4q47qH4XwDXw5ZhzoRF8zQEI+gFh5ZXX/sF1mhVUj7d4+frfz5CgzyBH25RvtO1HThjnxOu0p1idxlwads/ZdN81GZOuryRgbRf6Md8xnNhqPMEnsBso11CAxCzzq5VuHvbcd3kvcsxQPWa9AurmHuVuP6avl9jHGuQ2Llo/6gR47frvuGB9x/VAqH4Pdoju/aZ5OjCMCnCiWV/EDHxjcaKz8/RyLE0qs8QV0z03NZsoUdp2KTOUukILKTbAERGP3IaKbQlXqMC/C2n7uQvK06dk9gAOm9kfWksqvaxxYgVcKoB5o/5LxzAv0gHO1dvLOL7UFMhYDp9YptJQO4a7xrZL0u2XP4H1Hu1EOGmptIComGlODbX5a3UlnKUDLojrwvSbBnjBYLRsOBozyoz9/LGUh2OdxhvsIu/uDC40lWtZXLTOjlTXu8SukdvWmIG25prkeaL+GmZl+3loZBEjp6Ou4Sq2uLGxU9sSky0l9O1Y/VlCvOLlBjQ4WZqpMWG7FqqenbY79nEZ4A5xz2ZMi/iQlYxtGRNUVw5ovCzXE63PUYZPvMi5+dxuQowXCEgDE9Qu9VaC3GWz/1i6Q6ZTk4z4CVQ3OI/ncIcn0xNaaOjLDZ3FmdNjz77cBiffEWzduRzB3evYHYBfAddTYdcSc+WbfEaGX1KSuYWL/kdFCvwV3bobBH6bD7RVg2fAMY/2xlfoDSmlyEGpcNxjKMiMYCgEf7JtJKt7Bg/AQuOsAzQo5Q1MjVx5h5aWXZw9yvFSkD1vMdyfnfITpSAD51mM23iJ8TEn/MM4sB8Eq4ox8sUGsT3d5kuK+gCO79cwJC8VwCP2Btj6xxRf8Vw9ztIJXJNkam7wRHa2AOMrY/DHMrTjq4g7S+P+MGGtb4/BH866yLEQJjBZYIZ53vVaq/cCE/RTeGWyU+lrrUs1W+B8A5xkwxbJ2+EvOimF6w1f35Lyz7u72+/nVirIicUoUiCPI2maaaT9sp2dQB2w+cPii20NxB69SIDtAlCVgqu+VdlxWbqhvTBrrC+ERDnO1/DSyBK+apAcM4v/xfxlVwYuZQM5o8C1MlDPk/DEqj8XzKXUfGjOeDmmILIfZ8EKhh1/PAjTIqst7e7m9vv7t7ch4u9g9XbaYA4xYcnEo7uf7O7m1v5LuZuWrIabuy5MStp0yyOlJeDCnqbtxnw27NhlWwits5fjieEEI8OO+B5Xo0cTu45OnEyrOs/xxLrrzXsaPeXTz7p3739K2NKL5QBdvOzGzsS6SDN2epLUKC8zSWKY00r90lMg4ix5nWFeZkqxrJTi63aWM6HsvT2cu6sKD29XdS1morFCa68EpEEWlNuba2xpaebnfP4eORgncQ6aibvrpI3UPn+IbNTdu9UR0DArPYyo0ePnD+ch4vB4AT3mtm7rp0RRAh8B4o0/KBUdcKy0KTbtuag2bCmq1dq6npNY2wUAwxp4To9s3yYObHQ4rrF7GSxVWtJFFRfZjvQW5ZjlFbPStqbxuAYet9a4uxKMJMJ6gWEtrIcE52bcGIt+3r+Fo/bWlrbS1iZdeu2z44PSquO8u40Qs/+X30D5ucaBb0V6ewueVkS7IlsJja5vfm05OMGdrIKMrNAOC2opBdfGroxHkfrwjsNkcvl0d4cKwKqSRiJCIsD5OvI2aAH6EYAHBjEnp3xNnSJ8y5MmMJGucCebUui0VL/tqdRFrhNMtG9baK1Vs0sILa51pdZPKTj77QGkJS3dAVAvyKvGa4WcrZUjhWuAa9ElK+ZTRc4T8t5yzAdZrKdTWBhom4at/E5nsE8fhlQhhF4DsWy/oBwV6mXDfhyGnysrCKVxvrly0aqgvNLQqmQZ3oL0VUiJlV+trN22mw/xjkh648J+uGd/lyO9n9HnFt9D3+Mpq4WqFp0Gv60W82ox2V6M4rhUa6FtNGdiNWjP08Y7e3lGfRpFKbyye8y2N5q4o9w10Q6AXIdJtthmLtu8KMC9syBOJx7JMfS54Eu6yqqS4FEaZPJw/54b2J/MR/ka8xWoK6dxN5eiSwG2JbgvvICq2HFx86rIgjQGV+FD9B4DidQ+MaT+mhaMgNIBcoZX/e44vF6B9Yx+O/dlsFA3fIDfR6D7ACOxXhzq+YKuFjXtGc43gZER51RtGzjfcPHIgKzcVLpu/l1v27XmGgFG7R6+IX0S/SQ+tuIt1DUvL/BsM8Mz5InWP7z0+oiBnQS+1ZDlovcObUvo7ixbc9NHlOs1bVK2/YL6zhItwiPoSyU0RnGfyCnsl/XETZEdgxjgn2NWCGCSMdAa5JSzoKwWjKq1E7yhiRxNpEVJ8xYvHYEXgmRm6prBGOUwJfrHtVCAAiVbcDndW8AfBKHL7XW+eRseGGFeH2Iviy8YGovRfQ6Cw42n1Ag6Gtbz0A16UvgdAZtEIVpDnyB78VYQLhqMqT8300StEXBSCspNWKu07dLagm7FkU58VLymNR4fl4gMPrkIsd5YUMPCVMlQiq3ddGNUJgbnoeMyNCyH8txa65heV2kjPGSDA0wwwc/UajvA28j6KivGsijxnSSqRIx0jCe2AwcjAkvKaVjsSb2qIFrL+L5ZzGhx+f3pGOq/NdJZXOPHL3yqHOgYuSzEKbVpCPwV1Di+Cq1khulOrSfBpdYIaw1FmcaF7jmjG7AMqCu3YGresU2kEtGiZFAA166yIAJcz/gC63ztro2s8wo0F65Eqa8W4Wzb3HInOLRemLn7jSNi0ije9RfZTwYb6xD1FY3xu/Z+DVyWgCUqKqZpydxNj8ll65ag29F43Jo5nRcNyCB2UodJ1yzPBT8s+iiAKVypGfaovZ/GxfWUZWTYLW67PC0CCpNj+zboAZWja8kz1ctFovw8kLs1bvCvE/MTYpzAZAORdhCxn4ZGXPBXxpa3LalS0s+22+xMaRE7TF3KDP47FwT+3ssYzhXeJddm2vNrf7lj2ArNKc6ezOT3GXUbjiMwWbfOTMiUP/LSB20oOicN+IfWcYZHerugO3x1f9iSfrSYb1fP7RJ/uq3Bwv1WtlYd2uBujEIPN16P0Hnxdqfvwq/i137ZArCrGRbEy4OIv4XVML8T0cvc/5zrRUM4+rNVH0N4UcDJxGsj7XnxzWS8kXymWH9wUjlzacRj+rPHbAfz6YN1v4mx1wFxkZ7F3XiC8w2qCRt2CsoYrS8uuHO+oe6Epar5IEloMrfdsO1JouwpclxPqXijrSmxbaRNvO07aRXCPNWr4dtEcd2Tctbq3tOI2a6zBfyNvG3TJM2lCKHlqOXXvLkb4J4doN1qNJz72+Km6jTEJR14jP4tMI0bn2A/roHtamgkCmN05pcC9FoQSzvkaa4p2lSeKTvAlV7/McM5yxZYAcm8IrH9ksg4iOv6ydqkU5p1DNybjb+RZCUx10CQo42UYMC2iFR2evgnr2/eJ3Pkho0mOvVEf6/81+yub97HX+ZJ3I99GIkXoyohp0uaZwZZUekhUX1HqqHzpsAkFlCgeFBSI92stINm506l3RuJxrHVES5XSsJOftR5HMjhfkunnb7niXeuVnKDKVSCRIsq34Buga2/XcuwUq0rfey30UP3Pc/t5xsRwdsry4y9NCg8J6F0h0ex9o1d/nIe153/gFk4sSkqdxM0wcnjU627SG0SkoXo+e1Y3ngXqoakCTNW69FfV/cnVWWQwQNIDV8xzZhYqU6x9bN9A12/Q7dSFKVGn4Kc34uVOhZyF6DjoPsAUtnhUUu85qk36L9mr2d/a/1ybowO3UwJlW5g+ygkOaLVNUR8GuMKo6H7T+/dqk84thPygAQEAiUTW1OXCwIso6QvFnc9o1WiHxLZIV3w9Rck7lz7n8DDxGplKhuTyCagEKxPXBavNC7K5iDY9TtnbP6KskesLIl0n1jIRJNSOHUV2j2nX4zdE+CaLinIWvzX745xHIiWeMsETlHW8LVtIpQT+PoGLTFTRwBdS4m3UR6EPAVT3VG+UvGn0uwmcmkni7ITmfKySt84FHzWpID9dD2C+Pqdty9R6UNQ7QN9DPqjcdDenGtCdkq9gNlqdoVWUFBOX72e/cerUork9R5uQnkP0gfEZ/dqEkcyASxp5gJM3zkchSiT6cfscpOOwo092aavUCmB0FwnGV9GrSTdijjgeMGAvEFaVscuMQ0f6qwbKVzqGVkFbrZNEAGNKbNR7n/mv30007HAySm4jJdRpoLsTfoQZv9zd9AFaExwK1wNx3tNiF12w6weH2GlRE5tILcbUQm3ehqt0MBMAB8V7W8l8DtgYAL61vh11Bfw/wUAAP//qkt+EQ==" + return "eJzsXd2P27aWf89fQfQlyWLi3qa7C2ywuMB0ctubbZIO4pkA+6RLi8c2a4pUSGom7l+/4JdE2bQt68PTFNs+ZSzx/M4Hzwd5SL1CG9i+Qau8fIaQpprBG/T8FyFWDNANExVBtwzrpZDF82cISWCAFbxBK/wMIQIql7TUVPA36O/PEELol5tbVAhSMXiG0JICI+qN/eEV4riAQMj8p7el+bcUVfhL/Hz8DsMLYKr+c3hVLH6HXEd/TuAJ/zlcnGohKV+hArSkudofeRdCDKNSIGf/1vrpIBTzn/tj5p7YwPZRSJIcuACNCdZ4qsENq5OMrbZKQzHJ0BKUqGQOow0eBv6uFoj7/7vTdtUal4hqYa078WtW4LKkfOUf/a41+BHr/ODNUa+xRhJ0JTkQtJSiQK2peH37Dn2pQG5ne2wtKGOUrw7Raw3zk3s2mEb0Tnt+t8USz1SUnCoBSy6Uk8ezfcWldN5CeiOUts8qRHnOKgJIwqpiWF4hjb9eIUx+r5QugOsrhDlBUlScGKGDlELOEngofxA0h6wQXK/7YAoCk1AKqZEdJ0WolMLaAiV9qNy6t9G7t0gskV5DUGqguwAm+EohLVLEtdCYJegumcD6MNU781pNCRei4jo1vNpUPfm6W0PEU5jYxp0StNjaPyqQDzSHQ3Sj4foAuI7/uYfDDJIEg34WEsFXXJQMrhDeeWMppJ9Ocy0kXgGiCs015gTL5m/38yRPjsIo8vRjOd9h/lApDwYrJXKKNRD0SNMGG4AMFLBB1PIwLVDOeoFYdG0PZKNeypbxSiWAcFAajio6F4xBHvS8ge2rB8wqQCWm0vvXUooHSgBhQqh5ELMmALeGTuUCDcQNbHd+OSau5j2Lp/Ob4S1YLg1bD5CVkuYpx3pqmq8B5WssV0CQHcIasLMVb0ktDc5/vVfWvc5/vUeaglQz9AmWRrgK5YJriXNtRzJ6pEuEy5LRHC/MXBF6DfKRKrhCVD9XdnRGlXse9oNXjuWiZXmHY9eNfRT9LIQuJeX6WBBbgB4Sxkbz5oZ9xyJa1rg7+fQAwfxrCAjz/gAYw/zV3DuDd2+tzSVhWEOc0EMFDHEgOB+MhFVP+p/smwmSJyjWoGYqFyX8cP7En5v30A97XHag9nrGRI4PiLwT2dcp4RqHEQZ+tcAm8hag14IIJlbbTrgKLDegR0flhu2F6ce+YH48UzNiuVSgU9GxU6LniflREq5YFGWloZsvds9OVkcsqYRHzNiMSFGWQGaLrYYU58Z3HWb8Hc9FYdRrX0d+sJDtBSId6GclzjegVZbbJHk/kp+Fxg92Bh7KlcY8h1leVjMJxjUCyXIhQR0Es1eu7sD5WBULkCYFsOOgMCwS3MJZm3LMJwiB/ilo1ptlmhYwU5D3AHVv0xLjLDFjHhjlSEEuOFGdyM/KPOUfTlE2adLSpDZRpYCZ9VZA0M3tvcshqUJ5JSVwzbYGWaUgCKyLkAhVm5kE3Neib4z9GXjOos1IbrXADNyJcCbKYWZcQzBDOgTvfkOiBGn9+lElWRSPkmoYh38zlAaOtOgmAEt6ZAnYMbuLoIBCyO1sYWxL8JnERaboH9ATirFaW/77+t2gchSMcRqr/Pxhhu7WVHlfbQxYcLZF+AFTZpJ2O9s+f/A1kksHjUDNy/AaLXFBWTIOHmPJVNU9Wfrg4DezzFboT8aNesRlRnlPezX62dOMnTOUe1SrCpR2k5hqhcQjR4YmUiXO4Um4FZUek90wSS2LDcdaPAW/HPSjkJsZ5SsJSo3lhnOgD2Hh1oDxZM5B4rOCmfVM/RGF5GIQJhhROArMvx5A9gQxslzOhlOVR/KZ48SbFMsnMN6HoTVWaAHAkaw4p3x11GQdgMy6+V4w/sFwaXyoGQYpynMIOB6xQkpjqYFcRXnWDM3t2idB8AByi/7zb80v10sNEinzO+WrK7uQZyYqFxo9UEXDLK1KMzF/eN28uldxmFdLKfJOJcdb//DoNUdUAbFKaZCzNVmqmQHHBYGU4R0U+17GT6jJH53eeW0N/3z789wy9NEQcM4MSwjWYFRv1ylrRKfhKrfsnOW4xDnV20TwPZIJH8QdhqtRu42/GqngAYYxn19+OgNppSmjf9iEaRBYE3NKkDlw7Zc0HVC/Dt/OIzrgq/gaMNPrbbZgIt9Mof+aBHIkgsLtAlEXlf8uFrMlpgzIBOh+Fwtvk2v8AMjRMZruaI8GXPBrU6KLZ8x56FS1KExOcgHpWS9f0zMusSPOupqYTtFNwTJE3Q3Q6ZS+i7Sf6hukUxpAUqq9zWCLJZ/V6xCZS6mzxt+N7Dr/9/rTx7qAVM0CSBeQZTmFr8R288sB8ztNVrgdEOWCa0w5yClwWUARhdNwvO4OVPt9orNBFGvMDN0lDls4JrkpMMerCeVj8psPnsZudtPd/EtwbR5jCdDuiLq0oCnJv5ii9MpJ76pe6/OkzYRdAFpWbEkZixoG8jWQinXi4oFKXWHmV23HF7gfv1k0NRo4HQtzYTJ9M7Rbtz0s2NOdV2uqtFhJXJySvq1EXKQUYmOk62FAU67aZ1xbgyk4rO9UCJvXQslvH6kLA7ehqRB1q7R+wBOuy4iAVPJUGjoJ7zYyqNKYITVBbEUfTJTQWEPXQPZ02ovD3Nk6rF8eUZONUJ5An5E0Omk13mgylUq3bbefw9MTFsFE5FUBXM8IGKEPcFR3LfekqjwHpZYVq0kgR+JA7KyB2O2SKWEYAsoZ7ZcKJAWFhERMiE1VngLnNjOmRGcpJFZPVpuDRvP8X6sN/CtkJy5QFHGn6aMJXRKXda/pzS2aa5xviKQPIG27qX/b7o7v908vhbRv/fLrP56PZoXNVrRPq+yunglo2eCdxZuqqBi2ueTN7X3of+LRPmPcCVgj6LYqdpIHRguqB27YGtgOqR0tbFI2hAbji5dkxttFNbgd4q47qH4XwDXw5ZhzoRF8zQEI+gFh5ZXX/sF1mhVUj7d4+frfz5CgzyBH25RvtO1HThjnxOu0p1idxlwads/ZdN81GZOuryRgbRf6Md8xnNhqPMEnsBso11CAxCzzq5VuHvbcd3kvcsxQPWa9AurmHuVuP6avl9jHGuQ2Llo/6gR47frvuGB9x/VAqH4Pdoju/aZ5OjCMCnCiWV/EDHxjcaKz8/RyLE0qs8QV0z03NZsoUdp2KTOUukILKTbAERGP3IaKbQlXqMC/C2n7uQvK06dk9gAOm9kfWksqvaxxYgVcKoB5o/5LxzAv0gHO1dvLOL7UFMhYDp9YptJQO4a7xrZL0u2XP4H1Hu1EOGmptIComGlODbX5a3UlnKUDLojrwvSbBnjBYLRsOBozyoz9/LGUh2OdxhvsIu/uDC40lWtZXLTOjlTXu8SukdvWmIG25prkeaL+GmZl+3loZBEjp6Ou4Sq2uLGxU9sSky0l9O1Y/VlCvOLlBjQ4WZqpMWG7FqqenbY79nEZ4A5xz2ZMi/iQlYxtGRNUVw5ovCzXE63PUYZPvMi5+dxuQowXCEgDE9Qu9VaC3GWz/1i6Q6ZTk4z4CVQ3OI/ncIcn0xNaaOjLDZ3FmdNjz77cBiffEWzduRzB3evYHYBfAddTYdcSc+WbfEaGX1KSuYWL/kdFCvwV3bobBH6bD7RVg2fAMY/2xlfoDSmlyEGpcNxjKMiMYCgEf7JtJKt7Bg/AQuOsAzQo5Q1MjVx5h5aWXZw9yvFSkD1vMdyfnfITpSAD51mM23iJ8TEn/MM4sB8Eq4ox8sUGsT3d5kuK+gCO79cwJC8VwCP2Btj6xxRf8Vw9ztIJXJNkam7wRHa2AOMrY/DHMrTjq4g7S+P+MGGtb4/BH866yLEQJjBZYIZ53vVaq/cCE/RTeGWyU+lrrUs1W+B8A5xkwxbJ2+EvOimF6w1f35Lyz7u72+/nVirIicUoUiCPI2maaaT9sp2dQB2w+cPii20NxB69SIDtAlCVgqu+VdlxWbqhvTBrrC+ERDnO1/DSyBK+apAcM4v/xfxlVwYuZQM5o8C1MlDPk/DEqj8XzKXUfGjOeDmmILIfZ8EKhh1/PAjTIqst7e7m9vv7t7ch4u9g9XbaYA4xYcnEo7uf7O7m1v5LuZuWrIabuy5MStp0yyOlJeDCnqbtxnw27NhlWwits5fjieEEI8OO+B5Xo0cTu45OnEyrOs/xxLrrzXsaPeXTz7p3739K2NKL5QBdvOzGzsS6SDN2epLUKC8zSWKY00r90lMg4ix5nWFeZkqxrJTi63aWM6HsvT2cu6sKD29XdS1morFCa68EpEEWlNuba2xpaebnfP4eORgncQ6aibvrpI3UPn+IbNTdu9UR0DArPYyo0ePnD+ch4vB4AT3mtm7rp0RRAh8B4o0/KBUdcKy0KTbtuag2bCmq1dq6npNY2wUAwxp4To9s3yYObHQ4rrF7GSxVWtJFFRfZjvQW5ZjlFbPStqbxuAYet9a4uxKMJMJ6gWEtrIcE52bcGIt+3r+Fo/bWlrbS1iZdeu2z44PSquO8u40Qs/+X30D5ucaBb0V6ewueVkS7IlsJja5vfm05OMGdrIKMrNAOC2opBdfGroxHkfrwjsNkcvl0d4cKwKqSRiJCIsD5OvI2aAH6EYAHBjEnp3xNnSJ8y5MmMJGucCebUui0VL/tqdRFrhNMtG9baK1Vs0sILa51pdZPKTj77QGkJS3dAVAvyKvGa4WcrZUjhWuAa9ElK+ZTRc4T8t5yzAdZrKdTWBhom4at/E5nsE8fhlQhhF4DsWy/oBwV6mXDfhyGnysrCKVxvrly0aqgvNLQqmQZ3oL0VUiJlV+trN22mw/xjkh648J+uGd/lyO9n9HnFt9D3+Mpq4WqFp0Gv60W82ox2V6M4rhUa6FtNGdiNWjP08Y7e3lGfRpFKbyye8y2N5q4o9w10Q6AXIdJtthmLtu8KMC9syBOJx7JMfS54Eu6yqqS4FEaZPJw/54b2J/MR/ka8xWoK6dxN5eiSwG2JbgvvICq2HFx86rIgjQGV+FD9B4DidQ+MaT+mhaMgNIBcoZX/e44vF6B9Yx+O/dlsFA3fIDfR6D7ACOxXhzq+YKuFjXtGc43gZER51RtGzjfcPHIgKzcVLpu/l1v27XmGgFG7R6+IX0S/SQ+tuIt1DUvL/BsM8Mz5InWP7z0+oiBnQS+1ZDlovcObUvo7ixbc9NHlOs1bVK2/YL6zhItwiPoSyU0RnGfyCnsl/XETZEdgxjgn2NWCGCSMdAa5JSzoKwWjKq1E7yhiRxNpEVJ8xYvHYEXgmRm6prBGOUwJfrHtVCAAiVbcDndW8AfBKHL7XW+eRseGGFeH2Iviy8YGovRfQ6Cw42n1Ag6Gtbz0A16UvgdAZtEIVpDnyB78VYQLhqMqT8300StEXBSCspNWKu07dLagm7FkU58VLymNR4fl4gMPrkIsd5YUMPCVMlQiq3ddGNUJgbnoeMyNCyH8txa65heV2kjPGSDA0wwwc/UajvA28j6KivGsijxnSSqRIx0jCe2AwcjAkvKaVjsSb2qIFrL+L5ZzGhx+f3pGOq/NdJZXOPHL3yqHOgYuSzEKbVpCPwV1Di+Cq1khulOrSfBpdYIaw1FmcaF7jmjG7AMqCu3YGresU2kEtGiZFAA166yIAJcz/gC63ztro2s8wo0F65Eqa8W4Wzb3HInOLRemLn7jSNi0ije9RfZTwYb6xD1FY3xu/Z+DVyWgCUqKqZpydxNj8ll65ag29F43Jo5nRcNyCB2UodJ1yzPBT8s+iiAKVypGfaovZ/GxfWUZWTYLW67PC0CCpNj+zboAZWja8kz1ctFovw8kLs1bvCvE/MTYpzAZAORdhCxn4ZGXPBXxpa3LalS0s+22+xMaRE7TF3KDP47FwT+3ssYzhXeJddm2vNrf7lj2ArNKc6ezOT3GXUbjiMwWbfOTMiUP/LSB20oOicN+IfWcYZHerugO3x1f9iSfrSYb1fP7RJ/uq3Bwv1WtlYd2uBujEIPN16P0Hnxdqfvwq/i137ZArCrGRbEy4OIv4XVML8T0cvc/5zrRUM4+rNVH0N4UcDJxGsj7XnxzWS8kXymWH9wUjlzacRj+rPHbAfz6YN1v4mx1wFxkZ7F3XiC8w2qCRt2CsoYrS8uuHO+oe6Epar5IEloMrfdsO1JouwpclxPqXijrSmxbaRNvO07aRXCPNWr4dtEcd2Tctbq3tOI2a6zBfyNvG3TJM2lCKHlqOXXvLkb4J4doN1qNJz72+Km6jTEJR14jP4tMI0bn2A/roHtamgkCmN05pcC9FoQSzvkaa4p2lSeKTvAlV7/McM5yxZYAcm8IrH9ksg4iOv6ydqkU5p1DNybjb+RZCUx10CQo42UYMC2iFR2evgnr2/eJ3Pkho0mOvVEf6/81+yub97HX+ZJ3I99GIkXoyohp0uaZwZZUekhUX1HqqHzpsAkFlCgeFBSI92stINm506l3RuJxrHVES5XSsJOftR5HMjhfkunnb7niXeuVnKDKVSCRIsq34Buga2/XcuwUq0rfey30UP3Pc/t5xsRwdsry4y9NCg8J6F0h0ex9o1d/nIe153/gFk4sSkqdxM0wcnjU627SG0SkoXo+e1Y3ngXqoakCTNW69FfV/cnVWWQwQNIDV8xzZhYqU6x9bN9A12/Q7dSFKVGn4Kc34uVOhZyF6DjoPsAUtnhUUu85qk36L9mr2d/a/1ybowO3UwJlW5g+ygkOaLVNUR8GuMKo6H7T+/dqk84thPygAQEAiUTW1OXCwIso6QvFnc9o1WiHxLZIV3w9Rck7lz7n8DDxGplKhuTyCagEKxPXBavNC7K5iDY9TtnbP6KskesLIl0n1jIRJNSOHUV2j2nX4zdE+CaLinIWvzX745xHIiWeMsETlHW8LVtIpQT+PoGLTFTRwBdS4m3UR6EPAVT3VG+UvGn0uwmcmkni7ITmfKySt84FHzWpID9dD2C+Pqdty9R6UNQ7QN9DPqjcdDenGtCdkq9gNlqdoVWUFBOX72e/cerUork9R5uQnkP0gfEZ/dqEkcyASxp5gJM3zkchSiT6cfscpOOwo092aavUCmB0FwnGV9GrSTdijjgeMGAvEFaVscuMQ0f6qwbKVzqGVkFbrZNEAGNKbNR7n/mv30007HAySm4jJdRpoLsTfoQZv9zd9AFaExwK1wNx3tNiF12w6weH2GlRE5tILcbUQm3uov2/wIAAP//P7RJng==" } diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/data.json b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/data.json index 1bb2cee29a3d..7ea16beb6788 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/data.json +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/data.json @@ -78,10 +78,6 @@ "metadata": { "region": "us-central1", "zone": "us-central1-a" - }, - "otel_log": { - "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", - "span_id": "00f067aa0ba902b7" } } }, diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md index 50614879d094..20c755af0a8e 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/docs.md @@ -6,7 +6,6 @@ The logs include detailed information about: - API endpoints and deployed models - Request and response payloads - Model versions and API methods used -- OpenTelemetry trace data - Request metadata and timing information @@ -67,10 +66,6 @@ Here is a sample event for `vertexai_logs`: "metadata": { "user_id": "user123", "session_id": "session456" - }, - "otel_log": { - "trace_id": "abc123def456", - "span_id": "789ghi012jkl" } } } diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml index 2cf4c9d713c7..f3c3b2d7438a 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/_meta/fields.yml @@ -45,8 +45,4 @@ - name: metadata type: object enabled: true - description: Additional metadata associated with the AI interaction in JSON format. - - name: otel_log - type: object - enabled: true - description: OpenTelemetry log data associated with the AI interaction in JSON format. \ No newline at end of file + description: Additional metadata associated with the AI interaction in JSON format. \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/data.go b/x-pack/metricbeat/module/gcp/vertexai_logs/data.go index 854ec9d55777..c43f5f8dc89a 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/data.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/data.go @@ -30,7 +30,6 @@ type VertexAILogRow struct { FullRequest string `bigquery:"full_request"` FullResponse string `bigquery:"full_response"` Metadata string `bigquery:"metadata"` - OtelLog string `bigquery:"otel_log"` } // CreateEvent creates a single mb.Event from a VertexAILogRow @@ -65,10 +64,6 @@ func CreateEvent(row VertexAILogRow, projectID string, logger *logp.Logger) (mb. logger.Warnf("failed to process metadata: %v", err) } - if err := processJSONField(row.OtelLog, "otel_log", fields, logger); err != nil { - logger.Warnf("failed to process otel_log: %v", err) - } - event.MetricSetFields = fields // Set cloud provider information diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go index 86942b90d7f0..00b56a93a8c0 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -219,8 +219,7 @@ SELECT IFNULL(api_method, '') AS api_method, IFNULL(TO_JSON_STRING(full_request), '{}') AS full_request, IFNULL(TO_JSON_STRING(full_response), '{}') AS full_response, - IFNULL(TO_JSON_STRING(metadata), '{}') AS metadata, - IFNULL(TO_JSON_STRING(otel_log), '{}') AS otel_log + IFNULL(TO_JSON_STRING(metadata), '{}') AS metadata FROM %s WHERE diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index 62e844315e21..dab17304abb9 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -62,7 +62,6 @@ func TestCreateEvent(t *testing.T) { FullRequest: `{"inputs": ["test"]}`, FullResponse: `{"outputs": ["result"]}`, Metadata: `{"user_id": "user123"}`, - OtelLog: `{"trace_id": "abc123"}`, } projectID := "test-project" logger := logp.NewLogger("test") @@ -83,7 +82,6 @@ func TestCreateEvent(t *testing.T) { "full_request": map[string]interface{}{"inputs": []interface{}{"test"}}, "full_response": map[string]interface{}{"outputs": []interface{}{"result"}}, "metadata": map[string]interface{}{"user_id": "user123"}, - "otel_log": map[string]interface{}{"trace_id": "abc123"}, } assert.Equal(expectedFields, event.MetricSetFields) // Check RootFields @@ -113,7 +111,6 @@ func TestCreateEventWithInvalidJSON(t *testing.T) { FullRequest: `{"invalid": json}`, // Invalid JSON FullResponse: `{}`, Metadata: `{}`, - OtelLog: `{}`, } projectID := "test-project" logger := logp.NewLogger("test") @@ -160,8 +157,7 @@ func TestEventsMapping(t *testing.T) { FullRequest: `{}`, FullResponse: `{}`, Metadata: `{}`, - OtelLog: `{}`, - }, + }, { Endpoint: "https://us-west1-aiplatform.googleapis.com", DeployedModelID: "model-789012", @@ -175,8 +171,7 @@ func TestEventsMapping(t *testing.T) { FullRequest: `{}`, FullResponse: `{}`, Metadata: `{}`, - OtelLog: `{}`, - }, + }, } projectID := "test-project" logger := logp.NewLogger("test") From cecdb66c9e099cf8f172a10015d9c16e8eaa44b9 Mon Sep 17 00:00:00 2001 From: Ishleen Kaur <102962586+ishleenk17@users.noreply.github.com> Date: Tue, 16 Sep 2025 16:49:45 +0530 Subject: [PATCH 20/26] Update vertexai_logs.go --- x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go index 00b56a93a8c0..b2555135b059 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -12,8 +12,8 @@ import ( "time" "cloud.google.com/go/bigquery" - "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/api/iterator" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" From 02537e9136196c1be4fe8ecbdeb30be01ded5636 Mon Sep 17 00:00:00 2001 From: Ishleen Kaur <102962586+ishleenk17@users.noreply.github.com> Date: Tue, 16 Sep 2025 16:50:13 +0530 Subject: [PATCH 21/26] Update vertexai_logs_test.go --- .../metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index dab17304abb9..b3f2dfad4522 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -7,11 +7,10 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/stretchr/testify/assert" ) func TestGenerateQuery(t *testing.T) { From c3bd7a98ded8bf6d2f3831764dc3fcd84dd2e0f0 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Tue, 16 Sep 2025 18:13:58 +0530 Subject: [PATCH 22/26] Replace run with read --- .../module/gcp/vertexai_logs/vertexai_logs.go | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go index b2555135b059..19154de1cf58 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -154,23 +154,9 @@ func (m *MetricSet) queryVertexAILogs(ctx context.Context, client *bigquery.Clie q := client.Query(query) q.Location = location - job, err := q.Run(ctx) + it, err := q.Read(ctx) if err != nil { - return nil, fmt.Errorf("bigquery Run failed: %w", err) - } - - status, err := job.Wait(ctx) - if err != nil { - return nil, fmt.Errorf("bigquery Wait failed: %w", err) - } - - if err := status.Err(); err != nil { - return nil, fmt.Errorf("bigquery status error: %w", err) - } - - it, err := job.Read(ctx) - if err != nil { - return nil, fmt.Errorf("reading from bigquery job failed: %w", err) + return nil, fmt.Errorf("bigquery Read failed: %w", err) } var rows []VertexAILogRow From 9adad7b2182a7b80a67c1da50d9aba029b84722c Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Tue, 16 Sep 2025 18:31:17 +0530 Subject: [PATCH 23/26] Linter error --- .../metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index b3f2dfad4522..23dc9cd287ee 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -156,7 +156,7 @@ func TestEventsMapping(t *testing.T) { FullRequest: `{}`, FullResponse: `{}`, Metadata: `{}`, - }, + }, { Endpoint: "https://us-west1-aiplatform.googleapis.com", DeployedModelID: "model-789012", @@ -170,7 +170,7 @@ func TestEventsMapping(t *testing.T) { FullRequest: `{}`, FullResponse: `{}`, Metadata: `{}`, - }, + }, } projectID := "test-project" logger := logp.NewLogger("test") From 7f510051cac2465dab737d2ddd4b3bee00a48f67 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Tue, 16 Sep 2025 20:34:10 +0530 Subject: [PATCH 24/26] linter fix --- x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go index 19154de1cf58..c2d5d56e0f14 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs.go @@ -12,8 +12,8 @@ import ( "time" "cloud.google.com/go/bigquery" - "google.golang.org/api/option" "google.golang.org/api/iterator" + "google.golang.org/api/option" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" From ee38195deea8f97147438ed50a8146f6e8187935 Mon Sep 17 00:00:00 2001 From: ishleenk17 Date: Tue, 16 Sep 2025 20:53:20 +0530 Subject: [PATCH 25/26] LINTER ERROR --- .../metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go index 23dc9cd287ee..529a607178c5 100644 --- a/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go +++ b/x-pack/metricbeat/module/gcp/vertexai_logs/vertexai_logs_test.go @@ -7,10 +7,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/stretchr/testify/assert" ) func TestGenerateQuery(t *testing.T) { From 14f1834c543e54499ba3ef7f5f04a9880f576e0d Mon Sep 17 00:00:00 2001 From: Colleen McGinnis Date: Tue, 16 Sep 2025 13:28:32 -0500 Subject: [PATCH 26/26] rerun make update --- .../metricbeat/metricbeat-metricset-gcp-vertexai_logs.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md index 157605739d95..f9ebdcf68f44 100644 --- a/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md +++ b/docs/reference/metricbeat/metricbeat-metricset-gcp-vertexai_logs.md @@ -1,7 +1,6 @@ --- mapped_pages: - https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-metricset-gcp-vertexai_logs.html - applies_to: stack: beta 9.2.0 ---