|
| 1 | +--- |
| 2 | +mapped_pages: |
| 3 | + - https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-metricset-gcp-vertexai_logs.html |
| 4 | +applies_to: |
| 5 | + stack: beta 9.2.0 |
| 6 | +--- |
| 7 | + |
| 8 | +% This file is generated! See scripts/docs_collector.py |
| 9 | + |
| 10 | +# Google Cloud Platform vertexai_logs metricset [metricbeat-metricset-gcp-vertexai_logs] |
| 11 | + |
| 12 | +The `vertexai_logs` metricset is designed to collect Vertex AI prompt-response logs from GCP BigQuery. BigQuery is a fully-managed, serverless data warehouse that stores detailed logs of interactions with Vertex AI models. |
| 13 | + |
| 14 | +Vertex AI logs export to BigQuery enables you to export detailed Google Cloud Vertex AI interaction data (such as prompts, responses, model usage, and metadata) automatically to a BigQuery dataset that you specify. Then you can access your Vertex AI logs from BigQuery for detailed analysis and monitoring using Metricbeat. This enables comprehensive tracking of AI model usage, performance monitoring, and cost analysis. |
| 15 | + |
| 16 | +The logs include detailed information about: |
| 17 | +- API endpoints and deployed models |
| 18 | +- Request and response payloads |
| 19 | +- Model versions and API methods used |
| 20 | +- Request metadata and timing information |
| 21 | + |
| 22 | + |
| 23 | +## Metricset-specific configuration notes [_metricset_specific_configuration_notes_14] |
| 24 | + |
| 25 | +* **table_id**: (Required) Full table identifier in the format `project_id.dataset_id.table_name` that contains the Vertex AI logs data. You can copy this from the "Details" tab when viewing your table in the BigQuery web console, under the "Table ID" field. |
| 26 | + |
| 27 | + |
| 28 | +## Configuration example [_configuration_example_22] |
| 29 | + |
| 30 | +```yaml |
| 31 | +- module: gcp |
| 32 | + metricsets: |
| 33 | + - vertexai_logs |
| 34 | + period: 10m |
| 35 | + project_id: "your project id" |
| 36 | + credentials_file_path: "your JSON credentials file path" |
| 37 | + table_id: "your_project.your_dataset.your_vertex_ai_logs_table" |
| 38 | +``` |
| 39 | +
|
| 40 | +## Sample Event |
| 41 | +
|
| 42 | +Here is a sample event for `vertexai_logs`: |
| 43 | + |
| 44 | +```json |
| 45 | +{ |
| 46 | + "@timestamp": "2023-12-01T10:30:45.000Z", |
| 47 | + "cloud": { |
| 48 | + "provider": "gcp", |
| 49 | + "project": { |
| 50 | + "id": "my-gcp-project" |
| 51 | + } |
| 52 | + }, |
| 53 | + "gcp": { |
| 54 | + "vertexai_logs": { |
| 55 | + "endpoint": "https://us-central1-aiplatform.googleapis.com", |
| 56 | + "deployed_model_id": "1234567890123456789", |
| 57 | + "logging_time": "2023-12-01T10:30:45.000Z", |
| 58 | + "request_id": 98765432101234567, |
| 59 | + "request_payload": ["What is machine learning?"], |
| 60 | + "response_payload": ["Machine learning is a subset of artificial intelligence..."], |
| 61 | + "model": "gemini-2.5-pro", |
| 62 | + "model_version": "1.0", |
| 63 | + "api_method": "generateContent", |
| 64 | + "full_request": { |
| 65 | + "inputs": ["What is machine learning?"], |
| 66 | + "parameters": { |
| 67 | + "temperature": 0.7 |
| 68 | + } |
| 69 | + }, |
| 70 | + "full_response": { |
| 71 | + "outputs": ["Machine learning is a subset of artificial intelligence..."], |
| 72 | + "usage": { |
| 73 | + "input_tokens": 5, |
| 74 | + "output_tokens": 50 |
| 75 | + } |
| 76 | + }, |
| 77 | + "metadata": { |
| 78 | + "user_id": "user123", |
| 79 | + "session_id": "session456" |
| 80 | + } |
| 81 | + } |
| 82 | + } |
| 83 | +} |
| 84 | +``` |
| 85 | + |
| 86 | +## Fields [_fields] |
| 87 | + |
| 88 | +For a description of each field in the metricset, see the [exported fields](/reference/metricbeat/exported-fields-gcp.md) section. |
| 89 | + |
| 90 | +Here is an example document generated by this metricset: |
| 91 | + |
| 92 | +```json |
| 93 | +{ |
| 94 | + "@timestamp": "2025-09-02T10:14:50.313Z", |
| 95 | + "agent": { |
| 96 | + "hostname": "metricbeat-host", |
| 97 | + "name": "metricbeat-host" |
| 98 | + }, |
| 99 | + "cloud": { |
| 100 | + "account": { |
| 101 | + "id": "elastic-beats" |
| 102 | + }, |
| 103 | + "provider": "gcp", |
| 104 | + "project": { |
| 105 | + "id": "elastic-beats" |
| 106 | + } |
| 107 | + }, |
| 108 | + "event": { |
| 109 | + "dataset": "gcp.vertexai_logs", |
| 110 | + "duration": 123456789, |
| 111 | + "module": "gcp" |
| 112 | + }, |
| 113 | + "gcp": { |
| 114 | + "vertexai_logs": { |
| 115 | + "endpoint": "https://us-central1-aiplatform.googleapis.com/v1/projects/elastic-beats/locations/us-central1/endpoints/123456789", |
| 116 | + "deployed_model_id": "model-deployment-123", |
| 117 | + "logging_time": "2025-09-02T10:14:50.313Z", |
| 118 | + "request_id": 98765432101234567, |
| 119 | + "model": "gemini-2.5-pro", |
| 120 | + "model_version": "001", |
| 121 | + "api_method": "generateContent", |
| 122 | + "request_payload": [ |
| 123 | + "What is the weather like today?" |
| 124 | + ], |
| 125 | + "response_payload": [ |
| 126 | + "I don't have access to real-time weather information. Please check a weather service or app for current conditions." |
| 127 | + ], |
| 128 | + "full_request": { |
| 129 | + "contents": [ |
| 130 | + { |
| 131 | + "parts": [ |
| 132 | + { |
| 133 | + "text": "What is the weather like today?" |
| 134 | + } |
| 135 | + ], |
| 136 | + "role": "user" |
| 137 | + } |
| 138 | + ], |
| 139 | + "generationConfig": { |
| 140 | + "temperature": 0.7, |
| 141 | + "maxOutputTokens": 1024 |
| 142 | + } |
| 143 | + }, |
| 144 | + "full_response": { |
| 145 | + "candidates": [ |
| 146 | + { |
| 147 | + "content": { |
| 148 | + "parts": [ |
| 149 | + { |
| 150 | + "text": "I don't have access to real-time weather information. Please check a weather service or app for current conditions." |
| 151 | + } |
| 152 | + ], |
| 153 | + "role": "model" |
| 154 | + }, |
| 155 | + "finishReason": "STOP", |
| 156 | + "safetyRatings": [ |
| 157 | + { |
| 158 | + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", |
| 159 | + "probability": "NEGLIGIBLE" |
| 160 | + } |
| 161 | + ] |
| 162 | + } |
| 163 | + ], |
| 164 | + "usageMetadata": { |
| 165 | + "promptTokenCount": 8, |
| 166 | + "candidatesTokenCount": 24, |
| 167 | + "totalTokenCount": 32 |
| 168 | + } |
| 169 | + }, |
| 170 | + "metadata": { |
| 171 | + "region": "us-central1", |
| 172 | + "zone": "us-central1-a" |
| 173 | + } |
| 174 | + } |
| 175 | + }, |
| 176 | + "metricset": { |
| 177 | + "name": "vertexai_logs", |
| 178 | + "period": 300000 |
| 179 | + }, |
| 180 | + "service": { |
| 181 | + "type": "gcp" |
| 182 | + } |
| 183 | +} |
| 184 | +``` |
0 commit comments