Skip to content

Commit d3d5b02

Browse files
dkirov-dddkirov-dd
committed
Add pipeline
1 parent 00df174 commit d3d5b02

File tree

3 files changed

+387
-2
lines changed

3 files changed

+387
-2
lines changed

‎hugging_face_tgi/README.md

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ No additional installation is needed on your server.
3838
3939
#### Logs
4040
41-
The Hugging Face TGI integration can collect logs from the server container and forward them to Datadog.
41+
The Hugging Face TGI integration can collect logs from the server container and forward them to Datadog. The TGI server container needs to be started with the environment variable `NO_COLOR=1` and the option `--json-output` for the logs output to be correctly parsed by Datadog. If launching the server in this manner is not possible, the logs ingested by Datadog will likely be malformed, please refer to the troubleshooting section.
4242

4343
<!-- xxx tabs xxx -->
4444
<!-- xxx tab "Host" xxx -->
@@ -55,7 +55,8 @@ The Hugging Face TGI integration can collect logs from the server container and
5555
logs:
5656
- type: docker
5757
source: hugging_face_tgi
58-
service: hugging_face_tgi
58+
service: text-generation-inference
59+
auto_multi_line_detection: true
5960
```
6061

6162
<!-- xxz tab xxx -->
@@ -98,6 +99,22 @@ See [service_checks.json][8] for a list of service checks provided by this integ
9899

99100
In containerized environments, ensure that the Agent has network access to the TGI metrics endpoint specified in the `hugging_face_tgi.d/conf.yaml` file.
100101

102+
If you wish to ingest non JSON TGI logs, use the following logs configuration:
103+
104+
```yaml
105+
logs:
106+
- type: docker
107+
source: hugging_face_tgi
108+
service: text-generation-inference
109+
auto_multi_line_detection: true
110+
log_processing_rules:
111+
- type: mask_sequences
112+
name: strip_ansi
113+
pattern: "\\x1B\\[[0-9;]*m"
114+
replace_placeholder: ""
115+
116+
```
117+
101118
Need help? Contact [Datadog support][9].
102119

103120

‎hugging_face_tgi/assets/logs/hugging_face_tgi.yaml

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
id: hugging_face_tgi
2+
metric_id: hugging_face_tgi
3+
backend_only: false
4+
facets: null
5+
pipeline:
6+
type: pipeline
7+
name: Hugging Face TGI
8+
enabled: true
9+
filter:
10+
query: source:hugging_face_tgi
11+
processors:
12+
- type: grok-parser
13+
name: Non JSON and no color
14+
enabled: true
15+
source: message
16+
samples:
17+
- '2025-09-09T11:29:51.795563Z INFO
18+
generate_stream{parameters=GenerateParameters { best_of: None,
19+
temperature: None, repetition_penalty: None, frequency_penalty: None,
20+
top_k: None, top_p: None, typical_p: None, do_sample: false,
21+
max_new_tokens: Some(20), return_full_text: None, stop: [], truncate:
22+
None, watermark: false, details: false, decoder_input_details: false,
23+
seed: None, top_n_tokens: None, grammar: None, adapter_id: None }
24+
total_time="1.194364886s" validation_time="204.821µs"
25+
queue_time="53.525µs" inference_time="1.194106715s"
26+
time_per_token="59.705335ms" seed="None"}:
27+
text_generation_router::server: router/src/server.rs:637: Success'
28+
- '2025-09-09T11:28:03.840209Z ERROR
29+
chat_completions{parameters="GenerateParameters { best_of: None,
30+
temperature: None, repetition_penalty: None, frequency_penalty: None,
31+
top_k: None, top_p: None, typical_p: None, do_sample: true,
32+
max_new_tokens: Some(20), return_full_text: None, stop: [], truncate:
33+
None, watermark: false, details: true, decoder_input_details: false,
34+
seed: None, top_n_tokens: None, grammar: None, adapter_id: None
35+
}"}:async_stream:generate_stream: text_generation_router::infer:
36+
router/src/infer/mod.rs:126: `inputs` tokens + `max_new_tokens` must
37+
be <= 512. Given: 1864 `inputs` tokens and 20 `max_new_tokens`'
38+
- "2025-09-08T15:41:01.566464Z WARN text_generation_router::server:
39+
router/src/server.rs:1906: Invalid hostname, defaulting to 0.0.0.0"
40+
- "2025-09-08T15:38:42.366067Z INFO download: text_generation_launcher:
41+
Starting check and download process for
42+
teknium/OpenHermes-2.5-Mistral-7B"
43+
- |-
44+
2025-09-08T15:38:40.500145Z INFO text_generation_launcher: Args {
45+
model_id: "teknium/OpenHermes-2.5-Mistral-7B",
46+
revision: None,
47+
validation_workers: 2,
48+
sharded: None,
49+
num_shard: None,
50+
quantize: None,
51+
speculate: None,
52+
dtype: None,
53+
kv_cache_dtype: None,
54+
trust_remote_code: false,
55+
max_concurrent_requests: 128,
56+
max_best_of: 2,
57+
max_stop_sequences: 4,
58+
max_top_n_tokens: 5,
59+
max_input_tokens: None,
60+
max_input_length: None,
61+
max_total_tokens: None,
62+
waiting_served_ratio: 0.3,
63+
max_batch_prefill_tokens: Some(
64+
512,
65+
),
66+
max_batch_total_tokens: None,
67+
max_waiting_tokens: 20,
68+
max_batch_size: None,
69+
cuda_graphs: None,
70+
hostname: "ip-172-31-21-18",
71+
port: 80,
72+
prometheus_port: 9000,
73+
shard_uds_path: "/tmp/text-generation-server",
74+
master_addr: "localhost",
75+
master_port: 29500,
76+
huggingface_hub_cache: None,
77+
weights_cache_override: None,
78+
disable_custom_kernels: false,
79+
cuda_memory_fraction: 1.0,
80+
rope_scaling: None,
81+
rope_factor: None,
82+
json_output: false,
83+
otlp_endpoint: None,
84+
otlp_service_name: "text-generation-inference.router",
85+
cors_allow_origin: [],
86+
api_key: None,
87+
watermark_gamma: None,
88+
watermark_delta: None,
89+
ngrok: false,
90+
ngrok_authtoken: None,
91+
ngrok_edge: None,
92+
tokenizer_config_path: None,
93+
disable_grammar_support: false,
94+
env: false,
95+
max_client_batch_size: 4,
96+
lora_adapters: None,
97+
usage_stats: On,
98+
payload_limit: 2000000,
99+
enable_prefill_logprobs: false,
100+
graceful_termination_timeout: 90,
101+
}
102+
grok:
103+
supportRules: >-
104+
tgi_date %{date("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"):date}
105+
106+
success_params (\s+total_time="%{regex("(?<=\")[A-z0-9.µ]*(?=\")"):hugging_face_tgi.total_time}")?(\s+validation_time="%{regex("(?<=\")[A-z0-9.µ]*(?=\")"):hugging_face_tgi.validation_time}")?(\s+queue_time="%{regex("(?<=\")[A-z0-9.µ]*(?=\")"):hugging_face_tgi.queue_time}")?(\s+inference_time="%{regex("(?<=\")[A-z0-9.µ]*(?=\")"):hugging_face_tgi.inference_time}")?(\s+time_per_token="%{regex("(?<=\")[A-z0-9.µ]*(?=\")"):hugging_face_tgi.time_per_token}")?(\s+seed="%{regex("(?<=\")[A-z0-9.µ]*(?=\")"):hugging_face_tgi.seed}")?
107+
108+
log_tail %{word:hugging_face_tgi.component}(::%{word:hugging_face_tgi.sub_component})?:\s+(%{regex("[A-z0-9/\\.:]*(?=: )"):hugging_face_tgi.file}:\s+)?%{data:message}
109+
110+
general_params parameters="?%{regex(".*\\s+}"):hugging_face_tgi.parameters}
111+
112+
color (%{regex("\\[0-9]*m")})?
113+
matchRules: >-
114+
full_log
115+
%{tgi_date}\s+%{notSpace:status}\s+%{word:hugging_face_tgi.operation_type}\{%{general_params}"?%{success_params}\s*\}(:%{regex("[A-z0-9/\\.:]*(?=:
116+
)"):hugging_face_tgi.operation_sub_type})?:\s+%{log_tail}
117+
118+
119+
init_log %{tgi_date}\s+%{notSpace:status}\s+%{word:hugging_face_tgi.component}:\s+Args\s+\{%{data:hugging_face_tgi:keyvalue(": ","()\\[\\]",""," ,")}\s+\}
120+
121+
122+
short_log %{tgi_date}\s+%{notSpace:status}\s+(download:\s+)?%{log_tail}
123+
- type: status-remapper
124+
name: Status Remapper
125+
enabled: true
126+
sources:
127+
- status
128+
- type: date-remapper
129+
name: Date Remapper
130+
enabled: true
131+
sources:
132+
- date
133+
- type: attribute-remapper
134+
name: Span
135+
enabled: true
136+
sources:
137+
- span
138+
sourceType: attribute
139+
target: hugging_face_tgi
140+
targetType: attribute
141+
preserveSource: false
142+
overrideOnConflict: false
143+
- type: attribute-remapper
144+
name: Spans
145+
enabled: true
146+
sources:
147+
- spans
148+
sourceType: attribute
149+
target: hugging_face_tgi.spans
150+
targetType: attribute
151+
preserveSource: false
152+
overrideOnConflict: false
153+
- type: attribute-remapper
154+
name: Filename
155+
enabled: true
156+
sources:
157+
- filename
158+
sourceType: attribute
159+
target: hugging_face_tgi.filename
160+
targetType: attribute
161+
preserveSource: false
162+
overrideOnConflict: false
163+
- type: attribute-remapper
164+
name: Line number
165+
enabled: true
166+
sources:
167+
- line_number
168+
sourceType: attribute
169+
target: hugging_face_tgi.line_number
170+
targetType: attribute
171+
preserveSource: false
172+
overrideOnConflict: false
173+
- type: attribute-remapper
174+
name: Target
175+
enabled: true
176+
sources:
177+
- target
178+
sourceType: attribute
179+
target: hugging_face_tgi.target
180+
targetType: attribute
181+
preserveSource: false
182+
overrideOnConflict: false
183+
- type: message-remapper
184+
name: Message Remapper
185+
enabled: true
186+
sources:
187+
- message
188+
- fields.message
189+
- type: grok-parser
190+
name: JSON init
191+
enabled: true
192+
source: message
193+
samples:
194+
- |-
195+
Args {
196+
model_id: "teknium/OpenHermes-2.5-Mistral-7B",
197+
revision: None,
198+
validation_workers: 2,
199+
sharded: None,
200+
num_shard: None,
201+
quantize: None,
202+
speculate: None,
203+
dtype: None,
204+
kv_cache_dtype: None,
205+
trust_remote_code: false,
206+
max_concurrent_requests: 128,
207+
max_best_of: 2,
208+
max_stop_sequences: 4,
209+
max_top_n_tokens: 5,
210+
max_input_tokens: None,
211+
max_input_length: None,
212+
max_total_tokens: None,
213+
waiting_served_ratio: 0.3,
214+
max_batch_prefill_tokens: Some(
215+
512,
216+
),
217+
max_batch_total_tokens: None,
218+
max_waiting_tokens: 20,
219+
max_batch_size: None,
220+
cuda_graphs: None,
221+
hostname: "ip-172-31-21-18",
222+
port: 80,
223+
prometheus_port: 9000,
224+
shard_uds_path: "/tmp/text-generation-server",
225+
master_addr: "localhost",
226+
master_port: 29500,
227+
huggingface_hub_cache: None,
228+
weights_cache_override: None,
229+
disable_custom_kernels: false,
230+
cuda_memory_fraction: 1.0,
231+
rope_scaling: None,
232+
rope_factor: None,
233+
json_output: true,
234+
otlp_endpoint: None,
235+
otlp_service_name: "text-generation-inference.router",
236+
cors_allow_origin: [],
237+
api_key: None,
238+
watermark_gamma: None,
239+
watermark_delta: None,
240+
ngrok: false,
241+
ngrok_authtoken: None,
242+
ngrok_edge: None,
243+
tokenizer_config_path: None,
244+
disable_grammar_support: false,
245+
env: false,
246+
max_client_batch_size: 4,
247+
lora_adapters: None,
248+
usage_stats: On,
249+
payload_limit: 2000000,
250+
enable_prefill_logprobs: false,
251+
graceful_termination_timeout: 90,
252+
}
253+
grok:
254+
supportRules: ""
255+
matchRules: 'rule Args\s+\{\s+%{data:hugging_face_tgi:keyvalue(":
256+
","()\\[\\]",""," ,")}\s+\}'
257+
- type: grok-parser
258+
name: Parameters
259+
enabled: true
260+
source: hugging_face_tgi.parameters
261+
samples:
262+
- "GenerateParameters { best_of: None, temperature: None,
263+
repetition_penalty: None, frequency_penalty: None, top_k: None, top_p:
264+
None, typical_p: None, do_sample: false, max_new_tokens: Some(20),
265+
return_full_text: None, stop: [], truncate: None, watermark: false,
266+
details: false, decoder_input_details: false, seed: None,
267+
top_n_tokens: None, grammar: None, adapter_id: None }"
268+
grok:
269+
supportRules: ""
270+
matchRules: 'rule
271+
%{word:hugging_face_tgi.parameters.type}\s*\{\s+%{data:hugging_face_tgi.parameters:keyvalue(":
272+
","()\\[\\]",""," ,")}\s+\}'

0 commit comments

Comments
 (0)