Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
/*
* (c) Copyright 2025 Palantir Technologies Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.palantir.deadlines;

import com.google.common.annotations.VisibleForTesting;
import com.palantir.deadlines.DeadlineMetrics.Expired_Cause;
import com.palantir.deadlines.DeadlineMetrics.Expired_Intent;
import com.palantir.tritium.metrics.registry.SharedTaggedMetricRegistries;

/**
* Provides the ability to observe the number of deadline expirations that happen between points in time.
* <p/>
* Usage example:
* <pre>
* ExpirationObservation start = ExpirationObservation.start();
* // ... operation that may cause deadline expiration ...
* ExpirationObservation end = start.observeFrom();
* if (end.totalExpirations() > 0) {
* // at least one deadline expiration happened
* }
* </pre>
*
* Calling {@link #start()} will create a new observation with no recorded expirations.
*
* Calling {@link #observeFrom()} will return an observation comparing the current point in time to the count of
* expirations when the original observation was created.
*
* Calling {@link #totalExpirations()} will return the number of deadline expirations that occurred
* between two observations.
*/
public final class ExpirationObservation {

private static final DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());

private final MeterValues start;
private final MeterValues end;

private ExpirationObservation(MeterValues start, MeterValues end) {
this.start = start;
this.end = end;
}

/**
* Begin a new observation.
*/
public static ExpirationObservation start() {
MeterValues now = readMeters();
return new ExpirationObservation(now, now);
}

/**
* Return a new observation with the number of deadline expirations that have occurred since this observation was
* initialized.
*
* The current observation should have been previously initialized either via a call to this method, or
* a call to {@link #start()},
*/
public ExpirationObservation observeFrom() {
MeterValues now = readMeters();
return new ExpirationObservation(end, now);
}

/**
* Return the total number of expirations that have occurred since this observation was initialized.
*/
public long totalExpirations() {
return nExternalPropagate()
+ nExternalPropagateAlreadyExpired()
+ nExternalIgnore()
+ nInternalPropagate()
+ nInternalPropagateAlreadyExpired()
+ nInternalIgnore();
}

@VisibleForTesting
long totalWithExternalCause() {
return nExternalPropagate() + nExternalPropagateAlreadyExpired() + nExternalIgnore();
}

@VisibleForTesting
long totalWithInternalCause() {
return nInternalPropagate() + nInternalPropagateAlreadyExpired() + nInternalIgnore();
}

@VisibleForTesting
long totalWithPropagateIntent() {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could make this (or all of these) public to allow consumers to e.g. observe when a deadline expiration would throw an exception if enforcement is enabled.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What are your thoughts on something like #95? I think it's a bit simpler: I'm proposing we just expose the deadline metric values and provide a small utility to compute diffs between the metric value observations.

return nExternalPropagate() + nInternalPropagate();
}

@VisibleForTesting
long totalWithPropagateAlreadyExpiredIntent() {
return nExternalPropagateAlreadyExpired() + nInternalPropagateAlreadyExpired();
}

@VisibleForTesting
long totalWithIgnoreIntent() {
return nExternalIgnore() + nInternalIgnore();
}

@VisibleForTesting
long nExternalPropagate() {
return end.externalPropagate - start.externalPropagate;
}

@VisibleForTesting
long nExternalPropagateAlreadyExpired() {
return end.externalPropagateAlreadyExpired - start.externalPropagateAlreadyExpired;
}

@VisibleForTesting
long nExternalIgnore() {
return end.externalIgnore - start.externalIgnore;
}

@VisibleForTesting
long nInternalPropagate() {
return end.internalPropagate - start.internalPropagate;
}

@VisibleForTesting
long nInternalPropagateAlreadyExpired() {
return end.internalPropagateAlreadyExpired - start.internalPropagateAlreadyExpired;
}

@VisibleForTesting
long nInternalIgnore() {
return end.internalIgnore - start.internalIgnore;
}

private static MeterValues readMeters() {
long externalPropagate = getCountFor(Expired_Cause.EXTERNAL, Expired_Intent.PROPAGATE);
long externalPropagateAlreadyExpired =
getCountFor(Expired_Cause.EXTERNAL, Expired_Intent.PROPAGATE_ALREADY_EXPIRED);
long externalIgnore = getCountFor(Expired_Cause.EXTERNAL, Expired_Intent.IGNORE);
long internalPropagate = getCountFor(Expired_Cause.INTERNAL, Expired_Intent.PROPAGATE);
long internalPropagateAlreadyExpired =
getCountFor(Expired_Cause.INTERNAL, Expired_Intent.PROPAGATE_ALREADY_EXPIRED);
long internalIgnore = getCountFor(Expired_Cause.INTERNAL, Expired_Intent.IGNORE);

return new MeterValues(
externalPropagate,
externalPropagateAlreadyExpired,
externalIgnore,
internalPropagate,
internalPropagateAlreadyExpired,
internalIgnore);
}

private static long getCountFor(Expired_Cause cause, Expired_Intent intent) {
return metrics.expired().cause(cause).intent(intent).build().getCount();
}

private record MeterValues(
long externalPropagate,
long externalPropagateAlreadyExpired,
long externalIgnore,
long internalPropagate,
long internalPropagateAlreadyExpired,
long internalIgnore) {}
}
106 changes: 36 additions & 70 deletions deadlines/src/test/java/com/palantir/deadlines/DeadlinesTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,12 @@
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import com.codahale.metrics.Meter;
import com.palantir.deadlines.DeadlineMetrics.Expired_Cause;
import com.palantir.deadlines.DeadlineMetrics.Expired_Intent;
import com.palantir.deadlines.Deadlines.Enforcement;
import com.palantir.deadlines.Deadlines.RequestDecodingAdapter;
import com.palantir.deadlines.Deadlines.RequestEncodingAdapter;
import com.palantir.tracing.CloseableSpan;
import com.palantir.tracing.CloseableTracer;
import com.palantir.tracing.DetachedSpan;
import com.palantir.tritium.metrics.registry.SharedTaggedMetricRegistries;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -335,23 +331,15 @@ public void test_encode_to_request_expiration_external_deadline() {
Optional<Duration> remaining = Deadlines.getRemainingDeadline();
assertThat(remaining).hasValueSatisfying(d -> assertThat(d).isEqualTo(Duration.ZERO));

DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
Meter externalMeter = metrics.expired()
.cause(Expired_Cause.EXTERNAL)
.intent(Expired_Intent.PROPAGATE)
.build();
Meter internalMeter = metrics.expired()
.cause(Expired_Cause.INTERNAL)
.intent(Expired_Intent.PROPAGATE)
.build();
long originalExternalValue = externalMeter.getCount();
long originalInternalValue = internalMeter.getCount();
ExpirationObservation start = ExpirationObservation.start();

Map<String, String> outbound = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound, DummyRequestEncoder.INSTANCE);

assertThat(externalMeter.getCount()).isGreaterThan(originalExternalValue);
assertThat(internalMeter.getCount()).isEqualTo(originalInternalValue);
ExpirationObservation end = start.observeFrom();

assertThat(end.nExternalPropagate()).isGreaterThan(0);
assertThat(end.nInternalPropagate()).isEqualTo(0);
}
}

Expand All @@ -371,23 +359,15 @@ public void test_encode_to_request_expiration_internal_deadline() {
Optional<Duration> remaining = Deadlines.getRemainingDeadline();
assertThat(remaining).hasValueSatisfying(d -> assertThat(d).isEqualTo(Duration.ZERO));

DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
Meter externalMeter = metrics.expired()
.cause(Expired_Cause.EXTERNAL)
.intent(Expired_Intent.PROPAGATE)
.build();
Meter internalMeter = metrics.expired()
.cause(Expired_Cause.INTERNAL)
.intent(Expired_Intent.PROPAGATE)
.build();
long originalExternalValue = externalMeter.getCount();
long originalInternalValue = internalMeter.getCount();
ExpirationObservation start = ExpirationObservation.start();

Map<String, String> outbound = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound, DummyRequestEncoder.INSTANCE);

assertThat(internalMeter.getCount()).isGreaterThan(originalInternalValue);
assertThat(externalMeter.getCount()).isEqualTo(originalExternalValue);
ExpirationObservation end = start.observeFrom();

assertThat(end.nInternalPropagate()).isGreaterThan(0);
assertThat(end.nExternalPropagate()).isEqualTo(0);
}
}

Expand All @@ -407,35 +387,28 @@ public void disabled_propagation_reports_metrics_on_expiration() {
Optional<Duration> remaining = Deadlines.getRemainingDeadline();
assertThat(remaining).hasValueSatisfying(d -> assertThat(d).isEqualTo(Duration.ZERO));

DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
Meter externalMeterWillPropagate = metrics.expired()
.cause(Expired_Cause.EXTERNAL)
.intent(Expired_Intent.PROPAGATE)
.build();
Meter externalMeterWontPropagate = metrics.expired()
.cause(Expired_Cause.EXTERNAL)
.intent(Expired_Intent.IGNORE)
.build();
long originalWillPropagateValue = externalMeterWillPropagate.getCount();
long originalWontPropagateValue = externalMeterWontPropagate.getCount();
ExpirationObservation start = ExpirationObservation.start();

// first request is allowed to propagate the deadline, make sure the correct meter is marked
Map<String, String> outbound1 = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound1, DummyRequestEncoder.INSTANCE);
assertThat(externalMeterWillPropagate.getCount()).isGreaterThan(originalWillPropagateValue);
assertThat(externalMeterWontPropagate.getCount()).isEqualTo(originalWontPropagateValue);

originalWillPropagateValue = externalMeterWillPropagate.getCount();
originalWontPropagateValue = externalMeterWontPropagate.getCount();
ExpirationObservation obs1 = start.observeFrom();

assertThat(obs1.nExternalPropagate()).isGreaterThan(0);
assertThat(obs1.nExternalIgnore()).isEqualTo(0);

// and now disable propagation
Deadlines.disableFurtherDeadlinePropagation();

// second request is not allowed to propagate the deadline, make sure the correct meter is marked
Map<String, String> outbound2 = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound2, DummyRequestEncoder.INSTANCE);
assertThat(externalMeterWontPropagate.getCount()).isGreaterThan(originalWontPropagateValue);
assertThat(externalMeterWillPropagate.getCount()).isEqualTo(originalWillPropagateValue);

ExpirationObservation obs2 = obs1.observeFrom();

assertThat(obs2.nExternalIgnore()).isGreaterThan(0);
assertThat(obs2.nExternalPropagate()).isEqualTo(0);
}
}

Expand Down Expand Up @@ -481,29 +454,20 @@ public void multihop_expired_received_deadline_marks_propagate_already_expired_m
DetachedSpan server2Span = DetachedSpan.start("server2");

try (CloseableSpan ignored = server1Span.attach()) {
DeadlineMetrics metrics = DeadlineMetrics.of(SharedTaggedMetricRegistries.getSingleton());
Meter expiredMeterPropagateIntent = metrics.expired()
.cause(Expired_Cause.EXTERNAL)
.intent(Expired_Intent.PROPAGATE)
.build();
Meter expiredMeterPropagateAlreadyExpiredIntent = metrics.expired()
.cause(Expired_Cause.EXTERNAL)
.intent(Expired_Intent.PROPAGATE_ALREADY_EXPIRED)
.build();

long expiredMeterPropagateIntentValue = expiredMeterPropagateIntent.getCount();
long expiredMeterPropagateAlreadyExpiredIntentValue = expiredMeterPropagateAlreadyExpiredIntent.getCount();
ExpirationObservation start = ExpirationObservation.start();

// the first hop receives a valid, non-zero deadline on the wire
Map<String, String> request = new HashMap<>();
Duration providedDeadline = Duration.ofMillis(1);
request.put(
DeadlinesHttpHeaders.EXPECT_WITHIN, Deadlines.durationToHeaderValue(providedDeadline.toNanos()));
Deadlines.parseFromRequest(Optional.empty(), request, DummyRequestDecoder.INSTANCE, Enforcement.DISABLE);

// nothing yet...
assertThat(expiredMeterPropagateIntent.getCount()).isEqualTo(expiredMeterPropagateIntentValue);
assertThat(expiredMeterPropagateAlreadyExpiredIntent.getCount())
.isEqualTo(expiredMeterPropagateAlreadyExpiredIntentValue);
ExpirationObservation obs1 = start.observeFrom();

assertThat(obs1.nExternalPropagate()).isEqualTo(0);
assertThat(obs1.nExternalPropagateAlreadyExpired()).isEqualTo(0);

// force expiration within the first hop
clock.elapsed += 2_000_000;
Expand All @@ -515,23 +479,25 @@ public void multihop_expired_received_deadline_marks_propagate_already_expired_m
assertThat(outbound1.get(DeadlinesHttpHeaders.EXPECT_WITHIN))
.isNotNull()
.isEqualTo("0");
assertThat(expiredMeterPropagateIntent.getCount()).isGreaterThan(expiredMeterPropagateIntentValue);
assertThat(expiredMeterPropagateAlreadyExpiredIntent.getCount()).isZero();

ExpirationObservation obs2 = obs1.observeFrom();

assertThat(obs2.nExternalPropagate()).isGreaterThan(0);
assertThat(obs2.nExternalPropagateAlreadyExpired()).isEqualTo(0);

// next hop parses a zero deadline
try (CloseableSpan ignored2 = server2Span.attach()) {
expiredMeterPropagateIntentValue = expiredMeterPropagateIntent.getCount();
Deadlines.parseFromRequest(
Optional.empty(), outbound1, DummyRequestDecoder.INSTANCE, Enforcement.DISABLE);
// sending another request when the deadline has already expired should
// mark the meter with the "propagate-already-expired" intent
expiredMeterPropagateAlreadyExpiredIntentValue = expiredMeterPropagateAlreadyExpiredIntent.getCount();
Map<String, String> outbound2 = new HashMap<>();
Deadlines.encodeToRequest(Duration.ofSeconds(10), outbound2, DummyRequestEncoder.INSTANCE);
assertThat(expiredMeterPropagateAlreadyExpiredIntent.getCount())
.isGreaterThan(expiredMeterPropagateAlreadyExpiredIntentValue);
// meter with the "propagate" intent is unchanged
assertThat(expiredMeterPropagateIntent.getCount()).isEqualTo(expiredMeterPropagateIntentValue);

ExpirationObservation obs3 = obs2.observeFrom();

assertThat(obs3.nExternalPropagateAlreadyExpired()).isGreaterThan(0);
assertThat(obs3.nExternalPropagate()).isEqualTo(0);
}
}
}
Expand Down
Loading