Coroutines suspends

instrumentation/kotlin-coroutines-suspends/README.md

@@ -0,0 +1,29 @@
+Kotlin Coroutines Suspends Instrumentation
+===========================
+
+This instrumentation is used to cover Kotlin Coroutines versions 1.4.0 exclusive
+
+Provides instrumentation for Kotlin Suspend Functions.  It will not track internal Kotlin Coroutine Suspend functions.  Out of the box it will capture any Suspend Function outside of the internal Suspend Functions.   
+You can configure the agent to ignore certain suspend functions based on the classname or Continuation name using the actual value or a regular expression to ignore any suspend functions that match it.
+
+## Reporting of Suspend Functions
+The execution time of a suspend function will be captured with the metric(span) name that starts with Custom/Kotlin/Coroutines/SuspendFunction/ and a name.  Typically the name will be similar to this: Continuation at *classname*(*filename*.kt:*linenumber*)
+
+## Configuration
+The following configuration are based on a Kotlin item added to newrelic.yml.  The settings are dynamic and will change within a minute or so after saving newrelic.yml.
+
+### Suspends Ignores Configuration
+To stop tracking a Suspend whose metric name starts with "Custom/ContinuationWrapper/resumeWith/" use the remaining part of the metric name as the configuration or a regular expression that matches it and other Suspends that match, add a continuations item with the suspend functions to ignore.  
+Additional you can ignore Suspend functions based on the class name or you can use a regular expression to ignore Suspend functions from a class or package.
+
+**Example**
+
+  Kotlin:   
+    ignores:    
+      suspends: "Continuation at com.nrlabs.AsyncKt.main$task1(async.kt:11)"
+
+#### Notes on Regular Expressions
+This extension uses Java regular expression so it is recommended to consult a Java regular expression cheatsheet if you are not familar with it.   
+
+
+

instrumentation/kotlin-coroutines-suspends/build.gradle

@@ -0,0 +1,29 @@
+plugins {
+    id "org.jetbrains.kotlin.jvm"
+}
+
+dependencies {
+    implementation(project(":agent-bridge"))
+    implementation(project(":newrelic-agent"))
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.0")
+
+}
+
+jar {
+    manifest {
+        attributes 'Implementation-Title': 'com.newrelic.instrumentation.kotlin-coroutines-suspends'
+    }
+}
+
+verifyInstrumentation {
+    passes 'org.jetbrains.kotlinx:kotlinx-coroutines-core:[1.4.0,)'
+    passes 'org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:[1.4.0)'
+    excludeRegex '.*SNAPSHOT'
+    excludeRegex '.*alpha'
+    excludeRegex '.*Beta'
+    excludeRegex '.*-eap-.*'
+    excludeRegex '.*-native-.*'
+    excludeRegex '.*-M[0-9]'
+    excludeRegex '.*-rc'
+    excludeRegex '.*-RC'
+}

instrumentation/kotlin-coroutines-suspends/src/main/java/com/newrelic/instrumentation/kotlin/suspends/SuspendsUtils.java

@@ -0,0 +1,105 @@
+package com.newrelic.instrumentation.kotlin.suspends;
+
+import com.newrelic.agent.bridge.AgentBridge;
+import com.newrelic.agent.bridge.ExitTracer;
+import com.newrelic.agent.kotlincoroutines.KotlinCoroutinesService;
+import com.newrelic.agent.kotlincoroutines.SuspendsConfigListener;
+import com.newrelic.agent.service.ServiceFactory;
+import com.newrelic.agent.tracers.*;
+import com.newrelic.api.agent.NewRelic;
+import kotlin.coroutines.Continuation;
+import kotlinx.coroutines.AbstractCoroutine;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.logging.Level;
+import java.util.regex.Pattern;
+
+public class SuspendsUtils implements SuspendsConfigListener {
+
+    private static final HashSet<String> ignoredSuspends = new HashSet<>();
+    private static final HashSet<Pattern> ignoredSuspendsRegex = new HashSet<>();
+    public static final String CREATE_METHOD1 = "Continuation at kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt$createCoroutineUnintercepted$$inlined$createCoroutineFromSuspendFunction$IntrinsicsKt__IntrinsicsJvmKt$4";
+    public static final String CREATE_METHOD2 = "Continuation at kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt$createCoroutineUnintercepted$$inlined$createCoroutineFromSuspendFunction$IntrinsicsKt__IntrinsicsJvmKt$3";
+    private static final String CONT_LOC = "Continuation at";
+    public static String sub = "createCoroutineFromSuspendFunction";
+    public static final String KOTLIN_PACKAGE = "kotlin";
+    private static final String SUSPEND_FUNCTION_METRIC_NAME_PREFIX = "Custom/Kotlin/Coroutines/SuspendFunction/";
+
+    static {
+        KotlinCoroutinesService service = ServiceFactory.getKotlinCoroutinesService();
+        service.addSuspendsConfigListener(new SuspendsUtils());
+    }
+
+    private SuspendsUtils() {}
+
+    public static ExitTracer getSuspendTracer(Continuation<?> continuation) {
+        Class<?> clazz = continuation.getClass();
+        String className = clazz.getName();
+        // don't track suspend functions in internal Coroutines classes (i.e. starts with kotlin or kotlinx)
+        if(className.startsWith(KOTLIN_PACKAGE)) { return null; }
+        String continuationString = getContinuationString(continuation);
+        // ignore if can't determine continuation string
+        if(continuationString == null || continuationString.isEmpty()) { return null; }
+
+        for(Pattern ignoredSuspend : ignoredSuspendsRegex) {
+            if(ignoredSuspend.matcher(continuationString).matches() || ignoredSuspend.matcher(className).matches() ) {
+                return null;
+            }
+        }
+        if (ignoredSuspends.contains(continuationString) || ignoredSuspends.contains(className)) {
+            return null;
+        }
+        ClassMethodSignature signature = new ClassMethodSignature(clazz.getName(), "invokeSuspend", "(Ljava.lang.Object;)Ljava.lang.Object;");
+        int index = ClassMethodSignatures.get().getIndex(signature);
+        if(index == -1) {
+            index = ClassMethodSignatures.get().add(signature);
+        }
+
+        if(index >= 0) {
+            String metricName = SUSPEND_FUNCTION_METRIC_NAME_PREFIX + continuationString;
+            return AgentBridge.instrumentation.createTracer(continuation, index, metricName, DefaultTracer.DEFAULT_TRACER_FLAGS);
+        }
+        return null;
+    }
+
+    public static <T> String getContinuationString(Continuation<T> continuation) {
+        String contString = continuation.toString();
+
+        if(contString.equals(CREATE_METHOD1) || contString.equals(CREATE_METHOD2)) {
+            return sub;
+        }
+
+        if(contString.startsWith(CONT_LOC)) {
+            return contString;
+        }
+
+        if(continuation instanceof AbstractCoroutine) {
+            return ((AbstractCoroutine<?>)continuation).nameString$kotlinx_coroutines_core();
+        }
+
+        int index = contString.indexOf('@');
+        if(index > -1) {
+            return contString.substring(0, index);
+        }
+
+        return null;
+    }
+
+    @Override
+    public void configureSuspendsIgnores(String[] ignores, String[] ignoreRegexes) {
+        NewRelic.getAgent().getLogger().log(Level.FINE,"Will ignore Suspend Functions matching {0}", Arrays.toString(ignores));
+        NewRelic.getAgent().getLogger().log(Level.FINE,"Will ignore Suspend Functions matching regular expressions:  {0}", Arrays.toString(ignoreRegexes));
+
+        ignoredSuspends.clear();
+        if(ignores != null) {
+            ignoredSuspends.addAll(Arrays.asList(ignores));
+        }
+        ignoredSuspendsRegex.clear();
+        if(ignoreRegexes != null) {
+            for(String regex : ignoreRegexes) {
+                ignoredSuspendsRegex.add(Pattern.compile(regex));
+            }
+        }
+    }
+}

instrumentation/kotlin-coroutines-suspends/src/main/java/kotlin/coroutines/jvm/internal/BaseContinuationImpl_Instrumentation.java

@@ -0,0 +1,30 @@
+package kotlin.coroutines.jvm.internal;
+
+import com.newrelic.agent.bridge.ExitTracer;
+import com.newrelic.api.agent.weaver.MatchType;
+import com.newrelic.api.agent.weaver.Weave;
+import com.newrelic.api.agent.weaver.Weaver;
+import com.newrelic.instrumentation.kotlin.suspends.SuspendsUtils;
+import kotlin.coroutines.Continuation;
+
+@Weave(type = MatchType.BaseClass, originalName = "kotlin.coroutines.jvm.internal.BaseContinuationImpl")
+public abstract class BaseContinuationImpl_Instrumentation implements Continuation<Object> {
+
+	protected Object invokeSuspend(Object result) {
+		ExitTracer tracer = SuspendsUtils.getSuspendTracer(this);
+        Object value = null;
+        try {
+            value = Weaver.callOriginal();
+        } catch (Exception e) {
+			if(tracer != null) {
+				tracer.finish(e);
+			}
+			throw e;
+        }
+        if (tracer != null) {
+			tracer.finish(0, value);
+		}
+		return value;
+	}
+
+}

newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java

@@ -267,8 +267,8 @@ public class AgentConfigImpl extends BaseConfig implements AgentConfig {
     private final JarCollectorConfig jarCollectorConfig;
     private final JfrConfig jfrConfig;
     private final JmxConfig jmxConfig;
-    private final KeyTransactionConfig keyTransactionConfig;
     private final KotlinCoroutinesConfig kotlinCoroutinesConfig;
+    private final KeyTransactionConfig keyTransactionConfig;
     private final LabelsConfig labelsConfig;
     private final NormalizationRuleConfig normalizationRuleConfig;
     private final ReinstrumentConfig reinstrumentConfig;

newrelic-agent/src/main/java/com/newrelic/agent/config/KotlinCoroutinesConfig.java

@@ -14,6 +14,10 @@ public interface KotlinCoroutinesConfig {
 
     public String[] getIgnoredRegexDispatched();
 
+    public String[] getIgnoredSuspends();
+
+    public String[] getIgnoredRegexSuspends();
+
     public boolean isDelayedEnabled();
 
 }

newrelic-agent/src/main/java/com/newrelic/agent/config/KotlinCoroutinesConfigImpl.java

@@ -22,6 +22,8 @@ public class KotlinCoroutinesConfigImpl extends BaseConfig implements KotlinCoro
     private String[] ignoredRegexContinuations = null;
     private String[] ignoredRegexScopes = null;
     private String[] ignoredRegexDispatched = null;
+    private String[] ignoredSuspends = null;
+    private String[] ignoredRegexSuspends = null;
     private boolean delayedEnabled = true;
 
     public KotlinCoroutinesConfigImpl(Map<String, Object> props) {
@@ -112,6 +114,12 @@ public String[] getIgnoredRegexDispatched() {
         return ignoredRegexDispatched;
     }
 
+    @Override
+    public String[] getIgnoredSuspends() { return ignoredSuspends;}
+
+    @Override
+    public String[] getIgnoredRegexSuspends() { return ignoredRegexSuspends;}
+
     @Override
     public boolean isDelayedEnabled() {
         return delayedEnabled;

newrelic-agent/src/main/java/com/newrelic/agent/kotlincoroutines/KotlinCoroutinesService.java

@@ -13,6 +13,7 @@
 public class KotlinCoroutinesService extends AbstractService implements AgentConfigListener {
 
     private final Set<CoroutineConfigListener> listeners = new LinkedHashSet<>();
+    private final Set<SuspendsConfigListener> suspendListeners = new LinkedHashSet<>();
     private KotlinCoroutinesConfig coroutinesConfig;
 
     public KotlinCoroutinesService(KotlinCoroutinesConfig coroutinesConfig) {
@@ -31,6 +32,13 @@ public void addCoroutineConfigListener(CoroutineConfigListener listener) {
         }
     }
 
+    public void addSuspendsConfigListener(SuspendsConfigListener listener) {
+        if(listener != null) {
+            suspendListeners.add(listener);
+            listener.configureSuspendsIgnores(coroutinesConfig.getIgnoredSuspends(),coroutinesConfig.getIgnoredRegexSuspends());
+        }
+    }
+
     @Override
     protected void doStart() throws Exception {
         for (CoroutineConfigListener listener : listeners) {
@@ -61,6 +69,9 @@ public void configChanged(String appName, AgentConfig agentConfig) {
             listener.configureDispatchedTasksIgnores(coroutinesConfig.getIgnoredDispatched(),coroutinesConfig.getIgnoredRegexDispatched());
             listener.configureContinuationIgnores(coroutinesConfig.getIgnoredContinuations(),coroutinesConfig.getIgnoredRegExContinuations());
         }
+        for(SuspendsConfigListener listener : suspendListeners) {
+            listener.configureSuspendsIgnores(coroutinesConfig.getIgnoredSuspends(),coroutinesConfig.getIgnoredRegexSuspends());
+        }
 
     }
 }

newrelic-agent/src/main/java/com/newrelic/agent/kotlincoroutines/SuspendsConfigListener.java

@@ -0,0 +1,6 @@
+package com.newrelic.agent.kotlincoroutines;
+
+public interface SuspendsConfigListener {
+
+    void configureSuspendsIgnores(String[] ignores, String[] ignoresRegex);
+}

newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceFactory.java

@@ -217,4 +217,5 @@ public static ExpirationService getExpirationService() {
     }
 
     public static KotlinCoroutinesService getKotlinCoroutinesService() { return SERVICE_MANAGER.getKotlinCoroutinesService(); }
+
 }

settings.gradle

@@ -270,6 +270,7 @@ include 'instrumentation:kafka-streams-spans-2.1.0'
 include 'instrumentation:kafka-streams-spans-2.6.0'
 include 'instrumentation:kafka-streams-spans-3.2.0'
 include 'instrumentation:kafka-streams-spans-3.7.0'
+include 'instrumentation:kotlin-coroutines-suspends'
 include 'instrumentation:kotlin-coroutines-1.4'
 include 'instrumentation:kotlin-coroutines-1.7'
 include 'instrumentation:kotlin-coroutines-1.9'