Skip to content

Commit 9e73c68

Browse files
waldekmastykarzwaldekmastykarz
authored
Extends Dev Proxy with a language model. Closes #745 (#754)
1 parent af65ca6 commit 9e73c68

File tree

11 files changed

+228
-24
lines changed

11 files changed

+228
-24
lines changed

‎dev-proxy-abstractions/ILanguageModelClient.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
public interface ILanguageModelClient
2+
{
3+
Task<string?> GenerateCompletion(string prompt);
4+
}

‎dev-proxy-abstractions/PluginEvents.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public interface IProxyContext
1515
{
1616
IProxyConfiguration Configuration { get; }
1717
X509Certificate2? Certificate { get; }
18+
ILanguageModelClient LanguageModelClient { get; }
1819
}
1920

2021
public class ThrottlerInfo

‎dev-proxy-plugins/RequestLogs/OpenApiSpecGeneratorPlugin.cs‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -298,14 +298,14 @@ public override void Register()
298298
PluginEvents.AfterRecordingStop += AfterRecordingStop;
299299
}
300300

301-
private Task AfterRecordingStop(object? sender, RecordingArgs e)
301+
private async Task AfterRecordingStop(object? sender, RecordingArgs e)
302302
{
303303
Logger.LogInformation("Creating OpenAPI spec from recorded requests...");
304304

305305
if (!e.RequestLogs.Any())
306306
{
307307
Logger.LogDebug("No requests to process");
308-
return Task.CompletedTask;
308+
return;
309309
}
310310

311311
var openApiDocs = new List<OpenApiDocument>();
@@ -334,7 +334,16 @@ request.Context is null ||
334334
var pathItem = GetOpenApiPathItem(request.Context.Session);
335335
var parametrizedPath = ParametrizePath(pathItem, request.Context.Session.HttpClient.Request.RequestUri);
336336
var operationInfo = pathItem.Operations.First();
337-
operationInfo.Value.OperationId = GetOperationId(operationInfo.Key.ToString(), parametrizedPath);
337+
operationInfo.Value.OperationId = await GetOperationId(
338+
operationInfo.Key.ToString(),
339+
request.Context.Session.HttpClient.Request.RequestUri.GetLeftPart(UriPartial.Authority),
340+
parametrizedPath
341+
);
342+
operationInfo.Value.Description = await GetOperationDescription(
343+
operationInfo.Key.ToString(),
344+
request.Context.Session.HttpClient.Request.RequestUri.GetLeftPart(UriPartial.Authority),
345+
parametrizedPath
346+
);
338347
AddOrMergePathItem(openApiDocs, pathItem, request.Context.Session.HttpClient.Request.RequestUri, parametrizedPath);
339348
}
340349
catch (Exception ex)
@@ -370,8 +379,6 @@ request.Context is null ||
370379
// store the generated OpenAPI specs in the global data
371380
// for use by other plugins
372381
e.GlobalData[GeneratedOpenApiSpecsKey] = generatedOpenApiSpecs;
373-
374-
return Task.CompletedTask;
375382
}
376383

377384
/**
@@ -441,9 +448,18 @@ private string GetLastNonTokenSegment(string[] segments)
441448
return "item";
442449
}
443450

444-
private string GetOperationId(string method, string parametrizedPath)
451+
private async Task<string> GetOperationId(string method, string serverUrl, string parametrizedPath)
452+
{
453+
var prompt = $"For the specified request, generate an operation ID, compatible with an OpenAPI spec. Respond with just the ID in plain-text format. For example, for request such as `GET https://api.contoso.com/books/{{books-id}}` you return `getBookById`. For a request like `GET https://api.contoso.com/books/{{books-id}}/authors` you return `getAuthorsForBookById`. Request: {method.ToUpper()} {serverUrl}{parametrizedPath}";
454+
var id = await Context.LanguageModelClient.GenerateCompletion(prompt);
455+
return id ?? $"{method}{parametrizedPath.Replace('/', '.')}";
456+
}
457+
458+
private async Task<string> GetOperationDescription(string method, string serverUrl, string parametrizedPath)
445459
{
446-
return $"{method}{parametrizedPath.Replace('/', '.')}";
460+
var prompt = $"You're an expert in OpenAPI. You help developers build great OpenAPI specs for use with LLMs. For the specified request, generate a one-sentence description. Respond with just the description. For example, for a request such as `GET https://api.contoso.com/books/{{books-id}}` you return `Get a book by ID`. Request: {method.ToUpper()} {serverUrl}{parametrizedPath}";
461+
var description = await Context.LanguageModelClient.GenerateCompletion(prompt);
462+
return description ?? $"{method} {parametrizedPath}";
447463
}
448464

449465
/**
@@ -472,6 +488,7 @@ private OpenApiPathItem GetOpenApiPathItem(SessionEventArgs session)
472488
};
473489
var operation = new OpenApiOperation
474490
{
491+
// will be replaced later after the path has been parametrized
475492
Description = $"{method} {resource}",
476493
// will be replaced later after the path has been parametrized
477494
OperationId = $"{method}.{resource}"

‎dev-proxy/LanguageModel/LanguageModelClient.cs‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System.Diagnostics;
5+
using System.Net.Http.Json;
6+
using Microsoft.Extensions.Logging;
7+
8+
namespace Microsoft.DevProxy.LanguageModel;
9+
10+
public class LanguageModelClient(LanguageModelConfiguration? configuration, ILogger logger) : ILanguageModelClient
11+
{
12+
private readonly LanguageModelConfiguration? _configuration = configuration;
13+
private readonly ILogger _logger = logger;
14+
private bool? _lmAvailable;
15+
private Dictionary<string, string> _cache = new();
16+
17+
public async Task<string?> GenerateCompletion(string prompt)
18+
{
19+
using var scope = _logger.BeginScope("Language Model");
20+
21+
if (_configuration == null || !_configuration.Enabled)
22+
{
23+
// LM turned off. Nothing to do, nothing to report
24+
return null;
25+
}
26+
27+
if (!_lmAvailable.HasValue)
28+
{
29+
if (string.IsNullOrEmpty(_configuration.Url))
30+
{
31+
_logger.LogError("URL is not set. Language model will be disabled");
32+
_lmAvailable = false;
33+
return null;
34+
}
35+
36+
if (string.IsNullOrEmpty(_configuration.Model))
37+
{
38+
_logger.LogError("Model is not set. Language model will be disabled");
39+
_lmAvailable = false;
40+
return null;
41+
}
42+
43+
_logger.LogDebug("Checking availability...");
44+
_lmAvailable = await IsLmAvailable();
45+
46+
// we want to log this only once
47+
if (!_lmAvailable.Value)
48+
{
49+
_logger.LogError("{model} at {url} is not available", _configuration.Model, _configuration.Url);
50+
return null;
51+
}
52+
}
53+
54+
if (!_lmAvailable.Value)
55+
{
56+
return null;
57+
}
58+
59+
if (_configuration.CacheResponses && _cache.TryGetValue(prompt, out var cachedResponse))
60+
{
61+
_logger.LogDebug("Returning cached response for prompt: {prompt}", prompt);
62+
return cachedResponse;
63+
}
64+
65+
var response = await GenerateCompletionInternal(prompt);
66+
if (response == null)
67+
{
68+
return null;
69+
}
70+
if (response.Error is not null)
71+
{
72+
_logger.LogError(response.Error);
73+
return null;
74+
}
75+
else
76+
{
77+
if (_configuration.CacheResponses && response.Response is not null)
78+
{
79+
_cache[prompt] = response.Response;
80+
}
81+
82+
return response.Response;
83+
}
84+
}
85+
86+
private async Task<LanguageModelResponse?> GenerateCompletionInternal(string prompt)
87+
{
88+
Debug.Assert(_configuration != null, "Configuration is null");
89+
90+
try
91+
{
92+
using var client = new HttpClient();
93+
var url = $"{_configuration.Url}/api/generate";
94+
_logger.LogDebug("Requesting completion. Prompt: {prompt}", prompt);
95+
96+
var response = await client.PostAsJsonAsync(url,
97+
new
98+
{
99+
prompt,
100+
model = _configuration.Model,
101+
stream = false
102+
}
103+
);
104+
return await response.Content.ReadFromJsonAsync<LanguageModelResponse>();
105+
}
106+
catch (Exception ex)
107+
{
108+
_logger.LogError(ex, "Failed to generate completion");
109+
return null;
110+
}
111+
}
112+
113+
private async Task<bool> IsLmAvailable()
114+
{
115+
Debug.Assert(_configuration != null, "Configuration is null");
116+
117+
_logger.LogDebug("Checking LM availability at {url}...", _configuration.Url);
118+
119+
try
120+
{
121+
// check if lm is on
122+
using var client = new HttpClient();
123+
var response = await client.GetAsync(_configuration.Url);
124+
_logger.LogDebug("Response: {response}", response.StatusCode);
125+
126+
if (!response.IsSuccessStatusCode)
127+
{
128+
return false;
129+
}
130+
131+
var testCompletion = await GenerateCompletionInternal("Are you there? Reply with a yes or no.");
132+
if (testCompletion?.Error is not null)
133+
{
134+
_logger.LogError("Error: {error}", testCompletion.Error);
135+
return false;
136+
}
137+
138+
return true;
139+
}
140+
catch (Exception ex)
141+
{
142+
_logger.LogError(ex, "Couldn't reach language model at {url}", _configuration.Url);
143+
return false;
144+
}
145+
}
146+
}

‎dev-proxy/LanguageModel/LanguageModelConfiguration.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Microsoft.DevProxy.LanguageModel;
5+
6+
public class LanguageModelConfiguration
7+
{
8+
public bool Enabled { get; set; } = false;
9+
// default Ollama URL
10+
public string? Url { get; set; } = "http://localhost:11434";
11+
public string? Model { get; set; } = "phi3";
12+
public bool CacheResponses { get; set; } = true;
13+
}

‎dev-proxy/LanguageModel/LanguageModelResponse.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Microsoft.DevProxy.LanguageModel;
5+
6+
public class LanguageModelResponse
7+
{
8+
public string? Response { get; init; }
9+
public string? Error { get; init; }
10+
}

‎dev-proxy/Logging/ProxyConsoleFormatter.cs‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -102,34 +102,39 @@ private void LogMessage<TState>(in LogEntry<TState> logEntry, IExternalScopeProv
102102
var logLevel = logEntry.LogLevel;
103103
var message = logEntry.Formatter(logEntry.State, logEntry.Exception);
104104

105-
WriteMessageBoxedWithInvertedLabels(message, logLevel, textWriter);
105+
WriteMessageBoxedWithInvertedLabels(message, logLevel, scopeProvider, textWriter);
106106

107107
if (logEntry.Exception is not null)
108108
{
109109
textWriter.Write($" Exception Details: {logEntry.Exception}");
110110
}
111111

112-
if (_options.IncludeScopes && scopeProvider is not null)
113-
{
114-
scopeProvider.ForEachScope((scope, state) =>
115-
{
116-
state.Write(" => ");
117-
state.Write(scope);
118-
}, textWriter);
119-
}
120112
textWriter.WriteLine();
121113
}
122114

123-
private void WriteMessageBoxedWithInvertedLabels(string? message, LogLevel logLevel, TextWriter textWriter)
115+
private void WriteMessageBoxedWithInvertedLabels(string? message, LogLevel logLevel, IExternalScopeProvider? scopeProvider, TextWriter textWriter)
124116
{
117+
if (message is null)
118+
{
119+
return;
120+
}
121+
125122
var label = GetLogLevelString(logLevel);
126123
var (bgColor, fgColor) = GetLogLevelColor(logLevel);
127124

128-
if (message is not null)
125+
textWriter.WriteColoredMessage($" {label} ", bgColor, fgColor);
126+
textWriter.Write($"{labelSpacing}{_boxSpacing}{(logLevel == LogLevel.Debug ? $"[{DateTime.Now}] " : "")}");
127+
128+
if (_options.IncludeScopes && scopeProvider is not null)
129129
{
130-
textWriter.WriteColoredMessage($" {label} ", bgColor, fgColor);
131-
textWriter.Write($"{labelSpacing}{_boxSpacing}{(logLevel == LogLevel.Debug ? $"[{DateTime.Now}] " : "")}{message}");
130+
scopeProvider.ForEachScope((scope, state) =>
131+
{
132+
state.Write(scope);
133+
state.Write(": ");
134+
}, textWriter);
132135
}
136+
137+
textWriter.Write(message);
133138
}
134139

135140
private void WriteLogMessageBoxedWithInvertedLabels(string[] message, MessageType messageType, TextWriter textWriter, bool lastMessage = false)

‎dev-proxy/Program.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using Microsoft.DevProxy;
55
using Microsoft.DevProxy.Abstractions;
6+
using Microsoft.DevProxy.LanguageModel;
67
using Microsoft.DevProxy.Logging;
78
using Microsoft.Extensions.Logging;
89
using Microsoft.Extensions.Logging.Console;
@@ -20,7 +21,9 @@ ILogger BuildLogger()
2021
options.FormatterName = "devproxy";
2122
options.LogToStandardErrorThreshold = LogLevel.Warning;
2223
})
23-
.AddConsoleFormatter<ProxyConsoleFormatter, ConsoleFormatterOptions>()
24+
.AddConsoleFormatter<ProxyConsoleFormatter, ConsoleFormatterOptions>(options => {
25+
options.IncludeScopes = true;
26+
})
2427
.AddRequestLogger(pluginEvents)
2528
.SetMinimumLevel(ProxyHost.LogLevel ?? ProxyCommandHandler.Configuration.LogLevel);
2629
});
@@ -29,7 +32,8 @@ ILogger BuildLogger()
2932

3033
var logger = BuildLogger();
3134

32-
IProxyContext context = new ProxyContext(ProxyCommandHandler.Configuration, ProxyEngine.Certificate);
35+
var lmClient = new LanguageModelClient(ProxyCommandHandler.Configuration.LanguageModel, logger);
36+
IProxyContext context = new ProxyContext(ProxyCommandHandler.Configuration, ProxyEngine.Certificate, lmClient);
3337
ProxyHost proxyHost = new();
3438

3539
// this is where the root command is created which contains all commands and subcommands

‎dev-proxy/ProxyCommandHandler.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using Microsoft.DevProxy.Abstractions;
77
using System.CommandLine;
88
using System.CommandLine.Invocation;
9+
using Microsoft.DevProxy.LanguageModel;
910

1011
namespace Microsoft.DevProxy;
1112

‎dev-proxy/ProxyConfiguration.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Runtime.Serialization;
55
using System.Text.Json.Serialization;
66
using Microsoft.DevProxy.Abstractions;
7+
using Microsoft.DevProxy.LanguageModel;
78
using Microsoft.Extensions.Logging;
89

910
namespace Microsoft.DevProxy;
@@ -36,6 +37,6 @@ public class ProxyConfiguration : IProxyConfiguration
3637
public string ConfigFile { get; set; } = "devproxyrc.json";
3738
[JsonConverter(typeof(JsonStringEnumConverter))]
3839
public ReleaseType NewVersionNotification { get; set; } = ReleaseType.Stable;
40+
public LanguageModelConfiguration? LanguageModel { get; set; }
3941
public MockRequestHeader[]? FilterByHeaders { get; set; }
4042
}
41-

0 commit comments

Comments
 (0)