Extends Proxy to support other URLs. Closes #54 (#87)

Author: waldekmastykarz

msgraph-developer-proxy/ChaosEngine.cs

@@ -2,15 +2,13 @@
 // Licensed under the MIT License.
 
 using System.Net;
-using System.Security.Cryptography.X509Certificates;
 using System.Text.Json;
 using System.Text.RegularExpressions;
 using Titanium.Web.Proxy;
 using Titanium.Web.Proxy.EventArguments;
 using Titanium.Web.Proxy.Helpers;
 using Titanium.Web.Proxy.Http;
 using Titanium.Web.Proxy.Models;
-using Titanium.Web.Proxy.Network;
 
 namespace Microsoft.Graph.DeveloperProxy {
     internal enum FailMode {
@@ -79,6 +77,11 @@ public class ChaosEngine {
         private ExplicitProxyEndPoint? _explicitEndPoint;
         private readonly Dictionary<string, DateTime> _throttledRequests;
         private readonly ConsoleColor _color;
+        // lists of URLs to watch, used for intercepting requests
+        private List<Regex> urlsToWatch = new List<Regex>();
+        // lists of hosts to watch extracted from urlsToWatch,
+        // used for deciding which URLs to decrypt for further inspection
+        private List<Regex> hostsToWatch = new List<Regex>();
 
         public ChaosEngine(ProxyConfiguration config) {
             _config = config ?? throw new ArgumentNullException(nameof(config));
@@ -96,7 +99,13 @@ public ChaosEngine(ProxyConfiguration config) {
         }
 
         public async Task Run(CancellationToken? cancellationToken) {
-            Console.WriteLine($"Configuring proxy for cloud {_config.Cloud} - {_config.HostName}");
+            if (!_config.UrlsToWatch.Any()) {
+                Console.WriteLine("No URLs to watch configured. Please add URLs to watch in the appsettings.json config file.");
+                return;
+            }
+
+            LoadUrlsToWatch();
+
             _proxyServer = new ProxyServer();
 
             _proxyServer.CertificateManager.CertificateStorage = new CertificateDiskCache();
@@ -146,6 +155,36 @@ public async Task Run(CancellationToken? cancellationToken) {
             while (_proxyServer.ProxyRunning) { Thread.Sleep(10); }
         }
 
+        // Convert strings from config to regexes.
+        // From the list of URLs, extract host names and convert them to regexes.
+        // We need this because before we decrypt a request, we only have access
+        // to the host name, not the full URL.
+        private void LoadUrlsToWatch() {
+            foreach (var urlToWatch in _config.UrlsToWatch) {
+                // add the full URL
+                var urlToWatchRegexString = Regex.Escape(urlToWatch).Replace("\\*", ".*");
+                urlsToWatch.Add(new Regex(urlToWatchRegexString, RegexOptions.Compiled | RegexOptions.IgnoreCase));
+
+                // extract host from the URL
+                var hostToWatch = "";
+                if (urlToWatch.Contains("://")) {
+                    // if the URL contains a protocol, extract the host from the URL
+                    hostToWatch = urlToWatch.Split("://")[1].Substring(0, urlToWatch.Split("://")[1].IndexOf("/"));
+                }
+                else {
+                    // if the URL doesn't contain a protocol,
+                    // we assume the whole URL is a host name
+                    hostToWatch = urlToWatch;
+                }
+
+                var hostToWatchRegexString = Regex.Escape(hostToWatch).Replace("\\*", ".*");
+                // don't add the same host twice
+                if (!hostsToWatch.Any(h => h.ToString() == hostToWatchRegexString)) {
+                    hostsToWatch.Add(new Regex(hostToWatchRegexString, RegexOptions.Compiled | RegexOptions.IgnoreCase));
+                }
+            }
+        }
+
         private void Console_CancelKeyPress(object? sender, ConsoleCancelEventArgs e) {
             StopProxy();
         }
@@ -207,10 +246,8 @@ private FailMode ShouldFail(Request r) {
         }
 
         async Task OnBeforeTunnelConnectRequest(object sender, TunnelConnectSessionEventArgs e) {
-            string hostname = e.HttpClient.Request.RequestUri.Host;
-
             // Ensures that only the targeted Https domains are proxyied
-            if (!hostname.Contains(_config.HostName)) {
+            if (!ShouldDecryptRequest(e.HttpClient.Request.RequestUri.Host)) {
                 e.DecryptSsl = false;
             }
         }
@@ -231,14 +268,14 @@ async Task OnRequest(object sender, SessionEventArgs e) {
                 e.UserData = e.HttpClient.Request;
             }
 
-            // Chaos happens only for graph requests which are not OPTIONS
-            if (method is not "OPTIONS" && e.HttpClient.Request.RequestUri.Host.Contains(_config.HostName)) {
-                Console.WriteLine($"saw a graph request: {e.HttpClient.Request.Method} {e.HttpClient.Request.RequestUriString}");
-                HandleGraphRequest(e);
+            // Chaos happens only for requests which are not OPTIONS
+            if (method is not "OPTIONS" && ShouldWatchRequest(e.HttpClient.Request.Url)) {
+                Console.WriteLine($"saw a request: {e.HttpClient.Request.Method} {e.HttpClient.Request.Url}");
+                HandleRequest(e);
             }
         }
 
-        private void HandleGraphRequest(SessionEventArgs e) {
+        private void HandleRequest(SessionEventArgs e) {
             var responseComponents = ResponseComponents.Build();
             var matchingResponse = GetMatchingMockResponse(e.HttpClient.Request);
             if (matchingResponse is not null) {
@@ -254,12 +291,13 @@ private void HandleGraphRequest(SessionEventArgs e) {
                 }
 
                 if (failMode == FailMode.PassThru && _config.FailureRate != 100) {
-                    Console.WriteLine($"\tPassed through {e.HttpClient.Request.RequestUri.AbsolutePath}");
+                    Console.WriteLine($"\tPassed through {e.HttpClient.Request.Url}");
                     return;
                 }
 
                 FailResponse(e, responseComponents, failMode);
-                if (!IsSdkRequest(e.HttpClient.Request)) {
+                if (IsGraphRequest(e.HttpClient.Request) &&
+                    !IsSdkRequest(e.HttpClient.Request)) {
                     Console.ForegroundColor = ConsoleColor.Green;
                     Console.Error.WriteLine($"\tTIP: {BuildUseSdkMessage(e.HttpClient.Request)}");
                     Console.ForegroundColor = _color;
@@ -289,8 +327,14 @@ private static bool IsSdkRequest(Request request) {
             return request.Headers.HeaderExists("SdkVersion");
         }
 
+        private static bool IsGraphRequest(Request request) {
+            return request.RequestUri.Host.Contains("graph", StringComparison.OrdinalIgnoreCase);
+        }
+
         private static bool WarnNoSelect(Request request) {
-            return request.Method == "GET" && !request.Url.Contains("$select", StringComparison.OrdinalIgnoreCase);
+            return IsGraphRequest(request) &&
+                request.Method == "GET" &&
+                !request.Url.Contains("$select", StringComparison.OrdinalIgnoreCase);
         }
 
         private static string GetMoveToSdkUrl(Request request) {
@@ -343,28 +387,36 @@ private static void ProcessMockResponse(SessionEventArgs e, ResponseComponents r
             }
         }
 
+        private bool ShouldDecryptRequest(string hostName) {
+            return hostsToWatch.Any(h => h.IsMatch(hostName));
+        }
+
+        private bool ShouldWatchRequest(string requestUrl) {
+            return urlsToWatch.Any(u => u.IsMatch(requestUrl));
+        }
+
         private ProxyMockResponse? GetMatchingMockResponse(Request request) {
             if (_config.NoMocks ||
                 _config.Responses is null ||
                 !_config.Responses.Any()) {
                 return null;
             }
 
-            var mockResponse = _config.Responses.FirstOrDefault(r => {
-                if (r.Method != request.Method) return false;
-                if (r.Url == request.RequestUri.AbsolutePath) {
+            var mockResponse = _config.Responses.FirstOrDefault(mockResponse => {
+                if (mockResponse.Method != request.Method) return false;
+                if (mockResponse.Url == request.Url) {
                     return true;
                 }
 
                 // check if the URL contains a wildcard
                 // if it doesn't, it's not a match for the current request for sure
-                if (!r.Url.Contains('*')) {
+                if (!mockResponse.Url.Contains('*')) {
                     return false;
                 }
 
                 // turn mock URL with wildcard into a regex and match against the request URL
-                var urlRegex = Regex.Escape(r.Url).Replace("\\*", ".*");
-                return Regex.IsMatch(request.RequestUri.AbsolutePath, urlRegex);
+                var mockResponseUrlRegex = Regex.Escape(mockResponse.Url).Replace("\\*", ".*");
+                return Regex.IsMatch(request.Url, mockResponseUrlRegex);
             });
             return mockResponse;
         }
@@ -393,11 +445,11 @@ private void UpdateProxyResponse(SessionEventArgs e, ResponseComponents response
                     })
                 );
             }
-            Console.WriteLine($"\t{(matchingResponse is not null ? "Mocked" : "Failed")} {e.HttpClient.Request.RequestUri.AbsolutePath} with {responseComponents.ErrorStatus}");
+            Console.WriteLine($"\t{(matchingResponse is not null ? "Mocked" : "Failed")} {e.HttpClient.Request.Url} with {responseComponents.ErrorStatus}");
             e.GenericResponse(responseComponents.Body ?? string.Empty, responseComponents.ErrorStatus, responseComponents.Headers);
         }
 
-        private string BuildApiErrorMessage(Request r) => $"Some error was generated by the proxy. {(IsSdkRequest(r) ? "" : BuildUseSdkMessage(r))}";
+        private string BuildApiErrorMessage(Request r) => $"Some error was generated by the proxy. {(IsGraphRequest(r) ? (IsSdkRequest(r) ? "" : BuildUseSdkMessage(r)) : "")}";
 
         private string BuildThrottleKey(Request r) => $"{r.Method}-{r.Url}";
 

msgraph-developer-proxy/Properties/launchSettings.json

@@ -10,11 +10,6 @@
       "commandLineArgs": "--port 8080 --failure-rate 100 --no-mocks",
       "hotReloadEnabled": true
     },
-    "All chaos with mock responses - China Cloud": {
-      "commandName": "Project",
-      "commandLineArgs": "--port 8080 --failure-rate 100 --no-mocks --cloud china",
-      "hotReloadEnabled": true
-    },
     "High chaos only 429 503 and no mocks": {
       "commandName": "Project",
       "commandLineArgs": "--port 8080 --failure-rate 95 -a 429 503 --no-mocks",

msgraph-developer-proxy/ProxyCommandHandler.cs

@@ -4,25 +4,21 @@
 using Microsoft.Extensions.Configuration;
 using System.CommandLine;
 using System.CommandLine.Invocation;
-using System.Text.Json;
 
 namespace Microsoft.Graph.DeveloperProxy {
     public class ProxyCommandHandler : ICommandHandler {
         public Option<int> Port { get; set; }
         public Option<int> Rate { get; set; }
         public Option<bool> DisableMocks { get; set; }
-        public Option<string> Cloud { get; set; }
         public Option<IEnumerable<int>> AllowedErrors { get; }
 
-        public ProxyCommandHandler(Option<int> port, Option<int> rate, Option<bool> disableMocks, Option<string> cloud, Option<IEnumerable<int>> allowedErrors) {
+        public ProxyCommandHandler(Option<int> port, Option<int> rate, Option<bool> disableMocks, Option<IEnumerable<int>> allowedErrors) {
             Port = port ?? throw new ArgumentNullException(nameof(port));
             Rate = rate ?? throw new ArgumentNullException(nameof(rate));
             DisableMocks = disableMocks ?? throw new ArgumentNullException(nameof(disableMocks));
-            Cloud = cloud ?? throw new ArgumentNullException(nameof(cloud));
             AllowedErrors = allowedErrors ?? throw new ArgumentNullException(nameof(allowedErrors));
         }
 
-
         public int Invoke(InvocationContext context) {
             return InvokeAsync(context).GetAwaiter().GetResult();
         }
@@ -31,15 +27,12 @@ public async Task<int> InvokeAsync(InvocationContext context) {
             int port = context.ParseResult.GetValueForOption(Port);
             int failureRate = context.ParseResult.GetValueForOption(Rate);
             bool disableMocks = context.ParseResult.GetValueForOption(DisableMocks);
-            string? cloud = context.ParseResult.GetValueForOption(Cloud);
             IEnumerable<int> allowedErrors = context.ParseResult.GetValueForOption(AllowedErrors) ?? Enumerable.Empty<int>();
             CancellationToken? cancellationToken = (CancellationToken?)context.BindingContext.GetService(typeof(CancellationToken?));
             Configuration.Port = port;
             Configuration.FailureRate = failureRate;
             Configuration.NoMocks = disableMocks;
             Configuration.AllowedErrors = allowedErrors;
-            if (cloud is not null)
-                Configuration.Cloud = cloud;
 
             var newReleaseInfo = await UpdateNotification.CheckForNewVersion();
             if (newReleaseInfo != null) {
@@ -56,7 +49,7 @@ public async Task<int> InvokeAsync(InvocationContext context) {
                 return 0;
             }
             catch (Exception ex) {
-                Console.Error.WriteLine("An error occured while running the Developer Proxy");
+                Console.Error.WriteLine("An error occurred while running the Developer Proxy");
                 Console.Error.WriteLine(ex.Message.ToString());
                 Console.Error.WriteLine(ex.StackTrace?.ToString());
                 var inner = ex.InnerException;

msgraph-developer-proxy/ProxyConfiguration.cs

@@ -13,17 +13,11 @@ public class ProxyConfiguration : IDisposable
         public int FailureRate { get; set; } = 50;
         [JsonPropertyName("noMocks")]
         public bool NoMocks { get; set; } = false;
-        [JsonPropertyName("cloud")]
-        public string Cloud { get; set; } = "global";
-        [JsonPropertyName("cloudHosts")]
-        public Dictionary<string, string> CloudHosts { get; set; } = new();
+        [JsonPropertyName("urlsToWatch")]
+        public string[] UrlsToWatch { get; set; } = Array.Empty<string>();
         [JsonPropertyName("allowedErrors")]
         public IEnumerable<int> AllowedErrors { get; set; } = Array.Empty<int>();
 
-        public string HostName => CloudHosts.ContainsKey(Cloud) ? CloudHosts[Cloud] : throw new ArgumentOutOfRangeException(nameof(Cloud), InvalidCloudMessage);
-
-        private string InvalidCloudMessage => $"The value provided for the cloud: {Cloud} is not valid, current valid values are: {string.Join(", ", CloudHosts.Keys.ToArray())}.";
-
         [JsonPropertyName("responses")]
         public IEnumerable<ProxyMockResponse> Responses { get; set; } = Array.Empty<ProxyMockResponse>();
 

msgraph-developer-proxy/ProxyHost.cs

@@ -29,11 +29,6 @@ public RootCommand GetRootCommand() {
             noMocksOptions.ArgumentHelpName = "no mocks";
             noMocksOptions.SetDefaultValue(false);
 
-            var cloudOption = new Option<string>("--cloud", "Set the target cloud to proxy requests for");
-            cloudOption.AddAlias("-c");
-            cloudOption.ArgumentHelpName = "cloud";
-            cloudOption.SetDefaultValue("global");
-
             var allowedErrorsOption = new Option<IEnumerable<int>>("--allowed-errors", "List of errors that the developer proxy may produce");
             allowedErrorsOption.AddAlias("-a");
             allowedErrorsOption.ArgumentHelpName = "allowed errors";
@@ -45,11 +40,10 @@ public RootCommand GetRootCommand() {
                 portOption,
                 rateOption,
                 noMocksOptions,
-                cloudOption,
                 allowedErrorsOption
             };
             command.Description = "HTTP proxy to create random failures for calls to Microsoft Graph";
-            command.Handler = new ProxyCommandHandler(portOption, rateOption, noMocksOptions, cloudOption, allowedErrorsOption);
+            command.Handler = new ProxyCommandHandler(portOption, rateOption, noMocksOptions, allowedErrorsOption);
 
             return command;
         }

msgraph-developer-proxy/UpdateNotification.cs

@@ -1,16 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System;
-using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.Net.Http.Headers;
 using System.Reflection;
-using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using System.Threading.Tasks;
 
 namespace Microsoft.Graph.DeveloperProxy {
     internal class ReleaseInfo {

msgraph-developer-proxy/appsettings.json

@@ -1,8 +1,16 @@
 {
-  "cloudHosts": {
-    "global": "graph.microsoft.com",
-    "usGov": "graph.microsoft.us",
-    "dod": "dod-graph.microsoft.us",
-    "china": "microsoftgraph.chinacloudapi.cn"
-  }
+  "urlsToWatch": [
+    "https://graph.microsoft.com/v1.0/*",
+    "https://graph.microsoft.com/beta/*",
+    "https://graph.microsoft.us/v1.0/*",
+    "https://graph.microsoft.us/beta/*",
+    "https://dod-graph.microsoft.us/v1.0/*",
+    "https://dod-graph.microsoft.us/beta/*",
+    "https://microsoftgraph.chinacloudapi.cn/v1.0/*",
+    "https://microsoftgraph.chinacloudapi.cn/beta/*",
+    "https://*.sharepoint.*/*_api/*",
+    "https://*.sharepoint.*/*_vti_bin/*",
+    "https://*.sharepoint-df.*/*_api/*",
+    "https://*.sharepoint-df.*/*_vti_bin/*"
+  ]
 }

msgraph-developer-proxy/responses.json

@@ -1,7 +1,7 @@
 {
   "responses": [
     {
-      "url": "/v1.0/me",
+      "url": "https://graph.microsoft.com/v1.0/me",
       "responseBody": {
         "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users/$entity",
         "businessPhones": [
@@ -20,13 +20,13 @@
       }
     },
     {
-      "url": "/v1.0/me",
+      "url": "https://graph.microsoft.com/v1.0/me",
       "method": "PATCH",
       "responseBody": {},
       "responseCode":  204
     },
     {
-      "url": "/v1.0/users/48d31887-5fad-4d73-a9f5-3c356e68a038",
+      "url": "https://graph.microsoft.com/v1.0/users/48d31887-5fad-4d73-a9f5-3c356e68a038",
       "responseBody": {
         "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users/$entity",
         "businessPhones": [
@@ -45,18 +45,18 @@
       }
     },
     {
-      "url": "/v1.0/me/photo",
+      "url": "https://graph.microsoft.com/v1.0/me/photo",
       "responseCode": 404
     },
     {
-      "url": "/v1.0/users/*/photo/$value",
+      "url": "https://graph.microsoft.com/v1.0/users/*/photo/$value",
       "responseBody": "@picture.jpg",
       "responseHeaders": {
         "content-type": "image/jpeg"
       }
     },
     {
-      "url": "/v1.0/users/*",
+      "url": "https://graph.microsoft.com/v1.0/users/*",
       "responseBody": {
         "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users/$entity",
         "businessPhones": [