Improve configuration initialization

aspire/AppHost.cs

@@ -5,24 +5,9 @@
 
 using ConsoleAppFramework;
 using Elastic.Documentation;
-using Microsoft.Extensions.Logging;
 using static Elastic.Documentation.Aspire.ResourceNames;
 
-// ReSharper disable UnusedVariable
-// ReSharper disable RedundantAssignment
-// ReSharper disable NotAccessedVariable
-
-var logLevel = LogLevel.Information;
-GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories, out _);
-var globalArguments = new List<string>();
-if (skipPrivateRepositories)
-	globalArguments.Add("--skip-private-repositories");
-
-if (logLevel != LogLevel.Information)
-{
-	globalArguments.Add("--log-level");
-	globalArguments.Add(logLevel.ToString());
-}
+GlobalCli.Process(ref args, out _, out var globalArguments);
 
 await ConsoleApp.RunAsync(args, BuildAspireHost);
 return;
@@ -32,7 +17,6 @@
 async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skipPrivateRepositories, Cancel ctx)
 {
 	var builder = DistributedApplication.CreateBuilder(args);
-	skipPrivateRepositories = globalArguments.Contains("--skip-private-repositories");
 
 	var llmUrl = builder.AddParameter("LlmGatewayUrl", secret: true);
 	var llmServiceAccountPath = builder.AddParameter("LlmGatewayServiceAccountPath", secret: true);
@@ -63,6 +47,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
 		.WithExplicitStart();
 
+	// ReSharper disable once RedundantAssignment
 	api = startElasticsearch
 		? api
 			.WithReference(elasticsearchLocal)
@@ -78,6 +63,8 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 		.WithArgs(["assembler", "build", "--exporters", "elasticsearch", .. globalArguments])
 		.WithExplicitStart()
 		.WaitForCompletion(cloneAll);
+
+	// ReSharper disable once RedundantAssignment
 	indexElasticsearch = startElasticsearch
 		? indexElasticsearch
 			.WaitFor(elasticsearchLocal)
@@ -97,6 +84,8 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 		.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
 		.WithExplicitStart()
 		.WaitForCompletion(cloneAll);
+
+	// ReSharper disable once RedundantAssignment
 	indexElasticsearchSemantic = startElasticsearch
 		? indexElasticsearchSemantic
 			.WaitFor(elasticsearchLocal)
@@ -130,6 +119,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
 
 
+	// ReSharper disable once RedundantAssignment
 	serveStatic = startElasticsearch ? serveStatic.WaitFor(elasticsearchLocal) : serveStatic.WaitFor(buildAll);
 
 	await builder.Build().RunAsync(ctx);

src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs

@@ -6,40 +6,70 @@
 using System.Text.RegularExpressions;
 using Elastic.Documentation.Configuration.Assembler;
 using Microsoft.Extensions.DependencyInjection;
-using NetEscapades.EnumGenerators;
+using Microsoft.Extensions.Logging;
 
 namespace Elastic.Documentation.Configuration;
 
-[EnumExtensions]
-public enum ConfigurationSource
-{
-	Local,
-	Checkout,
-	Embedded
-}
-
 public partial class ConfigurationFileProvider
 {
 	private readonly IFileSystem _fileSystem;
 	private readonly string _assemblyName;
+	private readonly ILogger<ConfigurationFileProvider> _logger;
 
-	public ConfigurationSource ConfigurationSource { get; private set; } = ConfigurationSource.Embedded;
+	public ConfigurationSource ConfigurationSource { get; }
 	public string? GitReference { get; }
 
-	public ConfigurationFileProvider(IFileSystem fileSystem, bool skipPrivateRepositories = false)
+	public ConfigurationFileProvider(
+		ILoggerFactory logFactory,
+		IFileSystem fileSystem,
+		bool skipPrivateRepositories = false,
+		ConfigurationSource? configurationSource = null
+	)
 	{
+		_logger = logFactory.CreateLogger<ConfigurationFileProvider>();
 		_fileSystem = fileSystem;
 		_assemblyName = typeof(ConfigurationFileProvider).Assembly.GetName().Name!;
 		SkipPrivateRepositories = skipPrivateRepositories;
 		TemporaryDirectory = fileSystem.Directory.CreateTempSubdirectory("docs-builder-config");
 
+		ConfigurationSource = configurationSource ?? (
+			fileSystem.Directory.Exists(LocalConfigurationDirectory)
+				? ConfigurationSource.Local : ConfigurationSource.Embedded
+			);
+
+		if (ConfigurationSource == ConfigurationSource.Local && !fileSystem.Directory.Exists(LocalConfigurationDirectory))
+			throw new Exception($"Required directory form {nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Local)} directory {LocalConfigurationDirectory} does not exist.");
+
+		if (ConfigurationSource == ConfigurationSource.Remote && !fileSystem.Directory.Exists(AppDataConfigurationDirectory))
+			throw new Exception($"Required directory form {nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Remote)} directory {AppDataConfigurationDirectory} does not exist.");
+
+		var path = GetAppDataPath("git-ref.txt");
+		if (_fileSystem.File.Exists(path))
+			GitReference = _fileSystem.File.ReadAllText(path);
+		else if (ConfigurationSource == ConfigurationSource.Remote)
+			throw new Exception($"Can not read git-ref.txt in directory {LocalConfigurationDirectory}");
+
+		if (ConfigurationSource == ConfigurationSource.Remote)
+		{
+			_logger.LogInformation("{ConfigurationSource}: git ref '{GitReference}', in {Directory}",
+				$"{nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Remote)}", GitReference, AppDataConfigurationDirectory);
+		}
+
+		if (ConfigurationSource == ConfigurationSource.Local)
+		{
+			_logger.LogInformation("{ConfigurationSource}: located {Directory}",
+				$"{nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Local)}", LocalConfigurationDirectory);
+		}
+		if (ConfigurationSource == ConfigurationSource.Embedded)
+		{
+			_logger.LogInformation("{ConfigurationSource} using embedded in binary configuration",
+				$"{nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Embedded)}");
+		}
+
 		VersionFile = CreateTemporaryConfigurationFile("versions.yml");
 		AssemblerFile = CreateTemporaryConfigurationFile("assembler.yml");
 		NavigationFile = CreateTemporaryConfigurationFile("navigation.yml");
 		LegacyUrlMappingsFile = CreateTemporaryConfigurationFile("legacy-url-mappings.yml");
-		var path = GetAppDataPath("git-ref.txt");
-		if (ConfigurationSource == ConfigurationSource.Checkout && _fileSystem.File.Exists(path))
-			GitReference = _fileSystem.File.ReadAllText(path);
 	}
 
 	public bool SkipPrivateRepositories { get; }
@@ -66,6 +96,8 @@ public IFileInfo CreateNavigationFile(AssemblyConfiguration configuration)
 		if (_fileSystem.File.Exists(tempFile))
 			return NavigationFile;
 
+		_logger.LogInformation("Filtering navigation file to remove private repositories");
+
 		// This routine removes `toc: `'s linking to private repositories and reindents any later lines if needed.
 		// This will make any public children in the nav move up one place.
 		var spacing = -1;
@@ -139,19 +171,22 @@ private IFileInfo CreateTemporaryConfigurationFile(string fileName)
 	private StreamReader GetLocalOrEmbedded(string fileName)
 	{
 		var localPath = GetLocalPath(fileName);
-		var appDataPath = GetAppDataPath(fileName);
-		if (_fileSystem.File.Exists(localPath))
+		if (ConfigurationSource == ConfigurationSource.Local && _fileSystem.File.Exists(localPath))
 		{
-			ConfigurationSource = ConfigurationSource.Local;
 			var reader = _fileSystem.File.OpenText(localPath);
 			return reader;
 		}
-		if (_fileSystem.File.Exists(appDataPath))
+		if (ConfigurationSource == ConfigurationSource.Local)
+			throw new Exception($"Can not read {fileName} in directory {LocalConfigurationDirectory}");
+
+		var appDataPath = GetAppDataPath(fileName);
+		if (ConfigurationSource == ConfigurationSource.Remote && _fileSystem.File.Exists(appDataPath))
 		{
-			ConfigurationSource = ConfigurationSource.Checkout;
 			var reader = _fileSystem.File.OpenText(appDataPath);
 			return reader;
 		}
+		if (ConfigurationSource == ConfigurationSource.Remote)
+			throw new Exception($"Can not read {fileName} in directory {AppDataConfigurationDirectory}");
 		return GetEmbeddedStream(fileName);
 	}
 
@@ -163,24 +198,26 @@ private StreamReader GetEmbeddedStream(string fileName)
 		return reader;
 	}
 
-	private static string AppDataConfigurationDirectory { get; } = Path.Combine(Paths.ApplicationData.FullName, "config-clone", "config");
-	private static string LocalConfigurationDirectory { get; } = Path.Combine(Paths.WorkingDirectoryRoot.FullName, "config");
+	public static string AppDataConfigurationDirectory { get; } = Path.Combine(Paths.ApplicationData.FullName, "config-clone", "config");
+	public static string LocalConfigurationDirectory { get; } = Path.Combine(Paths.WorkingDirectoryRoot.FullName, "config");
 
 	private static string GetLocalPath(string file) => Path.Combine(LocalConfigurationDirectory, file);
 	private static string GetAppDataPath(string file) => Path.Combine(AppDataConfigurationDirectory, file);
+
 	[GeneratedRegex(@"^\s+-?\s?toc:\s?")]
 	private static partial Regex TocPrefixRegex();
 }
 
 public static class ConfigurationFileProviderServiceCollectionExtensions
 {
-	public static IServiceCollection AddConfigurationFileProvider(
-		this IServiceCollection services,
+	public static IServiceCollection AddConfigurationFileProvider(this IServiceCollection services,
 		bool skipPrivateRepositories,
-		Action<IServiceCollection, ConfigurationFileProvider> configure
-	)
+		ConfigurationSource? configurationSource,
+		Action<IServiceCollection, ConfigurationFileProvider> configure)
 	{
-		var provider = new ConfigurationFileProvider(new FileSystem(), skipPrivateRepositories);
+		using var sp = services.BuildServiceProvider();
+		var logFactory = sp.GetRequiredService<ILoggerFactory>();
+		var provider = new ConfigurationFileProvider(logFactory, new FileSystem(), skipPrivateRepositories, configurationSource);
 		_ = services.AddSingleton(provider);
 		configure(services, provider);
 		return services;

src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs

@@ -13,32 +13,28 @@
 
 namespace Elastic.Documentation.ServiceDefaults;
 
-public record CliInvocation(bool IsHelpOrVersion);
-
 public static class AppDefaultsExtensions
 {
 	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
 	{
 		var args = Array.Empty<string>();
 		return builder.AddDocumentationServiceDefaults(ref args);
 	}
-	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, Action<IServiceCollection, ConfigurationFileProvider> configure) where TBuilder : IHostApplicationBuilder =>
-		builder.AddDocumentationServiceDefaults(ref args, null, configure);
-
-	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, LogLevel? defaultLogLevel = null, Action<IServiceCollection, ConfigurationFileProvider>? configure = null) where TBuilder : IHostApplicationBuilder
+	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, Action<IServiceCollection, ConfigurationFileProvider>? configure = null)
+		where TBuilder : IHostApplicationBuilder
 	{
-		var logLevel = defaultLogLevel ?? LogLevel.Information;
-		GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories, out var isHelpOrVersion);
+		GlobalCli.Process(ref args, out var globalArgs);
 
 		var services = builder.Services;
+		_ = builder.Services.AddElasticDocumentationLogging(globalArgs.LogLevel);
 		_ = services
-			.AddConfigurationFileProvider(skipPrivateRepositories, (s, p) =>
+			.AddConfigurationFileProvider(globalArgs.SkipPrivateRepositories, globalArgs.ConfigurationSource, (s, p) =>
 			{
 				_ = s.AddSingleton(p.CreateVersionConfiguration());
 				configure?.Invoke(s, p);
 			});
-		_ = builder.Services.AddElasticDocumentationLogging(logLevel);
-		_ = services.AddSingleton(new CliInvocation(isHelpOrVersion));
+		_ = builder.Services.AddElasticDocumentationLogging(globalArgs.LogLevel);
+		_ = services.AddSingleton(globalArgs);
 
 		return builder.AddServiceDefaults();
 	}

src/Elastic.Documentation/ConfigurationSource.cs

@@ -0,0 +1,15 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using NetEscapades.EnumGenerators;
+
+namespace Elastic.Documentation;
+
+[EnumExtensions]
+public enum ConfigurationSource
+{
+	Local,
+	Remote,
+	Embedded
+}

src/Elastic.Documentation/GlobalCommandLine.cs

@@ -6,38 +6,61 @@
 
 namespace Elastic.Documentation;
 
-public static class GlobalCommandLine
+public record GlobalCliArgs
 {
-	public static void Process(
-		ref string[] args,
-		ref LogLevel defaultLogLevel,
-		out bool skipPrivateRepositories,
-		out bool isHelpOrVersion
-	)
+	public LogLevel LogLevel { get; init; } = LogLevel.Information;
+	public ConfigurationSource? ConfigurationSource { get; init; }
+	public bool SkipPrivateRepositories { get; init; }
+	public bool IsHelpOrVersion { get; init; }
+}
+public static class GlobalCli
+{
+	public static void Process(ref string[] args, out GlobalCliArgs cli) => Process(ref args, out cli, out _);
+	public static void Process(ref string[] args, out GlobalCliArgs cli, out string[] globalArguments)
 	{
-		skipPrivateRepositories = false;
-		isHelpOrVersion = false;
-		var newArgs = new List<string>();
+		cli = new GlobalCliArgs();
+		globalArguments = [];
+		var globalArgs = new List<string>();
+		var filteredArguments = new List<string>();
 		for (var i = 0; i < args.Length; i++)
 		{
 			if (args[i] == "--log-level")
 			{
 				if (args.Length > i + 1)
-					defaultLogLevel = GetLogLevel(args[i + 1]);
+				{
+					cli = cli with { LogLevel = GetLogLevel(args[i + 1]) };
+					globalArgs.Add("--log-level");
+					globalArgs.Add(args[i + 1]);
+				}
+				i++;
+			}
+			else if (args[i] is "--config-source" or "--configuration-source" or "-c")
+			{
+				if (args.Length > i + 1 && ConfigurationSourceExtensions.TryParse(args[i + 1], out var cs, true, true))
+				{
+					cli = cli with { ConfigurationSource = cs };
+					globalArgs.Add("--config-source");
+					globalArgs.Add(args[i + 1]);
+				}
 				i++;
 			}
 			else if (args[i] == "--skip-private-repositories")
-				skipPrivateRepositories = true;
+			{
+				cli = cli with { SkipPrivateRepositories = true };
+				globalArgs.Add("--skip-private-repositories");
+			}
 			else if (args[i] is "--help" or "--version")
 			{
-				isHelpOrVersion = true;
-				newArgs.Add(args[i]);
+				cli = cli with { IsHelpOrVersion = true };
+				globalArgs.Add(args[i]);
+				filteredArguments.Add(args[i]);
 			}
 			else
-				newArgs.Add(args[i]);
+				filteredArguments.Add(args[i]);
 		}
 
-		args = [.. newArgs];
+		args = [.. filteredArguments];
+		globalArguments = [.. globalArgs];
 	}
 
 	private static LogLevel GetLogLevel(string? logLevel) => logLevel switch