Improve configuration initialization

aspire/AppHost.cs

@@ -13,11 +13,17 @@
 // ReSharper disable NotAccessedVariable
 
 var logLevel = LogLevel.Information;
-GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories);
+GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories, out var configurationSource);
 var globalArguments = new List<string>();
 if (skipPrivateRepositories)
 	globalArguments.Add("--skip-private-repositories");
 
+if (configurationSource is { } cs)
+{
+	globalArguments.Add("--config-source");
+	globalArguments.Add(cs.ToStringFast(true));
+}
+
 if (logLevel != LogLevel.Information)
 {
 	globalArguments.Add("--log-level");

src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs

@@ -6,40 +6,70 @@
 using System.Text.RegularExpressions;
 using Elastic.Documentation.Configuration.Assembler;
 using Microsoft.Extensions.DependencyInjection;
-using NetEscapades.EnumGenerators;
+using Microsoft.Extensions.Logging;
 
 namespace Elastic.Documentation.Configuration;
 
-[EnumExtensions]
-public enum ConfigurationSource
-{
-	Local,
-	Checkout,
-	Embedded
-}
-
 public partial class ConfigurationFileProvider
 {
 	private readonly IFileSystem _fileSystem;
 	private readonly string _assemblyName;
+	private readonly ILogger<ConfigurationFileProvider> _logger;
 
-	public ConfigurationSource ConfigurationSource { get; private set; } = ConfigurationSource.Embedded;
+	public ConfigurationSource ConfigurationSource { get; }
 	public string? GitReference { get; }
 
-	public ConfigurationFileProvider(IFileSystem fileSystem, bool skipPrivateRepositories = false)
+	public ConfigurationFileProvider(
+		ILoggerFactory logFactory,
+		IFileSystem fileSystem,
+		bool skipPrivateRepositories = false,
+		ConfigurationSource? configurationSource = null
+	)
 	{
+		_logger = logFactory.CreateLogger<ConfigurationFileProvider>();
 		_fileSystem = fileSystem;
 		_assemblyName = typeof(ConfigurationFileProvider).Assembly.GetName().Name!;
 		SkipPrivateRepositories = skipPrivateRepositories;
 		TemporaryDirectory = fileSystem.Directory.CreateTempSubdirectory("docs-builder-config");
 
+		ConfigurationSource = configurationSource ?? (
+			fileSystem.Directory.Exists(LocalConfigurationDirectory)
+				? ConfigurationSource.Local : ConfigurationSource.Embedded
+			);
+
+		if (ConfigurationSource == ConfigurationSource.Local && !fileSystem.Directory.Exists(LocalConfigurationDirectory))
+			throw new Exception($"Required directory form {nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Local)} directory {LocalConfigurationDirectory} does not exist.");
+
+		if (ConfigurationSource == ConfigurationSource.Init && !fileSystem.Directory.Exists(AppDataConfigurationDirectory))
+			throw new Exception($"Required directory form {nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Init)} directory {AppDataConfigurationDirectory} does not exist.");
+
+		var path = GetAppDataPath("git-ref.txt");
+		if (_fileSystem.File.Exists(path))
+			GitReference = _fileSystem.File.ReadAllText(path);
+		else if (ConfigurationSource == ConfigurationSource.Init)
+			throw new Exception($"Can not read git-ref.txt in directory {LocalConfigurationDirectory}");
+
+		if (ConfigurationSource == ConfigurationSource.Init)
+		{
+			_logger.LogInformation("{ConfigurationSource}: git ref '{GitReference}', in {Directory}",
+				$"{nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Init)}", GitReference, AppDataConfigurationDirectory);
+		}
+
+		if (ConfigurationSource == ConfigurationSource.Local)
+		{
+			_logger.LogInformation("{ConfigurationSource}: located {Directory}",
+				$"{nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Local)}", AppDataConfigurationDirectory);
+		}
+		if (ConfigurationSource == ConfigurationSource.Embedded)
+		{
+			_logger.LogInformation("{ConfigurationSource} using embedded in binary configuration",
+				$"{nameof(ConfigurationSource)}.{nameof(ConfigurationSource.Embedded)}");
+		}
+
 		VersionFile = CreateTemporaryConfigurationFile("versions.yml");
 		AssemblerFile = CreateTemporaryConfigurationFile("assembler.yml");
 		NavigationFile = CreateTemporaryConfigurationFile("navigation.yml");
 		LegacyUrlMappingsFile = CreateTemporaryConfigurationFile("legacy-url-mappings.yml");
-		var path = GetAppDataPath("git-ref.txt");
-		if (ConfigurationSource == ConfigurationSource.Checkout && _fileSystem.File.Exists(path))
-			GitReference = _fileSystem.File.ReadAllText(path);
 	}
 
 	public bool SkipPrivateRepositories { get; }
@@ -66,6 +96,8 @@ public IFileInfo CreateNavigationFile(AssemblyConfiguration configuration)
 		if (_fileSystem.File.Exists(tempFile))
 			return NavigationFile;
 
+		_logger.LogInformation("Filtering navigation file to remove private repositories");
+
 		// This routine removes `toc: `'s linking to private repositories and reindents any later lines if needed.
 		// This will make any public children in the nav move up one place.
 		var spacing = -1;
@@ -139,19 +171,22 @@ private IFileInfo CreateTemporaryConfigurationFile(string fileName)
 	private StreamReader GetLocalOrEmbedded(string fileName)
 	{
 		var localPath = GetLocalPath(fileName);
-		var appDataPath = GetAppDataPath(fileName);
-		if (_fileSystem.File.Exists(localPath))
+		if (ConfigurationSource == ConfigurationSource.Local && _fileSystem.File.Exists(localPath))
 		{
-			ConfigurationSource = ConfigurationSource.Local;
 			var reader = _fileSystem.File.OpenText(localPath);
 			return reader;
 		}
-		if (_fileSystem.File.Exists(appDataPath))
+		if (ConfigurationSource == ConfigurationSource.Local)
+			throw new Exception($"Can not read {fileName} in directory {LocalConfigurationDirectory}");
+
+		var appDataPath = GetAppDataPath(fileName);
+		if (ConfigurationSource == ConfigurationSource.Init && _fileSystem.File.Exists(appDataPath))
 		{
-			ConfigurationSource = ConfigurationSource.Checkout;
 			var reader = _fileSystem.File.OpenText(appDataPath);
 			return reader;
 		}
+		if (ConfigurationSource == ConfigurationSource.Init)
+			throw new Exception($"Can not read {fileName} in directory {AppDataConfigurationDirectory}");
 		return GetEmbeddedStream(fileName);
 	}
 
@@ -168,19 +203,21 @@ private StreamReader GetEmbeddedStream(string fileName)
 
 	private static string GetLocalPath(string file) => Path.Combine(LocalConfigurationDirectory, file);
 	private static string GetAppDataPath(string file) => Path.Combine(AppDataConfigurationDirectory, file);
+
 	[GeneratedRegex(@"^\s+-?\s?toc:\s?")]
 	private static partial Regex TocPrefixRegex();
 }
 
 public static class ConfigurationFileProviderServiceCollectionExtensions
 {
-	public static IServiceCollection AddConfigurationFileProvider(
-		this IServiceCollection services,
+	public static IServiceCollection AddConfigurationFileProvider(this IServiceCollection services,
 		bool skipPrivateRepositories,
-		Action<IServiceCollection, ConfigurationFileProvider> configure
-	)
+		Documentation.ConfigurationSource? configurationSource,
+		Action<IServiceCollection, ConfigurationFileProvider> configure)
 	{
-		var provider = new ConfigurationFileProvider(new FileSystem(), skipPrivateRepositories);
+		using var sp = services.BuildServiceProvider();
+		var logFactory = sp.GetRequiredService<ILoggerFactory>();
+		var provider = new ConfigurationFileProvider(logFactory, new FileSystem(), skipPrivateRepositories, configurationSource);
 		_ = services.AddSingleton(provider);
 		configure(services, provider);
 		return services;

src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs

@@ -23,19 +23,20 @@ public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder b
 	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, Action<IServiceCollection, ConfigurationFileProvider> configure) where TBuilder : IHostApplicationBuilder =>
 		builder.AddDocumentationServiceDefaults(ref args, null, configure);
 
-	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, LogLevel? defaultLogLevel = null, Action<IServiceCollection, ConfigurationFileProvider>? configure = null) where TBuilder : IHostApplicationBuilder
+	public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder builder, ref string[] args, LogLevel? defaultLogLevel = null, Action<IServiceCollection, ConfigurationFileProvider>? configure = null)
+		where TBuilder : IHostApplicationBuilder
 	{
 		var logLevel = defaultLogLevel ?? LogLevel.Information;
-		GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories);
+		GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories, out var configurationSource);
 
 		var services = builder.Services;
+		_ = builder.Services.AddElasticDocumentationLogging(logLevel);
 		_ = services
-			.AddConfigurationFileProvider(skipPrivateRepositories, (s, p) =>
+			.AddConfigurationFileProvider(skipPrivateRepositories, configurationSource, (s, p) =>
 			{
 				_ = s.AddSingleton(p.CreateVersionConfiguration());
 				configure?.Invoke(s, p);
 			});
-		_ = builder.Services.AddElasticDocumentationLogging(logLevel);
 
 		return builder.AddServiceDefaults();
 	}

src/Elastic.Documentation/ConfigurationSource.cs

@@ -0,0 +1,15 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using NetEscapades.EnumGenerators;
+
+namespace Elastic.Documentation;
+
+[EnumExtensions]
+public enum ConfigurationSource
+{
+	Local,
+	Init,
+	Embedded
+}

src/Elastic.Documentation/GlobalCommandLine.cs

@@ -8,8 +8,9 @@ namespace Elastic.Documentation;
 
 public static class GlobalCommandLine
 {
-	public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out bool skipPrivateRepositories)
+	public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out bool skipPrivateRepositories, out ConfigurationSource? configurationSource)
 	{
+		configurationSource = null;
 		skipPrivateRepositories = false;
 		var newArgs = new List<string>();
 		for (var i = 0; i < args.Length; i++)
@@ -20,6 +21,12 @@ public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out
 					defaultLogLevel = GetLogLevel(args[i + 1]);
 				i++;
 			}
+			else if (args[i] == "--config-source")
+			{
+				if (args.Length > i + 1 && ConfigurationSourceExtensions.TryParse(args[i + 1], out var cs, true, true))
+					configurationSource = cs;
+				i++;
+			}
 			else if (args[i] == "--skip-private-repositories")
 				skipPrivateRepositories = true;
 			else if (args[i] == "--inject")

src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs

@@ -14,7 +14,7 @@ public class InfoLoggerFilter(ConsoleAppFilter next, ILogger<InfoLoggerFilter> l
 	public override async Task InvokeAsync(ConsoleAppContext context, Cancel cancellationToken)
 	{
 		logger.LogInformation("Configuration source: {ConfigurationSource}", fileProvider.ConfigurationSource.ToStringFast(true));
-		if (fileProvider.ConfigurationSource == ConfigurationSource.Checkout)
+		if (fileProvider.ConfigurationSource == ConfigurationSource.Init)
 			logger.LogInformation("Configuration source git reference: {ConfigurationSourceGitReference}", fileProvider.GitReference);
 		var assemblyVersion = Assembly.GetExecutingAssembly().GetCustomAttributes<AssemblyInformationalVersionAttribute>()
 			.FirstOrDefault()?.InformationalVersion;

src/tooling/docs-assembler/Cli/RepositoryCommands.cs

@@ -49,25 +49,45 @@ private void AssignOutputLogger()
 	public async Task<int> CloneConfigurationFolder(string? gitRef = null, Cancel ctx = default)
 	{
 		await using var collector = new ConsoleDiagnosticsCollector(logFactory, githubActionsService).StartAsync(ctx);
-
 		var fs = new FileSystem();
 		var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "config-clone");
 		var checkoutFolder = fs.DirectoryInfo.New(cachedPath);
 		var cloner = new RepositorySourcer(logFactory, checkoutFolder, fs, collector);
+		if (gitRef is not null && gitRef.Length != 32)
+		{
+			collector.EmitError("", "gitRef must be be 32 characters long");
+			await collector.StopAsync(ctx);
+			// deleting the cached path because its not in the state we want
+			_log.LogInformation("Deleting cached config folder");
+			fs.Directory.Delete(cachedPath, true);
+			return 1;
+		}
+
 
 		// relies on the embedded configuration, but we don't expect this to change
 		var repository = assemblyConfiguration.ReferenceRepositories["docs-builder"];
 		repository = repository with
 		{
 			SparsePaths = ["config"]
 		};
-		if (string.IsNullOrEmpty(gitRef))
-			gitRef = "main";
+		var gitReference = gitRef;
+		if (string.IsNullOrEmpty(gitReference))
+			gitReference = "main";
 
-		_log.LogInformation("Cloning configuration ({GitReference})", gitRef);
-		var checkout = cloner.CloneRef(repository, gitRef, appendRepositoryName: false);
+		_log.LogInformation("Cloning configuration ({GitReference})", gitReference);
+		var checkout = cloner.CloneRef(repository, gitReference, appendRepositoryName: false);
 		_log.LogInformation("Cloned configuration ({GitReference}) to {ConfigurationFolder}", checkout.HeadReference, checkout.Directory.FullName);
 
+		if (gitRef is not null && !checkout.HeadReference.StartsWith(gitRef))
+		{
+			collector.EmitError("", $"Checkout of {checkout.HeadReference} does start with requested gitRef {gitRef}.");
+			await collector.StopAsync(ctx);
+			// deleting the cached path because its not in the state we want
+			_log.LogInformation("Deleting cached config folder");
+			fs.Directory.Delete(cachedPath, true);
+			return 1;
+		}
+
 		var gitRefInformationFile = Path.Combine(cachedPath, "config", "git-ref.txt");
 		await fs.File.WriteAllTextAsync(gitRefInformationFile, checkout.HeadReference, ctx);
 

src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs

@@ -203,7 +203,7 @@ public Checkout CloneRef(Repository repository, string gitRef, bool pull = false
 				return new Checkout
 				{
 					Directory = checkoutFolder,
-					HeadReference = gitRef,
+					HeadReference = git.GetCurrentCommit(),
 					Repository = repository,
 				};
 			}
@@ -224,7 +224,7 @@ public Checkout CloneRef(Repository repository, string gitRef, bool pull = false
 		return new Checkout
 		{
 			Directory = checkoutFolder,
-			HeadReference = gitRef,
+			HeadReference = git.GetCurrentCommit(),
 			Repository = repository,
 		};
 	}

tests/Elastic.ApiExplorer.Tests/Elastic.ApiExplorer.Tests.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\src\Elastic.ApiExplorer\Elastic.ApiExplorer.csproj"/>
+    <ProjectReference Include="..\authoring\authoring.fsproj" />
   </ItemGroup>
 
 </Project>

tests/Elastic.ApiExplorer.Tests/TestHelpers.cs

@@ -6,6 +6,7 @@
 using Elastic.Documentation;
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Versions;
+using Microsoft.Extensions.Logging.Abstractions;
 
 namespace Elastic.ApiExplorer.Tests;
 
@@ -33,7 +34,7 @@ public static IConfigurationContext CreateConfigurationContext(IFileSystem fileS
 			{
 				Elasticsearch = ElasticsearchEndpoint.Default,
 			},
-			ConfigurationFileProvider = new ConfigurationFileProvider(fileSystem),
+			ConfigurationFileProvider = new ConfigurationFileProvider(NullLoggerFactory.Instance, fileSystem),
 			VersionsConfiguration = versionsConfiguration
 		};
 	}

tests/Elastic.Markdown.Tests/TestHelpers.cs

@@ -33,7 +33,7 @@ public static IConfigurationContext CreateConfigurationContext(IFileSystem fileS
 			{
 				Elasticsearch = ElasticsearchEndpoint.Default,
 			},
-			ConfigurationFileProvider = new ConfigurationFileProvider(fileSystem),
+			ConfigurationFileProvider = new ConfigurationFileProvider(new TestLoggerFactory(TestContext.Current.TestOutputHelper), fileSystem),
 			VersionsConfiguration = versionsConfiguration
 		};
 	}

tests/Elastic.Markdown.Tests/TestLogger.cs

@@ -6,7 +6,7 @@
 
 namespace Elastic.Markdown.Tests;
 
-public class TestLogger(ITestOutputHelper output) : ILogger
+public class TestLogger(ITestOutputHelper? output) : ILogger
 {
 	private sealed class NullScope : IDisposable
 	{
@@ -18,17 +18,17 @@ public void Dispose() { }
 	public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Trace;
 
 	public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) =>
-		output.WriteLine(formatter(state, exception));
+		output?.WriteLine(formatter(state, exception));
 }
 
-public class TestLoggerProvider(ITestOutputHelper output) : ILoggerProvider
+public class TestLoggerProvider(ITestOutputHelper? output) : ILoggerProvider
 {
 	public void Dispose() => GC.SuppressFinalize(this);
 
 	public ILogger CreateLogger(string categoryName) => new TestLogger(output);
 }
 
-public class TestLoggerFactory(ITestOutputHelper output) : ILoggerFactory
+public class TestLoggerFactory(ITestOutputHelper? output) : ILoggerFactory
 {
 	public void Dispose() => GC.SuppressFinalize(this);
 

tests/authoring/Framework/Setup.fs

@@ -210,7 +210,7 @@ type Setup =
             )
         )
         let versionConfig = VersionsConfiguration(VersioningSystems = versioningSystems)
-        let configurationFileProvider = ConfigurationFileProvider(fileSystem)
+        let configurationFileProvider = ConfigurationFileProvider(new TestLoggerFactory(), fileSystem)
         let configurationContext = ConfigurationContext(
             VersionsConfiguration = versionConfig,
             ConfigurationFileProvider = configurationFileProvider,