Add caching for Kroki diagrams

docs/_docset.yml

@@ -86,7 +86,7 @@ toc:
       - file: code.md
       - file: comments.md
       - file: conditionals.md
-      - hidden: diagrams.md
+      - file: diagrams.md
       - file: dropdowns.md
       - file: definition-lists.md
       - file: example_blocks.md

docs/syntax/diagrams.md

@@ -2,6 +2,10 @@
 
 The `diagram` directive allows you to render various types of diagrams using the [Kroki](https://kroki.io/) service. Kroki supports many diagram types including Mermaid, D2, Graphviz, PlantUML, and more.
 
+::::{warning}
+This is an experimental feature. It may change in the future.
+::::
+
 ## Basic usage
 
 The basic syntax for the diagram directive is:
@@ -84,7 +88,7 @@ sequenceDiagram
 :::::{tab-item} Rendered
 ::::{diagram} mermaid
 sequenceDiagram
-    participant A as Alice
+    participant A as Ada
     participant B as Bob
     A->>B: Hello Bob, how are you?
     B-->>A: Great!

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -16,6 +16,7 @@
 using Elastic.Markdown.Helpers;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.Links.CrossLinks;
+using Elastic.Markdown.Myst.Directives.Diagram;
 using Elastic.Markdown.Myst.Renderers;
 using Elastic.Markdown.Myst.Renderers.LlmMarkdown;
 using Markdig.Syntax;
@@ -106,6 +107,9 @@ public async Task ResolveDirectoryTree(Cancel ctx)
 
 	public async Task<GenerationResult> GenerateAll(Cancel ctx)
 	{
+		// Clear diagram registry for fresh tracking
+		DiagramRegistry.Clear();
+
 		var result = new GenerationResult();
 
 		var generationState = Context.SkipDocumentationState ? null : GetPreviousGenerationState();
@@ -142,6 +146,13 @@ public async Task<GenerationResult> GenerateAll(Cancel ctx)
 		_logger.LogInformation($"Generating links.json");
 		var linkReference = await GenerateLinkReference(ctx);
 
+		// Clean up unused diagram files
+		var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory.FullName);
+		if (cleanedCount > 0)
+		{
+			_logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount);
+		}
+
 		// ReSharper disable once WithExpressionModifiesAllMembers
 		return result with
 		{

src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs

@@ -2,6 +2,8 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Security.Cryptography;
+using System.Text;
 using Elastic.Markdown.Diagnostics;
 
 namespace Elastic.Markdown.Myst.Directives.Diagram;
@@ -25,6 +27,16 @@ public class DiagramBlock(DirectiveBlockParser parser, ParserContext context) :
 	/// </summary>
 	public string? EncodedUrl { get; private set; }
 
+	/// <summary>
+	/// The local SVG path relative to the output directory
+	/// </summary>
+	public string? LocalSvgPath { get; private set; }
+
+	/// <summary>
+	/// Content hash for unique identification and caching
+	/// </summary>
+	public string? ContentHash { get; private set; }
+
 	public override void FinalizeAndValidate(ParserContext context)
 	{
 		// Extract diagram type from arguments or default to "mermaid"
@@ -39,6 +51,12 @@ public override void FinalizeAndValidate(ParserContext context)
 			return;
 		}
 
+		// Generate content hash for caching
+		ContentHash = GenerateContentHash(DiagramType, Content);
+
+		// Generate local path for cached SVG
+		LocalSvgPath = GenerateLocalPath(context);
+
 		// Generate the encoded URL for Kroki
 		try
 		{
@@ -47,7 +65,15 @@ public override void FinalizeAndValidate(ParserContext context)
 		catch (Exception ex)
 		{
 			this.EmitError($"Failed to encode diagram: {ex.Message}", ex);
+			return;
 		}
+
+		// Register diagram for tracking and cleanup
+		DiagramRegistry.RegisterDiagram(LocalSvgPath);
+
+		// Cache diagram asynchronously - fire and forget
+		// Use simplified approach without lock files to avoid orphaned locks
+		_ = Task.Run(() => TryCacheDiagramAsync(context));
 	}
 
 	private string? ExtractContent()
@@ -68,4 +94,84 @@ public override void FinalizeAndValidate(ParserContext context)
 
 		return lines.Count > 0 ? string.Join("\n", lines) : null;
 	}
+
+	private string GenerateContentHash(string diagramType, string content)
+	{
+		var input = $"{diagramType}:{content}";
+		var bytes = Encoding.UTF8.GetBytes(input);
+		var hash = SHA256.HashData(bytes);
+		return Convert.ToHexString(hash)[..12].ToLowerInvariant();
+	}
+
+	private string GenerateLocalPath(ParserContext context)
+	{
+		var markdownFileName = "unknown";
+		if (context.MarkdownSourcePath?.FullName != null)
+		{
+			markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.FullName);
+		}
+
+		var filename = $"{markdownFileName}-diagram-{DiagramType}-{ContentHash}.svg";
+		return Path.Combine("images", "generated-graphs", filename);
+	}
+
+	private async Task TryCacheDiagramAsync(ParserContext context)
+	{
+		if (string.IsNullOrEmpty(EncodedUrl) || string.IsNullOrEmpty(LocalSvgPath))
+			return;
+
+		try
+		{
+			// Determine the full output path
+			var outputDirectory = context.Build.DocumentationOutputDirectory.FullName;
+			var fullPath = Path.Combine(outputDirectory, LocalSvgPath);
+
+			// Skip if file already exists - simple check without locking
+			if (File.Exists(fullPath))
+				return;
+
+			// Create directory if it doesn't exist
+			var directory = Path.GetDirectoryName(fullPath);
+			if (directory != null && !Directory.Exists(directory))
+			{
+				_ = Directory.CreateDirectory(directory);
+			}
+
+			// Download SVG from Kroki using shared HttpClient
+			var svgContent = await DiagramHttpClient.Instance.GetStringAsync(EncodedUrl);
+
+			// Basic validation - ensure we got SVG content
+			// SVG can start with XML declaration, DOCTYPE, or directly with <svg>
+			if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains("<svg", StringComparison.OrdinalIgnoreCase))
+			{
+				// Invalid content - don't cache
+				return;
+			}
+
+			// Write to local file atomically using a temp file
+			var tempPath = fullPath + ".tmp";
+			await File.WriteAllTextAsync(tempPath, svgContent);
+			File.Move(tempPath, fullPath);
+		}
+		catch (HttpRequestException)
+		{
+			// Network-related failures - silent fallback to Kroki URLs
+			// Caching is opportunistic, network issues shouldn't generate warnings
+		}
+		catch (TaskCanceledException ex) when (ex.InnerException is TimeoutException)
+		{
+			// Timeout - silent fallback to Kroki URLs
+			// Timeouts are expected in slow network conditions
+		}
+		catch (IOException)
+		{
+			// File system issues - silent fallback to Kroki URLs
+			// Disk space or permission issues shouldn't break builds
+		}
+		catch (Exception)
+		{
+			// Unexpected errors - silent fallback to Kroki URLs
+			// Caching is opportunistic, any failure should fallback gracefully
+		}
+	}
 }

src/Elastic.Markdown/Myst/Directives/Diagram/DiagramHttpClient.cs

@@ -0,0 +1,21 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Markdown.Myst.Directives.Diagram;
+
+/// <summary>
+/// Shared HttpClient for diagram downloads to avoid resource exhaustion
+/// </summary>
+public static class DiagramHttpClient
+{
+	private static readonly Lazy<HttpClient> LazyHttpClient = new(() => new HttpClient
+	{
+		Timeout = TimeSpan.FromSeconds(30)
+	});
+
+	/// <summary>
+	/// Shared HttpClient instance for diagram downloads
+	/// </summary>
+	public static HttpClient Instance => LazyHttpClient.Value;
+}

src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs

@@ -0,0 +1,146 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+
+namespace Elastic.Markdown.Myst.Directives.Diagram;
+
+/// <summary>
+/// Registry to track active diagrams and manage cleanup of outdated cached files
+/// </summary>
+public static class DiagramRegistry
+{
+	private static readonly HashSet<string> ActiveDiagrams = [];
+	private static readonly Lock Lock = new();
+
+	/// <summary>
+	/// Register a diagram as active during the current build
+	/// </summary>
+	/// <param name="localSvgPath">The local SVG path relative to output directory</param>
+	public static void RegisterDiagram(string localSvgPath)
+	{
+		if (string.IsNullOrEmpty(localSvgPath))
+			return;
+
+		lock (Lock)
+		{
+			_ = ActiveDiagrams.Add(localSvgPath);
+		}
+	}
+
+	/// <summary>
+	/// Get all currently registered active diagrams
+	/// </summary>
+	/// <returns>Collection of active diagram paths</returns>
+	public static IReadOnlyCollection<string> GetActiveDiagrams()
+	{
+		lock (Lock)
+		{
+			return ActiveDiagrams.ToArray();
+		}
+	}
+
+	/// <summary>
+	/// Clear all registered diagrams (typically called at start of build)
+	/// </summary>
+	public static void Clear()
+	{
+		lock (Lock)
+		{
+			ActiveDiagrams.Clear();
+		}
+	}
+
+	/// <summary>
+	/// Clean up unused diagram files from the output directory
+	/// </summary>
+	/// <param name="outputDirectory">The output directory path</param>
+	/// <returns>Number of files cleaned up</returns>
+	public static int CleanupUnusedDiagrams(string outputDirectory) =>
+		CleanupUnusedDiagrams(outputDirectory, new FileSystem());
+
+	/// <summary>
+	/// Clean up unused diagram files from the output directory
+	/// </summary>
+	/// <param name="outputDirectory">The output directory path</param>
+	/// <param name="fileSystem">File system abstraction for testing</param>
+	/// <returns>Number of files cleaned up</returns>
+	public static int CleanupUnusedDiagrams(string outputDirectory, IFileSystem fileSystem)
+	{
+		if (string.IsNullOrEmpty(outputDirectory))
+			return 0;
+
+		var graphsDir = fileSystem.Path.Combine(outputDirectory, "images", "generated-graphs");
+		if (!fileSystem.Directory.Exists(graphsDir))
+			return 0;
+
+		var cleanedCount = 0;
+		var activePaths = GetActiveDiagrams();
+
+		try
+		{
+			var existingFiles = fileSystem.Directory.GetFiles(graphsDir, "*.svg", SearchOption.AllDirectories);
+
+			foreach (var file in existingFiles)
+			{
+				var relativePath = fileSystem.Path.GetRelativePath(outputDirectory, file);
+
+				// Convert to forward slashes for consistent comparison
+				var normalizedPath = relativePath.Replace(fileSystem.Path.DirectorySeparatorChar, '/');
+
+				if (!activePaths.Any(active => active.Replace(fileSystem.Path.DirectorySeparatorChar, '/') == normalizedPath))
+				{
+					try
+					{
+						fileSystem.File.Delete(file);
+						cleanedCount++;
+					}
+					catch
+					{
+						// Silent failure - cleanup is opportunistic
+					}
+				}
+			}
+
+			// Clean up empty directories
+			CleanupEmptyDirectories(graphsDir, fileSystem);
+		}
+		catch
+		{
+			// Silent failure - cleanup is opportunistic
+		}
+
+		return cleanedCount;
+	}
+
+	/// <summary>
+	/// Remove empty directories recursively
+	/// </summary>
+	/// <param name="directory">Directory to clean up</param>
+	/// <param name="fileSystem">File system abstraction</param>
+	private static void CleanupEmptyDirectories(string directory, IFileSystem fileSystem)
+	{
+		try
+		{
+			if (!fileSystem.Directory.Exists(directory))
+				return;
+
+			// Clean up subdirectories first
+			foreach (var subDir in fileSystem.Directory.GetDirectories(directory))
+			{
+				CleanupEmptyDirectories(subDir, fileSystem);
+			}
+
+			// Remove directory if it's empty
+			if (!fileSystem.Directory.EnumerateFileSystemEntries(directory).Any())
+			{
+				fileSystem.Directory.Delete(directory);
+			}
+		}
+		catch
+		{
+			// Silent failure - cleanup is opportunistic
+		}
+	}
+}

src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml

@@ -4,7 +4,17 @@
 	if (diagram?.EncodedUrl != null)
 	{
 		<div class="diagram" data-diagram-type="@diagram.DiagramType">
-			<img src="@diagram.EncodedUrl" alt="@diagram.DiagramType diagram" loading="lazy" />
+			@if (!string.IsNullOrEmpty(diagram.LocalSvgPath))
+			{
+				<img src="/@diagram.LocalSvgPath" 
+				     alt="@diagram.DiagramType diagram" 
+				     loading="lazy"
+				     onerror="this.src='@diagram.EncodedUrl'" />
+			}
+			else
+			{
+				<img src="@diagram.EncodedUrl" alt="@diagram.DiagramType diagram" loading="lazy" />
+			}
 		</div>
 	}
 	else