From 9c3baad79ed10b584f692191eeb4c5b0e5925306 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 18 Sep 2024 15:50:33 +1000 Subject: [PATCH 01/12] WaitFor for Azure Storage --- .../AzureStorageEndToEnd.AppHost/Program.cs | 2 +- playground/mongo/Mongo.ApiService/Program.cs | 2 +- playground/mongo/Mongo.AppHost/Program.cs | 3 +- .../Aspire.Hosting.Azure.Storage.csproj | 5 +++ .../AzureStorageExtensions.cs | 35 +++++++++++++++ .../Provisioners/AzureProvisioner.cs | 24 ++++++++-- .../MilvusBuilderExtensions.cs | 19 ++------ .../MongoDBBuilderExtensions.cs | 15 +++++++ .../MySqlBuilderExtensions.cs | 16 ------- .../OracleDatabaseBuilderExtensions.cs | 16 ------- .../PostgresBuilderExtensions.cs | 16 ------- .../SqlServerBuilderExtensions.cs | 16 ------- src/Aspire.Hosting/Dcp/ApplicationExecutor.cs | 29 ++++++++---- .../AzureCosmosDBEmulatorFunctionalTests.cs | 1 - .../AzureStorageEmulatorFunctionalTests.cs | 45 +++++++++++++++++++ 15 files changed, 150 insertions(+), 94 deletions(-) diff --git a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs index 68e78dd822..4f025b4878 100644 --- a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs +++ b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs @@ -11,7 +11,7 @@ builder.AddProject("api") .WithExternalHttpEndpoints() - .WithReference(blobs); + .WithReference(blobs).WaitFor(blobs); #if !SKIP_DASHBOARD_REFERENCE // This project is only added in playground projects to support development/debugging diff --git a/playground/mongo/Mongo.ApiService/Program.cs b/playground/mongo/Mongo.ApiService/Program.cs index 714bfcce1b..c2ae7038b3 100644 --- a/playground/mongo/Mongo.ApiService/Program.cs +++ b/playground/mongo/Mongo.ApiService/Program.cs @@ -8,7 +8,7 @@ var builder = WebApplication.CreateBuilder(args); builder.AddServiceDefaults(); -builder.AddMongoDBClient("mongo"); +builder.AddMongoDBClient("db"); var app = builder.Build(); diff --git a/playground/mongo/Mongo.AppHost/Program.cs b/playground/mongo/Mongo.AppHost/Program.cs index e76f2c911c..576f36eb7b 100644 --- a/playground/mongo/Mongo.AppHost/Program.cs +++ b/playground/mongo/Mongo.AppHost/Program.cs @@ -5,7 +5,8 @@ var db = builder.AddMongoDB("mongo") .WithMongoExpress(c => c.WithHostPort(3022)) - .PublishAsContainer(); + .PublishAsContainer() + .AddDatabase("db"); builder.AddProject("api") .WithExternalHttpEndpoints() diff --git a/src/Aspire.Hosting.Azure.Storage/Aspire.Hosting.Azure.Storage.csproj b/src/Aspire.Hosting.Azure.Storage/Aspire.Hosting.Azure.Storage.csproj index ee4df8d25b..3df8ac9336 100644 --- a/src/Aspire.Hosting.Azure.Storage/Aspire.Hosting.Azure.Storage.csproj +++ b/src/Aspire.Hosting.Azure.Storage/Aspire.Hosting.Azure.Storage.csproj @@ -16,6 +16,11 @@ + + + + + diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index cb4bd9e957..bf66c4da05 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -6,6 +6,7 @@ using Aspire.Hosting.Azure; using Aspire.Hosting.Azure.Storage; using Aspire.Hosting.Utils; +using Azure.Identity; using Azure.Provisioning; using Azure.Provisioning.Storage; @@ -197,7 +198,41 @@ public static IResourceBuilder AddBlobs(this IResource { var resource = new AzureBlobStorageResource(name, builder.Resource); + BlobServiceClient? blobServiceClient = null; + + builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => + { + var connectionString = await resource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); + + if (connectionString == null) + { + throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); + } + + blobServiceClient = CreateBlobServiceClient(connectionString); + }); + + //var healthCheckKey = $"{name}_blob_check"; + //builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureBlobStorage(sp => + //{ + // return blobServiceClient ?? throw new InvalidOperationException("BlobServiceClient is not initialized."); + //}, name: healthCheckKey); + return builder.ApplicationBuilder.AddResource(resource); +// .WithHealthCheck(healthCheckKey); + + static BlobServiceClient CreateBlobServiceClient(string connectionString) + { + if (Uri.TryCreate(connectionString, UriKind.Absolute, out var uri)) + { + return new BlobServiceClient(uri, new DefaultAzureCredential()); + } + else + { + return new BlobServiceClient(connectionString); + } + } + } /// diff --git a/src/Aspire.Hosting.Azure/Provisioning/Provisioners/AzureProvisioner.cs b/src/Aspire.Hosting.Azure/Provisioning/Provisioners/AzureProvisioner.cs index 6cdf39fe8d..74732485b9 100644 --- a/src/Aspire.Hosting.Azure/Provisioning/Provisioners/AzureProvisioner.cs +++ b/src/Aspire.Hosting.Azure/Provisioning/Provisioners/AzureProvisioner.cs @@ -68,6 +68,8 @@ IDistributedApplicationEventing eventing return azureResources; } + private ILookup? _parentChildLookup; + public async Task BeforeStartAsync(DistributedApplicationModel appModel, CancellationToken cancellationToken = default) { var azureResources = GetAzureResourcesFromAppModel(appModel); @@ -92,8 +94,15 @@ public async Task BeforeStartAsync(DistributedApplicationModel appModel, Cancell return; } - // Create a map of parents to their children used to propagate state changes later. - var parentChildLookup = appModel.Resources.OfType().ToLookup(r => r.Parent); + static IResource? SelectParentResource(IResource resource) => resource switch + { + IAzureResource ar => ar, + IResourceWithParent rp => SelectParentResource(rp.Parent), + _ => null + }; + + // Create a map of parents to their children used to propogate state changes later. + _parentChildLookup = appModel.Resources.OfType().ToLookup(r => r.Parent); // Sets the state of the resource and all of its children async Task UpdateStateAsync((IResource Resource, IAzureResource AzureResource) resource, Func stateFactory) @@ -110,7 +119,7 @@ async Task UpdateStateAsync((IResource Resource, IAzureResource AzureResource) r // We basically want child resources to be moved into the same state as their parent resources whenever // there is a state update. This is done for us in DCP so we replicate the behavior here in the Azure Provisioner. - var childResources = parentChildLookup[resource.Resource]; + var childResources = _parentChildLookup[resource.Resource]; foreach (var child in childResources) { await notificationService.PublishUpdateAsync(child, stateFactory).ConfigureAwait(false); @@ -327,6 +336,15 @@ async Task PublishConnectionStringAvailableEventAsync() { var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(resource.Resource, serviceProvider); await eventing.PublishAsync(connectionStringAvailableEvent, cancellationToken).ConfigureAwait(false); + + if (_parentChildLookup![resource.Resource] is { } children) + { + foreach (var child in children.OfType()) + { + var childConnectionStringAvailableEvent = new ConnectionStringAvailableEvent(child, serviceProvider); + await eventing.PublishAsync(childConnectionStringAvailableEvent, cancellationToken).ConfigureAwait(false); + } + } } } diff --git a/src/Aspire.Hosting.Milvus/MilvusBuilderExtensions.cs b/src/Aspire.Hosting.Milvus/MilvusBuilderExtensions.cs index 74110349e5..cdc56e5175 100644 --- a/src/Aspire.Hosting.Milvus/MilvusBuilderExtensions.cs +++ b/src/Aspire.Hosting.Milvus/MilvusBuilderExtensions.cs @@ -65,19 +65,6 @@ public static IResourceBuilder AddMilvus(this IDistributed ?? throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{milvus.Name}' resource but the connection string was null."); milvusClient = CreateMilvusClient(@event.Services, connectionString); - var lookup = builder.Resources.OfType().ToDictionary(d => d.Name); - foreach (var databaseName in milvus.Databases) - { - if (!lookup.TryGetValue(databaseName.Key, out var databaseResource)) - { - throw new DistributedApplicationException($"Database resource '{databaseName}' under Milvus Server resource '{milvus.Name}' was not found in the model."); - } - var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(connectionStringAvailableEvent, ct).ConfigureAwait(false); - - var beforeResourceStartedEvent = new BeforeResourceStartedEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(beforeResourceStartedEvent, ct).ConfigureAwait(false); - } }); var healthCheckKey = $"{name}_check"; @@ -144,15 +131,17 @@ public static IResourceBuilder AddDatabase(this IResourc builder.Resource.AddDatabase(name, databaseName); var milvusDatabaseResource = new MilvusDatabaseResource(name, databaseName, builder.Resource); - string? connectionString = null; MilvusClient? milvusClient = null; + builder.ApplicationBuilder.Eventing.Subscribe(milvusDatabaseResource, async (@event, ct) => { - connectionString = await milvusDatabaseResource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); + var connectionString = await milvusDatabaseResource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); + if (connectionString == null) { throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{milvusDatabaseResource.Name}' resource but the connection string was null."); } + milvusClient = CreateMilvusClient(@event.Services, connectionString); }); diff --git a/src/Aspire.Hosting.MongoDB/MongoDBBuilderExtensions.cs b/src/Aspire.Hosting.MongoDB/MongoDBBuilderExtensions.cs index 2d23e65582..e6e7b68ef5 100644 --- a/src/Aspire.Hosting.MongoDB/MongoDBBuilderExtensions.cs +++ b/src/Aspire.Hosting.MongoDB/MongoDBBuilderExtensions.cs @@ -101,6 +101,21 @@ public static IResourceBuilder AddDatabase(this IResour builder.Resource.AddDatabase(name, databaseName); var mongoDBDatabase = new MongoDBDatabaseResource(name, databaseName, builder.Resource); + string? connectionString = null; + + builder.ApplicationBuilder.Eventing.Subscribe(mongoDBDatabase, async (@event, ct) => + { + connectionString = await mongoDBDatabase.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); + + if (connectionString == null) + { + throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{mongoDBDatabase.Name}' resource but the connection string was null."); + } + }); + + var healthCheckKey = $"{name}_check"; + builder.ApplicationBuilder.Services.AddHealthChecks().AddMongoDb(sp => connectionString ?? throw new InvalidOperationException("Connection string is unavailable"), name: healthCheckKey); + return builder.ApplicationBuilder .AddResource(mongoDBDatabase); } diff --git a/src/Aspire.Hosting.MySql/MySqlBuilderExtensions.cs b/src/Aspire.Hosting.MySql/MySqlBuilderExtensions.cs index 6d9cd0bb2e..570acf10e8 100644 --- a/src/Aspire.Hosting.MySql/MySqlBuilderExtensions.cs +++ b/src/Aspire.Hosting.MySql/MySqlBuilderExtensions.cs @@ -42,22 +42,6 @@ public static IResourceBuilder AddMySql(this IDistributedAp { throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); } - - var lookup = builder.Resources.OfType().ToDictionary(d => d.Name); - - foreach (var databaseName in resource.Databases) - { - if (!lookup.TryGetValue(databaseName.Key, out var databaseResource)) - { - throw new DistributedApplicationException($"Database resource '{databaseName}' under MySql resource '{resource.Name}' was not found in the model."); - } - - var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(connectionStringAvailableEvent, ct).ConfigureAwait(false); - - var beforeResourceStartedEvent = new BeforeResourceStartedEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(beforeResourceStartedEvent, ct).ConfigureAwait(false); - } }); var healthCheckKey = $"{name}_check"; diff --git a/src/Aspire.Hosting.Oracle/OracleDatabaseBuilderExtensions.cs b/src/Aspire.Hosting.Oracle/OracleDatabaseBuilderExtensions.cs index 59465784e9..42e6435f58 100644 --- a/src/Aspire.Hosting.Oracle/OracleDatabaseBuilderExtensions.cs +++ b/src/Aspire.Hosting.Oracle/OracleDatabaseBuilderExtensions.cs @@ -38,22 +38,6 @@ public static IResourceBuilder AddOracle(this IDis { throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{oracleDatabaseServer.Name}' resource but the connection string was null."); } - - var lookup = builder.Resources.OfType().ToDictionary(d => d.Name); - - foreach (var databaseName in oracleDatabaseServer.Databases) - { - if (!lookup.TryGetValue(databaseName.Key, out var databaseResource)) - { - throw new DistributedApplicationException($"Database resource '{databaseName}' under Oracle resource '{oracleDatabaseServer.Name}' was not found in the model."); - } - - var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(connectionStringAvailableEvent, ct).ConfigureAwait(false); - - var beforeResourceStartedEvent = new BeforeResourceStartedEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(beforeResourceStartedEvent, ct).ConfigureAwait(false); - } }); var healthCheckKey = $"{name}_check"; diff --git a/src/Aspire.Hosting.PostgreSQL/PostgresBuilderExtensions.cs b/src/Aspire.Hosting.PostgreSQL/PostgresBuilderExtensions.cs index 0f2b866d56..feea322bca 100644 --- a/src/Aspire.Hosting.PostgreSQL/PostgresBuilderExtensions.cs +++ b/src/Aspire.Hosting.PostgreSQL/PostgresBuilderExtensions.cs @@ -58,22 +58,6 @@ public static IResourceBuilder AddPostgres(this IDistrib { throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{postgresServer.Name}' resource but the connection string was null."); } - - var lookup = builder.Resources.OfType().ToDictionary(d => d.Name); - - foreach (var databaseName in postgresServer.Databases) - { - if (!lookup.TryGetValue(databaseName.Key, out var databaseResource)) - { - throw new DistributedApplicationException($"Database resource '{databaseName}' under Postgres server resource '{postgresServer.Name}' not in model."); - } - - var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(connectionStringAvailableEvent, ct).ConfigureAwait(false); - - var beforeResourceStartedEvent = new BeforeResourceStartedEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(beforeResourceStartedEvent, ct).ConfigureAwait(false); - } }); var healthCheckKey = $"{name}_check"; diff --git a/src/Aspire.Hosting.SqlServer/SqlServerBuilderExtensions.cs b/src/Aspire.Hosting.SqlServer/SqlServerBuilderExtensions.cs index 0bcdd6f384..809277b5d0 100644 --- a/src/Aspire.Hosting.SqlServer/SqlServerBuilderExtensions.cs +++ b/src/Aspire.Hosting.SqlServer/SqlServerBuilderExtensions.cs @@ -40,22 +40,6 @@ public static IResourceBuilder AddSqlServer(this IDistr { throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{sqlServer.Name}' resource but the connection string was null."); } - - var lookup = builder.Resources.OfType().ToDictionary(d => d.Name); - - foreach (var databaseName in sqlServer.Databases) - { - if (!lookup.TryGetValue(databaseName.Key, out var databaseResource)) - { - throw new DistributedApplicationException($"Database resource '{databaseName}' under SQL Server resource '{sqlServer.Name}' was not found in the model."); - } - - var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(connectionStringAvailableEvent, ct).ConfigureAwait(false); - - var beforeResourceStartedEvent = new BeforeResourceStartedEvent(databaseResource, @event.Services); - await builder.Eventing.PublishAsync(beforeResourceStartedEvent, ct).ConfigureAwait(false); - } }); var healthCheckKey = $"{name}_check"; diff --git a/src/Aspire.Hosting/Dcp/ApplicationExecutor.cs b/src/Aspire.Hosting/Dcp/ApplicationExecutor.cs index 8085a28b6e..a2dcda02e7 100644 --- a/src/Aspire.Hosting/Dcp/ApplicationExecutor.cs +++ b/src/Aspire.Hosting/Dcp/ApplicationExecutor.cs @@ -1250,14 +1250,31 @@ await notificationService.PublishUpdateAsync(cr.ModelResource, s => s with } } - private async Task CreateExecutableAsync(AppResource er, ILogger resourceLogger, CancellationToken cancellationToken) + private async Task PublishConnectionStringAvailableEvent(IResource resource, CancellationToken cancellationToken) { - if (er.ModelResource is IResourceWithConnectionString) + // If the resource itself has a connection string then publish that the connection string is available. + if (resource is IResourceWithConnectionString) { - var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(er.ModelResource, serviceProvider); + var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(resource, serviceProvider); await eventing.PublishAsync(connectionStringAvailableEvent, cancellationToken).ConfigureAwait(false); } + // Sometimes the container/executable itself does not have a connection string, and in those cases + // we need to dispatch the event for the children. + if (_parentChildLookup[resource] is { } children) + { + foreach (var child in children.OfType()) + { + var childConnectionStringAvailableEvent = new ConnectionStringAvailableEvent(child, serviceProvider); + await eventing.PublishAsync(childConnectionStringAvailableEvent, cancellationToken).ConfigureAwait(false); + } + } + } + + private async Task CreateExecutableAsync(AppResource er, ILogger resourceLogger, CancellationToken cancellationToken) + { + await PublishConnectionStringAvailableEvent(er.ModelResource, cancellationToken).ConfigureAwait(false); + var beforeResourceStartedEvent = new BeforeResourceStartedEvent(er.ModelResource, serviceProvider); await eventing.PublishAsync(beforeResourceStartedEvent, cancellationToken).ConfigureAwait(false); @@ -1549,11 +1566,7 @@ await notificationService.PublishUpdateAsync(cr.ModelResource, s => s with private async Task CreateContainerAsync(AppResource cr, ILogger resourceLogger, CancellationToken cancellationToken) { - if (cr.ModelResource is IResourceWithConnectionString) - { - var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(cr.ModelResource, serviceProvider); - await eventing.PublishAsync(connectionStringAvailableEvent, cancellationToken).ConfigureAwait(false); - } + await PublishConnectionStringAvailableEvent(cr.ModelResource, cancellationToken).ConfigureAwait(false); var beforeResourceStartedEvent = new BeforeResourceStartedEvent(cr.ModelResource, serviceProvider); await eventing.PublishAsync(beforeResourceStartedEvent, cancellationToken).ConfigureAwait(false); diff --git a/tests/Aspire.Hosting.Azure.Tests/AzureCosmosDBEmulatorFunctionalTests.cs b/tests/Aspire.Hosting.Azure.Tests/AzureCosmosDBEmulatorFunctionalTests.cs index 0ff84a13be..76107a010e 100644 --- a/tests/Aspire.Hosting.Azure.Tests/AzureCosmosDBEmulatorFunctionalTests.cs +++ b/tests/Aspire.Hosting.Azure.Tests/AzureCosmosDBEmulatorFunctionalTests.cs @@ -55,5 +55,4 @@ public async Task VerifyWaitForOnCosmosDBEmulatorBlocksDependentResources() await app.StopAsync(); } - } diff --git a/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs b/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs index 8662b9d432..3ea4d7a9df 100644 --- a/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs +++ b/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs @@ -2,9 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. using Aspire.Components.Common.Tests; +using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Utils; using Azure.Storage.Blobs; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Hosting; using Xunit; using Xunit.Abstractions; @@ -13,6 +15,49 @@ namespace Aspire.Hosting.Azure.Tests; public class AzureStorageEmulatorFunctionalTests(ITestOutputHelper testOutputHelper) { + [Fact] + [RequiresDocker] + public async Task VerifyWaitForOnAzureStorageEmulatorForBlobsBlocksDependentResources() + { + var cts = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + using var builder = TestDistributedApplicationBuilder.Create(testOutputHelper); + + var healthCheckTcs = new TaskCompletionSource(); + builder.Services.AddHealthChecks().AddAsyncCheck("blocking_check", () => + { + return healthCheckTcs.Task; + }); + + var resource = builder.AddAzureStorage("resource") + .RunAsEmulator() + .WithHealthCheck("blocking_check") + .AddBlobs("blobs"); + + var dependentResource = builder.AddAzureCosmosDB("dependentresource") + .RunAsEmulator() + .WaitFor(resource); + + using var app = builder.Build(); + + var pendingStart = app.StartAsync(cts.Token); + + var rns = app.Services.GetRequiredService(); + + await rns.WaitForResourceAsync(resource.Resource.Name, KnownResourceStates.Running, cts.Token); + + await rns.WaitForResourceAsync(dependentResource.Resource.Name, KnownResourceStates.Waiting, cts.Token); + + healthCheckTcs.SetResult(HealthCheckResult.Healthy()); + + await rns.WaitForResourceAsync(resource.Resource.Name, (re => re.Snapshot.HealthStatus == HealthStatus.Healthy), cts.Token); + + await rns.WaitForResourceAsync(dependentResource.Resource.Name, KnownResourceStates.Running, cts.Token); + + await pendingStart; + + await app.StopAsync(); + } + [Fact] [RequiresDocker] public async Task VerifyAzureStorageEmulatorResource() From f630e36a89d9f58b4ac436bb444c71c037439dc0 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 18 Sep 2024 16:29:41 +1000 Subject: [PATCH 02/12] Enable health checks on blobs resource. --- .../AzureStorageExtensions.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index bf66c4da05..3f3afc298e 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -9,6 +9,9 @@ using Azure.Identity; using Azure.Provisioning; using Azure.Provisioning.Storage; +using Azure.Storage.Blobs; +using Microsoft.Extensions.DependencyInjection; +//using Microsoft.Extensions.DependencyInjection; namespace Aspire.Hosting; @@ -212,14 +215,14 @@ public static IResourceBuilder AddBlobs(this IResource blobServiceClient = CreateBlobServiceClient(connectionString); }); - //var healthCheckKey = $"{name}_blob_check"; - //builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureBlobStorage(sp => - //{ - // return blobServiceClient ?? throw new InvalidOperationException("BlobServiceClient is not initialized."); - //}, name: healthCheckKey); + var healthCheckKey = $"{name}_blob_check"; + builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureBlobStorage(sp => + { + return blobServiceClient ?? throw new InvalidOperationException("BlobServiceClient is not initialized."); + }, name: healthCheckKey); - return builder.ApplicationBuilder.AddResource(resource); -// .WithHealthCheck(healthCheckKey); + return builder.ApplicationBuilder.AddResource(resource) + .WithHealthCheck(healthCheckKey); static BlobServiceClient CreateBlobServiceClient(string connectionString) { From a603507ced117f76d4cd53590670b925b5f95d3b Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 18 Sep 2024 16:34:46 +1000 Subject: [PATCH 03/12] Add Queues health check. --- .../AzureStorageExtensions.cs | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index 3f3afc298e..b0a7223a05 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -10,6 +10,7 @@ using Azure.Provisioning; using Azure.Provisioning.Storage; using Azure.Storage.Blobs; +using Azure.Storage.Queues; using Microsoft.Extensions.DependencyInjection; //using Microsoft.Extensions.DependencyInjection; @@ -215,7 +216,7 @@ public static IResourceBuilder AddBlobs(this IResource blobServiceClient = CreateBlobServiceClient(connectionString); }); - var healthCheckKey = $"{name}_blob_check"; + var healthCheckKey = $"{name}_check"; builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureBlobStorage(sp => { return blobServiceClient ?? throw new InvalidOperationException("BlobServiceClient is not initialized."); @@ -261,6 +262,39 @@ public static IResourceBuilder AddQueues(this IResour { var resource = new AzureQueueStorageResource(name, builder.Resource); - return builder.ApplicationBuilder.AddResource(resource); + QueueServiceClient? queueServiceClient = null; + + builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => + { + var connectionString = await resource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); + + if (connectionString == null) + { + throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); + } + + queueServiceClient = CreateQueueServiceClient(connectionString); + }); + + var healthCheckKey = $"{name}_check"; + builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureQueueStorage(sp => + { + return queueServiceClient ?? throw new InvalidOperationException("QueueServiceClient is not initialized."); + }, name: healthCheckKey); + + return builder.ApplicationBuilder.AddResource(resource) + .WithHealthCheck(healthCheckKey); + + static QueueServiceClient CreateQueueServiceClient(string connectionString) + { + if (Uri.TryCreate(connectionString, UriKind.Absolute, out var uri)) + { + return new QueueServiceClient(uri, new DefaultAzureCredential()); + } + else + { + return new QueueServiceClient(connectionString); + } + } } } From 7d22f6d6c3f31d79342728a6944958512a6db9c5 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 18 Sep 2024 16:38:04 +1000 Subject: [PATCH 04/12] Add queues logic to playground app. --- .../AzureStorageEndToEnd.ApiService.csproj | 1 + .../AzureStorageEndToEnd.ApiService/Program.cs | 8 +++++++- .../AzureStorageEndToEnd.AppHost/Program.cs | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.ApiService/AzureStorageEndToEnd.ApiService.csproj b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.ApiService/AzureStorageEndToEnd.ApiService.csproj index ed383d7858..66e0c13e3e 100644 --- a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.ApiService/AzureStorageEndToEnd.ApiService.csproj +++ b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.ApiService/AzureStorageEndToEnd.ApiService.csproj @@ -8,6 +8,7 @@ + diff --git a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.ApiService/Program.cs b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.ApiService/Program.cs index 60c8707bdd..16e56ead76 100644 --- a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.ApiService/Program.cs +++ b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.ApiService/Program.cs @@ -2,17 +2,19 @@ // The .NET Foundation licenses this file to you under the MIT license. using Azure.Storage.Blobs; +using Azure.Storage.Queues; var builder = WebApplication.CreateBuilder(args); builder.AddServiceDefaults(); builder.AddAzureBlobClient("blobs"); +builder.AddAzureQueueClient("queues"); var app = builder.Build(); app.MapDefaultEndpoints(); -app.MapGet("/", async (BlobServiceClient bsc) => +app.MapGet("/", async (BlobServiceClient bsc, QueueServiceClient qsc) => { var container = bsc.GetBlobContainerClient("mycontainer"); await container.CreateIfNotExistsAsync(); @@ -29,6 +31,10 @@ blobNames.Add(blob.Name); } + var queue = qsc.GetQueueClient("myqueue"); + await queue.CreateIfNotExistsAsync(); + await queue.SendMessageAsync("Hello, world!"); + return blobNames; }); diff --git a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs index 4f025b4878..bd37afb0ae 100644 --- a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs +++ b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs @@ -8,10 +8,12 @@ }); var blobs = storage.AddBlobs("blobs"); +var queues = storage.AddQueues("queues"); builder.AddProject("api") .WithExternalHttpEndpoints() - .WithReference(blobs).WaitFor(blobs); + .WithReference(blobs).WaitFor(blobs) + .WithReference(queues).WaitFor(queues); #if !SKIP_DASHBOARD_REFERENCE // This project is only added in playground projects to support development/debugging From 979a6b24beca245cdd53305f55ff71174b694850 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 30 Sep 2024 18:48:03 +1000 Subject: [PATCH 05/12] Basic working model. --- .../AzureStorageEndToEnd.AppHost/Program.cs | 1 + .../Aspire.Hosting.Azure.Storage.csproj | 1 - .../AzureStorageExtensions.cs | 123 +++++++++--------- 3 files changed, 61 insertions(+), 64 deletions(-) diff --git a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs index bd37afb0ae..be74f6869c 100644 --- a/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs +++ b/playground/AzureStorageEndToEnd/AzureStorageEndToEnd.AppHost/Program.cs @@ -1,5 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. + var builder = DistributedApplication.CreateBuilder(args); var storage = builder.AddAzureStorage("storage").RunAsEmulator(container => diff --git a/src/Aspire.Hosting.Azure.Storage/Aspire.Hosting.Azure.Storage.csproj b/src/Aspire.Hosting.Azure.Storage/Aspire.Hosting.Azure.Storage.csproj index 3df8ac9336..b934c144be 100644 --- a/src/Aspire.Hosting.Azure.Storage/Aspire.Hosting.Azure.Storage.csproj +++ b/src/Aspire.Hosting.Azure.Storage/Aspire.Hosting.Azure.Storage.csproj @@ -18,7 +18,6 @@ - diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index 747d2e1012..2325387892 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -10,7 +10,6 @@ using Azure.Provisioning; using Azure.Provisioning.Storage; using Azure.Storage.Blobs; -using Azure.Storage.Queues; using Microsoft.Extensions.DependencyInjection; //using Microsoft.Extensions.DependencyInjection; @@ -90,11 +89,58 @@ public static IResourceBuilder AddAzureStorage(this IDistr }; var resource = new AzureStorageResource(name, configureConstruct); + var lockObject = new object(); + BlobServiceClient? blobServiceClient = null; + + builder.Eventing.Subscribe(resource, async (@event, ct) => + { + // HACK: AzureStorageResource does not implement IResourceWithConnectionString which means that + // we don't fire off a ConnectionStringAvailableEvent for it. To work around this the blob, + // table, and queue resources each propogate the event to the parent (the eventing system + // doesn't care about connection strings. For the actual health check we just use the + // blob service client in the parent rather than each of the child resources and rely + // on the resource health check service to propogate the health checks. + // + // TODO: There is potential race condition here to solve. + if (blobServiceClient != null) + { + return; + } + + var connectionString = await resource.GetBlobConnectionString().GetValueAsync(ct).ConfigureAwait(false); + + if (connectionString == null) + { + throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); + } + + blobServiceClient = CreateBlobServiceClient(connectionString); + }); + + var healthCheckKey = $"{name}_check"; + builder.Services.AddHealthChecks().AddAzureBlobStorage(sp => + { + return blobServiceClient ?? throw new InvalidOperationException("BlobServiceClient is not initialized."); + }, name: healthCheckKey); + return builder.AddResource(resource) // These ambient parameters are only available in development time. .WithParameter(AzureBicepResource.KnownParameters.PrincipalId) .WithParameter(AzureBicepResource.KnownParameters.PrincipalType) - .WithManifestPublishingCallback(resource.WriteToManifest); + .WithManifestPublishingCallback(resource.WriteToManifest) + .WithHealthCheck(healthCheckKey); + + static BlobServiceClient CreateBlobServiceClient(string connectionString) + { + if (Uri.TryCreate(connectionString, UriKind.Absolute, out var uri)) + { + return new BlobServiceClient(uri, new DefaultAzureCredential()); + } + else + { + return new BlobServiceClient(connectionString); + } + } } /// @@ -202,41 +248,13 @@ public static IResourceBuilder AddBlobs(this IResource { var resource = new AzureBlobStorageResource(name, builder.Resource); - BlobServiceClient? blobServiceClient = null; - builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => { - var connectionString = await resource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); - - if (connectionString == null) - { - throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); - } - - blobServiceClient = CreateBlobServiceClient(connectionString); + var propogatedEvent = new ConnectionStringAvailableEvent(resource.Parent, @event.Services); + await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); }); - var healthCheckKey = $"{name}_check"; - builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureBlobStorage(sp => - { - return blobServiceClient ?? throw new InvalidOperationException("BlobServiceClient is not initialized."); - }, name: healthCheckKey); - - return builder.ApplicationBuilder.AddResource(resource) - .WithHealthCheck(healthCheckKey); - - static BlobServiceClient CreateBlobServiceClient(string connectionString) - { - if (Uri.TryCreate(connectionString, UriKind.Absolute, out var uri)) - { - return new BlobServiceClient(uri, new DefaultAzureCredential()); - } - else - { - return new BlobServiceClient(connectionString); - } - } - + return builder.ApplicationBuilder.AddResource(resource); } /// @@ -249,6 +267,12 @@ public static IResourceBuilder AddTables(this IResour { var resource = new AzureTableStorageResource(name, builder.Resource); + builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => + { + var propogatedEvent = new ConnectionStringAvailableEvent(resource.Parent, @event.Services); + await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); + }); + return builder.ApplicationBuilder.AddResource(resource); } @@ -262,39 +286,12 @@ public static IResourceBuilder AddQueues(this IResour { var resource = new AzureQueueStorageResource(name, builder.Resource); - QueueServiceClient? queueServiceClient = null; - builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => { - var connectionString = await resource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); - - if (connectionString == null) - { - throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); - } - - queueServiceClient = CreateQueueServiceClient(connectionString); + var propogatedEvent = new ConnectionStringAvailableEvent(resource.Parent, @event.Services); + await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); }); - var healthCheckKey = $"{name}_check"; - builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureQueueStorage(sp => - { - return queueServiceClient ?? throw new InvalidOperationException("QueueServiceClient is not initialized."); - }, name: healthCheckKey); - - return builder.ApplicationBuilder.AddResource(resource) - .WithHealthCheck(healthCheckKey); - - static QueueServiceClient CreateQueueServiceClient(string connectionString) - { - if (Uri.TryCreate(connectionString, UriKind.Absolute, out var uri)) - { - return new QueueServiceClient(uri, new DefaultAzureCredential()); - } - else - { - return new QueueServiceClient(connectionString); - } - } + return builder.ApplicationBuilder.AddResource(resource); } } From 20d907f3a10697ba83442acf369e3b2655e294e0 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 30 Sep 2024 18:53:37 +1000 Subject: [PATCH 06/12] Revise test --- .../AzureStorageEmulatorFunctionalTests.cs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs b/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs index 3ea4d7a9df..195ed93d65 100644 --- a/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs +++ b/tests/Aspire.Hosting.Azure.Tests/AzureStorageEmulatorFunctionalTests.cs @@ -28,14 +28,19 @@ public async Task VerifyWaitForOnAzureStorageEmulatorForBlobsBlocksDependentReso return healthCheckTcs.Task; }); - var resource = builder.AddAzureStorage("resource") + var storage = builder.AddAzureStorage("resource") .RunAsEmulator() - .WithHealthCheck("blocking_check") - .AddBlobs("blobs"); + .WithHealthCheck("blocking_check"); + + var blobs = storage.AddBlobs("blobs"); + var queues = storage.AddQueues("queues"); + var tables = storage.AddTables("tables"); var dependentResource = builder.AddAzureCosmosDB("dependentresource") .RunAsEmulator() - .WaitFor(resource); + .WaitFor(blobs) + .WaitFor(queues) + .WaitFor(tables); using var app = builder.Build(); @@ -43,13 +48,15 @@ public async Task VerifyWaitForOnAzureStorageEmulatorForBlobsBlocksDependentReso var rns = app.Services.GetRequiredService(); - await rns.WaitForResourceAsync(resource.Resource.Name, KnownResourceStates.Running, cts.Token); + await rns.WaitForResourceAsync(storage.Resource.Name, KnownResourceStates.Running, cts.Token); await rns.WaitForResourceAsync(dependentResource.Resource.Name, KnownResourceStates.Waiting, cts.Token); healthCheckTcs.SetResult(HealthCheckResult.Healthy()); - await rns.WaitForResourceAsync(resource.Resource.Name, (re => re.Snapshot.HealthStatus == HealthStatus.Healthy), cts.Token); + await rns.WaitForResourceHealthyAsync(blobs.Resource.Name, cts.Token); + await rns.WaitForResourceHealthyAsync(queues.Resource.Name, cts.Token); + await rns.WaitForResourceHealthyAsync(tables.Resource.Name, cts.Token); await rns.WaitForResourceAsync(dependentResource.Resource.Name, KnownResourceStates.Running, cts.Token); From 43b519835bbd1a48c3c99648bfd759c91d5fc5cf Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 1 Oct 2024 00:25:30 +1000 Subject: [PATCH 07/12] Update Program.cs Co-authored-by: David Fowler --- playground/mongo/Mongo.AppHost/Program.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/playground/mongo/Mongo.AppHost/Program.cs b/playground/mongo/Mongo.AppHost/Program.cs index 576f36eb7b..4127382c66 100644 --- a/playground/mongo/Mongo.AppHost/Program.cs +++ b/playground/mongo/Mongo.AppHost/Program.cs @@ -5,7 +5,6 @@ var db = builder.AddMongoDB("mongo") .WithMongoExpress(c => c.WithHostPort(3022)) - .PublishAsContainer() .AddDatabase("db"); builder.AddProject("api") From 4dceb996b81a4d68d95aefda3ff516783cb232ec Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 1 Oct 2024 09:21:40 +1000 Subject: [PATCH 08/12] Address race. --- .../AzureStorageExtensions.cs | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index 2325387892..21111e3c63 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -91,22 +91,16 @@ public static IResourceBuilder AddAzureStorage(this IDistr var lockObject = new object(); BlobServiceClient? blobServiceClient = null; + var blobServiceClientLock = new object(); builder.Eventing.Subscribe(resource, async (@event, ct) => { // HACK: AzureStorageResource does not implement IResourceWithConnectionString which means that // we don't fire off a ConnectionStringAvailableEvent for it. To work around this the blob, - // table, and queue resources each propogate the event to the parent (the eventing system + // table, and queue resources each propagate the event to the parent (the eventing system // doesn't care about connection strings. For the actual health check we just use the // blob service client in the parent rather than each of the child resources and rely - // on the resource health check service to propogate the health checks. - // - // TODO: There is potential race condition here to solve. - if (blobServiceClient != null) - { - return; - } - + // on the resource health check service to propagate the health checks. var connectionString = await resource.GetBlobConnectionString().GetValueAsync(ct).ConfigureAwait(false); if (connectionString == null) @@ -114,7 +108,19 @@ public static IResourceBuilder AddAzureStorage(this IDistr throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); } - blobServiceClient = CreateBlobServiceClient(connectionString); + // This handles a possible race condition where each storage service type propagates the + // ConnectionStringAvailableEvent to the parent resource - but we only want to instantiate + // the blob service client once. + if (blobServiceClient == null) + { + lock (blobServiceClientLock) + { + if (blobServiceClient == null) + { + blobServiceClient = CreateBlobServiceClient(connectionString); + } + } + } }); var healthCheckKey = $"{name}_check"; From 3ccff4ad21968e818d521d8705332e7a0f4df881 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 1 Oct 2024 13:31:35 +1000 Subject: [PATCH 09/12] Switched over to propogating an internal event. --- .../AzureStorageExtensions.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index 21111e3c63..6973480cd6 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -5,6 +5,7 @@ using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; using Aspire.Hosting.Azure.Storage; +using Aspire.Hosting.Eventing; using Aspire.Hosting.Utils; using Azure.Identity; using Azure.Provisioning; @@ -93,14 +94,8 @@ public static IResourceBuilder AddAzureStorage(this IDistr BlobServiceClient? blobServiceClient = null; var blobServiceClientLock = new object(); - builder.Eventing.Subscribe(resource, async (@event, ct) => + builder.Eventing.Subscribe(resource, async (@event, ct) => { - // HACK: AzureStorageResource does not implement IResourceWithConnectionString which means that - // we don't fire off a ConnectionStringAvailableEvent for it. To work around this the blob, - // table, and queue resources each propagate the event to the parent (the eventing system - // doesn't care about connection strings. For the actual health check we just use the - // blob service client in the parent rather than each of the child resources and rely - // on the resource health check service to propagate the health checks. var connectionString = await resource.GetBlobConnectionString().GetValueAsync(ct).ConfigureAwait(false); if (connectionString == null) @@ -108,9 +103,8 @@ public static IResourceBuilder AddAzureStorage(this IDistr throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); } - // This handles a possible race condition where each storage service type propagates the - // ConnectionStringAvailableEvent to the parent resource - but we only want to instantiate - // the blob service client once. + // We only need to process one of the events from the sub-resource so we skip + // over the rest of the events if more than one sub-resource is being used. if (blobServiceClient == null) { lock (blobServiceClientLock) @@ -256,7 +250,7 @@ public static IResourceBuilder AddBlobs(this IResource builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => { - var propogatedEvent = new ConnectionStringAvailableEvent(resource.Parent, @event.Services); + var propogatedEvent = new AzureStorageSubResourceConnectionStringAvailableEvent(resource.Parent, resource); await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); }); @@ -275,7 +269,7 @@ public static IResourceBuilder AddTables(this IResour builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => { - var propogatedEvent = new ConnectionStringAvailableEvent(resource.Parent, @event.Services); + var propogatedEvent = new AzureStorageSubResourceConnectionStringAvailableEvent(resource.Parent, resource); await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); }); @@ -294,10 +288,16 @@ public static IResourceBuilder AddQueues(this IResour builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => { - var propogatedEvent = new ConnectionStringAvailableEvent(resource.Parent, @event.Services); + var propogatedEvent = new AzureStorageSubResourceConnectionStringAvailableEvent(resource.Parent, resource); await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); }); return builder.ApplicationBuilder.AddResource(resource); } + + private sealed class AzureStorageSubResourceConnectionStringAvailableEvent(AzureStorageResource parent, IResourceWithParent childResource) : IDistributedApplicationResourceEvent + { + public IResource Resource => parent; + public IResourceWithParent ChildResource => childResource; + } } From 041dad7b51c2fcda2df34a3e596f98ceb7726bd5 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 2 Oct 2024 09:15:38 +1000 Subject: [PATCH 10/12] Update src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs Co-authored-by: David Fowler --- src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index 6973480cd6..f6f0ed50ab 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -12,7 +12,6 @@ using Azure.Provisioning.Storage; using Azure.Storage.Blobs; using Microsoft.Extensions.DependencyInjection; -//using Microsoft.Extensions.DependencyInjection; namespace Aspire.Hosting; From ddb18553f345f081af308cbc53117e1d05eb5433 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 2 Oct 2024 14:11:42 +1000 Subject: [PATCH 11/12] Change Storage and Cosmos so that health checks are only enabled when using the emulators. --- .../AzureCosmosDBExtensions.cs | 88 ++++++------- .../AzureStorageExtensions.cs | 118 +++++++----------- 2 files changed, 90 insertions(+), 116 deletions(-) diff --git a/src/Aspire.Hosting.Azure.CosmosDB/AzureCosmosDBExtensions.cs b/src/Aspire.Hosting.Azure.CosmosDB/AzureCosmosDBExtensions.cs index 3fa1739dd9..ea0e5eb757 100644 --- a/src/Aspire.Hosting.Azure.CosmosDB/AzureCosmosDBExtensions.cs +++ b/src/Aspire.Hosting.Azure.CosmosDB/AzureCosmosDBExtensions.cs @@ -108,52 +108,9 @@ public static IResourceBuilder AddAzureCosmosDB(this IDis }; var resource = new AzureCosmosDBResource(name, configureConstruct); - - CosmosClient? cosmosClient = null; - - builder.Eventing.Subscribe(resource, async (@event, ct) => - { - var connectionString = await resource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); - - if (connectionString == null) - { - throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); - } - - cosmosClient = CreateCosmosClient(connectionString); - }); - - var healthCheckKey = $"{name}_check"; - builder.Services.AddHealthChecks().AddAzureCosmosDB(sp => - { - return cosmosClient ?? throw new InvalidOperationException("CosmosClient is not initialized."); - }, name: healthCheckKey); - return builder.AddResource(resource) .WithParameter(AzureBicepResource.KnownParameters.KeyVaultName) - .WithManifestPublishingCallback(resource.WriteToManifest) - .WithHealthCheck(healthCheckKey); - - static CosmosClient CreateCosmosClient(string connectionString) - { - var clientOptions = new CosmosClientOptions(); - clientOptions.CosmosClientTelemetryOptions.DisableDistributedTracing = true; - - if (Uri.TryCreate(connectionString, UriKind.Absolute, out var uri)) - { - return new CosmosClient(uri.OriginalString, new DefaultAzureCredential(), clientOptions); - } - else - { - if (CosmosUtils.IsEmulatorConnectionString(connectionString)) - { - clientOptions.ConnectionMode = ConnectionMode.Gateway; - clientOptions.LimitToEndpoint = true; - } - - return new CosmosClient(connectionString, clientOptions); - } - } + .WithManifestPublishingCallback(resource.WriteToManifest); } /// @@ -182,6 +139,28 @@ public static IResourceBuilder RunAsEmulator(this IResour Tag = "latest" }); + CosmosClient? cosmosClient = null; + + builder.ApplicationBuilder.Eventing.Subscribe(builder.Resource, async (@event, ct) => + { + var connectionString = await builder.Resource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); + + if (connectionString == null) + { + throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{builder.Resource.Name}' resource but the connection string was null."); + } + + cosmosClient = CreateCosmosClient(connectionString); + }); + + var healthCheckKey = $"{builder.Resource.Name}_check"; + builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureCosmosDB(sp => + { + return cosmosClient ?? throw new InvalidOperationException("CosmosClient is not initialized."); + }, name: healthCheckKey); + + builder.WithHealthCheck(healthCheckKey); + if (configureContainer != null) { var surrogate = new AzureCosmosDBEmulatorResource(builder.Resource); @@ -190,6 +169,27 @@ public static IResourceBuilder RunAsEmulator(this IResour } return builder; + + static CosmosClient CreateCosmosClient(string connectionString) + { + var clientOptions = new CosmosClientOptions(); + clientOptions.CosmosClientTelemetryOptions.DisableDistributedTracing = true; + + if (Uri.TryCreate(connectionString, UriKind.Absolute, out var uri)) + { + return new CosmosClient(uri.OriginalString, new DefaultAzureCredential(), clientOptions); + } + else + { + if (CosmosUtils.IsEmulatorConnectionString(connectionString)) + { + clientOptions.ConnectionMode = ConnectionMode.Gateway; + clientOptions.LimitToEndpoint = true; + } + + return new CosmosClient(connectionString, clientOptions); + } + } } /// diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index 6973480cd6..0e33a35275 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -5,7 +5,6 @@ using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; using Aspire.Hosting.Azure.Storage; -using Aspire.Hosting.Eventing; using Aspire.Hosting.Utils; using Azure.Identity; using Azure.Provisioning; @@ -90,17 +89,47 @@ public static IResourceBuilder AddAzureStorage(this IDistr }; var resource = new AzureStorageResource(name, configureConstruct); + return builder.AddResource(resource) + // These ambient parameters are only available in development time. + .WithParameter(AzureBicepResource.KnownParameters.PrincipalId) + .WithParameter(AzureBicepResource.KnownParameters.PrincipalType) + .WithManifestPublishingCallback(resource.WriteToManifest); + } + + /// + /// Configures an Azure Storage resource to be emulated using Azurite. This resource requires an to be added to the application model. This version of the package defaults to the tag of the / container image. + /// + /// The Azure storage resource builder. + /// Callback that exposes underlying container used for emulation to allow for customization. + /// A reference to the . + public static IResourceBuilder RunAsEmulator(this IResourceBuilder builder, Action>? configureContainer = null) + { + if (builder.ApplicationBuilder.ExecutionContext.IsPublishMode) + { + return builder; + } + + builder.WithEndpoint(name: "blob", targetPort: 10000) + .WithEndpoint(name: "queue", targetPort: 10001) + .WithEndpoint(name: "table", targetPort: 10002) + .WithAnnotation(new ContainerImageAnnotation + { + Registry = StorageEmulatorContainerImageTags.Registry, + Image = StorageEmulatorContainerImageTags.Image, + Tag = StorageEmulatorContainerImageTags.Tag + }); + var lockObject = new object(); BlobServiceClient? blobServiceClient = null; var blobServiceClientLock = new object(); - builder.Eventing.Subscribe(resource, async (@event, ct) => + builder.ApplicationBuilder.Eventing.Subscribe(builder.Resource, async (@event, ct) => { - var connectionString = await resource.GetBlobConnectionString().GetValueAsync(ct).ConfigureAwait(false); + var connectionString = await builder.Resource.GetBlobConnectionString().GetValueAsync(ct).ConfigureAwait(false); if (connectionString == null) { - throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{resource.Name}' resource but the connection string was null."); + throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{builder.Resource.Name}' resource but the connection string was null."); } // We only need to process one of the events from the sub-resource so we skip @@ -117,18 +146,23 @@ public static IResourceBuilder AddAzureStorage(this IDistr } }); - var healthCheckKey = $"{name}_check"; - builder.Services.AddHealthChecks().AddAzureBlobStorage(sp => + var healthCheckKey = $"{builder.Resource.Name}_check"; + + builder.ApplicationBuilder.Services.AddHealthChecks().AddAzureBlobStorage(sp => { return blobServiceClient ?? throw new InvalidOperationException("BlobServiceClient is not initialized."); }, name: healthCheckKey); - return builder.AddResource(resource) - // These ambient parameters are only available in development time. - .WithParameter(AzureBicepResource.KnownParameters.PrincipalId) - .WithParameter(AzureBicepResource.KnownParameters.PrincipalType) - .WithManifestPublishingCallback(resource.WriteToManifest) - .WithHealthCheck(healthCheckKey); + builder.WithHealthCheck(healthCheckKey); + + if (configureContainer != null) + { + var surrogate = new AzureStorageEmulatorResource(builder.Resource); + var surrogateBuilder = builder.ApplicationBuilder.CreateResourceBuilder(surrogate); + configureContainer(surrogateBuilder); + } + + return builder; static BlobServiceClient CreateBlobServiceClient(string connectionString) { @@ -143,39 +177,6 @@ static BlobServiceClient CreateBlobServiceClient(string connectionString) } } - /// - /// Configures an Azure Storage resource to be emulated using Azurite. This resource requires an to be added to the application model. This version of the package defaults to the tag of the / container image. - /// - /// The Azure storage resource builder. - /// Callback that exposes underlying container used for emulation to allow for customization. - /// A reference to the . - public static IResourceBuilder RunAsEmulator(this IResourceBuilder builder, Action>? configureContainer = null) - { - if (builder.ApplicationBuilder.ExecutionContext.IsPublishMode) - { - return builder; - } - - builder.WithEndpoint(name: "blob", targetPort: 10000) - .WithEndpoint(name: "queue", targetPort: 10001) - .WithEndpoint(name: "table", targetPort: 10002) - .WithAnnotation(new ContainerImageAnnotation - { - Registry = StorageEmulatorContainerImageTags.Registry, - Image = StorageEmulatorContainerImageTags.Image, - Tag = StorageEmulatorContainerImageTags.Tag - }); - - if (configureContainer != null) - { - var surrogate = new AzureStorageEmulatorResource(builder.Resource); - var surrogateBuilder = builder.ApplicationBuilder.CreateResourceBuilder(surrogate); - configureContainer(surrogateBuilder); - } - - return builder; - } - /// /// Adds a bind mount for the data folder to an Azure Storage emulator resource. /// @@ -247,13 +248,6 @@ public static IResourceBuilder WithTablePort(this public static IResourceBuilder AddBlobs(this IResourceBuilder builder, [ResourceName] string name) { var resource = new AzureBlobStorageResource(name, builder.Resource); - - builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => - { - var propogatedEvent = new AzureStorageSubResourceConnectionStringAvailableEvent(resource.Parent, resource); - await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); - }); - return builder.ApplicationBuilder.AddResource(resource); } @@ -266,13 +260,6 @@ public static IResourceBuilder AddBlobs(this IResource public static IResourceBuilder AddTables(this IResourceBuilder builder, [ResourceName] string name) { var resource = new AzureTableStorageResource(name, builder.Resource); - - builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => - { - var propogatedEvent = new AzureStorageSubResourceConnectionStringAvailableEvent(resource.Parent, resource); - await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); - }); - return builder.ApplicationBuilder.AddResource(resource); } @@ -285,19 +272,6 @@ public static IResourceBuilder AddTables(this IResour public static IResourceBuilder AddQueues(this IResourceBuilder builder, [ResourceName] string name) { var resource = new AzureQueueStorageResource(name, builder.Resource); - - builder.ApplicationBuilder.Eventing.Subscribe(resource, async (@event, ct) => - { - var propogatedEvent = new AzureStorageSubResourceConnectionStringAvailableEvent(resource.Parent, resource); - await builder.ApplicationBuilder.Eventing.PublishAsync(propogatedEvent, ct).ConfigureAwait(false); - }); - return builder.ApplicationBuilder.AddResource(resource); } - - private sealed class AzureStorageSubResourceConnectionStringAvailableEvent(AzureStorageResource parent, IResourceWithParent childResource) : IDistributedApplicationResourceEvent - { - public IResource Resource => parent; - public IResourceWithParent ChildResource => childResource; - } } From 968bd1feb3c40edc8950653cab72b2f0b0a2dbcf Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 2 Oct 2024 14:36:54 +1000 Subject: [PATCH 12/12] No need for locking now. --- .../AzureStorageExtensions.cs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs index 604c53fbaf..ee2e3518e3 100644 --- a/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs +++ b/src/Aspire.Hosting.Azure.Storage/AzureStorageExtensions.cs @@ -118,9 +118,7 @@ public static IResourceBuilder RunAsEmulator(this IResourc Tag = StorageEmulatorContainerImageTags.Tag }); - var lockObject = new object(); BlobServiceClient? blobServiceClient = null; - var blobServiceClientLock = new object(); builder.ApplicationBuilder.Eventing.Subscribe(builder.Resource, async (@event, ct) => { @@ -131,18 +129,7 @@ public static IResourceBuilder RunAsEmulator(this IResourc throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{builder.Resource.Name}' resource but the connection string was null."); } - // We only need to process one of the events from the sub-resource so we skip - // over the rest of the events if more than one sub-resource is being used. - if (blobServiceClient == null) - { - lock (blobServiceClientLock) - { - if (blobServiceClient == null) - { - blobServiceClient = CreateBlobServiceClient(connectionString); - } - } - } + blobServiceClient = CreateBlobServiceClient(connectionString); }); var healthCheckKey = $"{builder.Resource.Name}_check";