Diagnostics

Verified by tests

SystemCommandsTests — library CI run #31657041675 (2026-08-13)

Whizbang provides system-level diagnostics through the DiagnosticsCommand, enabling you to collect health checks, resource metrics, pipeline status, and perspective state from all services in your distributed system.

Quick Start

Send Diagnostics Command

// Collect health checks from all services
await dispatcher.SendAsync(new DiagnosticsCommand(DiagnosticType.HealthCheck));

// Collect full diagnostics with correlation ID
var correlationId = Guid.NewGuid();
await dispatcher.SendAsync(new DiagnosticsCommand(
    DiagnosticType.Full,
    correlationId
));

System Diagnostics

DiagnosticsCommand

The DiagnosticsCommand is a system command that broadcasts diagnostic requests to all services. Each service that implements a handler for DiagnosticsCommand can respond with its current state.

DiagnosticsCommand Definition

public record DiagnosticsCommand(
    DiagnosticType Type,
    Guid? CorrelationId = null
) : ICommand;

Properties:

Property	Type	Description
Type	DiagnosticType	Type of diagnostics to collect
CorrelationId	Guid?	Optional correlation ID for tracking responses

DiagnosticType Enum

The DiagnosticType enum specifies what information services should report:

Type	Description	Typical Response Time
HealthCheck	Basic health check - is the service responsive?	< 100ms
ResourceMetrics	Memory usage, thread count, resource metrics	< 200ms
PipelineStatus	Current state of message processing pipelines	< 500ms
PerspectiveStatus	Perspective and projection state information	< 1s
Full	Full diagnostic dump including all categories	1-3s

DiagnosticType Values

public enum DiagnosticType {
  HealthCheck,
  ResourceMetrics,
  PipelineStatus,
  PerspectiveStatus,
  Full
}

Implementing Diagnostic Handlers

Services implement handlers for DiagnosticsCommand to report their status. The handler should examine the DiagnosticType and respond appropriately.

Health Check Handler

Health Check Handler

public class DiagnosticsReceptor : IReceptor<DiagnosticsCommand, DiagnosticResponse> {
  private readonly IServiceHealthProvider _healthProvider;
  private readonly ILogger<DiagnosticsReceptor> _logger;

  public DiagnosticsReceptor(
      IServiceHealthProvider healthProvider,
      ILogger<DiagnosticsReceptor> logger) {
    _healthProvider = healthProvider;
    _logger = logger;
  }

  public async ValueTask<DiagnosticResponse> HandleAsync(
      DiagnosticsCommand command,
      CancellationToken ct) {

    _logger.LogInformation(
        "Received diagnostics request: {Type}, CorrelationId: {CorrelationId}",
        command.Type,
        command.CorrelationId);

    var response = command.Type switch {
      DiagnosticType.HealthCheck => await _collectHealthCheckAsync(ct),
      DiagnosticType.ResourceMetrics => await _collectResourceMetricsAsync(ct),
      DiagnosticType.PipelineStatus => await _collectPipelineStatusAsync(ct),
      DiagnosticType.PerspectiveStatus => await _collectPerspectiveStatusAsync(ct),
      DiagnosticType.Full => await _collectFullDiagnosticsAsync(ct),
      _ => DiagnosticResponse.Unknown(command.CorrelationId)
    };

    return response;
  }

  private async ValueTask<DiagnosticResponse> _collectHealthCheckAsync(
      CancellationToken ct) {
    var isHealthy = await _healthProvider.IsHealthyAsync(ct);

    return new DiagnosticResponse(
        ServiceName: Environment.MachineName,
        Status: isHealthy ? "Healthy" : "Unhealthy",
        Timestamp: DateTimeOffset.UtcNow,
        Details: new Dictionary<string, object> {
          ["uptime"] = _healthProvider.GetUptime(),
          ["version"] = _healthProvider.GetVersion()
        }
    );
  }

  private async ValueTask<DiagnosticResponse> _collectResourceMetricsAsync(
      CancellationToken ct) {
    var process = Process.GetCurrentProcess();

    return new DiagnosticResponse(
        ServiceName: Environment.MachineName,
        Status: "OK",
        Timestamp: DateTimeOffset.UtcNow,
        Details: new Dictionary<string, object> {
          ["memory_mb"] = process.WorkingSet64 / 1024 / 1024,
          ["thread_count"] = process.Threads.Count,
          ["handle_count"] = process.HandleCount,
          ["cpu_time_ms"] = process.TotalProcessorTime.TotalMilliseconds
        }
    );
  }

  private async ValueTask<DiagnosticResponse> _collectPipelineStatusAsync(
      CancellationToken ct) {
    // Collect information about message processing pipelines
    // This is application-specific
    return DiagnosticResponse.NotImplemented();
  }

  private async ValueTask<DiagnosticResponse> _collectPerspectiveStatusAsync(
      CancellationToken ct) {
    // Collect information about perspective state
    // This is application-specific
    return DiagnosticResponse.NotImplemented();
  }

  private async ValueTask<DiagnosticResponse> _collectFullDiagnosticsAsync(
      CancellationToken ct) {
    var health = await _collectHealthCheckAsync(ct);
    var resources = await _collectResourceMetricsAsync(ct);
    var pipeline = await _collectPipelineStatusAsync(ct);
    var perspectives = await _collectPerspectiveStatusAsync(ct);

    // Merge all diagnostic information
    var allDetails = new Dictionary<string, object>();
    foreach (var detail in health.Details.Concat(resources.Details)
        .Concat(pipeline.Details).Concat(perspectives.Details)) {
      allDetails[detail.Key] = detail.Value;
    }

    return new DiagnosticResponse(
        ServiceName: Environment.MachineName,
        Status: "Full",
        Timestamp: DateTimeOffset.UtcNow,
        Details: allDetails
    );
  }
}

DiagnosticResponse Event

Create a response event to publish diagnostic results:

DiagnosticResponse Event

public record DiagnosticResponse(
    string ServiceName,
    string Status,
    DateTimeOffset Timestamp,
    IReadOnlyDictionary<string, object> Details,
    Guid? CorrelationId = null
) : IEvent {

  public static DiagnosticResponse Unknown(Guid? correlationId) {
    return new DiagnosticResponse(
        ServiceName: Environment.MachineName,
        Status: "Unknown",
        Timestamp: DateTimeOffset.UtcNow,
        Details: new Dictionary<string, object>(),
        CorrelationId: correlationId
    );
  }

  public static DiagnosticResponse NotImplemented() {
    return new DiagnosticResponse(
        ServiceName: Environment.MachineName,
        Status: "NotImplemented",
        Timestamp: DateTimeOffset.UtcNow,
        Details: new Dictionary<string, object> {
          ["message"] = "This diagnostic type is not implemented by this service"
        }
    );
  }
}

Broadcasting Diagnostics

System commands use the whizbang.system.commands routing namespace, which all services automatically subscribe to when using SharedTopicInboxStrategy.

Broadcast Diagnostics Request

public class DiagnosticsController : ControllerBase {
  private readonly IDispatcher _dispatcher;

  public DiagnosticsController(IDispatcher dispatcher) {
    _dispatcher = dispatcher;
  }

  [HttpPost("diagnostics/health")]
  public async Task<IActionResult> CheckHealthAsync() {
    var correlationId = Guid.NewGuid();

    // Send command to all services
    await _dispatcher.SendAsync(
        new DiagnosticsCommand(DiagnosticType.HealthCheck, correlationId)
    );

    // Responses will be published as DiagnosticResponse events
    // Services can collect these via a perspective or event handler
    return Accepted(new { correlationId });
  }

  [HttpPost("diagnostics/full")]
  public async Task<IActionResult> CollectFullDiagnosticsAsync() {
    var correlationId = Guid.NewGuid();

    await _dispatcher.SendAsync(
        new DiagnosticsCommand(DiagnosticType.Full, correlationId)
    );

    return Accepted(new { correlationId });
  }
}

Collecting Diagnostic Responses

Use a perspective to materialize diagnostic responses into a read model. Perspectives are pure functions (IPerspectiveFor<TModel, TEvent...>) — they hold no state of their own; Whizbang persists the returned model and you read it back through a lens query:

Diagnostic Report Perspective

// Read model materialized from DiagnosticResponse events
public class DiagnosticReport {
  public string ServiceName { get; set; } = "";
  public string Status { get; set; } = "";
  public DateTimeOffset Timestamp { get; set; }
  public Guid? CorrelationId { get; set; }
}

// Pure-function perspective — the latest response wins per stream
public class DiagnosticReportPerspective :
  IPerspectiveFor<DiagnosticReport, DiagnosticResponse> {

  public DiagnosticReport Apply(DiagnosticReport currentData, DiagnosticResponse @event) {
    return new DiagnosticReport {
      ServiceName = @event.ServiceName,
      Status = @event.Status,
      Timestamp = @event.Timestamp,
      CorrelationId = @event.CorrelationId
    };
  }
}

Read the materialized reports back with ILensQuery<TModel>:

Diagnostic Lens Queries

public class DiagnosticsLens(ILensQuery<DiagnosticReport> query) {
  public async Task<IReadOnlyList<DiagnosticReport>> GetByCorrelationAsync(
      Guid correlationId, CancellationToken ct = default) {
    return await query.DefaultScope.Query
        .Where(row => row.Data.CorrelationId == correlationId)
        .Select(row => row.Data)
        .ToListAsync(ct);
  }

  public async Task<bool> AreAllServicesHealthyAsync(CancellationToken ct = default) {
    return await query.DefaultScope.Query
        .AllAsync(row => row.Data.Status == "Healthy", ct);
  }
}

Monitoring Dashboard Integration

Integrate diagnostics with monitoring dashboards:

Dashboard Diagnostics

public class SystemDiagnosticsHub : Hub {
  private readonly IDispatcher _dispatcher;
  private readonly DiagnosticsLens _diagnosticsLens;

  public SystemDiagnosticsHub(
      IDispatcher dispatcher,
      DiagnosticsLens diagnosticsLens) {
    _dispatcher = dispatcher;
    _diagnosticsLens = diagnosticsLens;
  }

  public async Task RequestSystemHealthAsync() {
    var correlationId = Guid.NewGuid();

    // Broadcast health check to all services
    await _dispatcher.SendAsync(
        new DiagnosticsCommand(DiagnosticType.HealthCheck, correlationId)
    );

    // Wait briefly for responses to materialize through the perspective
    await Task.Delay(TimeSpan.FromSeconds(2));

    // Query the read model for aggregated results
    var responses = await _diagnosticsLens.GetByCorrelationAsync(correlationId);

    // Push to dashboard
    await Clients.Caller.SendAsync(
        "HealthCheckResults",
        responses
    );
  }

  public async Task RequestFullDiagnosticsAsync() {
    var correlationId = Guid.NewGuid();

    await _dispatcher.SendAsync(
        new DiagnosticsCommand(DiagnosticType.Full, correlationId)
    );

    await Task.Delay(TimeSpan.FromSeconds(5));

    var responses = await _diagnosticsLens.GetByCorrelationAsync(correlationId);

    await Clients.Caller.SendAsync(
        "FullDiagnosticResults",
        responses
    );
  }
}

Resource Metrics Collection

Advanced resource metrics collection:

Resource Metrics

public class ResourceMetricsCollector {
  public async ValueTask<Dictionary<string, object>> CollectAsync(
      CancellationToken ct) {
    var process = Process.GetCurrentProcess();
    var metrics = new Dictionary<string, object>();

    // Memory metrics
    metrics["memory_working_set_mb"] = process.WorkingSet64 / 1024 / 1024;
    metrics["memory_private_mb"] = process.PrivateMemorySize64 / 1024 / 1024;
    metrics["memory_virtual_mb"] = process.VirtualMemorySize64 / 1024 / 1024;

    // GC metrics
    var gcMemory = GC.GetTotalMemory(forceFullCollection: false);
    metrics["gc_memory_mb"] = gcMemory / 1024 / 1024;
    metrics["gc_gen0_collections"] = GC.CollectionCount(0);
    metrics["gc_gen1_collections"] = GC.CollectionCount(1);
    metrics["gc_gen2_collections"] = GC.CollectionCount(2);

    // Thread metrics
    metrics["thread_count"] = process.Threads.Count;
    metrics["thread_pool_available"] = ThreadPool.PendingWorkItemCount;

    // CPU metrics
    metrics["cpu_time_ms"] = process.TotalProcessorTime.TotalMilliseconds;
    metrics["cpu_privileged_time_ms"] =
        process.PrivilegedProcessorTime.TotalMilliseconds;
    metrics["cpu_user_time_ms"] =
        process.UserProcessorTime.TotalMilliseconds;

    // Handle metrics
    metrics["handle_count"] = process.HandleCount;

    // Uptime
    var uptime = DateTime.UtcNow - process.StartTime.ToUniversalTime();
    metrics["uptime_seconds"] = (long)uptime.TotalSeconds;

    return metrics;
  }
}

Best Practices

Diagnostic Handler Design

Diagnostic Handler Best Practices

// DiagnosticResponse.Timeout / .Error / .Healthy are application-defined
// factory helpers on your response record, like Unknown / NotImplemented above.
public class DiagnosticsReceptor : IReceptor<DiagnosticsCommand, DiagnosticResponse> {
  // DO: Implement timeouts for diagnostic collection
  private static readonly TimeSpan HealthCheckTimeout = TimeSpan.FromSeconds(5);
  private static readonly TimeSpan FullDiagnosticsTimeout = TimeSpan.FromSeconds(30);

  public async ValueTask<DiagnosticResponse> HandleAsync(
      DiagnosticsCommand command,
      CancellationToken ct) {

    // DO: Use appropriate timeout based on diagnostic type
    var timeout = command.Type switch {
      DiagnosticType.HealthCheck => HealthCheckTimeout,
      DiagnosticType.Full => FullDiagnosticsTimeout,
      _ => TimeSpan.FromSeconds(10)
    };

    using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
    cts.CancelAfter(timeout);

    try {
      // DO: Wrap diagnostic collection in try-catch
      return await _collectDiagnosticsAsync(command, cts.Token);
    } catch (OperationCanceledException) {
      // DO: Return timeout response instead of throwing
      return DiagnosticResponse.Timeout(command.CorrelationId);
    } catch (Exception ex) {
      // DO: Return error response instead of throwing
      return DiagnosticResponse.Error(command.CorrelationId, ex.Message);
    }
  }

  // DO: Keep health checks lightweight
  private async ValueTask<DiagnosticResponse> _collectHealthCheckAsync(
      CancellationToken ct) {
    // Don't perform expensive operations in health checks
    // Just verify the service can respond
    return DiagnosticResponse.Healthy();
  }

  // DON'T: Perform expensive operations in health checks
  // private async Task _healthCheckDontDoThis() {
  //   await _database.PingAsync();  // Bad - could timeout
  //   await _externalApi.TestAsync();  // Bad - network dependency
  // }
}

Correlation and Aggregation

Correlation Best Practices

// DO: Always use correlation IDs for request-response tracking
var correlationId = Guid.NewGuid();
await dispatcher.SendAsync(
    new DiagnosticsCommand(DiagnosticType.HealthCheck, correlationId)
);

// DO: Store correlation ID for later aggregation
_diagnosticRequests[correlationId] = new DiagnosticRequest {
  RequestedAt = DateTimeOffset.UtcNow,
  Type = DiagnosticType.HealthCheck,
  ExpectedResponses = _knownServiceCount
};

// DO: Set reasonable timeouts for response collection
await Task.Delay(TimeSpan.FromSeconds(2));

// DO: Handle partial responses gracefully
var responses = await _diagnosticsLens.GetByCorrelationAsync(correlationId);
if (responses.Count < _knownServiceCount) {
  _logger.LogWarning(
      "Received {Count} of {Expected} diagnostic responses",
      responses.Count,
      _knownServiceCount);
}

Performance Considerations

1. Health Check Performance: Keep health checks under 100ms - they may be called frequently
2. Full Diagnostics: Limit full diagnostics to admin/debug scenarios - can be expensive
3. Response Aggregation: Use perspectives to aggregate responses over time
4. Timeout Handling: Always implement timeouts to prevent hanging operations
5. Error Handling: Return diagnostic errors instead of throwing - helps identify partial failures

See Also

• Tracing - Handler-level distributed tracing
• OpenTelemetry Integration - Metrics and telemetry
• System Commands - System command routing
• Perspectives - State aggregation patterns