This sample demonstrates how to produce messages atomically using Kafka transactions, committing some transactions and aborting others.
What This Sample Shows
Enabling transactions on the producer with EnableTransactions, specifying the transactional id.
Starting a transaction with InitKafkaTransaction, publishing a set of 5 messages and committing it with Commit.
Aborting a transaction by disposing it without committing: an exception is thrown every few iterations (whenever the first number is a multiple of 3000), so the transaction is never committed.
Running two hosted services that use transactions concurrently within the same application.
Consuming the topic with a plain consumer, which retries each message twice and then skips it in case of error.
How to Run
Start the infrastructure as described in Samples, then run the two applications from the repository root.
dotnet run --project ./samples/Kafka/TransactionalProducer.Producer/.
dotnet run --project ./samples/Kafka/TransactionalProducer.Consumer/.
The producer logs when each transaction is initialized, when its messages are produced and when it is committed. Each transaction stays open for 3 seconds before being committed or aborted, and the consumer receives the messages only once their transaction is committed.
Common
The message being exchanged is defined in a common project.
namespace Silverback.Samples.Kafka.TransactionalProducer.Common;
public class SampleMessage
{
public int Number { get; set; }
}
using Microsoft.Extensions.DependencyInjection;
using Silverback.Configuration;
using Silverback.Messaging.Configuration;
namespace Silverback.Samples.Kafka.TransactionalProducer.Producer;
public class Startup
{
public void ConfigureServices(IServiceCollection services)
{
// Enable Silverback
services.AddSilverback()
// Use Apache Kafka as message broker
.WithConnectionToMessageBroker(options => options.AddKafka())
// Delegate the broker clients configuration to a separate class
.AddBrokerClientsConfigurator<BrokerClientsConfigurator>();
// Add the hosted service that produces the random sample messages
services.AddHostedService<ProducerBackgroundService>();
services.AddHostedService<ProducerBackgroundService2>();
}
public void Configure()
{
}
}
using Silverback.Messaging.Configuration;
using Silverback.Samples.Kafka.TransactionalProducer.Common;
namespace Silverback.Samples.Kafka.TransactionalProducer.Producer;
public class BrokerClientsConfigurator : IBrokerClientsConfigurator
{
public void Configure(BrokerClientsConfigurationBuilder builder)
{
builder
.AddKafkaClients(
clients => clients
// The bootstrap server address is needed to connect
.WithBootstrapServers("PLAINTEXT://localhost:19092")
// Add a producer
.AddProducer(
producer => producer
// Enable transactions and define the transactional id
.EnableTransactions("sample")
// Produce the SampleMessage to the samples-transactional-producer topic
.Produce<SampleMessage>(
endpoint => endpoint
.ProduceTo("samples-transactional-producer"))));
}
}
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Silverback.Messaging.Publishing;
using Silverback.Messaging.Transactions;
using Silverback.Samples.Kafka.TransactionalProducer.Common;
namespace Silverback.Samples.Kafka.TransactionalProducer.Producer;
public class ProducerBackgroundService : BackgroundService
{
private readonly IPublisher _publisher;
private readonly ILogger<ProducerBackgroundService> _logger;
public ProducerBackgroundService(
IPublisher publisher,
ILogger<ProducerBackgroundService> logger)
{
_publisher = publisher;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
int number = 0;
while (!stoppingToken.IsCancellationRequested)
{
await ProduceMessagesAsync(++number * 1000, stoppingToken);
await Task.Delay(100, stoppingToken);
}
}
private async Task ProduceMessagesAsync(int number, CancellationToken stoppingToken)
{
try
{
using IKafkaTransaction transaction = _publisher.InitKafkaTransaction();
_logger.LogInformation("Transaction '{TransactionalIdSuffix}' initialized", transaction.TransactionalIdSuffix);
await _publisher.PublishAsync(
new SampleMessage[]
{
new() { Number = number + 1 },
new() { Number = number + 2 },
new() { Number = number + 3 },
new() { Number = number + 4 },
new() { Number = number + 5 }
},
stoppingToken);
_logger.LogInformation("Produced {Number} in transaction '{TransactionalIdSuffix}'", 5, transaction.TransactionalIdSuffix);
await Task.Delay(3000, stoppingToken);
if (number % 3000 == 0)
throw new InvalidOperationException("Test!");
transaction.Commit();
_logger.LogInformation("Transaction '{TransactionalIdSuffix}' committed", transaction.TransactionalIdSuffix);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to produce {Number}", number);
}
}
}
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Silverback.Messaging.Publishing;
using Silverback.Messaging.Transactions;
using Silverback.Samples.Kafka.TransactionalProducer.Common;
namespace Silverback.Samples.Kafka.TransactionalProducer.Producer;
public class ProducerBackgroundService2 : BackgroundService
{
private readonly IPublisher _publisher;
private readonly ILogger<ProducerBackgroundService2> _logger;
public ProducerBackgroundService2(
IPublisher publisher,
ILogger<ProducerBackgroundService2> logger)
{
_publisher = publisher;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
int number = 0;
while (!stoppingToken.IsCancellationRequested)
{
await ProduceMessagesAsync(++number * 100, stoppingToken);
await Task.Delay(100, stoppingToken);
}
}
private async Task ProduceMessagesAsync(int number, CancellationToken stoppingToken)
{
try
{
using IKafkaTransaction transaction = _publisher.InitKafkaTransaction("secondary");
_logger.LogInformation("Transaction '{TransactionalIdSuffix}' initialized", transaction.TransactionalIdSuffix);
await _publisher.PublishAsync(
new SampleMessage[]
{
new() { Number = number + 1 },
new() { Number = number + 2 },
new() { Number = number + 3 },
new() { Number = number + 4 },
new() { Number = number + 5 }
},
stoppingToken);
_logger.LogInformation("Produced {Number} in transaction '{TransactionalIdSuffix}'", 5, transaction.TransactionalIdSuffix);
await Task.Delay(3000, stoppingToken);
if (number % 3000 == 0)
throw new InvalidOperationException("Test!");
transaction.Commit();
_logger.LogInformation("Transaction '{TransactionalIdSuffix}' committed", transaction.TransactionalIdSuffix);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to produce {Number}", number);
}
}
}
using Microsoft.Extensions.DependencyInjection;
using Silverback.Configuration;
using Silverback.Messaging.Configuration;
namespace Silverback.Samples.Kafka.TransactionalProducer.Consumer;
public class Startup
{
public void ConfigureServices(IServiceCollection services)
{
// Enable Silverback
services.AddSilverback()
// Use Apache Kafka as message broker
.WithConnectionToMessageBroker(options => options.AddKafka())
// Delegate the broker clients configuration to a separate class
.AddBrokerClientsConfigurator<BrokerClientsConfigurator>()
// Register the subscribers
.AddSingletonSubscriber<SampleMessageSubscriber>();
}
public void Configure()
{
}
}
using Silverback.Messaging.Configuration;
using Silverback.Samples.Kafka.TransactionalProducer.Common;
namespace Silverback.Samples.Kafka.TransactionalProducer.Consumer;
public class BrokerClientsConfigurator : IBrokerClientsConfigurator
{
public void Configure(BrokerClientsConfigurationBuilder builder)
{
builder
.AddKafkaClients(
clients => clients
// The bootstrap server address is needed to connect
.WithBootstrapServers("PLAINTEXT://localhost:19092")
// Add a consumer
.AddConsumer(
consumer => consumer
// Set the consumer group id
.WithGroupId("sample-consumer")
// AutoOffsetReset.Earliest means that the consumer
// must start consuming from the beginning of the topic,
// if no offset was stored for this consumer group
.AutoResetOffsetToEarliest()
// Consume the SampleMessage from the samples-transactional-producer topic
.Consume<SampleMessage>(
endpoint => endpoint
.ConsumeFrom("samples-transactional-producer")
// Retry twice to process each message in case of
// exception, then skip it
.OnError(policy => policy.Retry(2).ThenSkip()))));
}
}
using Microsoft.Extensions.Logging;
using Silverback.Samples.Kafka.TransactionalProducer.Common;
namespace Silverback.Samples.Kafka.TransactionalProducer.Consumer;
public class SampleMessageSubscriber
{
private readonly ILogger<SampleMessageSubscriber> _logger;
public SampleMessageSubscriber(ILogger<SampleMessageSubscriber> logger)
{
_logger = logger;
}
public void OnMessageReceived(SampleMessage message) =>
_logger.LogInformation("Received {MessageNumber}", message.Number);
}