This sample demonstrates how to deal with raw binary contents and large messages, to transfer some files through MQTT.
What This Sample Shows
Producing a file as a BinaryMessage that wraps the FileStream.
Extending BinaryMessage with a custom property mapped to a header (x-filename) via the [Header] attribute.
Producing to the samples/binary-files topic with at-least-once QoS and the retain flag.
Streaming the consumed content straight to a file and retrying up to 5 times in case of error.
How to Run
Start the infrastructure as described in Samples, then run the two applications from the repository root.
dotnet run --project ./samples/MQTT/BinaryFileStreaming.Producer/.
dotnet run --project ./samples/MQTT/BinaryFileStreaming.Consumer/.
The producer exposes a Swagger UI at the /swagger path of the address it logs at startup. Use it to call the binary-file or custom-binary-file endpoint, passing the path of a local file.
Note
The producer's launch profile binds to port 0, so a free port is picked at every start. Look for the Now listening on line in the console output to find the address.
The consumer saves the received files into a temp folder, two levels above its working directory (samples/temp when run from the project folder).
Producer
The producer exposes two REST API that receive the path of a local file to be streamed. The second API uses a custom BinaryMessage to forward further metadata (the file name in this example).
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using Silverback.Configuration;
using Silverback.Messaging.Configuration;
namespace Silverback.Samples.Mqtt.BinaryFileStreaming.Producer;
public class Startup
{
public void ConfigureServices(IServiceCollection services)
{
// Enable Silverback
services.AddSilverback()
// Use Apache Mqtt as message broker
.WithConnectionToMessageBroker(
options => options
.AddMqtt())
// Delegate the broker clients configuration to a separate class
.AddBrokerClientsConfigurator<BrokerClientsConfigurator>();
// Add API controllers and SwaggerGen
services.AddControllers();
services.AddSwaggerGen();
}
public void Configure(IApplicationBuilder app)
{
// Enable middlewares to serve generated Swagger JSON and UI
app.UseSwagger().UseSwaggerUI(
uiOptions =>
{
uiOptions.SwaggerEndpoint(
"/swagger/v1/swagger.json",
$"{GetType().Assembly.FullName} API");
});
// Enable routing and endpoints for controllers
app.UseRouting();
app.UseEndpoints(endpoints => { endpoints.MapControllers(); });
}
}
using Silverback.Messaging.Configuration;
using Silverback.Messaging.Messages;
namespace Silverback.Samples.Mqtt.BinaryFileStreaming.Producer;
public class BrokerClientsConfigurator : IBrokerClientsConfigurator
{
public void Configure(BrokerClientsConfigurationBuilder builder)
{
builder
.AddMqttClients(
clients => clients
// Configure connection
.ConnectViaTcp("localhost")
// Add an MQTT client
.AddClient(
client => client
.WithClientId("samples.binary-file.producer")
// Produce the binary files to the samples/binary-files topic
.Produce<BinaryMessage>(
endpoint => endpoint
.ProduceTo("samples/binary-files")
.WithAtLeastOnceQoS()
.Retain())));
}
}
using Silverback.Messaging.Messages;
namespace Silverback.Samples.Mqtt.BinaryFileStreaming.Producer.Messages;
public class CustomBinaryMessage : BinaryMessage
{
[Header("x-filename")]
public string? Filename { get; set; }
}
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Silverback.Messaging.Messages;
using Silverback.Messaging.Publishing;
using Silverback.Samples.Mqtt.BinaryFileStreaming.Producer.Messages;
namespace Silverback.Samples.Mqtt.BinaryFileStreaming.Producer.Controllers;
[ApiController]
[Route("[controller]")]
public class ProducerController : ControllerBase
{
private readonly IPublisher _publisher;
public ProducerController(IPublisher publisher)
{
_publisher = publisher;
}
[HttpPost("binary-file")]
public async Task<IActionResult> ProduceBinaryFileAsync(
string filePath,
string? contentType)
{
// Open specified file stream
await using FileStream fileStream = System.IO.File.OpenRead(filePath);
// Create a BinaryMessage that wraps the file stream
BinaryMessage binaryMessage = new(fileStream);
if (!string.IsNullOrEmpty(contentType))
binaryMessage.ContentType = contentType;
// Publish the BinaryMessage that will be routed to the outbound
// endpoint. The FileStream will be read and produced chunk by chunk,
// without the entire file being loaded into memory.
await _publisher.PublishAsync(binaryMessage);
return NoContent();
}
[HttpPost("custom-binary-file")]
public async Task<IActionResult> ProduceBinaryFileWithCustomHeadersAsync(
string filePath,
string? contentType)
{
// Open specified file stream
await using FileStream fileStream = System.IO.File.OpenRead(filePath);
// Create a CustomBinaryMessage that wraps the file stream. The
// CustomBinaryMessage extends the BinaryMessage adding an extra
// Filename property that is also exported as header.
CustomBinaryMessage binaryMessage = new()
{
Content = fileStream,
Filename = Path.GetFileName(filePath)
};
if (!string.IsNullOrEmpty(contentType))
binaryMessage.ContentType = contentType;
// Publish the BinaryMessage that will be routed to the outbound
// endpoint. The FileStream will be read and produced chunk by chunk,
// without the entire file being loaded into memory.
await _publisher.PublishAsync(binaryMessage);
return NoContent();
}
}
using Microsoft.Extensions.DependencyInjection;
using Silverback.Configuration;
using Silverback.Messaging.Configuration;
using Silverback.Samples.Mqtt.BinaryFileStreaming.Consumer.Subscribers;
namespace Silverback.Samples.Mqtt.BinaryFileStreaming.Consumer;
public class Startup
{
public void ConfigureServices(IServiceCollection services)
{
// Enable Silverback
services.AddSilverback()
// Use Apache Mqtt as message broker
.WithConnectionToMessageBroker(
options => options
.AddMqtt())
// Delegate the broker clients configuration to a separate class
.AddBrokerClientsConfigurator<BrokerClientsConfigurator>()
// Register the subscribers
.AddSingletonSubscriber<BinaryFileSubscriber>();
}
public void Configure()
{
}
}
using Silverback.Messaging.Configuration;
using Silverback.Samples.Mqtt.BinaryFileStreaming.Consumer.Messages;
namespace Silverback.Samples.Mqtt.BinaryFileStreaming.Consumer;
public class BrokerClientsConfigurator : IBrokerClientsConfigurator
{
public void Configure(BrokerClientsConfigurationBuilder builder)
{
builder
.AddMqttClients(
clients => clients
// Configure connection
.ConnectTo("localhost")
// Add an MQTT client
.AddClient(
client => client
.WithClientId("samples.binary-file.consumer")
// Consume the binary files from the samples/binary-files topic.
//
// A CustomBinaryMessage is used instead of the built-in
// BinaryMessage to be able to add the extra header
// containing the file name.
//
// Since the CustomBinaryMessage extends the built-in
// BinaryMessage, the serializer will be automatically
// switched to the BinaryMessageSerializer.
.Consume<CustomBinaryMessage>(
endpoint => endpoint
.ConsumeFrom("samples/binary-files")
.WithAtLeastOnceQoS()
// Retry each chunks sequence 5 times in case of an
// exception
.OnError(policy => policy.Retry(5)))));
}
}
using Silverback.Messaging.Messages;
namespace Silverback.Samples.Mqtt.BinaryFileStreaming.Consumer.Messages;
public class CustomBinaryMessage : BinaryMessage
{
[Header("x-filename")]
public string? Filename { get; set; }
}
using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Silverback.Samples.Mqtt.BinaryFileStreaming.Consumer.Messages;
namespace Silverback.Samples.Mqtt.BinaryFileStreaming.Consumer.Subscribers;
public class BinaryFileSubscriber
{
private const string OutputPath = "../../temp";
private readonly ILogger<BinaryFileSubscriber> _logger;
public BinaryFileSubscriber(ILogger<BinaryFileSubscriber> logger)
{
_logger = logger;
}
public async Task OnBinaryMessageReceivedAsync(CustomBinaryMessage binaryMessage)
{
EnsureTargetFolderExists();
string filename = Guid.NewGuid().ToString("N") + binaryMessage.Filename;
_logger.LogInformation("Saving binary file as {Filename}...", filename);
// Create a FileStream to save the file
using FileStream fileStream =
File.OpenWrite(Path.Combine(OutputPath, filename));
if (binaryMessage.Content != null)
{
// Asynchronously copy the message content to the FileStream.
// The message chunks are streamed directly and the entire file is
// never loaded into memory.
await binaryMessage.Content.CopyToAsync(fileStream);
}
_logger.LogInformation(
"Written {FileStreamLength} bytes into {Filename}",
fileStream.Length,
filename);
}
private static void EnsureTargetFolderExists()
{
if (!Directory.Exists(OutputPath))
Directory.CreateDirectory(OutputPath);
}
}