-
Notifications
You must be signed in to change notification settings - Fork 121
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add sample project to evaluate cooperative sticky strategy support
test: add unit tests and fix bug in ClusterConfiguration - Added 3 new test files to improve coverage: - ConsumerConfigurationBuilderTests.cs - KafkaConfigTests.cs - PartitionAssignmentStrategyTests.cs - Fixed a bug in ClusterConfiguration related to AutoCommitInterval initialization chore: update AutoCommitInterval to 100ms in Cooperative-sticky sample - Updated AutoCommitInterval in Cooperative-sticky sample Program.cs to 100ms. fix: resolve Codacy issues - Fixed various code quality issues flagged by Codacy.
- Loading branch information
1 parent
b925601
commit 83bce82
Showing
11 changed files
with
462 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 44 additions & 0 deletions
44
samples/KafkaFlow.Sample.CooperativeSticky/HostedService.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| using System; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using KafkaFlow.Producers; | ||
| using Microsoft.Extensions.Hosting; | ||
|
|
||
| namespace KafkaFlow.Sample.CooperativeSticky; | ||
|
|
||
| public class HostedService : IHostedService | ||
| { | ||
| private IMessageProducer _producer; | ||
| const string producerName = "PrintConsole"; | ||
| const string topicName = "sample-topic"; | ||
|
|
||
|
|
||
| public HostedService(IProducerAccessor producerAccessor) | ||
| { | ||
| _producer = producerAccessor.GetProducer(producerName); | ||
| } | ||
|
|
||
| public async Task StartAsync(CancellationToken cancellationToken) | ||
| { | ||
| try | ||
| { | ||
| while (true) | ||
| { | ||
| await _producer.ProduceAsync( | ||
| topicName, | ||
| Guid.NewGuid().ToString(), | ||
| new TestMessage { Text = $"Message: {Guid.NewGuid()}" }); | ||
| await Task.Delay(500, cancellationToken); | ||
| } | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| Console.WriteLine(e); | ||
| } | ||
| } | ||
|
|
||
| public Task StopAsync(CancellationToken cancellationToken) | ||
| { | ||
| return Task.CompletedTask; | ||
| } | ||
| } |
33 changes: 33 additions & 0 deletions
33
samples/KafkaFlow.Sample.CooperativeSticky/KafkaFlow.Sample.CooperativeSticky.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| <IsPackable>false</IsPackable> | ||
| <GenerateDocumentationFile>false</GenerateDocumentationFile> | ||
| <InvariantGlobalization>true</InvariantGlobalization> | ||
| </PropertyGroup> | ||
|
|
||
| <PropertyGroup Condition=" '$(Configuration)' == 'Debug' "> | ||
| <NoWarn>1701;1702;CS1591;SA1600</NoWarn> | ||
| </PropertyGroup> | ||
|
|
||
| <PropertyGroup Condition=" '$(Configuration)' == 'Release' "> | ||
| <NoWarn>1701;1702;CS1591;SA1600</NoWarn> | ||
| </PropertyGroup> | ||
|
|
||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\src\KafkaFlow.LogHandler.Console\KafkaFlow.LogHandler.Console.csproj" /> | ||
| <ProjectReference Include="..\..\src\KafkaFlow.Microsoft.DependencyInjection\KafkaFlow.Microsoft.DependencyInjection.csproj" /> | ||
| <ProjectReference Include="..\..\src\KafkaFlow.Serializer.ProtobufNet\KafkaFlow.Serializer.ProtobufNet.csproj" /> | ||
| <ProjectReference Include="..\..\src\KafkaFlow\KafkaFlow.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.1" /> | ||
| <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" /> | ||
| </ItemGroup> | ||
|
|
||
|
|
||
| </Project> |
18 changes: 18 additions & 0 deletions
18
samples/KafkaFlow.Sample.CooperativeSticky/PrintConsoleHandler.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| using System; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace KafkaFlow.Sample.CooperativeSticky; | ||
|
|
||
| public class PrintConsoleHandler : IMessageHandler<TestMessage> | ||
| { | ||
| public Task Handle(IMessageContext context, TestMessage message) | ||
| { | ||
| Console.WriteLine( | ||
| "Partition: {0} | Offset: {1} | Message: {2}", | ||
| context.ConsumerContext.Partition, | ||
| context.ConsumerContext.Offset, | ||
| message.Text); | ||
|
|
||
| return Task.CompletedTask; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| using Confluent.Kafka; | ||
| using KafkaFlow; | ||
| using KafkaFlow.Sample.CooperativeSticky; | ||
| using KafkaFlow.Serializer; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Hosting; | ||
| using AutoOffsetReset = KafkaFlow.AutoOffsetReset; | ||
|
|
||
| const string producerName = "PrintConsole"; | ||
| const string topicName = "sample-topic"; | ||
| var hostBuilder = new HostBuilder(); | ||
| hostBuilder.ConfigureServices(services => | ||
| services.AddHostedService<HostedService>().AddKafka( | ||
| kafka => kafka | ||
| .UseConsoleLog() | ||
| .AddCluster( | ||
| cluster => cluster | ||
| .WithBrokers(new[] { "localhost:9092" }) | ||
| .CreateTopicIfNotExists(topicName, 6, 1) | ||
| .AddProducer( | ||
| producerName, | ||
| producer => producer | ||
| .DefaultTopic(topicName) | ||
| .AddMiddlewares(m => m.AddSerializer<ProtobufNetSerializer>()) | ||
| ) | ||
| .AddConsumer( | ||
| consumer => consumer | ||
| .WithConsumerConfig(new ConsumerConfig | ||
| { | ||
| PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky}) | ||
| .Topic(topicName) | ||
| .WithGroupId("print-console-handler") | ||
| .WithBufferSize(100) | ||
| .WithWorkersCount(3) | ||
| .WithAutoCommitIntervalMs(100) | ||
| .WithAutoOffsetReset(AutoOffsetReset.Latest) | ||
| .AddMiddlewares( | ||
| middlewares => middlewares | ||
| .AddDeserializer<ProtobufNetDeserializer>() | ||
| .AddTypedHandlers(h => h.AddHandler<PrintConsoleHandler>()) | ||
| ) | ||
| ) | ||
| ) | ||
| )); | ||
|
|
||
| var build = hostBuilder.Build(); | ||
| var kafkaBus = build.Services.CreateKafkaBus(); | ||
| await kafkaBus.StartAsync(); | ||
|
|
||
| await build.RunAsync(); | ||
| await kafkaBus.StopAsync(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| # KafkaFlow.Sample | ||
|
|
||
| This is a simple sample that shows how to produce and consume messages. | ||
|
|
||
| ## How to run | ||
|
|
||
| ### Requirements | ||
|
|
||
| - [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) | ||
| - [Docker Desktop](https://www.docker.com/products/docker-desktop/) | ||
|
|
||
| ### Start the cluster | ||
|
|
||
| Using your terminal of choice, start the cluster. | ||
| You can find a docker-compose file at the root of this repository. | ||
| Position the terminal in that folder and run the following command. | ||
|
|
||
| ```bash | ||
| docker-compose up -d | ||
| ``` | ||
|
|
||
| ### Run the Sample | ||
|
|
||
| Using your terminal of choice, start the sample for the sample folder. | ||
|
|
||
| ```bash | ||
| dotnet run | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| using System.Runtime.Serialization; | ||
|
|
||
| namespace KafkaFlow.Sample.CooperativeSticky; | ||
|
|
||
| [DataContract] | ||
| public class TestMessage | ||
| { | ||
| [DataMember(Order = 1)] | ||
| public string Text { get; set; } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
62 changes: 62 additions & 0 deletions
62
tests/KafkaFlow.UnitTests/Consumer/ConsumerConfigurationBuilderTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| using System; | ||
| using AutoFixture; | ||
| using AutoFixture.AutoMoq; | ||
| using Confluent.Kafka; | ||
| using FluentAssertions; | ||
| using KafkaFlow.Configuration; | ||
| using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
| using Moq; | ||
|
|
||
| namespace KafkaFlow.UnitTests.Consumer; | ||
|
|
||
| [TestClass] | ||
| public class ConsumerConfigurationBuilderTests | ||
| { | ||
| private readonly Fixture _fixture = new(); | ||
|
|
||
| [TestInitialize] | ||
| public void Setup() | ||
| { | ||
| _fixture.Customize(new AutoMoqCustomization()); | ||
| } | ||
|
|
||
| [TestMethod] | ||
| public void ConfigurationBuild_CallBuild_WithSticky_EnableAutoCommit_True() | ||
| { | ||
| // Arrange | ||
| var consumerConfigurationBuilder = _fixture.Create<ConsumerConfigurationBuilder>(); | ||
| consumerConfigurationBuilder.WithConsumerConfig(new ConsumerConfig | ||
| { | ||
| PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky, | ||
| GroupId = "Test", | ||
| }).WithAutoCommitIntervalMs(500) | ||
| .WithBufferSize(3); | ||
|
|
||
| // Act | ||
| var consumerConfiguration = consumerConfigurationBuilder.Build(_fixture.Create<ClusterConfiguration>()); | ||
|
|
||
| // Assert | ||
| var consumerConfig = consumerConfiguration.GetKafkaConfig(); | ||
| consumerConfig.EnableAutoCommit.Should().BeTrue(); | ||
| consumerConfig.AutoCommitIntervalMs.Should().Be(500); | ||
| consumerConfiguration.AutoCommitInterval.Should().Be(TimeSpan.FromMilliseconds(500)); | ||
| } | ||
|
|
||
| [TestMethod] | ||
| public void ConfigurationBuild_CallBuild_WithSRoundRobin_EnableAutoCommit_False() | ||
| { | ||
| // Arrange | ||
| var consumerConfigurationBuilder = new ConsumerConfigurationBuilder(Mock.Of<IDependencyConfigurator>()); | ||
| consumerConfigurationBuilder.WithConsumerConfig(new ConsumerConfig | ||
| { | ||
| PartitionAssignmentStrategy = PartitionAssignmentStrategy.RoundRobin, | ||
| GroupId = "Test" | ||
| }).WithAutoCommitIntervalMs(500).WithBufferSize(3); | ||
| // Act | ||
| var consumerConfiguration = consumerConfigurationBuilder.Build(_fixture.Create<ClusterConfiguration>()); | ||
|
|
||
| // Assert | ||
| consumerConfiguration.GetKafkaConfig().EnableAutoCommit.Should().BeFalse(); | ||
| consumerConfiguration.AutoCommitInterval.Should().Be(TimeSpan.FromMilliseconds(500)); | ||
| } | ||
| } |
Oops, something went wrong.