From 9f1e3f9863e7f05982923ea99d120ef57ffa0f98 Mon Sep 17 00:00:00 2001 From: Daniel Krawczyk Date: Thu, 29 Sep 2016 09:26:57 +0200 Subject: [PATCH] Schema-registry integration --- build.gradle | 1 + .../pl/allegro/tech/hermes/api/Topic.java | 34 +-- .../pl/allegro/tech/hermes/api/TopicTest.java | 2 - .../tech/hermes/api/helper/PatchTest.java | 4 +- hermes-common/build.gradle | 5 +- .../tech/hermes/common/config/Configs.java | 4 +- .../tech/hermes/common/di/CommonBinder.java | 21 +- .../di/factories/BoonObjectMapperFactory.java | 18 -- .../exception/InvalidSchemaException.java | 26 -- .../common/message/AvroSchemaSource.java | 47 ---- .../DeserializationException.java | 13 - .../wrapper/AvroMessageContentWrapper.java | 2 +- .../wrapper/DeserializationException.java | 8 + .../wrapper/MessageContentWrapper.java | 58 ++++- .../SchemaAwarePayload.java | 4 +- .../SchemaAwareSerDe.java | 7 +- .../wrapper}/SchemaMissingException.java | 4 +- .../wrapper/UnwrappedMessageContent.java | 2 +- .../AvroCompiledSchemaRepositoryFactory.java | 16 +- .../schema/SchemaRepositoryFactory.java | 33 +++ .../common/schema/SchemaRepositoryType.java | 5 + .../schema/SchemaSourceClientFactory.java | 53 ++++ .../SchemaVersionsRepositoryFactory.java | 48 ++++ .../CouldNotCompileSchemaException.java | 17 -- .../schema/CouldNotLoadSchemaException.java | 20 -- .../topic/schema/SchemaCompilersFactory.java | 28 -- .../SchemaRepoSchemaSourceProvider.java | 34 --- .../domain/topic/schema/SchemaRepository.java | 59 ----- .../topic/schema/SchemaRepositoryType.java | 5 - .../topic/schema/SchemaSourceProvider.java | 23 -- .../domain/topic/schema/TopicWithSchema.java | 22 -- .../schema/UnknownSchemaVersionException.java | 17 -- .../schema/ZookeeperSchemaSourceProvider.java | 37 --- .../JsonCompiledSchemaRepositoryFactory.java | 37 --- .../schema/SchemaSourceProviderFactory.java | 53 ---- .../SchemaVersionsRepositoryFactory.java | 52 ---- .../schema/repo/SchemaRepoClient.java | 24 -- .../schema/repo/SchemaRepoClientFactory.java | 35 --- .../ZookeeperTopicRepositoryTest.groovy | 3 +- .../wrapper/MessageContentWrapperTest.java | 69 +++-- .../SchemaAwareSerDeTest.java | 4 +- .../SchemaRepoSchemaSourceProviderTest.java | 68 ----- .../ZookeeperSchemaSourceProviderTest.java | 65 ----- .../repo/JerseySchemaRepoClientTest.java | 226 ----------------- hermes-console/config.json.example | 3 +- .../js/console/topic/TopicController.js | 2 +- .../js/console/topic/TopicRepository.js | 21 +- .../static/partials/modal/editTopic.html | 25 +- hermes-console/static/partials/topic.html | 6 +- hermes-consumers/build.gradle | 3 +- .../hermes/consumers/consumer/Message.java | 2 +- .../filtering/avro/AvroPathPredicate.java | 3 +- .../kafka/KafkaMessageReceiverFactory.java | 2 +- .../KafkaSingleThreadedMessageReceiver.java | 4 +- .../AvroToJsonMessageConverterTest.java | 4 +- .../hermes/consumers/test/MessageBuilder.java | 4 +- hermes-frontend/build.gradle | 1 + .../hermes/frontend/di/FrontendBinder.java | 2 - .../MessageContentTypeEnforcer.java | 1 - .../publishing/PublishingServlet.java | 6 +- .../frontend/publishing/avro/AvroMessage.java | 2 +- .../publishing/message/JsonMessage.java | 16 -- .../frontend/publishing/message/Message.java | 4 +- .../publishing/message/MessageFactory.java | 30 +-- .../validator/JsonTopicMessageValidator.java | 69 ----- .../TopicMessageValidatorListFactory.java | 6 +- .../MessageContentTypeEnforcerTest.java | 6 +- hermes-management/build.gradle | 1 + .../hermes/management/api/SchemaEndpoint.java | 6 +- .../api/mappers/RuntimeExceptionMapper.java | 7 +- .../api/mappers/SchemaExceptionMapper.java | 22 ++ .../config/MessageConfiguration.java | 6 +- .../config/SchemaRepositoryConfiguration.java | 68 ++--- .../management/config/TopicProperties.java | 10 + .../config/kafka/KafkaConfiguration.java | 2 +- .../AvroSchemaRemovalDisabledException.java | 16 -- .../schema/CouldNotDeleteSchemaException.java | 16 -- ...va => SchemaRemovalDisabledException.java} | 8 +- .../topic/schema/SchemaSourceRepository.java | 18 -- .../topic/schema/SchemaSourceService.java | 39 +-- .../topic/validator/TopicValidator.java | 6 +- .../service/KafkaSingleMessageReader.java | 8 +- .../KafkaRetransmissionService.java | 2 +- .../retransmit/KafkaTimestampExtractor.java | 4 +- .../SchemaRepoSchemaSourceRepository.java | 33 --- .../ZookeeperSchemaSourceRepository.java | 41 --- .../schema/validator/AvroSchemaValidator.java | 1 - .../validator/InvalidSchemaException.java | 16 ++ .../schema/validator/JsonSchemaValidator.java | 44 ---- .../schema/validator/SchemaValidator.java | 2 - .../validator/SchemaValidatorProvider.java | 3 +- .../src/main/resources/application.yaml | 2 +- .../topic/validator/TopicValidatorTest.groovy | 12 +- ...ZookeeperSchemaSourceRepositorySpec.groovy | 63 ----- .../validator/AvroSchemaValidatorTest.groovy | 1 - .../validator/JsonSchemaValidatorTest.groovy | 45 ---- hermes-schema/build.gradle | 11 + .../CachedCompiledSchemaRepository.java | 2 +- .../CachedSchemaVersionsRepository.java | 27 +- .../tech/hermes}/schema/CompiledSchema.java | 2 +- .../schema/CompiledSchemaRepository.java | 3 +- .../schema/CouldNotLoadSchemaException.java | 19 ++ .../schema/CouldNotRemoveSchemaException.java | 15 ++ .../DirectCompiledSchemaRepository.java | 14 +- .../DirectSchemaVersionsRepository.java | 16 +- .../hermes/schema/InvalidSchemaException.java | 15 ++ .../tech/hermes}/schema/SchemaCompiler.java | 2 +- .../hermes/schema/SchemaCompilersFactory.java | 10 + .../tech/hermes/schema/SchemaException.java | 20 ++ .../tech/hermes/schema/SchemaRepository.java | 48 ++++ .../SchemaRepositoryServerException.java | 7 +- .../hermes/schema/SchemaSourceClient.java | 20 ++ .../schema/SchemaSourceNotFoundException.java | 13 +- .../tech/hermes}/schema/SchemaVersion.java | 2 +- .../schema/SchemaVersionsRepository.java | 5 +- .../confluent/SchemaRegistryRequest.java | 42 +++ .../confluent/SchemaRegistryResponse.java | 69 +++++ .../SchemaRegistrySchemaSourceClient.java | 146 +++++++++++ .../SchemaRepoSchemaSourceClient.java | 94 ++++--- .../CachedCompiledSchemaRepositoryTest.groovy | 2 +- .../CachedSchemaVersionsRepositoryTest.groovy | 28 +- .../DirectCompiledSchemaRepositoryTest.groovy | 16 +- .../SchemaRegistrySchemaSourceClientTest.java | 219 ++++++++++++++++ .../SchemaRepoSchemaSourceClientTest.java | 240 ++++++++++++++++++ .../__files/all-schemas-response.json | 0 .../hermes/test/helper/avro/AvroUser.java | 4 +- .../test/helper/builder/TopicBuilder.java | 18 +- .../helper/endpoint/HermesAPIOperations.java | 16 ++ .../src/main/resources/schema/example.json | 11 - .../src/main/resources/schema/user_v2.avsc | 2 +- integration/build.gradle | 7 +- .../hermes/integration/BatchDeliveryTest.java | 1 - .../hermes/integration/FilteringAvroTest.java | 2 - .../KafkaSingleMessageReaderTest.java | 2 - .../integration/MessageBufferLoadingTest.java | 2 +- .../integration/PublishingAvroTest.java | 14 +- .../hermes/integration/PublishingTest.java | 64 ----- .../env/ConfluentSchemaRegistryStarter.java | 49 ++++ .../integration/env/ConsumersStarter.java | 4 +- .../integration/env/EnvironmentAware.java | 4 +- .../integration/env/FrontendStarter.java | 4 +- .../env/HermesIntegrationEnvironment.java | 4 +- .../integration/helper/GraphiteEndpoint.java | 3 +- .../management/SchemaManagementTest.java | 30 +-- .../integration/resources/application.yaml | 1 - .../resources/application-integration.yaml | 2 +- .../src/test/resources/config.properties | 2 +- settings.gradle | 3 +- 148 files changed, 1554 insertions(+), 1839 deletions(-) delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/BoonObjectMapperFactory.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/InvalidSchemaException.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/AvroSchemaSource.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/DeserializationException.java create mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/DeserializationException.java rename hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/{serialization => wrapper}/SchemaAwarePayload.java (77%) rename hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/{serialization => wrapper}/SchemaAwareSerDe.java (80%) rename hermes-common/src/main/java/pl/allegro/tech/hermes/{domain/topic/schema => common/message/wrapper}/SchemaMissingException.java (80%) rename hermes-common/src/main/java/pl/allegro/tech/hermes/{infrastructure => common}/schema/AvroCompiledSchemaRepositoryFactory.java (58%) create mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryFactory.java create mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryType.java create mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaSourceClientFactory.java create mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionsRepositoryFactory.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CouldNotCompileSchemaException.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CouldNotLoadSchemaException.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaCompilersFactory.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepoSchemaSourceProvider.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepository.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepositoryType.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaSourceProvider.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/TopicWithSchema.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/UnknownSchemaVersionException.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/ZookeeperSchemaSourceProvider.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/JsonCompiledSchemaRepositoryFactory.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/SchemaSourceProviderFactory.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/SchemaVersionsRepositoryFactory.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/SchemaRepoClient.java delete mode 100644 hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/SchemaRepoClientFactory.java rename hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/{serialization => wrapper}/SchemaAwareSerDeTest.java (91%) delete mode 100644 hermes-common/src/test/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepoSchemaSourceProviderTest.java delete mode 100644 hermes-common/src/test/java/pl/allegro/tech/hermes/domain/topic/schema/ZookeeperSchemaSourceProviderTest.java delete mode 100644 hermes-common/src/test/java/pl/allegro/tech/hermes/infrastructure/schema/repo/JerseySchemaRepoClientTest.java delete mode 100644 hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/JsonTopicMessageValidator.java create mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/SchemaExceptionMapper.java delete mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/AvroSchemaRemovalDisabledException.java delete mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/CouldNotDeleteSchemaException.java rename hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/{CouldNotSaveSchemaException.java => SchemaRemovalDisabledException.java} (52%) delete mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaSourceRepository.java delete mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/SchemaRepoSchemaSourceRepository.java delete mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/ZookeeperSchemaSourceRepository.java create mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/InvalidSchemaException.java delete mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/JsonSchemaValidator.java delete mode 100644 hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/ZookeeperSchemaSourceRepositorySpec.groovy delete mode 100644 hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/validator/JsonSchemaValidatorTest.groovy create mode 100644 hermes-schema/build.gradle rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/CachedCompiledSchemaRepository.java (97%) rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/CachedSchemaVersionsRepository.java (71%) rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/CompiledSchema.java (95%) rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/CompiledSchemaRepository.java (75%) create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotLoadSchemaException.java create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotRemoveSchemaException.java rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/DirectCompiledSchemaRepository.java (64%) rename hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SimpleSchemaVersionsRepository.java => hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectSchemaVersionsRepository.java (54%) create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/InvalidSchemaException.java rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/SchemaCompiler.java (74%) create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompilersFactory.java create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaException.java create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepository.java rename hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/SchemaRepoException.java => hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepositoryServerException.java (50%) create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaSourceClient.java rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/SchemaSourceNotFoundException.java (53%) rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/SchemaVersion.java (95%) rename {hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/main/java/pl/allegro/tech/hermes}/schema/SchemaVersionsRepository.java (76%) create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRequest.java create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryResponse.java create mode 100644 hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistrySchemaSourceClient.java rename hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/JerseySchemaRepoClient.java => hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/schemarepo/SchemaRepoSchemaSourceClient.java (61%) rename {hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/test/groovy/pl/allegro/tech/hermes}/schema/CachedCompiledSchemaRepositoryTest.groovy (96%) rename {hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/test/groovy/pl/allegro/tech/hermes}/schema/CachedSchemaVersionsRepositoryTest.groovy (75%) rename {hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic => hermes-schema/src/test/groovy/pl/allegro/tech/hermes}/schema/DirectCompiledSchemaRepositoryTest.groovy (71%) create mode 100644 hermes-schema/src/test/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistrySchemaSourceClientTest.java create mode 100644 hermes-schema/src/test/java/pl/allegro/tech/hermes/schema/schemarepo/SchemaRepoSchemaSourceClientTest.java rename {hermes-common => hermes-schema}/src/test/resources/schema-repo-stub/__files/all-schemas-response.json (100%) delete mode 100644 hermes-test-helper/src/main/resources/schema/example.json create mode 100644 integration/src/integration/java/pl/allegro/tech/hermes/integration/env/ConfluentSchemaRegistryStarter.java diff --git a/build.gradle b/build.gradle index 0b3ebdcc3e..d6af522127 100644 --- a/build.gradle +++ b/build.gradle @@ -43,6 +43,7 @@ allprojects { guava : '19.0', jackson : '2.5.1', jersey : '2.23.2', + jetty : '9.3.6.v20151106', curator : '2.11.0', wiremock : '1.58', fongo : '1.6.1', diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java index 27939118bf..ccdb7b9279 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java @@ -17,10 +17,6 @@ public class Topic { @NotNull private String description; - private boolean validationEnabled = false; - - private boolean validationDryRunEnabled = false; - private boolean jsonToAvroDryRunEnabled = false; @NotNull @@ -44,14 +40,11 @@ public enum Ack { private boolean schemaVersionAwareSerializationEnabled = false; public Topic(TopicName name, String description, RetentionTime retentionTime, - boolean validationEnabled, boolean validationDryRunEnabled, boolean migratedFromJsonType, - Ack ack, boolean trackingEnabled, ContentType contentType, boolean jsonToAvroDryRunEnabled, - boolean schemaVersionAwareSerializationEnabled) { + boolean migratedFromJsonType, Ack ack, boolean trackingEnabled, ContentType contentType, + boolean jsonToAvroDryRunEnabled, boolean schemaVersionAwareSerializationEnabled) { this.name = name; this.description = description; this.retentionTime = retentionTime; - this.validationEnabled = validationEnabled; - this.validationDryRunEnabled = validationDryRunEnabled; this.ack = (ack == null ? Ack.LEADER : ack); this.trackingEnabled = trackingEnabled; this.migratedFromJsonType = migratedFromJsonType; @@ -65,17 +58,14 @@ public Topic( @JsonProperty("name") String qualifiedName, @JsonProperty("description") String description, @JsonProperty("retentionTime") RetentionTime retentionTime, - @JsonProperty("validation") boolean validationEnabled, - @JsonProperty("validationDryRun") boolean validationDryRunEnabled, @JsonProperty("jsonToAvroDryRun") boolean jsonToAvroDryRunEnabled, @JsonProperty("ack") Ack ack, @JsonProperty("trackingEnabled") boolean trackingEnabled, @JsonProperty("migratedFromJsonType") boolean migratedFromJsonType, @JsonProperty("schemaVersionAwareSerializationEnabled") boolean schemaVersionAwareSerializationEnabled, @JsonProperty("contentType") ContentType contentType) { - this(TopicName.fromQualifiedName(qualifiedName), description, retentionTime, validationEnabled, - validationDryRunEnabled, migratedFromJsonType, ack, trackingEnabled, contentType, jsonToAvroDryRunEnabled, - schemaVersionAwareSerializationEnabled); + this(TopicName.fromQualifiedName(qualifiedName), description, retentionTime, migratedFromJsonType, ack, + trackingEnabled, contentType, jsonToAvroDryRunEnabled, schemaVersionAwareSerializationEnabled); } public RetentionTime getRetentionTime() { @@ -84,8 +74,8 @@ public RetentionTime getRetentionTime() { @Override public int hashCode() { - return Objects.hash(name, description, retentionTime, validationEnabled, validationDryRunEnabled, - migratedFromJsonType, trackingEnabled, ack, contentType, jsonToAvroDryRunEnabled, schemaVersionAwareSerializationEnabled); + return Objects.hash(name, description, retentionTime, migratedFromJsonType, trackingEnabled, ack, contentType, + jsonToAvroDryRunEnabled, schemaVersionAwareSerializationEnabled); } @Override @@ -101,8 +91,6 @@ public boolean equals(Object obj) { return Objects.equals(this.name, other.name) && Objects.equals(this.description, other.description) && Objects.equals(this.retentionTime, other.retentionTime) - && Objects.equals(this.isValidationEnabled(), other.isValidationEnabled()) - && Objects.equals(this.validationDryRunEnabled, other.validationDryRunEnabled) && Objects.equals(this.jsonToAvroDryRunEnabled, other.jsonToAvroDryRunEnabled) && Objects.equals(this.trackingEnabled, other.trackingEnabled) && Objects.equals(this.migratedFromJsonType, other.migratedFromJsonType) @@ -133,16 +121,6 @@ public void setRetentionTime(RetentionTime retentionTime) { this.retentionTime = retentionTime; } - @JsonProperty("validation") - public boolean isValidationEnabled() { - return validationEnabled || ContentType.AVRO == contentType; - } - - @JsonProperty("validationDryRun") - public boolean isValidationDryRunEnabled() { - return validationDryRunEnabled; - } - @JsonProperty("jsonToAvroDryRun") public boolean isJsonToAvroDryRunEnabled() { return jsonToAvroDryRunEnabled; diff --git a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java index 9821eebdef..de703621f6 100644 --- a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java +++ b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java @@ -21,8 +21,6 @@ public void shouldDeserializeTopic() throws Exception { assertThat(topic.getName().getName()).isEqualTo("bar"); assertThat(topic.getName().getGroupName()).isEqualTo("foo"); assertThat(topic.getDescription()).isEqualTo("description"); - assertThat(topic.isValidationEnabled()).isFalse(); - assertThat(topic.isValidationDryRunEnabled()).isFalse(); } @Test diff --git a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java index 9682479519..b3ea8c72c7 100644 --- a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java +++ b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java @@ -73,13 +73,13 @@ public void shouldPatchNestedObjects() { public void shouldNotResetPrimitiveFields() { // given Topic topic = topic("group.topic").withTrackingEnabled(true).build(); - PatchData patch = patchData().set("validation", true).build(); + PatchData patch = patchData().set("schemaVersionAwareSerializationEnabled", true).build(); // when Topic patched = Patch.apply(topic, patch); // then assertThat(patched.isTrackingEnabled()).isTrue(); - assertThat(patched.isValidationEnabled()).isTrue(); + assertThat(patched.isSchemaVersionAwareSerializationEnabled()).isTrue(); } } diff --git a/hermes-common/build.gradle b/hermes-common/build.gradle index 45d14aa3c5..744171dcd5 100644 --- a/hermes-common/build.gradle +++ b/hermes-common/build.gradle @@ -3,6 +3,7 @@ apply plugin: 'groovy' dependencies { compile project(':hermes-api') compile project(':hermes-metrics') + compile project(':hermes-schema') compile group: 'com.netflix.archaius', name: 'archaius-core', version: '0.6.0' @@ -14,9 +15,6 @@ dependencies { exclude module: 'slf4j-log4j12' exclude module: 'log4j' } - compile (group: 'com.github.fge', name: 'json-schema-validator', version: '2.2.6') { - exclude group: 'net.sf.jopt-simple' - } compile group: 'org.glassfish.hk2', name: 'hk2-locator', version: '2.3.0-b01' compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: versions.jersey @@ -30,7 +28,6 @@ dependencies { compile group: 'com.google.guava', name: 'guava', version: versions.guava compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: versions.jackson - compile group: 'io.fastjson', name: 'boon', version: '0.28' compile group: 'org.apache.avro', name: 'avro', version: '1.7.7' compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '3.1.1' diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/Configs.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/Configs.java index 7ffe5b6030..384eb1bff3 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/Configs.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/Configs.java @@ -188,8 +188,8 @@ public enum Configs { SCHEMA_CACHE_RELOAD_THREAD_POOL_SIZE("schema.cache.reload.thread.pool.size", 2), SCHEMA_CACHE_ENABLED("schema.cache.enabled", true), SCHEMA_CACHE_COMPILED_MAXIMUM_SIZE("schema.cache.compiled.maximum.size", 2000), - SCHEMA_REPOSITORY_TYPE("schema.repository.type", "zookeeper"), - SCHEMA_REPOSITORY_SERVER_URL("schema.repository.serverUrl", "http://localhost:8888/schema-repo/"), + SCHEMA_REPOSITORY_TYPE("schema.repository.type", "schema_registry"), + SCHEMA_REPOSITORY_SERVER_URL("schema.repository.serverUrl", "http://localhost:8888/"), SCHEMA_REPOSITORY_HTTP_READ_TIMEOUT_MS("schema.repository.http.read.timeout.ms", 2000), SCHEMA_REPOSITORY_HTTP_CONNECT_TIMEOUT_MS("schema.repository.http.connect.timeout.ms", 2000), diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/CommonBinder.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/CommonBinder.java index 7ce8bf4dbb..79f52bf4dd 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/CommonBinder.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/CommonBinder.java @@ -1,6 +1,5 @@ package pl.allegro.tech.hermes.common.di; -import com.github.fge.jsonschema.main.JsonSchema; import com.yammer.metrics.core.HealthCheckRegistry; import org.apache.avro.Schema; import org.glassfish.hk2.api.TypeLiteral; @@ -8,7 +7,6 @@ import pl.allegro.tech.hermes.common.broker.ZookeeperBrokerStorage; import pl.allegro.tech.hermes.common.clock.ClockFactory; import pl.allegro.tech.hermes.common.config.ConfigFactory; -import pl.allegro.tech.hermes.common.di.factories.BoonObjectMapperFactory; import pl.allegro.tech.hermes.common.di.factories.CuratorClientFactory; import pl.allegro.tech.hermes.common.di.factories.DistributedEphemeralCounterFactory; import pl.allegro.tech.hermes.common.di.factories.GraphiteWebTargetFactory; @@ -34,17 +32,15 @@ import pl.allegro.tech.hermes.common.metric.HermesMetrics; import pl.allegro.tech.hermes.common.metric.counter.CounterStorage; import pl.allegro.tech.hermes.common.metric.counter.zookeeper.ZookeeperCounterStorage; +import pl.allegro.tech.hermes.common.schema.AvroCompiledSchemaRepositoryFactory; +import pl.allegro.tech.hermes.common.schema.SchemaRepositoryFactory; +import pl.allegro.tech.hermes.common.schema.SchemaSourceClientFactory; +import pl.allegro.tech.hermes.common.schema.SchemaVersionsRepositoryFactory; import pl.allegro.tech.hermes.common.util.HostnameResolver; import pl.allegro.tech.hermes.common.util.InetAddressHostnameResolver; import pl.allegro.tech.hermes.domain.notifications.InternalNotificationsBus; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchemaRepository; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; -import pl.allegro.tech.hermes.infrastructure.schema.AvroCompiledSchemaRepositoryFactory; -import pl.allegro.tech.hermes.infrastructure.schema.JsonCompiledSchemaRepositoryFactory; -import pl.allegro.tech.hermes.infrastructure.schema.SchemaSourceProviderFactory; -import pl.allegro.tech.hermes.infrastructure.schema.SchemaVersionsRepositoryFactory; -import pl.allegro.tech.hermes.infrastructure.schema.repo.SchemaRepoClientFactory; import pl.allegro.tech.hermes.infrastructure.zookeeper.notifications.ZookeeperInternalNotificationBus; +import pl.allegro.tech.hermes.schema.CompiledSchemaRepository; import javax.inject.Singleton; import java.time.Clock; @@ -58,12 +54,10 @@ protected void configure() { bindFactory(ClockFactory.class).in(Singleton.class).to(Clock.class); bind(ZookeeperBrokerStorage.class).to(BrokerStorage.class).in(Singleton.class); bind(InetAddressHostnameResolver.class).in(Singleton.class).to(HostnameResolver.class); - bindSingletonFactory(SchemaSourceProviderFactory.class); - bindSingletonFactory(SchemaRepoClientFactory.class); + bindSingletonFactory(SchemaSourceClientFactory.class); bindSingletonFactory(SchemaVersionsRepositoryFactory.class); - bindSingleton(SchemaRepository.class); - bindFactory(JsonCompiledSchemaRepositoryFactory.class).in(Singleton.class).to(new TypeLiteral>() {}); bindFactory(AvroCompiledSchemaRepositoryFactory.class).in(Singleton.class).to(new TypeLiteral>() {}); + bindSingletonFactory(SchemaRepositoryFactory.class); bindSingleton(CuratorClientFactory.class); bindSingleton(HermesMetrics.class); @@ -77,7 +71,6 @@ protected void configure() { bindSingletonFactory(KafkaCuratorClientFactory.class).named(CuratorType.KAFKA); bindSingletonFactory(GraphiteWebTargetFactory.class); bindSingletonFactory(ObjectMapperFactory.class); - bindSingletonFactory(BoonObjectMapperFactory.class); bindSingletonFactory(SharedCounterFactory.class); bindSingletonFactory(DistributedEphemeralCounterFactory.class); bindSingletonFactory(MetricRegistryFactory.class); diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/BoonObjectMapperFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/BoonObjectMapperFactory.java deleted file mode 100644 index 97caccf1e6..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/BoonObjectMapperFactory.java +++ /dev/null @@ -1,18 +0,0 @@ -package pl.allegro.tech.hermes.common.di.factories; - -import org.boon.json.JsonFactory; -import org.boon.json.ObjectMapper; -import org.glassfish.hk2.api.Factory; - -public class BoonObjectMapperFactory implements Factory { - - @Override - public ObjectMapper provide() { - return JsonFactory.create(); - } - - @Override - public void dispose(ObjectMapper instance) { - - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/InvalidSchemaException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/InvalidSchemaException.java deleted file mode 100644 index e8057a1858..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/InvalidSchemaException.java +++ /dev/null @@ -1,26 +0,0 @@ -package pl.allegro.tech.hermes.common.exception; - -import com.google.common.base.Joiner; -import pl.allegro.tech.hermes.api.ErrorCode; - -import java.util.List; - -public class InvalidSchemaException extends HermesException { - - public InvalidSchemaException(Throwable cause) { - super("Error while trying to validate schema", cause); - } - - public InvalidSchemaException(List reasons) { - super(String.format("Tried to set invalid topic schema: %s", Joiner.on(";").join(reasons))); - } - - public InvalidSchemaException(String message) { - super(message); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.FORMAT_ERROR; - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/AvroSchemaSource.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/AvroSchemaSource.java deleted file mode 100644 index 8985a0ac9b..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/AvroSchemaSource.java +++ /dev/null @@ -1,47 +0,0 @@ -package pl.allegro.tech.hermes.common.message; - -import org.apache.avro.Schema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaMissingException; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; - -import java.util.List; -import java.util.function.Function; - -public interface AvroSchemaSource { - CompiledSchema getAvroSchema(Topic topic, SchemaVersion version); - List versions(Topic topic, boolean online); - - - Logger logger = LoggerFactory.getLogger(AvroSchemaSource.class); - - default List versions(Topic topic) { - return versions(topic, false); - } - - default T invokeWithFallback(Topic topic, Function, T> processor, List availableVersions) { - for (SchemaVersion version : availableVersions) { - try { - return processor.apply(getAvroSchema(topic, version)); - } catch (Exception ex) { - logger.debug("Failed to match schema for message for topic {}, schema version {}, fallback to previous.", topic.getQualifiedName(), version.value()); - } - } - logger.error("Could not match schema from cache for message for topic {} {}", topic.getQualifiedName(), SchemaVersion.toString(availableVersions)); - throw new SchemaMissingException(topic); - } - - // try-harding to find proper schema - default T tryHard(Topic topic, Function, T> processor) { - try { - return invokeWithFallback(topic, processor, versions(topic)); - } catch (Exception ex) { - logger.info("Trying to find schema online for message for topic {}", topic.getQualifiedName()); - return invokeWithFallback(topic, processor, versions(topic, true)); - } - } - -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/DeserializationException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/DeserializationException.java deleted file mode 100644 index b5aedb68cf..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/DeserializationException.java +++ /dev/null @@ -1,13 +0,0 @@ -package pl.allegro.tech.hermes.common.message.serialization; - -import pl.allegro.tech.hermes.common.exception.InternalProcessingException; - -public class DeserializationException extends InternalProcessingException { - public DeserializationException(String message, Exception cause) { - super(message, cause); - } - - public DeserializationException(String message) { - super(message); - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapper.java index ac37c48a05..4bb2f41f99 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapper.java @@ -5,7 +5,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.util.Utf8; import pl.allegro.tech.hermes.common.util.MessageId; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchema; import javax.inject.Inject; import java.time.Clock; diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/DeserializationException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/DeserializationException.java new file mode 100644 index 0000000000..9cc62105f8 --- /dev/null +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/DeserializationException.java @@ -0,0 +1,8 @@ +package pl.allegro.tech.hermes.common.message.wrapper; + +public class DeserializationException extends RuntimeException { + + DeserializationException(String message) { + super(message); + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapper.java index e608cc3440..e227854f50 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapper.java @@ -1,26 +1,31 @@ package pl.allegro.tech.hermes.common.message.wrapper; import org.apache.avro.Schema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.message.AvroSchemaSource; -import pl.allegro.tech.hermes.common.message.serialization.SchemaAwarePayload; -import pl.allegro.tech.hermes.common.message.serialization.SchemaAwareSerDe; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaVersion; import javax.inject.Inject; +import java.util.List; import java.util.Map; -import static java.util.Optional.of; - public class MessageContentWrapper { + + private static final Logger logger = LoggerFactory.getLogger(MessageContentWrapper.class); + private final JsonMessageContentWrapper jsonMessageContentWrapper; private final AvroMessageContentWrapper avroMessageContentWrapper; + private final SchemaRepository schemaRepository; @Inject public MessageContentWrapper(JsonMessageContentWrapper jsonMessageContentWrapper, - AvroMessageContentWrapper avroMessageContentWrapper) { + AvroMessageContentWrapper avroMessageContentWrapper, SchemaRepository schemaRepository) { this.jsonMessageContentWrapper = jsonMessageContentWrapper; this.avroMessageContentWrapper = avroMessageContentWrapper; + this.schemaRepository = schemaRepository; } public UnwrappedMessageContent unwrapJson(byte[] data) { @@ -28,15 +33,42 @@ public UnwrappedMessageContent unwrapJson(byte[] data) { } public UnwrappedMessageContent unwrapAvro(byte[] data, - Topic topic, - AvroSchemaSource schemaSource) { - return topic.isSchemaVersionAwareSerializationEnabled() ? - deserialize(data, topic, schemaSource) : schemaSource.tryHard(topic, schema -> avroMessageContentWrapper.unwrapContent(data, schema)); + Topic topic) { + return topic.isSchemaVersionAwareSerializationEnabled() ? deserializeSchemaVersionAwarePayload(data, topic) : + tryDeserializingUsingAnySchemaVersion(data, topic); } - private UnwrappedMessageContent deserialize(byte[] data, Topic topic, AvroSchemaSource schemaSource) { + private UnwrappedMessageContent deserializeSchemaVersionAwarePayload(byte[] data, Topic topic) { SchemaAwarePayload payload = SchemaAwareSerDe.deserialize(data); - return avroMessageContentWrapper.unwrapContent(payload.getPayload(), schemaSource.getAvroSchema(topic, payload.getSchemaVersion())); + return avroMessageContentWrapper.unwrapContent(payload.getPayload(), + schemaRepository.getAvroSchema(topic, payload.getSchemaVersion())); + } + + // try-harding to find proper schema + private UnwrappedMessageContent tryDeserializingUsingAnySchemaVersion(byte[] data, Topic topic) { + try { + return tryDeserializingUsingAnySchemaVersion(data, topic, false); + } catch (Exception ex) { + logger.info("Trying to find schema online for message for topic {}", topic.getQualifiedName()); + return tryDeserializingUsingAnySchemaVersion(data, topic, true); + } + } + + private UnwrappedMessageContent tryDeserializingUsingAnySchemaVersion(byte[] data, Topic topic, boolean online) { + List versions = schemaRepository.getVersions(topic, online); + for (SchemaVersion version : versions) { + try { + CompiledSchema schema = online ? schemaRepository.getKnownAvroSchemaVersion(topic, version) : + schemaRepository.getAvroSchema(topic, version); + return avroMessageContentWrapper.unwrapContent(data, schema); + } catch (Exception ex) { + logger.debug("Failed to match schema for message for topic {}, schema version {}, fallback to previous.", + topic.getQualifiedName(), version.value()); + } + } + logger.error("Could not match schema from cache for message for topic {} {}", + topic.getQualifiedName(), SchemaVersion.toString(versions)); + throw new SchemaMissingException(topic); } public byte[] wrapAvro(byte[] data, String id, long timestamp, Topic topic, CompiledSchema schema, Map externalMetadata) { diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwarePayload.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwarePayload.java similarity index 77% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwarePayload.java rename to hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwarePayload.java index 47c8eb7cea..a7f9d2b74a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwarePayload.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwarePayload.java @@ -1,6 +1,6 @@ -package pl.allegro.tech.hermes.common.message.serialization; +package pl.allegro.tech.hermes.common.message.wrapper; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; +import pl.allegro.tech.hermes.schema.SchemaVersion; public class SchemaAwarePayload { private final byte[] payload; diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwareSerDe.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDe.java similarity index 80% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwareSerDe.java rename to hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDe.java index 0ee542cee1..3d9d0e3bcf 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwareSerDe.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDe.java @@ -1,11 +1,10 @@ -package pl.allegro.tech.hermes.common.message.serialization; +package pl.allegro.tech.hermes.common.message.wrapper; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; +import pl.allegro.tech.hermes.schema.SchemaVersion; import java.nio.ByteBuffer; import static java.lang.String.format; -import static pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion.valueOf; public class SchemaAwareSerDe { private static final byte MAGIC_BYTE = 0; @@ -28,7 +27,7 @@ public static SchemaAwarePayload deserialize(byte[] payloadWithHeader) { int schemaVersion = buffer.getInt(); byte[] payload = new byte[payloadWithHeader.length - HEADER_SIZE]; buffer.get(payload); - return new SchemaAwarePayload(payload, valueOf(schemaVersion)); + return new SchemaAwarePayload(payload, SchemaVersion.valueOf(schemaVersion)); } private static void assertMagicByte(byte magicByte) { diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaMissingException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaMissingException.java similarity index 80% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaMissingException.java rename to hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaMissingException.java index 86fe14554e..6df5563fe1 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaMissingException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaMissingException.java @@ -1,4 +1,4 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.common.message.wrapper; import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.api.Topic; @@ -6,7 +6,7 @@ public class SchemaMissingException extends HermesException { - public SchemaMissingException(Topic topic) { + SchemaMissingException(Topic topic) { super("Schema for topic " + topic.getQualifiedName() + " was not available"); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappedMessageContent.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappedMessageContent.java index a6f2926164..f531186a68 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappedMessageContent.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappedMessageContent.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.common.message.wrapper; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchema; import java.util.Optional; diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/AvroCompiledSchemaRepositoryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/AvroCompiledSchemaRepositoryFactory.java similarity index 58% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/AvroCompiledSchemaRepositoryFactory.java rename to hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/AvroCompiledSchemaRepositoryFactory.java index 09ecb65284..d876121a9b 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/AvroCompiledSchemaRepositoryFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/AvroCompiledSchemaRepositoryFactory.java @@ -1,28 +1,32 @@ -package pl.allegro.tech.hermes.infrastructure.schema; +package pl.allegro.tech.hermes.common.schema; import org.apache.avro.Schema; import org.glassfish.hk2.api.Factory; import pl.allegro.tech.hermes.common.config.ConfigFactory; import pl.allegro.tech.hermes.common.config.Configs; -import pl.allegro.tech.hermes.domain.topic.schema.*; +import pl.allegro.tech.hermes.schema.CachedCompiledSchemaRepository; +import pl.allegro.tech.hermes.schema.CompiledSchemaRepository; +import pl.allegro.tech.hermes.schema.DirectCompiledSchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaCompilersFactory; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; import javax.inject.Inject; public class AvroCompiledSchemaRepositoryFactory implements Factory> { - private final SchemaSourceProvider schemaSourceProvider; + private final SchemaSourceClient schemaSourceClient; private final ConfigFactory configFactory; @Inject - public AvroCompiledSchemaRepositoryFactory(SchemaSourceProvider schemaSourceProvider, ConfigFactory configFactory) { - this.schemaSourceProvider = schemaSourceProvider; + public AvroCompiledSchemaRepositoryFactory(SchemaSourceClient schemaSourceClient, ConfigFactory configFactory) { + this.schemaSourceClient = schemaSourceClient; this.configFactory = configFactory; } @Override public CompiledSchemaRepository provide() { return new CachedCompiledSchemaRepository<>( - new DirectCompiledSchemaRepository<>(schemaSourceProvider, SchemaCompilersFactory.avroSchemaCompiler()), + new DirectCompiledSchemaRepository<>(schemaSourceClient, SchemaCompilersFactory.avroSchemaCompiler()), configFactory.getIntProperty(Configs.SCHEMA_CACHE_COMPILED_MAXIMUM_SIZE), configFactory.getIntProperty(Configs.SCHEMA_CACHE_COMPILED_EXPIRE_AFTER_ACCESS_MINUTES)); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryFactory.java new file mode 100644 index 0000000000..f7139ebb35 --- /dev/null +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryFactory.java @@ -0,0 +1,33 @@ +package pl.allegro.tech.hermes.common.schema; + +import org.apache.avro.Schema; +import org.glassfish.hk2.api.Factory; +import pl.allegro.tech.hermes.schema.CompiledSchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaVersionsRepository; + +import javax.inject.Inject; + +public class SchemaRepositoryFactory implements Factory { + + private final SchemaVersionsRepository schemaVersionsRepository; + + private final CompiledSchemaRepository compiledAvroSchemaRepository; + + @Inject + public SchemaRepositoryFactory(SchemaVersionsRepository schemaVersionsRepository, + CompiledSchemaRepository compiledAvroSchemaRepository) { + this.schemaVersionsRepository = schemaVersionsRepository; + this.compiledAvroSchemaRepository = compiledAvroSchemaRepository; + } + + @Override + public SchemaRepository provide() { + return new SchemaRepository(schemaVersionsRepository, compiledAvroSchemaRepository); + } + + @Override + public void dispose(SchemaRepository instance) { + + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryType.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryType.java new file mode 100644 index 0000000000..3cb7a80d5d --- /dev/null +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryType.java @@ -0,0 +1,5 @@ +package pl.allegro.tech.hermes.common.schema; + +public enum SchemaRepositoryType { + SCHEMA_REPO, SCHEMA_REGISTRY +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaSourceClientFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaSourceClientFactory.java new file mode 100644 index 0000000000..04f1e40c0c --- /dev/null +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaSourceClientFactory.java @@ -0,0 +1,53 @@ +package pl.allegro.tech.hermes.common.schema; + +import org.glassfish.hk2.api.Factory; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; +import pl.allegro.tech.hermes.common.config.ConfigFactory; +import pl.allegro.tech.hermes.common.config.Configs; +import pl.allegro.tech.hermes.schema.confluent.SchemaRegistrySchemaSourceClient; +import pl.allegro.tech.hermes.schema.schemarepo.SchemaRepoSchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; + +import javax.inject.Inject; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import java.net.URI; + +public class SchemaSourceClientFactory implements Factory { + + private final ConfigFactory configFactory; + + @Inject + public SchemaSourceClientFactory(ConfigFactory configFactory) { + this.configFactory = configFactory; + } + + @Override + public SchemaSourceClient provide() { + + int httpReadTimeoutMs = configFactory.getIntProperty(Configs.SCHEMA_REPOSITORY_HTTP_READ_TIMEOUT_MS); + int httpConnectTimeoutMs = configFactory.getIntProperty(Configs.SCHEMA_REPOSITORY_HTTP_CONNECT_TIMEOUT_MS); + + ClientConfig config = new ClientConfig() + .property(ClientProperties.READ_TIMEOUT, httpReadTimeoutMs) + .property(ClientProperties.CONNECT_TIMEOUT, httpConnectTimeoutMs); + + String schemaRepositoryType = configFactory.getStringProperty(Configs.SCHEMA_REPOSITORY_TYPE).toUpperCase(); + Client client = ClientBuilder.newClient(config); + URI schemaRepositoryServerUri = URI.create(configFactory.getStringProperty(Configs.SCHEMA_REPOSITORY_SERVER_URL)); + switch (SchemaRepositoryType.valueOf(schemaRepositoryType)) { + case SCHEMA_REPO: + return new SchemaRepoSchemaSourceClient(client, schemaRepositoryServerUri); + case SCHEMA_REGISTRY: + return new SchemaRegistrySchemaSourceClient(client, schemaRepositoryServerUri); + default: + throw new IllegalStateException("Unknown schema repository type " + schemaRepositoryType); + } + } + + @Override + public void dispose(SchemaSourceClient instance) { + + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionsRepositoryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionsRepositoryFactory.java new file mode 100644 index 0000000000..0c2a6b5875 --- /dev/null +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionsRepositoryFactory.java @@ -0,0 +1,48 @@ +package pl.allegro.tech.hermes.common.schema; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.glassfish.hk2.api.Factory; +import pl.allegro.tech.hermes.common.config.ConfigFactory; +import pl.allegro.tech.hermes.common.config.Configs; +import pl.allegro.tech.hermes.schema.CachedSchemaVersionsRepository; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaVersionsRepository; +import pl.allegro.tech.hermes.schema.DirectSchemaVersionsRepository; + +import javax.inject.Inject; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class SchemaVersionsRepositoryFactory implements Factory { + + private final SchemaSourceClient schemaSourceClient; + private final ConfigFactory configFactory; + + @Inject + public SchemaVersionsRepositoryFactory(SchemaSourceClient schemaSourceClient, ConfigFactory configFactory) { + this.schemaSourceClient = schemaSourceClient; + this.configFactory = configFactory; + } + + @Override + public SchemaVersionsRepository provide() { + if (configFactory.getBooleanProperty(Configs.SCHEMA_CACHE_ENABLED)) { + return new CachedSchemaVersionsRepository(schemaSourceClient, + getVersionsReloader(), + configFactory.getIntProperty(Configs.SCHEMA_CACHE_REFRESH_AFTER_WRITE_MINUTES), + configFactory.getIntProperty(Configs.SCHEMA_CACHE_EXPIRE_AFTER_WRITE_MINUTES)); + } + return new DirectSchemaVersionsRepository(schemaSourceClient); + } + + private ExecutorService getVersionsReloader() { + return Executors.newFixedThreadPool( + configFactory.getIntProperty(Configs.SCHEMA_CACHE_RELOAD_THREAD_POOL_SIZE), + new ThreadFactoryBuilder().setNameFormat("schema-source-reloader-%d").build()); + } + + @Override + public void dispose(SchemaVersionsRepository instance) { + + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CouldNotCompileSchemaException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CouldNotCompileSchemaException.java deleted file mode 100644 index 8de6814328..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CouldNotCompileSchemaException.java +++ /dev/null @@ -1,17 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import pl.allegro.tech.hermes.api.ErrorCode; -import pl.allegro.tech.hermes.common.exception.HermesException; - -public class CouldNotCompileSchemaException extends HermesException { - - public CouldNotCompileSchemaException(Throwable cause) { - super(cause); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.OTHER; - } - -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CouldNotLoadSchemaException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CouldNotLoadSchemaException.java deleted file mode 100644 index 5739e5a493..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CouldNotLoadSchemaException.java +++ /dev/null @@ -1,20 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import pl.allegro.tech.hermes.api.ErrorCode; -import pl.allegro.tech.hermes.common.exception.HermesException; - -public class CouldNotLoadSchemaException extends HermesException { - - public CouldNotLoadSchemaException(Throwable cause) { - super(cause); - } - - public CouldNotLoadSchemaException(String message, Throwable cause) { - super(message, cause); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaCompilersFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaCompilersFactory.java deleted file mode 100644 index 1be5ccec0e..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaCompilersFactory.java +++ /dev/null @@ -1,28 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import org.apache.avro.Schema; - -import java.io.IOException; - -public interface SchemaCompilersFactory { - - static SchemaCompiler avroSchemaCompiler() { - return source -> new Schema.Parser().parse(source.value()); - } - - static SchemaCompiler jsonSchemaCompiler(ObjectMapper objectMapper) { - JsonSchemaFactory jsonSchemaFactory = JsonSchemaFactory.byDefault(); - return source -> { - try { - return jsonSchemaFactory.getJsonSchema(objectMapper.readTree(source.value())); - } catch (IOException | ProcessingException e) { - throw new CouldNotCompileSchemaException(e); - } - }; - } - -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepoSchemaSourceProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepoSchemaSourceProvider.java deleted file mode 100644 index a884c8d5c3..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepoSchemaSourceProvider.java +++ /dev/null @@ -1,34 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.infrastructure.schema.repo.SchemaRepoClient; - -import javax.inject.Inject; -import java.util.List; -import java.util.Optional; - -public class SchemaRepoSchemaSourceProvider implements SchemaSourceProvider { - - protected final SchemaRepoClient schemaRepoClient; - - @Inject - public SchemaRepoSchemaSourceProvider(SchemaRepoClient schemaRepoClient) { - this.schemaRepoClient = schemaRepoClient; - } - - @Override - public Optional get(Topic topic) { - return schemaRepoClient.getLatestSchema(topic.getQualifiedName()).map(SchemaSource::valueOf); - } - - @Override - public Optional get(Topic topic, SchemaVersion version) { - return schemaRepoClient.getSchema(topic.getQualifiedName(), version).map(SchemaSource::valueOf); - } - - @Override - public List versions(Topic topic) { - return schemaRepoClient.getSchemaVersions(topic.getQualifiedName()); - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepository.java deleted file mode 100644 index 31a9cfaa7b..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepository.java +++ /dev/null @@ -1,59 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import com.github.fge.jsonschema.main.JsonSchema; -import org.apache.avro.Schema; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.message.AvroSchemaSource; - -import javax.inject.Inject; -import java.util.List; -import java.util.Optional; - -public class SchemaRepository implements AvroSchemaSource { - - private final SchemaVersionsRepository schemaVersionsRepository; - private final CompiledSchemaRepository avroSchemaRepository; - private final CompiledSchemaRepository jsonSchemaRepository; - - @Inject - public SchemaRepository(SchemaVersionsRepository schemaVersionsRepository, - CompiledSchemaRepository avroSchemaRepository, - CompiledSchemaRepository jsonSchemaRepository) { - this.schemaVersionsRepository = schemaVersionsRepository; - this.avroSchemaRepository = avroSchemaRepository; - this.jsonSchemaRepository = jsonSchemaRepository; - } - - public CompiledSchema getAvroSchema(Topic topic) { - return getSchema(topic, avroSchemaRepository); - } - - public CompiledSchema getAvroSchema(Topic topic, SchemaVersion version) { - return getSchema(topic, version, avroSchemaRepository); - } - - public CompiledSchema getJsonSchema(Topic topic) { - return getSchema(topic, jsonSchemaRepository); - } - - public CompiledSchema getJsonSchema(Topic topic, SchemaVersion version) { - return getSchema(topic, version, jsonSchemaRepository); - } - - private CompiledSchema getSchema(Topic topic, CompiledSchemaRepository compiledSchemaRepository) { - SchemaVersion latestVersion = schemaVersionsRepository.latestSchemaVersion(topic).orElseThrow(() -> new SchemaMissingException(topic)); - return getSchema(topic, latestVersion, compiledSchemaRepository); - } - - private CompiledSchema getSchema(Topic topic, SchemaVersion version, CompiledSchemaRepository compiledSchemaRepository) { - if (!schemaVersionsRepository.schemaVersionExists(topic, version)) { - throw new UnknownSchemaVersionException(topic, version); - } - return compiledSchemaRepository.getSchema(topic, version); - } - - @Override - public List versions(Topic topic, boolean online) { - return schemaVersionsRepository.versions(topic, online); - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepositoryType.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepositoryType.java deleted file mode 100644 index 5c3c56bed5..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepositoryType.java +++ /dev/null @@ -1,5 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -public enum SchemaRepositoryType { - SCHEMA_REPO, ZOOKEEPER -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaSourceProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaSourceProvider.java deleted file mode 100644 index 863faef22d..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaSourceProvider.java +++ /dev/null @@ -1,23 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.api.Topic; - -import java.util.List; -import java.util.Optional; - -public interface SchemaSourceProvider { - - Optional get(Topic topic); - - default Optional get(Topic topic, SchemaVersion version) { - throw new UnsupportedOperationException("schema versioning not supported"); - } - - /** - * @return a sorted list of versions in descending order. - */ - default List versions(Topic topic) { - throw new UnsupportedOperationException("schema versioning not supported"); - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/TopicWithSchema.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/TopicWithSchema.java deleted file mode 100644 index 30f9bb0a5c..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/TopicWithSchema.java +++ /dev/null @@ -1,22 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import pl.allegro.tech.hermes.api.Topic; - -public class TopicWithSchema { - - private final Topic topic; - private final T schema; - - TopicWithSchema(Topic topic, T schema) { - this.topic = topic; - this.schema = schema; - } - - public Topic getTopic() { - return topic; - } - - public T getSchema() { - return schema; - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/UnknownSchemaVersionException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/UnknownSchemaVersionException.java deleted file mode 100644 index a708d00d56..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/UnknownSchemaVersionException.java +++ /dev/null @@ -1,17 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import pl.allegro.tech.hermes.api.ErrorCode; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.exception.HermesException; - -public class UnknownSchemaVersionException extends HermesException { - - public UnknownSchemaVersionException(Topic topic, SchemaVersion version) { - super("Unknown schema version " + version.value() + " for topic " + topic.getQualifiedName()); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/ZookeeperSchemaSourceProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/ZookeeperSchemaSourceProvider.java deleted file mode 100644 index 338f92cdfe..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/ZookeeperSchemaSourceProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.zookeeper.KeeperException; -import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.exception.InternalProcessingException; -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; - -import java.nio.charset.Charset; -import java.util.Optional; - -public class ZookeeperSchemaSourceProvider implements SchemaSourceProvider { - - protected static final String SCHEMA_SUFFIX = "schema"; - - protected final CuratorFramework curatorFramework; - protected final ZookeeperPaths zkPaths; - - public ZookeeperSchemaSourceProvider(CuratorFramework curatorFramework, ZookeeperPaths zkPaths) { - this.curatorFramework = curatorFramework; - this.zkPaths = zkPaths; - } - - @Override - public Optional get(Topic topic) { - try { - byte[] schemaBytes = curatorFramework.getData().forPath(zkPaths.topicPath(topic.getName(), SCHEMA_SUFFIX)); - return Optional.of(SchemaSource.valueOf(new String(schemaBytes, Charset.defaultCharset()))); - } catch (KeeperException.NoNodeException ex) { - return Optional.empty(); - } catch (Exception ex) { - throw new InternalProcessingException("Could not load from zookeeper schema for topic " + topic.getQualifiedName(), ex); - } - } - -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/JsonCompiledSchemaRepositoryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/JsonCompiledSchemaRepositoryFactory.java deleted file mode 100644 index 13f1b1446b..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/JsonCompiledSchemaRepositoryFactory.java +++ /dev/null @@ -1,37 +0,0 @@ -package pl.allegro.tech.hermes.infrastructure.schema; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.fge.jsonschema.main.JsonSchema; -import org.glassfish.hk2.api.Factory; -import pl.allegro.tech.hermes.common.config.ConfigFactory; -import pl.allegro.tech.hermes.common.config.Configs; -import pl.allegro.tech.hermes.domain.topic.schema.*; - -import javax.inject.Inject; - -public class JsonCompiledSchemaRepositoryFactory implements Factory> { - - private final SchemaSourceProvider schemaSourceProvider; - private final ConfigFactory configFactory; - private final ObjectMapper objectMapper; - - @Inject - public JsonCompiledSchemaRepositoryFactory(ObjectMapper objectMapper, SchemaSourceProvider schemaSourceProvider, ConfigFactory configFactory) { - this.objectMapper = objectMapper; - this.schemaSourceProvider = schemaSourceProvider; - this.configFactory = configFactory; - } - - @Override - public CompiledSchemaRepository provide() { - return new CachedCompiledSchemaRepository<>( - new DirectCompiledSchemaRepository<>(schemaSourceProvider, SchemaCompilersFactory.jsonSchemaCompiler(objectMapper)), - configFactory.getIntProperty(Configs.SCHEMA_CACHE_COMPILED_MAXIMUM_SIZE), - configFactory.getIntProperty(Configs.SCHEMA_CACHE_COMPILED_EXPIRE_AFTER_ACCESS_MINUTES)); - } - - @Override - public void dispose(CompiledSchemaRepository instance) { - - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/SchemaSourceProviderFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/SchemaSourceProviderFactory.java deleted file mode 100644 index 2dd414d765..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/SchemaSourceProviderFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -package pl.allegro.tech.hermes.infrastructure.schema; - -import org.apache.curator.framework.CuratorFramework; -import org.glassfish.hk2.api.Factory; -import pl.allegro.tech.hermes.common.config.ConfigFactory; -import pl.allegro.tech.hermes.common.config.Configs; -import pl.allegro.tech.hermes.common.di.CuratorType; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepoSchemaSourceProvider; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepositoryType; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaSourceProvider; -import pl.allegro.tech.hermes.domain.topic.schema.ZookeeperSchemaSourceProvider; -import pl.allegro.tech.hermes.infrastructure.schema.repo.SchemaRepoClient; -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; - -import javax.inject.Inject; -import javax.inject.Named; - -public class SchemaSourceProviderFactory implements Factory { - - private final ConfigFactory configFactory; - private final SchemaRepoClient schemaRepoClient; - private final CuratorFramework curatorFramework; - private final ZookeeperPaths zookeeperPaths; - - @Inject - public SchemaSourceProviderFactory(ConfigFactory configFactory, - SchemaRepoClient schemaRepoClient, - @Named(CuratorType.HERMES) CuratorFramework curatorFramework, - ZookeeperPaths zookeeperPaths) { - this.configFactory = configFactory; - this.schemaRepoClient = schemaRepoClient; - this.curatorFramework = curatorFramework; - this.zookeeperPaths = zookeeperPaths; - } - - @Override - public SchemaSourceProvider provide() { - String schemaRepositoryType = configFactory.getStringProperty(Configs.SCHEMA_REPOSITORY_TYPE).toUpperCase(); - switch (SchemaRepositoryType.valueOf(schemaRepositoryType)) { - case SCHEMA_REPO: - return new SchemaRepoSchemaSourceProvider(schemaRepoClient); - case ZOOKEEPER: - return new ZookeeperSchemaSourceProvider(curatorFramework, zookeeperPaths); - default: - throw new IllegalStateException("Unknown schema repository type " + schemaRepositoryType); - } - } - - @Override - public void dispose(SchemaSourceProvider instance) { - } - -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/SchemaVersionsRepositoryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/SchemaVersionsRepositoryFactory.java deleted file mode 100644 index a590a4df39..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/SchemaVersionsRepositoryFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -package pl.allegro.tech.hermes.infrastructure.schema; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.glassfish.hk2.api.Factory; -import pl.allegro.tech.hermes.common.config.ConfigFactory; -import pl.allegro.tech.hermes.domain.topic.schema.CachedSchemaVersionsRepository; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaSourceProvider; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersionsRepository; -import pl.allegro.tech.hermes.domain.topic.schema.SimpleSchemaVersionsRepository; - -import javax.inject.Inject; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static pl.allegro.tech.hermes.common.config.Configs.SCHEMA_CACHE_ENABLED; -import static pl.allegro.tech.hermes.common.config.Configs.SCHEMA_CACHE_EXPIRE_AFTER_WRITE_MINUTES; -import static pl.allegro.tech.hermes.common.config.Configs.SCHEMA_CACHE_REFRESH_AFTER_WRITE_MINUTES; -import static pl.allegro.tech.hermes.common.config.Configs.SCHEMA_CACHE_RELOAD_THREAD_POOL_SIZE; - -public class SchemaVersionsRepositoryFactory implements Factory { - - private final SchemaSourceProvider sourceProvider; - private final ConfigFactory configFactory; - - @Inject - public SchemaVersionsRepositoryFactory(SchemaSourceProvider sourceProvider, ConfigFactory configFactory) { - this.sourceProvider = sourceProvider; - this.configFactory = configFactory; - } - - @Override - public SchemaVersionsRepository provide() { - if (configFactory.getBooleanProperty(SCHEMA_CACHE_ENABLED)) { - return new CachedSchemaVersionsRepository(sourceProvider, - getVersionsReloader(), - configFactory.getIntProperty(SCHEMA_CACHE_REFRESH_AFTER_WRITE_MINUTES), - configFactory.getIntProperty(SCHEMA_CACHE_EXPIRE_AFTER_WRITE_MINUTES)); - } - return new SimpleSchemaVersionsRepository(sourceProvider); - } - - private ExecutorService getVersionsReloader() { - return Executors.newFixedThreadPool( - configFactory.getIntProperty(SCHEMA_CACHE_RELOAD_THREAD_POOL_SIZE), - new ThreadFactoryBuilder().setNameFormat("schema-source-reloader-%d").build()); - } - - @Override - public void dispose(SchemaVersionsRepository instance) { - - } -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/SchemaRepoClient.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/SchemaRepoClient.java deleted file mode 100644 index 2945e4c04a..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/SchemaRepoClient.java +++ /dev/null @@ -1,24 +0,0 @@ -package pl.allegro.tech.hermes.infrastructure.schema.repo; - -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; - -import java.util.List; -import java.util.Optional; - -public interface SchemaRepoClient { - - void registerSubject(String subject); - - boolean isSubjectRegistered(String subject); - - void registerSchema(String subject, String schema); - - Optional getLatestSchema(String subject); - - Optional getSchema(String subject, SchemaVersion version); - - /** - * @return a sorted list of versions in descending order. - */ - List getSchemaVersions(String subject); -} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/SchemaRepoClientFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/SchemaRepoClientFactory.java deleted file mode 100644 index 2c790eeae1..0000000000 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/SchemaRepoClientFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -package pl.allegro.tech.hermes.infrastructure.schema.repo; - -import org.glassfish.hk2.api.Factory; -import org.glassfish.jersey.client.ClientConfig; -import org.glassfish.jersey.client.ClientProperties; -import pl.allegro.tech.hermes.common.config.ConfigFactory; -import pl.allegro.tech.hermes.common.config.Configs; - -import javax.inject.Inject; -import javax.ws.rs.client.ClientBuilder; -import java.net.URI; - -public class SchemaRepoClientFactory implements Factory { - - private final ConfigFactory configFactory; - - @Inject - public SchemaRepoClientFactory(ConfigFactory configFactory) { - this.configFactory = configFactory; - } - - @Override - public SchemaRepoClient provide() { - ClientConfig config = new ClientConfig() - .property(ClientProperties.READ_TIMEOUT, configFactory.getIntProperty(Configs.SCHEMA_REPOSITORY_HTTP_READ_TIMEOUT_MS)) - .property(ClientProperties.CONNECT_TIMEOUT, configFactory.getIntProperty(Configs.SCHEMA_REPOSITORY_HTTP_CONNECT_TIMEOUT_MS)); - - return new JerseySchemaRepoClient(ClientBuilder.newClient(config), URI.create(configFactory.getStringProperty(Configs.SCHEMA_REPOSITORY_SERVER_URL))); - } - - @Override - public void dispose(SchemaRepoClient instance) { - - } -} diff --git a/hermes-common/src/test/groovy/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperTopicRepositoryTest.groovy b/hermes-common/src/test/groovy/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperTopicRepositoryTest.groovy index 84ebe47feb..13050d0cee 100644 --- a/hermes-common/src/test/groovy/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperTopicRepositoryTest.groovy +++ b/hermes-common/src/test/groovy/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperTopicRepositoryTest.groovy @@ -88,7 +88,7 @@ class ZookeeperTopicRepositoryTest extends IntegrationTest { def "should load topic details"() { given: - repository.createTopic(topic(GROUP, 'details').withDescription('description').withValidation(true).build()) + repository.createTopic(topic(GROUP, 'details').withDescription('description').build()) wait.untilTopicCreated(GROUP, 'details') when: @@ -96,7 +96,6 @@ class ZookeeperTopicRepositoryTest extends IntegrationTest { then: retrievedTopic.description == 'description' - retrievedTopic.validationEnabled } def "should update topic"() { diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapperTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapperTest.java index 5a5ec8180e..525cb94b90 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapperTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapperTest.java @@ -5,18 +5,17 @@ import org.apache.commons.collections4.map.HashedMap; import org.junit.Test; import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.message.AvroSchemaSource; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaMissingException; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; +import pl.allegro.tech.hermes.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaVersion; +import pl.allegro.tech.hermes.schema.SchemaVersionsRepository; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import pl.allegro.tech.hermes.test.helper.builder.TopicBuilder; import java.io.IOException; import java.time.Clock; -import java.util.List; -import static com.googlecode.catchexception.CatchException.catchException; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader.load; @@ -25,55 +24,47 @@ public class MessageContentWrapperTest { private final MessageContentWrapper messageContentWrapper = new MessageContentWrapper( new JsonMessageContentWrapper("message", "metadata", new ObjectMapper()), - new AvroMessageContentWrapper(Clock.systemDefaultZone()) - ); + new AvroMessageContentWrapper(Clock.systemDefaultZone()), schemaRepository); static Topic topic = TopicBuilder.topic("group", "topic").build(); - static AvroSchemaSource schemaSource = new AvroUserSchemaSource(); - - static class AvroUserSchemaSource implements AvroSchemaSource { - CompiledSchema schema1 = new CompiledSchema<>(load("/schema/user.avsc"), SchemaVersion.valueOf(1)); - CompiledSchema schema2 = new CompiledSchema<>(load("/schema/user_v2.avsc"), SchemaVersion.valueOf(2)); - CompiledSchema schema3 = new CompiledSchema<>(load("/schema/user_v3.avsc"), SchemaVersion.valueOf(3)); - - @Override - public CompiledSchema getAvroSchema(Topic topic, SchemaVersion version) { - switch (version.value()) { - case 1: return schema1; - case 2: return schema2; - case 3: return schema3; - default: throw new RuntimeException("sry"); - } - } - - @Override - public List versions(Topic topic, boolean online) { - return online? - asList(schema3.getVersion(), schema2.getVersion(), schema1.getVersion()) - : asList(schema2.getVersion(), schema1.getVersion()); - } - } + + static CompiledSchema schema1 = new CompiledSchema<>(load("/schema/user.avsc"), SchemaVersion.valueOf(1)); + static CompiledSchema schema2 = new CompiledSchema<>(load("/schema/user_v2.avsc"), SchemaVersion.valueOf(2)); + static CompiledSchema schema3 = new CompiledSchema<>(load("/schema/user_v3.avsc"), SchemaVersion.valueOf(3)); + + static SchemaVersionsRepository schemaVersionsRepository = (topic, online) -> + online? asList(schema3.getVersion(), schema2.getVersion(), schema1.getVersion()) + : asList(schema2.getVersion(), schema1.getVersion()); + + static CompiledSchemaRepository compiledSchemaRepository = (topic, version) -> { + switch (version.value()) { + case 1: return schema1; + case 2: return schema2; + case 3: return schema3; + default: throw new RuntimeException("sry");} + }; + static SchemaRepository schemaRepository = new SchemaRepository(schemaVersionsRepository, compiledSchemaRepository); @Test public void shouldUnwrapMessageUsingEverySchemaAvailable() throws IOException { // forcing offline latest - shouldUnwrapMessage(SchemaVersion.valueOf(2)); + shouldUnwrapMessageWrappedWithSchemaAtVersion(SchemaVersion.valueOf(2)); // forcing fallback - shouldUnwrapMessage(SchemaVersion.valueOf(1)); + shouldUnwrapMessageWrappedWithSchemaAtVersion(SchemaVersion.valueOf(1)); // forcing online check - shouldUnwrapMessage(SchemaVersion.valueOf(3)); + shouldUnwrapMessageWrappedWithSchemaAtVersion(SchemaVersion.valueOf(3)); } - public void shouldUnwrapMessage(SchemaVersion version) throws IOException { + public void shouldUnwrapMessageWrappedWithSchemaAtVersion(SchemaVersion version) throws IOException { // given - CompiledSchema schema = schemaSource.getAvroSchema(topic, version); + CompiledSchema schema = schemaRepository.getKnownAvroSchemaVersion(topic, version); AvroUser user = new AvroUser(schema, "Bob", 15, "blue"); byte[] wrapped = messageContentWrapper.wrapAvro(user.asBytes(), "uniqueId", 1234, topic, schema, new HashedMap<>()); // when - UnwrappedMessageContent unwrappedMessageContent = messageContentWrapper.unwrapAvro(wrapped, topic, schemaSource); + UnwrappedMessageContent unwrappedMessageContent = messageContentWrapper.unwrapAvro(wrapped, topic); // then assertThat(unwrappedMessageContent.getContent()).contains(user.asBytes()); @@ -88,6 +79,6 @@ public void shouldThrowExceptionWhenMessageCouldNotBeUnwrappedByAnySchema() thro byte[] doesNotMatchAnySchema = "{}".getBytes(); // when - messageContentWrapper.unwrapAvro(doesNotMatchAnySchema, topic, schemaSource); + messageContentWrapper.unwrapAvro(doesNotMatchAnySchema, topic); } } \ No newline at end of file diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwareSerDeTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDeTest.java similarity index 91% rename from hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwareSerDeTest.java rename to hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDeTest.java index 73e0ea97a5..3dbc261872 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/serialization/SchemaAwareSerDeTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDeTest.java @@ -1,7 +1,7 @@ -package pl.allegro.tech.hermes.common.message.serialization; +package pl.allegro.tech.hermes.common.message.wrapper; import org.testng.annotations.Test; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; +import pl.allegro.tech.hermes.schema.SchemaVersion; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import java.io.IOException; diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepoSchemaSourceProviderTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepoSchemaSourceProviderTest.java deleted file mode 100644 index 4ca4d639fa..0000000000 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaRepoSchemaSourceProviderTest.java +++ /dev/null @@ -1,68 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; -import com.github.tomakehurst.wiremock.client.UrlMatchingStrategy; -import com.github.tomakehurst.wiremock.junit.WireMockRule; -import org.junit.Rule; -import org.junit.Test; -import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.config.ConfigFactory; -import pl.allegro.tech.hermes.infrastructure.schema.repo.SchemaRepoClientFactory; - -import java.util.Optional; - -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - -public class SchemaRepoSchemaSourceProviderTest { - - private static final String ROOT_DIR = "/schema-repo/"; - - private static final SchemaRepoSchemaSourceProvider sourceProvider = new SchemaRepoSchemaSourceProvider(new SchemaRepoClientFactory(new ConfigFactory()).provide()); - - private static final Topic topic = topic("someGroup.someTopic").build(); - - @Rule - public final WireMockRule wireMockRule = new WireMockRule(wireMockConfig().port(8888)); - - @Test - public void shouldReturnEmptyOptionalOnMissingSchema() { - // given - wireMockRule.stubFor(get(topicLatestSchema()).willReturn(notFoundResponse())); - - // when - Optional source = sourceProvider.get(topic); - - // then - wireMockRule.verify(1, getRequestedFor(topicLatestSchema())); - assertThat(source).isEmpty(); - } - - @Test - public void shouldReturnSchemaSourceWhenAvailable() { - // given - wireMockRule.stubFor(get(topicLatestSchema()).willReturn(okResponse().withBody("0\tsomeSchema"))); - - // when - Optional source = sourceProvider.get(topic); - - // then - wireMockRule.verify(1, getRequestedFor(topicLatestSchema())); - assertThat(source).contains(SchemaSource.valueOf("someSchema")); - } - - private UrlMatchingStrategy topicLatestSchema() { - return urlEqualTo(ROOT_DIR + topic.getQualifiedName() + "/latest"); - } - - private ResponseDefinitionBuilder okResponse() { - return aResponse().withStatus(200); - } - - private ResponseDefinitionBuilder notFoundResponse() { - return aResponse().withStatus(404); - } -} \ No newline at end of file diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/domain/topic/schema/ZookeeperSchemaSourceProviderTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/domain/topic/schema/ZookeeperSchemaSourceProviderTest.java deleted file mode 100644 index b116d6bd76..0000000000 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/domain/topic/schema/ZookeeperSchemaSourceProviderTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package pl.allegro.tech.hermes.domain.topic.schema; - -import com.googlecode.catchexception.CatchException; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.junit.Test; -import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.exception.InternalProcessingException; -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperBaseTest; - -import java.util.Optional; - -import static com.googlecode.catchexception.CatchException.catchException; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - -public class ZookeeperSchemaSourceProviderTest extends ZookeeperBaseTest { - - private static final String AVRO_SCHEMA = "{type:\"record\",name:\"schema\",namespace:\"com.avro\",fields:[{name:\"username\",type:\"string\",doc:\"Name of the user account\"}],\"doc:\":\"A basic schema\"}"; - - private ZookeeperSchemaSourceProvider provider = new ZookeeperSchemaSourceProvider(zookeeperClient, new ZookeeperPaths("/test")); - - @Test - public void shouldGetTopicSchemaFromZookeeper() throws Exception { - // given - Topic topic = topic("org.hermes.schema", "existing").build(); - zookeeperClient.create().creatingParentsIfNeeded().forPath("/test/groups/org.hermes.schema/topics/existing/schema", AVRO_SCHEMA.getBytes()); - - // when - Optional schemaSource = provider.get(topic); - - // then - assertThat(schemaSource).isPresent().contains(SchemaSource.valueOf(AVRO_SCHEMA)); - } - - @Test - public void shouldReturnEmptyWhenSchemaNotFound() { - // given - Topic topic = topic("org.hermes.schema", "notExisting").build(); - - // when - Optional schemaSource = provider.get(topic); - - // then - assertThat(schemaSource).isEmpty(); - } - - @Test - public void shouldThrowExceptionWhenCouldNotConnectToZK() { - // given - Topic topic = topic("org.hermes.schema", "broken").build(); - CuratorFramework notStartedClient = CuratorFrameworkFactory.newClient(zookeeperServer.getConnectString(), new ExponentialBackoffRetry(100, 1)); - ZookeeperSchemaSourceProvider brokenProvider = new ZookeeperSchemaSourceProvider(notStartedClient, new ZookeeperPaths("/test")); - - // when - catchException(brokenProvider).get(topic); - - // then - assertThat(CatchException.caughtException()).isInstanceOf(InternalProcessingException.class); - } - -} diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/infrastructure/schema/repo/JerseySchemaRepoClientTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/infrastructure/schema/repo/JerseySchemaRepoClientTest.java deleted file mode 100644 index 2f65044730..0000000000 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/infrastructure/schema/repo/JerseySchemaRepoClientTest.java +++ /dev/null @@ -1,226 +0,0 @@ -package pl.allegro.tech.hermes.infrastructure.schema.repo; - -import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; -import com.github.tomakehurst.wiremock.client.UrlMatchingStrategy; -import com.github.tomakehurst.wiremock.junit.WireMockRule; -import org.junit.ClassRule; -import org.junit.Test; -import org.testng.annotations.BeforeTest; -import pl.allegro.tech.hermes.common.exception.InvalidSchemaException; -import pl.allegro.tech.hermes.common.exception.SchemaRepoException; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; -import pl.allegro.tech.hermes.test.helper.util.Ports; - -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.core.MediaType; -import java.net.URI; -import java.util.List; -import java.util.Optional; - -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; -import static com.googlecode.catchexception.CatchException.catchException; -import static com.googlecode.catchexception.CatchException.caughtException; -import static org.assertj.core.api.Assertions.assertThat; - -public class JerseySchemaRepoClientTest { - - private static final String ROOT_DIR = "/schema-repo/"; - private static final int PORT = Ports.nextAvailable(); - - private static final SchemaRepoClient client = new JerseySchemaRepoClient( - ClientBuilder.newClient(), - URI.create("http://localhost:" + PORT + ROOT_DIR) - ); - - @ClassRule - public static final WireMockRule wireMock = new WireMockRule( - wireMockConfig().port(PORT).usingFilesUnderClasspath("schema-repo-stub") - ); - - @BeforeTest - public void initialize() { - wireMock.resetRequests(); - wireMock.resetMappings(); - wireMock.resetScenarios(); - } - - @Test - public void shouldCheckIfSubjectIsRegistered() { - // given - wireMock.stubFor(get(subjectUrl("notRegistered.subject")).willReturn(notFoundResponse())); - - // when - boolean exists = client.isSubjectRegistered("notRegistered.subject"); - - // then - wireMock.verify(1, getRequestedFor(subjectUrl("notRegistered.subject"))); - assertThat(exists).isFalse(); - } - - @Test - public void shouldRegisterSubject() { - // given - wireMock.stubFor(put(subjectUrl("register.subject")).willReturn(okResponse())); - - // when - client.registerSubject("register.subject"); - - // then - verify(putRequestedFor(subjectUrl("register.subject")) - .withHeader("Content-type", equalTo(MediaType.APPLICATION_FORM_URLENCODED))); - } - - @Test - public void shouldThrowExceptionForUnsuccessfulSubjectRegistration() { - // given - wireMock.stubFor(put(subjectUrl("failed.subject")).willReturn(serverErrorResponse())); - - // when - catchException(client).registerSubject("failed.subject"); - - // then - assertThat((Throwable) caughtException()).isInstanceOf(SchemaRepoException.class); - } - - @Test - public void shouldReturnEmptyOptionalIfLatestSchemaDoesNotExist() { - // given - wireMock.stubFor(get(latestSchemaUrl("nolatest.subject")).willReturn(notFoundResponse())); - - // when - Optional schema = client.getLatestSchema("nolatest.subject"); - - // then - assertThat(schema.isPresent()).isFalse(); - } - - @Test - public void shouldReturnLatestSchemaIfExists() { - // given - wireMock.stubFor(get(latestSchemaUrl("latest.subject")).willReturn(okResponse().withBody("0\t{}"))); - - // when - Optional schema = client.getLatestSchema("latest.subject"); - - // then - assertThat(schema.get()).isEqualTo("{}"); - } - - @Test - public void shouldRegisterSchema() { - // given - wireMock.stubFor(put(registerSchemaUrl("registerSchema.subject")).willReturn(okResponse())); - - // when - client.registerSchema("registerSchema.subject", "{}"); - - // then - verify(1, putRequestedFor(registerSchemaUrl("registerSchema.subject")) - .withHeader("Content-type", equalTo(MediaType.TEXT_PLAIN)) - .withRequestBody(equalTo("{}")) - ); - } - - @Test - public void shouldThrowExceptionForInvalidSchemaRegistration() { - // given - wireMock.stubFor(put(registerSchemaUrl("invalidSchema.subject")).willReturn(forbiddenResponse("error"))); - - // when - catchException(client).registerSchema("invalidSchema.subject", "{}"); - - // then - assertThat((Throwable) caughtException()) - .isInstanceOf(InvalidSchemaException.class) - .hasMessageContaining("error"); - } - - @Test - public void shouldThrowSchemaRepoExceptionForSchemaRegistration() { - // given - wireMock.stubFor(put(registerSchemaUrl("repoException.subject")).willReturn(serverErrorResponse())); - - // when - catchException(client).registerSchema("repoException.subject", "{}"); - - // then - assertThat((Throwable) caughtException()).isInstanceOf(SchemaRepoException.class); - } - - @Test - public void shouldReturnSchemaVersions() { - // given - wireMock.stubFor(get(allSchemasUrl("allSchemas.subject")).willReturn( - okResponse().withBodyFile("all-schemas-response.json").withHeader("Content-Type", "application/json")) - ); - - // when - List versions = client.getSchemaVersions("allSchemas.subject"); - - // then - assertThat(versions).containsExactly( - SchemaVersion.valueOf(2), - SchemaVersion.valueOf(1), - SchemaVersion.valueOf(0) - ); - } - - @Test - public void shouldReturnEmptySchemaVersionsIfNoSchemasAreRegistered() { - // given - wireMock.stubFor(get(allSchemasUrl("noSchema.subject")).willReturn( - okResponse().withBody("").withHeader("Content-Type", "application/json")) - ); - - // when - List versions = client.getSchemaVersions("noSchema.subject"); - - // then - assertThat(versions).isEmpty(); - } - - @Test - public void shouldReturnEmptySchemaVersionsIfSubjectDoesntExist() { - // given - wireMock.stubFor(get(allSchemasUrl("noSubject.subject")).willReturn(notFoundResponse())); - - // when - List versions = client.getSchemaVersions("noSubject.subject"); - - // then - assertThat(versions).isEmpty(); - } - - private UrlMatchingStrategy subjectUrl(String name) { - return urlEqualTo(ROOT_DIR + name); - } - - private UrlMatchingStrategy latestSchemaUrl(String name) { - return urlEqualTo(ROOT_DIR + name + "/latest"); - } - - private UrlMatchingStrategy registerSchemaUrl(String name) { - return urlEqualTo(ROOT_DIR + name + "/register"); - } - - private UrlMatchingStrategy allSchemasUrl(String name) { - return urlEqualTo(ROOT_DIR + name + "/all"); - } - - private ResponseDefinitionBuilder okResponse() { - return aResponse().withStatus(200); - } - - private ResponseDefinitionBuilder serverErrorResponse() { - return aResponse().withStatus(500); - } - - private ResponseDefinitionBuilder notFoundResponse() { - return aResponse().withStatus(404); - } - - private ResponseDefinitionBuilder forbiddenResponse(String body) { - return aResponse().withStatus(403).withBody(body); - } -} \ No newline at end of file diff --git a/hermes-console/config.json.example b/hermes-console/config.json.example index a621344020..c72dd07277 100644 --- a/hermes-console/config.json.example +++ b/hermes-console/config.json.example @@ -46,7 +46,8 @@ "retentionTime": { "duration": 1 } - } + }, + "removeSchema": false }, "subscription": { "endpointAddressResolverMetadata": { diff --git a/hermes-console/static/js/console/topic/TopicController.js b/hermes-console/static/js/console/topic/TopicController.js index ecc3b429e1..4da21dc065 100644 --- a/hermes-console/static/js/console/topic/TopicController.js +++ b/hermes-console/static/js/console/topic/TopicController.js @@ -73,7 +73,7 @@ topics.controller('TopicController', ['TOPIC_CONFIG', 'TopicRepository', 'TopicM passwordHint: 'Password for group ' + groupName }).result.then(function (result) { passwordService.set(result.password); - topicRepository.remove($scope.topic.name).$promise + topicRepository.remove($scope.topic) .then(function () { toaster.pop('success', 'Success', 'Topic has been removed'); $location.path('/groups/' + groupName).replace(); diff --git a/hermes-console/static/js/console/topic/TopicRepository.js b/hermes-console/static/js/console/topic/TopicRepository.js index 57fcfeff7c..502055d1e3 100644 --- a/hermes-console/static/js/console/topic/TopicRepository.js +++ b/hermes-console/static/js/console/topic/TopicRepository.js @@ -1,7 +1,8 @@ var repository = angular.module('hermes.topic.repository', ['hermes.subscription.repository']); -repository.factory('TopicRepository', ['DiscoveryService', '$resource', '$location', 'SubscriptionRepository', 'SchemaRepository', - function (discovery, $resource, $location, subscriptionRepository, schemaRepository) { +repository.factory('TopicRepository', ['DiscoveryService', '$resource', '$location', 'SubscriptionRepository', + 'SchemaRepository', 'TOPIC_CONFIG', + function (discovery, $resource, $location, subscriptionRepository, schemaRepository, topicConfig) { var repository = $resource(discovery.resolve('/topics/:name'), null, {update: {method: 'PUT'}}); var previewRepository = $resource(discovery.resolve('/topics/:name/preview'), null); @@ -16,6 +17,15 @@ repository.factory('TopicRepository', ['DiscoveryService', '$resource', '$locati return promise; } + function wrapSchemaRemove(topic, promise) { + if (topicConfig.removeSchema && topic.contentType == 'AVRO') { + return promise.then(function () { + return schemaRepository.remove(topic.name).$promise; + }); + } + return promise; + } + function ngPromiseCleaner(key, value) { if (_.contains(["$promise", "$resolved"], key)) { return undefined; @@ -39,8 +49,8 @@ repository.factory('TopicRepository', ['DiscoveryService', '$resource', '$locati add: function (topic, schema) { return wrapSchemaSave(topic, schema, listing.save({}, topic).$promise); }, - remove: function (name) { - return repository.remove({name: name}); + remove: function (topic) { + return wrapSchemaRemove(topic, repository.delete({name: topic.name}).$promise); }, save: function (topic, schema) { return wrapSchemaSave(topic, schema, repository.update({name: topic.name}, topic).$promise); @@ -74,6 +84,9 @@ repository.factory('SchemaRepository', ['DiscoveryService', '$resource', }, save: function (name, schema) { return repository.save({name: name}, schema); + }, + remove: function(name) { + return repository.remove({name: name}); } }; }]); diff --git a/hermes-console/static/partials/modal/editTopic.html b/hermes-console/static/partials/modal/editTopic.html index e0d50468bb..61c4b24666 100644 --- a/hermes-console/static/partials/modal/editTopic.html +++ b/hermes-console/static/partials/modal/editTopic.html @@ -58,18 +58,12 @@

Edit topic: -
-
-
+
diff --git a/hermes-consumers/build.gradle b/hermes-consumers/build.gradle index 9abe805d49..d6cccb7972 100644 --- a/hermes-consumers/build.gradle +++ b/hermes-consumers/build.gradle @@ -2,8 +2,9 @@ dependencies { compile project(':hermes-common') compile project(':hermes-tracker') compile project(':hermes-metrics') + compile project(':hermes-schema') - compile group: 'org.eclipse.jetty', name: 'jetty-client', version: '9.3.6.v20151106' + compile group: 'org.eclipse.jetty', name: 'jetty-client', version: versions.jetty compile group: 'org.jctools', name: 'jctools-core', version: '1.2' diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Message.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Message.java index 71d9c058ea..b7ac0f2105 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Message.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Message.java @@ -7,7 +7,7 @@ import pl.allegro.tech.hermes.api.Header; import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchema; import java.net.URI; import java.util.*; diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/filtering/avro/AvroPathPredicate.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/filtering/avro/AvroPathPredicate.java index 5d9777d9c0..166a364779 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/filtering/avro/AvroPathPredicate.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/filtering/avro/AvroPathPredicate.java @@ -2,11 +2,10 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; -import org.apache.commons.lang.StringUtils; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.filtering.FilteringException; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchema; import java.io.IOException; import java.util.Arrays; diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaMessageReceiverFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaMessageReceiverFactory.java index ade41a7f56..18adb19860 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaMessageReceiverFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaMessageReceiverFactory.java @@ -14,7 +14,7 @@ import pl.allegro.tech.hermes.consumers.consumer.rate.ConsumerRateLimiter; import pl.allegro.tech.hermes.consumers.consumer.receiver.MessageReceiver; import pl.allegro.tech.hermes.consumers.consumer.receiver.ReceiverFactory; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.tracker.consumers.Trackers; import javax.inject.Inject; diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaSingleThreadedMessageReceiver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaSingleThreadedMessageReceiver.java index 2b6fcd7f79..970927560b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaSingleThreadedMessageReceiver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaSingleThreadedMessageReceiver.java @@ -23,7 +23,7 @@ import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset; import pl.allegro.tech.hermes.consumers.consumer.receiver.MessageReceiver; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaRepository; import java.time.Clock; import java.util.Collection; @@ -134,7 +134,7 @@ private Message convertToMessage(ConsumerRecord record) { private UnwrappedMessageContent getUnwrappedMessageContent(ConsumerRecord message) { if (topic.getContentType() == ContentType.AVRO) { - return messageContentWrapper.unwrapAvro(message.value(), topic, schemaRepository); + return messageContentWrapper.unwrapAvro(message.value(), topic); } else if (topic.getContentType() == ContentType.JSON) { return messageContentWrapper.unwrapJson(message.value()); } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverterTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverterTest.java index 327379f862..fb83b090b8 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverterTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverterTest.java @@ -5,8 +5,8 @@ import org.mockito.runners.MockitoJUnitRunner; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.consumers.consumer.Message; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.SchemaVersion; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import java.io.IOException; diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/MessageBuilder.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/MessageBuilder.java index 63ff170b82..4625b16880 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/MessageBuilder.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/MessageBuilder.java @@ -6,8 +6,8 @@ import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; import pl.allegro.tech.hermes.consumers.consumer.Message; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; +import pl.allegro.tech.hermes.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.SchemaVersion; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; diff --git a/hermes-frontend/build.gradle b/hermes-frontend/build.gradle index 5c725399f7..99c2650154 100644 --- a/hermes-frontend/build.gradle +++ b/hermes-frontend/build.gradle @@ -2,6 +2,7 @@ dependencies { compile project(':hermes-common') compile project(':hermes-tracker') compile project(':hermes-metrics') + compile project(':hermes-schema') compile group: 'io.undertow', name: 'undertow-servlet', version: '1.3.23.Final' compile group: 'org.apache.kafka', name: 'kafka-clients', version: '0.8.2.0' diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/di/FrontendBinder.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/di/FrontendBinder.java index f5ec575edd..f186b8b7d3 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/di/FrontendBinder.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/di/FrontendBinder.java @@ -21,7 +21,6 @@ import pl.allegro.tech.hermes.frontend.server.HermesServer; import pl.allegro.tech.hermes.frontend.services.HealthCheckService; import pl.allegro.tech.hermes.frontend.validator.AvroTopicMessageValidator; -import pl.allegro.tech.hermes.frontend.validator.JsonTopicMessageValidator; import pl.allegro.tech.hermes.frontend.validator.MessageValidators; import pl.allegro.tech.hermes.frontend.validator.TopicMessageValidator; import pl.allegro.tech.hermes.frontend.validator.TopicMessageValidatorListFactory; @@ -59,7 +58,6 @@ protected void configure() { bindFactory(TopicsCacheFactory.class).to(TopicsCache.class).in(Singleton.class); bindSingleton(MessagePublisher.class); bindSingleton(MessageContentTypeEnforcer.class); - bindSingleton(JsonTopicMessageValidator.class); bindSingleton(AvroTopicMessageValidator.class); bindFactory(TopicMessageValidatorListFactory.class).in(Singleton.class).to(new TypeLiteral>() { }); diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/MessageContentTypeEnforcer.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/MessageContentTypeEnforcer.java index 7b68c73cf8..c05299c257 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/MessageContentTypeEnforcer.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/MessageContentTypeEnforcer.java @@ -1,7 +1,6 @@ package pl.allegro.tech.hermes.frontend.publishing; import org.apache.avro.Schema; -import pl.allegro.tech.hermes.frontend.publishing.avro.AvroMessage; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; import static javax.ws.rs.core.MediaType.APPLICATION_JSON; diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/PublishingServlet.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/PublishingServlet.java index 329885aa66..c7d2a2a560 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/PublishingServlet.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/PublishingServlet.java @@ -3,13 +3,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import pl.allegro.tech.hermes.api.ErrorDescription; import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.common.config.ConfigFactory; import pl.allegro.tech.hermes.common.config.Configs; import pl.allegro.tech.hermes.common.message.wrapper.UnsupportedContentTypeException; import pl.allegro.tech.hermes.common.metric.HermesMetrics; -import pl.allegro.tech.hermes.domain.topic.schema.CouldNotLoadSchemaException; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaMissingException; import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache; import pl.allegro.tech.hermes.frontend.listeners.BrokerListeners; import pl.allegro.tech.hermes.frontend.publishing.callbacks.*; @@ -18,6 +15,8 @@ import pl.allegro.tech.hermes.frontend.publishing.message.MessageState; import pl.allegro.tech.hermes.frontend.publishing.preview.MessagePreviewLog; import pl.allegro.tech.hermes.frontend.validator.InvalidMessageException; +import pl.allegro.tech.hermes.schema.CouldNotLoadSchemaException; +import pl.allegro.tech.hermes.common.message.wrapper.SchemaMissingException; import pl.allegro.tech.hermes.tracker.frontend.Trackers; import tech.allegro.schema.json2avro.converter.AvroConversionException; @@ -35,7 +34,6 @@ import static org.apache.commons.lang.StringUtils.strip; import static org.apache.commons.lang.StringUtils.substringAfterLast; import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_NOT_EXISTS; -import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; import static pl.allegro.tech.hermes.common.config.Configs.FRONTEND_MESSAGE_PREVIEW_ENABLED; public class PublishingServlet extends HttpServlet { diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/avro/AvroMessage.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/avro/AvroMessage.java index ca4dc6cb9e..253800aaa0 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/avro/AvroMessage.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/avro/AvroMessage.java @@ -2,8 +2,8 @@ import org.apache.avro.Schema; import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; import pl.allegro.tech.hermes.frontend.publishing.message.Message; +import pl.allegro.tech.hermes.schema.CompiledSchema; import java.util.Optional; diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/JsonMessage.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/JsonMessage.java index 8624da8087..228d68e8b7 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/JsonMessage.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/JsonMessage.java @@ -1,27 +1,17 @@ package pl.allegro.tech.hermes.frontend.publishing.message; -import com.github.fge.jsonschema.main.JsonSchema; import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; - -import java.util.Optional; public class JsonMessage implements Message { private final String id; private final byte[] data; private final long timestamp; - private final Optional> schema; public JsonMessage(String id, byte[] data, long timestamp) { - this(id, data, timestamp, Optional.empty()); - } - - public JsonMessage(String id, byte[] data, long timestamp, Optional> schema) { this.id = id; this.data = data; this.timestamp = timestamp; - this.schema = schema; } @Override @@ -44,12 +34,6 @@ public ContentType getContentType() { return ContentType.JSON; } - @SuppressWarnings("unchecked") - @Override - public Optional> getCompiledSchema() { - return schema.map(schema -> (CompiledSchema)schema); - } - public JsonMessage withDataReplaced(byte[] newData) { return new JsonMessage(id, newData, timestamp); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/Message.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/Message.java index dfedc16c1c..c358952114 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/Message.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/Message.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.frontend.publishing.message; import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchema; import java.util.Optional; @@ -14,7 +14,7 @@ public interface Message { ContentType getContentType(); - Optional> getCompiledSchema(); + default Optional> getCompiledSchema() { return Optional.empty(); } default T getSchema() { return this.getCompiledSchema().get().getSchema(); diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageFactory.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageFactory.java index dc73c0d925..57d5adefb5 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageFactory.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageFactory.java @@ -1,6 +1,5 @@ package pl.allegro.tech.hermes.frontend.publishing.message; -import com.github.fge.jsonschema.main.JsonSchema; import com.google.common.collect.ImmutableMap; import org.apache.avro.Schema; import org.slf4j.Logger; @@ -9,13 +8,13 @@ import pl.allegro.tech.hermes.common.http.MessageMetadataHeaders; import pl.allegro.tech.hermes.common.message.wrapper.MessageContentWrapper; import pl.allegro.tech.hermes.common.message.wrapper.UnsupportedContentTypeException; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.frontend.publishing.MessageContentTypeEnforcer; import pl.allegro.tech.hermes.frontend.publishing.avro.AvroMessage; import pl.allegro.tech.hermes.frontend.publishing.metadata.HeadersPropagator; import pl.allegro.tech.hermes.frontend.validator.MessageValidators; +import pl.allegro.tech.hermes.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.SchemaVersion; import tech.allegro.schema.json2avro.converter.AvroConversionException; import javax.inject.Inject; @@ -65,7 +64,7 @@ public Message create(HttpServletRequest request, Topic topic, String messageId, topic.getQualifiedName(), exception); } } - return createJsonMessage(request, topic, messageId, messageContent, timestamp); + return createJsonMessage(request, messageId, messageContent, timestamp); } case AVRO: return createAvroMessage(request, topic, messageId, messageContent, timestamp); @@ -73,10 +72,16 @@ public Message create(HttpServletRequest request, Topic topic, String messageId, } } + private JsonMessage createJsonMessage(HttpServletRequest request, String messageId, byte[] messageContent, long timestamp) { + JsonMessage message = new JsonMessage(messageId, messageContent, timestamp); + byte[] wrapped = messageContentWrapper.wrapJson(message.getData(), message.getId(), message.getTimestamp(), headersPropagator.extract(toHeadersMap(request))); + return message.withDataReplaced(wrapped); + } + private AvroMessage createAvroMessage(HttpServletRequest request, Topic topic, String messageId, byte[] messageContent, long timestamp) { CompiledSchema schema = extractSchemaVersion(request) .map(version -> schemaRepository.getAvroSchema(topic, version)) - .orElse(schemaRepository.getAvroSchema(topic)); + .orElse(schemaRepository.getLatestAvroSchema(topic)); AvroMessage message = new AvroMessage( messageId, @@ -90,19 +95,6 @@ private AvroMessage createAvroMessage(HttpServletRequest request, Topic topic, S return message.withDataReplaced(wrapped); } - private JsonMessage createJsonMessage(HttpServletRequest request, Topic topic, String messageId, byte[] messageContent, long timestamp) { - JsonMessage message = new JsonMessage(messageId, messageContent, timestamp); - if (topic.isValidationEnabled()) { - CompiledSchema schema = extractSchemaVersion(request) - .map(version -> schemaRepository.getJsonSchema(topic, version)) - .orElse(schemaRepository.getJsonSchema(topic)); - message = new JsonMessage(messageId, messageContent, timestamp, of(schema)); - } - validators.check(topic, message); - byte[] wrapped = messageContentWrapper.wrapJson(message.getData(), message.getId(), message.getTimestamp(), headersPropagator.extract(toHeadersMap(request))); - return message.withDataReplaced(wrapped); - } - private Optional extractSchemaVersion(HttpServletRequest request) { int version = request.getIntHeader(MessageMetadataHeaders.SCHEMA_VERSION.getName()); return version < 0 ? empty() : of(SchemaVersion.valueOf(version)); diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/JsonTopicMessageValidator.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/JsonTopicMessageValidator.java deleted file mode 100644 index d46b3169c1..0000000000 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/JsonTopicMessageValidator.java +++ /dev/null @@ -1,69 +0,0 @@ -package pl.allegro.tech.hermes.frontend.validator; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchema; -import com.google.common.base.Joiner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; -import pl.allegro.tech.hermes.frontend.publishing.message.Message; - -import javax.inject.Inject; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -public class JsonTopicMessageValidator implements TopicMessageValidator { - - private static final Logger logger = LoggerFactory.getLogger(JsonTopicMessageValidator.class); - private static final Charset UTF_8 = Charset.forName("UTF-8"); - private final ObjectMapper objectMapper; - - @Inject - public JsonTopicMessageValidator(ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - } - - @Override - public void check(Message message, Topic topic) { - if (ContentType.JSON != topic.getContentType() || !topic.isValidationEnabled()) { - return; - } - - List errors = validate(message.getCompiledSchema(), message.getData()); - - if (!errors.isEmpty()) { - if (topic.isValidationDryRunEnabled()) { - logger.info("Message incompatible with JSON schema for topic {}, errors: {}, message body: {}", - topic.getQualifiedName(), Joiner.on(";").join(errors), new String(message.getData(), UTF_8)); - } else { - throw new InvalidMessageException("Message incompatible with JSON schema", errors); - } - } - } - - private List validate(Optional> jsonSchema, byte[] message) { - List errors = new ArrayList<>(); - - try { - JsonNode messageNode = objectMapper.readTree(message); - - ProcessingReport report = jsonSchema.get().getSchema().validateUnchecked(messageNode); - - report.forEach(processingMessage -> errors.add(processingMessage.getMessage())); - - } catch (Exception e) { - logger.warn("Error while deserializing message: " + new String(message), e); - errors.add("Problem with message deserialization. Is this correct JSON format?"); - } - - return errors; - } - -} diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/TopicMessageValidatorListFactory.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/TopicMessageValidatorListFactory.java index 34d2e7099d..0ae904a624 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/TopicMessageValidatorListFactory.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/TopicMessageValidatorListFactory.java @@ -9,17 +9,15 @@ public class TopicMessageValidatorListFactory implements Factory> { private final AvroTopicMessageValidator avroTopicMessageValidator; - private final JsonTopicMessageValidator jsonTopicMessageValidator; @Inject - public TopicMessageValidatorListFactory(AvroTopicMessageValidator avroTopicMessageValidator, JsonTopicMessageValidator jsonTopicMessageValidator) { + public TopicMessageValidatorListFactory(AvroTopicMessageValidator avroTopicMessageValidator) { this.avroTopicMessageValidator = avroTopicMessageValidator; - this.jsonTopicMessageValidator = jsonTopicMessageValidator; } @Override public List provide() { - return ImmutableList.of(avroTopicMessageValidator, jsonTopicMessageValidator); + return ImmutableList.of(avroTopicMessageValidator); } @Override diff --git a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/publishing/MessageContentTypeEnforcerTest.java b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/publishing/MessageContentTypeEnforcerTest.java index 9f437f1c1d..655475db65 100644 --- a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/publishing/MessageContentTypeEnforcerTest.java +++ b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/publishing/MessageContentTypeEnforcerTest.java @@ -2,10 +2,8 @@ import org.apache.avro.Schema; import org.junit.Test; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; -import pl.allegro.tech.hermes.frontend.publishing.avro.AvroMessage; -import pl.allegro.tech.hermes.frontend.publishing.message.Message; +import pl.allegro.tech.hermes.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.SchemaVersion; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import java.io.IOException; diff --git a/hermes-management/build.gradle b/hermes-management/build.gradle index 60b533bbf1..ed934b95da 100644 --- a/hermes-management/build.gradle +++ b/hermes-management/build.gradle @@ -4,6 +4,7 @@ dependencies { compile project(':hermes-api') compile project(':hermes-common') compile project(':hermes-tracker') + compile project(':hermes-schema') compile group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: versions.spring compile group: 'org.springframework.boot', name: 'spring-boot-starter-jersey', version: versions.spring diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SchemaEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SchemaEndpoint.java index dd4bd28e38..5e881d163b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SchemaEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SchemaEndpoint.java @@ -3,10 +3,10 @@ import com.wordnik.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; import pl.allegro.tech.hermes.management.api.auth.Roles; import pl.allegro.tech.hermes.management.domain.topic.TopicService; import pl.allegro.tech.hermes.management.domain.topic.schema.SchemaSourceService; +import pl.allegro.tech.hermes.schema.SchemaVersion; import javax.annotation.security.RolesAllowed; import javax.ws.rs.*; @@ -56,7 +56,7 @@ public Response get(@PathParam("topicName") String qualifiedTopicName, @PathPara public Response save(@PathParam("topicName") String qualifiedTopicName, @DefaultValue("true") @QueryParam(value = "validate") boolean validate, String schema) { - schemaSourceService.saveSchemaSource(qualifiedTopicName, schema, validate); + schemaSourceService.registerSchemaSource(qualifiedTopicName, schema, validate); notifyFrontendSchemaChanged(qualifiedTopicName); return Response.status(Response.Status.CREATED).build(); } @@ -69,7 +69,7 @@ private void notifyFrontendSchemaChanged(String qualifiedTopicName) { @RolesAllowed({Roles.GROUP_OWNER, Roles.ADMIN}) @ApiOperation(value = "Delete schema", httpMethod = HttpMethod.DELETE) public Response delete(@PathParam("topicName") String qualifiedTopicName) { - schemaSourceService.deleteSchemaSource(qualifiedTopicName); + schemaSourceService.deleteAllSchemaSources(qualifiedTopicName); return Response.status(Response.Status.OK).build(); } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/RuntimeExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/RuntimeExceptionMapper.java index d6c6c81220..e18419e1b1 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/RuntimeExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/RuntimeExceptionMapper.java @@ -1,8 +1,9 @@ package pl.allegro.tech.hermes.management.api.mappers; -import org.boon.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.api.ErrorCode; +import pl.allegro.tech.hermes.api.ErrorDescription; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -20,9 +21,7 @@ public Response toResponse(RuntimeException exception) { return Response .status(Response.Status.INTERNAL_SERVER_ERROR) .type(MediaType.APPLICATION_JSON_TYPE) - .entity(Maps.map( - "message", exception.getMessage() - )) + .entity(new ErrorDescription(exception.getMessage(), ErrorCode.OTHER)) .build(); } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/SchemaExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/SchemaExceptionMapper.java new file mode 100644 index 0000000000..0e645e043c --- /dev/null +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/SchemaExceptionMapper.java @@ -0,0 +1,22 @@ +package pl.allegro.tech.hermes.management.api.mappers; + +import pl.allegro.tech.hermes.api.ErrorDescription; +import pl.allegro.tech.hermes.schema.SchemaException; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + +@Provider +public class SchemaExceptionMapper implements ExceptionMapper { + + @Override + public Response toResponse(SchemaException exception) { + return Response + .status(exception.getCode().getHttpCode()) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity(new ErrorDescription(exception.getMessage(), exception.getCode())) + .build(); + } +} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageConfiguration.java index 07f0179dca..3297c5788a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageConfiguration.java @@ -8,6 +8,7 @@ import pl.allegro.tech.hermes.common.message.wrapper.AvroMessageContentWrapper; import pl.allegro.tech.hermes.common.message.wrapper.JsonMessageContentWrapper; import pl.allegro.tech.hermes.common.message.wrapper.MessageContentWrapper; +import pl.allegro.tech.hermes.schema.SchemaRepository; import java.time.Clock; @@ -24,9 +25,12 @@ public class MessageConfiguration { @Autowired ObjectMapper objectMapper; + @Autowired + SchemaRepository schemaRepository; + @Bean MessageContentWrapper messageContentWrapper() { - return new MessageContentWrapper(jsonMessageContentWrapper(), new AvroMessageContentWrapper(clock)); + return new MessageContentWrapper(jsonMessageContentWrapper(), new AvroMessageContentWrapper(clock), schemaRepository); } private JsonMessageContentWrapper jsonMessageContentWrapper() { diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryConfiguration.java index 83fe92f6dd..abd06ebed4 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryConfiguration.java @@ -1,9 +1,6 @@ package pl.allegro.tech.hermes.management.config; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.fge.jsonschema.main.JsonSchema; import org.apache.avro.Schema; -import org.apache.curator.framework.CuratorFramework; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -11,75 +8,52 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchemaRepository; -import pl.allegro.tech.hermes.domain.topic.schema.DirectCompiledSchemaRepository; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaCompilersFactory; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaSourceProvider; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersionsRepository; -import pl.allegro.tech.hermes.domain.topic.schema.SimpleSchemaVersionsRepository; -import pl.allegro.tech.hermes.infrastructure.schema.repo.JerseySchemaRepoClient; -import pl.allegro.tech.hermes.infrastructure.schema.repo.SchemaRepoClient; -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; import pl.allegro.tech.hermes.management.domain.topic.TopicService; -import pl.allegro.tech.hermes.management.domain.topic.schema.SchemaSourceRepository; -import pl.allegro.tech.hermes.management.infrastructure.schema.SchemaRepoSchemaSourceRepository; -import pl.allegro.tech.hermes.management.infrastructure.schema.ZookeeperSchemaSourceRepository; +import pl.allegro.tech.hermes.schema.CompiledSchemaRepository; +import pl.allegro.tech.hermes.schema.confluent.SchemaRegistrySchemaSourceClient; +import pl.allegro.tech.hermes.schema.DirectCompiledSchemaRepository; +import pl.allegro.tech.hermes.schema.DirectSchemaVersionsRepository; +import pl.allegro.tech.hermes.schema.SchemaCompilersFactory; +import pl.allegro.tech.hermes.schema.schemarepo.SchemaRepoSchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaVersionsRepository; import javax.ws.rs.client.Client; import java.net.URI; @Configuration -@EnableConfigurationProperties({SchemaRepositoryProperties.class, SchemaCacheProperties.class}) +@EnableConfigurationProperties({SchemaRepositoryProperties.class}) public class SchemaRepositoryConfiguration { - @Autowired - SchemaCacheProperties schemaCacheProperties; - @Autowired @Lazy TopicService topicService; - @Autowired - @Lazy - private CuratorFramework storageZookeeper; - - @Autowired - @Lazy - private ZookeeperPaths zookeeperPaths; - @Autowired private SchemaRepositoryProperties schemaRepositoryProperties; @Bean - @ConditionalOnMissingBean(SchemaSourceRepository.class) - @ConditionalOnProperty(value = "schema.repository.type", havingValue = "zookeeper", matchIfMissing = true) - public SchemaSourceRepository zookeeperSchemaSourceRepository() { - return new ZookeeperSchemaSourceRepository(storageZookeeper, zookeeperPaths); - } - - @Bean - @ConditionalOnMissingBean(SchemaSourceRepository.class) + @ConditionalOnMissingBean(SchemaSourceClient.class) @ConditionalOnProperty(value = "schema.repository.type", havingValue = "schema_repo") - public SchemaSourceRepository schemaRepoSchemaSourceRepository(Client jerseyClient) { - SchemaRepoClient client = new JerseySchemaRepoClient(jerseyClient, URI.create(schemaRepositoryProperties.getServerUrl())); - return new SchemaRepoSchemaSourceRepository(client); + public SchemaSourceClient schemaRepoSchemaSourceClient(Client httpClient) { + return new SchemaRepoSchemaSourceClient(httpClient, URI.create(schemaRepositoryProperties.getServerUrl())); } @Bean - public SchemaSourceProvider schemaSourceProvider(SchemaSourceRepository schemaSourceRepository) { - return schemaSourceRepository; + @ConditionalOnMissingBean(SchemaSourceClient.class) + @ConditionalOnProperty(value = "schema.repository.type", havingValue = "schema_registry") + public SchemaSourceClient confluentSchemaRegistrySchemaSourceClient(Client httpClient) { + return new SchemaRegistrySchemaSourceClient(httpClient, URI.create(schemaRepositoryProperties.getServerUrl())); } @Bean - public SchemaRepository aggregateSchemaRepository(SchemaSourceProvider schemaSourceProvider, ObjectMapper objectMapper) { - SchemaVersionsRepository versionsRepository = new SimpleSchemaVersionsRepository(schemaSourceProvider); + public SchemaRepository aggregateSchemaRepository(SchemaSourceClient schemaSourceClient) { + SchemaVersionsRepository versionsRepository = new DirectSchemaVersionsRepository(schemaSourceClient); CompiledSchemaRepository avroSchemaRepository = new DirectCompiledSchemaRepository<>( - schemaSourceProvider, SchemaCompilersFactory.avroSchemaCompiler()); - CompiledSchemaRepository jsonSchemaRepository = new DirectCompiledSchemaRepository<>( - schemaSourceProvider, SchemaCompilersFactory.jsonSchemaCompiler(objectMapper)); + schemaSourceClient, SchemaCompilersFactory.avroSchemaCompiler()); - return new SchemaRepository(versionsRepository, avroSchemaRepository, jsonSchemaRepository); + return new SchemaRepository(versionsRepository, avroSchemaRepository); } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/TopicProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/TopicProperties.java index 5f29914da6..26a505c51b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/TopicProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/TopicProperties.java @@ -11,6 +11,8 @@ public class TopicProperties { private boolean allowRemoval = false; + private boolean removeSchema = false; + private boolean uncleanLeaderElectionEnabled = false; public int getReplicationFactor() { @@ -37,6 +39,14 @@ public void setAllowRemoval(boolean allowRemoval) { this.allowRemoval = allowRemoval; } + public boolean isRemoveSchema() { + return removeSchema; + } + + public void setRemoveSchema(boolean removeSchema) { + this.removeSchema = removeSchema; + } + public boolean isUncleanLeaderElectionEnabled() { return uncleanLeaderElectionEnabled; } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaConfiguration.java index 69e5128508..03fd1be91b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaConfiguration.java @@ -13,7 +13,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaRepository; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; import pl.allegro.tech.hermes.common.admin.AdminTool; import pl.allegro.tech.hermes.common.broker.BrokerStorage; diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/AvroSchemaRemovalDisabledException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/AvroSchemaRemovalDisabledException.java deleted file mode 100644 index 281d998193..0000000000 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/AvroSchemaRemovalDisabledException.java +++ /dev/null @@ -1,16 +0,0 @@ -package pl.allegro.tech.hermes.management.domain.topic.schema; - -import pl.allegro.tech.hermes.api.ErrorCode; -import pl.allegro.tech.hermes.common.exception.HermesException; - -public class AvroSchemaRemovalDisabledException extends HermesException { - - public AvroSchemaRemovalDisabledException(String message) { - super(message); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.AVRO_SCHEMA_REMOVAL_DISABLED; - } -} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/CouldNotDeleteSchemaException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/CouldNotDeleteSchemaException.java deleted file mode 100644 index 72fef830e2..0000000000 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/CouldNotDeleteSchemaException.java +++ /dev/null @@ -1,16 +0,0 @@ -package pl.allegro.tech.hermes.management.domain.topic.schema; - -import pl.allegro.tech.hermes.api.ErrorCode; -import pl.allegro.tech.hermes.common.exception.HermesException; - -public class CouldNotDeleteSchemaException extends HermesException { - - public CouldNotDeleteSchemaException(String message, Throwable cause) { - super(message, cause); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_COULD_NOT_BE_DELETED; - } -} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/CouldNotSaveSchemaException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaRemovalDisabledException.java similarity index 52% rename from hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/CouldNotSaveSchemaException.java rename to hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaRemovalDisabledException.java index 39cd0ea3a3..e6f1b8aaff 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/CouldNotSaveSchemaException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaRemovalDisabledException.java @@ -3,14 +3,14 @@ import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.common.exception.HermesException; -public class CouldNotSaveSchemaException extends HermesException { +public class SchemaRemovalDisabledException extends HermesException { - public CouldNotSaveSchemaException(String message, Throwable t) { - super(message, t); + public SchemaRemovalDisabledException() { + super("Removing schema is disabled."); } @Override public ErrorCode getCode() { - return ErrorCode.SCHEMA_COULD_NOT_BE_SAVED; + return ErrorCode.OPERATION_DISABLED; } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaSourceRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaSourceRepository.java deleted file mode 100644 index a1d6c2ba25..0000000000 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaSourceRepository.java +++ /dev/null @@ -1,18 +0,0 @@ -package pl.allegro.tech.hermes.management.domain.topic.schema; - -import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaSourceProvider; - -public interface SchemaSourceRepository extends SchemaSourceProvider { - - void save(SchemaSource schemaSource, Topic topic); - default void save(SchemaSource schemaSource, Topic topic, int version) { - throw new UnsupportedOperationException("sry"); - } - - void delete(Topic topic); - default void delete(Topic topic, int version) { - throw new UnsupportedOperationException("sry"); - } -} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaSourceService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaSourceService.java index 03ff4a4ca6..c39f23142d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaSourceService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaSourceService.java @@ -4,58 +4,63 @@ import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.SchemaSource; import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.management.config.TopicProperties; import pl.allegro.tech.hermes.management.domain.topic.TopicService; import pl.allegro.tech.hermes.management.infrastructure.schema.validator.SchemaValidator; import pl.allegro.tech.hermes.management.infrastructure.schema.validator.SchemaValidatorProvider; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaVersion; import java.util.Optional; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; @Component public class SchemaSourceService { private final TopicService topicService; - private final SchemaSourceRepository schemaSourceRepository; + private final SchemaSourceClient schemaSourceClient; private final SchemaValidatorProvider validatorProvider; + private final TopicProperties topicProperties; @Autowired - public SchemaSourceService(TopicService topicService, SchemaSourceRepository schemaSourceRepository, SchemaValidatorProvider validatorProvider) { + public SchemaSourceService(TopicService topicService, SchemaSourceClient schemaSourceClient, + SchemaValidatorProvider validatorProvider, TopicProperties topicProperties) { this.topicService = topicService; - this.schemaSourceRepository = schemaSourceRepository; + this.schemaSourceClient = schemaSourceClient; this.validatorProvider = validatorProvider; + this.topicProperties = topicProperties; } public Optional getSchemaSource(String qualifiedTopicName) { Topic topic = findTopic(qualifiedTopicName); - return schemaSourceRepository.get(topic); + return schemaSourceClient.getLatestSchemaSource(topic.getName()); } - public void saveSchemaSource(String qualifiedTopicName, String schema, boolean validate) { + public void registerSchemaSource(String qualifiedTopicName, String schema, boolean validate) { Topic topic = findTopic(qualifiedTopicName); if (validate) { SchemaValidator validator = validatorProvider.provide(topic.getContentType()); validator.check(schema); } - schemaSourceRepository.save(SchemaSource.valueOf(schema), topic); + schemaSourceClient.registerSchemaSource(topic.getName(), SchemaSource.valueOf(schema)); } - public void deleteSchemaSource(String qualifiedTopicName) { + public Optional getSchemaSource(String qualifiedTopicName, SchemaVersion version) { Topic topic = findTopic(qualifiedTopicName); - if (topic.getContentType() == AVRO) { - throw new AvroSchemaRemovalDisabledException("Topic " + qualifiedTopicName + " has Avro content-type, schema removal is disabled"); + return schemaSourceClient.getSchemaSource(topic.getName(), version); + } + + public void deleteAllSchemaSources(String qualifiedTopicName) { + if (!topicProperties.isRemoveSchema()) { + throw new SchemaRemovalDisabledException(); } - schemaSourceRepository.delete(topic); + TopicName topicName = TopicName.fromQualifiedName(qualifiedTopicName); + schemaSourceClient.deleteAllSchemaSources(topicName); } private Topic findTopic(String qualifiedTopicName) { return topicService.getTopicDetails(fromQualifiedName(qualifiedTopicName)); } - - public Optional getSchemaSource(String qualifiedTopicName, SchemaVersion version) { - Topic topic = findTopic(qualifiedTopicName); - return schemaSourceRepository.get(topic, version); - } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidator.java index eb7f0e7e47..c7f5ead4a8 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidator.java @@ -4,8 +4,8 @@ import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; -import pl.allegro.tech.hermes.domain.topic.schema.CouldNotLoadSchemaException; +import pl.allegro.tech.hermes.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.CouldNotLoadSchemaException; @Component public class TopicValidator { @@ -30,7 +30,7 @@ public void ensureUpdatedTopicIsValid(Topic updated, Topic previous) { } try { - schemaRepository.getAvroSchema(updated); + schemaRepository.getLatestAvroSchema(updated); } catch (CouldNotLoadSchemaException e) { throw new TopicValidationException("Avro schema not available, migration not permitted", e); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaSingleMessageReader.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaSingleMessageReader.java index 9549f71665..e5e170adae 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaSingleMessageReader.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaSingleMessageReader.java @@ -3,10 +3,10 @@ import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.kafka.KafkaTopic; -import pl.allegro.tech.hermes.common.message.serialization.SchemaAwarePayload; -import pl.allegro.tech.hermes.common.message.serialization.SchemaAwareSerDe; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; +import pl.allegro.tech.hermes.common.message.wrapper.SchemaAwarePayload; +import pl.allegro.tech.hermes.common.message.wrapper.SchemaAwareSerDe; import pl.allegro.tech.hermes.management.domain.topic.SingleMessageReader; +import pl.allegro.tech.hermes.schema.SchemaRepository; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; import java.nio.charset.Charset; @@ -39,7 +39,7 @@ private byte[] convertAvroToJson(Topic topic, byte[] bytes) { return converter.convertToJson(payload.getPayload(), schemaRepository.getAvroSchema(topic, payload.getSchemaVersion()).getSchema()); } - return converter.convertToJson(bytes, schemaRepository.getAvroSchema(topic).getSchema()); + return converter.convertToJson(bytes, schemaRepository.getLatestAvroSchema(topic).getSchema()); } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaRetransmissionService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaRetransmissionService.java index e43ac66dac..78b252b2be 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaRetransmissionService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaRetransmissionService.java @@ -15,7 +15,7 @@ import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; import pl.allegro.tech.hermes.common.kafka.offset.SubscriptionOffsetChangeIndicator; import pl.allegro.tech.hermes.common.message.wrapper.MessageContentWrapper; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.management.domain.message.RetransmissionService; import pl.allegro.tech.hermes.management.infrastructure.kafka.service.KafkaRawMessageReader; diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaTimestampExtractor.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaTimestampExtractor.java index 1f6ba5f56d..3c45f093e7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaTimestampExtractor.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaTimestampExtractor.java @@ -5,7 +5,7 @@ import pl.allegro.tech.hermes.common.message.wrapper.MessageContentWrapper; import pl.allegro.tech.hermes.common.message.wrapper.UnsupportedContentTypeException; import pl.allegro.tech.hermes.common.message.wrapper.UnwrappedMessageContent; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository; +import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.management.infrastructure.kafka.service.KafkaRawMessageReader; class KafkaTimestampExtractor { @@ -36,7 +36,7 @@ public long extract(Long offset) { private UnwrappedMessageContent unwrapContent(byte[] message) { switch (kafkaTopic.contentType()) { case AVRO: - return messageContentWrapper.unwrapAvro(message, topic, schemaRepository); + return messageContentWrapper.unwrapAvro(message, topic); case JSON: return messageContentWrapper.unwrapJson(message); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/SchemaRepoSchemaSourceRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/SchemaRepoSchemaSourceRepository.java deleted file mode 100644 index bd1ebe247d..0000000000 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/SchemaRepoSchemaSourceRepository.java +++ /dev/null @@ -1,33 +0,0 @@ -package pl.allegro.tech.hermes.management.infrastructure.schema; - -import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.infrastructure.schema.repo.SchemaRepoClient; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepoSchemaSourceProvider; -import pl.allegro.tech.hermes.management.domain.topic.schema.SchemaSourceRepository; - -public class SchemaRepoSchemaSourceRepository extends SchemaRepoSchemaSourceProvider implements SchemaSourceRepository { - - public SchemaRepoSchemaSourceRepository(SchemaRepoClient schemaRepoClient) { - super(schemaRepoClient); - } - - @Override - public void save(SchemaSource schemaSource, Topic topic) { - String subjectName = registerSubjectIfNotRegistered(topic); - schemaRepoClient.registerSchema(subjectName, schemaSource.value()); - } - - private String registerSubjectIfNotRegistered(Topic topic) { - String subjectName = topic.getQualifiedName(); - if (!schemaRepoClient.isSubjectRegistered(subjectName)) { - schemaRepoClient.registerSubject(subjectName); - } - return subjectName; - } - - @Override - public void delete(Topic topic) { - throw new UnsupportedOperationException("Deleting schemas is not supported by this repository"); - } -} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/ZookeeperSchemaSourceRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/ZookeeperSchemaSourceRepository.java deleted file mode 100644 index 1141ae2078..0000000000 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/ZookeeperSchemaSourceRepository.java +++ /dev/null @@ -1,41 +0,0 @@ -package pl.allegro.tech.hermes.management.infrastructure.schema; - -import org.apache.curator.framework.CuratorFramework; -import pl.allegro.tech.hermes.api.SchemaSource; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.domain.topic.schema.ZookeeperSchemaSourceProvider; -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import pl.allegro.tech.hermes.management.domain.topic.schema.CouldNotDeleteSchemaException; -import pl.allegro.tech.hermes.management.domain.topic.schema.CouldNotSaveSchemaException; -import pl.allegro.tech.hermes.management.domain.topic.schema.SchemaSourceRepository; - -public class ZookeeperSchemaSourceRepository extends ZookeeperSchemaSourceProvider implements SchemaSourceRepository { - - public ZookeeperSchemaSourceRepository(CuratorFramework curatorFramework, ZookeeperPaths zkPaths) { - super(curatorFramework, zkPaths); - } - - @Override - public void save(SchemaSource schemaSource, Topic topic) { - try { - String schemaPath = zkPaths.topicPath(topic.getName(), SCHEMA_SUFFIX); - byte[] schema = schemaSource.value().getBytes(); - if (curatorFramework.checkExists().forPath(schemaPath) == null) { - curatorFramework.create().creatingParentsIfNeeded().forPath(schemaPath, schema); - } else { - curatorFramework.setData().forPath(schemaPath, schema); - } - } catch (Exception e) { - throw new CouldNotSaveSchemaException("Could not store in zookeeper schema for topic " + topic.getQualifiedName(), e); - } - } - - @Override - public void delete(Topic topic) { - try { - curatorFramework.delete().forPath(zkPaths.topicPath(topic.getName(), SCHEMA_SUFFIX)); - } catch (Exception e) { - throw new CouldNotDeleteSchemaException("Could not delete from zookeeper schema for topic " + topic.getQualifiedName(), e); - } - } -} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidator.java index 5aba91cfd0..95437929c2 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidator.java @@ -3,7 +3,6 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaParseException; import org.springframework.stereotype.Component; -import pl.allegro.tech.hermes.common.exception.InvalidSchemaException; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/InvalidSchemaException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/InvalidSchemaException.java new file mode 100644 index 0000000000..068257519b --- /dev/null +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/InvalidSchemaException.java @@ -0,0 +1,16 @@ +package pl.allegro.tech.hermes.management.infrastructure.schema.validator; + +import pl.allegro.tech.hermes.api.ErrorCode; +import pl.allegro.tech.hermes.common.exception.HermesException; + +public class InvalidSchemaException extends HermesException { + + InvalidSchemaException(Throwable cause) { + super("Error while trying to validate schema", cause); + } + + @Override + public ErrorCode getCode() { + return ErrorCode.FORMAT_ERROR; + } +} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/JsonSchemaValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/JsonSchemaValidator.java deleted file mode 100644 index 22814ec856..0000000000 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/JsonSchemaValidator.java +++ /dev/null @@ -1,44 +0,0 @@ -package pl.allegro.tech.hermes.management.infrastructure.schema.validator; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import pl.allegro.tech.hermes.common.exception.InvalidSchemaException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.StreamSupport; - -@Component -public class JsonSchemaValidator implements SchemaValidator { - - private final JsonSchemaFactory jsonSchemaFactory = JsonSchemaFactory.byDefault(); - private final ObjectMapper objectMapper; - - @Autowired - public JsonSchemaValidator(ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - } - - @Override - public void check(String schema) throws InvalidSchemaException { - List errors = new ArrayList<>(); - try { - JsonNode schemaNode = objectMapper.readTree(schema); - ProcessingReport report = jsonSchemaFactory.getSyntaxValidator().validateSchema(schemaNode); - - StreamSupport.stream(report.spliterator(), false).forEach(e -> errors.add(e.getMessage())); - } catch (IOException e) { - throw new InvalidSchemaException(e); - } - - if (!errors.isEmpty()) { - throw new InvalidSchemaException(errors); - } - } - -} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidator.java index 5a0b754014..0bec062cb1 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidator.java @@ -1,7 +1,5 @@ package pl.allegro.tech.hermes.management.infrastructure.schema.validator; -import pl.allegro.tech.hermes.common.exception.InvalidSchemaException; - public interface SchemaValidator { void check(String schema) throws InvalidSchemaException; } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidatorProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidatorProvider.java index d6bd7959e2..a61755e4ba 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidatorProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidatorProvider.java @@ -13,9 +13,8 @@ public class SchemaValidatorProvider { private final Map validators = new EnumMap<>(ContentType.class); @Autowired - public SchemaValidatorProvider(JsonSchemaValidator jsonSchemaValidator, AvroSchemaValidator avroSchemaValidator) { + public SchemaValidatorProvider(AvroSchemaValidator avroSchemaValidator) { validators.put(ContentType.AVRO, avroSchemaValidator); - validators.put(ContentType.JSON, jsonSchemaValidator); } public SchemaValidator provide(ContentType contentType) { diff --git a/hermes-management/src/main/resources/application.yaml b/hermes-management/src/main/resources/application.yaml index 5ecaad5c79..a4daf40163 100644 --- a/hermes-management/src/main/resources/application.yaml +++ b/hermes-management/src/main/resources/application.yaml @@ -26,6 +26,6 @@ management: connect-timeout: 2000 read-timeout: 2000 -schema.repository.type: zookeeper +schema.repository.type: schema_registry audit.enabled: false \ No newline at end of file diff --git a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidatorTest.groovy b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidatorTest.groovy index 8f37a2ae67..44f5a57465 100644 --- a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidatorTest.groovy +++ b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidatorTest.groovy @@ -3,10 +3,10 @@ package pl.allegro.tech.hermes.management.domain.topic.validator import org.apache.avro.Schema import pl.allegro.tech.hermes.api.ContentType import pl.allegro.tech.hermes.api.Topic -import pl.allegro.tech.hermes.domain.topic.schema.SchemaRepository -import pl.allegro.tech.hermes.domain.topic.schema.CouldNotLoadSchemaException -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion +import pl.allegro.tech.hermes.schema.CompiledSchema +import pl.allegro.tech.hermes.schema.CouldNotLoadSchemaException +import pl.allegro.tech.hermes.schema.SchemaRepository +import pl.allegro.tech.hermes.schema.SchemaVersion import pl.allegro.tech.hermes.test.helper.avro.AvroUser import spock.lang.Specification @@ -81,7 +81,7 @@ class TopicValidatorTest extends Specification { given: def jsonTopic = topic('group.topic').withContentType(ContentType.JSON).build() def migratedTopic = topic('group.topic').withContentType(ContentType.AVRO).migratedFromJsonType().build() - schemaRepository.getAvroSchema(migratedTopic) >> { throw new CouldNotLoadSchemaException("", new RuntimeException()) } + schemaRepository.getLatestAvroSchema(migratedTopic) >> { throw new CouldNotLoadSchemaException("", new RuntimeException()) } when: topicValidator.ensureUpdatedTopicIsValid(migratedTopic, jsonTopic) @@ -94,7 +94,7 @@ class TopicValidatorTest extends Specification { given: def jsonTopic = topic('group.topic').withContentType(ContentType.JSON).build() def migratedTopic = topic('group.topic').withContentType(ContentType.AVRO).migratedFromJsonType().build() - schemaRepository.getAvroSchema(migratedTopic) >> new CompiledSchema(new AvroUser().schema, SchemaVersion.valueOf(1)) + schemaRepository.getLatestAvroSchema(migratedTopic) >> new CompiledSchema(new AvroUser().schema, SchemaVersion.valueOf(1)) when: topicValidator.ensureUpdatedTopicIsValid(migratedTopic, jsonTopic) diff --git a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/ZookeeperSchemaSourceRepositorySpec.groovy b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/ZookeeperSchemaSourceRepositorySpec.groovy deleted file mode 100644 index 38312e3cba..0000000000 --- a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/ZookeeperSchemaSourceRepositorySpec.groovy +++ /dev/null @@ -1,63 +0,0 @@ -package pl.allegro.tech.hermes.management.infrastructure.schema - -import org.apache.curator.framework.CuratorFramework -import org.apache.curator.framework.CuratorFrameworkFactory -import org.apache.curator.retry.ExponentialBackoffRetry -import org.apache.curator.test.TestingServer -import pl.allegro.tech.hermes.api.SchemaSource -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths -import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperWaiter -import spock.lang.Shared -import spock.lang.Specification - -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic - -class ZookeeperSchemaSourceRepositorySpec extends Specification { - - @Shared - private TestingServer zookeeperServer = new TestingServer(6789) - - @Shared - private CuratorFramework zookeeperClient - - @Shared - private ZookeeperWaiter wait - - @Shared - private ZookeeperSchemaSourceRepository repository - - void setupSpec() { - zookeeperClient = CuratorFrameworkFactory.builder() - .connectString(zookeeperServer.getConnectString()) - .retryPolicy(new ExponentialBackoffRetry(1000, 3)) - .build() - zookeeperClient.start() - repository = new ZookeeperSchemaSourceRepository(zookeeperClient, new ZookeeperPaths("/test")) - wait = new ZookeeperWaiter(zookeeperClient) - wait.untilZookeeperClientStarted() - } - - def "should save schema in zookeeper"() { - given: - def topic = topic("org.hermes.schema", "save").build() - - when: - repository.save(SchemaSource.valueOf("schema"), topic) - - then: - zookeeperClient.getData().forPath("/test/groups/org.hermes.schema/topics/save/schema") == "schema".bytes - } - - def "should delete schema from zookeeper"() { - given: - def topic = topic("org.hermes.schema", "delete").build() - zookeeperClient.create().creatingParentsIfNeeded().forPath("/test/groups/org.hermes.schema/topics/delete/schema", "toBeRemoved".bytes) - - when: - repository.delete(topic) - - then: - !zookeeperClient.checkExists().forPath("/test/groups/org.hermes.schema/topics/delete/schema") - } - -} diff --git a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidatorTest.groovy b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidatorTest.groovy index 8f9addb2f8..b6de50f613 100644 --- a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidatorTest.groovy +++ b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidatorTest.groovy @@ -2,7 +2,6 @@ package pl.allegro.tech.hermes.management.infrastructure.schema.validator import groovy.json.JsonBuilder import org.apache.commons.io.IOUtils -import pl.allegro.tech.hermes.common.exception.InvalidSchemaException import spock.lang.Specification class AvroSchemaValidatorTest extends Specification { diff --git a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/validator/JsonSchemaValidatorTest.groovy b/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/validator/JsonSchemaValidatorTest.groovy deleted file mode 100644 index fe6277e01e..0000000000 --- a/hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/infrastructure/schema/validator/JsonSchemaValidatorTest.groovy +++ /dev/null @@ -1,45 +0,0 @@ -package pl.allegro.tech.hermes.management.infrastructure.schema.validator - -import com.fasterxml.jackson.databind.ObjectMapper -import groovy.json.JsonBuilder -import pl.allegro.tech.hermes.common.exception.InvalidSchemaException -import spock.lang.Specification - -class JsonSchemaValidatorTest extends Specification { - - private JsonSchemaValidator jsonSchemaValidator = new JsonSchemaValidator(new ObjectMapper()); - - def "should accept valid schema"() { - given: - def json = new JsonBuilder() - json { - type "boolean" - } - - expect: - jsonSchemaValidator.check(json.toString()) - } - - def "should throw exception for invalid schema"() { - given: - def json = new JsonBuilder() - json { - type "error" - } - - when: - jsonSchemaValidator.check(json.toString()) - - then: - thrown(InvalidSchemaException) - } - - - def "should throw exception for invalid json"() { - when: - jsonSchemaValidator.check('{') - - then: - thrown(InvalidSchemaException) - } -} diff --git a/hermes-schema/build.gradle b/hermes-schema/build.gradle new file mode 100644 index 0000000000..aa25a20776 --- /dev/null +++ b/hermes-schema/build.gradle @@ -0,0 +1,11 @@ +dependencies { + compile project(':hermes-api') + + compile group: 'org.apache.avro', name: 'avro', version: '1.7.7' + compile group: 'com.google.guava', name: 'guava', version: versions.guava + + testCompile project(path: ":hermes-test-helper") + + testCompile group: 'org.spockframework', name: 'spock-core', version: versions.spock + +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CachedCompiledSchemaRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepository.java similarity index 97% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CachedCompiledSchemaRepository.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepository.java index 27dec4cc32..3b7b7c43bd 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CachedCompiledSchemaRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepository.java @@ -1,4 +1,4 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.schema; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CachedSchemaVersionsRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepository.java similarity index 71% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CachedSchemaVersionsRepository.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepository.java index 1ba87cd8ff..6e29ed25a7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/CachedSchemaVersionsRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepository.java @@ -1,4 +1,4 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.schema; import com.google.common.base.Ticker; import com.google.common.cache.CacheBuilder; @@ -21,29 +21,29 @@ public class CachedSchemaVersionsRepository implements SchemaVersionsRepository private static final Logger logger = LoggerFactory.getLogger(CachedSchemaVersionsRepository.class); - private final SchemaSourceProvider schemaSourceProvider; + private final SchemaSourceClient schemaSourceClient; private final LoadingCache> versionsCache; - public CachedSchemaVersionsRepository(SchemaSourceProvider schemaSourceProvider, ExecutorService versionsReloader, + public CachedSchemaVersionsRepository(SchemaSourceClient schemaSourceClient, ExecutorService versionsReloader, int refreshAfterWriteMinutes, int expireAfterWriteMinutes) { - this(schemaSourceProvider, versionsReloader, refreshAfterWriteMinutes, expireAfterWriteMinutes, Ticker.systemTicker()); + this(schemaSourceClient, versionsReloader, refreshAfterWriteMinutes, expireAfterWriteMinutes, Ticker.systemTicker()); } - CachedSchemaVersionsRepository(SchemaSourceProvider schemaSourceProvider, ExecutorService versionsReloader, + CachedSchemaVersionsRepository(SchemaSourceClient schemaSourceClient, ExecutorService versionsReloader, int refreshAfterWriteMinutes, int expireAfterWriteMinutes, Ticker ticker) { - this.schemaSourceProvider = schemaSourceProvider; + this.schemaSourceClient = schemaSourceClient; this.versionsCache = CacheBuilder .newBuilder() .ticker(ticker) .refreshAfterWrite(refreshAfterWriteMinutes, TimeUnit.MINUTES) .expireAfterWrite(expireAfterWriteMinutes, TimeUnit.MINUTES) - .build(new SchemaVersionsLoader(schemaSourceProvider, versionsReloader)); + .build(new SchemaVersionsLoader(schemaSourceClient, versionsReloader)); } @Override public List versions(Topic topic, boolean online) { try { - return online? schemaSourceProvider.versions(topic) : versionsCache.get(topic); + return online? schemaSourceClient.getVersions(topic.getName()) : versionsCache.get(topic); } catch (ExecutionException e) { logger.error("Error while loading schema versions for topic {}", topic.getQualifiedName(), e); return emptyList(); @@ -52,18 +52,18 @@ public List versions(Topic topic, boolean online) { private static class SchemaVersionsLoader extends CacheLoader> { - private final SchemaSourceProvider schemaSourceProvider; + private final SchemaSourceClient schemaSourceClient; private final ExecutorService versionsReloader; - public SchemaVersionsLoader(SchemaSourceProvider schemaSourceProvider, ExecutorService versionsReloader) { - this.schemaSourceProvider = schemaSourceProvider; + public SchemaVersionsLoader(SchemaSourceClient schemaSourceClient, ExecutorService versionsReloader) { + this.schemaSourceClient = schemaSourceClient; this.versionsReloader = versionsReloader; } @Override public List load(Topic topic) throws Exception { logger.info("Loading schema versions for topic {}", topic.getQualifiedName()); - return schemaSourceProvider.versions(topic); + return schemaSourceClient.getVersions(topic.getName()); } @Override @@ -71,7 +71,7 @@ public ListenableFuture> reload(Topic topic, List> task = ListenableFutureTask.create(() -> { logger.info("Reloading schema versions for topic {}", topic.getQualifiedName()); try { - return schemaSourceProvider.versions(topic); + return schemaSourceClient.getVersions(topic.getName()); } catch (Exception e) { logger.warn("Could not reload schema versions for topic {}", topic.getQualifiedName(), e); throw e; @@ -81,5 +81,4 @@ public ListenableFuture> reload(Topic topic, List { CompiledSchema getSchema(Topic topic, SchemaVersion version); - } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotLoadSchemaException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotLoadSchemaException.java new file mode 100644 index 0000000000..c085816cc7 --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotLoadSchemaException.java @@ -0,0 +1,19 @@ +package pl.allegro.tech.hermes.schema; + +import pl.allegro.tech.hermes.api.ErrorCode; + +public class CouldNotLoadSchemaException extends SchemaException { + + CouldNotLoadSchemaException(Throwable cause) { + super(cause); + } + + CouldNotLoadSchemaException(String message, Throwable cause) { + super(message, cause); + } + + @Override + public ErrorCode getCode() { + return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; + } +} diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotRemoveSchemaException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotRemoveSchemaException.java new file mode 100644 index 0000000000..d92be57b3f --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotRemoveSchemaException.java @@ -0,0 +1,15 @@ +package pl.allegro.tech.hermes.schema; + +import pl.allegro.tech.hermes.api.ErrorCode; + +public class CouldNotRemoveSchemaException extends SchemaException { + + public CouldNotRemoveSchemaException(String message) { + super(message); + } + + @Override + public ErrorCode getCode() { + return ErrorCode.SCHEMA_COULD_NOT_BE_DELETED; + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/DirectCompiledSchemaRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepository.java similarity index 64% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/DirectCompiledSchemaRepository.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepository.java index e755f66d84..dece4565c1 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/DirectCompiledSchemaRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepository.java @@ -1,30 +1,28 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.schema; import pl.allegro.tech.hermes.api.SchemaSource; import pl.allegro.tech.hermes.api.Topic; -import static java.lang.String.format; - public class DirectCompiledSchemaRepository implements CompiledSchemaRepository { - private final SchemaSourceProvider schemaSourceProvider; + private final SchemaSourceClient schemaSourceClient; private final SchemaCompiler schemaCompiler; - public DirectCompiledSchemaRepository(SchemaSourceProvider schemaSourceProvider, + public DirectCompiledSchemaRepository(SchemaSourceClient schemaSourceClient, SchemaCompiler schemaCompiler) { - this.schemaSourceProvider = schemaSourceProvider; + this.schemaSourceClient = schemaSourceClient; this.schemaCompiler = schemaCompiler; } @Override public CompiledSchema getSchema(Topic topic, SchemaVersion version) { try { - SchemaSource schemaSource = schemaSourceProvider.get(topic, version) + SchemaSource schemaSource = schemaSourceClient.getSchemaSource(topic.getName(), version) .orElseThrow(() -> new SchemaSourceNotFoundException(topic, version)); return new CompiledSchema<>(schemaCompiler.compile(schemaSource), version); } catch (Exception e) { throw new CouldNotLoadSchemaException( - format("Could not load schema type of %s for topic %s", + String.format("Could not load schema type of %s for topic %s", topic.getContentType(), topic.getQualifiedName()), e); } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SimpleSchemaVersionsRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectSchemaVersionsRepository.java similarity index 54% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SimpleSchemaVersionsRepository.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectSchemaVersionsRepository.java index fa57b2826e..53eb0c8d40 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SimpleSchemaVersionsRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectSchemaVersionsRepository.java @@ -1,29 +1,27 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; -import javax.inject.Inject; import java.util.List; import static java.util.Collections.emptyList; -public class SimpleSchemaVersionsRepository implements SchemaVersionsRepository { +public class DirectSchemaVersionsRepository implements SchemaVersionsRepository { - private static final Logger logger = LoggerFactory.getLogger(SimpleSchemaVersionsRepository.class); + private static final Logger logger = LoggerFactory.getLogger(DirectSchemaVersionsRepository.class); - private final SchemaSourceProvider schemaSourceProvider; + private final SchemaSourceClient schemaSourceClient; - @Inject - public SimpleSchemaVersionsRepository(SchemaSourceProvider schemaSourceProvider) { - this.schemaSourceProvider = schemaSourceProvider; + public DirectSchemaVersionsRepository(SchemaSourceClient schemaSourceClient) { + this.schemaSourceClient = schemaSourceClient; } @Override public List versions(Topic topic, boolean online) { try { - return schemaSourceProvider.versions(topic); + return schemaSourceClient.getVersions(topic.getName()); } catch (Exception e) { logger.error("Error while loading schema versions for topic {}", topic.getQualifiedName(), e); return emptyList(); diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/InvalidSchemaException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/InvalidSchemaException.java new file mode 100644 index 0000000000..7d70501d32 --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/InvalidSchemaException.java @@ -0,0 +1,15 @@ +package pl.allegro.tech.hermes.schema; + +import pl.allegro.tech.hermes.api.ErrorCode; + +public class InvalidSchemaException extends SchemaException { + + public InvalidSchemaException(String message) { + super(message); + } + + @Override + public ErrorCode getCode() { + return ErrorCode.FORMAT_ERROR; + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaCompiler.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompiler.java similarity index 74% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaCompiler.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompiler.java index 96667ed766..948ee164c0 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaCompiler.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompiler.java @@ -1,4 +1,4 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.schema; import pl.allegro.tech.hermes.api.SchemaSource; diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompilersFactory.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompilersFactory.java new file mode 100644 index 0000000000..888ce6d423 --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompilersFactory.java @@ -0,0 +1,10 @@ +package pl.allegro.tech.hermes.schema; + +import org.apache.avro.Schema; + +public interface SchemaCompilersFactory { + + static SchemaCompiler avroSchemaCompiler() { + return source -> new Schema.Parser().parse(source.value()); + } +} diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaException.java new file mode 100644 index 0000000000..1ef1f117aa --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaException.java @@ -0,0 +1,20 @@ +package pl.allegro.tech.hermes.schema; + +import pl.allegro.tech.hermes.api.ErrorCode; + +public abstract class SchemaException extends RuntimeException { + + SchemaException(String message) { + super(message); + } + + SchemaException(Throwable cause) { + super(cause); + } + + SchemaException(String message, Throwable cause) { + super(message, cause); + } + + public abstract ErrorCode getCode(); +} \ No newline at end of file diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepository.java new file mode 100644 index 0000000000..bbd3e49341 --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepository.java @@ -0,0 +1,48 @@ +package pl.allegro.tech.hermes.schema; + +import org.apache.avro.Schema; +import pl.allegro.tech.hermes.api.Topic; + +import java.util.List; + +public class SchemaRepository { + + private final SchemaVersionsRepository schemaVersionsRepository; + private final CompiledSchemaRepository compiledAvroSchemaRepository; + + public SchemaRepository(SchemaVersionsRepository schemaVersionsRepository, + CompiledSchemaRepository compiledAvroSchemaRepository) { + this.schemaVersionsRepository = schemaVersionsRepository; + this.compiledAvroSchemaRepository = compiledAvroSchemaRepository; + } + + public CompiledSchema getLatestAvroSchema(Topic topic) { + SchemaVersion latestVersion = schemaVersionsRepository.latestSchemaVersion(topic) + .orElseThrow(() -> new SchemaSourceNotFoundException(topic)); + if (!schemaVersionsRepository.schemaVersionExists(topic, latestVersion)) { + throw new SchemaSourceNotFoundException(topic, latestVersion); + } + return compiledAvroSchemaRepository.getSchema(topic, latestVersion); + } + + public CompiledSchema getAvroSchema(Topic topic, SchemaVersion version) { + if (!schemaVersionsRepository.schemaVersionExists(topic, version)) { + throw new SchemaSourceNotFoundException(topic, version); + } + return compiledAvroSchemaRepository.getSchema(topic, version); + } + + /** + * This method should be used only for cache-backed repository implementations + * where we have possibly stale versions-cache and are 100% sure the requested version exists. + * If it does not exist, each method call will try to load the requested version + * from underlying schema repository. + */ + public CompiledSchema getKnownAvroSchemaVersion(Topic topic, SchemaVersion version) { + return compiledAvroSchemaRepository.getSchema(topic, version); + } + + public List getVersions(Topic topic, boolean online) { + return schemaVersionsRepository.versions(topic, online); + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/SchemaRepoException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepositoryServerException.java similarity index 50% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/SchemaRepoException.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepositoryServerException.java index 120af3b134..8789fcedc9 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/SchemaRepoException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepositoryServerException.java @@ -1,10 +1,10 @@ -package pl.allegro.tech.hermes.common.exception; +package pl.allegro.tech.hermes.schema; import pl.allegro.tech.hermes.api.ErrorCode; -public class SchemaRepoException extends HermesException { +public class SchemaRepositoryServerException extends SchemaException { - public SchemaRepoException(String message) { + public SchemaRepositoryServerException(String message) { super(message); } @@ -12,5 +12,4 @@ public SchemaRepoException(String message) { public ErrorCode getCode() { return ErrorCode.INTERNAL_ERROR; } - } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaSourceClient.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaSourceClient.java new file mode 100644 index 0000000000..60305c9018 --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaSourceClient.java @@ -0,0 +1,20 @@ +package pl.allegro.tech.hermes.schema; + +import pl.allegro.tech.hermes.api.SchemaSource; +import pl.allegro.tech.hermes.api.TopicName; + +import java.util.List; +import java.util.Optional; + +public interface SchemaSourceClient { + + Optional getSchemaSource(TopicName topic, SchemaVersion version); + + Optional getLatestSchemaSource(TopicName topic); + + List getVersions(TopicName topic); + + void registerSchemaSource(TopicName topic, SchemaSource schemaSource); + + void deleteAllSchemaSources(TopicName topic); +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaSourceNotFoundException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaSourceNotFoundException.java similarity index 53% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaSourceNotFoundException.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaSourceNotFoundException.java index b7aa04c1f6..2282c6b2a6 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaSourceNotFoundException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaSourceNotFoundException.java @@ -1,17 +1,16 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.schema; import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.exception.HermesException; -public class SchemaSourceNotFoundException extends HermesException { +class SchemaSourceNotFoundException extends SchemaException { - public SchemaSourceNotFoundException(Topic topic) { - super("No schema source for topic " + topic.getQualifiedName()); + SchemaSourceNotFoundException(Topic topic, SchemaVersion schemaVersion) { + super("No schema source for topic " + topic.getQualifiedName() + " of version " + schemaVersion.value()); } - public SchemaSourceNotFoundException(Topic topic, SchemaVersion schemaVersion) { - super("No schema source for topic " + topic.getQualifiedName() + " of version " + schemaVersion.value()); + SchemaSourceNotFoundException(Topic topic) { + super("No schema source for topic " + topic.getQualifiedName()); } @Override diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaVersion.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersion.java similarity index 95% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaVersion.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersion.java index 7db1b41958..80ccb485c2 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaVersion.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersion.java @@ -1,4 +1,4 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.schema; import com.google.common.base.Joiner; diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaVersionsRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsRepository.java similarity index 76% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaVersionsRepository.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsRepository.java index 650d21ae2c..7fa90efd3b 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/schema/SchemaVersionsRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsRepository.java @@ -1,7 +1,8 @@ -package pl.allegro.tech.hermes.domain.topic.schema; +package pl.allegro.tech.hermes.schema; import pl.allegro.tech.hermes.api.Topic; +import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -12,7 +13,7 @@ default boolean schemaVersionExists(Topic topic, SchemaVersion version) { } default Optional latestSchemaVersion(Topic topic) { - return versions(topic).stream().findFirst(); + return versions(topic).stream().max(Comparator.comparingInt(SchemaVersion::value)); } default List versions(Topic topic) { diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRequest.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRequest.java new file mode 100644 index 0000000000..11e57fcb20 --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRequest.java @@ -0,0 +1,42 @@ +package pl.allegro.tech.hermes.schema.confluent; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import pl.allegro.tech.hermes.api.SchemaSource; + +import java.util.Objects; + +class SchemaRegistryRequest { + + private final String schema; + + @JsonCreator + SchemaRegistryRequest(@JsonProperty("schema") String schema) { + this.schema = schema; + } + + static SchemaRegistryRequest fromSchemaSource(SchemaSource schemaSource) { + return new SchemaRegistryRequest(schemaSource.value()); + } + + public String getSchema() { + return schema; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SchemaRegistryRequest that = (SchemaRegistryRequest) o; + return Objects.equals(schema, that.schema); + } + + @Override + public int hashCode() { + return Objects.hash(schema); + } +} diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryResponse.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryResponse.java new file mode 100644 index 0000000000..332aa4ba39 --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryResponse.java @@ -0,0 +1,69 @@ +package pl.allegro.tech.hermes.schema.confluent; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.NotNull; +import java.util.Objects; + +class SchemaRegistryResponse { + + @NotNull + private final String subject; + + @NotNull + private final Integer id; + + @NotNull + private final Integer version; + + @NotNull + private final String schema; + + @JsonCreator + SchemaRegistryResponse(@JsonProperty("subject") String subject, + @JsonProperty("id") Integer id, + @JsonProperty("version") Integer version, + @JsonProperty("schema") String schema) { + this.subject = subject; + this.version = version; + this.id = id; + this.schema = schema; + } + + public String getSubject() { + return subject; + } + + public Integer getVersion() { + return version; + } + + public Integer getId() { + return id; + } + + public String getSchema() { + return schema; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SchemaRegistryResponse that = (SchemaRegistryResponse) o; + return Objects.equals(subject, that.subject) && + Objects.equals(version, that.version) && + Objects.equals(id, that.id) && + Objects.equals(schema, that.schema); + } + + @Override + public int hashCode() { + return Objects.hash(subject, version, id, schema); + } +} diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistrySchemaSourceClient.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistrySchemaSourceClient.java new file mode 100644 index 0000000000..341436ee70 --- /dev/null +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistrySchemaSourceClient.java @@ -0,0 +1,146 @@ +package pl.allegro.tech.hermes.schema.confluent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.api.SchemaSource; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.schema.CouldNotRemoveSchemaException; +import pl.allegro.tech.hermes.schema.InvalidSchemaException; +import pl.allegro.tech.hermes.schema.SchemaRepositoryServerException; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaVersion; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.net.URI; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * This implementation of SchemaSourceClient is compatible with Confluent Schema Registry API + * except for the deleteAllSchemaSources, which is basically not supported by the Confluent project + */ +public class SchemaRegistrySchemaSourceClient implements SchemaSourceClient { + + private static final Logger logger = LoggerFactory.getLogger(SchemaRegistrySchemaSourceClient.class); + + private static final String SCHEMA_REPO_CONTENT_TYPE = "application/vnd.schemaregistry.v1+json"; + + private final WebTarget target; + + public SchemaRegistrySchemaSourceClient(Client client, URI schemaRegistryUri) { + this.target = client.target(schemaRegistryUri); + } + + @Override + public Optional getSchemaSource(TopicName topic, SchemaVersion version) { + Response response = target.path("subjects") + .path(topic.qualifiedName()) + .path("versions") + .path(Integer.toString(version.value())) + .request() + .get(); + return extractSchema(topic.qualifiedName(), response).map(SchemaSource::valueOf); + } + + @Override + public Optional getLatestSchemaSource(TopicName topic) { + Response response = target.path("subjects") + .path(topic.qualifiedName()) + .path("versions") + .path("latest") + .request() + .get(); + return extractSchema(topic.qualifiedName(), response).map(SchemaSource::valueOf); + } + + private Optional extractSchema(String subject, Response response) { + if (response.getStatus() == Response.Status.OK.getStatusCode()) { + String schema = response.readEntity(SchemaRegistryResponse.class).getSchema(); + return Optional.of(schema); + } else { + logger.error("Could not find schema for subject {}, reason: {}", subject, response.getStatus()); + return Optional.empty(); + } + } + + @Override + public List getVersions(TopicName topic) { + Response response = target.path("subjects") + .path(topic.qualifiedName()) + .path("versions") + .request() + .get(); + return extractSchemaVersions(topic.qualifiedName(), response); + } + + private List extractSchemaVersions(String subject, Response response) { + if (response.getStatus() == Response.Status.OK.getStatusCode()) { + return Arrays.stream(response.readEntity(Integer[].class)) + .sorted(Comparator.reverseOrder()) + .map(SchemaVersion::valueOf) + .collect(Collectors.toList()); + } else { + logger.error("Could not find schema versions for subject {}, reason: {}", subject, response.getStatus()); + return Collections.emptyList(); + } + } + + @Override + public void registerSchemaSource(TopicName topic, SchemaSource schemaSource) { + Response response = target.path("subjects") + .path(topic.qualifiedName()) + .path("versions") + .request() + .accept(MediaType.APPLICATION_JSON_TYPE) + .post(Entity.entity(SchemaRegistryRequest.fromSchemaSource(schemaSource), SCHEMA_REPO_CONTENT_TYPE)); + checkSchemaRegistration(response.getStatusInfo(), topic.qualifiedName(), response.readEntity(String.class)); + } + + private void checkSchemaRegistration(Response.StatusType statusType, String topicName, String response) { + switch (statusType.getFamily()) { + case SUCCESSFUL: + logger.info("Successful write to schema registry for subject {}", topicName); + break; + case CLIENT_ERROR: + logger.warn("Invalid schema for subject {}. Details: {}", topicName, response); + throw new InvalidSchemaException("Invalid schema. Reason: " + response); + case SERVER_ERROR: + logger.error("Failure write to schema repo for subject {}. Reason: {}", topicName, response); + throw new SchemaRepositoryServerException("Failure writing to schema registry. Reason: " + response); + default: + logger.error("Unknown response from schema registry. Subject {}, HTTP status {}, Details: {}", + topicName, statusType.getStatusCode(), response); + throw new SchemaRepositoryServerException("Unknown response from schema registry. Reason: " + response); + } + } + + @Override + public void deleteAllSchemaSources(TopicName topic) { + Response response = target.path("subjects") + .path(topic.qualifiedName()) + .request() + .delete(); + checkSchemaRemoval(response.getStatusInfo(), topic.qualifiedName(), response.readEntity(String.class)); + } + + private void checkSchemaRemoval(Response.StatusType statusType, String topicName, String response) { + switch (statusType.getFamily()) { + case SUCCESSFUL: + logger.info("Successful removed schema subject {}", topicName); + break; + case CLIENT_ERROR: + case SERVER_ERROR: + default: + logger.warn("Could not remove schema subject {}. Reason: {}", topicName, response); + throw new CouldNotRemoveSchemaException("Could not remove schema subject. Reason: " + response); + } + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/JerseySchemaRepoClient.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/schemarepo/SchemaRepoSchemaSourceClient.java similarity index 61% rename from hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/JerseySchemaRepoClient.java rename to hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/schemarepo/SchemaRepoSchemaSourceClient.java index e87424215a..a128bea60a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/schema/repo/JerseySchemaRepoClient.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/schemarepo/SchemaRepoSchemaSourceClient.java @@ -1,12 +1,15 @@ -package pl.allegro.tech.hermes.infrastructure.schema.repo; +package pl.allegro.tech.hermes.schema.schemarepo; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.common.exception.InvalidSchemaException; -import pl.allegro.tech.hermes.common.exception.SchemaRepoException; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; +import pl.allegro.tech.hermes.api.SchemaSource; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.schema.InvalidSchemaException; +import pl.allegro.tech.hermes.schema.SchemaRepositoryServerException; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaVersion; import javax.ws.rs.client.Client; import javax.ws.rs.client.Entity; @@ -23,32 +26,67 @@ import static javax.ws.rs.core.Response.Status.Family.SUCCESSFUL; -public class JerseySchemaRepoClient implements SchemaRepoClient { +public class SchemaRepoSchemaSourceClient implements SchemaSourceClient { - private static final Logger logger = LoggerFactory.getLogger(JerseySchemaRepoClient.class); + private static final Logger logger = LoggerFactory.getLogger(SchemaRepoSchemaSourceClient.class); private final WebTarget target; - public JerseySchemaRepoClient(Client client, URI schemaRepoServerUri) { + public SchemaRepoSchemaSourceClient(Client client, URI schemaRepoServerUri) { this.target = client.target(schemaRepoServerUri); } @Override - public void registerSubject(String subject) { - Response response = target.path(subject).request().put(Entity.entity("", MediaType.APPLICATION_FORM_URLENCODED_TYPE)); - if (SUCCESSFUL != response.getStatusInfo().getFamily()) { - logger.error("Failure subject registration in schema repo. Subject: {}, response code: {}, details: {}", - subject, response.getStatus(), response.readEntity(String.class)); - throw new SchemaRepoException("Failure subject registration in schema-repo."); + public Optional getSchemaSource(TopicName topic, SchemaVersion version) { + String subject = topic.qualifiedName(); + Response response = target.path(subject).path("id").path(Integer.toString(version.value())).request().get(); + return extractSchema(subject, response).map(SchemaSource::valueOf); + } + + @Override + public Optional getLatestSchemaSource(TopicName topic) { + String subject = topic.qualifiedName(); + Response response = target.path(subject).path("latest").request().get(); + return extractSchema(subject, response).map(SchemaSource::valueOf); + } + + @Override + public List getVersions(TopicName topic) { + Response response = target.path(topic.qualifiedName()).path("all").request().accept(MediaType.APPLICATION_JSON_TYPE).get(); + if (response.getStatus() == Response.Status.OK.getStatusCode()) { + List schemasWithIds = Optional.ofNullable(response.readEntity(new GenericType>() {})) + .orElseGet(Collections::emptyList); + return schemasWithIds.stream() + .map(SchemaWithId::getId) + .sorted(Comparator.reverseOrder()) + .map(SchemaVersion::valueOf) + .collect(Collectors.toList()); } + return Collections.emptyList(); } @Override - public boolean isSubjectRegistered(String subject) { + public void registerSchemaSource(TopicName topic, SchemaSource schemaSource) { + String topicName = topic.qualifiedName(); + if (!isSubjectRegistered(topicName)) { + registerSubject(topicName); + } + registerSchema(topicName, schemaSource.value()); + } + + private boolean isSubjectRegistered(String subject) { return target.path(subject).request().get().getStatus() == Response.Status.OK.getStatusCode(); } - @Override + private void registerSubject(String subject) { + Response response = target.path(subject).request().put(Entity.entity("", MediaType.APPLICATION_FORM_URLENCODED_TYPE)); + if (SUCCESSFUL != response.getStatusInfo().getFamily()) { + logger.error("Failure subject registration in schema repo. Subject: {}, response code: {}, details: {}", + subject, response.getStatus(), response.readEntity(String.class)); + throw new SchemaRepositoryServerException("Failure subject registration in schema-repo."); + } + } + public void registerSchema(String subject, String schema) { Response response = target.path(subject).path("register").request().put(Entity.entity(schema, MediaType.TEXT_PLAIN)); checkSchemaRegistration(response.getStatusInfo(), subject, response.readEntity(String.class)); @@ -61,27 +99,20 @@ private void checkSchemaRegistration(Response.StatusType statusType, String subj break; case CLIENT_ERROR: logger.warn("Invalid schema for subject {}. Details: {}", subject, response); - throw new InvalidSchemaException(String.format("Invalid schema. Reason: %s", response)); + throw new InvalidSchemaException("Invalid schema. Reason: " + response); case SERVER_ERROR: logger.error("Failure write to schema repo for subject {}. Reason: {}", subject, response); - throw new SchemaRepoException("Failure writing to schema-repo."); + throw new SchemaRepositoryServerException("Failure writing to schema-repo. Reason: " + response); default: logger.error("Unknown response from schema-repo. Subject {}, http status {}, Details: {}", subject, statusType.getStatusCode(), response); - throw new SchemaRepoException("Unknown response from schema-repo"); + throw new SchemaRepositoryServerException("Unknown response from schema-repo. Reason: " + response); } } @Override - public Optional getLatestSchema(String subject) { - Response response = target.path(subject).path("latest").request().get(); - return extractSchema(subject, response); - } - - @Override - public Optional getSchema(String subject, SchemaVersion version) { - Response response = target.path(subject).path("id").path(Integer.toString(version.value())).request().get(); - return extractSchema(subject, response); + public void deleteAllSchemaSources(TopicName topic) { + throw new UnsupportedOperationException("Deleting schemas is not supported by this repository type"); } private Optional extractSchema(String subject, Response response) { @@ -94,15 +125,6 @@ private Optional extractSchema(String subject, Response response) { } } - public List getSchemaVersions(String subject) { - Response response = target.path(subject).path("all").request().accept(MediaType.APPLICATION_JSON_TYPE).get(); - if (response.getStatus() == Response.Status.OK.getStatusCode()) { - List schemasWithIds = Optional.ofNullable(response.readEntity(new GenericType>() {})).orElseGet(Collections::emptyList); - return schemasWithIds.stream().map(SchemaWithId::getId).sorted(Comparator.reverseOrder()).map(SchemaVersion::valueOf).collect(Collectors.toList()); - } - return Collections.emptyList(); - } - private String parseSchema(String schemaResponse) { return schemaResponse.substring(1 + schemaResponse.indexOf('\t')); } diff --git a/hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/CachedCompiledSchemaRepositoryTest.groovy b/hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepositoryTest.groovy similarity index 96% rename from hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/CachedCompiledSchemaRepositoryTest.groovy rename to hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepositoryTest.groovy index 138c4fccef..fdc7ab5c44 100644 --- a/hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/CachedCompiledSchemaRepositoryTest.groovy +++ b/hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepositoryTest.groovy @@ -1,4 +1,4 @@ -package pl.allegro.tech.hermes.domain.topic.schema +package pl.allegro.tech.hermes.schema import spock.lang.Specification diff --git a/hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/CachedSchemaVersionsRepositoryTest.groovy b/hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepositoryTest.groovy similarity index 75% rename from hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/CachedSchemaVersionsRepositoryTest.groovy rename to hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepositoryTest.groovy index 1d22d62f91..d6f60be095 100644 --- a/hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/CachedSchemaVersionsRepositoryTest.groovy +++ b/hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepositoryTest.groovy @@ -1,6 +1,7 @@ -package pl.allegro.tech.hermes.domain.topic.schema +package pl.allegro.tech.hermes.schema import com.google.common.util.concurrent.MoreExecutors +import pl.allegro.tech.hermes.api.TopicName import pl.allegro.tech.hermes.test.helper.cache.FakeTicker import spock.lang.Specification @@ -17,16 +18,16 @@ class CachedSchemaVersionsRepositoryTest extends Specification { static final SchemaVersion v1 = SchemaVersion.valueOf(1) static final SchemaVersion v0 = SchemaVersion.valueOf(0) - def schemaSourceProvider = Stub(SchemaSourceProvider) + def schemaSourceClient = Stub(SchemaSourceClient) def ticker = new FakeTicker() - def versionsRepository = new CachedSchemaVersionsRepository(schemaSourceProvider, MoreExecutors.sameThreadExecutor(), + def versionsRepository = new CachedSchemaVersionsRepository(schemaSourceClient, MoreExecutors.sameThreadExecutor(), (int) REFRESH_TIME.toMinutes(), (int) EXPIRE_TIME.toMinutes(), ticker) def topic = topic("group", "topic").build() def "should indicate if schema version exists"() { given: - schemaSourceProvider.versions(topic) >> [v1, v0] + schemaSourceClient.getVersions(topic.getName()) >> [v1, v0] expect: versionsRepository.schemaVersionExists(topic, version) == expectedResult @@ -40,15 +41,22 @@ class CachedSchemaVersionsRepositoryTest extends Specification { def "should provide latest schema version"() { given: - schemaSourceProvider.versions(topic) >> [v1, v0] + schemaSourceClient.getVersions(topic.getName()) >> versions expect: - versionsRepository.latestSchemaVersion(topic).get() == v1 + versionsRepository.latestSchemaVersion(topic).get() == latestVersion + + where: + versions | latestVersion + [v1, v0] | v1 + [v0, v1] | v1 + [v1, v2] | v2 + [v1, v2, v0] | v2 } def "should empty latest schema version for topic without schema versions available"() { given: - schemaSourceProvider.versions(topic("other", "topic").build()) >> [] + schemaSourceClient.getVersions(TopicName.fromQualifiedName("other.topic")) >> [] expect: !versionsRepository.latestSchemaVersion(topic).isPresent() @@ -56,7 +64,7 @@ class CachedSchemaVersionsRepositoryTest extends Specification { def "should respond using cached schema versions if schema versions didn't expire"() { given: - schemaSourceProvider.versions(topic) >>> [[v1, v0], [v2, v1, v0]] + schemaSourceClient.getVersions(topic.getName()) >>> [[v1, v0], [v2, v1, v0]] versionsRepository.latestSchemaVersion(topic) ticker.advance(REFRESH_TIME.minusMinutes(1)) @@ -67,7 +75,7 @@ class CachedSchemaVersionsRepositoryTest extends Specification { def "should respond using fresh schema versions if schema versions did expire"() { given: - schemaSourceProvider.versions(topic) >>> [[v1, v0], [v2, v1, v0]] + schemaSourceClient.getVersions(topic.getName()) >>> [[v1, v0], [v2, v1, v0]] versionsRepository.latestSchemaVersion(topic) ticker.advance(REFRESH_TIME.plusMinutes(1)) @@ -79,7 +87,7 @@ class CachedSchemaVersionsRepositoryTest extends Specification { def "should respond with stale data if refresh failed"() { given: def failing = false - schemaSourceProvider.versions(topic) >> { + schemaSourceClient.getVersions(topic.getName()) >> { if (failing) { throw new RuntimeException("failing mode on") } diff --git a/hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/DirectCompiledSchemaRepositoryTest.groovy b/hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepositoryTest.groovy similarity index 71% rename from hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/DirectCompiledSchemaRepositoryTest.groovy rename to hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepositoryTest.groovy index 5c686e4f5a..4d231e92a1 100644 --- a/hermes-common/src/test/groovy/pl/allegro/tech/hermes/domain/topic/schema/DirectCompiledSchemaRepositoryTest.groovy +++ b/hermes-schema/src/test/groovy/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepositoryTest.groovy @@ -1,4 +1,4 @@ -package pl.allegro.tech.hermes.domain.topic.schema +package pl.allegro.tech.hermes.schema import pl.allegro.tech.hermes.api.SchemaSource import spock.lang.Specification @@ -9,15 +9,15 @@ class DirectCompiledSchemaRepositoryTest extends Specification { static final SchemaVersion v1 = SchemaVersion.valueOf(1) - def schemaSourceProvider = Stub(SchemaSourceProvider) + def schemaSourceClient = Stub(SchemaSourceClient) def schemaCompiler = { it.value().toUpperCase() } - def repository = new DirectCompiledSchemaRepository(schemaSourceProvider, schemaCompiler) + def repository = new DirectCompiledSchemaRepository(schemaSourceClient, schemaCompiler) def topic = topic("group", "topic").build() def "should provide schema from source of given version"() { given: - schemaSourceProvider.get(topic, v1) >> Optional.of(SchemaSource.valueOf('stuff')) + schemaSourceClient.getSchemaSource(topic.getName(), v1) >> Optional.of(SchemaSource.valueOf('stuff')) expect: repository.getSchema(topic, v1) == new CompiledSchema('STUFF', v1) @@ -25,7 +25,7 @@ class DirectCompiledSchemaRepositoryTest extends Specification { def "should fail to provide schema if schema source is missing"() { given: - schemaSourceProvider.get(topic, v1) >> Optional.empty() + schemaSourceClient.getSchemaSource(topic.getName(), v1) >> Optional.empty() when: repository.getSchema(topic, v1) @@ -36,7 +36,7 @@ class DirectCompiledSchemaRepositoryTest extends Specification { def "should fail to provide schema if loading schema source failed"() { given: - schemaSourceProvider.get(topic, v1) >> { throw new RuntimeException("loading failed") } + schemaSourceClient.getSchemaSource(topic.getName(), v1) >> { throw new RuntimeException("loading failed") } when: repository.getSchema(topic, v1) @@ -47,8 +47,8 @@ class DirectCompiledSchemaRepositoryTest extends Specification { def "should fail to provide schema if schema compilation failed"() { given: - schemaSourceProvider.get(topic, v1) >> Optional.of(SchemaSource.valueOf('stuff')) - def repository = new DirectCompiledSchemaRepository(schemaSourceProvider, { + schemaSourceClient.getSchemaSource(topic.getName(), v1) >> Optional.of(SchemaSource.valueOf('stuff')) + def repository = new DirectCompiledSchemaRepository(schemaSourceClient, { throw new RuntimeException("compilation failed") }) diff --git a/hermes-schema/src/test/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistrySchemaSourceClientTest.java b/hermes-schema/src/test/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistrySchemaSourceClientTest.java new file mode 100644 index 0000000000..cf8125ade2 --- /dev/null +++ b/hermes-schema/src/test/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistrySchemaSourceClientTest.java @@ -0,0 +1,219 @@ +package pl.allegro.tech.hermes.schema.confluent; + +import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; +import com.github.tomakehurst.wiremock.client.UrlMatchingStrategy; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import org.assertj.core.api.Assertions; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import pl.allegro.tech.hermes.api.SchemaSource; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.schema.CouldNotRemoveSchemaException; +import pl.allegro.tech.hermes.schema.InvalidSchemaException; +import pl.allegro.tech.hermes.schema.SchemaRepositoryServerException; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaVersion; +import pl.allegro.tech.hermes.test.helper.util.Ports; + +import javax.ws.rs.client.ClientBuilder; +import java.net.URI; +import java.util.List; +import java.util.Optional; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.delete; +import static com.github.tomakehurst.wiremock.client.WireMock.deleteRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static com.googlecode.catchexception.CatchException.catchException; +import static com.googlecode.catchexception.CatchException.caughtException; +import static org.assertj.core.api.Assertions.assertThat; + +public class SchemaRegistrySchemaSourceClientTest { + + private static final int PORT = Ports.nextAvailable(); + + private static final String SCHEMA_REGISTRY_CONTENT_TYPE = "application/vnd.schemaregistry.v1+json"; + private static final String TOPIC_NAME = "group.topic"; + private static final TopicName topic = TopicName.fromQualifiedName(TOPIC_NAME); + private static final SchemaSource schemaSource = SchemaSource.valueOf("{}"); + + private static final SchemaSourceClient client = new SchemaRegistrySchemaSourceClient( + ClientBuilder.newClient(), + URI.create("http://localhost:" + PORT) + ); + + @ClassRule + public static final WireMockRule wireMock = new WireMockRule( + wireMockConfig().port(PORT).usingFilesUnderClasspath("schema-repo-stub") + ); + + @Before + public void initialize() { + wireMock.resetRequests(); + wireMock.resetMappings(); + wireMock.resetScenarios(); + } + + @Test + public void shouldRegisterSchemaSource() { + // given + stubFor(post(schemaVersionsUrl(TOPIC_NAME)).willReturn(okResponse())); + + // when + client.registerSchemaSource(topic, SchemaSource.valueOf("{}")); + + // then + verify(1, postRequestedFor(schemaVersionsUrl(TOPIC_NAME)) + .withHeader("Content-type", equalTo(SCHEMA_REGISTRY_CONTENT_TYPE)) + .withRequestBody(equalTo("{\"schema\":\"{}\"}"))); + } + + @Test + public void shouldThrowInvalidSchemaExceptionForInvalidSchemaRegistration() { + // given + stubFor(post(schemaVersionsUrl(TOPIC_NAME)).willReturn(badRequestResponse())); + + // when + catchException(client).registerSchemaSource(topic, schemaSource); + + // then + assertThat((Throwable) caughtException()).isInstanceOf(InvalidSchemaException.class); + } + + @Test + public void shouldThrowSchemaRepositoryServerExceptionForServerInternalErrorResponseWhenRegisteringSchema() { + // given + stubFor(post(schemaVersionsUrl(TOPIC_NAME)).willReturn(internalErrorResponse())); + + // when + catchException(client).registerSchemaSource(topic, schemaSource); + + // then + assertThat((Throwable) caughtException()).isInstanceOf(SchemaRepositoryServerException.class); + } + + @Test + public void shouldFetchSchemaAtVersion() { + // given + stubFor(get(getSchemaVersion(TOPIC_NAME, 5)) + .willReturn(okResponse() + .withHeader("Content-Type", "application/json") + .withBody("{\"subject\":\"group.topic\", \"id\":100, \"version\":5, \"schema\":\"{}\"}"))); + + // when + Optional schemaAtVersion5 = client.getSchemaSource(topic, SchemaVersion.valueOf(5)); + + // then + assertThat(schemaAtVersion5.get()).isEqualTo(SchemaSource.valueOf("{}")); + } + + @Test + public void shouldFetchLatestSchemaVersion() { + // given + stubFor(get(getSchemaLatestVersionUrl(TOPIC_NAME)) + .willReturn(okResponse() + .withHeader("Content-Type", "application/json") + .withBody("{\"subject\":\"group.topic\", \"id\":3, \"version\":2, \"schema\":\"{}\"}"))); + + // when + Optional latestSchema = client.getLatestSchemaSource(topic); + + // then + assertThat(latestSchema.get()).isEqualTo(SchemaSource.valueOf("{}")); + } + + @Test + public void shouldReturnAllSchemaVersions() { + // given + stubFor(get(schemaVersionsUrl(TOPIC_NAME)) + .willReturn(okResponse() + .withHeader("Content-Type", "application/json") + .withBody("[3,6,2]"))); + + // when + List versions = client.getVersions(topic); + + // then + Assertions.assertThat(versions).contains(SchemaVersion.valueOf(3), SchemaVersion.valueOf(6), SchemaVersion.valueOf(2)); + } + + @Test + public void shouldReturnEmptyListOnNotRegisteredSubject() { + // given + stubFor(get(schemaVersionsUrl(TOPIC_NAME)).willReturn(notFoundResponse())); + + // when + List versions = client.getVersions(topic); + + // then + Assertions.assertThat(versions).isEmpty(); + } + + @Test + public void shouldDeleteAllSchemaVersions() { + // given + stubFor(delete(subjectUrl(TOPIC_NAME)).willReturn(okResponse())); + + // when + client.deleteAllSchemaSources(topic); + + // then + verify(1, deleteRequestedFor(subjectUrl(TOPIC_NAME))); + } + + @Test + public void shouldThrowExceptionForMethodNotAllowedResponseWhenDeletingSchema() { + // given + stubFor(delete(subjectUrl(TOPIC_NAME)).willReturn(methodNotAllowedResponse())); + + // when + catchException(client).deleteAllSchemaSources(topic); + + // then + assertThat((Throwable) caughtException()).isInstanceOf(CouldNotRemoveSchemaException.class); + } + + private UrlMatchingStrategy getSchemaVersion(String topicName, int version) { + return urlEqualTo("/subjects/" + topicName + "/versions/" + version); + } + + private UrlMatchingStrategy getSchemaLatestVersionUrl(String topicName) { + return urlEqualTo("/subjects/" + topicName + "/versions/latest"); + } + + private UrlMatchingStrategy schemaVersionsUrl(String topicName) { + return urlEqualTo("/subjects/" + topicName + "/versions"); + } + + private UrlMatchingStrategy subjectUrl(String topicName) { + return urlEqualTo("/subjects/" + topicName); + } + + private ResponseDefinitionBuilder okResponse() { + return aResponse().withStatus(200); + } + + private ResponseDefinitionBuilder badRequestResponse() { + return aResponse().withStatus(400); + } + + private ResponseDefinitionBuilder methodNotAllowedResponse() { + return aResponse().withStatus(405); + } + + private ResponseDefinitionBuilder notFoundResponse() { + return aResponse().withStatus(404); + } + + private ResponseDefinitionBuilder internalErrorResponse() { + return aResponse().withStatus(500); + } +} \ No newline at end of file diff --git a/hermes-schema/src/test/java/pl/allegro/tech/hermes/schema/schemarepo/SchemaRepoSchemaSourceClientTest.java b/hermes-schema/src/test/java/pl/allegro/tech/hermes/schema/schemarepo/SchemaRepoSchemaSourceClientTest.java new file mode 100644 index 0000000000..bcdc233e3d --- /dev/null +++ b/hermes-schema/src/test/java/pl/allegro/tech/hermes/schema/schemarepo/SchemaRepoSchemaSourceClientTest.java @@ -0,0 +1,240 @@ +package pl.allegro.tech.hermes.schema.schemarepo; + +import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; +import com.github.tomakehurst.wiremock.client.UrlMatchingStrategy; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import org.assertj.core.api.Assertions; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import pl.allegro.tech.hermes.api.SchemaSource; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.schema.InvalidSchemaException; +import pl.allegro.tech.hermes.schema.SchemaRepositoryServerException; +import pl.allegro.tech.hermes.schema.SchemaSourceClient; +import pl.allegro.tech.hermes.schema.SchemaVersion; +import pl.allegro.tech.hermes.test.helper.util.Ports; + +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.core.MediaType; +import java.net.URI; +import java.util.List; +import java.util.Optional; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.put; +import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static com.googlecode.catchexception.CatchException.catchException; +import static com.googlecode.catchexception.CatchException.caughtException; +import static org.assertj.core.api.Assertions.assertThat; + +public class SchemaRepoSchemaSourceClientTest { + + private static final String ROOT_DIR = "/schema-repo/"; + private static final int PORT = Ports.nextAvailable(); + + private static final String TOPIC_NAME = "group.topic"; + private static final TopicName topic = TopicName.fromQualifiedName(TOPIC_NAME); + private static final SchemaSource schemaSource = SchemaSource.valueOf("{}"); + + private static final SchemaSourceClient client = new SchemaRepoSchemaSourceClient( + ClientBuilder.newClient(), + URI.create("http://localhost:" + PORT + ROOT_DIR) + ); + + @ClassRule + public static final WireMockRule wireMock = new WireMockRule( + wireMockConfig().port(PORT).usingFilesUnderClasspath("schema-repo-stub") + ); + + @Before + public void initialize() { + wireMock.resetRequests(); + wireMock.resetMappings(); + wireMock.resetScenarios(); + } + + @Test + public void shouldRegisterSubjectAndSchemaSource() { + // given + stubFor(get(subjectUrl(TOPIC_NAME)).willReturn(notFoundResponse())); + stubFor(put(subjectUrl(TOPIC_NAME)).willReturn(okResponse())); + stubFor(put(registerSchemaUrl(TOPIC_NAME)).willReturn(okResponse())); + + // when + client.registerSchemaSource(topic, schemaSource); + + // then + verify(1, getRequestedFor(subjectUrl(TOPIC_NAME))); + verify(1, putRequestedFor(subjectUrl(TOPIC_NAME))); + verify(1, putRequestedFor(registerSchemaUrl(TOPIC_NAME)) + .withHeader("Content-type", equalTo(MediaType.TEXT_PLAIN)) + .withRequestBody(equalTo(schemaSource.value()))); + } + + @Test + public void shouldRegisterSchemaSourceWhenSubjectIsAlreadyRegistered() { + // given + stubFor(get(subjectUrl(TOPIC_NAME)).willReturn(okResponse())); + stubFor(put(registerSchemaUrl(TOPIC_NAME)).willReturn(okResponse())); + + // when + client.registerSchemaSource(topic, schemaSource); + + // then + verify(1, getRequestedFor(subjectUrl(TOPIC_NAME))); + verify(0, putRequestedFor(subjectUrl(TOPIC_NAME))); + verify(1, putRequestedFor(registerSchemaUrl(TOPIC_NAME)) + .withHeader("Content-type", equalTo(MediaType.TEXT_PLAIN)) + .withRequestBody(equalTo(schemaSource.value()))); + } + + @Test + public void shouldThrowExceptionForUnsuccessfulSubjectRegistration() { + // given + stubFor(get(subjectUrl(TOPIC_NAME)).willReturn(notFoundResponse())); + stubFor(put(subjectUrl(TOPIC_NAME)).willReturn(serverErrorResponse())); + + // when + catchException(client).registerSchemaSource(topic, schemaSource); + + // then + assertThat((Throwable) caughtException()).isInstanceOf(SchemaRepositoryServerException.class); + } + + @Test + public void shouldReturnEmptyOptionalIfLatestSchemaDoesNotExist() { + // given + stubFor(get(latestSchemaUrl(TOPIC_NAME)).willReturn(notFoundResponse())); + + // when + Optional schema = client.getLatestSchemaSource(topic); + + // then + assertThat(schema.isPresent()).isFalse(); + } + + @Test + public void shouldReturnLatestSchemaIfExists() { + // given + stubFor(get(latestSchemaUrl(TOPIC_NAME)).willReturn(okResponse().withBody("0\t{}"))); + + // when + Optional schema = client.getLatestSchemaSource(topic); + + // then + assertThat(schema.get().value()).isEqualTo("{}"); + } + + + @Test + public void shouldThrowExceptionForInvalidSchemaRegistration() { + // given + stubFor(get(subjectUrl(TOPIC_NAME)).willReturn(okResponse())); + stubFor(put(registerSchemaUrl(TOPIC_NAME)).willReturn(forbiddenResponse("error"))); + + // when + catchException(client).registerSchemaSource(topic, schemaSource); + + // then + assertThat((Throwable) caughtException()) + .isInstanceOf(InvalidSchemaException.class) + .hasMessageContaining("error"); + } + + @Test + public void shouldThrowSchemaRepoExceptionForSchemaRegistration() { + // given + stubFor(get(subjectUrl(TOPIC_NAME)).willReturn(okResponse())); + stubFor(put(registerSchemaUrl(TOPIC_NAME)).willReturn(serverErrorResponse())); + + // when + catchException(client).registerSchemaSource(topic, schemaSource); + + // then + assertThat((Throwable) caughtException()).isInstanceOf(SchemaRepositoryServerException.class); + } + + @Test + public void shouldReturnSchemaVersions() { + // given + wireMock.stubFor(get(allSchemasUrl(TOPIC_NAME)).willReturn( + okResponse().withBodyFile("all-schemas-response.json").withHeader("Content-Type", "application/json")) + ); + + // when + List versions = client.getVersions(topic); + + // then + Assertions.assertThat(versions).containsExactly( + SchemaVersion.valueOf(2), + SchemaVersion.valueOf(1), + SchemaVersion.valueOf(0) + ); + } + + @Test + public void shouldReturnEmptySchemaVersionsIfNoSchemasAreRegistered() { + // given + wireMock.stubFor(get(allSchemasUrl(TOPIC_NAME)).willReturn( + okResponse().withBody("").withHeader("Content-Type", "application/json")) + ); + + // when + List versions = client.getVersions(topic); + + // then + Assertions.assertThat(versions).isEmpty(); + } + + @Test + public void shouldReturnEmptySchemaVersionsIfSubjectDoesntExist() { + // given + wireMock.stubFor(get(allSchemasUrl(TOPIC_NAME)).willReturn(notFoundResponse())); + + // when + List versions = client.getVersions(topic); + + // then + Assertions.assertThat(versions).isEmpty(); + } + + private UrlMatchingStrategy subjectUrl(String name) { + return urlEqualTo(ROOT_DIR + name); + } + + private UrlMatchingStrategy latestSchemaUrl(String name) { + return urlEqualTo(ROOT_DIR + name + "/latest"); + } + + private UrlMatchingStrategy registerSchemaUrl(String name) { + return urlEqualTo(ROOT_DIR + name + "/register"); + } + + private UrlMatchingStrategy allSchemasUrl(String name) { + return urlEqualTo(ROOT_DIR + name + "/all"); + } + + private ResponseDefinitionBuilder okResponse() { + return aResponse().withStatus(200); + } + + private ResponseDefinitionBuilder serverErrorResponse() { + return aResponse().withStatus(500); + } + + private ResponseDefinitionBuilder notFoundResponse() { + return aResponse().withStatus(404); + } + + private ResponseDefinitionBuilder forbiddenResponse(String body) { + return aResponse().withStatus(403).withBody(body); + } +} \ No newline at end of file diff --git a/hermes-common/src/test/resources/schema-repo-stub/__files/all-schemas-response.json b/hermes-schema/src/test/resources/schema-repo-stub/__files/all-schemas-response.json similarity index 100% rename from hermes-common/src/test/resources/schema-repo-stub/__files/all-schemas-response.json rename to hermes-schema/src/test/resources/schema-repo-stub/__files/all-schemas-response.json diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUser.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUser.java index 49ebe4e811..c030ce33e7 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUser.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUser.java @@ -3,8 +3,8 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import pl.allegro.tech.hermes.domain.topic.schema.SchemaVersion; -import pl.allegro.tech.hermes.domain.topic.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.CompiledSchema; +import pl.allegro.tech.hermes.schema.SchemaVersion; import pl.allegro.tech.hermes.test.helper.message.TestMessage; import java.io.IOException; diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/TopicBuilder.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/TopicBuilder.java index 041359f486..ee92c3ee03 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/TopicBuilder.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/TopicBuilder.java @@ -11,10 +11,6 @@ public class TopicBuilder { private String description = "description"; - private boolean validationEnabled = false; - - private boolean validationDryRunEnabled = false; - private boolean jsonToAvroDryRunEnabled = false; private Topic.Ack ack = Topic.Ack.LEADER; @@ -47,8 +43,8 @@ public static TopicBuilder topic(String qualifiedName) { public Topic build() { return new Topic( - name, description, retentionTime, validationEnabled, validationDryRunEnabled, migratedFromJsonType, - ack, trackingEnabled, contentType, jsonToAvroDryRunEnabled, schemaVersionAwareSerialization + name, description, retentionTime, migratedFromJsonType, ack, trackingEnabled, contentType, + jsonToAvroDryRunEnabled, schemaVersionAwareSerialization ); } @@ -67,16 +63,6 @@ public TopicBuilder withRetentionTime(int retentionTime) { return this; } - public TopicBuilder withValidation(boolean enabled) { - this.validationEnabled = enabled; - return this; - } - - public TopicBuilder withValidationDryRun(boolean enabled) { - this.validationDryRunEnabled = enabled; - return this; - } - public TopicBuilder withJsonToAvroDryRun(boolean enabled) { this.jsonToAvroDryRunEnabled = enabled; return this; diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/HermesAPIOperations.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/HermesAPIOperations.java index cb6cb788bc..9eed56905b 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/HermesAPIOperations.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/HermesAPIOperations.java @@ -52,6 +52,17 @@ public Topic createTopic(String group, String topic) { return created; } + public Topic createAvroTopic(String group, String topic) { + Topic created = topic(group, topic) + .withRetentionTime(1000) + .withDescription("Test topic") + .withContentType(ContentType.AVRO) + .build(); + + createTopic(created); + return created; + } + public Topic createTopic(Topic topic) { if (endpoints.findTopics(topic, topic.isTrackingEnabled()).contains(topic.getQualifiedName())) { return topic; @@ -108,6 +119,11 @@ public Topic buildTopic(String group, String topic) { return createTopic(group, topic); } + public Topic buildAvroTopic(String group, String topic) { + createGroup(group); + return createAvroTopic(group, topic); + } + public Topic buildTopic(Topic topic) { createGroup(topic.getName().getGroupName()); return createTopic(topic); diff --git a/hermes-test-helper/src/main/resources/schema/example.json b/hermes-test-helper/src/main/resources/schema/example.json deleted file mode 100644 index 98c435cf49..0000000000 --- a/hermes-test-helper/src/main/resources/schema/example.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "title": "Event Schema", - "type": "object", - "properties": { - "id": { - "description": "id of event", - "type": "integer" - } - }, - "required": ["id"] -} \ No newline at end of file diff --git a/hermes-test-helper/src/main/resources/schema/user_v2.avsc b/hermes-test-helper/src/main/resources/schema/user_v2.avsc index 1f27e92673..c7d3ba17da 100644 --- a/hermes-test-helper/src/main/resources/schema/user_v2.avsc +++ b/hermes-test-helper/src/main/resources/schema/user_v2.avsc @@ -11,6 +11,6 @@ {"name": "name", "type": "string"}, {"name": "age", "type": "int"}, {"name": "favoriteColor", "type": ["null", "string"]}, - {"name": "other", "type": ["null", "string"]} + {"name": "other", "type": ["null", "string"], "default": null} ] } diff --git a/integration/build.gradle b/integration/build.gradle index 0199c03857..d944a6c353 100644 --- a/integration/build.gradle +++ b/integration/build.gradle @@ -1,6 +1,7 @@ repositories { maven { url 'http://repository.jboss.org/nexus/content/groups/public' } + maven { url 'http://packages.confluent.io/maven/' } } project.configurations { @@ -66,9 +67,13 @@ dependencies { testCompile group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.2.0' testCompile files('./lib/schema-repo-server-0.1.3-jersey2.jar'); - testCompile 'org.glassfish.jersey.containers:jersey-container-grizzly2-http:2.23.2' + testCompile group: 'org.glassfish.jersey.containers', name: 'jersey-container-grizzly2-http', version: versions.jersey testCompile group: 'org.freemarker', name: 'freemarker', version: '2.3.21' testCompile group: 'org.schemarepo', name: 'schema-repo-common', version: '0.1.3' + testCompile group: 'io.confluent', name: 'kafka-schema-registry', version: '3.0.1' + testCompile group: 'org.eclipse.jetty', name: 'jetty-server', version: versions.jetty + testCompile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: versions.jetty + testCompile group: 'org.eclipse.jetty', name: 'jetty-servlets', version: versions.jetty testCompile(group: 'org.hornetq', name: 'hornetq-jms-server', version: '2.4.1.Final') { exclude module: 'hornetq-native' diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/BatchDeliveryTest.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/BatchDeliveryTest.java index 1c2fc73ed7..93fbe27901 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/BatchDeliveryTest.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/BatchDeliveryTest.java @@ -104,7 +104,6 @@ public void shouldDeliverAvroMessagesAsJsonBatch() { AvroUser user = new AvroUser("Bob", 50, "blue"); Topic topic = topic("batch.avro.topic") - .withValidation(true) .withContentType(AVRO).build(); operations.buildTopic(topic); operations.saveSchema(topic, user.getSchemaAsString()); diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/FilteringAvroTest.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/FilteringAvroTest.java index 44e48bf61a..279c2f0055 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/FilteringAvroTest.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/FilteringAvroTest.java @@ -42,7 +42,6 @@ public void initializeAlways() { public void shouldFilterIncomingEvents() { // given final Topic topic = topic("filteredTopic.topic") - .withValidation(true) .withContentType(AVRO).build(); operations.buildTopic(topic); operations.saveSchema(topic, AvroUserSchemaLoader.load().toString()); @@ -69,7 +68,6 @@ public void shouldFilterIncomingEvents() { public void shouldChainMultipleFilters() { // given final Topic topic = topic("filteredChainTopic.topic") - .withValidation(true) .withContentType(AVRO).build(); operations.buildTopic(topic); operations.saveSchema(topic, AvroUserSchemaLoader.load().toString()); diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/KafkaSingleMessageReaderTest.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/KafkaSingleMessageReaderTest.java index 0ffddf33cd..7988a2e89f 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/KafkaSingleMessageReaderTest.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/KafkaSingleMessageReaderTest.java @@ -64,7 +64,6 @@ public void shouldFetchSingleMessageByTopicPartitionAndOffset() { public void shouldFetchSingleAvroMessage() throws IOException { // given Topic topic = topic("avro.fetch") - .withValidation(true) .withContentType(AVRO).build(); operations.buildTopic(topic); operations.saveSchema(topic, avroUser.getSchemaAsString()); @@ -86,7 +85,6 @@ public void shouldFetchSingleAvroMessage() throws IOException { public void shouldFetchSingleAvroMessageWithSchemaAwareSerialization() throws IOException { // given Topic topic = topic("avro.fetchSchemaAwareSerialization") - .withValidation(true) .withSchemaVersionAwareSerialization() .withContentType(AVRO).build(); operations.buildTopic(topic); diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/MessageBufferLoadingTest.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/MessageBufferLoadingTest.java index fef19cec16..7f754fd544 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/MessageBufferLoadingTest.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/MessageBufferLoadingTest.java @@ -137,7 +137,7 @@ private File backupFileWithOneMessage(Topic topic) { File backup = new File(tempDir.getAbsoluteFile(), "messages.dat"); MessageRepository messageRepository = new ChronicleMapMessageRepository(backup); - MessageContentWrapper wrapper = new MessageContentWrapper(new JsonMessageContentWrapper(CONFIG_FACTORY, new ObjectMapper()), null); + MessageContentWrapper wrapper = new MessageContentWrapper(new JsonMessageContentWrapper(CONFIG_FACTORY, new ObjectMapper()), null, null); String messageId = randomUUID().toString(); long timestamp = now().toEpochMilli(); diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/PublishingAvroTest.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/PublishingAvroTest.java index 1540ff2619..0d4595f5ff 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/PublishingAvroTest.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/PublishingAvroTest.java @@ -55,7 +55,6 @@ public void initializeAlways() { public void shouldPublishAvroAndConsumeJsonMessage() throws InterruptedException, ExecutionException, TimeoutException, IOException { // given Topic topic = operations.buildTopic(topic("publishAvroConsumeJson.topic") - .withValidation(true) .withContentType(AVRO) .build() ); @@ -74,7 +73,6 @@ public void shouldPublishAvroAndConsumeJsonMessage() throws InterruptedException public void shouldPublishAvroAndConsumeAvroMessage() throws InterruptedException, ExecutionException, TimeoutException, IOException { // given Topic topic = operations.buildTopic(topic("publishAvroConsumeAvro.topic") - .withValidation(true) .withContentType(AVRO) .build() ); @@ -93,7 +91,6 @@ public void shouldPublishAvroAndConsumeAvroMessage() throws InterruptedException public void shouldSendAvroAfterSubscriptionContentTypeChanged() throws Exception { // given Topic topic = operations.buildTopic(topic("publishAvroAfterTopicEditing.topic") - .withValidation(true) .withContentType(AVRO) .build() ); @@ -125,7 +122,6 @@ public void shouldSendAvroAfterSubscriptionContentTypeChanged() throws Exception public void shouldGetBadRequestForPublishingInvalidMessageWithSchema() throws InterruptedException, ExecutionException, TimeoutException { // given Topic topic = operations.buildTopic(topic("invalidAvro.topic") - .withValidation(true) .withContentType(AVRO).build()); operations.saveSchema(topic, user.getSchemaAsString()); @@ -140,8 +136,6 @@ public void shouldGetBadRequestForPublishingInvalidMessageWithSchema() throws In public void shouldIgnoreValidationDryRunSettingForAvroTopic() { // given Topic topic = operations.buildTopic(topic("invalidAvro.topicWithValidationDryRun") - .withValidation(true) - .withValidationDryRun(true) .withContentType(AVRO).build()); operations.saveSchema(topic, user.getSchemaAsString()); @@ -156,7 +150,6 @@ public void shouldIgnoreValidationDryRunSettingForAvroTopic() { public void shouldPublishJsonMessageConvertedToAvroForAvroTopics() { // given Topic topic = operations.buildTopic(topic("avro.topic2") - .withValidation(true) .withContentType(AVRO) .build() ); @@ -172,7 +165,6 @@ public void shouldPublishJsonMessageConvertedToAvroForAvroTopics() { @Test public void shouldGetBadRequestForJsonInvalidWithAvroSchema() { Topic topic = topic("avro.invalidJson") - .withValidation(true) .withContentType(AVRO).build(); operations.buildTopic(topic); operations.saveSchema(topic, user.getSchemaAsString()); @@ -246,7 +238,6 @@ public void shouldPublishAvroAndConsumeJsonMessageWithTraceId() throws IOExcepti // and Topic topic = operations.buildTopic(topic("publishAvroConsumeJsonWithTraceId.topic") - .withValidation(true) .withContentType(AVRO).build() ); operations.saveSchema(topic, user.getSchemaAsString()); @@ -270,7 +261,6 @@ public void shouldPublishAvroAndConsumeJsonMessageWithTraceId() throws IOExcepti public void shouldUseExplicitSchemaVersionWhenPublishingAndConsuming() throws IOException { // given Topic topic = operations.buildTopic(topic("explicitSchemaVersion.topic") - .withValidation(true) .withContentType(AVRO) .withSchemaVersionAwareSerialization() .build() @@ -281,12 +271,12 @@ public void shouldUseExplicitSchemaVersionWhenPublishingAndConsuming() throws IO operations.saveSchema(topic, load("/schema/user_v2.avsc").toString()); // when - assertThat(publisher.publishAvro(topic.getQualifiedName(), user.asBytes(), of(SCHEMA_VERSION.getName(), "0"))) + assertThat(publisher.publishAvro(topic.getQualifiedName(), user.asBytes(), of(SCHEMA_VERSION.getName(), "1"))) .hasStatus(CREATED); // then remoteService.waitUntilRequestReceived(request -> { - assertThat(request.getHeaders().getHeader(SCHEMA_VERSION.getName()).firstValue()).isEqualTo("0"); + assertThat(request.getHeaders().getHeader(SCHEMA_VERSION.getName()).firstValue()).isEqualTo("1"); assertBodyDeserializesIntoUser(request.getBodyAsString(), user); }); } diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/PublishingTest.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/PublishingTest.java index 3d7a618819..6833d39887 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/PublishingTest.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/PublishingTest.java @@ -1,8 +1,6 @@ package pl.allegro.tech.hermes.integration; -import org.apache.commons.io.IOUtils; import org.glassfish.jersey.client.ClientConfig; -import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import pl.allegro.tech.hermes.api.EndpointAddress; @@ -46,13 +44,6 @@ public class PublishingTest extends IntegrationTest { private RemoteServiceEndpoint remoteService; - private String schema; - - @BeforeClass - public void initialize() throws IOException { - schema = IOUtils.toString(this.getClass().getResourceAsStream("/schema/example.json")); - } - @BeforeMethod public void initializeAlways() { this.remoteService = new RemoteServiceEndpoint(SharedServices.services().serviceMock()); @@ -235,61 +226,6 @@ public void shouldNotCreateTopicWhenPublishingToNonExistingTopic() throws Except assertThat(SharedServices.services().zookeeper().checkExists().forPath(paths.topicPath(nonExisting))).isNull(); } - @Test - public void shouldPublishValidMessageWithJsonSchema() { - //given - String message = "{\"id\": 6}"; - Topic topic = operations.buildTopic(topic("schema.topic.validJson") - .withValidation(true) - .withContentType(JSON) - .build() - ); - operations.saveSchema(topic, schema); - - //when - Response response = publisher.publish("schema.topic.validJson", message); - - //then - assertThat(response).hasStatus(CREATED); - } - - @Test - public void shouldNotPublishInvalidMessageWithJsonSchema() { - // given - String messageInvalidWithSchema = "{\"id\": \"shouldBeNumber\"}"; - Topic topic = operations.buildTopic(topic("schema.topic.invalidJson") - .withValidation(true) - .withContentType(JSON) - .build() - ); - operations.saveSchema(topic, schema); - - //when - Response response = publisher.publish("schema.topic.invalidJson", messageInvalidWithSchema); - - //then - assertThat(response).hasStatus(BAD_REQUEST); - } - - @Test - public void shouldPublishInvalidJsonMessageOnValidationDryRun() { - // given - String invalidMessage = "{\"id\": \"shouldBeNumber\"}"; - Topic topic = operations.buildTopic(topic("schema.topicWithValidationDryRun") - .withValidation(true) - .withValidationDryRun(true) - .withContentType(JSON) - .build() - ); - operations.saveSchema(topic, schema); - - //when - Response response = publisher.publish("schema.topicWithValidationDryRun", invalidMessage); - - //then - assertThat(response).hasStatus(CREATED); - } - @Test public void shouldPublishAndConsumeMessageWithTraceId() { diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/ConfluentSchemaRegistryStarter.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/ConfluentSchemaRegistryStarter.java new file mode 100644 index 0000000000..724eedd201 --- /dev/null +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/ConfluentSchemaRegistryStarter.java @@ -0,0 +1,49 @@ +package pl.allegro.tech.hermes.integration.env; + +import io.confluent.kafka.schemaregistry.rest.SchemaRegistryConfig; +import io.confluent.kafka.schemaregistry.rest.SchemaRegistryRestApplication; +import org.eclipse.jetty.server.Server; +import org.glassfish.jersey.internal.ServiceFinder; +import pl.allegro.tech.hermes.test.helper.environment.Starter; + +import java.util.Properties; + +public class ConfluentSchemaRegistryStarter implements Starter { + + private final Properties properties; + private SchemaRegistryRestApplication restApp; + private Server server; + + public ConfluentSchemaRegistryStarter(int port, String zkConnectionString) { + Properties properties = new Properties(); + properties.put("listeners", "http://0.0.0.0:" + port); + properties.put("kafkastore.connection.url", zkConnectionString); + this.properties = properties; + } + + @Override + public void start() throws Exception { + + // temporarily overriding Jersey's default ServiceIteratorProvider + // in order to filter out SpringComponentProvider (available in classpath) + // which tries to initialize custom application context from an XML file + ServiceFinder.setIteratorProvider(SpringlessServiceIteratorProvider.INSTANCE); + + SchemaRegistryConfig config = new SchemaRegistryConfig(properties); + restApp = new SchemaRegistryRestApplication(config); + server = restApp.createServer(); + server.start(); + + ServiceFinder.setIteratorProvider(null); + } + + @Override + public void stop() throws Exception { + server.stop(); + } + + @Override + public SchemaRegistryRestApplication instance() { + return restApp; + } +} diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/ConsumersStarter.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/ConsumersStarter.java index d61879b1ad..6aae3e631d 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/ConsumersStarter.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/ConsumersStarter.java @@ -16,7 +16,7 @@ import static pl.allegro.tech.hermes.common.config.Configs.KAFKA_CONSUMER_AUTO_OFFSET_RESET_CONFIG; import static pl.allegro.tech.hermes.common.config.Configs.SCHEMA_CACHE_ENABLED; import static pl.allegro.tech.hermes.common.config.Configs.SCHEMA_REPOSITORY_TYPE; -import static pl.allegro.tech.hermes.domain.topic.schema.SchemaRepositoryType.SCHEMA_REPO; +import static pl.allegro.tech.hermes.common.schema.SchemaRepositoryType.SCHEMA_REGISTRY; public class ConsumersStarter implements Starter { @@ -28,7 +28,7 @@ public class ConsumersStarter implements Starter { @Override public void start() throws Exception { LOGGER.info("Starting Hermes Consumers"); - configFactory.overrideProperty(SCHEMA_REPOSITORY_TYPE, SCHEMA_REPO.name()); + configFactory.overrideProperty(SCHEMA_REPOSITORY_TYPE, SCHEMA_REGISTRY.name()); configFactory.overrideProperty(SCHEMA_CACHE_ENABLED, false); configFactory.overrideProperty(KAFKA_CONSUMER_AUTO_OFFSET_RESET_CONFIG, "earliest"); consumers = HermesConsumers.consumers() diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/EnvironmentAware.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/EnvironmentAware.java index ea7e494aaa..f177e295d0 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/EnvironmentAware.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/EnvironmentAware.java @@ -26,13 +26,13 @@ public interface EnvironmentAware { String CLIENT_FRONTEND_URL = "http://localhost:" + FRONTEND_PORT; String FRONTEND_TOPICS_ENDPOINT = FRONTEND_URL + "topics"; - + String FRONTEND_HEALTH_ENDPOINT = FRONTEND_URL + "status/health"; String MANAGEMENT_ENDPOINT_URL = "http://localhost:" + MANAGEMENT_PORT + "/"; String CONSUMER_ENDPOINT_URL = "http://localhost:" + CONFIG_FACTORY.getIntProperty(Configs.CONSUMER_HEALTH_CHECK_PORT) + "/"; - + int GRAPHITE_HTTP_SERVER_PORT = 18089; int GRAPHITE_SERVER_PORT = 18023; diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/FrontendStarter.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/FrontendStarter.java index de07faf1b0..82cedefbff 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/FrontendStarter.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/FrontendStarter.java @@ -20,7 +20,7 @@ import static pl.allegro.tech.hermes.common.config.Configs.FRONTEND_SSL_ENABLED; import static pl.allegro.tech.hermes.common.config.Configs.SCHEMA_CACHE_ENABLED; import static pl.allegro.tech.hermes.common.config.Configs.SCHEMA_REPOSITORY_TYPE; -import static pl.allegro.tech.hermes.domain.topic.schema.SchemaRepositoryType.SCHEMA_REPO; +import static pl.allegro.tech.hermes.common.schema.SchemaRepositoryType.SCHEMA_REGISTRY; public class FrontendStarter implements Starter { @@ -36,7 +36,7 @@ public FrontendStarter(int port) { this.port = port; configFactory = new MutableConfigFactory(); configFactory.overrideProperty(FRONTEND_PORT, port); - configFactory.overrideProperty(SCHEMA_REPOSITORY_TYPE, SCHEMA_REPO.name()); + configFactory.overrideProperty(SCHEMA_REPOSITORY_TYPE, SCHEMA_REGISTRY.name()); configFactory.overrideProperty(SCHEMA_CACHE_ENABLED, false); } diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/HermesIntegrationEnvironment.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/HermesIntegrationEnvironment.java index 57200db050..712a935265 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/HermesIntegrationEnvironment.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/env/HermesIntegrationEnvironment.java @@ -36,10 +36,12 @@ public class HermesIntegrationEnvironment implements EnvironmentAware { STARTERS.put(OAuthServerMockStarter.class, new OAuthServerMockStarter()); STARTERS.put(CustomKafkaStarter.class, new CustomKafkaStarter(SECONDARY_KAFKA_PORT, SECONDARY_ZK_KAFKA_CONNECT)); STARTERS.put(JmsStarter.class, new JmsStarter()); - STARTERS.put(SchemaRepoStarter.class, new SchemaRepoStarter(SCHEMA_REPO_PORT)); + STARTERS.put(ConfluentSchemaRegistryStarter.class, new ConfluentSchemaRegistryStarter(SCHEMA_REPO_PORT, + SECONDARY_ZK_KAFKA_CONNECT)); STARTERS.put(ConsumersStarter.class, new ConsumersStarter()); STARTERS.put(FrontendStarter.class, new FrontendStarter(FRONTEND_PORT)); STARTERS.put(ManagementStarter.class, new ManagementStarter(MANAGEMENT_PORT)); + } private CuratorFramework zookeeper; diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/helper/GraphiteEndpoint.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/helper/GraphiteEndpoint.java index d39c998a14..57fa3931f8 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/helper/GraphiteEndpoint.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/helper/GraphiteEndpoint.java @@ -2,10 +2,11 @@ import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.client.WireMock; +import pl.allegro.tech.hermes.integration.env.EnvironmentAware; + import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; -import pl.allegro.tech.hermes.integration.env.EnvironmentAware; public class GraphiteEndpoint implements EnvironmentAware { diff --git a/integration/src/integration/java/pl/allegro/tech/hermes/integration/management/SchemaManagementTest.java b/integration/src/integration/java/pl/allegro/tech/hermes/integration/management/SchemaManagementTest.java index 4b5cf5082f..61b2b01184 100644 --- a/integration/src/integration/java/pl/allegro/tech/hermes/integration/management/SchemaManagementTest.java +++ b/integration/src/integration/java/pl/allegro/tech/hermes/integration/management/SchemaManagementTest.java @@ -14,6 +14,8 @@ public class SchemaManagementTest extends IntegrationTest { + private static final String EXAMPLE_SCHEMA = "\"string\""; + @Test public void shouldNotSaveSchemaForInvalidTopic() { // given && when @@ -26,10 +28,10 @@ public void shouldNotSaveSchemaForInvalidTopic() { @Test public void shouldSaveSchemaForExistingTopic() { // given - operations.buildTopic("schemaGroup1", "schemaTopic1"); + operations.buildAvroTopic("schemaGroup1", "schemaTopic1"); // when - Response response = management.schema().save("schemaGroup1.schemaTopic1", "{}"); + Response response = management.schema().save("schemaGroup1.schemaTopic1", EXAMPLE_SCHEMA); // then assertThat(response).hasStatus(Response.Status.CREATED); @@ -38,20 +40,20 @@ public void shouldSaveSchemaForExistingTopic() { @Test public void shouldReturnSchemaForTopic() { // given - operations.buildTopic("schemaGroup2", "schemaTopic2"); - management.schema().save("schemaGroup2.schemaTopic2", "{}"); + operations.buildAvroTopic("schemaGroup2", "schemaTopic2"); + management.schema().save("schemaGroup2.schemaTopic2", EXAMPLE_SCHEMA); // when Response response = management.schema().get("schemaGroup2.schemaTopic2"); // then - assertThat(response.readEntity(String.class)).isEqualTo("{}"); + assertThat(response.readEntity(String.class)).isEqualTo(EXAMPLE_SCHEMA); } @Test public void shouldRespondWithNoContentOnMissingSchema() { // given - operations.buildTopic("schemaGroup3", "schemaTopic3"); + operations.buildAvroTopic("schemaGroup3", "schemaTopic3"); // when Response response = management.schema().get("schemaGroup3.schemaTopic3"); @@ -61,19 +63,7 @@ public void shouldRespondWithNoContentOnMissingSchema() { } @Test - public void shouldNotSaveInvalidJsonSchema() throws IOException { - // given - operations.buildTopic("jsonGroup", "jsonTopic"); - - // when - Response response = management.schema().save("jsonGroup.jsonTopic", "{"); - - // then - assertThat(response).hasStatus(Response.Status.BAD_REQUEST); - } - - @Test - public void shouldNotRemoveAvroSchema() throws IOException { + public void shouldReturnMethodNotAcceptableWhenRemovingSchemaIsDisabled() throws IOException { // given AvroUser avroUser = new AvroUser(); Topic avroTopic = topic("avroGroup", "avroTopic") @@ -86,7 +76,7 @@ public void shouldNotRemoveAvroSchema() throws IOException { Response response = management.schema().delete("avroGroup.avroTopic"); // then - assertThat(response).hasStatus(Response.Status.BAD_REQUEST); + assertThat(response).hasStatus(Response.Status.NOT_ACCEPTABLE); } @Test diff --git a/integration/src/integration/resources/application.yaml b/integration/src/integration/resources/application.yaml index 52a021cc47..ecdc68f440 100644 --- a/integration/src/integration/resources/application.yaml +++ b/integration/src/integration/resources/application.yaml @@ -8,4 +8,3 @@ supportTeam: type: crowd crowd: path: http://localhost:18222/crowd - diff --git a/integration/src/test/resources/application-integration.yaml b/integration/src/test/resources/application-integration.yaml index b980e98477..c8280d04ef 100644 --- a/integration/src/test/resources/application-integration.yaml +++ b/integration/src/test/resources/application-integration.yaml @@ -40,4 +40,4 @@ spring: spring.groovy.template.check-template-location: false schema.repository: - type: schema_repo \ No newline at end of file + type: schema_registry \ No newline at end of file diff --git a/integration/src/test/resources/config.properties b/integration/src/test/resources/config.properties index 3d5946f06d..2b6897285a 100644 --- a/integration/src/test/resources/config.properties +++ b/integration/src/test/resources/config.properties @@ -39,6 +39,6 @@ report.period=1 admin.reaper.interval.ms=3000 oauth.server.url=http://localhost:19999 -schema.repository.type=schema_repo +schema.repository.type=schema_registry tracker.enabled=true \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 8704a745fe..f67bdcf5ce 100644 --- a/settings.gradle +++ b/settings.gradle @@ -11,5 +11,6 @@ include 'hermes-common', 'hermes-tracker', 'hermes-tracker-mongo', 'hermes-metrics', - 'hermes-tracker-elasticsearch' + 'hermes-tracker-elasticsearch', + 'hermes-schema'