From f1b0036e43cd352b8b89d00973fe09fb0363f404 Mon Sep 17 00:00:00 2001 From: LiRen Tu Date: Tue, 3 May 2022 16:40:15 -0700 Subject: [PATCH] Fix jitpack (#16) * Bump gradle wrapper * Clean up cache key * Fix non-nullable field * Upgrade spock and groovy * Use java 17 * Remove tool chain config (cannot be set together with source target compatibility * Set java version for all projects * Try toolchain in base * Try toolchain in projects * Revert build.gradle changes * Avoid using java 17 method --- .github/workflows/java_ci.yaml | 4 +- build.gradle | 14 ++--- .../converter/AdditionalPropertyField.java | 3 +- .../converter/JsonGenericRecordReader.java | 53 ++++++++++--------- .../converter/JsonAvroConverterSpec.groovy | 2 +- .../test/resources/json_avro_converter.json | 31 +++++++++++ 6 files changed, 69 insertions(+), 38 deletions(-) diff --git a/.github/workflows/java_ci.yaml b/.github/workflows/java_ci.yaml index d2f27b0..337ab99 100644 --- a/.github/workflows/java_ci.yaml +++ b/.github/workflows/java_ci.yaml @@ -15,7 +15,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v1 with: - java-version: '14' + java-version: '17' - name: Validate Gradle wrapper uses: gradle/wrapper-validation-action@v1 - name: Set up cache @@ -25,8 +25,6 @@ jobs: ~/.gradle/caches ~/.gradle/wrapper key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - name: Build with Gradle run: ./gradlew clean build - name: Cleanup Gradle Cache diff --git a/build.gradle b/build.gradle index c426423..5ff779a 100644 --- a/build.gradle +++ b/build.gradle @@ -15,22 +15,18 @@ scmVersion { } } -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(17)) - } -} - allprojects { project.group = 'tech.allegro.schema.json2avro' project.version = scmVersion.version + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 } subprojects { project.ext.versions = [ - spock: '1.3-groovy-2.5', - groovy: '2.5.10' + spock: '2.1-groovy-3.0', + groovy: '3.0.9' ] repositories { @@ -49,5 +45,5 @@ subprojects { } wrapper { - gradleVersion = '7.2' + gradleVersion = '7.4.2' } diff --git a/converter/src/main/java/tech/allegro/schema/json2avro/converter/AdditionalPropertyField.java b/converter/src/main/java/tech/allegro/schema/json2avro/converter/AdditionalPropertyField.java index 6fe8b2b..82c6d3e 100644 --- a/converter/src/main/java/tech/allegro/schema/json2avro/converter/AdditionalPropertyField.java +++ b/converter/src/main/java/tech/allegro/schema/json2avro/converter/AdditionalPropertyField.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -12,7 +13,7 @@ public final class AdditionalPropertyField { public static final String DEFAULT_AVRO_FIELD_NAME = "_airbyte_additional_properties"; - public static final Set DEFAULT_JSON_FIELD_NAMES = Set.of("_ab_additional_properties", DEFAULT_AVRO_FIELD_NAME); + public static final Set DEFAULT_JSON_FIELD_NAMES = ImmutableSet.of("_ab_additional_properties", DEFAULT_AVRO_FIELD_NAME); public static final Schema FIELD_SCHEMA = Schema.createUnion( Schema.create(Schema.Type.NULL), Schema.createMap(Schema.create(Type.STRING))); diff --git a/converter/src/main/java/tech/allegro/schema/json2avro/converter/JsonGenericRecordReader.java b/converter/src/main/java/tech/allegro/schema/json2avro/converter/JsonGenericRecordReader.java index 6e68cd4..13723ee 100644 --- a/converter/src/main/java/tech/allegro/schema/json2avro/converter/JsonGenericRecordReader.java +++ b/converter/src/main/java/tech/allegro/schema/json2avro/converter/JsonGenericRecordReader.java @@ -1,6 +1,25 @@ package tech.allegro.schema.json2avro.converter; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; +import static tech.allegro.schema.json2avro.converter.AdditionalPropertyField.DEFAULT_AVRO_FIELD_NAME; +import static tech.allegro.schema.json2avro.converter.AdditionalPropertyField.DEFAULT_JSON_FIELD_NAMES; +import static tech.allegro.schema.json2avro.converter.AvroTypeExceptions.enumException; +import static tech.allegro.schema.json2avro.converter.AvroTypeExceptions.typeException; +import static tech.allegro.schema.json2avro.converter.AvroTypeExceptions.unionException; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.avro.AvroRuntimeException; import org.apache.avro.AvroTypeException; @@ -13,25 +32,6 @@ import org.apache.avro.generic.GenericRecordBuilder; import tech.allegro.schema.json2avro.converter.util.DateTimeUtils; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; - -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; -import static tech.allegro.schema.json2avro.converter.AdditionalPropertyField.DEFAULT_AVRO_FIELD_NAME; -import static tech.allegro.schema.json2avro.converter.AdditionalPropertyField.DEFAULT_JSON_FIELD_NAMES; -import static tech.allegro.schema.json2avro.converter.AvroTypeExceptions.enumException; -import static tech.allegro.schema.json2avro.converter.AvroTypeExceptions.typeException; -import static tech.allegro.schema.json2avro.converter.AvroTypeExceptions.unionException; - public class JsonGenericRecordReader { private static final Object INCOMPATIBLE = new Object(); @@ -254,11 +254,16 @@ private Object read(Schema.Field field, Schema schema, Object value, Deque readArray(Schema.Field field, Schema schema, List items, Deque path) { // When all array elements are supposed to be null or string, we enforce array values to be string. // This is to properly handle Json arrays that do not follow the schema. - Set nonNullElementTypes = schema.getElementType() - .getTypes().stream() - .map(Schema::getType) - .filter(t -> t != Type.NULL) - .collect(Collectors.toSet()); + Set nonNullElementTypes; + if (schema.getElementType().isUnion()) { + nonNullElementTypes = schema.getElementType() + .getTypes().stream() + .map(Schema::getType) + .filter(t -> t != Type.NULL) + .collect(Collectors.toSet()); + } else { + nonNullElementTypes = Collections.singleton(schema.getElementType().getType()); + } boolean enforceString = nonNullElementTypes.size() == 1 && nonNullElementTypes.contains(Type.STRING); return items.stream() .map(item -> read(field, schema.getElementType(), item, path, false, enforceString)) diff --git a/converter/src/test/groovy/tech/allegro/schema/json2avro/converter/JsonAvroConverterSpec.groovy b/converter/src/test/groovy/tech/allegro/schema/json2avro/converter/JsonAvroConverterSpec.groovy index 8c79ce1..be69270 100644 --- a/converter/src/test/groovy/tech/allegro/schema/json2avro/converter/JsonAvroConverterSpec.groovy +++ b/converter/src/test/groovy/tech/allegro/schema/json2avro/converter/JsonAvroConverterSpec.groovy @@ -122,7 +122,7 @@ class JsonAvroConverterSpec extends Specification { then: def e = thrown AvroConversionException - e.message == "Failed to convert JSON to Avro: Field field_integer is expected to be type: java.lang.Number" + e.message == "Failed to convert JSON to Avro: Field field_integer is expected to be type: java.lang.Number, but it is: foobar" } def "should ignore unknown fields"() { diff --git a/converter/src/test/resources/json_avro_converter.json b/converter/src/test/resources/json_avro_converter.json index 048a3ec..0a89d2d 100644 --- a/converter/src/test/resources/json_avro_converter.json +++ b/converter/src/test/resources/json_avro_converter.json @@ -1,6 +1,37 @@ [ { "testCase": "simple_schema", + "avroSchema": { + "type": "record", + "name": "test_schema", + "fields": [ + { + "name": "username", + "type": "string" + }, + { + "name": "active", + "type": "boolean" + }, + { + "name": "age", + "type": "int" + } + ] + }, + "jsonObject": { + "username": "airbyte_developer", + "active": true, + "age": 150 + }, + "avroObject": { + "username": "airbyte_developer", + "active": true, + "age": 150 + } + }, + { + "testCase": "nullable_schema", "avroSchema": { "type": "record", "name": "test_schema",