diff --git a/airbyte-config/models/build.gradle b/airbyte-config/models/build.gradle index 9deb096a4832d..68eb849d9dadf 100644 --- a/airbyte-config/models/build.gradle +++ b/airbyte-config/models/build.gradle @@ -1,9 +1,12 @@ +import org.jsonschema2pojo.SourceType + plugins { id "com.github.eirnym.js2p" version "1.0" } jsonSchema2Pojo { - source = files("${sourceSets.main.output.resourcesDir}/json") + sourceType = SourceType.YAMLSCHEMA + source = files("${sourceSets.main.output.resourcesDir}/types") targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') targetPackage = 'io.airbyte.config' diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java b/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java index 9e392f022a3c5..5e520f1b570e3 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java @@ -36,27 +36,27 @@ public enum ConfigSchema { // workspace - STANDARD_WORKSPACE("StandardWorkspace.json"), + STANDARD_WORKSPACE("StandardWorkspace.yaml"), // source - STANDARD_SOURCE("StandardSource.json"), - SOURCE_CONNECTION_SPECIFICATION("SourceConnectionSpecification.json"), - SOURCE_CONNECTION_IMPLEMENTATION("SourceConnectionImplementation.json"), + STANDARD_SOURCE("StandardSource.yaml"), + SOURCE_CONNECTION_SPECIFICATION("SourceConnectionSpecification.yaml"), + SOURCE_CONNECTION_IMPLEMENTATION("SourceConnectionImplementation.yaml"), // destination - STANDARD_DESTINATION("StandardDestination.json"), - DESTINATION_CONNECTION_SPECIFICATION("DestinationConnectionSpecification.json"), - DESTINATION_CONNECTION_IMPLEMENTATION("DestinationConnectionImplementation.json"), + STANDARD_DESTINATION("StandardDestination.yaml"), + DESTINATION_CONNECTION_SPECIFICATION("DestinationConnectionSpecification.yaml"), + DESTINATION_CONNECTION_IMPLEMENTATION("DestinationConnectionImplementation.yaml"), // sync - STANDARD_SYNC("StandardSync.json"), - STANDARD_SYNC_SUMMARY("StandardSyncSummary.json"), - STANDARD_SYNC_SCHEDULE("StandardSyncSchedule.json"), + STANDARD_SYNC("StandardSync.yaml"), + STANDARD_SYNC_SUMMARY("StandardSyncSummary.yaml"), + STANDARD_SYNC_SCHEDULE("StandardSyncSchedule.yaml"), - STATE("State.json"); + STATE("State.yaml"); static final Path KNOWN_SCHEMAS_ROOT = prepareSchemas(); - private static final String RESOURCE_DIR = "json"; + private static final String RESOURCE_DIR = "types"; /* * JsonReferenceProcessor relies on all of the json in consumes being in a file system (not in a @@ -68,7 +68,7 @@ private static Path prepareSchemas() { try { final List filenames = MoreResources.listResources(ConfigSchema.class, RESOURCE_DIR) .map(p -> p.getFileName().toString()) - .filter(p -> p.endsWith(".json")) + .filter(p -> p.endsWith(".yaml")) .collect(Collectors.toList()); final Path configRoot = Files.createTempDirectory("schemas"); diff --git a/airbyte-config/models/src/main/resources/json/DataType.json b/airbyte-config/models/src/main/resources/json/DataType.json deleted file mode 100644 index 2854653881847..0000000000000 --- a/airbyte-config/models/src/main/resources/json/DataType.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/DataType.json", - "title": "DataType", - "description": "standard data types.", - "type": "string", - "enum": ["string", "number", "boolean", "object", "array"] -} diff --git a/airbyte-config/models/src/main/resources/json/DestinationConnectionImplementation.json b/airbyte-config/models/src/main/resources/json/DestinationConnectionImplementation.json deleted file mode 100644 index 3af15cb5f9341..0000000000000 --- a/airbyte-config/models/src/main/resources/json/DestinationConnectionImplementation.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/DestinationConnectionImplementation.json", - "title": "DestinationConnectionImplementation", - "description": "information required for connection to a destination.", - "type": "object", - "required": [ - "name", - "destinationSpecificationId", - "workspaceId", - "destinationImplementationId", - "configuration", - "tombstone" - ], - "additionalProperties": false, - "properties": { - "name": { - "type": "string" - }, - "destinationSpecificationId": { - "type": "string", - "format": "uuid" - }, - "workspaceId": { - "type": "string", - "format": "uuid" - }, - "destinationImplementationId": { - "type": "string", - "format": "uuid" - }, - "configuration": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "tombstone": { - "description": "if not set or false, the configuration is active. if true, then this configuration is permanently off.", - "type": "boolean" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/DestinationConnectionSpecification.json b/airbyte-config/models/src/main/resources/json/DestinationConnectionSpecification.json deleted file mode 100644 index 1d93b20b0d1a4..0000000000000 --- a/airbyte-config/models/src/main/resources/json/DestinationConnectionSpecification.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/DestinationSpecification.json", - "title": "DestinationConnectionSpecification", - "description": "specification for how to configure a connection to a destination", - "type": "object", - "required": ["destinationId", "destinationSpecificationId", "specification"], - "additionalProperties": false, - "properties": { - "destinationId": { - "type": "string", - "format": "uuid" - }, - "destinationSpecificationId": { - "type": "string", - "format": "uuid" - }, - "documentationUrl": { - "type": "string" - }, - "specification": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/JobCheckConnectionConfig.json b/airbyte-config/models/src/main/resources/json/JobCheckConnectionConfig.json deleted file mode 100644 index d30cdef7601c0..0000000000000 --- a/airbyte-config/models/src/main/resources/json/JobCheckConnectionConfig.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/JobCheckConnectionConfig.json", - "title": "JobCheckConnectionConfig", - "description": "job check connection config", - "type": "object", - "additionalProperties": false, - "required": ["connectionConfiguration", "dockerImage"], - "properties": { - "connectionConfiguration": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "dockerImage": { - "type": "string" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/JobConfig.json b/airbyte-config/models/src/main/resources/json/JobConfig.json deleted file mode 100644 index d333b757736f2..0000000000000 --- a/airbyte-config/models/src/main/resources/json/JobConfig.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/JobConfig.json", - "title": "JobConfig", - "description": "job config", - "type": "object", - "additionalProperties": false, - "required": ["configType"], - "properties": { - "configType": { - "type": "string", - "enum": [ - "checkConnectionSource", - "checkConnectionDestination", - "discoverSchema", - "sync" - ] - }, - "checkConnection": { - "$ref": "JobCheckConnectionConfig.json" - }, - "discoverSchema": { - "$ref": "JobDiscoverSchemaConfig.json" - }, - "sync": { - "$ref": "JobSyncConfig.json" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/JobDiscoverSchemaConfig.json b/airbyte-config/models/src/main/resources/json/JobDiscoverSchemaConfig.json deleted file mode 100644 index d30cdef7601c0..0000000000000 --- a/airbyte-config/models/src/main/resources/json/JobDiscoverSchemaConfig.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/JobCheckConnectionConfig.json", - "title": "JobCheckConnectionConfig", - "description": "job check connection config", - "type": "object", - "additionalProperties": false, - "required": ["connectionConfiguration", "dockerImage"], - "properties": { - "connectionConfiguration": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "dockerImage": { - "type": "string" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/JobOutput.json b/airbyte-config/models/src/main/resources/json/JobOutput.json deleted file mode 100644 index cf7189e1cf49f..0000000000000 --- a/airbyte-config/models/src/main/resources/json/JobOutput.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/JobOutput.json", - "title": "JobOutput", - "description": "job output", - "type": "object", - "additionalProperties": false, - "required": ["outputType"], - "properties": { - "outputType": { - "type": "string", - "enum": ["checkConnection", "discoverSchema", "sync"] - }, - "checkConnection": { - "$ref": "StandardCheckConnectionOutput.json" - }, - "discoverSchema": { - "$ref": "StandardDiscoverSchemaOutput.json" - }, - "sync": { - "$ref": "StandardSyncOutput.json" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/JobSyncConfig.json b/airbyte-config/models/src/main/resources/json/JobSyncConfig.json deleted file mode 100644 index 9e998a0738b08..0000000000000 --- a/airbyte-config/models/src/main/resources/json/JobSyncConfig.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/JobSyncConfig.json", - "title": "JobSyncConfig", - "description": "job sync config", - "type": "object", - "additionalProperties": false, - "required": [ - "sourceConnectionImplementation", - "destinationConnectionImplementation", - "standardSync", - "sourceDockerImage", - "destinationDockerImage" - ], - "properties": { - "sourceConnectionImplementation": { - "$ref": "SourceConnectionImplementation.json" - }, - "destinationConnectionImplementation": { - "$ref": "DestinationConnectionImplementation.json" - }, - "standardSync": { - "$ref": "StandardSync.json" - }, - "sourceDockerImage": { - "type": "string" - }, - "destinationDockerImage": { - "type": "string" - }, - "state": { - "description": "optional state of the previous run. this object is defined per integration.", - "$ref": "State.json" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/SourceConnectionImplementation.json b/airbyte-config/models/src/main/resources/json/SourceConnectionImplementation.json deleted file mode 100644 index 351f2a7116802..0000000000000 --- a/airbyte-config/models/src/main/resources/json/SourceConnectionImplementation.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/SourceConnectionConfiguration.json", - "title": "SourceConnectionConfiguration", - "description": "information required for connection to a destination.", - "type": "object", - "required": [ - "name", - "sourceSpecificationId", - "sourceImplementationId", - "workspaceId", - "configuration", - "tombstone" - ], - "additionalProperties": false, - "properties": { - "name": { - "type": "string" - }, - "sourceSpecificationId": { - "type": "string", - "format": "uuid" - }, - "workspaceId": { - "type": "string", - "format": "uuid" - }, - "sourceImplementationId": { - "type": "string", - "format": "uuid" - }, - "configuration": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "tombstone": { - "description": "if not set or false, the configuration is active. if true, then this configuration is permanently off.", - "type": "boolean" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/SourceConnectionSpecification.json b/airbyte-config/models/src/main/resources/json/SourceConnectionSpecification.json deleted file mode 100644 index 996e3caf44512..0000000000000 --- a/airbyte-config/models/src/main/resources/json/SourceConnectionSpecification.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/SourceSpecification.json", - "title": "SourceConnectionSpecification", - "description": "specification for how to configure a connection to a source", - "type": "object", - "required": ["sourceId", "sourceSpecificationId", "specification"], - "additionalProperties": false, - "properties": { - "sourceId": { - "type": "string", - "format": "uuid" - }, - "sourceSpecificationId": { - "type": "string", - "format": "uuid" - }, - "documentationUrl": { - "type": "string" - }, - "changelogUrl": { - "type": "string" - }, - "specification": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardCheckConnectionInput.json b/airbyte-config/models/src/main/resources/json/StandardCheckConnectionInput.json deleted file mode 100644 index f9a6090e9c787..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardCheckConnectionInput.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/PublicCheckConnection.json", - "title": "PublicCheckConnection", - "description": "information required for connection.", - "type": "object", - "required": ["connectionConfiguration"], - "additionalProperties": false, - "properties": { - "connectionConfiguration": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardCheckConnectionOutput.json b/airbyte-config/models/src/main/resources/json/StandardCheckConnectionOutput.json deleted file mode 100644 index 135a4410cd771..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardCheckConnectionOutput.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardCheckConnectionOutput.json", - "title": "StandardCheckConnectionOutput", - "description": "describes the result of a 'check connection' action.", - "type": "object", - "required": ["status"], - "additionalProperties": false, - "properties": { - "status": { - "type": "string", - "enum": ["success", "failure"] - }, - "message": { - "type": "string" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardDataSchema.json b/airbyte-config/models/src/main/resources/json/StandardDataSchema.json deleted file mode 100644 index 0fee6461119cd..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardDataSchema.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardDataSchema.json", - "title": "StandardDataSchema", - "type": "object", - "definitions": { - "schema": { - "description": "describes the available schema.", - "type": "object", - "required": ["streams"], - "additionalProperties": false, - "properties": { - "streams": { - "type": "array", - "items": { - "$ref": "StandardDataSchema.json#/definitions/stream" - } - } - } - }, - "stream": { - "type": "object", - "required": ["name", "fields"], - "additionalProperties": false, - "properties": { - "name": { - "type": "string" - }, - "selected": { - "type": "boolean" - }, - "fields": { - "type": "array", - "items": { - "$ref": "StandardDataSchema.json#/definitions/field" - } - } - } - }, - "field": { - "type": "object", - "required": ["name", "dataType", "selected"], - "additionalProperties": false, - "properties": { - "name": { - "type": "string" - }, - "dataType": { - "$ref": "DataType.json" - }, - "selected": { - "description": "whether or not the field will be replicated.", - "type": "boolean" - } - } - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardDestination.json b/airbyte-config/models/src/main/resources/json/StandardDestination.json deleted file mode 100644 index 6358341e99682..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardDestination.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/Destination.json", - "title": "Source", - "description": "describes a destination", - "type": "object", - "required": ["destinationId", "name"], - "additionalProperties": false, - "properties": { - "destinationId": { - "type": "string", - "format": "uuid" - }, - "name": { - "type": "string" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardDiscoverSchemaInput.json b/airbyte-config/models/src/main/resources/json/StandardDiscoverSchemaInput.json deleted file mode 100644 index 1e57c53152bc2..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardDiscoverSchemaInput.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardDiscoverSchemaOutput.json", - "title": "StandardDiscoverSchemaOutput", - "description": "information required for connection.", - "type": "object", - "required": ["connectionConfiguration"], - "additionalProperties": false, - "properties": { - "connectionConfiguration": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardDiscoverSchemaOutput.json b/airbyte-config/models/src/main/resources/json/StandardDiscoverSchemaOutput.json deleted file mode 100644 index 4fb8229d78751..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardDiscoverSchemaOutput.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardDiscoverSchemaOutput.json", - "title": "StandardDiscoverSchemaOutput", - "description": "describes the standard output for any discovery run.", - "type": "object", - "required": ["schema"], - "additionalProperties": false, - "properties": { - "schema": { - "description": "describes the available schema.", - "$ref": "StandardDataSchema.json#/definitions/schema" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardSource.json b/airbyte-config/models/src/main/resources/json/StandardSource.json deleted file mode 100644 index e557fb41fece2..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardSource.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/Source.json", - "title": "Source", - "description": "describes a source", - "type": "object", - "required": ["sourceId", "name"], - "additionalProperties": false, - "properties": { - "sourceId": { - "type": "string", - "format": "uuid" - }, - "name": { - "type": "string" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardSync.json b/airbyte-config/models/src/main/resources/json/StandardSync.json deleted file mode 100644 index 6f10d0907e6c2..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardSync.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardSync.json", - "title": "StandardSync", - "description": "configuration required for sync for ALL taps", - "type": "object", - "required": [ - "sourceImplementationId", - "destinationImplementationId", - "name", - "syncMode", - "schema" - ], - "additionalProperties": false, - "properties": { - "sourceImplementationId": { - "type": "string", - "format": "uuid" - }, - "destinationImplementationId": { - "type": "string", - "format": "uuid" - }, - "connectionId": { - "type": "string", - "format": "uuid" - }, - "name": { - "type": "string" - }, - "syncMode": { - "type": "string", - "enum": ["full_refresh", "append"] - }, - "schema": { - "$ref": "StandardDataSchema.json#/definitions/schema" - }, - "status": { - "type": "string", - "enum": ["active", "inactive", "deprecated"] - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardSyncInput.json b/airbyte-config/models/src/main/resources/json/StandardSyncInput.json deleted file mode 100644 index 4dcb25e26d946..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardSyncInput.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/JobSyncConfig.json", - "title": "JobSyncConfig", - "description": "job sync config", - "type": "object", - "additionalProperties": false, - "required": [ - "sourceConnectionImplementation", - "destinationConnectionImplementation", - "standardSync", - "state" - ], - "properties": { - "sourceConnectionImplementation": { - "$ref": "SourceConnectionImplementation.json" - }, - "destinationConnectionImplementation": { - "$ref": "DestinationConnectionImplementation.json" - }, - "standardSync": { - "$ref": "StandardSync.json" - }, - "state": { - "$ref": "State.json" - }, - "standardSyncSummary": { - "description": "optional state of the previous run. this object is standard for any sync run.", - "$ref": "StandardSyncSummary.json" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardSyncOutput.json b/airbyte-config/models/src/main/resources/json/StandardSyncOutput.json deleted file mode 100644 index f88335edb07a0..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardSyncOutput.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardSyncOutput.json", - "title": "StandardSyncOutput", - "description": "job sync config", - "type": "object", - "additionalProperties": false, - "required": ["standardSyncSummary", "state"], - "properties": { - "standardSyncSummary": { - "$ref": "StandardSyncSummary.json" - }, - "state": { - "$ref": "State.json" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardSyncSchedule.json b/airbyte-config/models/src/main/resources/json/StandardSyncSchedule.json deleted file mode 100644 index 4fe5015582d26..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardSyncSchedule.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardSyncSchedule.json", - "description": "ideally this should be a union but java codegen does not handle the union type properly.", - "title": "StandardScheduleConfiguration", - "type": "object", - "required": ["manual", "connectionId"], - "additionalProperties": false, - "properties": { - "connectionId": { - "type": "string", - "format": "uuid" - }, - "schedule": { - "type": "object", - "required": ["timeUnit", "units"], - "additionalProperties": false, - "properties": { - "timeUnit": { - "type": "string", - "enum": ["minutes", "hours", "days", "weeks", "months"] - }, - "units": { - "type": "integer" - } - } - }, - "manual": { - "type": "boolean" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardSyncSummary.json b/airbyte-config/models/src/main/resources/json/StandardSyncSummary.json deleted file mode 100644 index 77e71f0f4d923..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardSyncSummary.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardSyncSummary.json", - "title": "StandardSyncSummary", - "description": "standard information output by ALL taps for a sync step (our version of state.json)", - "type": "object", - "required": ["jobId", "recordsSynced", "status", "startTime", "endTime"], - "additionalProperties": false, - "properties": { - "status": { - "type": "string", - "enum": ["completed", "failed", "cancelled"] - }, - "recordsSynced": { - "type": "integer", - "minValue": 0 - }, - "startTime": { - "type": "integer" - }, - "endTime": { - "type": "integer" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardTapConfig.json b/airbyte-config/models/src/main/resources/json/StandardTapConfig.json deleted file mode 100644 index e64060b04b09e..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardTapConfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardTapConfig.json", - "title": "StandardTapConfig", - "description": "StandardTapConfig", - "type": "object", - "additionalProperties": false, - "required": ["sourceConnectionImplementation", "standardSync"], - "properties": { - "sourceConnectionImplementation": { - "$ref": "SourceConnectionImplementation.json" - }, - "standardSync": { - "$ref": "StandardSync.json" - }, - "state": { - "$ref": "State.json" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardTargetConfig.json b/airbyte-config/models/src/main/resources/json/StandardTargetConfig.json deleted file mode 100644 index d7de7d0ee089c..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardTargetConfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardTargetConfig.json", - "title": "StandardTargetConfig", - "description": "StandardTargetConfig", - "type": "object", - "additionalProperties": false, - "required": [ - "sourceConnectionImplementation", - "destinationConnectionImplementation", - "standardSync" - ], - "properties": { - "destinationConnectionImplementation": { - "$ref": "DestinationConnectionImplementation.json" - }, - "standardSync": { - "$ref": "StandardSync.json" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/StandardWorkspace.json b/airbyte-config/models/src/main/resources/json/StandardWorkspace.json deleted file mode 100644 index 8496e63aeea7c..0000000000000 --- a/airbyte-config/models/src/main/resources/json/StandardWorkspace.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/StandardWorkspaceConfiguration.json", - "title": "StandardWorkspace", - "description": "workspace configuration", - "type": "object", - "required": [ - "workspaceId", - "name", - "slug", - "initialSetupComplete", - "onboardingComplete" - ], - "additionalProperties": false, - "properties": { - "workspaceId": { - "type": "string", - "format": "uuid" - }, - "customerId": { - "type": "string", - "format": "uuid" - }, - "name": { - "type": "string" - }, - "slug": { - "type": "string" - }, - "email": { - "type": "string", - "format": "email" - }, - "initialSetupComplete": { - "type": "boolean" - }, - "onboardingComplete": { - "type": "boolean" - }, - "anonymousDataCollection": { - "type": "boolean" - }, - "news": { - "type": "boolean" - }, - "securityUpdates": { - "type": "boolean" - } - } -} diff --git a/airbyte-config/models/src/main/resources/json/State.json b/airbyte-config/models/src/main/resources/json/State.json deleted file mode 100644 index 7ef9ca178f19a..0000000000000 --- a/airbyte-config/models/src/main/resources/json/State.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/json/State.json", - "title": "State", - "description": "information output by the connection.", - "type": "object", - "required": ["connectionId", "state"], - "additionalProperties": false, - "properties": { - "connectionId": { - "type": "string", - "format": "uuid" - }, - "state": { - "description": "Integration specific blob. Must be a valid JSON string.", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - } - } -} diff --git a/airbyte-config/models/src/main/resources/types/DataType.yaml b/airbyte-config/models/src/main/resources/types/DataType.yaml new file mode 100644 index 0000000000000..a0b3d5d23a381 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/DataType.yaml @@ -0,0 +1,12 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/DataType.yaml +title: DataType +description: standard data types. +type: string +enum: + - string + - number + - boolean + - object + - array diff --git a/airbyte-config/models/src/main/resources/types/DestinationConnectionImplementation.yaml b/airbyte-config/models/src/main/resources/types/DestinationConnectionImplementation.yaml new file mode 100644 index 0000000000000..bf2aa53caa4e9 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/DestinationConnectionImplementation.yaml @@ -0,0 +1,35 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/DestinationConnectionImplementation.yaml +title: DestinationConnectionImplementation +description: information required for connection to a destination. +type: object +required: + - name + - destinationSpecificationId + - workspaceId + - destinationImplementationId + - configuration + - tombstone +additionalProperties: false +properties: + name: + type: string + destinationSpecificationId: + type: string + format: uuid + workspaceId: + type: string + format: uuid + destinationImplementationId: + type: string + format: uuid + configuration: + description: Integration specific blob. Must be a valid JSON string. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + tombstone: + description: + if not set or false, the configuration is active. if true, then this + configuration is permanently off. + type: boolean diff --git a/airbyte-config/models/src/main/resources/types/DestinationConnectionSpecification.yaml b/airbyte-config/models/src/main/resources/types/DestinationConnectionSpecification.yaml new file mode 100644 index 0000000000000..dd6945f54c150 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/DestinationConnectionSpecification.yaml @@ -0,0 +1,24 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/DestinationSpecification.yaml +title: DestinationConnectionSpecification +description: specification for how to configure a connection to a destination +type: object +required: + - destinationId + - destinationSpecificationId + - specification +additionalProperties: false +properties: + destinationId: + type: string + format: uuid + destinationSpecificationId: + type: string + format: uuid + documentationUrl: + type: string + specification: + description: Integration specific blob. Must be a valid JSON string. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-config/models/src/main/resources/types/JobCheckConnectionConfig.yaml b/airbyte-config/models/src/main/resources/types/JobCheckConnectionConfig.yaml new file mode 100644 index 0000000000000..15592f0a9a298 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/JobCheckConnectionConfig.yaml @@ -0,0 +1,17 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/JobCheckConnectionConfig.yaml +title: JobCheckConnectionConfig +description: job check connection config +type: object +additionalProperties: false +required: + - connectionConfiguration + - dockerImage +properties: + connectionConfiguration: + description: Integration specific blob. Must be a valid JSON string. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + dockerImage: + type: string diff --git a/airbyte-config/models/src/main/resources/types/JobConfig.yaml b/airbyte-config/models/src/main/resources/types/JobConfig.yaml new file mode 100644 index 0000000000000..013f544880023 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/JobConfig.yaml @@ -0,0 +1,23 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/JobConfig.yaml +title: JobConfig +description: job config +type: object +additionalProperties: false +required: + - configType +properties: + configType: + type: string + enum: + - checkConnectionSource + - checkConnectionDestination + - discoverSchema + - sync + checkConnection: + "$ref": JobCheckConnectionConfig.yaml + discoverSchema: + "$ref": JobDiscoverSchemaConfig.yaml + sync: + "$ref": JobSyncConfig.yaml diff --git a/airbyte-config/models/src/main/resources/types/JobDiscoverSchemaConfig.yaml b/airbyte-config/models/src/main/resources/types/JobDiscoverSchemaConfig.yaml new file mode 100644 index 0000000000000..15592f0a9a298 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/JobDiscoverSchemaConfig.yaml @@ -0,0 +1,17 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/JobCheckConnectionConfig.yaml +title: JobCheckConnectionConfig +description: job check connection config +type: object +additionalProperties: false +required: + - connectionConfiguration + - dockerImage +properties: + connectionConfiguration: + description: Integration specific blob. Must be a valid JSON string. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + dockerImage: + type: string diff --git a/airbyte-config/models/src/main/resources/types/JobOutput.yaml b/airbyte-config/models/src/main/resources/types/JobOutput.yaml new file mode 100644 index 0000000000000..a7caf0759899f --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/JobOutput.yaml @@ -0,0 +1,22 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/JobOutput.yaml +title: JobOutput +description: job output +type: object +additionalProperties: false +required: + - outputType +properties: + outputType: + type: string + enum: + - checkConnection + - discoverSchema + - sync + checkConnection: + "$ref": StandardCheckConnectionOutput.yaml + discoverSchema: + "$ref": StandardDiscoverSchemaOutput.yaml + sync: + "$ref": StandardSyncOutput.yaml diff --git a/airbyte-config/models/src/main/resources/types/JobSyncConfig.yaml b/airbyte-config/models/src/main/resources/types/JobSyncConfig.yaml new file mode 100644 index 0000000000000..9a1d0b1d64e1b --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/JobSyncConfig.yaml @@ -0,0 +1,27 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/JobSyncConfig.yaml +title: JobSyncConfig +description: job sync config +type: object +additionalProperties: false +required: + - sourceConnectionImplementation + - destinationConnectionImplementation + - standardSync + - sourceDockerImage + - destinationDockerImage +properties: + sourceConnectionImplementation: + "$ref": SourceConnectionImplementation.yaml + destinationConnectionImplementation: + "$ref": DestinationConnectionImplementation.yaml + standardSync: + "$ref": StandardSync.yaml + sourceDockerImage: + type: string + destinationDockerImage: + type: string + state: + description: optional state of the previous run. this object is defined per integration. + "$ref": State.yaml diff --git a/airbyte-config/models/src/main/resources/types/SourceConnectionImplementation.yaml b/airbyte-config/models/src/main/resources/types/SourceConnectionImplementation.yaml new file mode 100644 index 0000000000000..c282aba7fac17 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/SourceConnectionImplementation.yaml @@ -0,0 +1,35 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/SourceConnectionConfiguration.yaml +title: SourceConnectionConfiguration +description: information required for connection to a destination. +type: object +required: + - name + - sourceSpecificationId + - sourceImplementationId + - workspaceId + - configuration + - tombstone +additionalProperties: false +properties: + name: + type: string + sourceSpecificationId: + type: string + format: uuid + workspaceId: + type: string + format: uuid + sourceImplementationId: + type: string + format: uuid + configuration: + description: Integration specific blob. Must be a valid JSON string. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + tombstone: + description: + if not set or false, the configuration is active. if true, then this + configuration is permanently off. + type: boolean diff --git a/airbyte-config/models/src/main/resources/types/SourceConnectionSpecification.yaml b/airbyte-config/models/src/main/resources/types/SourceConnectionSpecification.yaml new file mode 100644 index 0000000000000..65a5455921ef9 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/SourceConnectionSpecification.yaml @@ -0,0 +1,26 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/SourceSpecification.yaml +title: SourceConnectionSpecification +description: specification for how to configure a connection to a source +type: object +required: + - sourceId + - sourceSpecificationId + - specification +additionalProperties: false +properties: + sourceId: + type: string + format: uuid + sourceSpecificationId: + type: string + format: uuid + documentationUrl: + type: string + changelogUrl: + type: string + specification: + description: Integration specific blob. Must be a valid JSON string. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-config/models/src/main/resources/types/StandardCheckConnectionInput.yaml b/airbyte-config/models/src/main/resources/types/StandardCheckConnectionInput.yaml new file mode 100644 index 0000000000000..46a1dee503f06 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardCheckConnectionInput.yaml @@ -0,0 +1,14 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/PublicCheckConnection.yaml +title: PublicCheckConnection +description: information required for connection. +type: object +required: + - connectionConfiguration +additionalProperties: false +properties: + connectionConfiguration: + description: Integration specific blob. Must be a valid JSON string. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-config/models/src/main/resources/types/StandardCheckConnectionOutput.yaml b/airbyte-config/models/src/main/resources/types/StandardCheckConnectionOutput.yaml new file mode 100644 index 0000000000000..3f63081eb9f56 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardCheckConnectionOutput.yaml @@ -0,0 +1,17 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardCheckConnectionOutput.yaml +title: StandardCheckConnectionOutput +description: describes the result of a 'check connection' action. +type: object +required: + - status +additionalProperties: false +properties: + status: + type: string + enum: + - success + - failure + message: + type: string diff --git a/airbyte-config/models/src/main/resources/types/StandardDataSchema.yaml b/airbyte-config/models/src/main/resources/types/StandardDataSchema.yaml new file mode 100644 index 0000000000000..ac70fc612241a --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardDataSchema.yaml @@ -0,0 +1,47 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardDataSchema.yaml +title: StandardDataSchema +type: object +definitions: + schema: + description: describes the available schema. + type: object + required: + - streams + additionalProperties: false + properties: + streams: + type: array + items: + "$ref": "#/definitions/stream" + stream: + type: object + required: + - name + - fields + additionalProperties: false + properties: + name: + type: string + selected: + type: boolean + fields: + type: array + items: + "$ref": "#/definitions/field" + field: + type: object + required: + - name + - dataType + - selected + additionalProperties: false + properties: + name: + type: string + dataType: + "$ref": DataType.yaml + selected: + description: whether or not the field will be replicated. + type: boolean diff --git a/airbyte-config/models/src/main/resources/types/StandardDestination.yaml b/airbyte-config/models/src/main/resources/types/StandardDestination.yaml new file mode 100644 index 0000000000000..5c584dc6a2559 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardDestination.yaml @@ -0,0 +1,16 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/Destination.yaml +title: Source +description: describes a destination +type: object +required: + - destinationId + - name +additionalProperties: false +properties: + destinationId: + type: string + format: uuid + name: + type: string diff --git a/airbyte-config/models/src/main/resources/types/StandardDiscoverSchemaInput.yaml b/airbyte-config/models/src/main/resources/types/StandardDiscoverSchemaInput.yaml new file mode 100644 index 0000000000000..032088f5d7d78 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardDiscoverSchemaInput.yaml @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardDiscoverSchemaOutput.yaml", + "title": "StandardDiscoverSchemaOutput", + "description": "information required for connection.", + "type": "object", + "required": ["connectionConfiguration"], + "additionalProperties": false, + "properties": + { + "connectionConfiguration": + { + "description": "Integration specific blob. Must be a valid JSON string.", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode", + }, + }, +} diff --git a/airbyte-config/models/src/main/resources/types/StandardDiscoverSchemaOutput.yaml b/airbyte-config/models/src/main/resources/types/StandardDiscoverSchemaOutput.yaml new file mode 100644 index 0000000000000..42536b5c63d56 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardDiscoverSchemaOutput.yaml @@ -0,0 +1,13 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardDiscoverSchemaOutput.yaml +title: StandardDiscoverSchemaOutput +description: describes the standard output for any discovery run. +type: object +required: + - schema +additionalProperties: false +properties: + schema: + description: describes the available schema. + "$ref": StandardDataSchema.yaml#/definitions/schema diff --git a/airbyte-config/models/src/main/resources/types/StandardSource.yaml b/airbyte-config/models/src/main/resources/types/StandardSource.yaml new file mode 100644 index 0000000000000..cfca7f8dd1c51 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardSource.yaml @@ -0,0 +1,16 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/Source.yaml +title: Source +description: describes a source +type: object +required: + - sourceId + - name +additionalProperties: false +properties: + sourceId: + type: string + format: uuid + name: + type: string diff --git a/airbyte-config/models/src/main/resources/types/StandardSync.yaml b/airbyte-config/models/src/main/resources/types/StandardSync.yaml new file mode 100644 index 0000000000000..3ba1a9f8e1e59 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardSync.yaml @@ -0,0 +1,38 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardSync.yaml +title: StandardSync +description: configuration required for sync for ALL taps +type: object +required: + - sourceImplementationId + - destinationImplementationId + - name + - syncMode + - schema +additionalProperties: false +properties: + sourceImplementationId: + type: string + format: uuid + destinationImplementationId: + type: string + format: uuid + connectionId: + type: string + format: uuid + name: + type: string + syncMode: + type: string + enum: + - full_refresh + - append + schema: + "$ref": StandardDataSchema.yaml#/definitions/schema + status: + type: string + enum: + - active + - inactive + - deprecated diff --git a/airbyte-config/models/src/main/resources/types/StandardSyncInput.yaml b/airbyte-config/models/src/main/resources/types/StandardSyncInput.yaml new file mode 100644 index 0000000000000..b4a7e8e171d05 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardSyncInput.yaml @@ -0,0 +1,26 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/JobSyncConfig.yaml +title: JobSyncConfig +description: job sync config +type: object +additionalProperties: false +required: + - sourceConnectionImplementation + - destinationConnectionImplementation + - standardSync + - state +properties: + sourceConnectionImplementation: + "$ref": SourceConnectionImplementation.yaml + destinationConnectionImplementation: + "$ref": DestinationConnectionImplementation.yaml + standardSync: + "$ref": StandardSync.yaml + state: + "$ref": State.yaml + standardSyncSummary: + description: + optional state of the previous run. this object is standard for any + sync run. + "$ref": StandardSyncSummary.yaml diff --git a/airbyte-config/models/src/main/resources/types/StandardSyncOutput.yaml b/airbyte-config/models/src/main/resources/types/StandardSyncOutput.yaml new file mode 100644 index 0000000000000..492fcd89a23d3 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardSyncOutput.yaml @@ -0,0 +1,15 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardSyncOutput.yaml +title: StandardSyncOutput +description: job sync config +type: object +additionalProperties: false +required: + - standardSyncSummary + - state +properties: + standardSyncSummary: + "$ref": StandardSyncSummary.yaml + state: + "$ref": State.yaml diff --git a/airbyte-config/models/src/main/resources/types/StandardSyncSchedule.yaml b/airbyte-config/models/src/main/resources/types/StandardSyncSchedule.yaml new file mode 100644 index 0000000000000..a144afdecc8c1 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardSyncSchedule.yaml @@ -0,0 +1,35 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardSyncSchedule.yaml +description: + ideally this should be a union but java codegen does not handle the union + type properly. +title: StandardScheduleConfiguration +type: object +required: + - manual + - connectionId +additionalProperties: false +properties: + connectionId: + type: string + format: uuid + schedule: + type: object + required: + - timeUnit + - units + additionalProperties: false + properties: + timeUnit: + type: string + enum: + - minutes + - hours + - days + - weeks + - months + units: + type: integer + manual: + type: boolean diff --git a/airbyte-config/models/src/main/resources/types/StandardSyncSummary.yaml b/airbyte-config/models/src/main/resources/types/StandardSyncSummary.yaml new file mode 100644 index 0000000000000..e8475b4f783c9 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardSyncSummary.yaml @@ -0,0 +1,29 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardSyncSummary.yaml +title: StandardSyncSummary +description: + standard information output by ALL taps for a sync step (our version + of state.json) +type: object +required: + - jobId + - recordsSynced + - status + - startTime + - endTime +additionalProperties: false +properties: + status: + type: string + enum: + - completed + - failed + - cancelled + recordsSynced: + type: integer + minValue: 0 + startTime: + type: integer + endTime: + type: integer diff --git a/airbyte-config/models/src/main/resources/types/StandardTapConfig.yaml b/airbyte-config/models/src/main/resources/types/StandardTapConfig.yaml new file mode 100644 index 0000000000000..aaf009ed336c6 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardTapConfig.yaml @@ -0,0 +1,17 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardTapConfig.yaml +title: StandardTapConfig +description: StandardTapConfig +type: object +additionalProperties: false +required: + - sourceConnectionImplementation + - standardSync +properties: + sourceConnectionImplementation: + "$ref": SourceConnectionImplementation.yaml + standardSync: + "$ref": StandardSync.yaml + state: + "$ref": State.yaml diff --git a/airbyte-config/models/src/main/resources/types/StandardTargetConfig.yaml b/airbyte-config/models/src/main/resources/types/StandardTargetConfig.yaml new file mode 100644 index 0000000000000..a15b07d4b63a4 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardTargetConfig.yaml @@ -0,0 +1,16 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardTargetConfig.yaml +title: StandardTargetConfig +description: StandardTargetConfig +type: object +additionalProperties: false +required: + - sourceConnectionImplementation + - destinationConnectionImplementation + - standardSync +properties: + destinationConnectionImplementation: + "$ref": DestinationConnectionImplementation.yaml + standardSync: + "$ref": StandardSync.yaml diff --git a/airbyte-config/models/src/main/resources/types/StandardWorkspace.yaml b/airbyte-config/models/src/main/resources/types/StandardWorkspace.yaml new file mode 100644 index 0000000000000..660d09e63e803 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/StandardWorkspace.yaml @@ -0,0 +1,37 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardWorkspaceConfiguration.yaml +title: StandardWorkspace +description: workspace configuration +type: object +required: + - workspaceId + - name + - slug + - initialSetupComplete + - onboardingComplete +additionalProperties: false +properties: + workspaceId: + type: string + format: uuid + customerId: + type: string + format: uuid + name: + type: string + slug: + type: string + email: + type: string + format: email + initialSetupComplete: + type: boolean + onboardingComplete: + type: boolean + anonymousDataCollection: + type: boolean + news: + type: boolean + securityUpdates: + type: boolean diff --git a/airbyte-config/models/src/main/resources/types/State.yaml b/airbyte-config/models/src/main/resources/types/State.yaml new file mode 100644 index 0000000000000..fa41c0f7877d3 --- /dev/null +++ b/airbyte-config/models/src/main/resources/types/State.yaml @@ -0,0 +1,18 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/State.yaml +title: State +description: information output by the connection. +type: object +required: + - connectionId + - state +additionalProperties: false +properties: + connectionId: + type: string + format: uuid + state: + description: Integration specific blob. Must be a valid JSON string. + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_schema.yaml b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_catalog.yaml similarity index 92% rename from airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_schema.yaml rename to airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_catalog.yaml index 3b9bc2a2f85af..f9ad94612d8c4 100644 --- a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_schema.yaml +++ b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_catalog.yaml @@ -11,8 +11,7 @@ properties: streams: type: array items: - "$ref": airbyte_schema.yaml#/definitions/AirbyteStream - + "$ref": "#/definitions/AirbyteStream" definitions: AirbyteStream: type: object diff --git a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_message.yaml b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_message.yaml index a1ca3e478a95a..0f9b9dd16cdda 100644 --- a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_message.yaml +++ b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_message.yaml @@ -24,7 +24,6 @@ properties: log: description: "log message: any kind of logging you want the platform to know about." "$ref": "#/definitions/AirbyteLogMessage" - definitions: AirbyteRecordMessage: type: object @@ -41,7 +40,6 @@ definitions: description: "when the data was emitted from the source." type: string format: date-time - AirbyteStateMessage: type: object additionalProperties: false @@ -52,7 +50,6 @@ definitions: description: "the state data" type: object existingJavaType: com.fasterxml.jackson.databind.JsonNode - AirbyteLogMessage: type: object additionalProperties: false diff --git a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_types.yaml b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_type.yaml similarity index 100% rename from airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_types.yaml rename to airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_type.yaml diff --git a/airbyte-singer/build.gradle b/airbyte-singer/build.gradle index b3f8d42ef20ce..d30c79c9bd797 100644 --- a/airbyte-singer/build.gradle +++ b/airbyte-singer/build.gradle @@ -1,3 +1,5 @@ +import org.jsonschema2pojo.SourceType + plugins { id "com.github.eirnym.js2p" version "1.0" } @@ -11,7 +13,8 @@ jsonSchema2Pojo { // otherwise they clobber each other, when we are accessing them as java resources. // we need to find a more graceful way of handling this issue, but for now this keeps // us moving forward. - source = files("${sourceSets.main.output.resourcesDir}/singer_json") + sourceType = SourceType.YAMLSCHEMA + source = files("${sourceSets.main.output.resourcesDir}/singer_types") targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') targetPackage = 'io.airbyte.singer' diff --git a/airbyte-singer/src/main/java/io/airbyte/singer/SingerConfigSchema.java b/airbyte-singer/src/main/java/io/airbyte/singer/SingerConfigSchema.java index c0cfd98d01364..1e33370107056 100644 --- a/airbyte-singer/src/main/java/io/airbyte/singer/SingerConfigSchema.java +++ b/airbyte-singer/src/main/java/io/airbyte/singer/SingerConfigSchema.java @@ -36,10 +36,10 @@ // todo (cgardens) - dedupe this with ConfigSchema. public enum SingerConfigSchema { - SINGER_MESSAGE("SingerMessage.json"); + SINGER_MESSAGE("SingerMessage.yaml"); static final Path KNOWN_SCHEMAS_ROOT = prepareSchemas(); - private static final String RESOURCE_DIR = "singer_json"; + private static final String RESOURCE_DIR = "singer_types"; /* * JsonReferenceProcessor relies on all of the json in consumes being in a file system (not in a @@ -51,7 +51,8 @@ private static Path prepareSchemas() { try { final List filenames = MoreResources.listResources(SingerConfigSchema.class, RESOURCE_DIR) .map(p -> p.getFileName().toString()) - .filter(p -> p.endsWith(".json")) + .peek(x -> System.out.println("x = " + x)) + .filter(p -> p.endsWith(".yaml")) .collect(Collectors.toList()); final Path configRoot = Files.createTempDirectory("schemas"); diff --git a/airbyte-singer/src/main/resources/singer_json/SingerCatalog.json b/airbyte-singer/src/main/resources/singer_json/SingerCatalog.json deleted file mode 100644 index 323e180e8ee24..0000000000000 --- a/airbyte-singer/src/main/resources/singer_json/SingerCatalog.json +++ /dev/null @@ -1,232 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-singer/src/main/resources/singer_json/SingerCatalog.json", - "title": "SingerCatalog", - "description": "the schema of the singer catalog", - "type": "object", - "additionalProperties": false, - "required": ["streams"], - "properties": { - "streams": { - "type": "array", - "items": { - "$ref": "SingerCatalog.json#/definitions/SingerStream" - } - } - }, - "definitions": { - "SingerStream": { - "type": "object", - "additionalProperties": false, - "required": ["tap_stream_id", "stream", "schema"], - "properties": { - "stream": { - "type": "string", - "description": "The name of the stream." - }, - "tap_stream_id": { - "type": "string", - "description": "The unique identifier for the stream. This is allowed to be different from the name of the stream in order to allow for sources that have duplicate stream names." - }, - "schema": { - "$ref": "SingerCatalog.json#/definitions/SingerTableSchema" - }, - "table_name": { - "type": "string", - "description": "For a database source, the name of the table." - }, - "metadata": { - "type": "array", - "items": { - "$ref": "SingerCatalog.json#/definitions/SingerMetadata" - } - }, - "key_properties": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "SingerTableSchema": { - "type": "object", - "additionalProperties": false, - "required": ["properties"], - "description": "The JSON schema for the stream. This struct is weird because it's essentially modeling the properties field of a jsonschema object.", - "properties": { - "type": { - "$ref": "SingerCatalog.json#/definitions/SingerType" - }, - "properties": { - "$ref": "SingerCatalog.json#/definitions/SingerColumnMap" - }, - "definitions": { - "description": "placeholder for definitions that are included since it is all jsonchema." - } - } - }, - "SingerColumnMap": { - "type": "object", - "additionalProperties": { - "$ref": "SingerCatalog.json#/definitions/SingerColumn" - } - }, - "SingerColumn": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "$ref": "SingerCatalog.json#/definitions/SingerType" - }, - "format": { - "type": "string" - }, - "selected": { - "description": "Deprecated: Some legacy Taps handle stream and field selection by looking for \"selected\": true directly in the stream's schema in the catalog.json file (called properties.json in legacy taps).", - "type": "boolean" - }, - "minimum": { - "type": "integer" - }, - "maximum": { - "type": "integer" - }, - "maxLength": { - "type": "integer" - }, - "properties": { - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "items": { - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "anyOf": { - "type": "array", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "allOf": { - "type": "array", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "oneOf": { - "type": "array", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "not": { - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - } - } - }, - "SingerMetadata": { - "type": "object", - "additionalProperties": false, - "required": ["metadata", "breadcrumb"], - "properties": { - "metadata": { - "$ref": "SingerCatalog.json#/definitions/SingerMetadataChild" - }, - "breadcrumb": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "SingerMetadataChild": { - "type": "object", - "additionalProperties": false, - "required": [], - "properties": { - "selected": { - "description": "Non-Discoverable: Indicates that this node in the schema has been selected by the user for replication.", - "type": "boolean" - }, - "replication-method": { - "description": "Non-Discoverable: Either FULL_TABLE, INCREMENTAL, or LOG_BASED. The replication method to use for a stream.", - "type": "string", - "enum": ["FULL_TABLE", "INCREMENTAL", "LOG_BASED"] - }, - "replication-key": { - "description": "Non-Discoverable: The name of a property in the source to use as a \"bookmark\". For example, this will often be an \"updated-at\" field or an auto-incrementing primary key (requires replication-method", - "type": "array", - "items": { - "type": "string" - } - }, - "view-key-properties": { - "description": "Non Discoverable: List of key properties for a database view.", - "type": "array", - "items": { - "type": "string" - } - }, - "inclusion": { - "description": "Discoverable: available means the field is available for selection, and the tap will only emit values for that field if it is marked with \"selected\": true. automatic means that the tap will emit values for the field. unsupported means that the field exists in the source data but the tap is unable to provide it.", - "type": "string", - "enum": ["available", "automatic", "unsupported"] - }, - "selected-by-default": { - "description": "Discoverable: Indicates if a node in the schema should be replicated if a user has not expressed any opinion on whether or not to replicate it.", - "type": "boolean" - }, - "valid-replication-keys": { - "description": "Discoverable: List of the fields that could be used as replication keys.", - "type": "array", - "items": { - "type": "string" - } - }, - "forced-replication-method": { - "description": "Discoverable: Used to force the replication method to either FULL_TABLE or INCREMENTAL.", - "type": "string" - }, - "table-key-properties": { - "description": "Discoverable: List of key properties for a database table.", - "type": "array", - "items": { - "type": "string" - } - }, - "schema-name": { - "description": "Discoverable: The name of the stream.", - "type": "string" - }, - "is-view": { - "description": "Discoverable: Indicates whether a stream corresponds to a database view.", - "type": "boolean" - }, - "row-count": { - "description": "Discoverable: Number of rows in a database table/view.", - "type": "integer" - }, - "database-name": { - "description": "Discoverable: Name of database.", - "type": "string" - }, - "sql-datatype": { - "description": "Discoverable: Represents the datatype of a database column." - } - } - }, - "SingerType": { - "type": ["array", "string"], - "items": { - "type": "string", - "enum": [ - "null", - "boolean", - "object", - "array", - "number", - "string", - "integer" - ] - } - } - } -} diff --git a/airbyte-singer/src/main/resources/singer_json/SingerMessage.json b/airbyte-singer/src/main/resources/singer_json/SingerMessage.json deleted file mode 100644 index 21311ded5e2ed..0000000000000 --- a/airbyte-singer/src/main/resources/singer_json/SingerMessage.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-singer/src/main/resources/singer_json/SingerMessage.json", - "title": "SingerMessage", - "description": "This is a compromise. Java codegen doesn't support union types. So we need to just make a flat json schema that looks like at of the possible permutations of a singer message. See the description for each field to see which message type that field belongs to.", - "type": "object", - "additionalProperties": true, - "required": ["type"], - "properties": { - "type": { - "description": "record, schema, and state message: used to determine record type. used in all singer messages.", - "type": "string", - "enum": ["RECORD", "SCHEMA", "STATE"] - }, - "stream": { - "description": "record AND schema message: name of stream", - "type": "string" - }, - "time_extracted": { - "description": "record message: timestamp", - "type": "string" - }, - "record": { - "description": "record message: must be a JSON object", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - }, - "schema": { - "description": "schema message: the schema", - "$ref": "SingerCatalog.json#/definitions/SingerTableSchema" - }, - "key_properties": { - "description": "schema message: names of keys", - "type": "array", - "items": { - "type": "string" - } - }, - "bookmark_properties": { - "description": "schema message: names of bookmarks", - "type": "array", - "items": { - "type": "string" - } - }, - "value": { - "description": "value message: must be a JSON object", - "type": "object", - "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" - } - } -} diff --git a/airbyte-singer/src/main/resources/singer_types/SingerCatalog.yaml b/airbyte-singer/src/main/resources/singer_types/SingerCatalog.yaml new file mode 100644 index 0000000000000..5f5f36abe5f42 --- /dev/null +++ b/airbyte-singer/src/main/resources/singer_types/SingerCatalog.yaml @@ -0,0 +1,213 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-singer/src/main/resources/singer_types/SingerCatalog.yaml +title: SingerCatalog +description: the schema of the singer catalog +type: object +additionalProperties: false +required: + - streams +properties: + streams: + type: array + items: + "$ref": "#/definitions/SingerStream" +definitions: + SingerStream: + type: object + additionalProperties: false + required: + - tap_stream_id + - stream + - schema + properties: + stream: + type: string + description: The name of the stream. + tap_stream_id: + type: string + description: + The unique identifier for the stream. This is allowed to be different + from the name of the stream in order to allow for sources that have duplicate + stream names. + schema: + "$ref": "#/definitions/SingerTableSchema" + table_name: + type: string + description: For a database source, the name of the table. + metadata: + type: array + items: + "$ref": "#/definitions/SingerMetadata" + key_properties: + type: array + items: + type: string + SingerTableSchema: + type: object + additionalProperties: false + required: + - properties + description: + The JSON schema for the stream. This struct is weird because it's + essentially modeling the properties field of a jsonschema object. + properties: + type: + "$ref": "#/definitions/SingerType" + properties: + "$ref": "#/definitions/SingerColumnMap" + definitions: + description: + placeholder for definitions that are included since it is all + jsonchema. + SingerColumnMap: + type: object + additionalProperties: + "$ref": "#/definitions/SingerColumn" + SingerColumn: + type: object + additionalProperties: false + properties: + type: + "$ref": "#/definitions/SingerType" + format: + type: string + selected: + description: + 'Deprecated: Some legacy Taps handle stream and field selection + by looking for "selected": true directly in the stream''s schema in the + catalog.json file (called properties.json in legacy taps).' + type: boolean + minimum: + type: integer + maximum: + type: integer + maxLength: + type: integer + properties: + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + items: + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + anyOf: + type: array + existingJavaType: com.fasterxml.jackson.databind.JsonNode + allOf: + type: array + existingJavaType: com.fasterxml.jackson.databind.JsonNode + oneOf: + type: array + existingJavaType: com.fasterxml.jackson.databind.JsonNode + not: + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + SingerMetadata: + type: object + additionalProperties: false + required: + - metadata + - breadcrumb + properties: + metadata: + "$ref": "#/definitions/SingerMetadataChild" + breadcrumb: + type: array + items: + type: string + SingerMetadataChild: + type: object + additionalProperties: false + required: [] + properties: + selected: + description: + "Non-Discoverable: Indicates that this node in the schema has + been selected by the user for replication." + type: boolean + replication-method: + description: + "Non-Discoverable: Either FULL_TABLE, INCREMENTAL, or LOG_BASED. + The replication method to use for a stream." + type: string + enum: + - FULL_TABLE + - INCREMENTAL + - LOG_BASED + replication-key: + description: + 'Non-Discoverable: The name of a property in the source to use + as a "bookmark". For example, this will often be an "updated-at" field or + an auto-incrementing primary key (requires replication-method' + type: array + items: + type: string + view-key-properties: + description: "Non Discoverable: List of key properties for a database view." + type: array + items: + type: string + inclusion: + description: + 'Discoverable: available means the field is available for selection, + and the tap will only emit values for that field if it is marked with "selected": + true. automatic means that the tap will emit values for the field. unsupported + means that the field exists in the source data but the tap is unable to + provide it.' + type: string + enum: + - available + - automatic + - unsupported + selected-by-default: + description: + "Discoverable: Indicates if a node in the schema should be replicated + if a user has not expressed any opinion on whether or not to replicate it." + type: boolean + valid-replication-keys: + description: + "Discoverable: List of the fields that could be used as replication + keys." + type: array + items: + type: string + forced-replication-method: + description: + "Discoverable: Used to force the replication method to either + FULL_TABLE or INCREMENTAL." + type: string + table-key-properties: + description: "Discoverable: List of key properties for a database table." + type: array + items: + type: string + schema-name: + description: "Discoverable: The name of the stream." + type: string + is-view: + description: + "Discoverable: Indicates whether a stream corresponds to a database + view." + type: boolean + row-count: + description: "Discoverable: Number of rows in a database table/view." + type: integer + database-name: + description: "Discoverable: Name of database." + type: string + sql-datatype: + description: "Discoverable: Represents the datatype of a database column." + SingerType: + type: + - array + - string + items: + type: string + enum: + - "null" + - boolean + - object + - array + - number + - string + - integer diff --git a/airbyte-singer/src/main/resources/singer_types/SingerMessage.yaml b/airbyte-singer/src/main/resources/singer_types/SingerMessage.yaml new file mode 100644 index 0000000000000..a815906b98fa4 --- /dev/null +++ b/airbyte-singer/src/main/resources/singer_types/SingerMessage.yaml @@ -0,0 +1,50 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-singer/src/main/resources/singer_types/SingerMessage.yaml +title: SingerMessage +description: + This is a compromise. Java codegen doesn't support union types. So we + need to just make a flat json schema that looks like at of the possible permutations + of a singer message. See the description for each field to see which message type + that field belongs to. +type: object +additionalProperties: true +required: + - type +properties: + type: + description: + "record, schema, and state message: used to determine record type. + used in all singer messages." + type: string + enum: + - RECORD + - SCHEMA + - STATE + stream: + description: "record AND schema message: name of stream" + type: string + time_extracted: + description: "record message: timestamp" + type: string + record: + description: "record message: must be a JSON object" + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + schema: + description: "schema message: the schema" + "$ref": "SingerCatalog.yaml#/definitions/SingerTableSchema" + key_properties: + description: "schema message: names of keys" + type: array + items: + type: string + bookmark_properties: + description: "schema message: names of bookmarks" + type: array + items: + type: string + value: + description: "value message: must be a JSON object" + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/build.gradle b/build.gradle index a265207be5654..a481090c31bde 100644 --- a/build.gradle +++ b/build.gradle @@ -146,6 +146,7 @@ subprojects { // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here. implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.core:jackson-annotations' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation group: 'com.google.guava', name: 'guava', version: '29.0-jre'