Skip to content

Commit

Permalink
bulk-cdk: reorganize source structure (#43937)
Browse files Browse the repository at this point in the history
  • Loading branch information
Marius Posta authored Aug 12, 2024
1 parent 8ab89d3 commit 1535cad
Show file tree
Hide file tree
Showing 82 changed files with 241 additions and 318 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
package io.airbyte.cdk

import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
import io.airbyte.cdk.consumers.OutputConsumer
import io.airbyte.cdk.operation.Operation
import io.airbyte.cdk.output.OutputConsumer
import io.airbyte.cdk.util.ApmTraceUtils
import io.github.oshai.kotlinlogging.KotlinLogging
import io.micronaut.context.annotation.Value
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.exceptions
package io.airbyte.cdk

/**
* An exception that indicates that there is something wrong with the user's connector setup. This
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.operation
package io.airbyte.cdk

/** Interface that defines a CLI operation. */
fun interface Operation {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
package io.airbyte.cdk.command

import com.fasterxml.jackson.databind.JsonNode
import io.airbyte.cdk.exceptions.ConfigErrorException
import io.airbyte.cdk.ConfigErrorException
import io.airbyte.cdk.util.Jsons
import io.micronaut.context.annotation.Value
import jakarta.inject.Singleton
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.command

import io.airbyte.cdk.exceptions.ConfigErrorException
import io.airbyte.cdk.ConfigErrorException
import io.airbyte.cdk.util.ResourceUtils
import io.airbyte.protocol.models.v0.AirbyteStream
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.command

import io.airbyte.cdk.operation.Operation
import io.airbyte.cdk.Operation
import io.github.oshai.kotlinlogging.KotlinLogging
import io.micronaut.context.env.MapPropertySource
import io.micronaut.core.cli.CommandLine
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
package io.airbyte.cdk.command

import com.fasterxml.jackson.databind.JsonNode
import io.airbyte.cdk.exceptions.ConfigErrorException
import io.airbyte.cdk.ConfigErrorException
import io.airbyte.cdk.util.Jsons
import io.airbyte.cdk.util.ResourceUtils
import io.airbyte.protocol.models.v0.AirbyteGlobalState
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import com.networknt.schema.JsonSchema
import com.networknt.schema.JsonSchemaFactory
import com.networknt.schema.SchemaValidatorsConfig
import com.networknt.schema.SpecVersion
import io.airbyte.cdk.exceptions.ConfigErrorException
import io.airbyte.cdk.ConfigErrorException
import io.airbyte.cdk.util.Jsons

object ValidatedJsonUtils {
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.consumers
package io.airbyte.cdk.output

import io.airbyte.cdk.exceptions.ConfigErrorException
import io.airbyte.cdk.ConfigErrorException
import io.airbyte.cdk.util.Jsons
import io.airbyte.protocol.models.v0.AirbyteAnalyticsTraceMessage
import io.airbyte.protocol.models.v0.AirbyteCatalog
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.operation
package io.airbyte.cdk.spec

import io.airbyte.cdk.Operation
import io.airbyte.cdk.command.ConfigurationJsonObjectSupplier
import io.airbyte.cdk.consumers.OutputConsumer
import io.airbyte.cdk.output.OutputConsumer
import io.airbyte.protocol.models.v0.ConnectorSpecification
import io.micronaut.context.annotation.Requires
import io.micronaut.context.annotation.Value
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaDefault
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaDescription
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle
import io.airbyte.cdk.ConfigErrorException
import io.airbyte.cdk.command.CONNECTOR_CONFIG_PREFIX
import io.airbyte.cdk.exceptions.ConfigErrorException
import io.micronaut.context.annotation.ConfigurationProperties

/** Union type for SSH tunnel method configuration in connector configurations. */
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.ssh

import io.airbyte.cdk.exceptions.ConfigErrorException
import io.airbyte.cdk.ConfigErrorException
import io.github.oshai.kotlinlogging.KotlinLogging
import java.io.StringReader
import java.net.InetSocketAddress
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.jdbc
package io.airbyte.cdk.h2

import java.sql.DriverManager
import org.junit.jupiter.api.Assertions
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import io.airbyte.cdk.AirbyteConnectorRunner
import io.airbyte.cdk.AirbyteDestinationRunner
import io.airbyte.cdk.AirbyteSourceRunner
import io.airbyte.cdk.TestClockFactory
import io.airbyte.cdk.consumers.BufferingOutputConsumer
import io.airbyte.cdk.output.BufferingOutputConsumer
import io.airbyte.cdk.util.Jsons
import io.airbyte.protocol.models.v0.AirbyteMessage
import io.airbyte.protocol.models.v0.AirbyteStateMessage
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.jdbc
/*
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.cdk.h2

import io.github.oshai.kotlinlogging.KotlinLogging
import io.micronaut.context.annotation.Value
Expand All @@ -10,13 +13,13 @@ import java.sql.ResultSet
import org.apache.commons.lang3.RandomStringUtils
import org.h2.tools.Server

private val log = KotlinLogging.logger {}

/** Wraps an H2 in-memory database and exposes a TCP server for it. */
@Singleton
class H2TestFixture(
@Value("\${h2.database.name}") database: String? = null,
) : AutoCloseable {
private val log = KotlinLogging.logger {}

private val server: Server
private val internalConnection: Connection

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.consumers
package io.airbyte.cdk.output

import io.airbyte.protocol.models.v0.AirbyteCatalog
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.consumers
package io.airbyte.cdk.output

import io.airbyte.cdk.util.Jsons
import io.airbyte.protocol.models.v0.AirbyteMessage
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.operation
package io.airbyte.cdk.check

import io.airbyte.cdk.command.ConfigurationJsonObjectBase
import io.airbyte.cdk.command.ConfigurationJsonObjectSupplier
import io.airbyte.cdk.command.SourceConfiguration
import io.airbyte.cdk.command.SourceConfigurationFactory
import io.airbyte.cdk.consumers.OutputConsumer
import io.airbyte.cdk.source.MetadataQuerier
import io.airbyte.cdk.Operation
import io.airbyte.cdk.command.*
import io.airbyte.cdk.discover.MetadataQuerier
import io.airbyte.cdk.output.OutputConsumer
import io.airbyte.cdk.util.ApmTraceUtils
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus
import io.airbyte.protocol.models.v0.AirbyteErrorTraceMessage
Expand All @@ -16,8 +14,6 @@ import jakarta.inject.Singleton
import java.sql.SQLException
import org.apache.commons.lang3.exception.ExceptionUtils

private val log = KotlinLogging.logger {}

@Singleton
@Requires(property = Operation.PROPERTY, value = "check")
@Requires(env = ["source"])
Expand All @@ -27,6 +23,8 @@ class CheckOperation<T : ConfigurationJsonObjectBase>(
val metadataQuerierFactory: MetadataQuerier.Factory<SourceConfiguration>,
val outputConsumer: OutputConsumer,
) : Operation {
private val log = KotlinLogging.logger {}

/** Wraps all checks in exception handling because CHECK must always exit cleanly. */
override fun execute() {
try {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.command

import io.airbyte.cdk.exceptions.ConfigErrorException
import io.airbyte.cdk.ConfigErrorException

/**
* Each connector contains an implementation of this interface in a stateless class which maps the
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.source
package io.airbyte.cdk.discover

import io.airbyte.protocol.models.v0.AirbyteStream

Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.operation
package io.airbyte.cdk.discover

import io.airbyte.cdk.Operation
import io.airbyte.cdk.command.SourceConfiguration
import io.airbyte.cdk.consumers.OutputConsumer
import io.airbyte.cdk.source.AirbyteStreamDecorator
import io.airbyte.cdk.source.Field
import io.airbyte.cdk.source.MetadataQuerier
import io.airbyte.cdk.output.OutputConsumer
import io.airbyte.protocol.models.Field as AirbyteField
import io.airbyte.protocol.models.v0.AirbyteCatalog
import io.airbyte.protocol.models.v0.AirbyteStream
Expand All @@ -14,8 +12,6 @@ import io.github.oshai.kotlinlogging.KotlinLogging
import io.micronaut.context.annotation.Requires
import jakarta.inject.Singleton

private val log = KotlinLogging.logger {}

@Singleton
@Requires(property = Operation.PROPERTY, value = "discover")
@Requires(env = ["source"])
Expand All @@ -25,6 +21,8 @@ class DiscoverOperation(
val airbyteStreamDecorator: AirbyteStreamDecorator,
val outputConsumer: OutputConsumer,
) : Operation {
private val log = KotlinLogging.logger {}

override fun execute() {
val airbyteStreams = mutableListOf<AirbyteStream>()
metadataQuerierFactory.session(config).use { metadataQuerier: MetadataQuerier ->
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.source
package io.airbyte.cdk.discover

import io.airbyte.cdk.data.AirbyteType
import io.airbyte.cdk.data.JsonDecoder
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.source
package io.airbyte.cdk.discover

import io.airbyte.cdk.command.SourceConfiguration

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.consumers
package io.airbyte.cdk.output

import io.airbyte.cdk.data.AirbyteType
import io.github.oshai.kotlinlogging.KotlinLogging
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.read

import io.airbyte.cdk.source.Field
import io.airbyte.cdk.source.FieldOrMetaField
import io.airbyte.cdk.discover.Field
import io.airbyte.cdk.discover.FieldOrMetaField
import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair
import io.airbyte.protocol.models.v0.StreamDescriptor
import io.airbyte.protocol.models.v0.SyncMode
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@
package io.airbyte.cdk.read

import io.airbyte.cdk.command.OpaqueStateValue
import io.airbyte.cdk.source.PartitionReadCheckpoint
import io.airbyte.cdk.source.PartitionReader
import io.airbyte.cdk.source.PartitionsCreator
import io.airbyte.cdk.util.ThreadRenamingCoroutineName
import io.airbyte.protocol.models.v0.AirbyteStateMessage
import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.source
package io.airbyte.cdk.read

import io.airbyte.cdk.command.OpaqueStateValue
import io.airbyte.cdk.read.Feed
import io.airbyte.cdk.read.StateQuerier
import io.airbyte.cdk.source.PartitionsCreator.TryAcquireResourcesStatus
import io.airbyte.cdk.read.PartitionsCreator.TryAcquireResourcesStatus

/**
* [PartitionsCreatorFactory] must be implemented by each source connector and serves as the
Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,14 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.operation
package io.airbyte.cdk.read

import hu.webarticum.treeprinter.SimpleTreeNode
import hu.webarticum.treeprinter.TreeNode
import hu.webarticum.treeprinter.printer.TreePrinter
import hu.webarticum.treeprinter.printer.listing.ListingTreePrinter
import io.airbyte.cdk.Operation
import io.airbyte.cdk.command.InputState
import io.airbyte.cdk.command.SourceConfiguration
import io.airbyte.cdk.consumers.OutputConsumer
import io.airbyte.cdk.read.Feed
import io.airbyte.cdk.read.RootReader
import io.airbyte.cdk.read.StateManager
import io.airbyte.cdk.read.StateManagerFactory
import io.airbyte.cdk.source.PartitionsCreatorFactory
import io.airbyte.cdk.output.OutputConsumer
import io.airbyte.cdk.util.ThreadRenamingCoroutineName
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog
import io.github.oshai.kotlinlogging.KotlinLogging
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
package io.airbyte.cdk.read

import io.airbyte.cdk.consumers.OutputConsumer
import io.airbyte.cdk.source.PartitionsCreatorFactory
import io.airbyte.cdk.output.OutputConsumer
import io.airbyte.cdk.util.ThreadRenamingCoroutineName
import io.github.oshai.kotlinlogging.KotlinLogging
import java.time.Duration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,28 @@
package io.airbyte.cdk.read

import com.fasterxml.jackson.databind.JsonNode
import io.airbyte.cdk.ConfigErrorException
import io.airbyte.cdk.command.EmptyInputState
import io.airbyte.cdk.command.GlobalInputState
import io.airbyte.cdk.command.InputState
import io.airbyte.cdk.command.SourceConfiguration
import io.airbyte.cdk.command.StreamInputState
import io.airbyte.cdk.consumers.CatalogValidationFailureHandler
import io.airbyte.cdk.consumers.FieldNotFound
import io.airbyte.cdk.consumers.FieldTypeMismatch
import io.airbyte.cdk.consumers.InvalidIncrementalSyncMode
import io.airbyte.cdk.consumers.InvalidPrimaryKey
import io.airbyte.cdk.consumers.MultipleStreamsFound
import io.airbyte.cdk.consumers.StreamHasNoFields
import io.airbyte.cdk.consumers.StreamNotFound
import io.airbyte.cdk.data.AirbyteType
import io.airbyte.cdk.data.ArrayAirbyteType
import io.airbyte.cdk.data.LeafAirbyteType
import io.airbyte.cdk.exceptions.ConfigErrorException
import io.airbyte.cdk.source.CommonMetaField
import io.airbyte.cdk.source.Field
import io.airbyte.cdk.source.FieldOrMetaField
import io.airbyte.cdk.source.MetaField
import io.airbyte.cdk.source.MetadataQuerier
import io.airbyte.cdk.discover.CommonMetaField
import io.airbyte.cdk.discover.Field
import io.airbyte.cdk.discover.FieldOrMetaField
import io.airbyte.cdk.discover.MetaField
import io.airbyte.cdk.discover.MetadataQuerier
import io.airbyte.cdk.output.CatalogValidationFailureHandler
import io.airbyte.cdk.output.FieldNotFound
import io.airbyte.cdk.output.FieldTypeMismatch
import io.airbyte.cdk.output.InvalidIncrementalSyncMode
import io.airbyte.cdk.output.InvalidPrimaryKey
import io.airbyte.cdk.output.MultipleStreamsFound
import io.airbyte.cdk.output.StreamHasNoFields
import io.airbyte.cdk.output.StreamNotFound
import io.airbyte.protocol.models.v0.AirbyteStream
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog
import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream
Expand Down
Loading

0 comments on commit 1535cad

Please sign in to comment.