Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use the word catalog instead of schema in javabase #515

Merged
merged 2 commits into from
Oct 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public class IntegrationCliParser {
.build());
COMMAND_GROUP.addOption(Option.builder()
.longOpt(Command.DISCOVER.toString().toLowerCase())
.desc("outputs a catalog describing the source's schema")
.desc("outputs a catalog describing the source's catalog")
.build());
COMMAND_GROUP.addOption(Option.builder()
.longOpt(Command.READ.toString().toLowerCase())
Expand Down Expand Up @@ -113,15 +113,15 @@ private static IntegrationConfig parseOptions(String[] args, Command command) {
options.addOption(Option
.builder().longOpt(JavaBaseConstants.ARGS_CONFIG_KEY).desc(JavaBaseConstants.ARGS_CONFIG_DESC).hasArg(true).required(true).build());
options.addOption(Option
.builder().longOpt(JavaBaseConstants.ARGS_SCHEMA_KEY).desc(JavaBaseConstants.ARGS_SCHEMA_DESC).hasArg(true).build());
.builder().longOpt(JavaBaseConstants.ARGS_CATALOG_KEY).desc(JavaBaseConstants.ARGS_CATALOG_DESC).hasArg(true).build());
options.addOption(Option
.builder().longOpt(JavaBaseConstants.ARGS_STATE_KEY).desc(JavaBaseConstants.ARGS_PATH_DESC).hasArg(true).build());
}
case WRITE -> {
options.addOption(Option
.builder().longOpt(JavaBaseConstants.ARGS_CONFIG_KEY).desc(JavaBaseConstants.ARGS_CONFIG_DESC).hasArg(true).required(true).build());
options.addOption(Option
.builder().longOpt(JavaBaseConstants.ARGS_SCHEMA_KEY).desc(JavaBaseConstants.ARGS_SCHEMA_DESC).hasArg(true).build());
.builder().longOpt(JavaBaseConstants.ARGS_CATALOG_KEY).desc(JavaBaseConstants.ARGS_CATALOG_DESC).hasArg(true).build());
}
default -> throw new IllegalStateException("Unexpected value: " + command);
}
Expand All @@ -147,13 +147,13 @@ private static IntegrationConfig parseOptions(String[] args, Command command) {
case READ -> {
return IntegrationConfig.read(
Path.of(argsMap.get(JavaBaseConstants.ARGS_CONFIG_KEY)),
Path.of(argsMap.get(JavaBaseConstants.ARGS_SCHEMA_KEY)),
Path.of(argsMap.get(JavaBaseConstants.ARGS_CATALOG_KEY)),
argsMap.containsKey(JavaBaseConstants.ARGS_STATE_KEY) ? Path.of(argsMap.get(JavaBaseConstants.ARGS_STATE_KEY)) : null);
}
case WRITE -> {
return IntegrationConfig.write(
Path.of(argsMap.get(JavaBaseConstants.ARGS_CONFIG_KEY)),
Path.of(argsMap.get(JavaBaseConstants.ARGS_SCHEMA_KEY)));
Path.of(argsMap.get(JavaBaseConstants.ARGS_CATALOG_KEY)));
}
default -> throw new IllegalStateException("Unexpected value: " + command);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ public class IntegrationConfig {

private final Command command;
private final Path configPath;
private final Path schemaPath;
private final Path catalogPath;
private final Path statePath;

private IntegrationConfig(Command command, Path configPath, Path schemaPath, Path statePath) {
private IntegrationConfig(Command command, Path configPath, Path catalogPath, Path statePath) {
this.command = command;
this.configPath = configPath;
this.schemaPath = schemaPath;
this.catalogPath = catalogPath;
this.statePath = statePath;
}

Expand All @@ -57,16 +57,16 @@ public static IntegrationConfig discover(Path config) {
return new IntegrationConfig(Command.DISCOVER, config, null, null);
}

public static IntegrationConfig read(Path config, Path schema, Path state) {
Preconditions.checkNotNull(config);
Preconditions.checkNotNull(schema);
return new IntegrationConfig(Command.READ, config, schema, state);
public static IntegrationConfig read(Path configPath, Path catalogPath, Path statePath) {
Preconditions.checkNotNull(configPath);
Preconditions.checkNotNull(catalogPath);
return new IntegrationConfig(Command.READ, configPath, catalogPath, statePath);
}

public static IntegrationConfig write(Path config, Path schema) {
Preconditions.checkNotNull(config);
Preconditions.checkNotNull(schema);
return new IntegrationConfig(Command.WRITE, config, schema, null);
public static IntegrationConfig write(Path configPath, Path catalogPath) {
Preconditions.checkNotNull(configPath);
Preconditions.checkNotNull(catalogPath);
return new IntegrationConfig(Command.WRITE, configPath, catalogPath, null);
}

public Command getCommand() {
Expand All @@ -78,9 +78,9 @@ public Path getConfigPath() {
return configPath;
}

public Path getSchemaPath() {
public Path getCatalogPath() {
Preconditions.checkState(command == Command.READ || command == Command.WRITE);
return schemaPath;
return catalogPath;
}

public Optional<Path> getStatePath() {
Expand All @@ -93,7 +93,7 @@ public String toString() {
return "IntegrationConfig{" +
"command=" + command +
", configPath='" + configPath + '\'' +
", schemaPath='" + schemaPath + '\'' +
", catalogPath='" + catalogPath + '\'' +
", statePath='" + statePath + '\'' +
'}';
}
Expand All @@ -109,13 +109,13 @@ public boolean equals(Object o) {
IntegrationConfig that = (IntegrationConfig) o;
return command == that.command &&
Objects.equals(configPath, that.configPath) &&
Objects.equals(schemaPath, that.schemaPath) &&
Objects.equals(catalogPath, that.catalogPath) &&
Objects.equals(statePath, that.statePath);
}

@Override
public int hashCode() {
return Objects.hash(command, configPath, schemaPath, statePath);
return Objects.hash(command, configPath, catalogPath, statePath);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public void run(String[] args) throws Exception {
throw new RuntimeException("Not implemented");
case WRITE -> {
final JsonNode config = parseConfig(parsed.getConfigPath());
final Schema schema = parseConfig(parsed.getSchemaPath(), Schema.class);
final Schema schema = parseConfig(parsed.getCatalogPath(), Schema.class);
final DestinationConsumer<SingerMessage> consumer = destination.write(config, schema);
consumeWriteStream(consumer);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,12 @@

public class JavaBaseConstants {

public static String ENV_DESTINATION_CLASS = "DESTINATION_CLASS";
public static String ENV_DESTINATION_JAR_PATH = "DESTINATION_JAR_PATH";

public static String ARGS_CONFIG_KEY = "config";
public static String ARGS_SCHEMA_KEY = "schema";
public static String ARGS_CATALOG_KEY = "catalog";
public static String ARGS_STATE_KEY = "state";

public static String ARGS_CONFIG_DESC = "path to the json configuration file";
public static String ARGS_SCHEMA_DESC = "input path for the schema";
public static String ARGS_CATALOG_DESC = "input path for the catalog";
public static String ARGS_PATH_DESC = "path to the json-encoded state file";

// todo (cgardens) - this mount path should be passed in by the worker and read as an arg or
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
class IntegrationCliParserTest {

private static final String CONFIG_FILENAME = "config.json";
private static final String SCHEMA_FILENAME = "schema.json";
private static final String CATALOG_FILENAME = "catalog.json";
private static final String STATE_FILENAME = "state.json";

@Test
Expand All @@ -59,23 +59,23 @@ void testDiscover() {

@Test
void testWrite() {
final String[] args = new String[] {"--write", "--config", CONFIG_FILENAME, "--schema", SCHEMA_FILENAME};
final String[] args = new String[] {"--write", "--config", CONFIG_FILENAME, "--catalog", CATALOG_FILENAME};
final IntegrationConfig actual = new IntegrationCliParser().parse(args);
assertEquals(IntegrationConfig.write(Path.of(CONFIG_FILENAME), Path.of(SCHEMA_FILENAME)), actual);
assertEquals(IntegrationConfig.write(Path.of(CONFIG_FILENAME), Path.of(CATALOG_FILENAME)), actual);
}

@Test
void testReadWithoutState() {
final String[] args = new String[] {"--read", "--config", CONFIG_FILENAME, "--schema", SCHEMA_FILENAME};
final String[] args = new String[] {"--read", "--config", CONFIG_FILENAME, "--catalog", CATALOG_FILENAME};
final IntegrationConfig actual = new IntegrationCliParser().parse(args);
assertEquals(IntegrationConfig.read(Path.of(CONFIG_FILENAME), Path.of(SCHEMA_FILENAME), null), actual);
assertEquals(IntegrationConfig.read(Path.of(CONFIG_FILENAME), Path.of(CATALOG_FILENAME), null), actual);
}

@Test
void testReadWithState() {
final String[] args = new String[] {"--read", "--config", CONFIG_FILENAME, "--schema", SCHEMA_FILENAME, "--state", STATE_FILENAME};
final String[] args = new String[] {"--read", "--config", CONFIG_FILENAME, "--catalog", CATALOG_FILENAME, "--state", STATE_FILENAME};
final IntegrationConfig actual = new IntegrationCliParser().parse(args);
assertEquals(IntegrationConfig.read(Path.of(CONFIG_FILENAME), Path.of(SCHEMA_FILENAME), Path.of(STATE_FILENAME)), actual);
assertEquals(IntegrationConfig.read(Path.of(CONFIG_FILENAME), Path.of(CATALOG_FILENAME), Path.of(STATE_FILENAME)), actual);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,15 @@
class IntegrationConfigTest {

private static final Path CONFIG_PATH = Path.of("config.json");
private static final Path SCHEMA_PATH = Path.of("schema.json");
private static final Path CATALOG_PATH = Path.of("catalog.json");
private static final Path STATE_PATH = Path.of("state.json");

@Test
void testSpec() {
final IntegrationConfig config = IntegrationConfig.spec();
assertEquals(Command.SPEC, config.getCommand());
assertThrows(IllegalStateException.class, config::getConfigPath);
assertThrows(IllegalStateException.class, config::getSchemaPath);
assertThrows(IllegalStateException.class, config::getCatalogPath);
assertThrows(IllegalStateException.class, config::getStatePath);
}

Expand All @@ -53,7 +53,7 @@ void testCheck() {
final IntegrationConfig config = IntegrationConfig.check(CONFIG_PATH);
assertEquals(Command.CHECK, config.getCommand());
assertEquals(CONFIG_PATH, config.getConfigPath());
assertThrows(IllegalStateException.class, config::getSchemaPath);
assertThrows(IllegalStateException.class, config::getCatalogPath);
assertThrows(IllegalStateException.class, config::getStatePath);
}

Expand All @@ -64,43 +64,43 @@ void testDiscover() {
final IntegrationConfig config = IntegrationConfig.discover(CONFIG_PATH);
assertEquals(Command.DISCOVER, config.getCommand());
assertEquals(CONFIG_PATH, config.getConfigPath());
assertThrows(IllegalStateException.class, config::getSchemaPath);
assertThrows(IllegalStateException.class, config::getCatalogPath);
assertThrows(IllegalStateException.class, config::getStatePath);
}

@Test
void testWrite() {
assertThrows(NullPointerException.class, () -> IntegrationConfig.write(null, SCHEMA_PATH));
assertThrows(NullPointerException.class, () -> IntegrationConfig.write(null, CATALOG_PATH));
assertThrows(NullPointerException.class, () -> IntegrationConfig.write(CONFIG_PATH, null));

final IntegrationConfig config = IntegrationConfig.write(CONFIG_PATH, SCHEMA_PATH);
final IntegrationConfig config = IntegrationConfig.write(CONFIG_PATH, CATALOG_PATH);
assertEquals(Command.WRITE, config.getCommand());
assertEquals(CONFIG_PATH, config.getConfigPath());
assertEquals(SCHEMA_PATH, config.getSchemaPath());
assertEquals(CATALOG_PATH, config.getCatalogPath());
assertThrows(IllegalStateException.class, config::getStatePath);
}

@Test
void testReadWithState() {
assertThrows(NullPointerException.class, () -> IntegrationConfig.read(null, SCHEMA_PATH, STATE_PATH));
assertThrows(NullPointerException.class, () -> IntegrationConfig.read(null, CATALOG_PATH, STATE_PATH));
assertThrows(NullPointerException.class, () -> IntegrationConfig.read(CONFIG_PATH, null, STATE_PATH));

final IntegrationConfig config = IntegrationConfig.read(CONFIG_PATH, SCHEMA_PATH, STATE_PATH);
final IntegrationConfig config = IntegrationConfig.read(CONFIG_PATH, CATALOG_PATH, STATE_PATH);
assertEquals(Command.READ, config.getCommand());
assertEquals(CONFIG_PATH, config.getConfigPath());
assertEquals(SCHEMA_PATH, config.getSchemaPath());
assertEquals(CATALOG_PATH, config.getCatalogPath());
assertEquals(Optional.of(STATE_PATH), config.getStatePath());
}

@Test
void testReadWithoutState() {
assertThrows(NullPointerException.class, () -> IntegrationConfig.read(null, SCHEMA_PATH, null));
assertThrows(NullPointerException.class, () -> IntegrationConfig.read(null, CATALOG_PATH, null));
assertThrows(NullPointerException.class, () -> IntegrationConfig.read(CONFIG_PATH, null, null));

final IntegrationConfig config = IntegrationConfig.read(CONFIG_PATH, SCHEMA_PATH, null);
final IntegrationConfig config = IntegrationConfig.read(CONFIG_PATH, CATALOG_PATH, null);
assertEquals(Command.READ, config.getCommand());
assertEquals(CONFIG_PATH, config.getConfigPath());
assertEquals(SCHEMA_PATH, config.getSchemaPath());
assertEquals(CATALOG_PATH, config.getCatalogPath());
assertEquals(Optional.empty(), config.getStatePath());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
class IntegrationRunnerTest {

private static final String CONFIG_FILE_NAME = "config.json";
private static final String SCHEMA_FILE_NAME = "schema.json";
private static final String CATALOG_FILE_NAME = "catalog.json";

private static final String[] ARGS = new String[] {"args"};

Expand All @@ -70,7 +70,7 @@ class IntegrationRunnerTest {
private Consumer<String> stdoutConsumer;
private Destination destination;
private Path configPath;
private Path schemaPath;
private Path catalogPath;

@SuppressWarnings("unchecked")
@BeforeEach
Expand All @@ -81,7 +81,7 @@ void setup() throws IOException {
Path configDir = Files.createTempDirectory("test");

configPath = IOs.writeFile(configDir, CONFIG_FILE_NAME, CONFIG_STRING);
schemaPath = IOs.writeFile(configDir, SCHEMA_FILE_NAME, Jsons.serialize(SCHEMA));
catalogPath = IOs.writeFile(configDir, CATALOG_FILE_NAME, Jsons.serialize(SCHEMA));
}

@Test
Expand Down Expand Up @@ -129,7 +129,7 @@ void testDiscover() throws Exception {
@SuppressWarnings("unchecked")
@Test
void testWrite() throws Exception {
final IntegrationConfig intConfig = IntegrationConfig.write(Path.of(configPath.toString()), Path.of(schemaPath.toString()));
final IntegrationConfig intConfig = IntegrationConfig.write(Path.of(configPath.toString()), Path.of(catalogPath.toString()));
final DestinationConsumer<SingerMessage> destinationConsumerMock = mock(DestinationConsumer.class);
when(cliParser.parse(ARGS)).thenReturn(intConfig);
when(destination.write(CONFIG, SCHEMA)).thenReturn(destinationConsumerMock);
Expand Down