Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove json schema validator #1043

Merged
merged 4 commits into from
Feb 8, 2023
Merged

Remove json schema validator #1043

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a0600a4
Remove json schema validator dep
windoze Feb 8, 2023
66c41d6
No longer need this
windoze Feb 8, 2023
150297b
Cleanup
windoze Feb 8, 2023
60f6a4c
Bump version for maven test
blrchen Feb 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 1 addition & 10 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,6 @@ plugins {

repositories {
mavenCentral()
mavenLocal()
maven {
url "https://repository.mulesoft.org/nexus/content/repositories/public/"
}
maven {
url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus
}

}

configurations {
Expand Down Expand Up @@ -149,11 +141,10 @@ project.ext.spec = [
"mockito_inline": "org.mockito:mockito-inline:2.28.2",
'testing' : "org.testng:testng:6.14.3",
'jdiagnostics' : "org.anarres.jdiagnostics:jdiagnostics:1.0.7",
'jsonSchemaVali': "com.github.everit-org.json-schema:org.everit.json.schema:1.9.1",
"antlr": "org.antlr:antlr4:4.8",
"antlrRuntime": "org.antlr:antlr4-runtime:4.8",
"jsqlparser": "com.github.jsqlparser:jsqlparser:3.1",

'common_collections': 'commons-collections:commons-collections:3.2.2',
]
]

Expand Down
3 changes: 2 additions & 1 deletion feathr-config/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,11 @@ dependencies {
provided spec.product.pegasus.data
implementation spec.product.typesafe_config
implementation spec.product.log4j
implementation spec.product.jsonSchemaVali
implementation spec.product.jackson.jackson_databind
implementation spec.product.mvel
implementation spec.product.json
implementation spec.product.guava
implementation spec.product.common_collections

testImplementation spec.product.testing
testImplementation spec.product.mockito
Expand Down
70 changes: 0 additions & 70 deletions .../main/java/com/linkedin/feathr/core/configvalidator/typesafe/TypesafeConfigValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@
import java.util.regex.Pattern;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.everit.json.schema.Schema;
import org.everit.json.schema.ValidationException;
import org.everit.json.schema.loader.SchemaLoader;
import org.json.JSONObject;
import org.json.JSONTokener;

Expand All @@ -52,14 +49,6 @@ public class TypesafeConfigValidator implements ConfigValidator {
// Used when rendering the parsed config to JSON string (which is then used in validation)
private ConfigRenderOptions _renderOptions;

// Schema for FeatureDef config
private Schema _featureDefSchema;

// Schema for Join config
private Schema _joinConfigSchema;

private Schema _presentationConfigSchema;

private final static String FEATUREDEF_CONFIG_SCHEMA = "/FeatureDefConfigSchema.json";

private final static String JOIN_CONFIG_SCHEMA = "/JoinConfigSchema.json";
Expand Down Expand Up @@ -201,42 +190,22 @@ public ValidationResult validateSyntax(ConfigType configType, Config config) {

switch (configType) {
case FeatureDef:
if (_featureDefSchema == null) {
_featureDefSchema = loadFeatureDefSchema();
logger.info("FeatureDef config schema loaded");
}
_featureDefSchema.validate(root);

// validate naming convention
result = validateFeatureDefNames(config);
break;

case Join:
if (_joinConfigSchema == null) {
_joinConfigSchema = loadJoinConfigSchema();
logger.info("Join config schema loaded");
}
_joinConfigSchema.validate(root);
result = new ValidationResult(SYNTACTIC, VALID);
break;

case Presentation:
if (_presentationConfigSchema == null) {
_presentationConfigSchema = loadPresentationConfigSchema();
logger.info("Presentation config schema loaded");
}
_presentationConfigSchema.validate(root);
result = new ValidationResult(SYNTACTIC, VALID);
break;
default:
throw new ConfigValidationException("Unknown config type: " + configType);
}
} catch (ConfigValidationException e) {
throw e;
} catch (ValidationException e) {
String header = configType + " config syntax is invalid. Details:";
String details = String.join("\n", header, String.join("\n", e.getAllMessages()));
result = new ValidationResult(SYNTACTIC, INVALID, details, e);
} catch (Exception e) {
throw new ConfigValidationException("Config validation error", e);
}
Expand Down Expand Up @@ -408,43 +377,4 @@ private Config buildTypesafeConfig(ConfigType configType, ConfigDataProvider con
TypesafeConfigBuilder builder = new TypesafeConfigBuilder();
return builder.buildTypesafeConfig(configType, configDataProvider);
}

/*
* Loads schema for FeatureDef config using Everit JSON Schema Validator
* (https://github.com/everit-org/json-schema)
*/
private Schema loadFeatureDefSchema() {
try (InputStream inputStream = getClass().getResourceAsStream(FEATUREDEF_CONFIG_SCHEMA)) {
JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream));
return SchemaLoader.load(rawSchema);
} catch (Exception e) {
throw new ConfigValidationException("Error in loading FeatureDef schema", e);
}
}

/*
* Loads schema for Join config using Everit JSON Schema Validator
* (https://github.com/everit-org/json-schema)
*/
private Schema loadJoinConfigSchema() {
try (InputStream inputStream = getClass().getResourceAsStream(JOIN_CONFIG_SCHEMA)) {
JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream));
return SchemaLoader.load(rawSchema);
} catch (Exception e) {
throw new ConfigValidationException("Error in loading FeatureDef schema", e);
}
}

/*
* Loads schema for Presentation config using Everit JSON Schema Validator
* (https://github.com/everit-org/json-schema)
*/
private Schema loadPresentationConfigSchema() {
try (InputStream inputStream = getClass().getResourceAsStream(PRESENTATION_CONFIG_SCHEMA)) {
JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream));
return SchemaLoader.load(rawSchema);
} catch (Exception e) {
throw new ConfigValidationException("Error in loading PresentationConfig schema", e);
}
}
}
7 changes: 0 additions & 7 deletions .../test/java/com/linkedin/feathr/core/configbuilder/typesafe/TypesafeConfigBuilderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,6 @@ public void testFeatureDefConfig() {
}
}

@Test(expectedExceptions = ConfigBuilderException.class, description = "Tests build of invalid FeatureDef config")
public void testFeatureDefConfig2() {
String featureDefConfigStr = "{invalidSectionName: {}}";
FeatureDefConfig obsFeatureDefConfigObj = configBuilder.buildFeatureDefConfigFromString(featureDefConfigStr);
fail("Test shouldn't pass for invalid config");
}

@Test(description = "Include of another config and selective overrides")
public void includeTest() {
String expEspressoConfigName = "MemberPreferenceData";
Expand Down
56 changes: 8 additions & 48 deletions ...hr-config/src/test/java/com/linkedin/feathr/core/configvalidator/ConfigValidatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,32 +1,22 @@
package com.linkedin.feathr.core.configvalidator;

import com.linkedin.feathr.core.config.ConfigType;
import com.linkedin.feathr.core.config.consumer.JoinConfig;
import com.linkedin.feathr.core.configdataprovider.ConfigDataProvider;
import com.linkedin.feathr.core.configdataprovider.ResourceConfigDataProvider;
import com.linkedin.feathr.core.configdataprovider.StringConfigDataProvider;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigException;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigParseOptions;
import com.typesafe.config.ConfigRenderOptions;
import com.typesafe.config.ConfigSyntax;
import java.io.InputStream;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.everit.json.schema.Schema;
import org.everit.json.schema.ValidationException;
import org.everit.json.schema.loader.SchemaLoader;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import static com.linkedin.feathr.core.config.ConfigType.*;
import static com.linkedin.feathr.core.configvalidator.ValidationStatus.*;
import static com.linkedin.feathr.core.configvalidator.ValidationType.*;
import static com.linkedin.feathr.core.config.ConfigType.FeatureDef;
import static com.linkedin.feathr.core.config.ConfigType.Join;
import static com.linkedin.feathr.core.configvalidator.ValidationStatus.INVALID;
import static com.linkedin.feathr.core.configvalidator.ValidationStatus.VALID;
import static com.linkedin.feathr.core.configvalidator.ValidationType.SYNTACTIC;
import static org.testng.Assert.*;


Expand Down Expand Up @@ -77,23 +67,6 @@ public void testFeatureDefConfigWithValidSyntax() {
}
}

@Test(description = "Tests syntax validation of an invalid FeatureDef config")
public void testFeatureDefConfigWithInvalidSyntax() {
try (ConfigDataProvider cdp = new StringConfigDataProvider(ConfigValidatorFixture.invalidFeatureDefConfig)) {
ValidationResult obsResult = _validator.validate(FeatureDef, SYNTACTIC, cdp);

assertEquals(obsResult.getValidationStatus(), INVALID);
assertTrue(obsResult.getDetails().isPresent());
assertTrue(obsResult.getCause().isPresent());

// Get details and verify that there are no error messages related to (syntactially valid) anchor A3
String details = obsResult.getDetails().get();
assertFalse(details.contains("#/anchors/A3"));
} catch (Exception e) {
fail("Caught exception: " + e.getMessage(), e);
}
}

@Test(description = "Tests syntax validation of a valid Join config")
public void testJoinConfigWithValidSyntax() {
List<String> configStrings = Arrays.asList(ConfigValidatorFixture.validJoinConfigWithSingleFeatureBag, ConfigValidatorFixture.validJoinConfigWithMultFeatureBags);
Expand All @@ -111,19 +84,6 @@ public void testJoinConfigWithValidSyntax() {
}
}

@Test(description = "Tests syntax validation of an invalid Join config")
public void testJoinConfigWithInvalidSyntax() {
try (ConfigDataProvider cdp = new StringConfigDataProvider(ConfigValidatorFixture.invalidJoinConfig)) {
ValidationResult obsResult = _validator.validate(Join, SYNTACTIC, cdp);

assertEquals(obsResult.getValidationStatus(), INVALID);
assertTrue(obsResult.getDetails().isPresent());
assertTrue(obsResult.getCause().isPresent());
} catch (Exception e) {
fail("Caught exception: " + e.getMessage(), e);
}
}

@Test(description = "Tests syntax validation of both FeatureDef and Join config together")
public void testFeatureDefAndJoinConfigSyntax() {
Map<ConfigType, ConfigDataProvider> configTypeWithDataProvider = new HashMap<>();
Expand Down
Loading