From ec340b1517aa8b2c368ad90999969880d17ac348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A5le=20Pedersen?= Date: Thu, 2 Nov 2023 11:49:35 +0100 Subject: [PATCH] Added TestExport and SchemaExport to avoid creating unstructured json, fixes #860 --- .../en/docs/Tasks/import-export/index.md | 64 + docs/site/content/en/openapi/openapi.yaml | 531 +- .../horreum/api/alerting/ChangeDetection.java | 1 - .../tools/horreum/api/alerting/Variable.java | 11 +- .../horreum/api/data/ExperimentProfile.java | 3 + .../tools/horreum/api/data/Label.java | 5 + .../tools/horreum/api/data/Schema.java | 11 + .../tools/horreum/api/data/SchemaExport.java | 17 + .../tools/horreum/api/data/Test.java | 19 + .../tools/horreum/api/data/TestExport.java | 68 + .../horreum/api/data/datastore/Datastore.java | 7 +- .../horreum/api/services/SchemaService.java | 15 +- .../horreum/api/services/TestService.java | 16 +- .../horreum/entity/ExperimentProfileDAO.java | 3 +- .../mapper/ExperimentProfileMapper.java | 7 +- .../tools/horreum/mapper/LabelMapper.java | 2 + .../tools/horreum/mapper/VariableMapper.java | 36 +- .../tools/horreum/svc/ActionServiceImpl.java | 65 +- .../horreum/svc/AlertingServiceImpl.java | 104 +- .../horreum/svc/ExperimentServiceImpl.java | 55 +- .../tools/horreum/svc/SchemaServiceImpl.java | 83 +- .../tools/horreum/svc/ServiceMediator.java | 34 +- .../horreum/svc/SubscriptionServiceImpl.java | 33 +- .../tools/horreum/svc/TestServiceImpl.java | 118 +- .../horreum/svc/AlertingServiceTest.java | 24 +- .../tools/horreum/svc/BaseServiceTest.java | 40 +- .../tools/horreum/svc/LogServiceTest.java | 3 +- .../tools/horreum/svc/RunServiceTest.java | 23 +- .../tools/horreum/svc/SchemaServiceTest.java | 2 + .../tools/horreum/svc/TestServiceTest.java | 20 +- .../tools/horreum/it/HorreumClientIT.java | 2 +- horreum-web/package-lock.json | 10274 ++++++++-------- .../example-data/quarkus_sb_test.json | 240 +- 33 files changed, 6261 insertions(+), 5675 deletions(-) create mode 100644 docs/site/content/en/docs/Tasks/import-export/index.md create mode 100644 horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/SchemaExport.java create mode 100644 horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/TestExport.java diff --git a/docs/site/content/en/docs/Tasks/import-export/index.md b/docs/site/content/en/docs/Tasks/import-export/index.md new file mode 100644 index 000000000..fc5465952 --- /dev/null +++ b/docs/site/content/en/docs/Tasks/import-export/index.md @@ -0,0 +1,64 @@ +--- +title: Import and Export Tests and Schemas +date: 2023-11-30 +description: How to import and export Tests and Schemas in Horreum +categories: [Tutorial] +weight: 3 +--- + +> **Prerequisites**: + +> 1. Horreum is running +> 2. To export you have previously defined a `Schema` for the JSON data you wish to analyze, please see [Define a Schema](/docs/tasks/define-schema-and-views/) +> 3. To export you have previously defined a Test, please see [Create new Test](/docs/tasks/create-new-test/) + +## Background + +To simplify copying Tests and Schemas between Horreum instances Horreum provides a simple API to export and import new Tests and Schemas. Horreum also support updating exising Schemas and Tests by importing Tests or Schemas with existing Id's. + +## TestExport + +The export object for Tests is called [TestExport](https://github.com/Hyperfoil/Horreum/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/TestExport.java) and contains a lot of other fields in addition to what's defined in [Test](https://github.com/Hyperfoil/Horreum/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Test.java). This includes, variables, experiments, actions, subscriptions, datastore and missingDataRules. This is to simplify the import/export experience and make sure that all the data related to a Test has a single entrypoint with regards to import and export. Note that secrets defined on [Action](https://github.com/Hyperfoil/Horreum/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Action.java) are not portable between Horreum instances and there might be security concerns so they are omitted. The apiKey and password attributs defined on the config field in [Datastore](https://github.com/Hyperfoil/Horreum/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Datastore.java) are also omitted and will have to be manually added in a separate step. + +## TestSchema + +The export object for Schemas is called [SchemaExport](https://github.com/Hyperfoil/Horreum/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/SchemaExport.java) and contains other fields in addition to what's defined in [Schema](https://github.com/Hyperfoil/Horreum/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java). This includes, labels, extractors and transformers. This is to simplify the import/export experience and make sure that all the data related to a Schema has a single entrypoint with regards to import and export. + +## Import Schemas + +```bash +curl 'http://localhost:8080/api/schema/import/' \ + -s -X POST -H 'content-type: application/json' \ + -H 'Authorization: Bearer '$TOKEN \ + -d @/path/to/schema.json +``` + +If you are unfamiliar with creating the auth token please see [Upload Run](/docs/tasks/upload-new-run/). + +## Import Tests + +```bash +curl 'http://localhost:8080/api/test/import/' \ + -s -X POST -H 'content-type: application/json' \ + -H 'Authorization: Bearer '$TOKEN \ + -d @/path/to/test.json +``` + +## Export Schemas + +```bash +SCHEMAID='123' +curl 'http://localhost:8080/api/schema/export/?id='$SCHEMAID \ + -H 'Authorization: Bearer '$TOKEN \ + -O --output-dir /path/to/folder +``` + +## Export Tests + +```bash +TESTID='123' +curl 'http://localhost:8080/api/test/export/?id=$TESTID' \ + -s -X POST -H 'content-type: application/json' \ + -H 'Authorization: Bearer '$TOKEN \ + -O --output-dir /path/to/folder +``` diff --git a/docs/site/content/en/openapi/openapi.yaml b/docs/site/content/en/openapi/openapi.yaml index 8fb15e934..d37b8294f 100644 --- a/docs/site/content/en/openapi/openapi.yaml +++ b/docs/site/content/en/openapi/openapi.yaml @@ -4,7 +4,7 @@ info: title: Horreum REST API description: "Horreum automated change anomaly detection. For more information,\ \ please see [https://horreum.hyperfoil.io/](https://horreum.hyperfoil.io/)" - version: "0.11" + version: "0.12" tags: - name: Config description: Endpoint providing configuration for the Horreum System @@ -1355,16 +1355,81 @@ paths: post: tags: - Schema - description: Import an previously exported Schema + description: Import an previously exported Schema either as a new Schema or + to update an existing Schema operationId: importSchema requestBody: content: application/json: schema: + required: + - access + - owner + - id + - uri + - name type: string + properties: + access: + description: Access rights for the test. This defines the visibility + of the Test in the UI + enum: + - PUBLIC + - PROTECTED + - PRIVATE + type: string + allOf: + - $ref: '#/components/schemas/Access' + example: PUBLIC + nullable: false + owner: + description: Name of the team that owns the test. Users must belong + to the team that owns a test to make modifications + type: string + example: performance-team + nullable: false + id: + format: int32 + description: Unique Schema ID + type: integer + example: 101 + uri: + description: "Unique, versioned schema URI" + type: string + example: uri:my-schema:0.1 + nullable: false + name: + description: Schema name + type: string + example: My benchmark schema + nullable: false + description: + description: Schema Description + type: string + example: Schema for processing my benchmark + schema: + description: JSON validation schema. Used to validate uploaded JSON + documents + type: string + example: "{ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\"\ + , \"$id\": \"https://example.com/product.schema.json\", \"title\"\ + : \"Product\", \"description\": \"A product in the catalog\"\ + , \"type\": \"object\"}" + token: + description: Array of API tokens associated with test + type: string + example: "" + labels: + type: array + items: + $ref: '#/components/schemas/Label' + transformers: + type: array + items: + $ref: '#/components/schemas/Transformer' responses: - "201": - description: Created + "204": + description: Import a new Schema or update an existing Schema /api/schema/{id}: get: tags: @@ -1449,11 +1514,11 @@ paths: example: 101 responses: "200": - description: A JSON representation of the Schema object + description: A JSON representation of the SchemaExport object content: application/json: schema: - type: string + $ref: '#/components/schemas/SchemaExport' /api/schema/{id}/resetToken: post: tags: @@ -1780,16 +1845,134 @@ paths: post: tags: - Test - description: Import a previously exported Test + description: Import a previously exported Test either as a new Test or to update + an existing Test operationId: importTest requestBody: content: application/json: schema: + required: + - access + - owner + - id + - name + - datastoreId + - notificationsEnabled type: string + properties: + access: + description: Access rights for the test. This defines the visibility + of the Test in the UI + enum: + - PUBLIC + - PROTECTED + - PRIVATE + type: string + allOf: + - $ref: '#/components/schemas/Access' + example: PUBLIC + nullable: false + owner: + description: Name of the team that owns the test. Users must belong + to the team that owns a test to make modifications + type: string + example: performance-team + nullable: false + id: + format: int32 + description: Unique Test id + type: integer + example: 101 + name: + description: Test name + type: string + example: my-comprehensive-benchmark + nullable: false + folder: + description: Name of folder that the test is stored in. Folders + allow tests to be organised in the UI + type: string + example: My Team Folder + description: + description: Description of the test + type: string + example: Comprehensive benchmark to tests the limits of any system + it is run against + datastoreId: + format: int32 + description: backend ID for backing datastore + type: integer + nullable: false + tokens: + description: Array of API tokens associated with test + type: array + items: + $ref: '#/components/schemas/TestToken' + timelineLabels: + description: List of label names that are used for determining metric + to use as the time series + type: array + items: + type: string + example: + - timestamp + timelineFunction: + description: Label function to modify timeline labels to a produce + a value used for ordering datapoints + type: string + example: timestamp => timestamp + fingerprintLabels: + description: 'Array of Label names that are used to create a fingerprint ' + type: array + items: + type: string + example: + - build_tag + fingerprintFilter: + description: Filter function to filter out datasets that are comparable + for the purpose of change detection + type: string + example: value => value === "true" + compareUrl: + description: URL to external service that can be called to compare + runs. This is typically an external reporting/visulization service + type: string + example: "(ids, token) => 'http://repoting.example.com/report/specj?q='\ + \ + ids.join('&q=') + \"&token=\"+token" + transformers: + description: Array for transformers defined for the Test + type: array + items: + $ref: '#/components/schemas/Transformer' + notificationsEnabled: + description: Are notifications enabled for the test + type: boolean + example: true + nullable: false + variables: + type: array + items: + $ref: '#/components/schemas/Variable' + missingDataRules: + type: array + items: + $ref: '#/components/schemas/MissingDataRule' + experiments: + type: array + items: + $ref: '#/components/schemas/ExperimentProfile' + actions: + type: array + items: + $ref: '#/components/schemas/Action' + subscriptions: + $ref: '#/components/schemas/Watch' + datastore: + $ref: '#/components/schemas/Datastore' responses: "204": - description: Import a new test + description: Import a new Test or update an existing Test /api/test/summary: get: tags: @@ -1898,11 +2081,11 @@ paths: type: integer responses: "200": - description: A Test defintion formatted as json + description: A Test definition formatted as json content: application/json: schema: - type: string + $ref: '#/components/schemas/TestExport' /api/test/{id}/fingerprint: get: tags: @@ -2174,6 +2357,41 @@ components: - PROTECTED - PRIVATE type: string + Action: + required: + - id + - event + - type + - config + - testId + - active + - runAlways + type: object + properties: + id: + format: int32 + type: integer + event: + type: string + nullable: false + type: + type: string + nullable: false + config: + type: array + nullable: false + testId: + format: int32 + type: integer + nullable: false + active: + type: boolean + nullable: false + runAlways: + type: boolean + nullable: false + secrets: + type: array ActionLog: description: Action Log required: @@ -2199,6 +2417,21 @@ components: - SAME - WORSE type: string + ChangeDetection: + required: + - id + - model + - config + type: object + properties: + id: + format: int32 + type: integer + model: + type: string + config: + type: array + nullable: false ComparisonResult: description: Result of performing a Comparison type: object @@ -2983,6 +3216,34 @@ components: \ array or JSON object" type: string example: "1724" + MissingDataRule: + required: + - id + - maxStaleness + - testId + type: object + properties: + id: + format: int32 + type: integer + name: + type: string + labels: + type: array + items: + type: string + condition: + type: string + maxStaleness: + format: int64 + type: integer + nullable: false + lastNotification: + format: date-time + type: string + testId: + format: int32 + type: integer PersistentLog: description: Persistent Log required: @@ -3489,6 +3750,71 @@ components: type: string example: uri:my-schmea:0.1 nullable: false + SchemaExport: + required: + - access + - owner + - id + - uri + - name + type: object + properties: + access: + description: Access rights for the test. This defines the visibility of + the Test in the UI + enum: + - PUBLIC + - PROTECTED + - PRIVATE + type: string + allOf: + - $ref: '#/components/schemas/Access' + example: PUBLIC + nullable: false + owner: + description: Name of the team that owns the test. Users must belong to the + team that owns a test to make modifications + type: string + example: performance-team + nullable: false + id: + format: int32 + description: Unique Schema ID + type: integer + example: 101 + uri: + description: "Unique, versioned schema URI" + type: string + example: uri:my-schema:0.1 + nullable: false + name: + description: Schema name + type: string + example: My benchmark schema + nullable: false + description: + description: Schema Description + type: string + example: Schema for processing my benchmark + schema: + description: JSON validation schema. Used to validate uploaded JSON documents + type: string + example: "{ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\"\ + , \"$id\": \"https://example.com/product.schema.json\", \"title\": \"\ + Product\", \"description\": \"A product in the catalog\", \"type\":\ + \ \"object\"}" + token: + description: Array of API tokens associated with test + type: string + example: "" + labels: + type: array + items: + $ref: '#/components/schemas/Label' + transformers: + type: array + items: + $ref: '#/components/schemas/Transformer' SchemaQueryResult: required: - schemas @@ -3966,6 +4292,125 @@ components: type: boolean example: true nullable: false + TestExport: + required: + - access + - owner + - id + - name + - datastoreId + - notificationsEnabled + type: object + properties: + access: + description: Access rights for the test. This defines the visibility of + the Test in the UI + enum: + - PUBLIC + - PROTECTED + - PRIVATE + type: string + allOf: + - $ref: '#/components/schemas/Access' + example: PUBLIC + nullable: false + owner: + description: Name of the team that owns the test. Users must belong to the + team that owns a test to make modifications + type: string + example: performance-team + nullable: false + id: + format: int32 + description: Unique Test id + type: integer + example: 101 + name: + description: Test name + type: string + example: my-comprehensive-benchmark + nullable: false + folder: + description: Name of folder that the test is stored in. Folders allow tests + to be organised in the UI + type: string + example: My Team Folder + description: + description: Description of the test + type: string + example: Comprehensive benchmark to tests the limits of any system it is + run against + datastoreId: + format: int32 + description: backend ID for backing datastore + type: integer + nullable: false + tokens: + description: Array of API tokens associated with test + type: array + items: + $ref: '#/components/schemas/TestToken' + timelineLabels: + description: List of label names that are used for determining metric to + use as the time series + type: array + items: + type: string + example: + - timestamp + timelineFunction: + description: Label function to modify timeline labels to a produce a value + used for ordering datapoints + type: string + example: timestamp => timestamp + fingerprintLabels: + description: 'Array of Label names that are used to create a fingerprint ' + type: array + items: + type: string + example: + - build_tag + fingerprintFilter: + description: Filter function to filter out datasets that are comparable + for the purpose of change detection + type: string + example: value => value === "true" + compareUrl: + description: URL to external service that can be called to compare runs. This + is typically an external reporting/visulization service + type: string + example: "(ids, token) => 'http://repoting.example.com/report/specj?q='\ + \ + ids.join('&q=') + \"&token=\"+token" + transformers: + description: Array for transformers defined for the Test + type: array + items: + $ref: '#/components/schemas/Transformer' + notificationsEnabled: + description: Are notifications enabled for the test + type: boolean + example: true + nullable: false + variables: + type: array + items: + $ref: '#/components/schemas/Variable' + missingDataRules: + type: array + items: + $ref: '#/components/schemas/MissingDataRule' + experiments: + type: array + items: + $ref: '#/components/schemas/ExperimentProfile' + actions: + type: array + items: + $ref: '#/components/schemas/Action' + subscriptions: + $ref: '#/components/schemas/Watch' + datastore: + $ref: '#/components/schemas/Datastore' TestListing: type: object properties: @@ -4209,6 +4654,43 @@ components: allOf: - $ref: '#/components/schemas/ErrorDetails' - description: Validation Error Details + Variable: + required: + - id + - testId + - name + - order + - labels + - changeDetection + type: object + properties: + id: + format: int32 + type: integer + testId: + format: int32 + type: integer + nullable: false + name: + type: string + nullable: false + group: + type: string + order: + format: int32 + type: integer + nullable: false + labels: + type: array + items: + type: string + nullable: false + calculation: + type: string + changeDetection: + type: array + items: + $ref: '#/components/schemas/ChangeDetection' VersionInfo: required: - version @@ -4225,3 +4707,32 @@ components: description: Timestamp of server startup type: integer example: 2023-10-18 18:00:57 + Watch: + required: + - users + - optout + - teams + - testId + type: object + properties: + id: + format: int32 + type: integer + users: + type: array + items: + type: string + nullable: false + optout: + type: array + items: + type: string + nullable: false + teams: + type: array + items: + type: string + nullable: false + testId: + format: int32 + type: integer diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/ChangeDetection.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/ChangeDetection.java index 6daa212b7..6f42e2402 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/ChangeDetection.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/ChangeDetection.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import jakarta.validation.constraints.NotNull; -import org.eclipse.microprofile.openapi.annotations.media.Schema; public class ChangeDetection { @JsonProperty( required = true ) diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/Variable.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/Variable.java index a2ee91934..732a8b0d5 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/Variable.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/Variable.java @@ -2,10 +2,10 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonNode; import jakarta.validation.constraints.NotNull; import org.eclipse.microprofile.openapi.annotations.media.Schema; +import java.util.List; import java.util.Set; public class Variable { @@ -23,19 +23,16 @@ public class Variable { public int order; @NotNull @JsonProperty(required = true) - public JsonNode labels; + public List labels; @JsonInclude(JsonInclude.Include.NON_NULL) public String calculation; - @Schema( - required = true, - implementation = ChangeDetection[].class - ) + @Schema(required = true, implementation = ChangeDetection[].class) public Set changeDetection; public Variable() { } - public Variable(Integer id, int testId, String name, String group, int order, JsonNode labels, String calculation, + public Variable(Integer id, int testId, String name, String group, int order, List labels, String calculation, Set changeDetection) { this.id = id; this.testId = testId; diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/ExperimentProfile.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/ExperimentProfile.java index 52ec52bc1..4e217bbb2 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/ExperimentProfile.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/ExperimentProfile.java @@ -1,6 +1,8 @@ package io.hyperfoil.tools.horreum.api.data; +import com.fasterxml.jackson.annotation.JsonIdentityInfo; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.ObjectIdGenerators; import com.fasterxml.jackson.databind.JsonNode; import jakarta.validation.constraints.NotNull; import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; @@ -8,6 +10,7 @@ import java.util.Collection; +@JsonIdentityInfo( property = "id", generator = ObjectIdGenerators.PropertyGenerator.class) @Schema(description = "An Experiment Profile defines the labels and filters for the dataset and baseline") public class ExperimentProfile { @JsonProperty(required = true ) diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Label.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Label.java index 32f940890..15263006b 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Label.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Label.java @@ -44,6 +44,11 @@ public class Label extends ProtectedType { public Label() { } + public Label(String name, int schemaId) { + this.name = name; + this.schemaId = schemaId; + } + public static class Value implements Serializable { public int datasetId; public int labelId; diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java index 2c0915274..d8d333187 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java @@ -46,6 +46,17 @@ public Schema() { access = Access.PUBLIC; } + public Schema(Schema s) { + this.id = s.id; + this.uri = s.uri; + this.name = s.name; + this.description = s.description; + this.schema = s.schema; + this.token = s.token; + this.access = s.access; + this.owner = s.owner; + } + public static class ValidationEvent { public int id; public Collection errors; diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/SchemaExport.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/SchemaExport.java new file mode 100644 index 000000000..1d7f8e6aa --- /dev/null +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/SchemaExport.java @@ -0,0 +1,17 @@ +package io.hyperfoil.tools.horreum.api.data; + +import java.util.List; + +public class SchemaExport extends Schema { + + public List