diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/asset.v1.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/asset.v1.json new file mode 100644 index 0000000000000..071ae8285c938 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/asset.v1.json @@ -0,0 +1,17 @@ +{ + "title": "Asset", + "type": "object", + "description": "Item installed for Kibana (e.g. dashboard, visualization), Elasticsearch (e.g. ingest pipeline, ILM policy), or a Kibana plugin (e.g. ML job)", + "properties": { + "id": { + "type": "string" + }, + "type": { + "$ref": "./asset_type.v1.json" + } + }, + "required": [ + "id", + "type" + ] +} diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/asset_type.v1.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/asset_type.v1.json new file mode 100644 index 0000000000000..837115982f198 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/asset_type.v1.json @@ -0,0 +1,13 @@ +{ + "type": "string", + "title": "AssetType", + "description": "Types of assets which can be installed/removed", + "enum": [ + "index-template", + "ingest-pipeline", + "ilm-policy", + "rollup-job", + "ml-job", + "data-frame-transform" + ] + } diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/datasource.v1.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/datasource.v1.json new file mode 100644 index 0000000000000..b25d5040d149f --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/datasource.v1.json @@ -0,0 +1,31 @@ +{ + "title": "Datasource", + "type": "object", + "description": "A logical grouping of places where data is coming from, such as \"Production\", \"Staging\", \"Production East-1\", \"Metrics Cluster\", etc. -- these groupings are user-defined. We store information collected from the user about this logical grouping such as a name and any other information we need about it to generate the associated config. A package defines its own data source templates that can use user-provided values to generate the data source config. A single data source will typically enable users to collect both logs and metrics. A data source can be in multiple policies at the same time. A datasource can have multiple streams.", + "properties": { + "name": { + "type": "string" + }, + "package": { + "$ref": "./package.v1.json" + }, + "streams": { + "type": "array", + "items": { + "$ref": "./stream.v1.json" + } + }, + "id": { + "type": "string", + "format": "uuid" + }, + "read_alias": { + "type": "string" + } + }, + "required": [ + "name", + "package", + "streams" + ] +} diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/input.v1.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/input.v1.json new file mode 100644 index 0000000000000..01fe26df97cee --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/input.v1.json @@ -0,0 +1,50 @@ +{ + "title": "Input", + "type": "object", + "description": "Where the data comes from", + "properties": { + "type": { + "type": "string", + "enum": [ + "log", + "metric/system", + "metric/docker", + "etc" + ] + }, + "config": { + "type": "object", + "example": "{paths: \"/var/log/*.log\"} or {metricsets: [\"container\", \"cpu\"]} or {username: \"elastic\", password: \"changeme\"}", + "description": "Mix of configurable and required properties still TBD. Object for now might become string" + }, + "ingest_pipelines": { + "type": "array", + "description": "Need a distinction for \"main\" ingest pipeline. Should be handled during install. Likely by package/manifest format", + "items": { + "type": "string" + } + }, + "id": { + "type": "string", + "format": "uuid" + }, + "index_template": { + "type": "string" + }, + "ilm_policy": { + "type": "string" + }, + "fields": { + "type": "array", + "description": "", + "items": { + "type": "object", + "description": "contents from fields.yml" + } + } + }, + "required": [ + "type", + "config" + ] +} diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/output.v1.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/output.v1.json new file mode 100644 index 0000000000000..c21cc27b80fcf --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/output.v1.json @@ -0,0 +1,47 @@ +{ + "title": "Output", + "type": "object", + "description": "Where to send the data", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string", + "example": "\"default\" or \"infosec1\"" + }, + "type": { + "type": "string", + "enum": [ + "elasticsearch", + "something", + "else" + ] + }, + "url": { + "type": "string", + "format": "uri" + }, + "api_token": { + "type": "string" + }, + "index_name": { + "type": "string", + "example": "metrics-mysql-prod_west-access", + "description": "unique alias with write index" + }, + "ingest_pipeline": { + "type": "string", + "example": "metrics-mysql-prod_west-access" + }, + "config": { + "type": "object", + "description": "contains everything not otherwise specified (e.g. TLS, etc)" + } + }, + "required": [ + "id", + "name", + "type" + ] +} diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/package.v1.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/package.v1.json new file mode 100644 index 0000000000000..68b16f5a24202 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/package.v1.json @@ -0,0 +1,34 @@ +{ + "title": "Package", + "type": "object", + "description": "A group of items related to a data ingestion source (e.g. MySQL, nginx, AWS). Can include Kibana assets, ES assets, data source configuration templates, manual install steps, etc.", + "properties": { + "name": { + "type": "string", + "example": "coredns" + }, + "version": { + "type": "string", + "example": "1.0.1, 1.3.1" + }, + "description": { + "type": "string", + "example": "CoreDNS logs and metrics integration.\nThe CoreDNS integrations allows to gather logs and metrics from the CoreDNS DNS server to get better insights.\n" + }, + "title": { + "type": "string", + "example": "CoreDNS" + }, + "assets": { + "type": "array", + "items": { + "$ref": "./asset.v1.json" + } + } + }, + "required": [ + "name", + "version", + "assets" + ] +} diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/policy.v1.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/policy.v1.json new file mode 100644 index 0000000000000..1228e9a97d168 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/policy.v1.json @@ -0,0 +1,37 @@ +{ + "title": "Policy", + "type": "object", + "description": "The entire config for the Beats agent, including all assigned data source config outputs along with agent-wide configuration values", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "use_case": { + "type": "string" + }, + "datasources": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "./datasource.v1.json" + } + }, + "description": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + } + }, + "required": [ + "id", + "status" + ] +} diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/stream.v1.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/stream.v1.json new file mode 100644 index 0000000000000..76c531d3cd317 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/models/stream.v1.json @@ -0,0 +1,31 @@ +{ + "title": "Stream", + "type": "object", + "description": "A combination of an input type, the required config, an output, and any processors", + "properties": { + "id": { + "type": "string" + }, + "input": { + "$ref": "./input.v1.json" + }, + "config": { + "type": "object", + "example": "{paths: \"/var/log/*.log\"} or {metricsets: [\"container\", \"cpu\"]} or {username: \"elastic\", password: \"changeme\"}" + }, + "output": { + "$ref": "./output.v1.json" + }, + "processors": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "id", + "input", + "output" + ] +} diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/openapi.json b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/openapi.json new file mode 100755 index 0000000000000..bc3e8aeffdb24 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/openapi.json @@ -0,0 +1,160 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Ingest", + "version": "0.2", + "description": "Strawman API for fka Ingest Plugin", + "license": { + "url": "https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt", + "name": "Elastic" + }, + "contact": { + "name": "Elastic Observability Team" + } + }, + "servers": [ + { + "url": "http://localhost:5601/api/epm", + "description": "EPM API Root" + } + ], + "paths": { + "/policies": { + "get": { + "summary": "Get policies", + "tags": [], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "./models/policy.v1.json" + } + } + } + } + } + }, + "parameters": [ + { + "schema": { + "type": "string" + }, + "in": "query", + "name": "datasource" + }, + { + "schema": { + "type": "string" + }, + "in": "query", + "name": "use_case" + }, + { + "$ref": "#/components/parameters/pageIndexParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" + } + ], + "description": "Return one or many polices based on the given filter", + "operationId": "getPolicies" + }, + "parameters": [] + }, + "/datasources": { + "get": { + "summary": "Get datasources", + "tags": [], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "./models/datasource.v1.json" + } + } + } + } + } + }, + "parameters": [ + { + "schema": { + "type": "string" + }, + "in": "query", + "name": "policy_id" + }, + { + "$ref": "#/components/parameters/pageIndexParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" + } + ], + "description": "Return datasources", + "operationId": "getDatasources" + } + }, + "/policy/{policy_id}": { + "parameters": [ + { + "name": "policy_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "get": { + "summary": "Get policy by id", + "tags": [], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "./models/policy.v1.json" + } + } + } + } + }, + "description": "Get Policy by id", + "operationId": "getPolicyById" + } + } + }, + "components": { + "parameters": { + "pageSizeParam": { + "name": "per_page", + "in": "query", + "description": "The number of items to return", + "required": false, + "schema": { + "type": "integer", + "default": 50 + } + }, + "pageIndexParam": { + "name": "page", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "default": 1 + } + } + } + } +} diff --git a/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/types.ts b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/types.ts new file mode 100644 index 0000000000000..967676b080871 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/common/api_specs/ingest/types.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/** + * The entire config for the Beats agent, including all assigned data source config outputs + * along with agent-wide configuration values + */ +export interface Policy { + datasources?: Datasource[]; + description?: string; + id: string; + name?: string; + status: Status; + use_case?: string; +} + +/** + * A logical grouping of places where data is coming from, such as "Production", "Staging", + * "Production East-1", "Metrics Cluster", etc. -- these groupings are user-defined. We + * store information collected from the user about this logical grouping such as a name and + * any other information we need about it to generate the associated config. A package + * defines its own data source templates that can use user-provided values to generate the + * data source config. A single data source will typically enable users to collect both logs + * and metrics. A data source can be in multiple policies at the same time. A datasource can + * have multiple streams. + */ +export interface Datasource { + id?: string; + name: string; + package: Package; + read_alias?: string; + streams: Stream[]; +} + +/** + * A group of items related to a data ingestion source (e.g. MySQL, nginx, AWS). Can include + * Kibana assets, ES assets, data source configuration templates, manual install steps, etc. + */ +export interface Package { + assets: Asset[]; + description?: string; + name: string; + title?: string; + version: string; +} + +/** + * Item installed for Kibana (e.g. dashboard, visualization), Elasticsearch (e.g. ingest + * pipeline, ILM policy), or a Kibana plugin (e.g. ML job) + */ +export interface Asset { + id: string; + type: AssetType; +} + +/** + * Types of assets which can be installed/removed + */ +export enum AssetType { + DataFrameTransform = 'data-frame-transform', + IlmPolicy = 'ilm-policy', + IndexTemplate = 'index-template', + IngestPipeline = 'ingest-pipeline', + MlJob = 'ml-job', + RollupJob = 'rollup-job', +} + +/** + * A combination of an input type, the required config, an output, and any processors + */ +export interface Stream { + config?: { [key: string]: any }; + id: string; + input: Input; + output: Output; + processors?: string[]; +} + +/** + * Where the data comes from + */ +export interface Input { + /** + * Mix of configurable and required properties still TBD. Object for now might become string + */ + config: { [key: string]: any }; + fields?: Array<{ [key: string]: any }>; + id?: string; + ilm_policy?: string; + index_template?: string; + /** + * Need a distinction for "main" ingest pipeline. Should be handled during install. Likely + * by package/manifest format + */ + ingest_pipelines?: string[]; + type: InputType; +} + +export enum InputType { + Etc = 'etc', + Log = 'log', + MetricDocker = 'metric/docker', + MetricSystem = 'metric/system', +} + +/** + * Where to send the data + */ +export interface Output { + api_token?: string; + /** + * contains everything not otherwise specified (e.g. TLS, etc) + */ + config?: { [key: string]: any }; + id: string; + /** + * unique alias with write index + */ + index_name?: string; + ingest_pipeline?: string; + name: string; + type: OutputType; + url?: string; +} + +export enum OutputType { + Elasticsearch = 'elasticsearch', + Else = 'else', + Something = 'something', +} + +export enum Status { + Active = 'active', + Inactive = 'inactive', +}