diff --git a/x-pack/plugins/dataset_quality/public/components/dataset_quality/dataset_quality.tsx b/x-pack/plugins/dataset_quality/public/components/dataset_quality/dataset_quality.tsx index 9fe6ca8db3b2f..9e4d38bb86283 100644 --- a/x-pack/plugins/dataset_quality/public/components/dataset_quality/dataset_quality.tsx +++ b/x-pack/plugins/dataset_quality/public/components/dataset_quality/dataset_quality.tsx @@ -7,30 +7,31 @@ import React from 'react'; import { CoreStart } from '@kbn/core/public'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { DataStreamsStatsService } from '../../services/data_streams_stats/data_streams_stats_service'; import { DatasetQualityContext, DatasetQualityContextValue } from './context'; import { useKibanaContextForPluginProvider } from '../../utils'; import { DatasetQualityStartDeps } from '../../types'; import { Header } from './header'; import { Table } from './table'; +import { DataStreamsStatsServiceStart } from '../../services/data_streams_stats'; export interface CreateDatasetQualityArgs { core: CoreStart; plugins: DatasetQualityStartDeps; + dataStreamsStatsService: DataStreamsStatsServiceStart; } -export const createDatasetQuality = ({ core, plugins }: CreateDatasetQualityArgs) => { +export const createDatasetQuality = ({ + core, + plugins, + dataStreamsStatsService, +}: CreateDatasetQualityArgs) => { + const datasetQualityProviderValue: DatasetQualityContextValue = { + dataStreamsStatsServiceClient: dataStreamsStatsService.client, + }; + return () => { const KibanaContextProviderForPlugin = useKibanaContextForPluginProvider(core, plugins); - const dataStreamsStatsServiceClient = new DataStreamsStatsService().start({ - http: core.http, - }).client; - - const datasetQualityProviderValue: DatasetQualityContextValue = { - dataStreamsStatsServiceClient, - }; - return ( diff --git a/x-pack/plugins/dataset_quality/public/index.ts b/x-pack/plugins/dataset_quality/public/index.ts index e57d36776edfd..e79effd160a38 100644 --- a/x-pack/plugins/dataset_quality/public/index.ts +++ b/x-pack/plugins/dataset_quality/public/index.ts @@ -9,6 +9,8 @@ import type { PluginInitializerContext } from '@kbn/core/public'; import { DatasetQualityConfig } from '../common/plugin_config'; import { DatasetQualityPlugin } from './plugin'; +export type { IDataStreamsStatsClient } from './services/data_streams_stats'; + export type { DatasetQualityPluginSetup, DatasetQualityPluginStart } from './types'; export function plugin(context: PluginInitializerContext) { diff --git a/x-pack/plugins/dataset_quality/public/plugin.tsx b/x-pack/plugins/dataset_quality/public/plugin.tsx index c2ab655422631..b278463201980 100644 --- a/x-pack/plugins/dataset_quality/public/plugin.tsx +++ b/x-pack/plugins/dataset_quality/public/plugin.tsx @@ -7,6 +7,7 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import { createDatasetQuality } from './components/dataset_quality'; +import { DataStreamsStatsService } from './services/data_streams_stats'; import { DatasetQualityPluginSetup, DatasetQualityPluginStart, @@ -17,18 +18,30 @@ import { export class DatasetQualityPlugin implements Plugin { + dataStreamsStatsService = new DataStreamsStatsService(); + constructor(context: PluginInitializerContext) {} public setup(core: CoreSetup, plugins: DatasetQualitySetupDeps) { + this.dataStreamsStatsService.setup(); + return {}; } public start(core: CoreStart, plugins: DatasetQualityStartDeps): DatasetQualityPluginStart { + const dataStreamsStatsService = this.dataStreamsStatsService.start({ + http: core.http, + }); + const DatasetQuality = createDatasetQuality({ core, plugins, + dataStreamsStatsService, }); - return { DatasetQuality }; + return { + DatasetQuality, + dataStreamsStatsService, + }; } } diff --git a/x-pack/plugins/dataset_quality/public/types.ts b/x-pack/plugins/dataset_quality/public/types.ts index 482aff3b242b9..b60e2b40cdb12 100644 --- a/x-pack/plugins/dataset_quality/public/types.ts +++ b/x-pack/plugins/dataset_quality/public/types.ts @@ -5,15 +5,17 @@ * 2.0. */ -import { ComponentType } from 'react'; +import type { ComponentType } from 'react'; import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public'; -import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; +import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; +import type { DataStreamsStatsServiceStart } from './services/data_streams_stats'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface DatasetQualityPluginSetup {} export interface DatasetQualityPluginStart { DatasetQuality: ComponentType; + dataStreamsStatsService: DataStreamsStatsServiceStart; } export interface DatasetQualityStartDeps { diff --git a/x-pack/plugins/observability_log_explorer/public/applications/observability_log_explorer.tsx b/x-pack/plugins/observability_log_explorer/public/applications/observability_log_explorer.tsx index a823ad1a840cd..bc7d10515a452 100644 --- a/x-pack/plugins/observability_log_explorer/public/applications/observability_log_explorer.tsx +++ b/x-pack/plugins/observability_log_explorer/public/applications/observability_log_explorer.tsx @@ -10,6 +10,7 @@ import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { Route, Router, Routes } from '@kbn/shared-ux-router'; import React from 'react'; import ReactDOM from 'react-dom'; +import { IngestPathwaysRoute } from '../routes/ingest_pathways'; import { DatasetQualityRoute, ObservabilityLogExplorerMainRoute } from '../routes/main'; import { ObservabilityLogExplorerAppMountParameters, @@ -76,6 +77,7 @@ export const ObservabilityLogExplorerApp = ({ exact={true} render={() => } /> + diff --git a/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/graph_visualization.tsx b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/graph_visualization.tsx new file mode 100644 index 0000000000000..8e2f53e8c58b3 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/graph_visualization.tsx @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { css } from '@emotion/react'; +import { useActor } from '@xstate/react'; +import cytoscape, { CytoscapeOptions, EdgeSingular, NodeSingular } from 'cytoscape'; +import dagre from 'cytoscape-dagre'; +import React, { useCallback, useEffect, useState } from 'react'; +import { Agent, useIngestPathwaysPageStateContext } from '../../state_machines/ingest_pathways'; + +export const ConnectedGraphVisualization = React.memo(() => { + const [state, send] = useActor(useIngestPathwaysPageStateContext()); + + const onSelectPathway = useCallback( + (pathwayId: string) => { + send({ + type: 'selectPathway', + pathwayId, + }); + }, + [send] + ); + + return ( + + ); +}); + +export const GraphVisualization = React.memo( + ({ + graphOptions, + onSelectPathway, + }: { + graphOptions: CytoscapeOptions; + onSelectPathway: (pathwayId: string) => void; + }) => { + const [cytoscapeInstance] = useState(() => { + cytoscape.use(dagre); + const newCytoscapeInstance = cytoscape(initialGraphOptions); + newCytoscapeInstance.on('select', 'edge[pathwayId]', (evt) => { + const pathwayId = evt.target.data('pathwayId'); + // onSelectPathway(pathwayId); + // console.log(evt); + const edges = evt.cy.$(`edge[pathwayId="${pathwayId}"]`); + const nodes = edges.connectedNodes(); + edges.select(); + nodes.select(); + }); + return newCytoscapeInstance; + }); + + useEffect(() => { + cytoscapeInstance.json(graphOptions); + cytoscapeInstance.layout(initialGraphOptions.layout!).run(); + }, [cytoscapeInstance, graphOptions]); + + return ( +
{ + if (elem != null) { + cytoscapeInstance.mount(elem); + } else { + cytoscapeInstance.unmount(); + } + }} + /> + ); + } +); + +const initialGraphOptions: CytoscapeOptions & { layout: Record } = { + elements: [], + autoungrabify: true, + style: [ + { + selector: '*', + style: { + 'font-size': '12px', + }, + }, + { + selector: 'node.agent', + style: { + label: (elem: NodeSingular) => { + const agent = elem.data('agent'); + return `${agent.name}\n${agent.type} ${agent.version}`; + }, + 'text-wrap': 'wrap', + shape: 'ellipse', + 'text-valign': 'center', + 'text-halign': 'left', + }, + }, + { + selector: 'node.dataStream', + style: { + label: 'data(dataStream.id)', + shape: 'hexagon', + 'text-valign': 'center', + 'text-halign': 'right', + }, + }, + { + selector: 'node.ingestPipeline', + style: { + label: 'data(ingestPipeline.id)', + shape: 'diamond', + }, + }, + { + selector: 'edge', + style: { + width: 1, + 'curve-style': 'bezier', + }, + }, + { + selector: 'edge.shipsTo', + style: { + 'target-arrow-shape': 'chevron', + 'target-arrow-fill': 'filled', + }, + }, + { + selector: 'edge.agentShipsTo', + style: { + // label: 'data(shipsTo.signalCount)', + width: (edge: EdgeSingular) => { + const agent: Agent = edge.data('agent'); + const totalSignalCount = agent.shipsTo.reduce( + (accumulatedSignalCount, { signalCount }) => accumulatedSignalCount + signalCount, + 0 + ); + return 1 + (9.0 / totalSignalCount) * edge.data('shipsTo').signalCount; + }, + }, + }, + ], + layout: { + name: 'dagre', + rankDir: 'LR', + rankSep: 300, + nodeSep: 30, + ranker: 'longest-path', + }, +}; + +const graphStyles = css({ + flex: '1 0 0%', + alignSelf: 'stretch', +}); diff --git a/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/index.ts b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/index.ts new file mode 100644 index 0000000000000..fa24e5980ae76 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './ingest_pathways'; diff --git a/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/ingest_pathways.tsx b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/ingest_pathways.tsx new file mode 100644 index 0000000000000..a614c4741501a --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/ingest_pathways.tsx @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiPageHeaderProps } from '@elastic/eui'; +import React, { useMemo } from 'react'; +import { ObservabilityLogExplorerPageTemplate } from '../page_template'; +import { ConnectedGraphVisualization } from './graph_visualization'; +import { ConnectedLoadingIndicator } from './loading_indicator'; +import { ConnectedReloadButton } from './reload_button'; + +export const IngestPathways = React.memo(() => { + const pageHeaderProps = useMemo( + (): EuiPageHeaderProps => ({ + alignItems: 'center', + bottomBorder: 'extended', + pageTitle: 'Ingest Pathways', + rightSideItems: [, ], + }), + [] + ); + + return ( + + + + ); +}); diff --git a/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/loading_indicator.tsx b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/loading_indicator.tsx new file mode 100644 index 0000000000000..1c9dbe84fda5d --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/loading_indicator.tsx @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiBadge } from '@elastic/eui'; +import { useActor } from '@xstate/react'; +import React from 'react'; +import { useIngestPathwaysPageStateContext } from '../../state_machines/ingest_pathways'; + +export const ConnectedLoadingIndicator = React.memo(() => { + const [state] = useActor(useIngestPathwaysPageStateContext()); + + return {`${state.value}`}; +}); diff --git a/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/reload_button.tsx b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/reload_button.tsx new file mode 100644 index 0000000000000..f6f580c174582 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/reload_button.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButton } from '@elastic/eui'; +import { useActor } from '@xstate/react'; +import React from 'react'; +import { useIngestPathwaysPageStateContext } from '../../state_machines/ingest_pathways'; + +export const ConnectedReloadButton = React.memo(() => { + const [, send] = useActor(useIngestPathwaysPageStateContext()); + + return ( + { + send({ + type: 'load', + }); + }} + > + Reload + + ); +}); diff --git a/x-pack/plugins/observability_log_explorer/public/components/page_template.tsx b/x-pack/plugins/observability_log_explorer/public/components/page_template.tsx index 5b57333dd8c86..db00544640987 100644 --- a/x-pack/plugins/observability_log_explorer/public/components/page_template.tsx +++ b/x-pack/plugins/observability_log_explorer/public/components/page_template.tsx @@ -5,15 +5,17 @@ * 2.0. */ -import { EuiPageSectionProps } from '@elastic/eui'; +import { EuiPageHeaderProps, EuiPageSectionProps } from '@elastic/eui'; import { css } from '@emotion/react'; import React from 'react'; import { useKibanaContextForPlugin } from '../utils/use_kibana'; export const ObservabilityLogExplorerPageTemplate = ({ children, + pageHeaderProps, pageProps, }: React.PropsWithChildren<{ + pageHeaderProps?: EuiPageHeaderProps; pageProps?: EuiPageSectionProps; }>) => { const { @@ -22,6 +24,7 @@ export const ObservabilityLogExplorerPageTemplate = ({ return ( {children} diff --git a/x-pack/plugins/observability_log_explorer/public/routes/ingest_pathways/index.ts b/x-pack/plugins/observability_log_explorer/public/routes/ingest_pathways/index.ts new file mode 100644 index 0000000000000..90e15e0d9b538 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/routes/ingest_pathways/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './ingest_pathways_route'; diff --git a/x-pack/plugins/observability_log_explorer/public/routes/ingest_pathways/ingest_pathways_route.tsx b/x-pack/plugins/observability_log_explorer/public/routes/ingest_pathways/ingest_pathways_route.tsx new file mode 100644 index 0000000000000..af588c4265141 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/routes/ingest_pathways/ingest_pathways_route.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiEmptyPrompt, EuiLoadingLogo } from '@elastic/eui'; +import { useActor } from '@xstate/react'; +import React from 'react'; +import { IngestPathways } from '../../components/ingest_pathways'; +import { ObservabilityLogExplorerPageTemplate } from '../../components/page_template'; +import { + IngestPathwaysPageStateProvider, + useIngestPathwaysPageStateContext, +} from '../../state_machines/ingest_pathways'; +import { ObservabilityLogExplorerHistory } from '../../types'; +import { noBreadcrumbs, useBreadcrumbs } from '../../utils/breadcrumbs'; +// import { useKbnUrlStateStorageFromRouterContext } from '../../utils/kbn_url_state_context'; +import { useKibanaContextForPlugin } from '../../utils/use_kibana'; + +export const IngestPathwaysRoute = () => { + const { + services: { + chrome, + data: { + search: { search }, + }, + datasetQuality: { + dataStreamsStatsService: { client: dataStreamsStatsClient }, + }, + http, + serverless, + }, + } = useKibanaContextForPlugin(); + + useBreadcrumbs(noBreadcrumbs, chrome, serverless); + + // const urlStateStorageContainer = useKbnUrlStateStorageFromRouterContext(); + + return ( + + + + ); +}; + +const ConnectedContent = React.memo(() => { + const { + services: { + appParams: { history }, + }, + } = useKibanaContextForPlugin(); + + const [state] = useActor(useIngestPathwaysPageStateContext()); + + if (state.matches('uninitialized')) { + return ; + } else { + return ; + } +}); + +const InitializingContent = React.memo(() => ( + + } + title={

Initializing

} + /> + +)); + +const InitializedContent = React.memo( + ({ history }: { history: ObservabilityLogExplorerHistory }) => { + return ; + } +); diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/graph/calculate_graph.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/graph/calculate_graph.ts new file mode 100644 index 0000000000000..031245c9b4b27 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/graph/calculate_graph.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElementDefinition } from 'cytoscape'; +import { + Agent, + DataStreamEntry, + IndexTemplate, + IngestPathwaysData, + IngestPipelineEntry, +} from '../types'; + +export const calculateGraph = ({ + agents, + dataStreams, + indexTemplates, + ingestPipelines, +}: IngestPathwaysData): { elements: ElementDefinition[] } => { + const dataStreamElements = Object.values(dataStreams).flatMap(convertDataStreamToGraphElements); + const agentElements = Object.values(agents).flatMap( + convertAgentToGraphElements({ dataStreams, indexTemplates }) + ); + const ingestPipelineElements = Object.values(ingestPipelines).flatMap( + convertIngestPipelineToGraphElements + ); + + return { + elements: [...dataStreamElements, ...agentElements, ...ingestPipelineElements], + }; +}; + +const convertDataStreamToGraphElements = (dataStream: DataStreamEntry): ElementDefinition[] => [ + { + group: 'nodes', + classes: 'dataStream', + data: { + id: getDataStreamElementId(dataStream), + dataStream, + }, + }, +]; + +const convertAgentToGraphElements = + ({ + dataStreams, + indexTemplates, + }: { + dataStreams: Record; + indexTemplates: Record; + }) => + (agent: Agent): ElementDefinition[] => + [ + { + group: 'nodes', + classes: 'agent', + data: { + id: getAgentElementId(agent), + agent, + }, + }, + ...agent.shipsTo.flatMap((shipsTo): ElementDefinition[] => { + const dataStream = dataStreams[shipsTo.dataStreamId]; + const source = getAgentElementId(agent); + const target = getDataStreamElementId({ + type: 'dataStreamStub', + id: shipsTo.dataStreamId, + }); + const agentDataStreamEdge: ElementDefinition = { + group: 'edges', + classes: 'shipsTo agentShipsTo', + data: { + id: `relation-${source}-ships-to-${target}`, + pathwayId: `pathway-${source}-to-${target}`, + source, + target, + agent, + shipsTo, + }, + }; + + if (dataStream.type === 'dataStream') { + const indexTemplate = indexTemplates[dataStream.indexTemplateId]; + + return indexTemplate.ingestPipelineIds.reduce( + (edges, ingestPipelineId, ingestPipelineIndex, ingestPipelineIds) => { + const lastEdge = edges[edges.length - 1]; + const leadingEdges = edges.slice(0, -1); + + const ingestPipelineElementId = getIngestPipelineElementId({ + type: 'ingestPipelineStub', + id: ingestPipelineId, + }); + + const splitEdges: ElementDefinition[] = [ + { + group: 'edges', + classes: lastEdge.classes, + data: { + id: `relation-${lastEdge.data.source}-ships-to-${ingestPipelineElementId}`, + pathwayId: lastEdge.data.pathwayId, + source: lastEdge.data.source, + target: ingestPipelineElementId, + shipsTo: lastEdge.data.shipsTo, + agent, + }, + }, + { + group: 'edges', + classes: 'shipsTo', + data: { + id: `relation-${ingestPipelineElementId}-ships-to-${lastEdge.data.target}`, + pathwayId: lastEdge.data.pathwayId, + source: ingestPipelineElementId, + target: lastEdge.data.target, + shipsTo: {}, + agent, + }, + }, + ]; + + return [...leadingEdges, ...splitEdges]; + }, + [agentDataStreamEdge] + ); + } else { + return [agentDataStreamEdge]; + } + }), + ]; + +const convertIngestPipelineToGraphElements = ( + ingestPipeline: IngestPipelineEntry +): ElementDefinition[] => [ + { + group: 'nodes', + classes: 'ingestPipeline', + data: { + id: getIngestPipelineElementId(ingestPipeline), + ingestPipeline, + }, + }, +]; + +const getDataStreamElementId = ({ id }: DataStreamEntry) => `dataStream-${id}`; + +const getAgentElementId = ({ id }: Agent) => `agent-${id}`; + +const getIngestPipelineElementId = ({ id }: IngestPipelineEntry) => `ingestPipeline-${id}`; diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/graph/index.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/graph/index.ts new file mode 100644 index 0000000000000..79b191a0455bd --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/graph/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './calculate_graph'; diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/index.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/index.ts new file mode 100644 index 0000000000000..c59cef8ac558b --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './ingest_pathways_provider'; +export * from './ingest_pathways_state_machine'; +export * from './types'; diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/ingest_pathways_provider.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/ingest_pathways_provider.ts new file mode 100644 index 0000000000000..a89c325c5d856 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/ingest_pathways_provider.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getDevToolsOptions } from '@kbn/xstate-utils'; +import { useInterpret } from '@xstate/react'; +import createContainer from 'constate'; +import { + createIngestPathwaysStateMachine, + IngestPathwaysStateMachineDependencies, +} from './ingest_pathways_state_machine'; + +export const useIngestPathwaysPageState = (deps: IngestPathwaysStateMachineDependencies) => { + const ingestPathwaysPageStateService = useInterpret( + () => createIngestPathwaysStateMachine(deps), + { devTools: getDevToolsOptions() } + ); + + return ingestPathwaysPageStateService; +}; + +export const [IngestPathwaysPageStateProvider, useIngestPathwaysPageStateContext] = createContainer( + useIngestPathwaysPageState +); diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/ingest_pathways_state_machine.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/ingest_pathways_state_machine.ts new file mode 100644 index 0000000000000..3b6ba3c17c630 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/ingest_pathways_state_machine.ts @@ -0,0 +1,291 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpStart } from '@kbn/core-http-browser'; +import { ISearchGeneric } from '@kbn/data-plugin/common'; +import { IDataStreamsStatsClient } from '@kbn/dataset-quality-plugin/public'; +import { ElementDefinition } from 'cytoscape'; +import moment from 'moment'; +import { ActionTypes, assign, createMachine } from 'xstate'; +import { calculateGraph } from './graph'; +import { loadDataStreams } from './services/load_data_streams'; +import { loadIndexTemplates } from './services/load_index_templates'; +import { loadIngestPipelines } from './services/load_ingest_pipelines'; +import { loadSignalData } from './services/load_signal_data'; +import { + Agent, + DataStream, + DataStreamStub, + GraphSelection, + IndexTemplate, + IngestPathwaysData, + IngestPathwaysParameters, + IngestPipeline, + IngestPipelineStub, +} from './types'; +import { mergeIngestPathwaysData } from './utils'; + +export const createPureIngestPathwaysStateMachine = (initialContext: IngestPathwaysContext) => + createMachine( + { + /** @xstate-layout N4IgpgJg5mDOIC5QEkB2NYBcAKBDTAFgO64CesAdAK6oCWdmtuANrQF6QDEA2gAwC6iUAAcA9rFqNRqISAAeiAGwB2CgA4VARjXKArABoQpRJoAspigE5r15crUaATGs2OAvm8NoMOfMTKUzKK4EPRQ3nA4tMJgrKhwnBDSYBT0AG6iANYpQSERWNjRsfRwfIJIIGISUjIVCgi86pa8io4GRiaOvADMFKY2lnYOrWruniD5voQk5BS5oegAyrRQqCwAIvi4icmpqBnZc8EQy6sbW2WyVZK00rL1Zn263a7txgjdpppWA0NOLh4vOhInhpgEjiEwmgIGA5AAVMAAW2EzHwCSS8T2Bxyx2hsIRyNRmFKAiu4hudzqiAAtI1FKY1KZunpDO9LG0KMp+pZGVzXJpeJZARNgQU-DNAscwptMLhFpgAE5gXCI2A7THpLI4kIyuWK5Wqy4Va41e6IVTKSyKHRvJSPLnWUyKXSWUyObpqbrCyag-zkTjzI0icmmqkfRwWN3siM9ZnOtSsxAR3QUXhptPdXSObSOCOKb2iqZ+tWwWJgADGRZmQcqIdutVA9V0ml6VpeTtMujsildiYQWd6PczjkUmcsuhU+eFqFEMPgFR94oCZOq9bNCGprksVmZto3ilT6ZaujTX26Ly940XYNmNHoNxY7EgK4pDfkiDdfZcfV+9n+rgLHxfQlCEYQgF9Q0bRBmysMwWgjAZrA0PsWy7H5rFaF5m0UACr0LYDwXmKF8KKOI4Agtcw2pex1C0G0UN4DR0NsP8RjGIEgKXWYiKWFY1mYXUKMpKCNxgrls3ojoEFzb5uRY4ZnE0QCQS4yVIXQPF4SRFE0XnYNV2E98N2-Sxdz7PQ1AoBCeVMPls0FZSxRvNSFigXV5SVFU9NrAy33qalnVTXQnUk95+kaay7E0aKXHpDwPCAA */ + context: initialContext, + + predictableActionArguments: true, + id: 'IngestPathways', + initial: 'uninitialized', + + schema: { + context: {} as IngestPathwaysContext, + events: {} as IngestPathwaysEvent, + services: {} as IngestPathwaysServices, + }, + + states: { + uninitialized: { + always: 'loadingSignalData', + }, + + loaded: {}, + + loadingIngestPipelines: { + invoke: { + src: 'loadIngestPipelines', + + onDone: { + target: 'loaded', + actions: ['storeIngestPipelines', 'updateGraph'], + }, + + id: 'loadIngestPipelines', + }, + }, + + loadingSignalData: { + invoke: { + src: 'loadSignalData', + + onDone: { + target: 'loadingDataStreams', + actions: ['storeSignalData', 'updateGraph'], + }, + + id: 'loadSignalData', + }, + }, + + loadingIndexTemplates: { + invoke: { + src: 'loadIndexTemplates', + id: 'loadIndexTemplates', + + onDone: { + target: 'loadingIngestPipelines', + actions: ['storeIndexTemplates', 'updateGraph'], + }, + }, + }, + + loadingDataStreams: { + invoke: { + src: 'loadDataStreams', + id: 'loadDataStreams', + + onDone: { + target: 'loadingIndexTemplates', + actions: 'storeDataStreams', + }, + }, + }, + }, + + on: { + load: '.loadingSignalData', + + selectPathway: { + target: '#IngestPathways', + internal: true, + actions: 'storeSelectedPathway', + }, + }, + }, + { + actions: { + storeSignalData: assign({ + data: (context, event) => { + if (event.type !== 'done.invoke.loadSignalData') { + return context.data; + } + + return mergeIngestPathwaysData(context.data, event.data); + }, + }), + storeDataStreams: assign({ + data: (context, event) => { + if (event.type !== 'done.invoke.loadDataStreams') { + return context.data; + } + + return mergeIngestPathwaysData(context.data, { + dataStreams: event.data, + }); + }, + }), + storeIndexTemplates: assign({ + data: (context, event) => { + if (event.type !== 'done.invoke.loadIndexTemplates') { + return context.data; + } + + return mergeIngestPathwaysData(context.data, { + indexTemplates: event.data.indexTemplates, + ingestPipelines: event.data.ingestPipelines, + }); + }, + }), + storeIngestPipelines: assign({ + data: (context, event) => { + if (event.type !== 'done.invoke.loadIngestPipelines') { + return context.data; + } + + return mergeIngestPathwaysData(context.data, { + ingestPipelines: event.data, + }); + }, + }), + updateGraph: assign({ + graph: (context, event) => { + const graph = calculateGraph(context.data); + + return { + ...context.graph, + ...graph, + }; + }, + }), + storeSelectedPathway: assign({ + graph: (context, event) => { + if (event.type !== 'selectPathway') { + return context.graph; + } + + return { + ...context.graph, + selection: { + ...context.graph.selection, + pathwayId: event.pathwayId, + }, + }; + }, + }), + }, + } + ); + +export interface IngestPathwaysStateMachineDependencies { + dataStreamsStatsClient: IDataStreamsStatsClient; + http: HttpStart; + search: ISearchGeneric; +} + +export const createIngestPathwaysStateMachine = ({ + dataStreamsStatsClient, + http, + search, +}: IngestPathwaysStateMachineDependencies) => { + const currentDate = new Date(); + const from = moment(currentDate).subtract(moment.duration(1, 'days')).toISOString(); + const to = currentDate.toISOString(); + + return createPureIngestPathwaysStateMachine({ + parameters: { + dataStreamPattern: 'logs-*-*,metrics-*-*', + timeRange: { + from, + to, + }, + }, + data: { + dataStreams: {}, + agents: {}, + indexTemplates: {}, + ingestPipelines: {}, + }, + graph: { + elements: [], + selection: { + pathwayId: null, + }, + }, + }).withConfig({ + services: { + loadSignalData: loadSignalData({ dataStreamsStatsClient, search }), + loadDataStreams: loadDataStreams({ http }), + loadIndexTemplates: loadIndexTemplates({ http }), + loadIngestPipelines: loadIngestPipelines({ http }), + }, + }); +}; + +export interface IngestPathwaysContext { + parameters: IngestPathwaysParameters; + data: IngestPathwaysData; + graph: { + elements: ElementDefinition[]; + selection: GraphSelection; + }; +} + +export interface IngestPathwaysServices { + [service: string]: { + data: any; + }; + loadSignalData: { + data: { + agents: Record; + dataStreams: Record; + }; + }; + loadDataStreams: { + data: Record; + }; + loadIndexTemplates: { + data: { + indexTemplates: Record; + ingestPipelines: Record; + }; + }; + loadIngestPipelines: { + data: Record; + }; +} + +export type IngestPathwaysEvent = + | { + type: 'load'; + } + | { + type: 'selectPathway'; + pathwayId: string; + } + | { + type: `${ActionTypes.DoneInvoke}.loadSignalData`; + data: IngestPathwaysServices['loadSignalData']['data']; + } + | { + type: `${ActionTypes.DoneInvoke}.loadDataStreams`; + data: IngestPathwaysServices['loadDataStreams']['data']; + } + | { + type: `${ActionTypes.DoneInvoke}.loadIndexTemplates`; + data: IngestPathwaysServices['loadIndexTemplates']['data']; + } + | { + type: `${ActionTypes.DoneInvoke}.loadIngestPipelines`; + data: IngestPathwaysServices['loadIngestPipelines']['data']; + }; diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_data_streams.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_data_streams.ts new file mode 100644 index 0000000000000..c5167ba4d37f7 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_data_streams.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpStart } from '@kbn/core-http-browser'; +import { decodeOrThrow } from '@kbn/io-ts-utils'; +import * as rt from 'io-ts'; +import { IngestPathwaysContext, IngestPathwaysServices } from '../ingest_pathways_state_machine'; +import { DataStream } from '../types'; +import { INDEX_MANAGEMENT_PREFIX } from '../utils'; + +type LoadDataStreamsResult = IngestPathwaysServices['loadDataStreams']['data']; + +export const loadDataStreams = + ({ http }: { http: HttpStart }) => + async ({ data: { dataStreams } }: IngestPathwaysContext): Promise => { + const updatedDataStreams: Record = Object.fromEntries( + await Promise.all( + Object.entries(dataStreams).map(async ([, dataStream]) => { + const rawResponse = await http.get( + `${INDEX_MANAGEMENT_PREFIX}/data_streams/${dataStream.id}` + ); + const response = decodeOrThrow(dataStreamResponseRT)(rawResponse); + + const newDataStream: DataStream = { + type: 'dataStream', + id: dataStream.id, + indexTemplateId: response.indexTemplateName, + }; + + return [dataStream.id, newDataStream] as const; + }) + ) + ); + + return updatedDataStreams; + }; + +const dataStreamResponseRT = rt.strict({ + indexTemplateName: rt.string, +}); diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_index_templates.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_index_templates.ts new file mode 100644 index 0000000000000..b78144c47a29f --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_index_templates.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpStart } from '@kbn/core-http-browser'; +import { decodeOrThrow } from '@kbn/io-ts-utils'; +import * as rt from 'io-ts'; +import { IngestPathwaysContext, IngestPathwaysServices } from '../ingest_pathways_state_machine'; +import { IndexTemplate, IngestPipelineStub } from '../types'; +import { INDEX_MANAGEMENT_PREFIX } from '../utils'; + +type LoadIndexTemplatesResult = IngestPathwaysServices['loadIndexTemplates']['data']; + +export const loadIndexTemplates = + ({ http }: { http: HttpStart }) => + async ({ data: { dataStreams } }: IngestPathwaysContext): Promise => { + // load main index templates + const mentionedIndexTemplateNames = new Set( + Object.values(dataStreams).flatMap((dataStream) => + dataStream.type === 'dataStream' ? [dataStream.indexTemplateId] : [] + ) + ); + + const indexTemplateInfos = Object.fromEntries( + await Promise.all( + Array.from(mentionedIndexTemplateNames).map(async (indexTemplateName) => { + const rawResponse = await http + .get(`${INDEX_MANAGEMENT_PREFIX}/index_templates/${indexTemplateName}`) + .catch((err) => ({ + composedOf: [], + template: {}, + })); + return [indexTemplateName, decodeOrThrow(indexTemplateResponseRT)(rawResponse)] as const; + }) + ) + ); + + // load component templates + const mentionedComponentTemplateNames = new Set( + Object.values(indexTemplateInfos).flatMap(({ composedOf }) => composedOf) + ); + + const componentTemplateInfos: Record = Object.fromEntries( + await Promise.all( + Array.from(mentionedComponentTemplateNames).map(async (componentTemplateName) => { + const rawResponse = await http + .get(`${INDEX_MANAGEMENT_PREFIX}/component_templates/${componentTemplateName}`) + .catch((err) => ({ + template: {}, + })); + return [ + componentTemplateName, + decodeOrThrow(componentTemplateResponseRT)(rawResponse), + ] as const; + }) + ) + ); + + // combine the index and component templates + const indexTemplates: Record = Object.fromEntries( + Object.entries(indexTemplateInfos).map(([indexTemplateId, indexTemplateInfo]) => { + const defaultPipeline = reduceToLastTemplateName( + indexTemplateInfo, + componentTemplateInfos, + 'default_pipeline' + ); + const finalPipeline = reduceToLastTemplateName( + indexTemplateInfo, + componentTemplateInfos, + 'final_pipeline' + ); + + return [ + indexTemplateId, + { + id: indexTemplateId, + ingestPipelineIds: [ + ...(defaultPipeline != null ? [defaultPipeline] : []), + ...(finalPipeline != null ? [finalPipeline] : []), + ], + }, + ]; + }) + ); + + // derive ingest pipeline stubs + const ingestPipelines: Record = Object.fromEntries( + Array.from( + new Set(Object.values(indexTemplates).flatMap(({ ingestPipelineIds }) => ingestPipelineIds)) + ).map((ingestPipelineId) => [ + ingestPipelineId, + { + type: 'ingestPipelineStub', + id: ingestPipelineId, + }, + ]) + ); + + return { + indexTemplates, + ingestPipelines, + }; + }; + +const reduceToLastTemplateName = ( + mainIndexTemplateInfo: IndexTemplateInfo, + availableComponentTemplateInfos: Record, + templateType: 'default_pipeline' | 'final_pipeline' +): string | undefined => { + return [ + mainIndexTemplateInfo.template.settings?.index?.[templateType], + ...mainIndexTemplateInfo.composedOf.map( + (composedOfTemplateName) => + availableComponentTemplateInfos[composedOfTemplateName].template.settings?.index?.[ + templateType + ] + ), + ].reduce((lastPipelineName, currentPipelineName) => { + return currentPipelineName ?? lastPipelineName; + }, undefined); +}; + +const templateSettingsRT = rt.exact( + rt.partial({ + settings: rt.exact( + rt.partial({ + index: rt.exact( + rt.partial({ + default_pipeline: rt.string, + final_pipeline: rt.string, + }) + ), + }) + ), + }) +); + +const indexTemplateResponseRT = rt.strict({ + composedOf: rt.array(rt.string), + template: templateSettingsRT, +}); + +type IndexTemplateInfo = rt.TypeOf; + +const componentTemplateResponseRT = rt.strict({ + template: templateSettingsRT, +}); + +type ComponentTemplateInfo = rt.TypeOf; diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_ingest_pipelines.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_ingest_pipelines.ts new file mode 100644 index 0000000000000..da0e3f8f2f8c7 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_ingest_pipelines.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpStart } from '@kbn/core-http-browser'; +import { decodeOrThrow } from '@kbn/io-ts-utils'; +import * as rt from 'io-ts'; +import { IngestPathwaysContext, IngestPathwaysServices } from '../ingest_pathways_state_machine'; +import { IngestPipeline } from '../types'; +import { INGEST_PIPELINES_PREFIX } from '../utils'; + +type LoadIngestPipelinesResult = IngestPathwaysServices['loadIngestPipelines']['data']; + +export const loadIngestPipelines = + ({ http }: { http: HttpStart }) => + async ({ + data: { ingestPipelines }, + }: IngestPathwaysContext): Promise => { + const updatedIngestPipelines: Record = Object.fromEntries( + await Promise.all( + Object.entries(ingestPipelines).map(async ([, ingestPipeline]) => { + const rawResponse = await http.get(`${INGEST_PIPELINES_PREFIX}/${ingestPipeline.id}`); + const response = decodeOrThrow(ingestPipelineResponseRT)(rawResponse); + + const newIngestPipeline: IngestPipeline = { + type: 'ingestPipeline', + id: ingestPipeline.id, + description: response.description, + processors: response.processors, + }; + + return [ingestPipeline.id, newIngestPipeline] as const; + }) + ) + ); + + return updatedIngestPipelines; + }; + +const ingestProcessorRT = rt.union([ + rt.strict({ + pipeline: rt.strict({ + name: rt.string, + }), + }), + rt.strict({ + rename: rt.strict({ + field: rt.string, + target_field: rt.string, + }), + }), + rt.unknown, +]); + +const ingestPipelineResponseRT = rt.intersection([ + rt.strict({ + name: rt.string, + processors: rt.array(ingestProcessorRT), + }), + rt.exact( + rt.partial({ + description: rt.string, + }) + ), +]); diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_signal_data.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_signal_data.ts new file mode 100644 index 0000000000000..779fc21f72e91 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/services/load_signal_data.ts @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEsSearchRequest, ISearchGeneric, ISearchRequestParams } from '@kbn/data-plugin/common'; +import { IDataStreamsStatsClient } from '@kbn/dataset-quality-plugin/public'; +import { decodeOrThrow } from '@kbn/io-ts-utils'; +import * as rt from 'io-ts'; +import { lastValueFrom } from 'rxjs'; +import { IngestPathwaysContext, IngestPathwaysServices } from '../ingest_pathways_state_machine'; + +type LoadSignalDataResult = IngestPathwaysServices['loadSignalData']['data']; + +export const loadSignalData = + ({ + dataStreamsStatsClient, + search, + }: { + dataStreamsStatsClient: IDataStreamsStatsClient; + search: ISearchGeneric; + }) => + async ({ + parameters: { dataStreamPattern, timeRange }, + }: IngestPathwaysContext): Promise => { + const request: IEsSearchRequest = { + params: { + index: dataStreamPattern, + allow_no_indices: true, + ignore_unavailable: true, + body: { + size: 0, + query: { + bool: { + filter: [ + { + exists: { + field: 'data_stream.type', + }, + }, + { + range: { + '@timestamp': { + gt: timeRange.from, + lte: timeRange.to, + }, + }, + }, + ], + }, + }, + aggregations: { + relations: { + composite: { + size: 1000, + sources: [ + { + agentId: { + terms: { + field: 'agent.id', + }, + }, + }, + { + dataStreamType: { + terms: { + field: 'data_stream.type', + }, + }, + }, + { + dataStreamDataset: { + terms: { + field: 'data_stream.dataset', + }, + }, + }, + { + dataStreamNamespace: { + terms: { + field: 'data_stream.namespace', + }, + }, + }, + ], + }, + aggregations: { + agent: { + top_metrics: { + metrics: [ + { + field: 'agent.name', + }, + { + field: 'agent.type', + }, + { + field: 'agent.version', + }, + ], + sort: { + '@timestamp': 'desc', + }, + }, + }, + }, + }, + }, + }, + }, + }; + + const { rawResponse } = await lastValueFrom(search(request)); + + const response = decodeOrThrow(signalResponseRT)(rawResponse); + + return response.aggregations.relations.buckets.reduce( + ( + currentData, + { + key: { agentId: unsafeAgentId, dataStreamType, dataStreamDataset, dataStreamNamespace }, + doc_count: signalCount, + agent, + } + ) => { + const dataStreamId = `${dataStreamType}-${dataStreamDataset}-${dataStreamNamespace}`; + if (currentData.dataStreams[dataStreamId] == null) { + currentData.dataStreams[dataStreamId] = { + type: 'dataStreamStub', + id: dataStreamId, + }; + } + + const agentMetadata = agent.top[0]?.metrics; + const agentId = `${agentMetadata['agent.type'] ?? 'unknown'}-${unsafeAgentId}`; + const previousAgent = currentData.agents[agentId]; + + if (previousAgent == null) { + currentData.agents[agentId] = { + id: agentId, + type: agentMetadata['agent.type'] ?? 'unknown', + name: agentMetadata['agent.name'] ?? agentId, + version: agentMetadata['agent.version'] ?? 'unknown', + shipsTo: [ + { + dataStreamId, + signalCount, + }, + ], + }; + } else { + currentData.agents[agentId] = { + ...previousAgent, + shipsTo: [ + ...previousAgent.shipsTo, + { + dataStreamId, + signalCount, + }, + ], + }; + } + + return currentData; + }, + { + dataStreams: {}, + agents: {}, + } + ); + }; + +const signalResponseRT = rt.strict({ + aggregations: rt.strict({ + relations: rt.strict({ + buckets: rt.array( + rt.strict({ + key: rt.strict({ + agentId: rt.string, + dataStreamType: rt.string, + dataStreamDataset: rt.string, + dataStreamNamespace: rt.string, + }), + doc_count: rt.number, + agent: rt.strict({ + top: rt.array( + rt.strict({ + metrics: rt.strict({ + 'agent.name': rt.union([rt.string, rt.null]), + 'agent.type': rt.union([rt.string, rt.null]), + 'agent.version': rt.union([rt.string, rt.null]), + }), + }) + ), + }), + }) + ), + }), + }), +}); diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/types.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/types.ts new file mode 100644 index 0000000000000..8bd54cf52b5c8 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/types.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface IngestPathwaysParameters { + timeRange: TimeRange; + dataStreamPattern: string; +} + +export interface IngestPathwaysData { + dataStreams: Record; + agents: Record; + indexTemplates: Record; + ingestPipelines: Record; +} + +export interface TimeRange { + from: string; + to: string; +} + +export interface DataStreamStub { + type: 'dataStreamStub'; + id: string; +} + +export interface DataStream { + type: 'dataStream'; + id: string; + indexTemplateId: string; +} + +export type DataStreamEntry = DataStream | DataStreamStub; + +export interface IndexTemplate { + id: string; + ingestPipelineIds: string[]; +} + +export interface IngestPipelineStub { + type: 'ingestPipelineStub'; + id: string; +} + +export interface IngestPipeline { + type: 'ingestPipeline'; + id: string; + description?: string; + processors: unknown[]; +} + +export type IngestPipelineEntry = IngestPipeline | IngestPipelineStub; + +export interface Agent { + id: string; + type: string; + name: string; + version: string; + shipsTo: Array<{ + dataStreamId: string; + signalCount: number; + }>; +} + +export interface GraphSelection { + pathwayId: string | null; +} diff --git a/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/utils.ts b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/utils.ts new file mode 100644 index 0000000000000..d99e2b3c42d68 --- /dev/null +++ b/x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/utils.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IngestPathwaysData } from './types'; + +export const INDEX_MANAGEMENT_PREFIX = '/api/index_management'; +export const INGEST_PIPELINES_PREFIX = '/api/ingest_pipelines'; + +export const mergeIngestPathwaysData = ( + firstData: IngestPathwaysData, + secondData: Partial +): IngestPathwaysData => ({ + agents: { ...firstData.agents, ...(secondData.agents ?? {}) }, + dataStreams: { ...firstData.dataStreams, ...(secondData.dataStreams ?? {}) }, + indexTemplates: { ...firstData.indexTemplates, ...(secondData.indexTemplates ?? {}) }, + ingestPipelines: { ...firstData.ingestPipelines, ...(secondData.ingestPipelines ?? {}) }, +});