diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index fb18faed8a74..dbf68dd939ba 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -13,11 +13,19 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; +import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT; + import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.ColumnsEntityInterface; import org.openmetadata.schema.api.lineage.AddLineage; import org.openmetadata.schema.entity.data.Table; @@ -30,6 +38,8 @@ import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.search.SearchClient; +import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @@ -37,6 +47,8 @@ public class LineageRepository { private final CollectionDAO dao; + public SearchClient searchClient = Entity.getSearchRepository().getSearchClient(); + public LineageRepository() { this.dao = Entity.getCollectionDAO(); Entity.setLineageRepository(this); @@ -88,6 +100,72 @@ public void addLineage(AddLineage addLineage) { to.getType(), Relationship.UPSTREAM.ordinal(), detailsJson); + addLineageToSearch(from, to, addLineage.getEdge().getLineageDetails()); + } + + private void addLineageToSearch( + EntityReference fromEntity, EntityReference toEntity, LineageDetails lineageDetails) { + IndexMapping sourceIndexMapping = + Entity.getSearchRepository().getIndexMapping(fromEntity.getType()); + String sourceIndexName = + sourceIndexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()); + IndexMapping destinationIndexMapping = + Entity.getSearchRepository().getIndexMapping(toEntity.getType()); + String destinationIndexName = + destinationIndexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()); + Map relationshipDetails = new HashMap<>(); + Pair from = new ImmutablePair<>("_id", fromEntity.getId().toString()); + Pair to = new ImmutablePair<>("_id", toEntity.getId().toString()); + processLineageData(fromEntity, toEntity, lineageDetails, relationshipDetails); + searchClient.updateLineage(sourceIndexName, from, relationshipDetails); + searchClient.updateLineage(destinationIndexName, to, relationshipDetails); + } + + private void processLineageData( + EntityReference fromEntity, + EntityReference toEntity, + LineageDetails lineageDetails, + Map relationshipDetails) { + Map fromDetails = new HashMap<>(); + Map toDetails = new HashMap<>(); + fromDetails.put("id", fromEntity.getId().toString()); + fromDetails.put("type", fromEntity.getType()); + fromDetails.put("fqn", fromEntity.getFullyQualifiedName()); + toDetails.put("id", toEntity.getId().toString()); + toDetails.put("type", toEntity.getType()); + toDetails.put("fqn", toEntity.getFullyQualifiedName()); + relationshipDetails.put( + "doc_id", fromEntity.getId().toString() + "-" + toEntity.getId().toString()); + relationshipDetails.put("fromEntity", fromDetails); + relationshipDetails.put("toEntity", toDetails); + if (lineageDetails != null) { + relationshipDetails.put( + "pipeline", + JsonUtils.getMap( + CommonUtil.nullOrEmpty(lineageDetails.getPipeline()) + ? null + : lineageDetails.getPipeline())); + relationshipDetails.put( + "description", + CommonUtil.nullOrEmpty(lineageDetails.getDescription()) + ? null + : lineageDetails.getDescription()); + if (!CommonUtil.nullOrEmpty(lineageDetails.getColumnsLineage())) { + List> colummnLineageList = new ArrayList<>(); + for (ColumnLineage columnLineage : lineageDetails.getColumnsLineage()) { + colummnLineageList.add(JsonUtils.getMap(columnLineage)); + } + relationshipDetails.put("columns", colummnLineageList); + } + relationshipDetails.put( + "sqlQuery", + CommonUtil.nullOrEmpty(lineageDetails.getSqlQuery()) + ? null + : lineageDetails.getSqlQuery()); + relationshipDetails.put( + "source", + CommonUtil.nullOrEmpty(lineageDetails.getSource()) ? null : lineageDetails.getSource()); + } } private String validateLineageDetails( @@ -143,14 +221,30 @@ public boolean deleteLineage(String fromEntity, String fromId, String toEntity, Entity.getEntityReferenceById(toEntity, UUID.fromString(toId), Include.NON_DELETED); // Finally, delete lineage relationship - return dao.relationshipDAO() - .delete( - from.getId(), - from.getType(), - to.getId(), - to.getType(), - Relationship.UPSTREAM.ordinal()) - > 0; + boolean result = + dao.relationshipDAO() + .delete( + from.getId(), + from.getType(), + to.getId(), + to.getType(), + Relationship.UPSTREAM.ordinal()) + > 0; + deleteLineageFromSearch(from, to); + return result; + } + + private void deleteLineageFromSearch(EntityReference fromEntity, EntityReference toEntity) { + searchClient.updateChildren( + GLOBAL_SEARCH_ALIAS, + new ImmutablePair<>( + "lineage.doc_id.keyword", + fromEntity.getId().toString() + "-" + toEntity.getId().toString()), + new ImmutablePair<>( + String.format( + REMOVE_LINEAGE_SCRIPT, + fromEntity.getId().toString() + "-" + toEntity.getId().toString()), + null)); } private EntityLineage getLineage( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java index de51cbe0a3a5..b9f46fe8fb57 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java @@ -15,6 +15,7 @@ import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import es.org.elasticsearch.action.search.SearchResponse; import io.dropwizard.jersey.errors.ErrorMessage; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -22,6 +23,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.IOException; import java.util.List; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -165,6 +167,41 @@ public EntityLineage getByName( return addHref(uriInfo, dao.getByName(entity, fqn, upstreamDepth, downStreamDepth)); } + @GET + @Path("/getLineage") + @Operation( + operationId = "searchLineage", + summary = "Search lineage", + responses = { + @ApiResponse( + responseCode = "200", + description = "search response", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SearchResponse.class))) + }) + public Response searchLineage( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "fqn") @QueryParam("fqn") String fqn, + @Parameter(description = "upstreamDepth") @QueryParam("upstreamDepth") int upstreamDepth, + @Parameter(description = "downstreamDepth") @QueryParam("downstreamDepth") + int downstreamDepth, + @Parameter( + description = + "Elasticsearch query that will be combined with the query_string query generator from the `query` argument") + @QueryParam("query_filter") + String queryFilter, + @Parameter(description = "Filter documents by deleted param. By default deleted is false") + @QueryParam("includeDeleted") + boolean deleted) + throws IOException { + + return Entity.getSearchRepository() + .searchLineage(fqn, upstreamDepth, downstreamDepth, queryFilter, deleted); + } + @PUT @Operation( operationId = "addLineageEdge", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index 886e720fe135..616fd1a27b36 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -44,6 +44,11 @@ public interface SearchClient { String REMOVE_TAGS_CHILDREN_SCRIPT = "for (int i = 0; i < ctx._source.tags.length; i++) { if (ctx._source.tags[i].tagFQN == '%s') { ctx._source.tags.remove(i) }}"; + String REMOVE_LINEAGE_SCRIPT = + "for (int i = 0; i < ctx._source.lineage.length; i++) { if (ctx._source.lineage[i].doc_id == '%s') { ctx._source.lineage.remove(i) }}"; + + String ADD_UPDATE_LINEAGE = + "boolean docIdExists = false; for (int i = 0; i < ctx._source.lineage.size(); i++) { if (ctx._source.lineage[i].doc_id.equalsIgnoreCase(params.lineageData.doc_id)) { ctx._source.lineage[i] = params.lineageData; docIdExists = true; break;}}if (!docIdExists) {ctx._source.lineage.add(params.lineageData);}"; String UPDATE_ADDED_DELETE_GLOSSARY_TAGS = "if (ctx._source.tags != null) { for (int i = ctx._source.tags.size() - 1; i >= 0; i--) { if (params.tagDeleted != null) { for (int j = 0; j < params.tagDeleted.size(); j++) { if (ctx._source.tags[i].tagFQN.equalsIgnoreCase(params.tagDeleted[j].tagFQN)) { ctx._source.tags.remove(i); } } } } } if (ctx._source.tags == null) { ctx._source.tags = []; } if (params.tagAdded != null) { ctx._source.tags.addAll(params.tagAdded); } ctx._source.tags = ctx._source.tags .stream() .distinct() .sorted((o1, o2) -> o1.tagFQN.compareTo(o2.tagFQN)) .collect(Collectors.toList());"; String REMOVE_TEST_SUITE_CHILDREN_SCRIPT = @@ -67,6 +72,10 @@ public interface SearchClient { Response searchBySourceUrl(String sourceUrl) throws IOException; + Response searchLineage( + String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) + throws IOException; + Response searchByField(String fieldName, String fieldValue, String index) throws IOException; Response aggregate(String index, String fieldName, String value, String query) throws IOException; @@ -95,6 +104,9 @@ void updateChildren( Pair fieldAndValue, Pair> updates); + void updateLineage( + String indexName, Pair fieldAndValue, Map lineagaData); + TreeMap> getSortedDate( String team, Long scheduleTime, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index c2878c36cf76..e3ae8aa81532 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -637,6 +637,12 @@ public Response searchBySourceUrl(String sourceUrl) throws IOException { return searchClient.searchBySourceUrl(sourceUrl); } + public Response searchLineage( + String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) + throws IOException { + return searchClient.searchLineage(fqn, upstreamDepth, downstreamDepth, queryFilter, deleted); + } + public Response searchByField(String fieldName, String fieldValue, String index) throws IOException { return searchClient.searchByField(fieldName, fieldValue, index); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index b4d3d9b85e8a..4cbaf3b0db50 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -111,6 +111,7 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; +import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.DataInsightInterface; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; @@ -419,6 +420,95 @@ public Response search(SearchRequest request) throws IOException { return Response.status(OK).entity(response).build(); } + @Override + public Response searchLineage( + String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) + throws IOException { + Map responseMap = new HashMap<>(); + List> edges = new ArrayList<>(); + Set> nodes = new HashSet<>(); + es.org.elasticsearch.action.search.SearchRequest searchRequest = + new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query( + QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + for (var hit : searchResponse.getHits().getHits()) { + responseMap.put("entity", hit.getSourceAsMap()); + } + getLineage( + fqn, downstreamDepth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword", deleted); + getLineage( + fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword", deleted); + responseMap.put("edges", edges); + responseMap.put("nodes", nodes); + return Response.status(OK).entity(responseMap).build(); + } + + private void getLineage( + String fqn, + int depth, + List> edges, + Set> nodes, + String queryFilter, + String direction, + boolean deleted) + throws IOException { + if (depth <= 0) { + return; + } + es.org.elasticsearch.action.search.SearchRequest searchRequest = + new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query( + QueryBuilders.boolQuery().must(QueryBuilders.termQuery(direction, fqn))); + if (CommonUtil.nullOrEmpty(deleted)) { + searchSourceBuilder.query( + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(direction, fqn)) + .must(QueryBuilders.termQuery("deleted", deleted))); + } + if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) { + try { + XContentParser filterParser = + XContentType.JSON + .xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, queryFilter); + QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query(); + BoolQueryBuilder newQuery = + QueryBuilders.boolQuery().must(searchSourceBuilder.query()).filter(filter); + searchSourceBuilder.query(newQuery); + } catch (Exception ex) { + LOG.warn("Error parsing query_filter from query parameters, ignoring filter", ex); + } + } + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + for (var hit : searchResponse.getHits().getHits()) { + List> lineage = + (List>) hit.getSourceAsMap().get("lineage"); + nodes.add(hit.getSourceAsMap()); + for (Map lin : lineage) { + HashMap fromEntity = (HashMap) lin.get("fromEntity"); + HashMap toEntity = (HashMap) lin.get("toEntity"); + if (direction.equalsIgnoreCase("lineage.fromEntity.fqn.keyword")) { + if (!edges.contains(lin) && fromEntity.get("fqn").equals(fqn)) { + edges.add(lin); + getLineage( + toEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction, deleted); + } + } else { + if (!edges.contains(lin) && toEntity.get("fqn").equals(fqn)) { + edges.add(lin); + getLineage( + fromEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction, deleted); + } + } + } + } + } + @Override public Response searchBySourceUrl(String sourceUrl) throws IOException { es.org.elasticsearch.action.search.SearchRequest searchRequest = @@ -1136,6 +1226,26 @@ public void updateChildren( } } + /** + * @param indexName + * @param fieldAndValue + */ + @Override + public void updateLineage( + String indexName, Pair fieldAndValue, Map lineageData) { + if (isClientAvailable) { + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); + updateByQueryRequest.setQuery( + new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue()) + .operator(Operator.AND)); + Map params = Collections.singletonMap("lineageData", lineageData); + Script script = + new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, ADD_UPDATE_LINEAGE, params); + updateByQueryRequest.setScript(script); + updateElasticSearchByQuery(updateByQueryRequest); + } + } + public void updateElasticSearch(UpdateRequest updateRequest) { if (updateRequest != null && isClientAvailable) { updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java index 91a41bbd507d..88b2e798e8f4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java @@ -63,6 +63,7 @@ public Map buildESDoc() { doc.put("column_suggest", columnSuggest); doc.put("entityType", Entity.CONTAINER); doc.put("serviceType", container.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(container.getEntityReference())); doc.put( "fqnParts", getFQNParts( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java index 31bc3210bc02..6f48ed2d6175 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java @@ -66,6 +66,7 @@ public Map buildESDoc() { doc.put("tier", parseTags.getTierTag()); doc.put("owner", getEntityWithDisplayName(dashboardDataModel.getOwner())); doc.put("service", getEntityWithDisplayName(dashboardDataModel.getService())); + doc.put("lineage", SearchIndex.getLineageData(dashboardDataModel.getEntityReference())); doc.put("domain", getEntityWithDisplayName(dashboardDataModel.getDomain())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java index bf29276875f0..ce7bc7f7e03e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java @@ -55,6 +55,7 @@ public Map buildESDoc() { doc.put("service_suggest", serviceSuggest); doc.put("entityType", Entity.DASHBOARD); doc.put("serviceType", dashboard.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(dashboard.getEntityReference())); doc.put("fqnParts", suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())); doc.put("owner", getEntityWithDisplayName(dashboard.getOwner())); doc.put("service", getEntityWithDisplayName(dashboard.getService())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java index ca44e3a1708c..80a00f3a55c5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java @@ -36,6 +36,7 @@ public Map buildESDoc() { doc.put("suggest", suggest); doc.put("entityType", Entity.MLMODEL); doc.put("serviceType", mlModel.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(mlModel.getEntityReference())); doc.put( "fqnParts", getFQNParts( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java index 9505034d34a4..3bd7712b459d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java @@ -48,6 +48,7 @@ public Map buildESDoc() { doc.put("service_suggest", serviceSuggest); doc.put("entityType", Entity.PIPELINE); doc.put("serviceType", pipeline.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(pipeline.getEntityReference())); doc.put( "fqnParts", getFQNParts( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java index 89cef6889540..fb6e7a0e09f5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java @@ -33,6 +33,7 @@ public Map buildESDoc() { doc.put("tier", parseTags.getTierTag()); doc.put("owner", getEntityWithDisplayName(searchIndex.getOwner())); doc.put("service", getEntityWithDisplayName(searchIndex.getService())); + doc.put("lineage", SearchIndex.getLineageData(searchIndex.getEntityReference())); doc.put("domain", getEntityWithDisplayName(searchIndex.getDomain())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index 69ea34362b9e..8f0f898bb17d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -10,13 +10,20 @@ import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.type.ColumnLineage; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.LineageDetails; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @@ -47,6 +54,84 @@ default EntityReference getEntityWithDisplayName(EntityReference entity) { return cloneEntity; } + static List> getLineageData(EntityReference entity) { + List> data = new ArrayList<>(); + CollectionDAO dao = Entity.getCollectionDAO(); + List toRelationshipsRecords = + dao.relationshipDAO() + .findTo(entity.getId(), entity.getType(), Relationship.UPSTREAM.ordinal()); + for (CollectionDAO.EntityRelationshipRecord entityRelationshipRecord : toRelationshipsRecords) { + EntityReference ref = + Entity.getEntityReferenceById( + entityRelationshipRecord.getType(), entityRelationshipRecord.getId(), Include.ALL); + LineageDetails lineageDetails = + JsonUtils.readValue(entityRelationshipRecord.getJson(), LineageDetails.class); + SearchIndex.getLineageDataDirection(entity, ref, lineageDetails, data); + } + List fromRelationshipsRecords = + dao.relationshipDAO() + .findFrom(entity.getId(), entity.getType(), Relationship.UPSTREAM.ordinal()); + for (CollectionDAO.EntityRelationshipRecord entityRelationshipRecord : + fromRelationshipsRecords) { + EntityReference ref = + Entity.getEntityReferenceById( + entityRelationshipRecord.getType(), entityRelationshipRecord.getId(), Include.ALL); + LineageDetails lineageDetails = + JsonUtils.readValue(entityRelationshipRecord.getJson(), LineageDetails.class); + SearchIndex.getLineageDataDirection(ref, entity, lineageDetails, data); + } + return data; + } + + static void getLineageDataDirection( + EntityReference fromEntity, + EntityReference toEntity, + LineageDetails lineageDetails, + List> data) { + HashMap fromDetails = new HashMap<>(); + HashMap toDetails = new HashMap<>(); + HashMap relationshipDetails = new HashMap<>(); + fromDetails.put("id", fromEntity.getId().toString()); + fromDetails.put("type", fromEntity.getType()); + fromDetails.put("fqn", fromEntity.getFullyQualifiedName()); + toDetails.put("id", toEntity.getId().toString()); + toDetails.put("type", toEntity.getType()); + toDetails.put("fqn", toEntity.getFullyQualifiedName()); + relationshipDetails.put( + "doc_id", fromEntity.getId().toString() + "-" + toEntity.getId().toString()); + relationshipDetails.put("fromEntity", fromDetails); + relationshipDetails.put("toEntity", toDetails); + if (lineageDetails != null) { + relationshipDetails.put( + "pipeline", + JsonUtils.getMap( + CommonUtil.nullOrEmpty(lineageDetails.getPipeline()) + ? null + : lineageDetails.getPipeline())); + relationshipDetails.put( + "description", + CommonUtil.nullOrEmpty(lineageDetails.getDescription()) + ? null + : lineageDetails.getDescription()); + if (!CommonUtil.nullOrEmpty(lineageDetails.getColumnsLineage())) { + List> colummnLineageList = new ArrayList<>(); + for (ColumnLineage columnLineage : lineageDetails.getColumnsLineage()) { + colummnLineageList.add(JsonUtils.getMap(columnLineage)); + } + relationshipDetails.put("columns", colummnLineageList); + } + relationshipDetails.put( + "sqlQuery", + CommonUtil.nullOrEmpty(lineageDetails.getSqlQuery()) + ? null + : lineageDetails.getSqlQuery()); + relationshipDetails.put( + "source", + CommonUtil.nullOrEmpty(lineageDetails.getSource()) ? null : lineageDetails.getSource()); + } + data.add(relationshipDetails); + } + static Map getDefaultFields() { Map fields = new HashMap<>(); fields.put(FIELD_DISPLAY_NAME, 15.0f); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java index 7ec89e4f4d0b..d6cae6b3fae0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java @@ -84,6 +84,7 @@ public Map buildESDoc() { doc.put("service", getEntityWithDisplayName(table.getService())); doc.put("domain", getEntityWithDisplayName(table.getDomain())); doc.put("database", getEntityWithDisplayName(table.getDatabase())); + doc.put("lineage", SearchIndex.getLineageData(table.getEntityReference())); doc.put("databaseSchema", getEntityWithDisplayName(table.getDatabaseSchema())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java index 3541164fa110..f448d57571b2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java @@ -75,6 +75,7 @@ public Map buildESDoc() { doc.put("service_suggest", serviceSuggest); doc.put("entityType", Entity.TOPIC); doc.put("serviceType", topic.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(topic.getEntityReference())); doc.put("messageSchema", topic.getMessageSchema() != null ? topic.getMessageSchema() : null); doc.put( "fqnParts", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 16258d4e7d8d..0d6dc095fe8d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -28,8 +28,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; @@ -42,6 +44,7 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; +import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.DataInsightInterface; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; @@ -426,6 +429,96 @@ public Response searchBySourceUrl(String sourceUrl) throws IOException { return Response.status(OK).entity(response).build(); } + @Override + public Response searchLineage( + String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) + throws IOException { + Map responseMap = new HashMap<>(); + List> edges = new ArrayList<>(); + Set> nodes = new HashSet<>(); + os.org.opensearch.action.search.SearchRequest searchRequest = + new os.org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query( + QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + for (var hit : searchResponse.getHits().getHits()) { + responseMap.put("entity", hit.getSourceAsMap()); + } + getLineage( + fqn, downstreamDepth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword", deleted); + getLineage( + fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword", deleted); + responseMap.put("edges", edges); + responseMap.put("nodes", nodes); + return Response.status(OK).entity(responseMap).build(); + } + + private void getLineage( + String fqn, + int depth, + List> edges, + Set> nodes, + String queryFilter, + String direction, + boolean deleted) + throws IOException { + if (depth <= 0) { + return; + } + os.org.opensearch.action.search.SearchRequest searchRequest = + new os.org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query( + QueryBuilders.boolQuery().must(QueryBuilders.termQuery(direction, fqn))); + if (CommonUtil.nullOrEmpty(deleted)) { + searchSourceBuilder.query( + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(direction, fqn)) + .must(QueryBuilders.termQuery("deleted", deleted))); + } + if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) { + try { + XContentParser filterParser = + XContentType.JSON + .xContent() + .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, queryFilter); + QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query(); + BoolQueryBuilder newQuery = + QueryBuilders.boolQuery().must(searchSourceBuilder.query()).filter(filter); + searchSourceBuilder.query(newQuery); + } catch (Exception ex) { + LOG.warn("Error parsing query_filter from query parameters, ignoring filter", ex); + } + } + searchRequest.source(searchSourceBuilder); + os.org.opensearch.action.search.SearchResponse searchResponse = + client.search(searchRequest, RequestOptions.DEFAULT); + for (var hit : searchResponse.getHits().getHits()) { + List> lineage = + (List>) hit.getSourceAsMap().get("lineage"); + nodes.add(hit.getSourceAsMap()); + for (Map lin : lineage) { + HashMap fromEntity = (HashMap) lin.get("fromEntity"); + HashMap toEntity = (HashMap) lin.get("toEntity"); + if (direction.equalsIgnoreCase("lineage.fromEntity.fqn.keyword")) { + if (!edges.contains(lin) && fromEntity.get("fqn").equals(fqn)) { + edges.add(lin); + getLineage( + toEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction, deleted); + } + } else { + if (!edges.contains(lin) && toEntity.get("fqn").equals(fqn)) { + edges.add(lin); + getLineage( + fromEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction, deleted); + } + } + } + } + } + @Override public Response searchByField(String fieldName, String fieldValue, String index) throws IOException { @@ -1141,6 +1234,27 @@ public void updateChildren( } } + /** + * @param indexName + * @param fieldAndValue + * @param updates + */ + @Override + public void updateLineage( + String indexName, Pair fieldAndValue, Map lineagaData) { + if (isClientAvailable) { + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); + updateByQueryRequest.setQuery( + new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue()) + .operator(Operator.AND)); + Map params = Collections.singletonMap("lineageData", lineagaData); + Script script = + new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, ADD_UPDATE_LINEAGE, params); + updateByQueryRequest.setScript(script); + updateOpenSearchByQuery(updateByQueryRequest); + } + } + private void updateOpenSearchByQuery(UpdateByQueryRequest updateByQueryRequest) { if (updateByQueryRequest != null && isClientAvailable) { updateByQueryRequest.setRefresh(true); diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json index e9e5917033e2..5677fa6983bc 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json @@ -93,6 +93,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "parent": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json index e174afb528e5..3d6dd74b5fd5 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json @@ -300,6 +300,9 @@ "type": "keyword", "normalizer": "lowercase_normalizer" }, + "lineage": { + "type" : "object" + }, "dataModelType": { "type": "keyword", "normalizer": "lowercase_normalizer" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json index 4177a573e054..8faa1298175d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json @@ -97,6 +97,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "charts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json index c748a443b529..0f37b2880f0b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json @@ -97,6 +97,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "dataProducts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json index 312e03ec3087..79cf5d1b20a3 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json @@ -136,6 +136,9 @@ "href": { "type": "text" }, + "lineage": { + "type" : "object" + }, "sourceUrl": { "type": "text" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json index f63f1084027a..54b4e8682209 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json @@ -212,6 +212,9 @@ "entityType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json index f462c8391e4f..b5c775d27bbd 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json @@ -565,6 +565,9 @@ } } }, + "lineage": { + "type" : "object" + }, "serviceType": { "type": "keyword", "normalizer": "lowercase_normalizer" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json index 86e86d42254a..15096ab2856b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json @@ -454,6 +454,9 @@ } } }, + "lineage": { + "type" : "object" + }, "lifeCycle": { "type": "keyword" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json index 080127ae1dab..07c77972d7d7 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json @@ -95,6 +95,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "parent": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json index 24c154325f29..604b23393902 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json @@ -86,6 +86,9 @@ "entityType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json index 0c16ba6cdc26..6f43943ecd0a 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json @@ -94,6 +94,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "domain" : { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json index 01e1ea352f29..44a40effb124 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json @@ -99,6 +99,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "dataProducts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json index d188600ae825..005db6bf06a9 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json @@ -141,6 +141,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "domain" : { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json index 89dc1236ba6d..118945f978b3 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json @@ -391,6 +391,9 @@ "lifeCycle": { "type": "object" }, + "lineage": { + "type" : "object" + }, "dataProducts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json index b0d1209f56b4..ea6dc0e740b8 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json @@ -416,6 +416,9 @@ "lifeCycle": { "type": "object" }, + "lineage": { + "type" : "object" + }, "location": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json index 741f7add9ba6..a113a6fcba36 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json @@ -95,6 +95,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "messageSchema": { "properties": { "schemaType": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json index 259995c5c950..9659c06efd8d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json @@ -85,6 +85,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "parent": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json index 61555a8f6950..65c69122afc4 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json @@ -130,6 +130,9 @@ "entityType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json index 1271e9efe113..3e5a2abdb14c 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json @@ -77,6 +77,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "charts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json index 6ad1bf057a35..8f453e43149b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json @@ -68,6 +68,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "dataProducts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json index 001ade10c7ec..ff2705907461 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json @@ -117,6 +117,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "domain" : { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json index 9038c22dda82..199b838cc150 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json @@ -232,6 +232,9 @@ "entityType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json index b547bf5cdbff..f7ebba449f87 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json @@ -77,6 +77,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "tableType": { "type": "keyword" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json index 99c456a5b7da..0d04d3fdba83 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json @@ -365,6 +365,9 @@ "serviceType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "entityType": { "type": "keyword" }, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js new file mode 100644 index 000000000000..6e0dd1a4c99d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js @@ -0,0 +1,54 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DATA_ASSETS, SEARCH_INDEX } from './constants'; + +export const PIPELINE_SUPPORTED_TYPES = ['Table', 'Topic']; + +export const LINEAGE_ITEMS = [ + { + term: 'raw_customer', + displayName: 'raw_customer', + entity: DATA_ASSETS.tables, + serviceName: 'sample_data', + entityType: 'Table', + fqn: 'sample_data.ecommerce_db.shopify.raw_customer', + searchIndex: SEARCH_INDEX.tables, + }, + { + term: 'fact_session', + displayName: 'fact_session', + entity: DATA_ASSETS.tables, + serviceName: 'sample_data', + schemaName: 'shopify', + entityType: 'Table', + fqn: 'sample_data.ecommerce_db.shopify.fact_session', + searchIndex: SEARCH_INDEX.tables, + }, + { + term: 'shop_products', + displayName: 'shop_products', + entity: DATA_ASSETS.topics, + serviceName: 'sample_kafka', + fqn: 'sample_kafka.shop_products', + entityType: 'Topic', + searchIndex: SEARCH_INDEX.topics, + }, + { + term: 'forecast_sales', + entity: DATA_ASSETS.mlmodels, + serviceName: 'mlflow_svc', + entityType: 'ML Model', + fqn: 'mlflow_svc.forecast_sales', + searchIndex: SEARCH_INDEX.mlmodels, + }, +]; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js index 378d21dd0475..2b6a80b6c96d 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js @@ -11,79 +11,126 @@ * limitations under the License. */ -import { visitEntityDetailsPage } from '../../common/common'; import { - SEARCH_ENTITY_PIPELINE, - SEARCH_ENTITY_TABLE, - SEARCH_ENTITY_TOPIC, -} from '../../constants/constants'; - -const tableEntity = SEARCH_ENTITY_TABLE.table_1; -const topicEntity = SEARCH_ENTITY_TOPIC.topic_1; -const pipelineEntity = SEARCH_ENTITY_PIPELINE.pipeline_1; -// Todo:- skipping flaky test -// const dashboardEntity = SEARCH_ENTITY_DASHBOARD.dashboard_1; - -const ENTITIES_LIST = [ - tableEntity, - topicEntity, - pipelineEntity, - // dashboardEntity, -]; + interceptURL, + verifyResponseStatusCode, + visitEntityDetailsPage, +} from '../../common/common'; +import { LINEAGE_ITEMS } from '../../constants/lineage.constants'; + +const dataTransfer = new DataTransfer(); + +const dragConnection = (sourceFqn, targetFqn) => { + cy.get( + `[data-testid="lineage-node-${sourceFqn}"] .react-flow__handle-right` + ).click({ force: true }); // Adding force true for handles because it can be hidden behind the node + + return cy + .get(`[data-testid="lineage-node-${targetFqn}"] .react-flow__handle-left`) + .click({ force: true }); // Adding force true for handles because it can be hidden behind the node +}; + +const connectEdgeBetweenNodes = (fromNode, toNode) => { + interceptURL('PUT', '/api/v1/lineage', 'lineageApi'); + const type = toNode.searchIndex; + + cy.get(`[data-testid="${type}-draggable-icon"]`) + .invoke('attr', 'draggable') + .should('contain', 'true'); + + cy.get(`[data-testid="${type}-draggable-icon"]`).trigger('dragstart', { + dataTransfer, + }); + + cy.get('[data-testid="lineage-details"]') + .trigger('drop', { dataTransfer }) + .trigger('dragend'); + + cy.get(`[data-testid="${type}-draggable-icon"]`) + .invoke('attr', 'draggable') + .should('contain', 'false'); + + cy.get('[data-testid="suggestion-node"]').click(); + cy.get('[data-testid="suggestion-node"] input').click().type(toNode.term); + cy.get('.ant-select-dropdown .ant-select-item').eq(0).click(); + + dragConnection(fromNode.fqn, toNode.fqn); + verifyResponseStatusCode('@lineageApi', 200); +}; + +const verifyNodePresent = (node) => { + cy.get('.react-flow__controls-fitview').click(); + cy.get(`[data-testid="lineage-node-${node.fqn}"]`).should('be.visible'); + cy.get( + `[data-testid="lineage-node-${node.fqn}"] [data-testid="entity-header-name"]` + ).should('have.text', node.term); +}; + +const deleteNode = (node) => { + interceptURL('DELETE', '/api/v1/lineage/**', 'lineageDeleteApi'); + cy.get(`[data-testid="lineage-node-${node.fqn}"]`).click(); + // Adding force true for handles because it can be hidden behind the node + cy.get('[data-testid="lineage-node-remove-btn"]').click({ force: true }); + verifyResponseStatusCode('@lineageDeleteApi', 200); +}; describe('Entity Details Page', () => { beforeEach(() => { cy.login(); }); - ENTITIES_LIST.map((entity) => { - it(`Edit lineage should work for ${entity.entity} entity`, () => { + LINEAGE_ITEMS.forEach((entity, index) => { + it(`Lineage Add Node for entity ${entity.entityType}`, () => { visitEntityDetailsPage({ term: entity.term, serviceName: entity.serviceName, entity: entity.entity, }); - cy.get('[data-testid="lineage"]').should('be.visible').click(); - // Check edit button should not be disabled - cy.get('[data-testid="edit-lineage"]') - .should('be.visible') - .should('not.be.disabled'); + cy.get('[data-testid="lineage"]').click(); + cy.get('[data-testid="edit-lineage"]').click(); + + // Connect the current entity to all others in the array except itself + for (let i = 0; i < LINEAGE_ITEMS.length; i++) { + if (i !== index) { + connectEdgeBetweenNodes(entity, LINEAGE_ITEMS[i]); + } + } + + cy.get('[data-testid="edit-lineage"]').click(); + cy.reload(); + + // Verify Added Nodes + for (let i = 0; i < LINEAGE_ITEMS.length; i++) { + if (i !== index) { + verifyNodePresent(LINEAGE_ITEMS[i]); + } + } + + cy.get('[data-testid="edit-lineage"]').click(); }); - }); -}); -describe('Lineage functionality', () => { - beforeEach(() => { - cy.login(); - }); + it(`Lineage Remove Node between ${entity.entityType}`, () => { + visitEntityDetailsPage({ + term: entity.term, + serviceName: entity.serviceName, + entity: entity.entity, + }); + + cy.get('[data-testid="lineage"]').click(); + cy.get('[data-testid="edit-lineage"]').click(); + + // Delete Nodes + for (let i = 0; i < LINEAGE_ITEMS.length; i++) { + if (i !== index) { + deleteNode(LINEAGE_ITEMS[i]); + cy.get(`[data-testid="lineage-node-${LINEAGE_ITEMS[i].fqn}"]`).should( + 'not.exist' + ); + } + } - it('toggle fullscreen mode', () => { - visitEntityDetailsPage({ - term: tableEntity.term, - serviceName: tableEntity.serviceName, - entity: tableEntity.entity, + cy.get('[data-testid="edit-lineage"]').click(); }); - cy.get('[data-testid="lineage"]').click(); - - // Enable fullscreen - cy.get('[data-testid="full-screen"]').click(); - cy.url().should('include', 'fullscreen=true'); - cy.get('[data-testid="breadcrumb"]') - .should('be.visible') - .and('contain', 'Lineage'); - cy.get('[data-testid="lineage-details"]').should( - 'have.class', - 'full-screen-lineage' - ); - - // Exit fullscreen - cy.get('[data-testid="exit-full-screen"]').click(); - cy.url().should('not.include', 'fullscreen=true'); - cy.get('[data-testid="breadcrumb"]').should('not.contain', 'Lineage'); - cy.get('[data-testid="lineage-details"]').should( - 'not.have.class', - 'full-screen-lineage' - ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx index 1404129cff04..402a7b519d69 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx @@ -30,7 +30,6 @@ import { import { ItemType } from 'antd/lib/menu/hooks/useItems'; import { AxiosError } from 'axios'; import classNames from 'classnames'; -import { isEmpty } from 'lodash'; import { EntityDetailUnion } from 'Models'; import VirtualList from 'rc-virtual-list'; import { @@ -52,11 +51,7 @@ import { Status, } from '../../../generated/type/bulkOperationResult'; import { Aggregations } from '../../../interface/search.interface'; -import { - QueryFieldInterface, - QueryFieldValueInterface, - QueryFilterInterface, -} from '../../../pages/ExplorePage/ExplorePage.interface'; +import { QueryFilterInterface } from '../../../pages/ExplorePage/ExplorePage.interface'; import { addAssetsToDataProduct, getDataProductByName, @@ -71,6 +66,7 @@ import { getAssetsPageQuickFilters } from '../../../utils/AdvancedSearchUtils'; import { getEntityReferenceFromEntity } from '../../../utils/EntityUtils'; import { getAggregations, + getQuickFilterQuery, getSelectedValuesFromQuickFilter, } from '../../../utils/Explore.utils'; import { getCombinedQueryFilterObject } from '../../../utils/ExplorePage/ExplorePageUtils'; @@ -422,30 +418,7 @@ export const AssetSelectionModal = ({ ); const handleQuickFiltersChange = (data: ExploreQuickFilterField[]) => { - const must: QueryFieldInterface[] = []; - data.forEach((filter) => { - if (!isEmpty(filter.value)) { - const should: QueryFieldValueInterface[] = []; - if (filter.value) { - filter.value.forEach((filterValue) => { - const term: Record = {}; - term[filter.key] = filterValue.key; - should.push({ term }); - }); - } - - must.push({ - bool: { should }, - }); - } - }); - - const quickFilterQuery = isEmpty(must) - ? undefined - : { - query: { bool: { must } }, - }; - + const quickFilterQuery = getQuickFilterQuery(data); setQuickFilterQuery(quickFilterQuery); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.component.tsx index 294ee8df05c9..67d8d7a35635 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.component.tsx @@ -27,7 +27,6 @@ import { withActivityFeed } from '../../components/AppRouter/withActivityFeed'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import { ColumnFilter } from '../../components/Table/ColumnFilter/ColumnFilter.component'; @@ -62,9 +61,12 @@ import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThr import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import { CustomPropertyTable } from '../common/CustomPropertyTable/CustomPropertyTable'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; import { ModalWithMarkdownEditor } from '../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor'; import { usePermissionProvider } from '../PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../PermissionProvider/PermissionProvider.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; import { ChartsPermissions, ChartType, @@ -659,12 +661,14 @@ const DashboardDetails = ({ label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.test.tsx index 7fa6e9d762e1..3b6100629b0b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.test.tsx @@ -124,12 +124,9 @@ jest.mock('../FeedEditor/FeedEditor', () => { return jest.fn().mockReturnValue(

FeedEditor

); }); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest.fn().mockReturnValue(

Lineage

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockReturnValue(

Lineage

); +}); jest.mock('../common/CustomPropertyTable/CustomPropertyTable', () => ({ CustomPropertyTable: jest .fn() diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataModels/DataModelDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataModels/DataModelDetails.component.tsx index f0c2dfe4ff0f..8eb4e56c0c27 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataModels/DataModelDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataModels/DataModelDetails.component.tsx @@ -24,7 +24,6 @@ import ActivityThreadPanel from '../../components/ActivityFeed/ActivityThreadPan import { withActivityFeed } from '../../components/AppRouter/withActivityFeed'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import TabsLabel from '../../components/TabsLabel/TabsLabel.component'; @@ -44,6 +43,8 @@ import { getTagsWithoutTier } from '../../utils/TableUtils'; import { createTagObject } from '../../utils/TagsUtils'; import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; import SchemaEditor from '../SchemaEditor/SchemaEditor'; import { SourceType } from '../SearchedData/SearchedData.interface'; import { DataModelDetailsProps } from './DataModelDetails.interface'; @@ -332,12 +333,14 @@ const DataModelDetails = ({ ), key: EntityTabs.LINEAGE, children: ( - + + + ), }, ]; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx index a321e2c79ed1..2367013cd2a8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx @@ -12,20 +12,24 @@ */ import { CloseOutlined } from '@ant-design/icons'; -import { Col, Divider, Drawer, Row, Typography } from 'antd'; +import { Button, Col, Divider, Drawer, Row, Typography } from 'antd'; import { isUndefined } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; import { Node } from 'reactflow'; +import { ReactComponent as EditIcon } from '../../../assets/svg/edit-new.svg'; import DescriptionV1 from '../../../components/common/EntityDescription/DescriptionV1'; +import { DE_ACTIVE_COLOR } from '../../../constants/constants'; import { CSMode } from '../../../enums/codemirror.enum'; import { EntityType } from '../../../enums/entity.enum'; import { getNameFromFQN } from '../../../utils/CommonUtils'; +import { getLineageDetailsObject } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; import { getEncodedFqn } from '../../../utils/StringsUtils'; import { getEntityLink } from '../../../utils/TableUtils'; import Loader from '../../Loader/Loader'; +import { ModalWithQueryEditor } from '../../Modals/ModalWithQueryEditor/ModalWithQueryEditor'; import SchemaEditor from '../../SchemaEditor/SchemaEditor'; import './entity-info-drawer.less'; import { @@ -39,13 +43,14 @@ const EdgeInfoDrawer = ({ onClose, nodes, hasEditAccess, - onEdgeDescriptionUpdate, + onEdgeDetailsUpdate, }: EdgeInfoDrawerInfo) => { const [edgeData, setEdgeData] = useState(); const [mysqlQuery, setMysqlQuery] = useState(''); const [isLoading, setIsLoading] = useState(false); const [isDescriptionEditable, setIsDescriptionEditable] = useState(false); + const [showSqlQueryModal, setShowSqlQueryModal] = useState(false); const { t } = useTranslation(); @@ -95,12 +100,14 @@ const EdgeInfoDrawer = ({ }, pipeline: { key: t('label.edge'), - value: data?.pipeline ? getEntityName(data?.pipeline) : undefined, + value: data?.edge?.pipeline + ? getEntityName(data?.edge?.pipeline) + : undefined, link: - data?.pipeline && + data?.edge?.pipeline && getEntityLink( - data?.pipeline.type, - getEncodedFqn(data?.pipeline.fullyQualifiedName) + data?.edge?.pipeline.type, + getEncodedFqn(data?.edge?.pipeline.fullyQualifiedName) ), }, functionInfo: { @@ -112,114 +119,171 @@ const EdgeInfoDrawer = ({ }; const edgeDescription = useMemo(() => { - return edgeEntity?.lineageDetails?.description ?? ''; + return edgeEntity?.description ?? ''; }, [edgeEntity]); const onDescriptionUpdate = useCallback( async (updatedHTML: string) => { - if (edgeDescription !== updatedHTML && edgeEntity) { + if (edgeDescription !== updatedHTML && edge) { const lineageDetails = { - ...edgeEntity.lineageDetails, + ...getLineageDetailsObject(edge), description: updatedHTML, }; + + const updatedEdgeDetails = { + edge: { + fromEntity: { + id: edgeEntity.fromEntity.id, + type: edgeEntity.fromEntity.type, + }, + toEntity: { + id: edgeEntity.toEntity.id, + type: edgeEntity.toEntity.type, + }, + lineageDetails, + }, + }; + await onEdgeDetailsUpdate?.(updatedEdgeDetails); + } + setIsDescriptionEditable(false); + }, + [edgeDescription, edgeEntity, edge] + ); + + const onSqlQueryUpdate = useCallback( + async (updatedQuery: string) => { + if (mysqlQuery !== updatedQuery && edge) { + const lineageDetails = { + ...getLineageDetailsObject(edge), + sqlQuery: updatedQuery, + }; + const updatedEdgeDetails = { edge: { fromEntity: { - id: edgeEntity.fromEntity, - type: edge.data.sourceType, + id: edgeEntity.fromEntity.id, + type: edgeEntity.fromEntity.type, }, toEntity: { - id: edgeEntity.toEntity, - type: edge.data.sourceType, + id: edgeEntity.toEntity.id, + type: edgeEntity.toEntity.type, }, lineageDetails, }, }; - await onEdgeDescriptionUpdate(updatedEdgeDetails); - setIsDescriptionEditable(false); - } else { - setIsDescriptionEditable(false); + await onEdgeDetailsUpdate?.(updatedEdgeDetails); + setMysqlQuery(updatedQuery); } + setShowSqlQueryModal(false); }, - [edgeDescription, edgeEntity, edge.data] + [edgeEntity, edge, mysqlQuery] ); useEffect(() => { setIsLoading(true); getEdgeInfo(); - setMysqlQuery(edge.data.edge?.lineageDetails?.sqlQuery); + setMysqlQuery(edge.data.edge?.sqlQuery); }, [edge, visible]); return ( - } - getContainer={false} - headerStyle={{ padding: 16 }} - mask={false} - open={visible} - style={{ position: 'absolute' }} - title={t('label.edge-information')}> - {isLoading ? ( - - ) : ( - - {edgeData && - Object.values(edgeData).map( - (data) => - data.value && ( - - - {`${data.key}:`} - + <> + } + getContainer={false} + headerStyle={{ padding: 16 }} + mask={false} + open={visible} + style={{ position: 'absolute' }} + title={t('label.edge-information')}> + {isLoading ? ( + + ) : ( + + {edgeData && + Object.values(edgeData).map( + (data) => + data.value && ( + + + {`${data.key}:`} + - {isUndefined(data.link) ? ( - {data.value} - ) : ( - - {data.value} - - )} - - ) - )} - - - setIsDescriptionEditable(false)} - onDescriptionEdit={() => setIsDescriptionEditable(true)} - onDescriptionUpdate={onDescriptionUpdate} - /> - - - - - {`${t('label.sql-uppercase-query')}:`} - - {mysqlQuery ? ( - {data.value} + ) : ( + + {data.value} + + )} + + ) + )} + + + setIsDescriptionEditable(false)} + onDescriptionEdit={() => setIsDescriptionEditable(true)} + onDescriptionUpdate={onDescriptionUpdate} /> - ) : ( - - {t('server.no-query-available')} - - )} - - + + + +
+ + {`${t('label.sql-uppercase-query')}`} + + {hasEditAccess && ( +
+ {mysqlQuery ? ( + + ) : ( + + {t('server.no-query-available')} + + )} + + + )} + + {showSqlQueryModal && ( + setShowSqlQueryModal(false)} + onSave={onSqlQueryUpdate} + /> )} - + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx index 35ea60c1d541..7ae2cdff0307 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx @@ -27,21 +27,11 @@ import { SearchIndex } from '../../../generated/entity/data/searchIndex'; import { StoredProcedure } from '../../../generated/entity/data/storedProcedure'; import { Table } from '../../../generated/entity/data/table'; import { Topic } from '../../../generated/entity/data/topic'; -import { getDashboardByFqn } from '../../../rest/dashboardAPI'; -import { getDataModelsByName } from '../../../rest/dataModelsAPI'; -import { getMlModelByFQN } from '../../../rest/mlModelAPI'; -import { getPipelineByFqn } from '../../../rest/pipelineAPI'; -import { getSearchIndexDetailsByFQN } from '../../../rest/SearchIndexAPI'; -import { getContainerByName } from '../../../rest/storageAPI'; -import { getStoredProceduresByName } from '../../../rest/storedProceduresAPI'; -import { getTableDetailsByFQN } from '../../../rest/tableAPI'; -import { getTopicByFqn } from '../../../rest/topicsAPI'; import { getHeaderLabel } from '../../../utils/EntityLineageUtils'; import { DRAWER_NAVIGATION_OPTIONS, getEntityTags, } from '../../../utils/EntityUtils'; -import { getEncodedFqn } from '../../../utils/StringsUtils'; import { getEntityIcon } from '../../../utils/TableUtils'; import ContainerSummary from '../../Explore/EntitySummaryPanel/ContainerSummary/ContainerSummary.component'; import DashboardSummary from '../../Explore/EntitySummaryPanel/DashboardSummary/DashboardSummary.component'; @@ -52,7 +42,6 @@ import SearchIndexSummary from '../../Explore/EntitySummaryPanel/SearchIndexSumm import StoredProcedureSummary from '../../Explore/EntitySummaryPanel/StoredProcedureSummary/StoredProcedureSummary.component'; import TableSummary from '../../Explore/EntitySummaryPanel/TableSummary/TableSummary.component'; import TopicSummary from '../../Explore/EntitySummaryPanel/TopicSummary/TopicSummary.component'; -import { SelectedNode } from '../EntityLineage/EntityLineage.interface'; import './entity-info-drawer.less'; import { LineageDrawerProps } from './EntityInfoDrawer.interface'; @@ -60,114 +49,24 @@ const EntityInfoDrawer = ({ show, onCancel, selectedNode, - isMainNode = false, }: LineageDrawerProps) => { const [entityDetail, setEntityDetail] = useState( {} as EntityDetailUnion ); - const [isLoading, setIsLoading] = useState(false); - - const fetchEntityDetail = async (selectedNode: SelectedNode) => { - let response = {}; - const encodedFqn = getEncodedFqn(selectedNode.fqn); - const commonFields = ['tags', 'owner']; - - setIsLoading(true); - try { - switch (selectedNode.type) { - case EntityType.TABLE: { - response = await getTableDetailsByFQN(encodedFqn, [ - ...commonFields, - 'columns', - 'usageSummary', - 'profile', - ]); - - break; - } - case EntityType.PIPELINE: { - response = await getPipelineByFqn(encodedFqn, [ - ...commonFields, - 'followers', - 'tasks', - ]); - - break; - } - case EntityType.TOPIC: { - response = await getTopicByFqn(encodedFqn ?? '', commonFields); - - break; - } - case EntityType.DASHBOARD: { - response = await getDashboardByFqn(encodedFqn, [ - ...commonFields, - 'charts', - ]); - - break; - } - case EntityType.MLMODEL: { - response = await getMlModelByFQN(encodedFqn, [ - ...commonFields, - 'dashboard', - ]); - - break; - } - case EntityType.CONTAINER: { - response = await getContainerByName( - encodedFqn, - 'dataModel,owner,tags' - ); - - break; - } - - case EntityType.DASHBOARD_DATA_MODEL: { - response = await getDataModelsByName( - encodedFqn, - 'owner,tags,followers' - ); - - break; - } - - case EntityType.STORED_PROCEDURE: { - response = await getStoredProceduresByName(encodedFqn, 'owner,tags'); - - break; - } - - case EntityType.SEARCH_INDEX: { - response = await getSearchIndexDetailsByFQN(encodedFqn, 'owner,tags'); - - break; - } - - default: - break; - } - setEntityDetail(response); - } finally { - setIsLoading(false); - } - }; - const tags = useMemo( - () => getEntityTags(selectedNode.type, entityDetail), + () => + getEntityTags(selectedNode.entityType ?? EntityType.TABLE, entityDetail), [entityDetail, selectedNode] ); const summaryComponent = useMemo(() => { - switch (selectedNode.type) { + switch (selectedNode.entityType) { case EntityType.TABLE: return ( ); @@ -177,7 +76,6 @@ const EntityInfoDrawer = ({ ); @@ -187,7 +85,6 @@ const EntityInfoDrawer = ({ ); @@ -197,7 +94,6 @@ const EntityInfoDrawer = ({ ); @@ -207,7 +103,6 @@ const EntityInfoDrawer = ({ ); @@ -216,7 +111,6 @@ const EntityInfoDrawer = ({ ); @@ -226,7 +120,6 @@ const EntityInfoDrawer = ({ ); @@ -236,7 +129,6 @@ const EntityInfoDrawer = ({ ); @@ -246,7 +138,6 @@ const EntityInfoDrawer = ({ ); @@ -254,10 +145,10 @@ const EntityInfoDrawer = ({ default: return null; } - }, [entityDetail, fetchEntityDetail, tags, selectedNode, isLoading]); + }, [entityDetail, tags, selectedNode]); useEffect(() => { - fetchEntityDetail(selectedNode); + setEntityDetail(selectedNode); }, [selectedNode]); return ( @@ -277,7 +168,7 @@ const EntityInfoDrawer = ({ open={show} style={{ position: 'absolute' }} title={ - + {'databaseSchema' in entityDetail && 'database' in entityDetail && ( + className={classNames( + 'flex items-center text-base entity-info-header-link' + )}> - {getEntityIcon(selectedNode.type)} + {getEntityIcon(selectedNode.entityType as string)} {getHeaderLabel( selectedNode.displayName ?? selectedNode.name, - selectedNode.fqn, - selectedNode.type, - isMainNode + selectedNode.fullyQualifiedName, + selectedNode.entityType as string, + false )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts index 6e1eaeea48ce..454f3b7c4d42 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts @@ -13,13 +13,12 @@ import { Edge, Node } from 'reactflow'; import { AddLineage } from '../../../generated/api/lineage/addLineage'; -import { SelectedNode } from '../EntityLineage/EntityLineage.interface'; +import { SourceType } from '../../SearchedData/SearchedData.interface'; export interface LineageDrawerProps { show: boolean; onCancel: () => void; - selectedNode: SelectedNode; - isMainNode: boolean; + selectedNode: SourceType; } export interface EdgeInfoDrawerInfo { @@ -28,7 +27,7 @@ export interface EdgeInfoDrawerInfo { visible: boolean; hasEditAccess: boolean; onClose: () => void; - onEdgeDescriptionUpdate: (updatedEdgeDetails: AddLineage) => Promise; + onEdgeDetailsUpdate?: (updatedEdgeDetails: AddLineage) => Promise; } type InfoType = { key: string; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/entity-info-drawer.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/entity-info-drawer.less index fd23f7bb7bc5..cd87d66c1b5a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/entity-info-drawer.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/entity-info-drawer.less @@ -29,7 +29,7 @@ } .entity-panel-container { - margin-top: 60px; + margin-top: 54px; .ant-drawer-header { border-bottom: none; padding-bottom: 0 !important; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.test.tsx index 7f4c5c9e1cb8..7c222411fe05 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.test.tsx @@ -52,9 +52,7 @@ describe('Test CustomEdge Component', () => { }); it('CTA should work properly', async () => { - render( - - ); + render(); const removeEdge = await screen.findByTestId('remove-edge-button'); const saveButton = await screen.findByTestId('save-button'); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx index 843ce506cf6d..c22df4e72333 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx @@ -12,44 +12,78 @@ */ import { Button, Input, Modal } from 'antd'; +import { AxiosError } from 'axios'; import classNames from 'classnames'; import { t } from 'i18next'; -import { isEmpty, isUndefined } from 'lodash'; -import React, { useMemo } from 'react'; +import { isUndefined } from 'lodash'; +import React, { useEffect, useMemo, useState } from 'react'; +import { Edge } from 'reactflow'; +import { PAGE_SIZE } from '../../../../constants/constants'; import { ERROR_PLACEHOLDER_TYPE, SIZE } from '../../../../enums/common.enum'; +import { EntityType } from '../../../../enums/entity.enum'; +import { SearchIndex } from '../../../../enums/search.enum'; import { EntityReference } from '../../../../generated/entity/type'; -import { getEntityName } from '../../../../utils/EntityUtils'; +import { searchData } from '../../../../rest/miscAPI'; +import { + getEntityName, + getEntityReferenceFromEntity, +} from '../../../../utils/EntityUtils'; import Fqn from '../../../../utils/Fqn'; import { getEntityIcon } from '../../../../utils/TableUtils'; +import { showErrorToast } from '../../../../utils/ToastUtils'; import ErrorPlaceHolder from '../../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import '../../../FeedEditor/feed-editor.less'; import './add-pipeline-modal.less'; interface AddPipeLineModalType { showAddEdgeModal: boolean; - edgeSearchValue: string; - selectedEdgeId: string | undefined; - edgeOptions: EntityReference[]; + selectedEdge?: Edge; onModalCancel: () => void; - onSave: () => void; + onSave: (value?: EntityReference) => void; onRemoveEdgeClick: (evt: React.MouseEvent) => void; - onSearch: (value: string) => void; - onSelect: (value: string) => void; } const AddPipeLineModal = ({ showAddEdgeModal, - edgeOptions, - edgeSearchValue, - selectedEdgeId, + selectedEdge, onRemoveEdgeClick, onModalCancel, onSave, - onSearch, - onSelect, }: AddPipeLineModalType) => { + const currentPipeline = selectedEdge?.data.edge.pipeline; + const [edgeSearchValue, setEdgeSearchValue] = useState(''); + const [edgeSelection, setEdgeSelection] = useState( + currentPipeline ?? {} + ); + const [edgeOptions, setEdgeOptions] = useState([]); + + const getSearchResults = async (value = '*') => { + try { + const data = await searchData(value, 1, PAGE_SIZE, '', '', '', [ + SearchIndex.PIPELINE, + SearchIndex.STORED_PROCEDURE, + ]); + + const edgeOptions = data.data.hits.hits.map((hit) => + getEntityReferenceFromEntity( + hit._source, + hit._source.entityType as EntityType + ) + ); + + setEdgeOptions(edgeOptions); + } catch (error) { + showErrorToast( + error as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.suggestion-lowercase-plural'), + }) + ); + } + }; + const errorPlaceholderEdge = useMemo(() => { - if (isEmpty(edgeOptions)) { + if (isUndefined(selectedEdge)) { if (edgeSearchValue) { return ( { + getSearchResults(edgeSearchValue); + }, [edgeSearchValue]); return ( + onClick={() => onSave(edgeSelection)}> {t('label.save')} , ]} maskClosable={false} open={showAddEdgeModal} - title={t(`label.${isUndefined(selectedEdgeId) ? 'add' : 'edit'}-entity`, { + title={t(`label.${isUndefined(selectedEdge) ? 'add' : 'edit'}-entity`, { entity: t('label.edge'), })} onCancel={onModalCancel}> @@ -98,7 +136,7 @@ const AddPipeLineModal = ({ data-testid="field-input" placeholder={t('message.search-for-edge')} value={edgeSearchValue} - onChange={(e) => onSearch(e.target.value)} + onChange={(e) => setEdgeSearchValue(e.target.value)} />
@@ -109,10 +147,10 @@ const AddPipeLineModal = ({ return (
onSelect(item.id)}> + onClick={() => setEdgeSelection(item)}>
{icon}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx index 5ede34d32961..3bc65210196b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx @@ -11,9 +11,9 @@ * limitations under the License. */ -import { SettingOutlined } from '@ant-design/icons'; -import { Button, Col, Row, Select, Space } from 'antd'; -import Input from 'antd/lib/input/Input'; +import { RightOutlined, SettingOutlined } from '@ant-design/icons'; +import { Button, Col, Dropdown, Row, Select, Space } from 'antd'; +import { ItemType } from 'antd/lib/menu/hooks/useItems'; import classNames from 'classnames'; import React, { FC, @@ -24,7 +24,7 @@ import React, { useState, } from 'react'; import { useTranslation } from 'react-i18next'; -import { useReactFlow } from 'reactflow'; +import { Node } from 'reactflow'; import { ReactComponent as ExitFullScreen } from '../../../assets/svg/exit-full-screen.svg'; import { ReactComponent as FullScreen } from '../../../assets/svg/full-screen.svg'; import { ReactComponent as EditIconColor } from '../../../assets/svg/ic-edit-lineage-colored.svg'; @@ -32,94 +32,91 @@ import { ReactComponent as EditIcon } from '../../../assets/svg/ic-edit-lineage. import { PRIMERY_COLOR } from '../../../constants/constants'; import { NO_PERMISSION_FOR_ACTION } from '../../../constants/HelperTextUtil'; import { - MAX_ZOOM_VALUE, - MIN_ZOOM_VALUE, - ZOOM_BUTTON_STEP, - ZOOM_SLIDER_STEP, + LINEAGE_DEFAULT_QUICK_FILTERS, ZOOM_TRANSITION_DURATION, } from '../../../constants/Lineage.constants'; +import { SearchIndex } from '../../../enums/search.enum'; +import { getAssetsPageQuickFilters } from '../../../utils/AdvancedSearchUtils'; import { handleSearchFilterOption } from '../../../utils/CommonUtils'; import { getLoadingStatusValue } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; -import SVGIcons, { Icons } from '../../../utils/SvgUtils'; +import { + getQuickFilterQuery, + getSelectedValuesFromQuickFilter, +} from '../../../utils/Explore.utils'; +import { ExploreQuickFilterField } from '../../Explore/ExplorePage.interface'; +import ExploreQuickFilters from '../../Explore/ExploreQuickFilters'; +import { useLineageProvider } from '../../LineageProvider/LineageProvider'; import { ControlProps, LineageConfig } from './EntityLineage.interface'; import LineageConfigModal from './LineageConfigModal'; const CustomControls: FC = ({ style, - isColumnsExpanded, - showFitView = true, - showZoom = true, - fitViewParams, className, deleted, - isEditMode, hasEditAccess, - onEditLinageClick, - onExpandColumnClick, handleFullScreenViewClick, onExitFullScreenViewClick, - loading, - status, - zoomValue, - lineageData, - lineageConfig, - onOptionSelect, - onLineageConfigUpdate, }: ControlProps) => { const { t } = useTranslation(); - const { fitView, zoomTo } = useReactFlow(); - const [zoom, setZoom] = useState(zoomValue); const [dialogVisible, setDialogVisible] = useState(false); + const { + nodes, + lineageConfig, + expandAllColumns, + onLineageEditClick, + zoomValue, + loading, + status, + reactFlowInstance, + toggleColumnView, + isEditMode, + onLineageConfigUpdate, + onQueryFilterUpdate, + onNodeClick, + } = useLineageProvider(); + const [selectedFilter, setSelectedFilter] = useState([]); + const [selectedQuickFilters, setSelectedQuickFilters] = useState< + ExploreQuickFilterField[] + >([]); + const [filters, setFilters] = useState([]); - const onZoomHandler = useCallback( - (zoomLevel: number) => { - zoomTo?.(zoomLevel, { duration: ZOOM_TRANSITION_DURATION }); - }, - [zoomTo] - ); - - const onZoomInHandler = useCallback(() => { - setZoom((pre) => { - const zoomInValue = pre < MAX_ZOOM_VALUE ? pre + ZOOM_BUTTON_STEP : pre; - onZoomHandler(zoomInValue); - - return zoomInValue; - }); - }, [onZoomHandler]); + const handleMenuClick = ({ key }: { key: string }) => { + setSelectedFilter((prevSelected) => [...prevSelected, key]); + }; - const onZoomOutHandler = useCallback(() => { - setZoom((pre) => { - const zoomOutValue = pre > MIN_ZOOM_VALUE ? pre - ZOOM_BUTTON_STEP : pre; - onZoomHandler(zoomOutValue); + const filterMenu: ItemType[] = useMemo(() => { + return filters.map((filter) => ({ + key: filter.key, + label: filter.label, + onClick: handleMenuClick, + })); + }, [filters]); - return zoomOutValue; - }); - }, [onZoomHandler]); + useEffect(() => { + const dropdownItems = getAssetsPageQuickFilters(); - const onFitViewHandler = useCallback(() => { - fitView?.(fitViewParams); - }, [fitView, fitViewParams]); + setFilters( + dropdownItems.map((item) => ({ + ...item, + value: getSelectedValuesFromQuickFilter(item, dropdownItems), + })) + ); - const onRangeChange = (event: React.ChangeEvent) => { - const zoomValue = parseFloat(event.target.value); - onZoomHandler(zoomValue); - setZoom(zoomValue); - }; + const defaultFilterValues = dropdownItems + .filter((item) => LINEAGE_DEFAULT_QUICK_FILTERS.includes(item.key)) + .map((item) => item.key); - useEffect(() => { - if (zoomValue !== zoom) { - setZoom(zoomValue); - } - }, [zoomValue]); + setSelectedFilter(defaultFilterValues); + }, []); const nodeOptions = useMemo( () => - [lineageData.entity, ...(lineageData.nodes || [])].map((node) => ({ - label: getEntityName(node), + [...(nodes || [])].map((node) => ({ + label: getEntityName(node.data.node), value: node.id, })), - [lineageData] + [nodes] ); const editIcon = useMemo(() => { @@ -136,19 +133,82 @@ const CustomControls: FC = ({ const handleDialogSave = useCallback( (config: LineageConfig) => { - onLineageConfigUpdate(config); + onLineageConfigUpdate?.(config); setDialogVisible(false); }, [onLineageConfigUpdate, setDialogVisible] ); + const onOptionSelect = useCallback( + (value?: string) => { + const selectedNode = nodes.find((node: Node) => node.id === value); + if (selectedNode) { + const { position } = selectedNode; + onNodeClick(selectedNode); + // moving selected node in center + reactFlowInstance?.setCenter(position.x, position.y, { + duration: ZOOM_TRANSITION_DURATION, + zoom: zoomValue, + }); + } + }, + [onNodeClick, reactFlowInstance] + ); + + const handleQuickFiltersChange = (data: ExploreQuickFilterField[]) => { + const quickFilterQuery = getQuickFilterQuery(data); + onQueryFilterUpdate(JSON.stringify(quickFilterQuery)); + }; + + const handleQuickFiltersValueSelect = useCallback( + (field: ExploreQuickFilterField) => { + setSelectedQuickFilters((pre) => { + const data = pre.map((preField) => { + if (preField.key === field.key) { + return field; + } else { + return preField; + } + }); + + handleQuickFiltersChange(data); + + return data; + }); + }, + [setSelectedQuickFilters] + ); + + useEffect(() => { + const updatedQuickFilters = filters + .filter((filter) => selectedFilter.includes(filter.key)) + .map((selectedFilterItem) => { + const originalFilterItem = selectedQuickFilters?.find( + (filter) => filter.key === selectedFilterItem.key + ); + + return originalFilterItem || selectedFilterItem; + }); + + const newItems = updatedQuickFilters.filter( + (item) => + !selectedQuickFilters.some( + (existingItem) => item.key === existingItem.key + ) + ); + + if (newItems.length > 0) { + setSelectedQuickFilters((prevSelected) => [...prevSelected, ...newItems]); + } + }, [selectedFilter, selectedQuickFilters, filters]); + return ( <> - + - -
- )} - {showFitView && ( -
+ } + type="text" + onClick={(e) => { + e.stopPropagation(); + setIsExpanded((prevIsExpanded: boolean) => !prevIsExpanded); + }}> + {t('label.column-plural')} + {isExpanded ? ( + + ) : ( + + )} + + {node.entityType === EntityType.TABLE && testSuite && ( +
+
+
+ {formTwoDigitNumber(testSuite?.summary?.success ?? 0)} +
+
+
+
+ {formTwoDigitNumber(testSuite?.summary?.aborted ?? 0)} +
+
+
+
+ {formTwoDigitNumber(testSuite?.summary?.failed ?? 0)} +
+
+
)} - +
{isExpanded && (
@@ -182,8 +410,8 @@ const CustomNodeV1 = (props: NodeProps) => {
{filteredColumns.map((column) => { - const isColumnTraced = selectedColumns.includes( - column.fullyQualifiedName + const isColumnTraced = tracedColumns.includes( + column.fullyQualifiedName ?? '' ); return ( @@ -198,7 +426,7 @@ const CustomNodeV1 = (props: NodeProps) => { key={column.fullyQualifiedName} onClick={(e) => { e.stopPropagation(); - handleColumnClick(column.fullyQualifiedName); + onColumnClick(column.fullyQualifiedName ?? ''); }}> {getColumnHandle( column.type, @@ -217,6 +445,7 @@ const CustomNodeV1 = (props: NodeProps) => { {!showAllColumns && (
- ), - removeNodeHandler, - onNodeExpand: handleNodeExpand, - isEditMode, - isNewNode: true, - }, - }; - setNewAddedNode(newNode as Node); - - setNodes( - (es) => getUniqueFlowElements(es.concat(newNode as Node)) as Node[] - ); - } - }; - - /** - * After dropping node to graph user will search and select entity - * and this method will take care of changing node information based on selected entity. - */ - const onEntitySelect = () => { - if (!isEmpty(selectedEntity)) { - const isExistingNode = nodes.some((n) => n.id === selectedEntity.id); - if (isExistingNode) { - setNodes((es) => - es - .map((n) => - n.id.includes(selectedEntity.id) - ? { - ...n, - selectable: true, - className: `${n.className} selected`, - } - : n - ) - .filter((es) => es.id !== newAddedNode.id) - ); - resetSelectedData(); - } else { - setNodes((es) => { - return es.map((el) => { - if (el.id === newAddedNode.id) { - return { - ...el, - connectable: true, - selectable: true, - id: selectedEntity.id, - data: { - ...el.data, - removeNodeHandler, - isEditMode, - node: selectedEntity, - }, - }; - } else { - return el; - } - }); - }); - } - } - }; - - /** - * This method will handle the delete edge modal confirmation - */ - const onRemove = useCallback(() => { - setDeletionState({ ...ELEMENT_DELETE_STATE, loading: true }); - setTimeout(() => { - setDeletionState({ ...ELEMENT_DELETE_STATE, status: 'success' }); - setTimeout(() => { - setShowDeleteModal(false); - setConfirmDelete(true); - setDeletionState((pre) => ({ ...pre, status: 'initial' })); - }, 500); - }, 500); - }, []); - - const handleEditLineageClick = useCallback(() => { - setEditMode((pre) => !pre && !deleted); - resetSelectedData(); - setIsDrawerOpen(false); - }, [deleted]); - - const handleEdgeClick = useCallback( - (_e: React.MouseEvent, edge: Edge) => { - setSelectedEdgeInfo(edge); - setIsDrawerOpen(true); - }, - [] - ); - - const toggleColumnView = (value: boolean) => { - setExpandAllColumns(value); - setEdges((prevEdges) => { - return prevEdges.map((edge) => { - edge.data.isExpanded = value; - - return edge; - }); - }); - setNodes((prevNodes) => { - const updatedNode = prevNodes.map((node) => { - node.data.isExpanded = value; - - return node; - }); - const { edge, node } = getLayoutedElements({ - node: updatedNode, - edge: edges, - }); - setEdges(edge); - - return node; - }); - }; - - const handleExpandColumnClick = () => { - if (!updatedLineageData) { - return; - } - if (expandAllColumns) { - toggleColumnView(false); - } else { - const allTableNodes = nodes - .map((item) => item.data.node) - .filter( - (node) => - [EntityType.TABLE, EntityType.DASHBOARD_DATA_MODEL].includes( - node.type as EntityType - ) && isUndefined(tableColumnsRef.current[node.id]) - ); - - allTableNodes.length && - allTableNodes.map(async (node) => await getTableColumns(node)); - toggleColumnView(true); - } - }; - - const getSearchResults = async (value = '*') => { - try { - const data = await searchData(value, 1, PAGE_SIZE, '', '', '', [ - SearchIndex.PIPELINE, - SearchIndex.STORED_PROCEDURE, - ]); - - const selectedPipeline = selectedEdge.data?.pipeline; - - const edgeOptions = data.data.hits.hits.map((hit) => - getEntityReferenceFromEntity( - hit._source, - hit._source.entityType as EntityType - ) - ); - - const optionContainItem = edgeOptions.find( - (item) => item.id === selectedEdgeId - ); - - setEdgeOptions([ - ...(selectedPipeline && - isEmpty(optionContainItem) && - isEmpty(edgeSearchValue) && - selectedPipeline.id === selectedEdgeId - ? [selectedPipeline] - : []), - ...edgeOptions, - ]); - } catch (error) { - showErrorToast( - error as AxiosError, - t('server.entity-fetch-error', { - entity: t('label.suggestion-lowercase-plural'), - }) - ); - } - }; - - const handleLineageConfigUpdate = useCallback((config: LineageConfig) => { - setLineageConfig(config); - fetchLineageData(config); - }, []); - const selectNode = (node: Node) => { - const { position } = node; - onNodeClick(node); - // moving selected node in center - reactFlowInstance && - reactFlowInstance.setCenter(position.x, position.y, { - duration: ZOOM_TRANSITION_DURATION, - zoom: zoomValue, - }); - }; - - const handleOptionSelect = (value?: string) => { - if (value) { - const selectedNode = nodes.find((node) => node.id === value); - - if (selectedNode) { - selectNode(selectedNode); - } else { - const path = findNodeById(value, childMap?.children, []) || []; - const lastNode = path[path?.length - 1]; - if (updatedLineageData) { - const { nodes, edges } = getPaginatedChildMap( - updatedLineageData, - childMap, - paginationData, - lineageConfig.nodesPerLayer - ); - const newNodes = union(nodes, path); - setElementsHandle( - { - ...updatedLineageData, - nodes: newNodes, - downstreamEdges: [ - ...(updatedLineageData.downstreamEdges || []), - ...edges, - ], - }, - lastNode.id - ); - } - } - } - }; - - const onEdgeDescriptionUpdate = useCallback( - async (updatedEdgeDetails: AddLineage) => { - try { - await updateLineageEdge(updatedEdgeDetails); - if (selectedEdgeInfo) { - const updatedSelectedEdgeInfo = { - ...selectedEdgeInfo, - data: { - ...selectedEdgeInfo.data, - edge: { - ...selectedEdgeInfo.data.edge, - lineageDetails: updatedEdgeDetails.edge.lineageDetails, - }, - }, - }; - - const updatedEdges = edges.map((edge) => - edge.id === selectedEdgeInfo.id ? updatedSelectedEdgeInfo : edge - ); - - setEdges(updatedEdges); - setSelectedEdgeInfo(updatedSelectedEdgeInfo); - - setUpdatedLineageData((pre) => { - if (!pre) { - return; - } - - const newData = { - ...pre, - downstreamEdges: updateEdgesWithLineageDetails( - pre.downstreamEdges ?? [], - updatedEdgeDetails - ), - upstreamEdges: updateEdgesWithLineageDetails( - pre.upstreamEdges ?? [], - updatedEdgeDetails - ), - }; - - return newData; - }); - } - } catch (err) { - showErrorToast(err as AxiosError); - } - }, - [edges, selectedEdgeInfo, updatedLineageData, setUpdatedLineageData] - ); - - /** - * Handle updated lineage nodes - * Change newly added node label based on entity:EntityReference - */ - const handleUpdatedLineageNode = () => { - const uNodes = updatedLineageData?.nodes; - const newlyAddedNodeElement = nodes.find((el) => el?.data?.isNewNode); - const newlyAddedNode = uNodes?.find( - (node) => node.id === newlyAddedNodeElement?.id - ); - - setNodes((els) => { - return (els || []).map((el) => { - if (el.id === newlyAddedNode?.id) { - return { - ...el, - data: { - ...el.data, - }, - }; - } else { - return el; - } - }); - }); - }; - - const handleZoomLevel = debounce((value: number) => { - setZoomValue(value); - }, 150); - - const initLineageChildMaps = ( - lineageData: EntityLineage, - childMapObj: EntityReferenceChild | undefined, - paginationObj: Record - ) => { - if (lineageData && childMapObj) { - const { nodes: newNodes, edges } = getPaginatedChildMap( - lineageData, - childMapObj, - paginationObj, - lineageConfig.nodesPerLayer - ); - setElementsHandle({ - ...lineageData, - nodes: newNodes, - downstreamEdges: [...(lineageData.downstreamEdges || []), ...edges], - }); - } - }; - - useEffect(() => { - if (!entity?.deleted) { - fetchLineageData(lineageConfig); - } - }, [entity?.deleted]); - - useEffect(() => { - if (!entityLineage) { - return; - } - if ( - !isEmpty(entityLineage) && - !isUndefined(entityLineage.entity) && - !deleted - ) { - const childMapObj: EntityReferenceChild = getChildMap(entityLineage); - setChildMap(childMapObj); - initLineageChildMaps(entityLineage, childMapObj, paginationData); - } - }, [entityLineage]); - - useEffect(() => { - if (!updatedLineageData) { - return; - } - setEntityLineage({ - ...updatedLineageData, - nodes: getNewNodes(updatedLineageData), - }); - }, [isEditMode]); - - useEffect(() => { - handleUpdatedLineageNode(); - }, [updatedLineageData]); - - useEffect(() => { - onEntitySelect(); - }, [selectedEntity]); - - useEffect(() => { - if (selectedEdge.data?.isColumnLineage) { - removeColumnEdge(selectedEdge, confirmDelete); - } else { - removeEdgeHandler(selectedEdge, confirmDelete); - } - }, [selectedEdge, confirmDelete]); - - useEffect(() => { - if (showAddEdgeModal) { - getSearchResults(edgeSearchValue); - } - }, [edgeSearchValue, showAddEdgeModal]); - - useEffect(() => { - edgesRef.current = edges; - }, [edges]); - - if (isLineageLoading || (nodes.length === 0 && !deleted)) { - return ; - } - - if (deleted) { - return getDeletedLineagePlaceholder(); - } - - return ( - - {isFullScreen && ( - - )} -
-
- - { - onLoad(reactFlowInstance); - setReactFlowInstance(reactFlowInstance); - }} - onMove={(_e, viewPort) => handleZoomLevel(viewPort.zoom)} - onNodeClick={(_e, node) => { - onNodeClick(node); - _e.stopPropagation(); - }} - onNodeContextMenu={onNodeContextMenu} - onNodeDrag={dragHandle} - onNodeDragStart={dragHandle} - onNodeDragStop={dragHandle} - onNodeMouseEnter={onNodeMouseEnter} - onNodeMouseLeave={onNodeMouseLeave} - onNodeMouseMove={onNodeMouseMove} - onNodesChange={onNodesChange} - onPaneClick={onPaneClick}> - {updatedLineageData && ( - - )} - - - -
- {isDrawerOpen && - !isEditMode && - (selectedEdgeInfo ? ( - { - setIsDrawerOpen(false); - setSelectedEdgeInfo(undefined); - }} - onEdgeDescriptionUpdate={onEdgeDescriptionUpdate} - /> - ) : ( - - ))} - - {showDeleteModal && ( - { - setShowDeleteModal(false); - }} - onOk={onRemove}> - {getModalBodyText(selectedEdge)} - - )} - - setEdgeSearchValue(value)} - onSelect={handleEdgeSelection} - /> -
- - ); -}; - -export default withLoader(EntityLineageComponent); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts index 41b618ffe1b1..25913d9a67b9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts @@ -13,11 +13,10 @@ import { LoadingState } from 'Models'; import { HTMLAttributes } from 'react'; -import { Edge as FlowEdge, FitViewOptions, Node } from 'reactflow'; +import { Edge as FlowEdge, Node } from 'reactflow'; import { EntityType } from '../../../enums/entity.enum'; import { Column } from '../../../generated/entity/data/container'; import { EntityReference } from '../../../generated/entity/type'; -import { EntityLineage } from '../../../generated/type/entityLineage'; import { SourceType } from '../../SearchedData/SearchedData.interface'; export interface SelectedNode { @@ -108,27 +107,13 @@ export enum EdgeTypeEnum { } export interface ControlProps extends HTMLAttributes { - showZoom?: boolean; - showFitView?: boolean; - fitViewParams?: FitViewOptions; - onZoomIn?: () => void; - onZoomOut?: () => void; - onFitView?: () => void; handleFullScreenViewClick?: () => void; onExitFullScreenViewClick?: () => void; deleted: boolean | undefined; - isEditMode: boolean; hasEditAccess: boolean | undefined; - isColumnsExpanded: boolean; - onEditLinageClick: () => void; - onExpandColumnClick: () => void; - loading: boolean; - status: LoadingState; - zoomValue: number; - lineageData: EntityLineage; - lineageConfig: LineageConfig; - onOptionSelect: (value?: string) => void; - onLineageConfigUpdate: (config: LineageConfig) => void; + onExpandColumnClick?: () => void; + onOptionSelect?: (value?: string) => void; + onLineageConfigUpdate?: (config: LineageConfig) => void; } export type LineagePos = 'from' | 'to'; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineageSidebar.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineageSidebar.component.tsx index 5144d2a81d16..3d2a34d27d98 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineageSidebar.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineageSidebar.component.tsx @@ -40,14 +40,15 @@ const EntityNode: FC = ({ type, label, draggable }) => { }; return ( -
+
onDragStart(event, `${type}-default`)}> + onDragStart={(event) => onDragStart(event, type)}> { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/Entitylineage.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/Entitylineage.component.test.tsx deleted file mode 100644 index bc66a51b879f..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/Entitylineage.component.test.tsx +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2022 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { fireEvent, render, screen } from '@testing-library/react'; -import React from 'react'; -import { act } from 'react-dom/test-utils'; -import { MemoryRouter } from 'react-router-dom'; -import { EntityType } from '../../../enums/entity.enum'; -import { MOCK_CHILD_MAP, MOCK_LINEAGE_DATA } from '../../../mocks/Lineage.mock'; -import EntityLineage from './EntityLineage.component'; - -const mockEntityLineageProp = { - deleted: false, - entityType: EntityType.TABLE, - hasEditAccess: true, -}; - -const mockFlowData = { - node: [ - { - id: 'a4b21449-b03b-4527-b482-148f52f92ff2', - sourcePosition: 'right', - targetPosition: 'left', - type: 'default', - className: 'leaf-node core', - data: { - label: 'dim_address etl', - isEditMode: false, - columns: {}, - isExpanded: false, - }, - position: { - x: 0, - y: 0, - }, - }, - ], - edge: [], -}; - -const mockPaginatedData = { - nodes: [...mockFlowData.node], - edges: [], -}; - -jest.mock('../../../utils/EntityLineageUtils', () => ({ - dragHandle: jest.fn(), - getDeletedLineagePlaceholder: jest - .fn() - .mockReturnValue( -

Lineage data is not available for deleted entities.

- ), - getHeaderLabel: jest.fn().mockReturnValue(

Header label

), - getLoadingStatusValue: jest.fn().mockReturnValue(

Confirm

), - getLayoutedElements: jest.fn().mockImplementation(() => mockFlowData), - getLineageData: jest.fn().mockImplementation(() => mockFlowData), - getPaginatedChildMap: jest.fn().mockImplementation(() => mockPaginatedData), - getChildMap: jest.fn().mockImplementation(() => MOCK_CHILD_MAP), - getModalBodyText: jest.fn(), - onLoad: jest.fn(), - onNodeContextMenu: jest.fn(), - onNodeMouseEnter: jest.fn(), - onNodeMouseLeave: jest.fn(), - onNodeMouseMove: jest.fn(), - getUniqueFlowElements: jest.fn().mockReturnValue([]), - getParamByEntityType: jest.fn().mockReturnValue('entityFQN'), -})); - -jest.mock('../../../rest/lineageAPI', () => ({ - getLineageByFQN: jest.fn().mockImplementation(() => - Promise.resolve({ - ...MOCK_LINEAGE_DATA, - }) - ), -})); - -jest.mock('../EntityInfoDrawer/EntityInfoDrawer.component', () => { - return jest.fn().mockReturnValue(

EntityInfoDrawerComponent

); -}); - -const mockPush = jest.fn(); - -jest.mock('react-router-dom', () => ({ - ...jest.requireActual('react-router-dom'), - useHistory: () => ({ - push: mockPush, - }), -})); - -describe('Test EntityLineage Component', () => { - it('Check if EntityLineage is rendering all the nodes', async () => { - act(() => { - render(, { - wrapper: MemoryRouter, - }); - }); - - const lineageContainer = await screen.findByTestId('lineage-container'); - const reactFlowElement = await screen.findByTestId('rf__wrapper'); - - expect(lineageContainer).toBeInTheDocument(); - expect(reactFlowElement).toBeInTheDocument(); - }); - - it('Check if EntityLineage has deleted as true', async () => { - act(() => { - render(, { - wrapper: MemoryRouter, - }); - }); - - const lineageContainer = screen.queryByTestId('lineage-container'); - const reactFlowElement = screen.queryByTestId('rf__wrapper'); - const deletedMessage = await screen.findByText( - /Lineage data is not available for deleted entities/i - ); - - expect(deletedMessage).toBeInTheDocument(); - - expect(reactFlowElement).not.toBeInTheDocument(); - - expect(lineageContainer).not.toBeInTheDocument(); - }); - - it('should add fullscreen true in url on fullscreen button click', async () => { - render(, { - wrapper: MemoryRouter, - }); - - const lineageContainer = await screen.findByTestId('lineage-container'); - const reactFlowElement = await screen.findByTestId('rf__wrapper'); - const fullscreenButton = await screen.getByTestId('full-screen'); - - expect(lineageContainer).toBeInTheDocument(); - expect(reactFlowElement).toBeInTheDocument(); - - act(() => { - fireEvent.click(fullscreenButton); - }); - - expect(mockPush).toHaveBeenCalledTimes(1); - expect(mockPush).toHaveBeenCalledWith({ - search: 'fullscreen=true', - }); - }); - - it('should show breadcrumbs when URL has fullscreen=true', async () => { - act(() => { - render( - - - - ); - }); - const lineageContainer = await screen.findByTestId('lineage-container'); - const reactFlowElement = await screen.findByTestId('rf__wrapper'); - - expect(lineageContainer).toBeInTheDocument(); - expect(reactFlowElement).toBeInTheDocument(); - - const breadcrumbs = await screen.getByTestId('breadcrumb'); - - expect(breadcrumbs).toBeInTheDocument(); - - const mainRootElement = await screen.getByTestId('lineage-details'); - - expect(mainRootElement).toHaveClass('full-screen-lineage'); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx index 7f8a2254be4d..83ec104af80b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Form, InputNumber, Modal, Select } from 'antd'; +import { Form, Input, Modal } from 'antd'; import React, { useState } from 'react'; import { useTranslation } from 'react-i18next'; import { @@ -18,12 +18,6 @@ import { LineageConfigModalProps, } from './EntityLineage.interface'; -const SELECT_OPTIONS = [1, 2, 3].map((value) => ( - - {value} - -)); - const LineageConfigModal: React.FC = ({ visible, config, @@ -33,13 +27,13 @@ const LineageConfigModal: React.FC = ({ const { t } = useTranslation(); const [form] = Form.useForm(); const [upstreamDepth, setUpstreamDepth] = useState( - config.upstreamDepth || 1 + config?.upstreamDepth || 1 ); const [downstreamDepth, setDownstreamDepth] = useState( - config.downstreamDepth || 1 + config?.downstreamDepth || 1 ); const [nodesPerLayer, setNodesPerLayer] = useState( - config.nodesPerLayer || 1 + config?.nodesPerLayer || 1 ); const handleSave = () => { @@ -73,11 +67,12 @@ const LineageConfigModal: React.FC = ({ }, ]} tooltip={t('message.upstream-depth-tooltip')}> - + type="number" + value={upstreamDepth} + onChange={(e) => setUpstreamDepth(Number(e.target.value))} + /> = ({ }, ]} tooltip={t('message.downstream-depth-tooltip')}> - + type="number" + value={downstreamDepth} + onChange={(e) => setDownstreamDepth(Number(e.target.value))} + /> = ({ }, ]} tooltip={t('message.nodes-per-layer-tooltip')}> - setNodesPerLayer(value as number)} + type="number" + onChange={(e) => setNodesPerLayer(Number(e.target.value))} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx index 7a577d81d7d3..db80940f3116 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx @@ -18,7 +18,8 @@ import { useTranslation } from 'react-i18next'; import { EntityLineageNodeType } from '../../../enums/entity.enum'; import { EntityReference } from '../../../generated/entity/type'; import { getBreadcrumbsFromFqn } from '../../../utils/EntityUtils'; -import { getEntityIcon } from '../../../utils/TableUtils'; +import { getServiceIcon } from '../../../utils/TableUtils'; +import { SourceType } from '../../SearchedData/SearchedData.interface'; import './lineage-node-label.less'; interface LineageNodeLabelProps { @@ -41,12 +42,10 @@ const EntityLabel = ({ node }: Pick) => { return ( - -
- {getEntityIcon(node.type || '')} + +
+ {getServiceIcon(node as SourceType)}
- - { return (
- - {breadcrumbs.map((breadcrumb, index) => ( - - - {breadcrumb.name} - - {index !== breadcrumbs.length - 1 && ( - - {t('label.slash-symbol')} +
+ + {breadcrumbs.map((breadcrumb, index) => ( + + + {breadcrumb.name} - )} - - ))} - -
- + {index !== breadcrumbs.length - 1 && ( + + {t('label.slash-symbol')} + + )} + + ))} +
+
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx index d3d31b7ee37e..bbc4e4f1fc50 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx @@ -26,15 +26,13 @@ import { useTranslation } from 'react-i18next'; import { FQN_SEPARATOR_CHAR } from '../../../constants/char.constants'; import { PAGE_SIZE } from '../../../constants/constants'; import { EntityType, FqnPart } from '../../../enums/entity.enum'; +import { SearchIndex } from '../../../enums/search.enum'; import { EntityReference } from '../../../generated/entity/type'; import { SearchSourceAlias } from '../../../interface/search.interface'; import { searchData } from '../../../rest/miscAPI'; import { formatDataResponse } from '../../../utils/APIUtils'; import { getPartialNameFromTableFQN } from '../../../utils/CommonUtils'; -import { - getEntityNodeIcon, - getSearchIndexFromNodeType, -} from '../../../utils/EntityLineageUtils'; +import { getEntityNodeIcon } from '../../../utils/EntityLineageUtils'; import serviceUtilClassBase from '../../../utils/ServiceUtilClassBase'; import { showErrorToast } from '../../../utils/ToastUtils'; import { ExploreSearchIndex } from '../../Explore/ExplorePage.interface'; @@ -77,7 +75,7 @@ const NodeSuggestions: FC = ({ '', '', '', - getSearchIndexFromNodeType(entityType) + (entityType as ExploreSearchIndex) ?? SearchIndex.TABLE ); setData(formatDataResponse(data.data.hits.hits)); } catch (error) { @@ -98,8 +96,8 @@ const NodeSuggestions: FC = ({ debouncedOnSearch, ]); - const handleChange = (e: React.ChangeEvent<{ value: string }>): void => { - const searchText = e.target.value; + const handleChange = (value: string): void => { + const searchText = value; setSearchValue(searchText); debounceOnSearch(searchText); }; @@ -124,17 +122,10 @@ const NodeSuggestions: FC = ({ label: ( <>
{ - onSelectHandler?.({ - description: entity.description, - displayName: entity.displayName, - id: entity.id, - type: entity.entityType as string, - name: entity.name, - fullyQualifiedName: entity.fullyQualifiedName, - }); + onSelectHandler?.(entity as EntityReference); }}> {entity.serviceType} ({ searchData: jest.fn().mockImplementation(() => Promise.resolve()), @@ -50,9 +60,7 @@ describe('Test NodeSuggestions Component', () => { // 1st call on page load with empty search string and respective searchIndex expect(mockSearchData.mock.calls[0][0]).toBe(''); - expect(mockSearchData.mock.calls[0][6]).toEqual( - SearchIndex[value as keyof typeof SearchIndex] - ); + expect(mockSearchData.mock.calls[0][6]).toEqual(value); const suggestionNode = await screen.findByTestId('suggestion-node'); const searchInput = await screen.findByRole('combobox'); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less index f773c82ce55d..662251410390 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less @@ -21,8 +21,8 @@ } .lineage-expand-icon { - width: 12px; - height: 12px; + width: 16px; + height: 16px; color: inherit; } @@ -34,17 +34,31 @@ } } -.lineage-node-content { - margin: 1px; -} - .react-flow__node { box-shadow: none; } .lineage-node { border: 1px solid @lineage-border; - border-radius: 4px; + border-radius: 10px; + overflow: hidden; + .profiler-item { + width: 36px; + height: 36px; + margin-right: 4px; + border-radius: 4px; + line-height: 21px; + font-size: 14px; + &.green { + border: 1px solid @green-5; + } + &.amber { + border: 1px solid @yellow-4; + } + &.red { + border: 1px solid @red-5; + } + } } .react-flow__node-default, @@ -100,7 +114,20 @@ } .lineage-node-handle { border-color: @primary-color; + svg { + color: @primary-color; + } + } + .label-container { + background: @primary-color-hover; + } + .column-container { + background: @primary-color-hover; + border-top: 1px solid @border-color; } +} + +.custom-node-header-active { .label-container { background: @primary-color-hover; } @@ -116,21 +143,24 @@ .react-flow { .lineage-node-handle.react-flow__handle-left { - left: -10px; + left: -22px; } .lineage-node-handle.react-flow__handle-right { - right: -10px; + right: -22px; } } .react-flow .lineage-node-handle { - width: 20px; - min-width: 20px; - height: 20px; + width: 35px; + min-width: 35px; + height: 35px; border-radius: 50%; border-color: @lineage-border; background: @white; - top: 38px; // Need to show handles on top half + top: 43px; // Need to show handles on top half + svg { + color: @text-grey-muted; + } } .react-flow .lineage-column-node-handle { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less index 80595bb20f92..cb54f5daac1c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less @@ -13,7 +13,8 @@ @import (reference) url('../../../styles/variables.less'); -@lineage-sidebar-width: 100px; +@lineage-sidebar-width: 110px; +@lineage-breadcrumb-panel: 50px; .custom-react-flow { .react-flow__node-input.selectable.selected { @@ -36,6 +37,12 @@ .react-flow__handle { border-color: @primary-color; } + .lineage-node-handle { + border: 1px solid @primary-color; + svg { + color: @primary-color; + } + } } .custom-node-header-active { border-color: @primary-color; @@ -67,10 +74,6 @@ position: relative; } -.custom-edge-pipeline-button.ant-btn { - background-color: @body-bg-color; -} - .custom-node { .custom-node-header-active { .react-flow__handle { @@ -83,7 +86,7 @@ } .custom-control-search-box { - width: 400px; + width: 220px; height: 32px; } .ant-select.custom-control-search-box-edit-mode { @@ -151,7 +154,7 @@ } .entity-lineage.sidebar { - height: 110%; + height: 100%; background: @white; position: absolute; top: 0; @@ -177,7 +180,7 @@ left: @sidebar-width; position: fixed !important; width: calc(100vw - @sidebar-width); - height: calc(100vh - @navbar-height); + height: calc(100vh - (@navbar-height + @lineage-breadcrumb-panel)); } .lineage-node-remove-btn { @@ -186,3 +189,35 @@ right: -20px; cursor: pointer; } + +.react-flow__attribution { + display: none; +} + +.react-flow__controls-button { + width: 20px; + height: 20px; +} + +.custom-edge-pipeline-button { + background-color: @body-bg-color; + color: @primary-color; + &.green { + background: @green-2; + border: 1px solid @green-5; + color: @green-6; + } + &.amber { + background: @yellow-1; + border: 1px solid @yellow-4; + color: @yellow-4; + } + &.red { + background: @red-2; + border: 1px solid @red-5; + color: @red-5; + } + svg { + color: inherit; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/TableSummary/TableSummary.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/TableSummary/TableSummary.component.tsx index 755085315af3..e87fd55ccc07 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/TableSummary/TableSummary.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/TableSummary/TableSummary.component.tsx @@ -31,7 +31,7 @@ import { getLatestTableProfileByFqn, getTableDetailsByFQN, } from '../../../../rest/tableAPI'; -import { formTwoDigitNmber as formTwoDigitNumber } from '../../../../utils/CommonUtils'; +import { formTwoDigitNumber } from '../../../../utils/CommonUtils'; import { getFormattedEntityData, getSortedTagsWithHighlight, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx index fcec4499c510..f41fb5b89ca8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx @@ -61,10 +61,6 @@ import { DataProduct } from '../../../../generated/entity/domains/dataProduct'; import { Domain } from '../../../../generated/entity/domains/domain'; import { usePaging } from '../../../../hooks/paging/usePaging'; import { Aggregations } from '../../../../interface/search.interface'; -import { - QueryFieldInterface, - QueryFieldValueInterface, -} from '../../../../pages/ExplorePage/ExplorePage.interface'; import { getDataProductByName, removeAssetsFromDataProduct, @@ -86,6 +82,7 @@ import { } from '../../../../utils/EntityUtils'; import { getAggregations, + getQuickFilterQuery, getSelectedValuesFromQuickFilter, } from '../../../../utils/Explore.utils'; import { @@ -516,30 +513,7 @@ const AssetsTabs = forwardRef( }, []); const handleQuickFiltersChange = (data: ExploreQuickFilterField[]) => { - const must: QueryFieldInterface[] = []; - data.forEach((filter) => { - if (!isEmpty(filter.value)) { - const should: QueryFieldValueInterface[] = []; - if (filter.value) { - filter.value.forEach((filterValue) => { - const term: Record = {}; - term[filter.key] = filterValue.key; - should.push({ term }); - }); - } - - must.push({ - bool: { should }, - }); - } - }); - - const quickFilterQuery = isEmpty(must) - ? undefined - : { - query: { bool: { must } }, - }; - + const quickFilterQuery = getQuickFilterQuery(data); setQuickFilterQuery(quickFilterQuery); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx new file mode 100644 index 000000000000..53c50de4b5d5 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx @@ -0,0 +1,180 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Card } from 'antd'; +import classNames from 'classnames'; +import { debounce } from 'lodash'; +import Qs from 'qs'; +import React, { DragEvent, useCallback, useMemo, useRef } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useHistory, useLocation, useParams } from 'react-router-dom'; +import ReactFlow, { Background, Controls, ReactFlowProvider } from 'reactflow'; +import { + MAX_ZOOM_VALUE, + MIN_ZOOM_VALUE, +} from '../../constants/Lineage.constants'; +import { + customEdges, + dragHandle, + nodeTypes, + onNodeContextMenu, + onNodeMouseEnter, + onNodeMouseLeave, + onNodeMouseMove, +} from '../../utils/EntityLineageUtils'; +import { getEntityBreadcrumbs } from '../../utils/EntityUtils'; +import TitleBreadcrumb from '../common/TitleBreadcrumb/TitleBreadcrumb.component'; +import CustomControlsComponent from '../Entity/EntityLineage/CustomControls.component'; +import { useLineageProvider } from '../LineageProvider/LineageProvider'; +import { LineageProps } from './Lineage.interface'; + +const Lineage = ({ + deleted, + hasEditAccess, + entity, + entityType, +}: LineageProps) => { + const { t } = useTranslation(); + const history = useHistory(); + const reactFlowWrapper = useRef(null); + const location = useLocation(); + const { + nodes, + edges, + isEditMode, + onNodeClick, + onEdgeClick, + onNodeDrop, + onNodesChange, + onEdgesChange, + entityLineage, + onPaneClick, + onConnect, + onZoomUpdate, + onInitReactFlow, + } = useLineageProvider(); + const { fqn: entityFQN } = useParams<{ fqn: string }>(); + const queryParams = new URLSearchParams(location.search); + const isFullScreen = queryParams.get('fullscreen') === 'true'; + + const onFullScreenClick = useCallback(() => { + history.push({ + search: Qs.stringify({ fullscreen: true }), + }); + }, [entityFQN]); + + const onExitFullScreenViewClick = useCallback(() => { + history.push({ + search: '', + }); + }, [entityFQN]); + + const onDragOver = useCallback((event: DragEvent) => { + event.preventDefault(); + event.dataTransfer.dropEffect = 'move'; + }, []); + + const handleZoomLevel = debounce((value: number) => { + onZoomUpdate(value); + }, 150); + + const breadcrumbs = useMemo( + () => + entity + ? [ + ...getEntityBreadcrumbs(entity, entityType), + { + name: t('label.lineage'), + url: '', + activeTitle: true, + }, + ] + : [], + [entity] + ); + + return ( + +
+ {isFullScreen && ( + + )} + + + onNodeDrop( + _e, + reactFlowWrapper.current?.getBoundingClientRect() as DOMRect + ) + } + onEdgeClick={(_e, data) => { + onEdgeClick(data); + _e.stopPropagation(); + }} + onEdgesChange={onEdgesChange} + onInit={onInitReactFlow} + onMove={(_e, viewPort) => handleZoomLevel(viewPort.zoom)} + onNodeClick={(_e, node) => { + onNodeClick(node); + _e.stopPropagation(); + }} + onNodeContextMenu={onNodeContextMenu} + onNodeDrag={dragHandle} + onNodeDragStart={dragHandle} + onNodeDragStop={dragHandle} + onNodeMouseEnter={onNodeMouseEnter} + onNodeMouseLeave={onNodeMouseLeave} + onNodeMouseMove={onNodeMouseMove} + onNodesChange={onNodesChange} + onPaneClick={onPaneClick}> + {entityLineage && ( + + )} + + + + +
+ + ); +}; + +export default Lineage; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts new file mode 100644 index 000000000000..9ec07612dd0b --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityType } from '../../enums/entity.enum'; +import { EntityReference } from '../../generated/entity/type'; +import { ColumnLineage } from '../../generated/type/entityLineage'; +import { SourceType } from '../SearchedData/SearchedData.interface'; + +export interface LineageProps { + entityType: EntityType; + deleted?: boolean; + hasEditAccess: boolean; + isFullScreen?: boolean; + entity?: SourceType; +} + +export interface EntityLineageReponse { + entity: EntityReference; + nodes?: EntityReference[]; + edges?: EdgeDetails[]; +} + +export type LineageRequest = { + upstreamDepth?: number; + downstreamDepth?: number; + nodesPerLayer?: number; +}; + +export interface EdgeFromToData { + fqn: string; + id: string; + type: string; +} + +export interface EdgeDetails { + fromEntity: EdgeFromToData; + toEntity: EdgeFromToData; + pipeline?: EntityReference; + source?: string; + sqlQuery?: string; + columns?: ColumnLineage[]; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.test.tsx new file mode 100644 index 000000000000..69e2d69f6ed5 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.test.tsx @@ -0,0 +1,138 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { render, screen } from '@testing-library/react'; +import React from 'react'; +import { EntityType } from '../../enums/entity.enum'; +import { MOCK_EXPLORE_SEARCH_RESULTS } from '../Explore/Explore.mock'; +import Lineage from './Lineage.component'; +import { EntityLineageReponse } from './Lineage.interface'; + +let entityLineage: EntityLineageReponse | undefined = { + entity: { + name: 'fact_sale', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.fact_sale', + id: '5a1947bb-84eb-40de-a5c5-2b7b80c834c3', + type: 'table', + }, + nodes: [ + { + name: 'dim_location', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_location', + id: '30e9170c-0e07-4e55-bf93-2d2dfab3a36e', + type: 'table', + }, + { + name: 'dim_address_clean', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address_clean', + id: '6059959e-96c8-4b61-b905-fc5d88b33293', + type: 'table', + }, + ], + edges: [ + { + toEntity: { + fqn: 'sample_data.ecommerce_db.shopify.dim_location', + id: '30e9170c-0e07-4e55-bf93-2d2dfab3a36e', + type: 'table', + }, + fromEntity: { + fqn: 'sample_data.ecommerce_db.shopify.fact_sale', + id: '5a1947bb-84eb-40de-a5c5-2b7b80c834c3', + type: 'table', + }, + sqlQuery: '', + source: 'Manual', + }, + { + toEntity: { + fqn: 'mlflow_svc.eta_predictions', + id: 'b81f6bad-42f3-4216-8505-cf6f0c0a8897', + type: 'mlmodel', + }, + fromEntity: { + fqn: 'sample_data.ecommerce_db.shopify.fact_sale', + id: '5a1947bb-84eb-40de-a5c5-2b7b80c834c3', + type: 'table', + }, + }, + { + toEntity: { + fqn: 'sample_data.ecommerce_db.shopify.fact_sale', + id: '5a1947bb-84eb-40de-a5c5-2b7b80c834c3', + type: 'table', + }, + fromEntity: { + fqn: 'sample_data.ecommerce_db.shopify.dim_address_clean', + id: '6059959e-96c8-4b61-b905-fc5d88b33293', + type: 'table', + }, + sqlQuery: '', + source: 'Manual', + }, + ], +}; + +jest.mock('../LineageProvider/LineageProvider', () => ({ + useLineageProvider: jest.fn().mockImplementation(() => ({ + tracedNodes: [], + tracedColumns: [], + entityLineage: entityLineage, + })), +})); + +jest.mock('react-router-dom', () => ({ + useHistory: jest.fn().mockReturnValue({ push: jest.fn(), listen: jest.fn() }), + useLocation: jest.fn().mockReturnValue({ pathname: 'pathname' }), + useParams: jest.fn().mockReturnValue({ + fqn: 'fqn', + }), +})); + +jest.mock('../Entity/EntityLineage/CustomControls.component', () => { + return jest.fn().mockImplementation(() => { + return

Controls Component

; + }); +}); + +describe('Lineage', () => { + const mockProps = { + entity: MOCK_EXPLORE_SEARCH_RESULTS.hits.hits[0]._source, + deleted: false, + hasEditAccess: true, + entityType: EntityType.TABLE, + }; + + beforeEach(() => { + render(); + }); + + it('renders Lineage component', () => { + const customControlsComponent = screen.getByText('Controls Component'); + const lineageComponent = screen.getByTestId('lineage-container'); + + expect(customControlsComponent).toBeInTheDocument(); + expect(lineageComponent).toBeInTheDocument(); + }); + + it('does not render CustomControlsComponent when entityLineage is falsy', () => { + const mockPropsWithoutEntity = { + ...mockProps, + entity: undefined, + }; + entityLineage = undefined; + render(); + const customControlsComponent = screen.getByText('Controls Component'); + + expect(customControlsComponent).toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.interface.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.interface.tsx new file mode 100644 index 000000000000..4c86ff6ab9fc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.interface.tsx @@ -0,0 +1,91 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { LoadingState } from 'Models'; +import { DragEvent, ReactNode } from 'react'; +import { + Connection, + Edge, + EdgeChange, + Node, + NodeChange, + NodeProps, + ReactFlowInstance, +} from 'reactflow'; +import { PipelineStatus } from '../../generated/entity/data/pipeline'; +import { EntityReference } from '../../generated/entity/type'; +import { + EdgeTypeEnum, + LineageConfig, +} from '../Entity/EntityLineage/EntityLineage.interface'; +import { + EdgeDetails, + EntityLineageReponse, +} from '../Lineage/Lineage.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; + +export interface LineageProviderProps { + children: ReactNode; +} + +export type UpstreamDownstreamData = { + downstreamEdges: EdgeDetails[]; + upstreamEdges: EdgeDetails[]; + downstreamNodes: EntityReference[]; + upstreamNodes: EntityReference[]; +}; + +export interface LineageContextType { + reactFlowInstance?: ReactFlowInstance; + nodes: Node[]; + edges: Edge[]; + expandedNodes: string[]; + tracedNodes: string[]; + tracedColumns: string[]; + lineageConfig: LineageConfig; + zoomValue: number; + isDrawerOpen: boolean; + loading: boolean; + status: LoadingState; + isEditMode: boolean; + entityLineage: EntityLineageReponse; + selectedNode: SourceType; + upstreamDownstreamData: UpstreamDownstreamData; + selectedColumn: string; + expandAllColumns: boolean; + pipelineStatus: Record; + onInitReactFlow: (reactFlowInstance: ReactFlowInstance) => void; + onPaneClick: () => void; + onNodeClick: (node: Node) => void; + onEdgeClick: (edge: Edge) => void; + onColumnClick: (node: string) => void; + onLineageEditClick: () => void; + onZoomUpdate: (value: number) => void; + onLineageConfigUpdate: (config: any) => void; + onQueryFilterUpdate: (query: string) => void; + onDrawerClose: () => void; + onNodeDrop: (event: DragEvent, reactFlowBounds: DOMRect) => void; + onNodeCollapse: (node: Node | NodeProps, direction: EdgeTypeEnum) => void; + onNodesChange: (changes: NodeChange[]) => void; + onEdgesChange: (changes: EdgeChange[]) => void; + toggleColumnView: () => void; + loadChildNodesHandler: ( + node: EntityReference, + direction: EdgeTypeEnum + ) => Promise; + fetchLineageData: (entityFqn: string, lineageConfig: LineageConfig) => void; + fetchPipelineStatus: (pipelineFqn: string) => void; + removeNodeHandler: (node: Node | NodeProps) => void; + onColumnEdgeRemove: () => void; + onAddPipelineClick: () => void; + onConnect: (connection: Edge | Connection) => void; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.test.tsx new file mode 100644 index 000000000000..f95e9f771459 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.test.tsx @@ -0,0 +1,141 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, fireEvent, render, screen } from '@testing-library/react'; +import React from 'react'; +import { Edge } from 'reactflow'; +import { getLineageDataByFQN } from '../../rest/lineageAPI'; +import { EdgeTypeEnum } from '../Entity/EntityLineage/EntityLineage.interface'; +import LineageProvider, { useLineageProvider } from './LineageProvider'; + +const mockLocation = { + search: '', + pathname: '/lineage', +}; + +const DummyChildrenComponent = () => { + const { loadChildNodesHandler, onEdgeClick } = useLineageProvider(); + + const nodeData = { + name: 'table1', + type: 'table', + fullyQualifiedName: 'table1', + id: 'table1', + }; + + const MOCK_EDGE = { + id: 'test', + source: 'test', + target: 'test', + type: 'test', + data: { + edge: { + fromEntity: { + id: 'test', + type: 'test', + }, + toEntity: { + id: 'test', + type: 'test', + }, + }, + }, + }; + const handleButtonClick = () => { + // Trigger the loadChildNodesHandler method when the button is clicked + loadChildNodesHandler(nodeData, EdgeTypeEnum.DOWN_STREAM); + }; + + return ( +
+ + + +
+ ); +}; + +jest.mock('react-router-dom', () => ({ + useHistory: jest.fn().mockReturnValue({ push: jest.fn(), listen: jest.fn() }), + useLocation: jest.fn().mockImplementation(() => mockLocation), + useParams: jest.fn().mockReturnValue({ + fqn: 'table1', + }), +})); + +jest.mock('../Entity/EntityInfoDrawer/EdgeInfoDrawer.component', () => { + return jest.fn().mockImplementation(() => { + return

Edge Info Drawer

; + }); +}); + +jest.mock('../../rest/lineageAPI', () => ({ + getLineageDataByFQN: jest.fn(), +})); + +describe('LineageProvider', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('renders Lineage component and fetches data', async () => { + await act(async () => { + render( + +
Children
+ + ); + }); + + expect(getLineageDataByFQN).toHaveBeenCalled(); + }); + + it('should call loadChildNodesHandler', async () => { + await act(async () => { + render( + + + + ); + }); + + const loadButton = await screen.getByTestId('load-nodes'); + fireEvent.click(loadButton); + + expect(getLineageDataByFQN).toHaveBeenCalled(); + }); + + it('should show delete modal', async () => { + await act(async () => { + render( + + + + ); + }); + + const edgeClick = await screen.getByTestId('edge-click'); + fireEvent.click(edgeClick); + + const edgeDrawer = screen.getByText('Edge Info Drawer'); + + expect(edgeDrawer).toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx new file mode 100644 index 000000000000..9eee79c5918c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -0,0 +1,1045 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Modal } from 'antd'; +import { AxiosError } from 'axios'; +import { isEqual, isUndefined, uniqueId, uniqWith } from 'lodash'; +import { LoadingState } from 'Models'; +import React, { + createContext, + DragEvent, + useCallback, + useContext, + useEffect, + useMemo, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; +import { + Connection, + Edge, + getConnectedEdges, + Node, + NodeProps, + ReactFlowInstance, + useEdgesState, + useNodesState, +} from 'reactflow'; +import { + ELEMENT_DELETE_STATE, + ZOOM_VALUE, +} from '../../constants/Lineage.constants'; +import { mockDatasetData } from '../../constants/mockTourData.constants'; +import { + EntityLineageDirection, + EntityLineageNodeType, +} from '../../enums/entity.enum'; +import { AddLineage } from '../../generated/api/lineage/addLineage'; +import { EntityReference } from '../../generated/type/entityLineage'; +import { getLineageDataByFQN, updateLineageEdge } from '../../rest/lineageAPI'; +import { + addLineageHandler, + createEdges, + createNewEdge, + createNodes, + getAllTracedColumnEdge, + getAllTracedNodes, + getClassifiedEdge, + getConnectedNodesEdges, + getLayoutedElements, + getLineageEdge, + getLineageEdgeForAPI, + getLoadingStatusValue, + getModalBodyText, + getNewLineageConnectionDetails, + getUpdatedColumnsFromEdge, + getUpstreamDownstreamNodesEdges, + onLoad, + removeLineageHandler, +} from '../../utils/EntityLineageUtils'; + +import { useParams } from 'react-router-dom'; +import { PipelineStatus } from '../../generated/entity/data/pipeline'; +import { getPipelineStatus } from '../../rest/pipelineAPI'; +import { getEpochMillisForPastDays } from '../../utils/date-time/DateTimeUtils'; +import { getDecodedFqn } from '../../utils/StringsUtils'; +import SVGIcons from '../../utils/SvgUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; +import EdgeInfoDrawer from '../Entity/EntityInfoDrawer/EdgeInfoDrawer.component'; +import EntityInfoDrawer from '../Entity/EntityInfoDrawer/EntityInfoDrawer.component'; +import AddPipeLineModal from '../Entity/EntityLineage/AppPipelineModel/AddPipeLineModal'; +import { + EdgeData, + EdgeTypeEnum, + ElementLoadingState, + LineageConfig, +} from '../Entity/EntityLineage/EntityLineage.interface'; +import EntityLineageSidebar from '../Entity/EntityLineage/EntityLineageSidebar.component'; +import NodeSuggestions from '../Entity/EntityLineage/NodeSuggestions.component'; +import { + EdgeDetails, + EntityLineageReponse, +} from '../Lineage/Lineage.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; +import { useTourProvider } from '../TourProvider/TourProvider'; +import { + LineageContextType, + LineageProviderProps, + UpstreamDownstreamData, +} from './LineageProvider.interface'; + +export const LineageContext = createContext({} as LineageContextType); + +const LineageProvider = ({ children }: LineageProviderProps) => { + const { t } = useTranslation(); + const { fqn: entityFqn } = useParams<{ fqn: string }>(); + const decodedFqn = getDecodedFqn(entityFqn); + const { isTourOpen } = useTourProvider(); + const [reactFlowInstance, setReactFlowInstance] = + useState(); + const [isDrawerOpen, setIsDrawerOpen] = useState(false); + const [isEditMode, setIsEditMode] = useState(false); + const [selectedNode, setSelectedNode] = useState( + {} as SourceType + ); + const [selectedColumn, setSelectedColumn] = useState(''); + const [showAddEdgeModal, setShowAddEdgeModal] = useState(false); + const [expandedNodes, setExpandedNodes] = useState([]); + const [expandAllColumns, setExpandAllColumns] = useState(false); + const [selectedEdge, setSelectedEdge] = useState(); + const [entityLineage, setEntityLineage] = useState({ + nodes: [], + edges: [], + entity: {} as EntityReference, + }); + const [updatedEntityLineage, setUpdatedEntityLineage] = + useState(null); + const [showDeleteModal, setShowDeleteModal] = useState(false); + const [upstreamDownstreamData, setUpstreamDownstreamData] = + useState({ + downstreamEdges: [], + upstreamEdges: [], + downstreamNodes: [], + upstreamNodes: [], + }); + const [deletionState, setDeletionState] = useState<{ + loading: boolean; + status: ElementLoadingState; + }>(ELEMENT_DELETE_STATE); + const [nodes, setNodes, onNodesChange] = useNodesState([]); + const [edges, setEdges, onEdgesChange] = useEdgesState([]); + const [loading, setLoading] = useState(true); + const [zoomValue, setZoomValue] = useState(ZOOM_VALUE); + const [tracedNodes, setTracedNodes] = useState([]); + const [tracedColumns, setTracedColumns] = useState([]); + const [status, setStatus] = useState('initial'); + const [newAddedNode, setNewAddedNode] = useState({} as Node); + const [lineageConfig, setLineageConfig] = useState({ + upstreamDepth: 3, + downstreamDepth: 3, + nodesPerLayer: 50, + }); + const [queryFilter, setQueryFilter] = useState(''); + const [pipelineStatus, setPipelineStatus] = useState< + Record + >({}); + + const fetchLineageData = async (fqn: string, config?: LineageConfig) => { + if (isTourOpen) { + setEntityLineage(mockDatasetData.entityLineage); + } else { + setLoading(true); + try { + const res = await getLineageDataByFQN(fqn, config, queryFilter); + if (res) { + const allNodes = uniqWith( + [...(res.nodes ?? []), res.entity], + isEqual + ); + setEntityLineage({ + ...res, + nodes: allNodes, + }); + } else { + showErrorToast( + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } finally { + setLoading(false); + } + } + }; + + const loadChildNodesHandler = useCallback( + async (node: EntityReference, direction: EdgeTypeEnum) => { + try { + const res = await getLineageDataByFQN( + node.fullyQualifiedName ?? '', + { + upstreamDepth: direction === EdgeTypeEnum.UP_STREAM ? 1 : 0, + downstreamDepth: direction === EdgeTypeEnum.DOWN_STREAM ? 1 : 0, + nodesPerLayer: lineageConfig.nodesPerLayer, + }, // load only one level of child nodes + queryFilter + ); + + const allNodes = uniqWith( + [...(entityLineage?.nodes ?? []), ...(res.nodes ?? []), res.entity], + isEqual + ); + const allEdges = uniqWith( + [...(entityLineage?.edges ?? []), ...(res.edges ?? [])], + isEqual + ); + + setEntityLineage((prev) => { + return { + ...prev, + nodes: allNodes, + edges: allEdges, + }; + }); + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } + }, + [nodes, edges, lineageConfig, entityLineage, setEntityLineage, queryFilter] + ); + + const fetchPipelineStatus = useCallback(async (pipelineFQN: string) => { + try { + const currentTime = Date.now(); + // past 1 day + const startDay = getEpochMillisForPastDays(1); + const response = await getPipelineStatus(pipelineFQN, { + startTs: startDay, + endTs: currentTime, + }); + setPipelineStatus((prev) => { + return { + ...prev, + [pipelineFQN]: response.data[0], + }; + }); + } catch (error) { + showErrorToast( + error as AxiosError, + t('message.fetch-pipeline-status-error') + ); + } + }, []); + + const handleLineageTracing = useCallback( + (selectedNode: Node) => { + const { normalEdge } = getClassifiedEdge(edges); + const incomingNode = getAllTracedNodes( + selectedNode, + nodes, + normalEdge, + [], + true + ); + const outgoingNode = getAllTracedNodes( + selectedNode, + nodes, + normalEdge, + [], + false + ); + const incomerIds = incomingNode.map((incomer) => incomer.id); + const outgoerIds = outgoingNode.map((outGoer) => outGoer.id); + const connectedNodeIds = [...outgoerIds, ...incomerIds, selectedNode.id]; + setTracedNodes(connectedNodeIds); + setTracedColumns([]); + }, + [nodes, edges] + ); + + const onColumnClick = useCallback( + (column: string) => { + setSelectedColumn(column); + const { columnEdge } = getClassifiedEdge(edges); + const { connectedColumnEdges } = getAllTracedColumnEdge( + column, + columnEdge + ); + + setTracedColumns(connectedColumnEdges); + }, + [nodes, edges] + ); + + const removeEdgeHandler = async ( + edge: Edge, + confirmDelete: boolean + ): Promise => { + if (!confirmDelete || !entityLineage) { + return; + } + + const { data } = edge; + + const edgeData: EdgeData = { + fromEntity: data.edge.fromEntity.type, + fromId: data.edge.fromEntity.id, + toEntity: data.edge.toEntity.type, + toId: data.edge.toEntity.id, + }; + + await removeLineageHandler(edgeData); + + const filteredEdges = (entityLineage.edges ?? []).filter( + (item) => + !( + item.fromEntity.id === edgeData.fromId && + item.toEntity.id === edgeData.toId + ) + ); + + setEdges((prev) => { + return prev.filter( + (item) => + !(item.source === edgeData.fromId && item.target === edgeData.toId) + ); + }); + + // On deleting of edge storing the result in a separate state. + // This state variable is applied to main entityLineage state variable when the edit operation is + // closed. This is done to perform the redrawing of the lineage graph on exit of edit mode. + setUpdatedEntityLineage(() => { + return { + ...entityLineage, + edges: filteredEdges, + }; + }); + }; + + const removeColumnEdge = async (edge: Edge, confirmDelete: boolean) => { + if (!confirmDelete || !entityLineage) { + return; + } + + const { data } = edge; + const selectedEdge = createNewEdge(edge); + const updatedCols = selectedEdge.edge.lineageDetails?.columnsLineage ?? []; + await addLineageHandler(selectedEdge); + + const updatedEdgeWithColumns = (entityLineage.edges ?? []).map((obj) => { + if ( + obj.fromEntity.id === data.edge.fromEntity.id && + obj.toEntity.id === data.edge.toEntity.id + ) { + return { + ...obj, + columns: updatedCols, + }; + } + + return obj; + }); + + setEntityLineage((prev) => { + return { + ...prev, + edges: updatedEdgeWithColumns, + }; + }); + + setShowDeleteModal(false); + }; + + const removeNodeHandler = useCallback( + (node: Node | NodeProps) => { + if (!entityLineage) { + return; + } + // Get edges connected to selected node + const edgesToRemove = getConnectedEdges([node as Node], edges); + edgesToRemove.forEach((edge) => { + removeEdgeHandler(edge, true); + }); + + setEntityLineage((prev) => { + return { + ...prev, + nodes: (prev.nodes ?? []).filter( + (previousNode) => previousNode.id !== node.id + ), + }; + }); + + setNewAddedNode({} as Node); + }, + [nodes, entityLineage] + ); + + const onNodeDrop = (event: DragEvent, reactFlowBounds: DOMRect) => { + event.preventDefault(); + const entityType = event.dataTransfer.getData('application/reactflow'); + if (entityType) { + const position = reactFlowInstance?.project({ + x: event.clientX - (reactFlowBounds?.left ?? 0), + y: event.clientY - (reactFlowBounds?.top ?? 0), + }); + const nodeId = uniqueId(); + const newNode = { + id: nodeId, + nodeType: EntityLineageNodeType.DEFAULT, + position, + className: '', + connectable: false, + selectable: false, + type: EntityLineageNodeType.DEFAULT, + data: { + label: ( +
+
+ ), + isEditMode, + isNewNode: true, + }, + }; + setNodes([...nodes, newNode as Node]); + setNewAddedNode(newNode as Node); + } + }; + + const onQueryFilterUpdate = useCallback((query: string) => { + setQueryFilter(query); + }, []); + + const onNodeClick = useCallback( + (node: Node) => { + if (node) { + setSelectedEdge(undefined); + setSelectedNode(node.data.node as SourceType); + setIsDrawerOpen(true); + handleLineageTracing(node); + } + }, + [handleLineageTracing] + ); + + const onPaneClick = useCallback(() => { + setIsDrawerOpen(false); + setTracedNodes([]); + setTracedColumns([]); + setSelectedNode({} as SourceType); + }, []); + + const onEdgeClick = useCallback((edge: Edge) => { + setSelectedEdge(edge); + setSelectedNode({} as SourceType); + setIsDrawerOpen(true); + }, []); + + const onLineageEditClick = useCallback(() => { + setIsEditMode((pre) => !pre); + setSelectedNode({} as SourceType); + setIsDrawerOpen(false); + }, []); + + const onInitReactFlow = (reactFlowInstance: ReactFlowInstance) => { + setTimeout(() => { + onLoad(reactFlowInstance); + }, 500); + + setReactFlowInstance(reactFlowInstance); + }; + + const onLineageConfigUpdate = useCallback((config) => { + setLineageConfig(config); + }, []); + + const onDrawerClose = useCallback(() => { + setIsDrawerOpen(false); + }, []); + + const onZoomUpdate = useCallback((value) => { + setZoomValue(value); + }, []); + + const toggleColumnView = useCallback(() => { + const updatedVal = !expandAllColumns; + setExpandAllColumns(updatedVal); + setNodes((prevNodes) => { + const updatedNode = prevNodes.map((node) => { + const nodeId = node.data.node.id; + + // Update the expandedNodes state based on the toggle value + if (updatedVal && !expandedNodes.includes(nodeId)) { + setExpandedNodes((prevExpandedNodes) => [ + ...prevExpandedNodes, + nodeId, + ]); + } else if (!updatedVal) { + setExpandedNodes((prevExpandedNodes) => + prevExpandedNodes.filter((id) => id !== nodeId) + ); + } + + return node; + }); + + const { edge, node } = getLayoutedElements( + { + node: updatedNode, + edge: edges, + }, + EntityLineageDirection.LEFT_RIGHT, + updatedVal + ); + + setEdges(edge); + + return node; + }); + }, [expandAllColumns, expandedNodes, edges]); + + const onRemove = useCallback(async () => { + try { + setDeletionState({ ...ELEMENT_DELETE_STATE, loading: true }); + + if (selectedEdge?.data?.isColumnLineage) { + await removeColumnEdge(selectedEdge, true); + } else { + await removeEdgeHandler(selectedEdge as Edge, true); + } + + setShowDeleteModal(false); + } catch (err) { + showErrorToast(err as AxiosError); + } finally { + setDeletionState((pre) => ({ + ...pre, + status: 'initial', + loading: false, + })); + } + }, [selectedEdge, setShowDeleteModal]); + + const onConnect = useCallback( + (params: Edge | Connection) => { + const { target, source, sourceHandle, targetHandle } = params; + + if (target === source) { + return; + } + + const columnConnection = + source !== sourceHandle && target !== targetHandle; + + setStatus('waiting'); + setLoading(true); + + const targetNode = nodes?.find((n) => target === n.id); + const sourceNode = nodes?.find((n) => source === n.id); + + if (!isUndefined(sourceNode) && !isUndefined(targetNode)) { + const currentEdge = (entityLineage.edges ?? []).find( + (edge) => edge.fromEntity.id === source && edge.toEntity.id === target + ); + + const newEdgeWithFqn = getLineageEdge( + sourceNode.data.node, + targetNode.data.node + ); + + const newEdgeWithoutFqn = getLineageEdgeForAPI( + sourceNode.data.node, + targetNode.data.node + ); + + if (columnConnection && currentEdge) { + const updatedColumns = getUpdatedColumnsFromEdge(params, currentEdge); + if (newEdgeWithoutFqn.edge.lineageDetails) { + newEdgeWithoutFqn.edge.lineageDetails.columnsLineage = + updatedColumns; + } + currentEdge.columns = updatedColumns; // update current edge with new columns + } + + addLineageHandler(newEdgeWithoutFqn) + .then(() => { + if (!entityLineage) { + return; + } + setStatus('success'); + setLoading(false); + + const allNodes = [ + ...(entityLineage.nodes ?? []), + sourceNode?.data.node as EntityReference, + targetNode?.data.node as EntityReference, + ]; + + const allEdges = isUndefined(currentEdge) + ? [...(entityLineage.edges ?? []), newEdgeWithFqn.edge] + : entityLineage.edges ?? []; + + setEntityLineage((pre) => { + const newData = { + ...pre, + nodes: uniqWith([pre.entity, ...allNodes], isEqual), + edges: uniqWith(allEdges, isEqual), + }; + + return newData; + }); + + setNewAddedNode({} as Node); + }) + .catch((err) => { + showErrorToast(err); + }) + .finally(() => { + setStatus('initial'); + setLoading(false); + }); + } + }, + [selectedNode, entityLineage, nodes, edges] + ); + + const onAddPipelineClick = useCallback(() => { + setShowAddEdgeModal(true); + }, []); + + const handleModalCancel = useCallback(() => { + setShowAddEdgeModal(false); + setSelectedEdge({} as Edge); + }, []); + + const onEntitySelect = (selectedEntity: EntityReference, nodeId: string) => { + const isExistingNode = nodes.some( + (n) => + n.data.node.fullyQualifiedName === selectedEntity.fullyQualifiedName + ); + if (isExistingNode) { + setNodes((es) => + es + .map((n) => + n.id.includes(nodeId) + ? { + ...n, + selectable: true, + className: `${n.className} selected`, + } + : n + ) + .filter((es) => es.id !== nodeId) + ); + setNewAddedNode({} as Node); + } else { + setNodes((es) => { + return es.map((el) => { + if (el.id === nodeId) { + return { + ...el, + connectable: true, + selectable: true, + id: selectedEntity.id, + data: { + saved: false, + node: selectedEntity, + }, + }; + } else { + return el; + } + }); + }); + } + }; + + const onAddPipelineModalSave = useCallback( + async (pipelineData?: EntityReference) => { + if (!selectedEdge || !entityLineage) { + return; + } + + setStatus('waiting'); + setLoading(true); + + const { source, target } = selectedEdge; + const existingEdge = (entityLineage.edges ?? []).find( + (ed) => ed.fromEntity.id === source && ed.toEntity.id === target + ); + + let edgeIndex = -1; + if (existingEdge) { + edgeIndex = (entityLineage.edges ?? []).indexOf(existingEdge); + + if (pipelineData) { + existingEdge.pipeline = pipelineData; + } + } + + const { newEdge } = getNewLineageConnectionDetails( + selectedEdge, + pipelineData + ); + + try { + await addLineageHandler(newEdge); + + setStatus('success'); + setLoading(false); + + setEntityLineage((pre) => { + if (!selectedEdge.data || !pre) { + return pre; + } + + const newEdges = [...(pre.edges ?? [])]; + + if (newEdges[edgeIndex]) { + newEdges[edgeIndex] = existingEdge as EdgeDetails; + } + + return { + ...pre, + edges: newEdges, + }; + }); + } catch (error) { + setLoading(false); + } finally { + setStatus('initial'); + handleModalCancel(); + } + }, + [selectedEdge, entityLineage] + ); + + const onEdgeDetailsUpdate = useCallback( + async (updatedEdgeDetails: AddLineage) => { + const { description, sqlQuery } = + updatedEdgeDetails.edge.lineageDetails ?? {}; + + try { + await updateLineageEdge(updatedEdgeDetails); + const updatedEdges = (entityLineage.edges ?? []).map((edge) => { + if ( + edge.fromEntity.id === updatedEdgeDetails.edge.fromEntity.id && + edge.toEntity.id === updatedEdgeDetails.edge.toEntity.id + ) { + return { + ...edge, + description, + sqlQuery, + }; + } + + return edge; + }); + setEntityLineage((prev) => { + return { + ...prev, + edges: updatedEdges, + }; + }); + } catch (err) { + showErrorToast(err as AxiosError); + } + }, + [edges, entityLineage, selectedEdge] + ); + + const onColumnEdgeRemove = useCallback(() => { + setShowDeleteModal(true); + }, []); + + const onNodeCollapse = useCallback( + (node: Node | NodeProps, direction: EdgeTypeEnum) => { + const { nodeFqn, edges: connectedEdges } = getConnectedNodesEdges( + node as Node, + nodes, + edges, + direction + ); + + const updatedNodes = (entityLineage.nodes ?? []).filter( + (item) => !nodeFqn.includes(item.fullyQualifiedName ?? '') + ); + const updatedEdges = (entityLineage.edges ?? []).filter((val) => { + return !connectedEdges.some( + (connectedEdge) => connectedEdge.data.edge === val + ); + }); + + setEntityLineage((pre) => { + return { + ...pre, + nodes: updatedNodes, + edges: updatedEdges, + }; + }); + }, + [nodes, edges, entityLineage] + ); + + const redrawLineage = useCallback( + (lineageData: EntityLineageReponse) => { + const allNodes = uniqWith( + [...(lineageData.nodes ?? []), lineageData.entity], + isEqual + ); + const updatedNodes = createNodes( + allNodes, + lineageData.edges ?? [], + entityFqn + ); + const updatedEdges = createEdges(allNodes, lineageData.edges ?? []); + setNodes(updatedNodes); + setEdges(updatedEdges); + + // Get upstream downstream nodes and edges data + const data = getUpstreamDownstreamNodesEdges( + lineageData.edges ?? [], + lineageData.nodes ?? [], + decodedFqn + ); + setUpstreamDownstreamData(data); + }, + [decodedFqn] + ); + + useEffect(() => { + if (decodedFqn) { + fetchLineageData(decodedFqn, lineageConfig); + } + }, [lineageConfig, decodedFqn, queryFilter]); + + useEffect(() => { + if (!loading) { + redrawLineage(entityLineage); + } + }, [entityLineage, loading]); + + useEffect(() => { + if (!isEditMode && updatedEntityLineage !== null) { + // On exit of edit mode, use updatedEntityLineage and update data. + const { downstreamEdges, upstreamEdges } = + getUpstreamDownstreamNodesEdges( + updatedEntityLineage.edges ?? [], + updatedEntityLineage.nodes ?? [], + decodedFqn + ); + + const updatedNodes = + updatedEntityLineage.nodes?.filter( + (n) => + !isUndefined( + downstreamEdges?.find((d) => d.toEntity.id === n.id) + ) || + !isUndefined(upstreamEdges?.find((u) => u.fromEntity.id === n.id)) + ) ?? []; + + setEntityLineage({ + ...updatedEntityLineage, + nodes: updatedNodes as EntityReference[], + }); + } + }, [isEditMode, updatedEntityLineage, decodedFqn]); + + useEffect(() => { + if (isEditMode) { + setUpdatedEntityLineage(null); + } + }, [isEditMode]); + + const activityFeedContextValues = useMemo(() => { + return { + isDrawerOpen, + loading, + isEditMode, + nodes, + edges, + reactFlowInstance, + entityLineage, + lineageConfig, + selectedNode, + selectedColumn, + zoomValue, + status, + expandedNodes, + tracedNodes, + tracedColumns, + expandAllColumns, + pipelineStatus, + upstreamDownstreamData, + onInitReactFlow, + onPaneClick, + onConnect, + onNodeDrop, + onNodeCollapse, + onColumnClick, + onNodesChange, + onEdgesChange, + onQueryFilterUpdate, + onZoomUpdate, + onDrawerClose, + toggleColumnView, + loadChildNodesHandler, + fetchLineageData, + fetchPipelineStatus, + removeNodeHandler, + onNodeClick, + onEdgeClick, + onColumnEdgeRemove, + onLineageConfigUpdate, + onLineageEditClick, + onAddPipelineClick, + }; + }, [ + isDrawerOpen, + loading, + isEditMode, + nodes, + edges, + entityLineage, + reactFlowInstance, + lineageConfig, + selectedNode, + selectedColumn, + zoomValue, + status, + expandedNodes, + tracedNodes, + tracedColumns, + expandAllColumns, + pipelineStatus, + upstreamDownstreamData, + onInitReactFlow, + onPaneClick, + onConnect, + onNodeDrop, + onNodeCollapse, + onColumnClick, + onQueryFilterUpdate, + onNodesChange, + onEdgesChange, + onZoomUpdate, + onDrawerClose, + loadChildNodesHandler, + fetchLineageData, + fetchPipelineStatus, + toggleColumnView, + removeNodeHandler, + onNodeClick, + onEdgeClick, + onColumnEdgeRemove, + onLineageConfigUpdate, + onLineageEditClick, + onAddPipelineClick, + ]); + + return ( + + {children} + + + {isDrawerOpen && + !isEditMode && + (selectedEdge ? ( + { + setIsDrawerOpen(false); + setSelectedEdge(undefined); + }} + onEdgeDetailsUpdate={onEdgeDetailsUpdate} + /> + ) : ( + setIsDrawerOpen(false)} + /> + ))} + + {showDeleteModal && ( + { + setShowDeleteModal(false); + }} + onOk={onRemove}> + {getModalBodyText(selectedEdge as Edge)} + + )} + {showAddEdgeModal && ( + { + setShowDeleteModal(true); + setShowAddEdgeModal(false); + }} + onSave={onAddPipelineModalSave} + /> + )} + + ); +}; + +export const useLineageProvider = () => useContext(LineageContext); + +export default LineageProvider; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.test.tsx index 63d8e2e6f51f..d2c1b487e4e4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.test.tsx @@ -206,12 +206,9 @@ jest.mock('../common/RichTextEditor/RichTextEditorPreviewer', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviewer

); }); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest.fn().mockReturnValue(

EntityLineage.component

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockReturnValue(

EntityLineage.component

); +}); jest.mock('./MlModelFeaturesList', () => { return jest.fn().mockReturnValue(

MlModelFeaturesList

); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.tsx index 74447ef1ebba..a67a36477a3b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.tsx @@ -24,7 +24,6 @@ import { ActivityFeedTab } from '../../components/ActivityFeed/ActivityFeedTab/A import { withActivityFeed } from '../../components/AppRouter/withActivityFeed'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import TabsLabel from '../../components/TabsLabel/TabsLabel.component'; @@ -48,8 +47,11 @@ import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThr import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import { CustomPropertyTable } from '../common/CustomPropertyTable/CustomPropertyTable'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; import { usePermissionProvider } from '../PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../PermissionProvider/PermissionProvider.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; import { MlModelDetailProp } from './MlModelDetail.interface'; import MlModelFeaturesList from './MlModelFeaturesList'; @@ -470,12 +472,14 @@ const MlModelDetail: FC = ({ label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.interface.ts new file mode 100644 index 000000000000..462ce9ce2d01 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.interface.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export type ModalWithQueryEditorProps = { + header: string; + value: string; + onSave?: (text: string) => void; + onCancel?: () => void; + visible: boolean; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.test.tsx new file mode 100644 index 000000000000..2a9d4ead22bc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.test.tsx @@ -0,0 +1,77 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, fireEvent, render } from '@testing-library/react'; +import React from 'react'; +import { ModalWithQueryEditor } from './ModalWithQueryEditor'; + +describe('ModalWithQueryEditor', () => { + const onSaveMock = jest.fn(); + const onCancelMock = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('renders the modal with the correct header', () => { + const header = 'Test Header'; + const { getByTestId } = render( + + ); + const headerElement = getByTestId('header'); + + expect(headerElement).toBeInTheDocument(); + expect(headerElement).toHaveTextContent(header); + }); + + it('calls onSave with the correct value when save button is clicked', async () => { + const value = 'Test Query'; + const { getByTestId } = render( + + ); + const saveButton = getByTestId('save'); + + await act(async () => { + fireEvent.click(saveButton); + }); + + expect(onSaveMock).toHaveBeenCalledWith(value); + }); + + it('calls onCancel when cancel button is clicked', () => { + const { getByTestId } = render( + + ); + const cancelButton = getByTestId('cancel'); + fireEvent.click(cancelButton); + + expect(onCancelMock).toHaveBeenCalled(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.tsx new file mode 100644 index 000000000000..803f6b42044c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.tsx @@ -0,0 +1,112 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Form, Modal, Typography } from 'antd'; +import { FormProps, useForm } from 'antd/lib/form/Form'; +import { AxiosError } from 'axios'; +import React, { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { CSMode } from '../../../enums/codemirror.enum'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import Loader from '../../Loader/Loader'; +import SchemaEditor from '../../SchemaEditor/SchemaEditor'; +import { ModalWithQueryEditorProps } from './ModalWithQueryEditor.interface'; + +export const ModalWithQueryEditor = ({ + header, + value, + onSave, + onCancel, + visible, +}: ModalWithQueryEditorProps) => { + const { t } = useTranslation(); + const [form] = useForm(); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + + const onFinish: FormProps['onFinish'] = async (value) => { + setIsSaving(true); + try { + await onSave?.(value.query ?? ''); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsSaving(false); + } + }; + + useEffect(() => { + if (visible) { + form.setFieldsValue({ query: value }); + setIsLoading(false); + } + }, [form, visible]); + + return ( + + {t('label.cancel')} + , + , + ]} + maskClosable={false} + open={visible} + title={{header}} + width="90%" + onCancel={onCancel}> + {isLoading ? ( + + ) : ( +
+ + + +
+ )} + + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx index 6aeeaf1b263d..d83c4b60e5f9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx @@ -26,7 +26,6 @@ import { ActivityFeedTab } from '../../components/ActivityFeed/ActivityFeedTab/A import { CustomPropertyTable } from '../../components/common/CustomPropertyTable/CustomPropertyTable'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import ExecutionsTab from '../../components/Execution/Execution.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; @@ -72,9 +71,12 @@ import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThr import { withActivityFeed } from '../AppRouter/withActivityFeed'; import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; import { ModalWithMarkdownEditor } from '../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor'; import { usePermissionProvider } from '../PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../PermissionProvider/PermissionProvider.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; import './pipeline-details.style.less'; import { PipeLineDetailsProp } from './PipelineDetails.interface'; @@ -654,12 +656,14 @@ const PipelineDetails = ({ label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx index 97817294ddcc..759d7540cf5f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx @@ -143,7 +143,7 @@ jest.mock('../FeedEditor/FeedEditor', () => { return jest.fn().mockReturnValue(

FeedEditor

); }); -jest.mock('../Entity/EntityLineage/EntityLineage.component', () => { +jest.mock('../Lineage/Lineage.component', () => { return jest .fn() .mockReturnValue(

Lineage

); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.test.tsx index 84eacc6a6dda..a1d20f1a009b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.test.tsx @@ -59,7 +59,7 @@ jest.mock('./Component/ColumnProfileTable', () => { jest.mock('../../utils/CommonUtils', () => ({ formatNumberWithComma: jest.fn(), - formTwoDigitNmber: jest.fn(), + formTwoDigitNumber: jest.fn(), getStatisticsDisplayValue: jest.fn(), getEntityDeleteMessage: jest.fn(), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx index 0fa1fdb1fcf9..d835f7459ca8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx @@ -64,15 +64,11 @@ const TasksDAGViewProps = { jest.mock('../../utils/EntityLineageUtils', () => ({ dragHandle: jest.fn(), - getDeletedLineagePlaceholder: jest - .fn() - .mockReturnValue(

Task data is not available for deleted entities.

), getHeaderLabel: jest.fn().mockReturnValue(

Header label

), getLayoutedElements: jest.fn().mockImplementation(() => ({ node: mockNodes, edge: mockEdges, })), - getLineageData: jest.fn().mockReturnValue([]), getModalBodyText: jest.fn(), onLoad: jest.fn(), onNodeContextMenu: jest.fn(), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.component.tsx index b2a2c0601c54..b4105d7cbc61 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.component.tsx @@ -24,7 +24,6 @@ import DescriptionV1 from '../../components/common/EntityDescription/Description import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import QueryViewer from '../../components/common/QueryViewer/QueryViewer.component'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import SampleDataWithMessages from '../../components/SampleDataWithMessages/SampleDataWithMessages'; @@ -51,6 +50,9 @@ import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThr import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import { CustomPropertyTable } from '../common/CustomPropertyTable/CustomPropertyTable'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; +import { SourceType } from '../SearchedData/SearchedData.interface'; import { TopicDetailsProps } from './TopicDetails.interface'; import TopicSchemaFields from './TopicSchema/TopicSchema'; @@ -392,12 +394,14 @@ const TopicDetails: React.FC = ({ label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.test.tsx index d717b53f1eb6..35464135daef 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.test.tsx @@ -73,12 +73,9 @@ jest.mock('react-router-dom', () => ({ useParams: jest.fn().mockImplementation(() => mockParams), })); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest.fn().mockReturnValue(

EntityLineage.component

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockReturnValue(

EntityLineage.component

); +}); jest.mock('../common/EntityDescription/Description', () => { return jest.fn().mockReturnValue(

Description Component

); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TriggerReIndexing/ElasticSearchReIndexModal.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TriggerReIndexing/ElasticSearchReIndexModal.component.tsx deleted file mode 100644 index 4c0a64ff73fd..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/TriggerReIndexing/ElasticSearchReIndexModal.component.tsx +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2022 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Form, Input, Modal, Select, TreeSelect } from 'antd'; -import React from 'react'; -import { useTranslation } from 'react-i18next'; -import { - ELASTIC_SEARCH_INITIAL_VALUES, - ENTITY_TREE_OPTIONS, - RECREATE_INDEX_OPTIONS, - RE_INDEX_LANG_OPTIONS, -} from '../../constants/elasticsearch.constant'; -import { CreateEventPublisherJob } from '../../generated/api/createEventPublisherJob'; - -interface ReIndexAllModalInterface { - visible: boolean; - onCancel: () => void; - onSave?: (data: CreateEventPublisherJob) => void; - confirmLoading: boolean; -} - -const ReIndexAllModal = ({ - visible, - onCancel, - onSave, - confirmLoading, -}: ReIndexAllModalInterface) => { - const { t } = useTranslation(); - - return ( - -
- - - - -