From f880e28b0cbdd9d560e6c604890051a2e3211428 Mon Sep 17 00:00:00 2001 From: 07Himank Date: Thu, 30 Nov 2023 11:52:53 +0530 Subject: [PATCH 01/44] working on lineage es integration --- .../service/jdbi3/LineageRepository.java | 58 +++++++++++++ .../resources/search/SearchResource.java | 26 ++++++ .../service/search/SearchClient.java | 6 ++ .../service/search/SearchRepository.java | 4 + .../elasticsearch/ElasticSearchClient.java | 81 +++++++++++++++++++ .../search/indexes/ContainerIndex.java | 1 + .../indexes/DashboardDataModelIndex.java | 1 + .../search/indexes/DashboardIndex.java | 1 + .../service/search/indexes/MlModelIndex.java | 1 + .../service/search/indexes/PipelineIndex.java | 1 + .../search/indexes/SearchEntityIndex.java | 1 + .../service/search/indexes/SearchIndex.java | 66 +++++++++++++++ .../service/search/indexes/TableIndex.java | 1 + .../service/search/indexes/TopicIndex.java | 1 + .../search/opensearch/OpenSearchClient.java | 20 +++++ .../en/container_index_mapping.json | 3 + .../dashboard_data_model_index_mapping.json | 3 + .../en/dashboard_index_mapping.json | 3 + .../en/mlmodel_index_mapping.json | 3 + .../en/pipeline_index_mapping.json | 3 + .../en/search_entity_index_mapping.json | 3 + .../elasticsearch/en/table_index_mapping.json | 3 + .../elasticsearch/en/topic_index_mapping.json | 3 + .../jp/container_index_mapping.json | 3 + .../dashboard_data_model_index_mapping.json | 3 + .../jp/dashboard_index_mapping.json | 3 + .../jp/mlmodel_index_mapping.json | 3 + .../jp/pipeline_index_mapping.json | 3 + .../jp/search_entity_index_mapping.json | 3 + .../elasticsearch/jp/table_index_mapping.json | 3 + .../elasticsearch/jp/topic_index_mapping.json | 3 + .../zh/container_index_mapping.json | 3 + .../dashboard_data_model_index_mapping.json | 3 + .../zh/dashboard_index_mapping.json | 3 + .../zh/mlmodel_index_mapping.json | 3 + .../zh/pipeline_index_mapping.json | 3 + .../zh/search_entity_index_mapping.json | 3 + .../elasticsearch/zh/table_index_mapping.json | 3 + .../elasticsearch/zh/topic_index_mapping.json | 3 + 39 files changed, 341 insertions(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index 369fb799d9d7..bb482b0f83c9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -14,10 +14,15 @@ package org.openmetadata.service.jdbi3; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.ColumnsEntityInterface; import org.openmetadata.schema.api.lineage.AddLineage; import org.openmetadata.schema.entity.data.Table; @@ -30,6 +35,8 @@ import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.search.SearchClient; +import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @@ -37,6 +44,8 @@ public class LineageRepository { private final CollectionDAO dao; + public SearchClient searchClient = Entity.getSearchRepository().getSearchClient(); + public LineageRepository() { this.dao = Entity.getCollectionDAO(); Entity.setLineageRepository(this); @@ -79,6 +88,55 @@ public void addLineage(AddLineage addLineage) { // Finally, add lineage relationship dao.relationshipDAO() .insert(from.getId(), to.getId(), from.getType(), to.getType(), Relationship.UPSTREAM.ordinal(), detailsJson); + addLineageToSearch(from, to, addLineage.getEdge().getLineageDetails()); + } + + private void addLineageToSearch(EntityReference fromEntity, EntityReference toEntity, LineageDetails lineageDetails) { + IndexMapping sourceIndexMapping = Entity.getSearchRepository().getIndexMapping(fromEntity.getType()); + String sourceIndexName = sourceIndexMapping.getIndexName(); + IndexMapping destinationIndexMapping = Entity.getSearchRepository().getIndexMapping(toEntity.getType()); + String destinationIndexName = destinationIndexMapping.getIndexName(); + // HashMap data = new HashMap<>(); + HashMap relationshipDetails = new HashMap<>(); + Pair from = new ImmutablePair<>("_id", fromEntity.getId().toString()); + Pair to = new ImmutablePair<>("_id", toEntity.getId().toString()); + processLineageData(fromEntity, toEntity, lineageDetails, relationshipDetails); + searchClient.updateLineage(sourceIndexName, from, relationshipDetails); + searchClient.updateLineage(destinationIndexName, to, relationshipDetails); + } + + private void processLineageData( + EntityReference fromEntity, + EntityReference toEntity, + LineageDetails lineageDetails, + HashMap relationshipDetails) { + HashMap fromDetails = new HashMap<>(); + HashMap toDetails = new HashMap<>(); + // HashMap relationshipDetails = new HashMap<>(); + fromDetails.put("id", fromEntity.getId().toString()); + fromDetails.put("type", fromEntity.getType()); + fromDetails.put("fqn", fromEntity.getFullyQualifiedName()); + toDetails.put("id", toEntity.getId().toString()); + toDetails.put("type", toEntity.getType()); + toDetails.put("fqn", toEntity.getFullyQualifiedName()); + relationshipDetails.put("fromEntity", fromDetails); + relationshipDetails.put("toEntity", toDetails); + if (lineageDetails != null) { + relationshipDetails.put( + "pipeline", + JsonUtils.getMap(CommonUtil.nullOrEmpty(lineageDetails.getPipeline()) ? null : lineageDetails.getPipeline())); + if (!CommonUtil.nullOrEmpty(lineageDetails.getColumnsLineage())) { + List> colummnLineageList = new ArrayList<>(); + for (ColumnLineage columnLineage : lineageDetails.getColumnsLineage()) { + colummnLineageList.add(JsonUtils.getMap(columnLineage)); + } + relationshipDetails.put("columns", colummnLineageList); + } + relationshipDetails.put( + "sqlQuery", CommonUtil.nullOrEmpty(lineageDetails.getSqlQuery()) ? null : lineageDetails.getSqlQuery()); + relationshipDetails.put( + "source", CommonUtil.nullOrEmpty(lineageDetails.getSource()) ? null : lineageDetails.getSource()); + } } private String validateLineageDetails(EntityReference from, EntityReference to, LineageDetails details) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java index cc0770e0bcb7..69f9d0ee3bc3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java @@ -207,6 +207,32 @@ public Response searchBySourceUrl( return searchRepository.searchBySourceUrl(sourceUrl); } + @GET + @Path("/getLineage") + @Operation( + operationId = "searchLineage", + summary = "Search lineage", + responses = { + @ApiResponse( + responseCode = "200", + description = "search response", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchResponse.class))) + }) + public Response searchLineage( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "fqn") @QueryParam("fqn") String fqn, + @Parameter(description = "depth") @QueryParam("depth") int depth, + @Parameter( + description = + "Elasticsearch query that will be combined with the query_string query generator from the `query` argument") + @QueryParam("query_filter") + String queryFilter) + throws IOException { + + return searchRepository.searchLineage(fqn, depth, queryFilter); + } + @GET @Path("/suggest") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index 336ef0ec0f7d..2fa78b61c375 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.security.KeyStoreException; import java.text.ParseException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -63,6 +64,8 @@ public interface SearchClient { Response searchBySourceUrl(String sourceUrl) throws IOException; + Response searchLineage(String fqn, int Depth, String queryFilter) throws IOException; + Response searchByField(String fieldName, String fieldValue, String index) throws IOException; Response aggregate(String index, String fieldName, String value, String query) throws IOException; @@ -86,6 +89,9 @@ public interface SearchClient { void softDeleteOrRestoreChildren(String indexName, String scriptTxt, List> fieldAndValue); void updateChildren(String indexName, Pair fieldAndValue, Pair> updates); + // void updateLineage(String indexName, Pair fieldAndValue, Pair> + // lineagaData); + void updateLineage(String indexName, Pair fieldAndValue, HashMap lineagaData); TreeMap> getSortedDate( String team, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 9e32fd90645e..7e54a9313148 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -508,6 +508,10 @@ public Response searchBySourceUrl(String sourceUrl) throws IOException { return searchClient.searchBySourceUrl(sourceUrl); } + public Response searchLineage(String fqn, int depth, String queryFilter) throws IOException { + return searchClient.searchLineage(fqn, depth, queryFilter); + } + public Response searchByField(String fieldName, String fieldValue, String index) throws IOException { return searchClient.searchByField(fieldName, fieldValue, index); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 20b637588c5a..473b8de15148 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -401,6 +401,69 @@ public Response search(SearchRequest request) throws IOException { return Response.status(OK).entity(response).build(); } + @Override + public Response searchLineage(String fqn, int depth, String queryFilter) throws IOException { + Map responseMap = new HashMap<>(); + List> edges = new ArrayList<>(); + List> nodes = new ArrayList<>(); + getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.fromEntity.fqn"); + getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.toEntity.fqn"); + responseMap.put("edges", edges); + responseMap.put("nodes", nodes); + return Response.status(OK).entity(responseMap).build(); + } + + private void getLineage( + String fqn, + int depth, + List> edges, + List> nodes, + String queryFilter, + String direction) + throws IOException { + if (depth <= 0) { + return; + } + es.org.elasticsearch.action.search.SearchRequest searchRequest = + new es.org.elasticsearch.action.search.SearchRequest("all"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(direction, fqn))); + if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) { + try { + XContentParser filterParser = + XContentType.JSON + .xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, queryFilter); + QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query(); + BoolQueryBuilder newQuery = QueryBuilders.boolQuery().must(searchSourceBuilder.query()).filter(filter); + searchSourceBuilder.query(newQuery); + } catch (Exception ex) { + LOG.warn("Error parsing query_filter from query parameters, ignoring filter", ex); + } + } + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + for (var hit : searchResponse.getHits().getHits()) { + List> lineage = (List>) hit.getSourceAsMap().get("lineage"); + nodes.add(hit.getSourceAsMap()); + for (Map lin : lineage) { + HashMap fromEntity = (HashMap) lin.get("fromEntity"); + HashMap toEntity = (HashMap) lin.get("toEntity"); + if (direction.equalsIgnoreCase("lineage.fromEntity.fqn")) { + if (!edges.contains(lin) && fromEntity.get("fqn").equals(fqn)) { + edges.add(lin); + } + getLineage(toEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction); + } else { + if (!edges.contains(lin) && toEntity.get("fqn").equals(fqn)) { + edges.add(lin); + } + getLineage(fromEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction); + } + } + } + } + @Override public Response searchBySourceUrl(String sourceUrl) throws IOException { es.org.elasticsearch.action.search.SearchRequest searchRequest = @@ -1056,6 +1119,24 @@ public void updateChildren( } } + /** + * @param indexName + * @param fieldAndValue + */ + @Override + public void updateLineage(String indexName, Pair fieldAndValue, HashMap lineageData) { + if (isClientAvailable) { + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); + updateByQueryRequest.setQuery( + new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue()).operator(Operator.AND)); + String scriptTxt = "ctx._source.lineage.add(params.lineageData)"; + Map params = Collections.singletonMap("lineageData", lineageData); + Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptTxt, params); + updateByQueryRequest.setScript(script); + updateElasticSearchByQuery(updateByQueryRequest); + } + } + public void updateElasticSearch(UpdateRequest updateRequest) { if (updateRequest != null && isClientAvailable) { updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java index c85faaa05acf..ca63316d126a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java @@ -64,6 +64,7 @@ public Map buildESDoc() { doc.put("column_suggest", columnSuggest); doc.put("entityType", Entity.CONTAINER); doc.put("serviceType", container.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(container.getEntityReference())); doc.put( "fqnParts", getFQNParts( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java index 5a9dc9528065..47d1895a20f5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java @@ -65,6 +65,7 @@ public Map buildESDoc() { doc.put("tier", parseTags.getTierTag()); doc.put("owner", getEntityWithDisplayName(dashboardDataModel.getOwner())); doc.put("service", getEntityWithDisplayName(dashboardDataModel.getService())); + doc.put("lineage", SearchIndex.getLineageData(dashboardDataModel.getEntityReference())); doc.put("domain", getEntityWithDisplayName(dashboardDataModel.getDomain())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java index 72bf68096da7..2ef133165988 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java @@ -52,6 +52,7 @@ public Map buildESDoc() { doc.put("service_suggest", serviceSuggest); doc.put("entityType", Entity.DASHBOARD); doc.put("serviceType", dashboard.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(dashboard.getEntityReference())); doc.put("fqnParts", suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())); doc.put("owner", getEntityWithDisplayName(dashboard.getOwner())); doc.put("service", getEntityWithDisplayName(dashboard.getService())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java index 506009133060..a48eb78cd29b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java @@ -34,6 +34,7 @@ public Map buildESDoc() { doc.put("suggest", suggest); doc.put("entityType", Entity.MLMODEL); doc.put("serviceType", mlModel.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(mlModel.getEntityReference())); doc.put( "fqnParts", getFQNParts( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java index 0ea482fbd696..39b7a7678c4e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java @@ -45,6 +45,7 @@ public Map buildESDoc() { doc.put("service_suggest", serviceSuggest); doc.put("entityType", Entity.PIPELINE); doc.put("serviceType", pipeline.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(pipeline.getEntityReference())); doc.put( "fqnParts", getFQNParts( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java index d1c481593168..457ebec0cf25 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java @@ -38,6 +38,7 @@ public Map buildESDoc() { doc.put("tier", parseTags.getTierTag()); doc.put("owner", getEntityWithDisplayName(searchIndex.getOwner())); doc.put("service", getEntityWithDisplayName(searchIndex.getService())); + doc.put("lineage", SearchIndex.getLineageData(searchIndex.getEntityReference())); doc.put("domain", getEntityWithDisplayName(searchIndex.getDomain())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index 6059c238fe60..5f9f0d3f8056 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -9,13 +9,20 @@ import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.type.ColumnLineage; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.LineageDetails; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @@ -44,6 +51,65 @@ default EntityReference getEntityWithDisplayName(EntityReference entity) { return cloneEntity; } + static List> getLineageData(EntityReference entity) { + List> data = new ArrayList<>(); + CollectionDAO dao = Entity.getCollectionDAO(); + List toRelationshipsRecords = + dao.relationshipDAO().findTo(entity.getId(), entity.getType(), Relationship.UPSTREAM.ordinal()); + for (CollectionDAO.EntityRelationshipRecord entityRelationshipRecord : toRelationshipsRecords) { + EntityReference ref = + Entity.getEntityReferenceById( + entityRelationshipRecord.getType(), entityRelationshipRecord.getId(), Include.ALL); + LineageDetails lineageDetails = JsonUtils.readValue(entityRelationshipRecord.getJson(), LineageDetails.class); + SearchIndex.getLineageDataDirection(entity, ref, lineageDetails, data); + } + List fromRelationshipsRecords = + dao.relationshipDAO().findFrom(entity.getId(), entity.getType(), Relationship.UPSTREAM.ordinal()); + for (CollectionDAO.EntityRelationshipRecord entityRelationshipRecord : fromRelationshipsRecords) { + EntityReference ref = + Entity.getEntityReferenceById( + entityRelationshipRecord.getType(), entityRelationshipRecord.getId(), Include.ALL); + LineageDetails lineageDetails = JsonUtils.readValue(entityRelationshipRecord.getJson(), LineageDetails.class); + SearchIndex.getLineageDataDirection(ref, entity, lineageDetails, data); + } + return data; + } + + static void getLineageDataDirection( + EntityReference fromEntity, + EntityReference toEntity, + LineageDetails lineageDetails, + List> data) { + HashMap fromDetails = new HashMap<>(); + HashMap toDetails = new HashMap<>(); + HashMap relationshipDetails = new HashMap<>(); + fromDetails.put("id", fromEntity.getId().toString()); + fromDetails.put("type", fromEntity.getType()); + fromDetails.put("fqn", fromEntity.getFullyQualifiedName()); + toDetails.put("id", toEntity.getId().toString()); + toDetails.put("type", toEntity.getType()); + toDetails.put("fqn", toEntity.getFullyQualifiedName()); + relationshipDetails.put("fromEntity", fromDetails); + relationshipDetails.put("toEntity", toDetails); + if (lineageDetails != null) { + relationshipDetails.put( + "pipeline", + JsonUtils.getMap(CommonUtil.nullOrEmpty(lineageDetails.getPipeline()) ? null : lineageDetails.getPipeline())); + if (!CommonUtil.nullOrEmpty(lineageDetails.getColumnsLineage())) { + List> colummnLineageList = new ArrayList<>(); + for (ColumnLineage columnLineage : lineageDetails.getColumnsLineage()) { + colummnLineageList.add(JsonUtils.getMap(columnLineage)); + } + relationshipDetails.put("columns", colummnLineageList); + } + relationshipDetails.put( + "sqlQuery", CommonUtil.nullOrEmpty(lineageDetails.getSqlQuery()) ? null : lineageDetails.getSqlQuery()); + relationshipDetails.put( + "source", CommonUtil.nullOrEmpty(lineageDetails.getSource()) ? null : lineageDetails.getSource()); + } + data.add(relationshipDetails); + } + static Map getDefaultFields() { Map fields = new HashMap<>(); fields.put(FIELD_DISPLAY_NAME, 15.0f); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java index 038312b7918a..7e646f67dfc6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java @@ -83,6 +83,7 @@ public Map buildESDoc() { doc.put("service", getEntityWithDisplayName(table.getService())); doc.put("domain", getEntityWithDisplayName(table.getDomain())); doc.put("database", getEntityWithDisplayName(table.getDatabase())); + doc.put("lineage", SearchIndex.getLineageData(table.getEntityReference())); doc.put("databaseSchema", getEntityWithDisplayName(table.getDatabaseSchema())); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java index fa4697677275..c32b61676952 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java @@ -70,6 +70,7 @@ public Map buildESDoc() { doc.put("service_suggest", serviceSuggest); doc.put("entityType", Entity.TOPIC); doc.put("serviceType", topic.getServiceType()); + doc.put("lineage", SearchIndex.getLineageData(topic.getEntityReference())); doc.put("messageSchema", topic.getMessageSchema() != null ? topic.getMessageSchema() : null); doc.put( "fqnParts", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 74ba3277b667..15a66489af0f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -404,6 +404,17 @@ public Response searchBySourceUrl(String sourceUrl) throws IOException { return Response.status(OK).entity(response).build(); } + @Override + public Response searchLineage(String fqn, int Depth, String queryFilter) throws IOException { + os.org.opensearch.action.search.SearchRequest searchRequest = + new os.org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("sourceUrl", fqn))); + searchRequest.source(searchSourceBuilder); + String response = client.search(searchRequest, RequestOptions.DEFAULT).toString(); + return Response.status(OK).entity(response).build(); + } + @Override public Response searchByField(String fieldName, String fieldValue, String index) throws IOException { os.org.opensearch.action.search.SearchRequest searchRequest = @@ -1060,6 +1071,15 @@ public void updateChildren( } } + /** + * @param indexName + * @param fieldAndValue + * @param updates + */ + @Override + public void updateLineage( + String indexName, Pair fieldAndValue, HashMap lineagaData) {} + private void updateOpenSearchByQuery(UpdateByQueryRequest updateByQueryRequest) { if (updateByQueryRequest != null && isClientAvailable) { updateByQueryRequest.setRefresh(true); diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json index 3c10dc9db2f3..0e2a47442be9 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json @@ -88,6 +88,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "parent": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json index f3ac1d31043a..304558ee20de 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json @@ -287,6 +287,9 @@ "serviceType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "dataModelType": { "type": "keyword" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json index ff1fa958ce64..8e6dce5ef976 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json @@ -97,6 +97,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "charts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json index a65c8e32f8b5..b998e6197d34 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json @@ -97,6 +97,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "dataProducts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json index eb9866cfa67f..c8cc76343ee7 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json @@ -136,6 +136,9 @@ "href": { "type": "text" }, + "lineage": { + "type" : "object" + }, "sourceUrl": { "type": "text" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json index d6b29d6f19ba..1d6e52fa1dbf 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json @@ -206,6 +206,9 @@ "entityType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json index 8b7448998d94..b8e0342785d0 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json @@ -517,6 +517,9 @@ } } }, + "lineage": { + "type" : "object" + }, "serviceType": { "type": "keyword" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json index cffdd7b29240..30f0730b90d1 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json @@ -427,6 +427,9 @@ } } }, + "lineage": { + "type" : "object" + }, "lifeCycle": { "type": "keyword" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json index 5a928bdbd758..8faf1b6a297c 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json @@ -90,6 +90,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "parent": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json index 24c154325f29..604b23393902 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json @@ -86,6 +86,9 @@ "entityType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json index 0c16ba6cdc26..6f43943ecd0a 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json @@ -94,6 +94,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "domain" : { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json index 01e1ea352f29..44a40effb124 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json @@ -99,6 +99,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "dataProducts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json index d188600ae825..005db6bf06a9 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json @@ -141,6 +141,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "domain" : { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json index f4199e03a926..755fed8768a9 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json @@ -386,6 +386,9 @@ "lifeCycle": { "type": "object" }, + "lineage": { + "type" : "object" + }, "dataProducts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json index 4ef1a5b46fa7..ccbc8af4bc15 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json @@ -411,6 +411,9 @@ "lifeCycle": { "type": "object" }, + "lineage": { + "type" : "object" + }, "location": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json index e737eac0f68d..bf0dc62e6aa8 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json @@ -95,6 +95,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "messageSchema": { "properties": { "schemaType": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json index 259995c5c950..9659c06efd8d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json @@ -85,6 +85,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "parent": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json index 61555a8f6950..65c69122afc4 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json @@ -130,6 +130,9 @@ "entityType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json index 1271e9efe113..3e5a2abdb14c 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json @@ -77,6 +77,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "charts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json index 6ad1bf057a35..8f453e43149b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json @@ -68,6 +68,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "dataProducts": { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json index 001ade10c7ec..ff2705907461 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json @@ -117,6 +117,9 @@ "sourceUrl": { "type": "text" }, + "lineage": { + "type" : "object" + }, "domain" : { "properties": { "id": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json index 9038c22dda82..199b838cc150 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json @@ -232,6 +232,9 @@ "entityType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "suggest": { "type": "completion", "contexts": [ diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json index 19906fa794c4..b9b979eb2101 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json @@ -77,6 +77,9 @@ "sourceUrl": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "tableType": { "type": "keyword" }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json index e750b894f122..194af9a8548a 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json @@ -363,6 +363,9 @@ "serviceType": { "type": "keyword" }, + "lineage": { + "type" : "object" + }, "entityType": { "type": "keyword" }, From 2ca0d14bc8276e388461a42d6ec40918c4f16d0d Mon Sep 17 00:00:00 2001 From: 07Himank Date: Thu, 30 Nov 2023 11:57:39 +0530 Subject: [PATCH 02/44] os change --- .../service/search/opensearch/OpenSearchClient.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 15a66489af0f..e8b93fe85a64 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -406,13 +406,7 @@ public Response searchBySourceUrl(String sourceUrl) throws IOException { @Override public Response searchLineage(String fqn, int Depth, String queryFilter) throws IOException { - os.org.opensearch.action.search.SearchRequest searchRequest = - new os.org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("sourceUrl", fqn))); - searchRequest.source(searchSourceBuilder); - String response = client.search(searchRequest, RequestOptions.DEFAULT).toString(); - return Response.status(OK).entity(response).build(); + return Response.status(OK).build(); } @Override From aa2cf3509e03156db316157c8b97f56428f9c61b Mon Sep 17 00:00:00 2001 From: 07Himank Date: Fri, 8 Dec 2023 15:39:14 +0530 Subject: [PATCH 03/44] working --- .../elasticsearch/ElasticSearchClient.java | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index d68a0c70879c..701b4a4a20da 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -1,26 +1,5 @@ package org.openmetadata.service.search.elasticsearch; -import static javax.ws.rs.core.Response.Status.OK; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; -import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; -import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.Entity.QUERY; -import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; -import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES; -import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; -import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; -import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; -import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; -import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES; -import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; -import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; - import com.fasterxml.jackson.databind.JsonNode; import es.org.elasticsearch.action.ActionListener; import es.org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -89,20 +68,6 @@ import es.org.elasticsearch.xcontent.NamedXContentRegistry; import es.org.elasticsearch.xcontent.XContentParser; import es.org.elasticsearch.xcontent.XContentType; -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; -import javax.net.ssl.SSLContext; -import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -155,6 +120,42 @@ import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.JsonUtils; +import javax.net.ssl.SSLContext; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + +import static javax.ws.rs.core.Response.Status.OK; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; +import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_NAME; +import static org.openmetadata.service.Entity.QUERY; +import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES; +import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; +import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; +import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; +import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; +import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES; +import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; +import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; + @Slf4j // Not tagged with Repository annotation as it is programmatically initialized public class ElasticSearchClient implements SearchClient { @@ -411,6 +412,9 @@ public Response searchLineage(String fqn, int depth, String queryFilter) throws Map responseMap = new HashMap<>(); List> edges = new ArrayList<>(); List> nodes = new ArrayList<>(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); + responseMap.put("entity",Response.status(OK).entity(client.search(new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS).source(searchSourceBuilder), RequestOptions.DEFAULT).toString()).build()); getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.fromEntity.fqn"); getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.toEntity.fqn"); responseMap.put("edges", edges); From c0d326971d5cab3868ead3e4a04a0b1dae5536e6 Mon Sep 17 00:00:00 2001 From: 07Himank Date: Fri, 8 Dec 2023 17:06:17 +0530 Subject: [PATCH 04/44] working on lineage es integration --- .../service/search/elasticsearch/ElasticSearchClient.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 701b4a4a20da..0e34e151b9db 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -412,9 +412,15 @@ public Response searchLineage(String fqn, int depth, String queryFilter) throws Map responseMap = new HashMap<>(); List> edges = new ArrayList<>(); List> nodes = new ArrayList<>(); + es.org.elasticsearch.action.search.SearchRequest searchRequest = + new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); - responseMap.put("entity",Response.status(OK).entity(client.search(new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS).source(searchSourceBuilder), RequestOptions.DEFAULT).toString()).build()); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + for (var hit : searchResponse.getHits().getHits()) { + responseMap.put("entity", hit.getSourceAsMap()); + } getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.fromEntity.fqn"); getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.toEntity.fqn"); responseMap.put("edges", edges); From e7c79ab85da7f88831d9495f95e22fd180304885 Mon Sep 17 00:00:00 2001 From: 07Himank Date: Tue, 12 Dec 2023 16:34:19 +0530 Subject: [PATCH 05/44] add curremt lineage, special character handling, delete lineage --- .../service/jdbi3/LineageRepository.java | 38 +++++++++++++------ .../service/search/SearchClient.java | 6 ++- .../elasticsearch/ElasticSearchClient.java | 21 +++++++--- .../service/search/indexes/SearchIndex.java | 1 + 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index bb482b0f83c9..58062faf6845 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -13,12 +13,6 @@ package org.openmetadata.service.jdbi3; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -40,6 +34,16 @@ import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; +import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT; + @Repository public class LineageRepository { private final CollectionDAO dao; @@ -96,7 +100,6 @@ private void addLineageToSearch(EntityReference fromEntity, EntityReference toEn String sourceIndexName = sourceIndexMapping.getIndexName(); IndexMapping destinationIndexMapping = Entity.getSearchRepository().getIndexMapping(toEntity.getType()); String destinationIndexName = destinationIndexMapping.getIndexName(); - // HashMap data = new HashMap<>(); HashMap relationshipDetails = new HashMap<>(); Pair from = new ImmutablePair<>("_id", fromEntity.getId().toString()); Pair to = new ImmutablePair<>("_id", toEntity.getId().toString()); @@ -112,13 +115,13 @@ private void processLineageData( HashMap relationshipDetails) { HashMap fromDetails = new HashMap<>(); HashMap toDetails = new HashMap<>(); - // HashMap relationshipDetails = new HashMap<>(); fromDetails.put("id", fromEntity.getId().toString()); fromDetails.put("type", fromEntity.getType()); fromDetails.put("fqn", fromEntity.getFullyQualifiedName()); toDetails.put("id", toEntity.getId().toString()); toDetails.put("type", toEntity.getType()); toDetails.put("fqn", toEntity.getFullyQualifiedName()); + relationshipDetails.put("doc_id", fromEntity.getId().toString() + "-" + toEntity.getId().toString()); relationshipDetails.put("fromEntity", fromDetails); relationshipDetails.put("toEntity", toDetails); if (lineageDetails != null) { @@ -188,9 +191,22 @@ public boolean deleteLineage(String fromEntity, String fromId, String toEntity, EntityReference to = Entity.getEntityReferenceById(toEntity, UUID.fromString(toId), Include.NON_DELETED); // Finally, delete lineage relationship - return dao.relationshipDAO() - .delete(from.getId(), from.getType(), to.getId(), to.getType(), Relationship.UPSTREAM.ordinal()) - > 0; + boolean result = + dao.relationshipDAO() + .delete(from.getId(), from.getType(), to.getId(), to.getType(), Relationship.UPSTREAM.ordinal()) + > 0; + deleteLineageFromSearch(from, to); + return result; + } + + private void deleteLineageFromSearch(EntityReference fromEntity, EntityReference toEntity) { + searchClient.updateChildren( + GLOBAL_SEARCH_ALIAS, + new ImmutablePair<>( + "lineage.doc_id.keyword", fromEntity.getId().toString() + "-" + toEntity.getId().toString()), + new ImmutablePair<>( + String.format(REMOVE_LINEAGE_SCRIPT, fromEntity.getId().toString() + "-" + toEntity.getId().toString()), + null)); } private EntityLineage getLineage(EntityReference primary, int upstreamDepth, int downstreamDepth) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index 953bb5cb00b9..f129f0fe9263 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -44,6 +44,9 @@ public interface SearchClient { String REMOVE_TAGS_CHILDREN_SCRIPT = "for (int i = 0; i < ctx._source.tags.length; i++) { if (ctx._source.tags[i].tagFQN == '%s') { ctx._source.tags.remove(i) }}"; + String REMOVE_LINEAGE_SCRIPT = + "for (int i = 0; i < ctx._source.lineage.length; i++) { if (ctx._source.lineage[i].doc_id == '%s') { ctx._source.lineage.remove(i) }}"; + String UPDATE_ADDED_DELETE_GLOSSARY_TAGS = "if (ctx._source.tags != null) { for (int i = ctx._source.tags.size() - 1; i >= 0; i--) { if (params.tagDeleted != null) { for (int j = 0; j < params.tagDeleted.size(); j++) { if (ctx._source.tags[i].tagFQN.equalsIgnoreCase(params.tagDeleted[j].tagFQN)) { ctx._source.tags.remove(i); } } } } } if (ctx._source.tags == null) { ctx._source.tags = []; } if (params.tagAdded != null) { ctx._source.tags.addAll(params.tagAdded); } ctx._source.tags = ctx._source.tags .stream() .distinct() .sorted((o1, o2) -> o1.tagFQN.compareTo(o2.tagFQN)) .collect(Collectors.toList());"; String REMOVE_TEST_SUITE_CHILDREN_SCRIPT = @@ -92,8 +95,7 @@ public interface SearchClient { void softDeleteOrRestoreChildren(String indexName, String scriptTxt, List> fieldAndValue); void updateChildren(String indexName, Pair fieldAndValue, Pair> updates); - // void updateLineage(String indexName, Pair fieldAndValue, Pair> - // lineagaData); + void updateLineage(String indexName, Pair fieldAndValue, HashMap lineagaData); TreeMap> getSortedDate( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 0e34e151b9db..5553128717dc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -411,7 +411,7 @@ public Response search(SearchRequest request) throws IOException { public Response searchLineage(String fqn, int depth, String queryFilter) throws IOException { Map responseMap = new HashMap<>(); List> edges = new ArrayList<>(); - List> nodes = new ArrayList<>(); + Set> nodes = new HashSet<>(); es.org.elasticsearch.action.search.SearchRequest searchRequest = new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -421,8 +421,8 @@ public Response searchLineage(String fqn, int depth, String queryFilter) throws for (var hit : searchResponse.getHits().getHits()) { responseMap.put("entity", hit.getSourceAsMap()); } - getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.fromEntity.fqn"); - getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.toEntity.fqn"); + getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword"); + getLineage(fqn, depth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword"); responseMap.put("edges", edges); responseMap.put("nodes", nodes); return Response.status(OK).entity(responseMap).build(); @@ -432,7 +432,7 @@ private void getLineage( String fqn, int depth, List> edges, - List> nodes, + Set> nodes, String queryFilter, String direction) throws IOException { @@ -464,7 +464,7 @@ private void getLineage( for (Map lin : lineage) { HashMap fromEntity = (HashMap) lin.get("fromEntity"); HashMap toEntity = (HashMap) lin.get("toEntity"); - if (direction.equalsIgnoreCase("lineage.fromEntity.fqn")) { + if (direction.equalsIgnoreCase("lineage.fromEntity.fqn.keyword")) { if (!edges.contains(lin) && fromEntity.get("fqn").equals(fqn)) { edges.add(lin); } @@ -1101,6 +1101,17 @@ public void updateChildren( UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); updateByQueryRequest.setQuery( new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue()).operator(Operator.AND)); + es.org.elasticsearch.action.search.SearchRequest searchRequest = + new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query( + new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue()).operator(Operator.AND)); + searchRequest.source(searchSourceBuilder); + try { + String response = client.search(searchRequest, RequestOptions.DEFAULT).toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } Script script = new Script( ScriptType.INLINE, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index e065cda2fa20..2efbaa17d085 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -90,6 +90,7 @@ static void getLineageDataDirection( toDetails.put("id", toEntity.getId().toString()); toDetails.put("type", toEntity.getType()); toDetails.put("fqn", toEntity.getFullyQualifiedName()); + relationshipDetails.put("doc_id", fromEntity.getId().toString() + "-" + toEntity.getId().toString()); relationshipDetails.put("fromEntity", fromDetails); relationshipDetails.put("toEntity", toDetails); if (lineageDetails != null) { From 4a1942da94a837b10f73b983ca2cf8cd000cd0f1 Mon Sep 17 00:00:00 2001 From: 07Himank Date: Tue, 12 Dec 2023 16:37:03 +0530 Subject: [PATCH 06/44] remove test code --- .../service/jdbi3/LineageRepository.java | 19 +++++++++---------- .../elasticsearch/ElasticSearchClient.java | 11 ----------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index 58062faf6845..8b5d85368fce 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -13,6 +13,15 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; +import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -34,16 +43,6 @@ import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; -import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT; - @Repository public class LineageRepository { private final CollectionDAO dao; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 5553128717dc..6eb5ff0f6e94 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -1101,17 +1101,6 @@ public void updateChildren( UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); updateByQueryRequest.setQuery( new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue()).operator(Operator.AND)); - es.org.elasticsearch.action.search.SearchRequest searchRequest = - new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query( - new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue()).operator(Operator.AND)); - searchRequest.source(searchSourceBuilder); - try { - String response = client.search(searchRequest, RequestOptions.DEFAULT).toString(); - } catch (IOException e) { - throw new RuntimeException(e); - } Script script = new Script( ScriptType.INLINE, From 8cc98686c6c1088aaee060eda10182164e9dcd2c Mon Sep 17 00:00:00 2001 From: 07Himank Date: Wed, 13 Dec 2023 17:01:16 +0530 Subject: [PATCH 07/44] update lineage index work done --- .../service/search/SearchClient.java | 2 + .../elasticsearch/ElasticSearchClient.java | 75 +++++++++---------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index f129f0fe9263..8663b638058f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -47,6 +47,8 @@ public interface SearchClient { String REMOVE_LINEAGE_SCRIPT = "for (int i = 0; i < ctx._source.lineage.length; i++) { if (ctx._source.lineage[i].doc_id == '%s') { ctx._source.lineage.remove(i) }}"; + String ADD_UPDATE_LINEAGE = + "boolean docIdExists = false; for (int i = 0; i < ctx._source.lineage.size(); i++) { if (ctx._source.lineage[i].doc_id.equalsIgnoreCase(params.lineageData.doc_id)) { ctx._source.lineage[i] = params.lineageData; docIdExists = true; break;}}if (!docIdExists) {ctx._source.lineage.add(params.lineageData);}"; String UPDATE_ADDED_DELETE_GLOSSARY_TAGS = "if (ctx._source.tags != null) { for (int i = ctx._source.tags.size() - 1; i >= 0; i--) { if (params.tagDeleted != null) { for (int j = 0; j < params.tagDeleted.size(); j++) { if (ctx._source.tags[i].tagFQN.equalsIgnoreCase(params.tagDeleted[j].tagFQN)) { ctx._source.tags.remove(i); } } } } } if (ctx._source.tags == null) { ctx._source.tags = []; } if (params.tagAdded != null) { ctx._source.tags.addAll(params.tagAdded); } ctx._source.tags = ctx._source.tags .stream() .distinct() .sorted((o1, o2) -> o1.tagFQN.compareTo(o2.tagFQN)) .collect(Collectors.toList());"; String REMOVE_TEST_SUITE_CHILDREN_SCRIPT = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 6eb5ff0f6e94..80f2fcc92103 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -1,5 +1,26 @@ package org.openmetadata.service.search.elasticsearch; +import static javax.ws.rs.core.Response.Status.OK; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; +import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_NAME; +import static org.openmetadata.service.Entity.QUERY; +import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES; +import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; +import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; +import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; +import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; +import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES; +import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; +import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; + import com.fasterxml.jackson.databind.JsonNode; import es.org.elasticsearch.action.ActionListener; import es.org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -68,6 +89,20 @@ import es.org.elasticsearch.xcontent.NamedXContentRegistry; import es.org.elasticsearch.xcontent.XContentParser; import es.org.elasticsearch.xcontent.XContentType; +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLContext; +import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -120,42 +155,6 @@ import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.JsonUtils; -import javax.net.ssl.SSLContext; -import javax.ws.rs.core.Response; -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; - -import static javax.ws.rs.core.Response.Status.OK; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; -import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; -import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.Entity.QUERY; -import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; -import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES; -import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; -import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; -import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; -import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; -import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES; -import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; -import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; - @Slf4j // Not tagged with Repository annotation as it is programmatically initialized public class ElasticSearchClient implements SearchClient { @@ -1122,9 +1121,9 @@ public void updateLineage(String indexName, Pair fieldAndValue, UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); updateByQueryRequest.setQuery( new MatchQueryBuilder(fieldAndValue.getKey(), fieldAndValue.getValue()).operator(Operator.AND)); - String scriptTxt = "ctx._source.lineage.add(params.lineageData)"; + // String scriptTxt = "ctx._source.lineage.add(params.lineageData)"; Map params = Collections.singletonMap("lineageData", lineageData); - Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptTxt, params); + Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, ADD_UPDATE_LINEAGE, params); updateByQueryRequest.setScript(script); updateElasticSearchByQuery(updateByQueryRequest); } From 83347d710fd236ba39f1ad568a2d850429944d55 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Thu, 14 Dec 2023 16:07:59 +0530 Subject: [PATCH 08/44] fix: add lineage ui changes --- .../DashboardDetails.component.tsx | 18 +- .../DataModels/DataModelDetails.component.tsx | 17 +- .../EdgeInfoDrawer.component.tsx | 12 +- .../EntityInfoDrawer.component.tsx | 124 +--- .../EntityInfoDrawer.interface.ts | 6 +- .../CustomControls.component.tsx | 196 ++++- .../EntityLineage/CustomEdge.component.tsx | 59 +- .../Entity/EntityLineage/CustomNode.utils.tsx | 16 +- .../EntityLineage/CustomNodeV1.component.tsx | 260 +++++-- .../EntityLineage/EntityLineage.component.tsx | 17 +- .../EntityLineage/EntityLineage.interface.ts | 17 +- .../EntityLineage/LineageConfigModal.tsx | 6 +- .../EntityLineage/LineageNodeLabelV1.tsx | 53 +- .../NodeSuggestions.component.tsx | 13 +- .../Entity/EntityLineage/custom-node.less | 17 + .../EntityLineage/entity-lineage.style.less | 4 +- .../TableSummary/TableSummary.component.tsx | 2 +- .../components/Lineage/Lineage.component.tsx | 159 ++++ .../components/Lineage/Lineage.interface.ts | 51 ++ .../LineageProvider/LineageProvider.tsx | 697 ++++++++++++++++++ .../MlModelDetail/MlModelDetail.component.tsx | 18 +- .../PipelineDetails.component.tsx | 18 +- .../TableProfiler/TableProfilerV1.test.tsx | 2 +- .../TopicDetails/TopicDetails.component.tsx | 18 +- .../src/pages/ContainerPage/ContainerPage.tsx | 18 +- .../SearchIndexDetailsPage.tsx | 17 +- .../StoredProcedure/StoredProcedurePage.tsx | 17 +- .../TableDetailsPageV1/TableDetailsPageV1.tsx | 17 +- .../main/resources/ui/src/rest/lineageAPI.ts | 15 + .../resources/ui/src/styles/variables.less | 3 + .../ui/src/utils/AdvancedSearchUtils.tsx | 2 +- .../resources/ui/src/utils/CommonUtils.tsx | 2 +- .../ui/src/utils/EntityLineageUtils.tsx | 4 +- .../resources/ui/src/utils/LineageV1Utils.ts | 178 +++++ 34 files changed, 1714 insertions(+), 359 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.component.tsx index 294ee8df05c9..67d8d7a35635 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.component.tsx @@ -27,7 +27,6 @@ import { withActivityFeed } from '../../components/AppRouter/withActivityFeed'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import { ColumnFilter } from '../../components/Table/ColumnFilter/ColumnFilter.component'; @@ -62,9 +61,12 @@ import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThr import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import { CustomPropertyTable } from '../common/CustomPropertyTable/CustomPropertyTable'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; import { ModalWithMarkdownEditor } from '../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor'; import { usePermissionProvider } from '../PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../PermissionProvider/PermissionProvider.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; import { ChartsPermissions, ChartType, @@ -659,12 +661,14 @@ const DashboardDetails = ({ label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataModels/DataModelDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataModels/DataModelDetails.component.tsx index f0c2dfe4ff0f..8eb4e56c0c27 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataModels/DataModelDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataModels/DataModelDetails.component.tsx @@ -24,7 +24,6 @@ import ActivityThreadPanel from '../../components/ActivityFeed/ActivityThreadPan import { withActivityFeed } from '../../components/AppRouter/withActivityFeed'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import TabsLabel from '../../components/TabsLabel/TabsLabel.component'; @@ -44,6 +43,8 @@ import { getTagsWithoutTier } from '../../utils/TableUtils'; import { createTagObject } from '../../utils/TagsUtils'; import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; import SchemaEditor from '../SchemaEditor/SchemaEditor'; import { SourceType } from '../SearchedData/SearchedData.interface'; import { DataModelDetailsProps } from './DataModelDetails.interface'; @@ -332,12 +333,14 @@ const DataModelDetails = ({ ), key: EntityTabs.LINEAGE, children: ( - + + + ), }, ]; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx index a321e2c79ed1..2d30e1a0a1d1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx @@ -95,12 +95,14 @@ const EdgeInfoDrawer = ({ }, pipeline: { key: t('label.edge'), - value: data?.pipeline ? getEntityName(data?.pipeline) : undefined, + value: data?.edge?.pipeline + ? getEntityName(data?.edge?.pipeline) + : undefined, link: - data?.pipeline && + data?.edge?.pipeline && getEntityLink( - data?.pipeline.type, - getEncodedFqn(data?.pipeline.fullyQualifiedName) + data?.edge?.pipeline.type, + getEncodedFqn(data?.edge?.pipeline.fullyQualifiedName) ), }, functionInfo: { @@ -135,7 +137,7 @@ const EdgeInfoDrawer = ({ lineageDetails, }, }; - await onEdgeDescriptionUpdate(updatedEdgeDetails); + await onEdgeDescriptionUpdate?.(updatedEdgeDetails); setIsDescriptionEditable(false); } else { setIsDescriptionEditable(false); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx index 35ea60c1d541..665666a3777d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx @@ -27,21 +27,11 @@ import { SearchIndex } from '../../../generated/entity/data/searchIndex'; import { StoredProcedure } from '../../../generated/entity/data/storedProcedure'; import { Table } from '../../../generated/entity/data/table'; import { Topic } from '../../../generated/entity/data/topic'; -import { getDashboardByFqn } from '../../../rest/dashboardAPI'; -import { getDataModelsByName } from '../../../rest/dataModelsAPI'; -import { getMlModelByFQN } from '../../../rest/mlModelAPI'; -import { getPipelineByFqn } from '../../../rest/pipelineAPI'; -import { getSearchIndexDetailsByFQN } from '../../../rest/SearchIndexAPI'; -import { getContainerByName } from '../../../rest/storageAPI'; -import { getStoredProceduresByName } from '../../../rest/storedProceduresAPI'; -import { getTableDetailsByFQN } from '../../../rest/tableAPI'; -import { getTopicByFqn } from '../../../rest/topicsAPI'; import { getHeaderLabel } from '../../../utils/EntityLineageUtils'; import { DRAWER_NAVIGATION_OPTIONS, getEntityTags, } from '../../../utils/EntityUtils'; -import { getEncodedFqn } from '../../../utils/StringsUtils'; import { getEntityIcon } from '../../../utils/TableUtils'; import ContainerSummary from '../../Explore/EntitySummaryPanel/ContainerSummary/ContainerSummary.component'; import DashboardSummary from '../../Explore/EntitySummaryPanel/DashboardSummary/DashboardSummary.component'; @@ -52,7 +42,6 @@ import SearchIndexSummary from '../../Explore/EntitySummaryPanel/SearchIndexSumm import StoredProcedureSummary from '../../Explore/EntitySummaryPanel/StoredProcedureSummary/StoredProcedureSummary.component'; import TableSummary from '../../Explore/EntitySummaryPanel/TableSummary/TableSummary.component'; import TopicSummary from '../../Explore/EntitySummaryPanel/TopicSummary/TopicSummary.component'; -import { SelectedNode } from '../EntityLineage/EntityLineage.interface'; import './entity-info-drawer.less'; import { LineageDrawerProps } from './EntityInfoDrawer.interface'; @@ -66,108 +55,19 @@ const EntityInfoDrawer = ({ {} as EntityDetailUnion ); - const [isLoading, setIsLoading] = useState(false); - - const fetchEntityDetail = async (selectedNode: SelectedNode) => { - let response = {}; - const encodedFqn = getEncodedFqn(selectedNode.fqn); - const commonFields = ['tags', 'owner']; - - setIsLoading(true); - try { - switch (selectedNode.type) { - case EntityType.TABLE: { - response = await getTableDetailsByFQN(encodedFqn, [ - ...commonFields, - 'columns', - 'usageSummary', - 'profile', - ]); - - break; - } - case EntityType.PIPELINE: { - response = await getPipelineByFqn(encodedFqn, [ - ...commonFields, - 'followers', - 'tasks', - ]); - - break; - } - case EntityType.TOPIC: { - response = await getTopicByFqn(encodedFqn ?? '', commonFields); - - break; - } - case EntityType.DASHBOARD: { - response = await getDashboardByFqn(encodedFqn, [ - ...commonFields, - 'charts', - ]); - - break; - } - case EntityType.MLMODEL: { - response = await getMlModelByFQN(encodedFqn, [ - ...commonFields, - 'dashboard', - ]); - - break; - } - case EntityType.CONTAINER: { - response = await getContainerByName( - encodedFqn, - 'dataModel,owner,tags' - ); - - break; - } - - case EntityType.DASHBOARD_DATA_MODEL: { - response = await getDataModelsByName( - encodedFqn, - 'owner,tags,followers' - ); - - break; - } - - case EntityType.STORED_PROCEDURE: { - response = await getStoredProceduresByName(encodedFqn, 'owner,tags'); - - break; - } - - case EntityType.SEARCH_INDEX: { - response = await getSearchIndexDetailsByFQN(encodedFqn, 'owner,tags'); - - break; - } - - default: - break; - } - setEntityDetail(response); - } finally { - setIsLoading(false); - } - }; - const tags = useMemo( - () => getEntityTags(selectedNode.type, entityDetail), + () => + getEntityTags(selectedNode.entityType ?? EntityType.TABLE, entityDetail), [entityDetail, selectedNode] ); const summaryComponent = useMemo(() => { - switch (selectedNode.type) { + switch (selectedNode.entityType) { case EntityType.TABLE: return ( ); @@ -177,7 +77,6 @@ const EntityInfoDrawer = ({ ); @@ -187,7 +86,6 @@ const EntityInfoDrawer = ({ ); @@ -197,7 +95,6 @@ const EntityInfoDrawer = ({ ); @@ -207,7 +104,6 @@ const EntityInfoDrawer = ({ ); @@ -216,7 +112,6 @@ const EntityInfoDrawer = ({ ); @@ -226,7 +121,6 @@ const EntityInfoDrawer = ({ ); @@ -236,7 +130,6 @@ const EntityInfoDrawer = ({ ); @@ -246,7 +139,6 @@ const EntityInfoDrawer = ({ ); @@ -254,10 +146,10 @@ const EntityInfoDrawer = ({ default: return null; } - }, [entityDetail, fetchEntityDetail, tags, selectedNode, isLoading]); + }, [entityDetail, tags, selectedNode]); useEffect(() => { - fetchEntityDetail(selectedNode); + setEntityDetail(selectedNode); }, [selectedNode]); return ( @@ -291,12 +183,12 @@ const EntityInfoDrawer = ({ 'entity-info-header-link': !isMainNode, })}> - {getEntityIcon(selectedNode.type)} + {getEntityIcon(selectedNode.entityType as string)} {getHeaderLabel( selectedNode.displayName ?? selectedNode.name, - selectedNode.fqn, - selectedNode.type, + selectedNode.fullyQualifiedName, + selectedNode.entityType as string, isMainNode )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts index 6e1eaeea48ce..471272daf5f4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts @@ -13,12 +13,12 @@ import { Edge, Node } from 'reactflow'; import { AddLineage } from '../../../generated/api/lineage/addLineage'; -import { SelectedNode } from '../EntityLineage/EntityLineage.interface'; +import { SourceType } from '../../SearchedData/SearchedData.interface'; export interface LineageDrawerProps { show: boolean; onCancel: () => void; - selectedNode: SelectedNode; + selectedNode: SourceType; isMainNode: boolean; } @@ -28,7 +28,7 @@ export interface EdgeInfoDrawerInfo { visible: boolean; hasEditAccess: boolean; onClose: () => void; - onEdgeDescriptionUpdate: (updatedEdgeDetails: AddLineage) => Promise; + onEdgeDescriptionUpdate?: (updatedEdgeDetails: AddLineage) => Promise; } type InfoType = { key: string; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx index 5ede34d32961..5c949ac84690 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx @@ -11,10 +11,12 @@ * limitations under the License. */ -import { SettingOutlined } from '@ant-design/icons'; -import { Button, Col, Row, Select, Space } from 'antd'; +import { FilterOutlined, SettingOutlined } from '@ant-design/icons'; +import { Button, Col, Dropdown, Row, Select, Space } from 'antd'; import Input from 'antd/lib/input/Input'; +import { ItemType } from 'antd/lib/menu/hooks/useItems'; import classNames from 'classnames'; +import { isEmpty } from 'lodash'; import React, { FC, memo, @@ -24,7 +26,7 @@ import React, { useState, } from 'react'; import { useTranslation } from 'react-i18next'; -import { useReactFlow } from 'reactflow'; +import { Node, useReactFlow } from 'reactflow'; import { ReactComponent as ExitFullScreen } from '../../../assets/svg/exit-full-screen.svg'; import { ReactComponent as FullScreen } from '../../../assets/svg/full-screen.svg'; import { ReactComponent as EditIconColor } from '../../../assets/svg/ic-edit-lineage-colored.svg'; @@ -38,39 +40,70 @@ import { ZOOM_SLIDER_STEP, ZOOM_TRANSITION_DURATION, } from '../../../constants/Lineage.constants'; +import { SearchIndex } from '../../../enums/search.enum'; +import { + QueryFieldInterface, + QueryFieldValueInterface, +} from '../../../pages/ExplorePage/ExplorePage.interface'; +import { getAssetsPageQuickFilters } from '../../../utils/AdvancedSearchUtils'; import { handleSearchFilterOption } from '../../../utils/CommonUtils'; import { getLoadingStatusValue } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; +import { getSelectedValuesFromQuickFilter } from '../../../utils/Explore.utils'; import SVGIcons, { Icons } from '../../../utils/SvgUtils'; +import { ExploreQuickFilterField } from '../../Explore/ExplorePage.interface'; +import ExploreQuickFilters from '../../Explore/ExploreQuickFilters'; +import { useLineageProvider } from '../../LineageProvider/LineageProvider'; import { ControlProps, LineageConfig } from './EntityLineage.interface'; import LineageConfigModal from './LineageConfigModal'; const CustomControls: FC = ({ style, - isColumnsExpanded, showFitView = true, showZoom = true, fitViewParams, className, deleted, - isEditMode, hasEditAccess, - onEditLinageClick, - onExpandColumnClick, handleFullScreenViewClick, onExitFullScreenViewClick, - loading, - status, - zoomValue, - lineageData, - lineageConfig, - onOptionSelect, - onLineageConfigUpdate, }: ControlProps) => { const { t } = useTranslation(); const { fitView, zoomTo } = useReactFlow(); - const [zoom, setZoom] = useState(zoomValue); const [dialogVisible, setDialogVisible] = useState(false); + const { + nodes, + lineageConfig, + expandAllColumns, + onLineageEditClick, + zoomValue, + loading, + status, + reactFlowInstance, + toggleColumnView, + isEditMode, + onLineageConfigUpdate, + onQueryFilterUpdate, + onNodeClick, + } = useLineageProvider(); + const [zoom, setZoom] = useState(zoomValue); + const [selectedFilter, setSelectedFilter] = useState([]); + const [selectedQuickFilters, setSelectedQuickFilters] = useState< + ExploreQuickFilterField[] + >([]); + const [filters, setFilters] = useState([]); + + const handleMenuClick = ({ key }: { key: string }) => { + setSelectedFilter((prevSelected) => [...prevSelected, key]); + }; + + const filterMenu: ItemType[] = useMemo(() => { + return filters.map((filter) => ({ + key: filter.key, + label: filter.label, + onClick: handleMenuClick, + })); + }, [filters]); const onZoomHandler = useCallback( (zoomLevel: number) => { @@ -113,13 +146,28 @@ const CustomControls: FC = ({ } }, [zoomValue]); + useEffect(() => { + const dropdownItems = getAssetsPageQuickFilters(); + + setFilters( + dropdownItems.map((item) => ({ + ...item, + value: getSelectedValuesFromQuickFilter( + item, + dropdownItems, + undefined // pass in state variable + ), + })) + ); + }, []); + const nodeOptions = useMemo( () => - [lineageData.entity, ...(lineageData.nodes || [])].map((node) => ({ - label: getEntityName(node), + [...(nodes || [])].map((node) => ({ + label: getEntityName(node.data.node), value: node.id, })), - [lineageData] + [nodes] ); const editIcon = useMemo(() => { @@ -136,12 +184,99 @@ const CustomControls: FC = ({ const handleDialogSave = useCallback( (config: LineageConfig) => { - onLineageConfigUpdate(config); + onLineageConfigUpdate?.(config); setDialogVisible(false); }, [onLineageConfigUpdate, setDialogVisible] ); + const onOptionSelect = useCallback( + (value?: string) => { + const selectedNode = nodes.find((node: Node) => node.id === value); + if (selectedNode) { + const { position } = selectedNode; + onNodeClick(selectedNode); + // moving selected node in center + reactFlowInstance && + reactFlowInstance.setCenter(position.x, position.y, { + duration: ZOOM_TRANSITION_DURATION, + zoom: zoomValue, + }); + } + }, + [onNodeClick, reactFlowInstance] + ); + + const handleQuickFiltersChange = (data: ExploreQuickFilterField[]) => { + const must: QueryFieldInterface[] = []; + data.forEach((filter) => { + if (!isEmpty(filter.value)) { + const should: QueryFieldValueInterface[] = []; + if (filter.value) { + filter.value.forEach((filterValue) => { + const term: Record = {}; + term[filter.key] = filterValue.key; + should.push({ term }); + }); + } + + must.push({ + bool: { should }, + }); + } + }); + + const quickFilterQuery = isEmpty(must) + ? undefined + : { + query: { bool: { must } }, + }; + + onQueryFilterUpdate(JSON.stringify(quickFilterQuery)); + }; + + const handleQuickFiltersValueSelect = useCallback( + (field: ExploreQuickFilterField) => { + setSelectedQuickFilters((pre) => { + const data = pre.map((preField) => { + if (preField.key === field.key) { + return field; + } else { + return preField; + } + }); + + handleQuickFiltersChange(data); + + return data; + }); + }, + [setSelectedQuickFilters] + ); + + useEffect(() => { + const updatedQuickFilters = filters + .filter((filter) => selectedFilter.includes(filter.key)) + .map((selectedFilterItem) => { + const originalFilterItem = selectedQuickFilters?.find( + (filter) => filter.key === selectedFilterItem.key + ); + + return originalFilterItem || selectedFilterItem; + }); + + const newItems = updatedQuickFilters.filter( + (item) => + !selectedQuickFilters.some( + (existingItem) => item.key === existingItem.key + ) + ); + + if (newItems.length > 0) { + setSelectedQuickFilters((prevSelected) => [...prevSelected, ...newItems]); + } + }, [selectedFilter, selectedQuickFilters, filters]); + return ( <> = ({ })} onChange={onOptionSelect} /> + + + @@ -292,7 +444,7 @@ const CustomControls: FC = ({ ? t('label.edit-entity', { entity: t('label.lineage') }) : NO_PERMISSION_FOR_ACTION } - onClick={onEditLinageClick} + onClick={onLineageEditClick} /> )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx index 19fdae105439..f8ea34d52416 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx @@ -12,13 +12,16 @@ */ import { Button } from 'antd'; -import React, { Fragment, useCallback } from 'react'; +import React, { Fragment, useCallback, useMemo } from 'react'; import { EdgeProps, getBezierPath } from 'reactflow'; import { ReactComponent as FunctionIcon } from '../../../assets/svg/ic-function.svg'; import { ReactComponent as PipelineIcon } from '../../../assets/svg/pipeline-grey.svg'; +import { INFO_COLOR } from '../../../constants/constants'; import { FOREIGN_OBJECT_SIZE } from '../../../constants/Lineage.constants'; import { EntityType } from '../../../enums/entity.enum'; +import { getEntityName } from '../../../utils/EntityUtils'; import SVGIcons from '../../../utils/SvgUtils'; +import { useLineageProvider } from '../../LineageProvider/LineageProvider'; import { CustomEdgeData } from './EntityLineage.interface'; interface LineageEdgeIconProps { @@ -59,9 +62,23 @@ export const CustomEdge = ({ data, selected, }: EdgeProps) => { - const { onEdgeClick, addPipelineClick, ...rest } = data; + const { onEdgeClick, addPipelineClick, edge, isColumnLineage, ...rest } = + data; const offset = 4; + const { tracedNodes, tracedColumns } = useLineageProvider(); + + const isColumnHighlighted = useMemo(() => { + if (!isColumnLineage) { + return false; + } + + return ( + tracedColumns.includes(data.sourceHandle) && + tracedColumns.includes(data.targetHandle) + ); + }, [isColumnLineage, tracedColumns]); + const [edgePath, edgeCenterX, edgeCenterY] = getBezierPath({ sourceX, sourceY, @@ -87,6 +104,23 @@ export const CustomEdge = ({ targetPosition, }); + const updatedStyle = useMemo(() => { + const isNodeTraced = + tracedNodes.includes(edge.fromEntity.id) && + tracedNodes.includes(edge.toEntity.id); + + let isStrokeNeeded = isNodeTraced; + + if (isColumnLineage) { + isStrokeNeeded = isColumnHighlighted; + } + + return { + ...style, + ...{ stroke: isStrokeNeeded ? INFO_COLOR : undefined }, + }; + }, [style, tracedNodes, edge, isColumnHighlighted, isColumnLineage]); + const isPipelineEdgeAllowed = ( sourceType: EntityType, targetType: EntityType @@ -98,9 +132,22 @@ export const CustomEdge = ({ }; const isColumnLineageAllowed = - !data.isColumnLineage && - isPipelineEdgeAllowed(data.sourceType, data.targetType); - const hasLabel = data.label; + !isColumnLineage && + isPipelineEdgeAllowed(data.edge.fromEntity.type, data.edge.toEntity.type); + + const hasLabel = useMemo(() => { + if (isColumnLineage) { + return false; + } + if (data.edge?.pipeline) { + return getEntityName(data.edge?.pipeline); + } + + return false; + }, [isColumnLineage, data]); + + // const hasLabel = isColumnLineage ?? getEntityName(data.edge?.pipeline); + const isSelectedEditMode = selected && data.isEditMode; const isSelected = selected; @@ -179,7 +226,7 @@ export const CustomEdge = ({ data-testid="react-flow-edge-path" id={id} markerEnd={markerEnd} - style={style} + style={updatedStyle} /> {getInvisiblePath(invisibleEdgePath)} {getInvisiblePath(invisibleEdgePath1)} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx index 1129286d0c60..8a13f2d94676 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx @@ -17,10 +17,8 @@ import { Handle, HandleProps, HandleType, Position } from 'reactflow'; import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-outlined.svg'; import { EntityLineageNodeType } from '../../../enums/entity.enum'; import { EntityReference } from '../../../generated/entity/type'; -import { isLeafNode } from '../../../utils/EntityUtils'; import { getEncodedFqn } from '../../../utils/StringsUtils'; import { - LeafNodes, LineagePos, LoadingNodeState, SelectedNode, @@ -30,7 +28,7 @@ export const getHandle = ( node: EntityReference, nodeType: string, isConnectable: HandleProps['isConnectable'], - lineageLeafNodes: LeafNodes, + isLeafNode: boolean, isNodeLoading: LoadingNodeState, className?: string, onSelect?: (state: boolean, value: SelectedNode) => void, @@ -41,7 +39,7 @@ export const getHandle = ( <> {getHandleByType(isConnectable, Position.Left, 'target', className)} {generateExpandButton( - lineageLeafNodes, + isLeafNode, node, isNodeLoading, 'to', @@ -55,7 +53,7 @@ export const getHandle = ( return ( <> {generateExpandButton( - lineageLeafNodes, + isLeafNode, node, isNodeLoading, 'from', @@ -79,7 +77,7 @@ export const getHandle = ( }; const generateExpandButton = ( - lineageLeafNodes: LeafNodes, + isLeaf: boolean, node: EntityReference, isNodeLoading: LoadingNodeState, direction: LineagePos, @@ -87,10 +85,10 @@ const generateExpandButton = ( onSelect?: (state: boolean, value: SelectedNode) => void, loadNodeHandler?: (node: EntityReference, pos: LineagePos) => void ) => { - const isLeaf = isLeafNode(lineageLeafNodes, node?.id as string, direction); - const isLoading = node.id.includes(isNodeLoading.id as string); + // const isLoading = node.id.includes(isNodeLoading.id as string); + const isLoading = false; - if (!isLeaf && !isLoading) { + if (isLeaf && !isLoading) { return ( + {node.entityType === EntityType.TABLE && testSuite && ( +
+
+
+ {formTwoDigitNumber(testSuite?.summary?.success ?? 0)} +
+
+
+
+ {formTwoDigitNumber(testSuite?.summary?.aborted ?? 0)} +
+
+
+
+ {formTwoDigitNumber(testSuite?.summary?.failed ?? 0)} +
+
+
)} - + {isExpanded && (
@@ -182,7 +326,7 @@ const CustomNodeV1 = (props: NodeProps) => {
{filteredColumns.map((column) => { - const isColumnTraced = selectedColumns.includes( + const isColumnTraced = tracedColumns.includes( column.fullyQualifiedName ); @@ -198,7 +342,7 @@ const CustomNodeV1 = (props: NodeProps) => { key={column.fullyQualifiedName} onClick={(e) => { e.stopPropagation(); - handleColumnClick(column.fullyQualifiedName); + onColumnClick(column.fullyQualifiedName); }}> {getColumnHandle( column.type, @@ -233,4 +377,4 @@ const CustomNodeV1 = (props: NodeProps) => { ); }; -export default CustomNodeV1; +export default memo(CustomNodeV1); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.component.tsx index 6f2de7b357e8..8d474b604ba6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.component.tsx @@ -115,6 +115,7 @@ import SVGIcons from '../../../utils/SvgUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import TitleBreadcrumb from '../../common/TitleBreadcrumb/TitleBreadcrumb.component'; import Loader from '../../Loader/Loader'; +import { SourceType } from '../../SearchedData/SearchedData.interface'; import { useTourProvider } from '../../TourProvider/TourProvider'; import EdgeInfoDrawer from '../EntityInfoDrawer/EdgeInfoDrawer.component'; import EntityInfoDrawer from '../EntityInfoDrawer/EntityInfoDrawer.component'; @@ -1347,12 +1348,6 @@ const EntityLineageComponent: FunctionComponent = ({ }, 500); }, []); - const handleEditLineageClick = useCallback(() => { - setEditMode((pre) => !pre && !deleted); - resetSelectedData(); - setIsDrawerOpen(false); - }, [deleted]); - const handleEdgeClick = useCallback( (_e: React.MouseEvent, edge: Edge) => { setSelectedEdgeInfo(edge); @@ -1721,14 +1716,6 @@ const EntityLineageComponent: FunctionComponent = ({ !isFullScreen ? onFullScreenClick : undefined } hasEditAccess={hasEditAccess} - isColumnsExpanded={expandAllColumns} - isEditMode={isEditMode} - lineageConfig={lineageConfig} - lineageData={updatedLineageData} - loading={loading} - status={status} - zoomValue={zoomValue} - onEditLinageClick={handleEditLineageClick} onExitFullScreenViewClick={ isFullScreen ? onExitFullScreenViewClick : undefined } @@ -1760,7 +1747,7 @@ const EntityLineageComponent: FunctionComponent = ({ isMainNode={ selectedNode.name === updatedLineageData?.entity?.name } - selectedNode={selectedNode} + selectedNode={selectedNode as SourceType} show={isDrawerOpen} onCancel={closeDrawer} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts index 41b618ffe1b1..07dec2c44ed0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts @@ -17,7 +17,6 @@ import { Edge as FlowEdge, FitViewOptions, Node } from 'reactflow'; import { EntityType } from '../../../enums/entity.enum'; import { Column } from '../../../generated/entity/data/container'; import { EntityReference } from '../../../generated/entity/type'; -import { EntityLineage } from '../../../generated/type/entityLineage'; import { SourceType } from '../../SearchedData/SearchedData.interface'; export interface SelectedNode { @@ -111,24 +110,14 @@ export interface ControlProps extends HTMLAttributes { showZoom?: boolean; showFitView?: boolean; fitViewParams?: FitViewOptions; - onZoomIn?: () => void; - onZoomOut?: () => void; onFitView?: () => void; handleFullScreenViewClick?: () => void; onExitFullScreenViewClick?: () => void; deleted: boolean | undefined; - isEditMode: boolean; hasEditAccess: boolean | undefined; - isColumnsExpanded: boolean; - onEditLinageClick: () => void; - onExpandColumnClick: () => void; - loading: boolean; - status: LoadingState; - zoomValue: number; - lineageData: EntityLineage; - lineageConfig: LineageConfig; - onOptionSelect: (value?: string) => void; - onLineageConfigUpdate: (config: LineageConfig) => void; + onExpandColumnClick?: () => void; + onOptionSelect?: (value?: string) => void; + onLineageConfigUpdate?: (config: LineageConfig) => void; } export type LineagePos = 'from' | 'to'; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx index 7f8a2254be4d..7eec343b6a90 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx @@ -33,13 +33,13 @@ const LineageConfigModal: React.FC = ({ const { t } = useTranslation(); const [form] = Form.useForm(); const [upstreamDepth, setUpstreamDepth] = useState( - config.upstreamDepth || 1 + config?.upstreamDepth || 1 ); const [downstreamDepth, setDownstreamDepth] = useState( - config.downstreamDepth || 1 + config?.downstreamDepth || 1 ); const [nodesPerLayer, setNodesPerLayer] = useState( - config.nodesPerLayer || 1 + config?.nodesPerLayer || 1 ); const handleSave = () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx index 7a577d81d7d3..db80940f3116 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx @@ -18,7 +18,8 @@ import { useTranslation } from 'react-i18next'; import { EntityLineageNodeType } from '../../../enums/entity.enum'; import { EntityReference } from '../../../generated/entity/type'; import { getBreadcrumbsFromFqn } from '../../../utils/EntityUtils'; -import { getEntityIcon } from '../../../utils/TableUtils'; +import { getServiceIcon } from '../../../utils/TableUtils'; +import { SourceType } from '../../SearchedData/SearchedData.interface'; import './lineage-node-label.less'; interface LineageNodeLabelProps { @@ -41,12 +42,10 @@ const EntityLabel = ({ node }: Pick) => { return ( - -
- {getEntityIcon(node.type || '')} + +
+ {getServiceIcon(node as SourceType)}
- - { return (
- - {breadcrumbs.map((breadcrumb, index) => ( - - - {breadcrumb.name} - - {index !== breadcrumbs.length - 1 && ( - - {t('label.slash-symbol')} +
+ + {breadcrumbs.map((breadcrumb, index) => ( + + + {breadcrumb.name} - )} - - ))} - -
- + {index !== breadcrumbs.length - 1 && ( + + {t('label.slash-symbol')} + + )} + + ))} +
+
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx index a2fcd98f5672..3a34401d24e2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx @@ -97,8 +97,8 @@ const NodeSuggestions: FC = ({ debouncedOnSearch, ]); - const handleChange = (e: React.ChangeEvent<{ value: string }>): void => { - const searchText = e.target.value; + const handleChange = (value: string): void => { + const searchText = value; setSearchValue(searchText); debounceOnSearch(searchText); }; @@ -126,14 +126,7 @@ const NodeSuggestions: FC = ({ className="d-flex items-center text-sm" key={entity.fullyQualifiedName} onClick={() => { - onSelectHandler?.({ - description: entity.description, - displayName: entity.displayName, - id: entity.id, - type: entity.entityType as string, - name: entity.name, - fullyQualifiedName: entity.fullyQualifiedName, - }); + onSelectHandler?.(entity as EntityReference); }}> {entity.serviceType} { + const history = useHistory(); + const reactFlowWrapper = useRef(null); + const location = useLocation(); + const { + nodes, + edges, + isEditMode, + onNodeClick, + onEdgeClick, + onNodeDrop, + onNodesChange, + onEdgesChange, + entityLineage, + onPaneClick, + onConnect, + onZoomUpdate, + onInitReactFlow, + onEntityFqnUpdate, + } = useLineageProvider(); + const { fqn: entityFQN } = useParams<{ fqn: string }>(); + const queryParams = new URLSearchParams(location.search); + const isFullScreen = queryParams.get('fullscreen') === 'true'; + + const onFullScreenClick = useCallback(() => { + history.push({ + search: Qs.stringify({ fullscreen: true }), + }); + }, [entityFQN]); + + const onExitFullScreenViewClick = useCallback(() => { + history.push({ + search: '', + }); + }, [entityFQN]); + + const onDragOver = useCallback((event: DragEvent) => { + event.preventDefault(); + event.dataTransfer.dropEffect = 'move'; + }, []); + + const handleZoomLevel = debounce((value: number) => { + onZoomUpdate(value); + }, 150); + + useEffect(() => { + if (entity) { + onEntityFqnUpdate(entity.fullyQualifiedName ?? ''); + } + }, [entity]); + + return ( + +
+ + + onNodeDrop(_e, reactFlowWrapper.current?.getBoundingClientRect()) + } + onEdgeClick={(_e, data) => { + onEdgeClick(data); + _e.stopPropagation(); + }} + onEdgesChange={onEdgesChange} + onInit={onInitReactFlow} + onMove={(_e, viewPort) => handleZoomLevel(viewPort.zoom)} + onNodeClick={(_e, node) => { + onNodeClick(node); + _e.stopPropagation(); + }} + onNodeContextMenu={onNodeContextMenu} + onNodeDrag={dragHandle} + onNodeDragStart={dragHandle} + onNodeDragStop={dragHandle} + onNodeMouseEnter={onNodeMouseEnter} + onNodeMouseLeave={onNodeMouseLeave} + onNodeMouseMove={onNodeMouseMove} + onNodesChange={onNodesChange} + onPaneClick={onPaneClick}> + {entityLineage && ( + + )} + + + +
+ + ); +}; + +export default Lineage; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts new file mode 100644 index 000000000000..9ec07612dd0b --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityType } from '../../enums/entity.enum'; +import { EntityReference } from '../../generated/entity/type'; +import { ColumnLineage } from '../../generated/type/entityLineage'; +import { SourceType } from '../SearchedData/SearchedData.interface'; + +export interface LineageProps { + entityType: EntityType; + deleted?: boolean; + hasEditAccess: boolean; + isFullScreen?: boolean; + entity?: SourceType; +} + +export interface EntityLineageReponse { + entity: EntityReference; + nodes?: EntityReference[]; + edges?: EdgeDetails[]; +} + +export type LineageRequest = { + upstreamDepth?: number; + downstreamDepth?: number; + nodesPerLayer?: number; +}; + +export interface EdgeFromToData { + fqn: string; + id: string; + type: string; +} + +export interface EdgeDetails { + fromEntity: EdgeFromToData; + toEntity: EdgeFromToData; + pipeline?: EntityReference; + source?: string; + sqlQuery?: string; + columns?: ColumnLineage[]; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx new file mode 100644 index 000000000000..4f71194d4a41 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -0,0 +1,697 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button } from 'antd'; +import { AxiosError } from 'axios'; +import { isEqual, isNil, isUndefined, uniqueId, uniqWith } from 'lodash'; +import { LoadingState } from 'Models'; +import React, { + createContext, + DragEvent, + ReactNode, + useCallback, + useContext, + useEffect, + useMemo, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; +import { + Connection, + Edge, + getConnectedEdges, + Node, + ReactFlowInstance, + useEdgesState, + useNodesState, +} from 'reactflow'; +import { ZOOM_VALUE } from '../../constants/Lineage.constants'; +import { mockDatasetData } from '../../constants/mockTourData.constants'; +import { + EntityLineageDirection, + EntityLineageNodeType, +} from '../../enums/entity.enum'; +import { EntityReference } from '../../generated/type/entityLineage'; +import { getLineageDataByFQN } from '../../rest/lineageAPI'; +import { + addLineageHandler, + getAllTracedColumnEdge, + getAllTracedNodes, + getClassifiedEdge, + getLayoutedElements, + getUniqueFlowElements, + onLoad, + removeLineageHandler, +} from '../../utils/EntityLineageUtils'; +import { createEdges, createNodes } from '../../utils/LineageV1Utils'; +import SVGIcons from '../../utils/SvgUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; +import EdgeInfoDrawer from '../Entity/EntityInfoDrawer/EdgeInfoDrawer.component'; +import EntityInfoDrawer from '../Entity/EntityInfoDrawer/EntityInfoDrawer.component'; +import { + EdgeData, + LineageConfig, + SelectedEdge, +} from '../Entity/EntityLineage/EntityLineage.interface'; +import EntityLineageSidebar from '../Entity/EntityLineage/EntityLineageSidebar.component'; +import NodeSuggestions from '../Entity/EntityLineage/NodeSuggestions.component'; +import { EntityLineageReponse } from '../Lineage/Lineage.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; +import { useTourProvider } from '../TourProvider/TourProvider'; + +interface LineageProviderProps { + children: ReactNode; +} + +export const LineageContext = createContext({} as any); + +const LineageProvider = ({ children }: LineageProviderProps) => { + const { t } = useTranslation(); + const { isTourOpen } = useTourProvider(); + const [reactFlowInstance, setReactFlowInstance] = + useState(); + const [isDrawerOpen, setIsDrawerOpen] = useState(false); + const [isEditMode, setIsEditMode] = useState(false); + const [selectedNode, setSelectedNode] = useState( + {} as SourceType + ); + const [selectedColumn, setSelectedColumn] = useState(''); + const [expandedNodes, setExpandedNodes] = useState([]); + const [expandAllColumns, setExpandAllColumns] = useState(false); + const [selectedEdge, setSelectedEdge] = useState(); + const [entityLineage, setEntityLineage] = useState({ + nodes: [], + edges: [], + entity: {} as EntityReference, + }); + const [nodes, setNodes, onNodesChange] = useNodesState([]); + const [edges, setEdges, onEdgesChange] = useEdgesState([]); + const [loading, setLoading] = useState(false); + const [zoomValue, setZoomValue] = useState(ZOOM_VALUE); + const [tracedNodes, setTracedNodes] = useState([]); + const [tracedColumns, setTracedColumns] = useState([]); + const [entityFqn, setEntityFqn] = useState(''); + const [status, setStatus] = useState('initial'); + const [newAddedNode, setNewAddedNode] = useState({} as Node); + const [lineageConfig, setLineageConfig] = useState({ + upstreamDepth: 1, + downstreamDepth: 1, + nodesPerLayer: 50, + }); + const [queryFilter, setQueryFilter] = useState(''); + + const fetchLineageData = async (fqn: string, config?: LineageConfig) => { + if (isTourOpen) { + setEntityLineage(mockDatasetData.entityLineage); + } else { + setLoading(true); + try { + const res = await getLineageDataByFQN(fqn, config, queryFilter); + if (res) { + setEntityLineage(res); + const allNodes = [res.entity, ...(res.nodes ?? [])]; + const updatedNodes = createNodes(allNodes, res.edges ?? []); + const updatedEdges = createEdges(allNodes, res.edges ?? []); + + setNodes(updatedNodes); + setEdges(updatedEdges); + } else { + showErrorToast( + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } finally { + setLoading(false); + } + } + }; + + const loadChildNodesHandler = useCallback( + async (node: EntityReference) => { + try { + const res = await getLineageDataByFQN( + node.fullyQualifiedName ?? '', + lineageConfig, + queryFilter + ); + + const allNodes = uniqWith( + [...(entityLineage?.nodes ?? []), ...(res.nodes ?? [])], + isEqual + ); + const allEdges = uniqWith( + [...(entityLineage?.edges ?? []), ...(res.edges ?? [])], + isEqual + ); + + const newNodes = createNodes(allNodes, allEdges); + const newEdges = createEdges(allNodes, allEdges); + + setEntityLineage((prev) => { + return { + ...prev, + nodes: allNodes, + edges: allEdges, + }; + }); + + setNodes(newNodes); + setEdges(newEdges); + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } + }, + [nodes, edges, lineageConfig, entityLineage, setEntityLineage, queryFilter] + ); + + const handleLineageTracing = useCallback( + (selectedNode: Node) => { + const { normalEdge } = getClassifiedEdge(edges); + const incomingNode = getAllTracedNodes( + selectedNode, + nodes, + normalEdge, + [], + true + ); + const outgoingNode = getAllTracedNodes( + selectedNode, + nodes, + normalEdge, + [], + false + ); + const incomerIds = incomingNode.map((incomer) => incomer.id); + const outgoerIds = outgoingNode.map((outGoer) => outGoer.id); + const connectedNodeIds = [...outgoerIds, ...incomerIds, selectedNode.id]; + setTracedNodes(connectedNodeIds); + setTracedColumns([]); + }, + [nodes, edges, selectedNode] + ); + + const onColumnClick = useCallback( + (column: string) => { + setSelectedColumn(column); + const { columnEdge } = getClassifiedEdge(edges); + const { connectedColumnEdges } = getAllTracedColumnEdge( + column, + columnEdge + ); + + setTracedColumns(connectedColumnEdges); + }, + [nodes, edges] + ); + + const removeEdgeHandler = ( + { source, target }: SelectedEdge, + confirmDelete: boolean + ) => { + if (confirmDelete && entityLineage) { + const edgeData: EdgeData = { + fromEntity: source.type, + fromId: source.id, + toEntity: target.type, + toId: target.id, + }; + removeLineageHandler(edgeData); + setEdges((prevEdges) => { + return prevEdges.filter((edge) => { + const isRemovedEdge = + edge.source === source.id && edge.target === target.id; + + return !isRemovedEdge; + }); + }); + } + }; + + const removeNodeHandler = useCallback( + (node: Node) => { + if (!entityLineage) { + return; + } + // Get edges connected to selected node + const edgesToRemove = getConnectedEdges([node], edges); + + // edgesToRemove.forEach((edge) => { + // removeEdgeHandler( + // getRemovedNodeData( + // entityLineage.nodes || [], + // edge, + // entityLineage.entity, + // selectedEntity + // ), + // true + // ); + // }); + + setNodes( + (previousNodes) => + getUniqueFlowElements( + previousNodes.filter((previousNode) => previousNode.id !== node.id) + ) as Node[] + ); + setNewAddedNode({} as Node); + }, + [nodes, entityLineage] + ); + + const onNodeDrop = (event: DragEvent, reactFlowBounds: DOMRect) => { + event.preventDefault(); + const type = event.dataTransfer.getData('application/reactflow'); + if (type.trim()) { + const position = reactFlowInstance?.project({ + x: event.clientX - (reactFlowBounds?.left ?? 0), + y: event.clientY - (reactFlowBounds?.top ?? 0), + }); + const [label, nodeType] = type.split('-'); + const nodeId = uniqueId(); + const newNode = { + id: nodeId, + nodeType, + position, + className: '', + connectable: false, + selectable: false, + type: EntityLineageNodeType.DEFAULT, + data: { + label: ( +
+
+ ), + isEditMode, + isNewNode: true, + }, + }; + setNodes([...nodes, newNode as Node]); + setNewAddedNode(newNode as Node); + } + }; + + const onQueryFilterUpdate = useCallback((query: string) => { + setQueryFilter(query); + }, []); + + const onNodeClick = (node: Node) => { + if (node) { + setSelectedEdge(undefined); + setSelectedNode(node.data.node as SourceType); + setIsDrawerOpen(true); + handleLineageTracing(node); + } + }; + + const onPaneClick = useCallback(() => { + setIsDrawerOpen(false); + setTracedNodes([]); + setTracedColumns([]); + setSelectedNode({} as SourceType); + }, []); + + const onEdgeClick = (edge: Edge) => { + setSelectedEdge(edge); + setSelectedNode({} as SourceType); + setIsDrawerOpen(true); + }; + + const onLineageEditClick = () => { + setIsEditMode((pre) => !pre); + setSelectedNode({} as SourceType); + setIsDrawerOpen(false); + }; + + const onInitReactFlow = (reactFlowInstance: ReactFlowInstance) => { + onLoad(reactFlowInstance); + setReactFlowInstance(reactFlowInstance); + }; + + const onLineageConfigUpdate = useCallback((config) => { + setLineageConfig(config); + }, []); + + const onDrawerClose = useCallback(() => { + setIsDrawerOpen(false); + }, []); + + const onZoomUpdate = useCallback((value) => { + setZoomValue(value); + }, []); + + const onEntityFqnUpdate = useCallback((value) => { + setEntityFqn(value); + }, []); + + const toggleColumnView = () => { + const updatedVal = !expandAllColumns; + setExpandAllColumns(updatedVal); + setNodes((prevNodes) => { + const updatedNode = prevNodes.map((node) => { + const nodeId = node.data.node.id; + // Update the expandedNodes state based on the toggle value + if (updatedVal && !expandedNodes.includes(nodeId)) { + setExpandedNodes((prevExpandedNodes) => [ + ...prevExpandedNodes, + nodeId, + ]); + } else if (!updatedVal) { + setExpandedNodes((prevExpandedNodes) => + prevExpandedNodes.filter((id) => id !== nodeId) + ); + } + + return node; + }); + + const { edge, node } = getLayoutedElements( + { + node: updatedNode, + edge: edges, + }, + EntityLineageDirection.LEFT_RIGHT, + updatedVal + ); + + setEdges(edge); + + return node; + }); + }; + + const onConnect = useCallback( + (params: Edge | Connection) => { + if (!entityLineage) { + return; + } + const { target, source, sourceHandle, targetHandle } = params; + + if (target === source) { + return; + } + + const columnConnection = !isNil(sourceHandle) && !isNil(targetHandle); + + setStatus('waiting'); + setLoading(true); + + // const nodes = [ + // ...(entityLineage.nodes as EntityReference[]), + // entityLineage.entity, + // ]; + + const targetNode = nodes?.find((n) => target === n.id); + const sourceNode = nodes?.find((n) => source === n.id); + + // if (isUndefined(targetNode) && sourceNode?.id !== selectedNode?.id) { + // targetNode = getSourceOrTargetNode(target || ''); + // } + // if (isUndefined(sourceNode) && targetNode?.id !== selectedNode?.id) { + // sourceNode = getSourceOrTargetNode(source || ''); + // } + + if (!isUndefined(sourceNode) && !isUndefined(targetNode)) { + const { + id: sourceId, + entityType: sourceType, + fullyQualifiedName: sourceFqn, + } = sourceNode.data.node; + const { + id: targetId, + entityType: targetType, + fullyQualifiedName: targetFqn, + } = targetNode.data.node; + + const newEdgeWithFqn = { + edge: { + fromEntity: { id: sourceId, type: sourceType, fqn: sourceFqn }, + toEntity: { id: targetId, type: targetType, fqn: targetFqn }, + }, + }; + + // Create another variable without the fqn field + const newEdgeWithoutFqn = { + edge: { + fromEntity: { id: sourceId, type: sourceType }, + toEntity: { id: targetId, type: targetType }, + }, + }; + + addLineageHandler(newEdgeWithoutFqn) + .then(() => { + if (!entityLineage) { + return; + } + setStatus('success'); + setLoading(false); + + const allNodes = [ + ...(entityLineage.nodes ?? []), + sourceNode?.data.node as EntityReference, + targetNode?.data.node as EntityReference, + ]; + + const allEdges = [ + ...(entityLineage.edges ?? []), + newEdgeWithFqn.edge, + ]; + + const updatedNodes = createNodes( + allNodes as EntityReference[], + allEdges + ); + const updatedEdges = createEdges( + allNodes as EntityReference[], + allEdges + ); + + setEntityLineage((pre) => { + const newData = { + ...pre, + nodes: uniqWith([pre.entity, ...allNodes], isEqual), + edges: uniqWith(allEdges, isEqual), + }; + + return newData; + }); + + setNodes(updatedNodes); + setEdges(updatedEdges); + + setTimeout(() => { + setStatus('initial'); + }, 100); + setNewAddedNode({} as Node); + }) + .catch(() => { + setStatus('initial'); + setLoading(false); + }); + } + }, + [selectedNode, entityLineage, nodes] + ); + + const onEntitySelect = (selectedEntity: EntityReference, nodeId: string) => { + const isExistingNode = nodes.some( + (n) => + n.data.node.fullyQualifiedName === selectedEntity.fullyQualifiedName + ); + if (isExistingNode) { + setNodes((es) => + es + .map((n) => + n.id.includes(nodeId) + ? { + ...n, + selectable: true, + className: `${n.className} selected`, + } + : n + ) + .filter((es) => es.id !== nodeId) + ); + setNewAddedNode({} as Node); + } else { + setNodes((es) => { + return es.map((el) => { + if (el.id === nodeId) { + return { + ...el, + connectable: true, + selectable: true, + id: nodeId, + data: { + saved: false, + node: selectedEntity, + }, + }; + } else { + return el; + } + }); + }); + } + }; + + useEffect(() => { + if (entityFqn) { + fetchLineageData(entityFqn, lineageConfig); + } + }, [lineageConfig, entityFqn, queryFilter]); + + useEffect(() => { + // filter out the unsaved nodes + }, []); + + const activityFeedContextValues = useMemo(() => { + return { + isDrawerOpen, + loading, + isEditMode, + nodes, + edges, + reactFlowInstance, + entityLineage, + lineageConfig, + selectedNode, + selectedColumn, + zoomValue, + status, + expandedNodes, + tracedNodes, + tracedColumns, + expandAllColumns, + onInitReactFlow, + onPaneClick, + onConnect, + onNodeDrop, + onColumnClick, + onNodesChange, + onEdgesChange, + onQueryFilterUpdate, + onZoomUpdate, + onDrawerClose, + toggleColumnView, + loadChildNodesHandler, + fetchLineageData, + onEntityFqnUpdate, + removeNodeHandler, + onNodeClick, + onEdgeClick, + onLineageConfigUpdate, + onLineageEditClick, + }; + }, [ + isDrawerOpen, + loading, + isEditMode, + nodes, + edges, + entityLineage, + reactFlowInstance, + lineageConfig, + selectedNode, + selectedColumn, + zoomValue, + status, + expandedNodes, + tracedNodes, + tracedColumns, + expandAllColumns, + onInitReactFlow, + onPaneClick, + onConnect, + onNodeDrop, + onColumnClick, + onQueryFilterUpdate, + onNodesChange, + onEdgesChange, + onZoomUpdate, + onDrawerClose, + loadChildNodesHandler, + fetchLineageData, + onEntityFqnUpdate, + toggleColumnView, + removeNodeHandler, + onNodeClick, + onEdgeClick, + onLineageConfigUpdate, + onLineageEditClick, + ]); + + return ( + + {children} + + + {isDrawerOpen && + !isEditMode && + (selectedEdge ? ( + { + setIsDrawerOpen(false); + setSelectedEdge(undefined); + }} + /> + ) : ( + setIsDrawerOpen(false)} + /> + ))} + + ); +}; + +export const useLineageProvider = () => useContext(LineageContext); + +export default LineageProvider; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.tsx index 74447ef1ebba..a67a36477a3b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.tsx @@ -24,7 +24,6 @@ import { ActivityFeedTab } from '../../components/ActivityFeed/ActivityFeedTab/A import { withActivityFeed } from '../../components/AppRouter/withActivityFeed'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import TabsLabel from '../../components/TabsLabel/TabsLabel.component'; @@ -48,8 +47,11 @@ import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThr import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import { CustomPropertyTable } from '../common/CustomPropertyTable/CustomPropertyTable'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; import { usePermissionProvider } from '../PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../PermissionProvider/PermissionProvider.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; import { MlModelDetailProp } from './MlModelDetail.interface'; import MlModelFeaturesList from './MlModelFeaturesList'; @@ -470,12 +472,14 @@ const MlModelDetail: FC = ({ label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx index 58895209bcae..76cb605159b2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx @@ -26,7 +26,6 @@ import { ActivityFeedTab } from '../../components/ActivityFeed/ActivityFeedTab/A import { CustomPropertyTable } from '../../components/common/CustomPropertyTable/CustomPropertyTable'; import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import ExecutionsTab from '../../components/Execution/Execution.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; @@ -72,9 +71,12 @@ import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThr import { withActivityFeed } from '../AppRouter/withActivityFeed'; import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; import { ModalWithMarkdownEditor } from '../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor'; import { usePermissionProvider } from '../PermissionProvider/PermissionProvider'; import { ResourceEntity } from '../PermissionProvider/PermissionProvider.interface'; +import { SourceType } from '../SearchedData/SearchedData.interface'; import { PipeLineDetailsProp } from './PipelineDetails.interface'; const PipelineDetails = ({ @@ -653,12 +655,14 @@ const PipelineDetails = ({ label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.test.tsx index 7bee1e58d3fc..79a3634b7504 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.test.tsx @@ -61,7 +61,7 @@ jest.mock('./Component/ColumnProfileTable', () => { jest.mock('../../utils/CommonUtils', () => ({ formatNumberWithComma: jest.fn(), - formTwoDigitNmber: jest.fn(), + formTwoDigitNumber: jest.fn(), getStatisticsDisplayValue: jest.fn(), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.component.tsx index b2a2c0601c54..b4105d7cbc61 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.component.tsx @@ -24,7 +24,6 @@ import DescriptionV1 from '../../components/common/EntityDescription/Description import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import QueryViewer from '../../components/common/QueryViewer/QueryViewer.component'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import SampleDataWithMessages from '../../components/SampleDataWithMessages/SampleDataWithMessages'; @@ -51,6 +50,9 @@ import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThr import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; import { CustomPropertyTable } from '../common/CustomPropertyTable/CustomPropertyTable'; import EntityRightPanel from '../Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../Lineage/Lineage.component'; +import LineageProvider from '../LineageProvider/LineageProvider'; +import { SourceType } from '../SearchedData/SearchedData.interface'; import { TopicDetailsProps } from './TopicDetails.interface'; import TopicSchemaFields from './TopicSchema/TopicSchema'; @@ -392,12 +394,14 @@ const TopicDetails: React.FC = ({ label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx index 81329581b3b7..62e0dd37dba0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.tsx @@ -29,8 +29,9 @@ import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/Error import ContainerChildren from '../../components/ContainerDetail/ContainerChildren/ContainerChildren'; import ContainerDataModel from '../../components/ContainerDetail/ContainerDataModel/ContainerDataModel'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import EntityRightPanel from '../../components/Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../../components/Lineage/Lineage.component'; +import LineageProvider from '../../components/LineageProvider/LineageProvider'; import Loader from '../../components/Loader/Loader'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; @@ -39,6 +40,7 @@ import { OperationPermission, ResourceEntity, } from '../../components/PermissionProvider/PermissionProvider.interface'; +import { SourceType } from '../../components/SearchedData/SearchedData.interface'; import { QueryVote } from '../../components/TableQueries/TableQueries.interface'; import TabsLabel from '../../components/TabsLabel/TabsLabel.component'; import { @@ -609,12 +611,14 @@ const ContainerPage = () => { label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexDetailsPage/SearchIndexDetailsPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexDetailsPage/SearchIndexDetailsPage.tsx index 462b1af4a5b8..f8db2522b060 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexDetailsPage/SearchIndexDetailsPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/SearchIndexDetailsPage/SearchIndexDetailsPage.tsx @@ -30,8 +30,9 @@ import DescriptionV1 from '../../components/common/EntityDescription/Description import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import QueryViewer from '../../components/common/QueryViewer/QueryViewer.component'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import EntityRightPanel from '../../components/Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../../components/Lineage/Lineage.component'; +import LineageProvider from '../../components/LineageProvider/LineageProvider'; import Loader from '../../components/Loader/Loader'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; @@ -468,12 +469,14 @@ function SearchIndexDetailsPage() { label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.tsx index 62ac1d88c6e2..58524c9a5dcb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.tsx @@ -26,8 +26,9 @@ import { CustomPropertyTable } from '../../components/common/CustomPropertyTable import DescriptionV1 from '../../components/common/EntityDescription/DescriptionV1'; import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import EntityRightPanel from '../../components/Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../../components/Lineage/Lineage.component'; +import LineageProvider from '../../components/LineageProvider/LineageProvider'; import Loader from '../../components/Loader/Loader'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; @@ -595,12 +596,14 @@ const StoredProcedurePage = () => { label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx index db4a4464f85e..d0c2273e7159 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx @@ -29,8 +29,9 @@ import DescriptionV1 from '../../components/common/EntityDescription/Description import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import QueryViewer from '../../components/common/QueryViewer/QueryViewer.component'; import { DataAssetsHeader } from '../../components/DataAssets/DataAssetsHeader/DataAssetsHeader.component'; -import EntityLineageComponent from '../../components/Entity/EntityLineage/EntityLineage.component'; import EntityRightPanel from '../../components/Entity/EntityRightPanel/EntityRightPanel'; +import Lineage from '../../components/Lineage/Lineage.component'; +import LineageProvider from '../../components/LineageProvider/LineageProvider'; import Loader from '../../components/Loader/Loader'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; @@ -665,12 +666,14 @@ const TableDetailsPageV1 = () => { label: , key: EntityTabs.LINEAGE, children: ( - + + + ), }, diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts index beac4e64f091..8f9274da23f1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts @@ -11,6 +11,8 @@ * limitations under the License. */ +import { LineageConfig } from '../components/Entity/EntityLineage/EntityLineage.interface'; +import { EntityLineageReponse } from '../components/Lineage/Lineage.interface'; import { AddLineage } from '../generated/api/lineage/addLineage'; import { EntityLineage } from '../generated/type/entityLineage'; import APIClient from './index'; @@ -33,3 +35,16 @@ export const updateLineageEdge = async (edge: AddLineage) => { return response.data; }; + +export const getLineageDataByFQN = async ( + fqn: string, + config?: LineageConfig, + queryFilter?: string +) => { + const { upstreamDepth = 1, downstreamDepth = 1 } = config ?? {}; + const response = await APIClient.get( + `/search/getLineage?fqn=${fqn}&depth=${upstreamDepth}&upstreamDepth=${upstreamDepth}&downstreamDepth=${downstreamDepth}&query_filter=${queryFilter}` + ); + + return response.data; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/styles/variables.less b/openmetadata-ui/src/main/resources/ui/src/styles/variables.less index d6234fc583a9..a844bf5ebcc7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/styles/variables.less +++ b/openmetadata-ui/src/main/resources/ui/src/styles/variables.less @@ -20,10 +20,12 @@ @green-2: #ebf9f4; @green-3: #48ca9e; @green-4: #48ca9e30; +@green-5: #91ffd8; @warning-color: #ffc34e; @yellow-1: #fbf2db; @yellow-2: #ffbe0e; @yellow-3: #ffbe0e1a; +@yellow-4: #ffd978; @error-color: #ff4c3b; @error-light-color: #e5493740; @failed-color: #cb2431; @@ -31,6 +33,7 @@ @red-2: #faf1f1; @red-3: #f24822; @red-4: #f2482230; +@red-5: #ffc2c2; @purple-1: #f2edfd; @purple-2: #7147e8; @blue-1: #ebf6fe; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx index 95a810e5642e..31174750f8d8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx @@ -91,7 +91,7 @@ export const getDropDownItems = (index: string) => { } }; -export const getAssetsPageQuickFilters = (type: AssetsOfEntity) => { +export const getAssetsPageQuickFilters = (type?: AssetsOfEntity) => { switch (type) { case AssetsOfEntity.DOMAIN: case AssetsOfEntity.DATA_PRODUCT: diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx index 7b09234c0358..6e6227687d85 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx @@ -564,7 +564,7 @@ export const getStatisticsDisplayValue = ( return formatNumberWithComma(displayValue); }; -export const formTwoDigitNmber = (number: number) => { +export const formTwoDigitNumber = (number: number) => { return number.toLocaleString('en-US', { minimumIntegerDigits: 2, useGrouping: false, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index e080f1d7ba5e..277d67d6b18a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -411,7 +411,8 @@ export const getDeletedLineagePlaceholder = () => { export const getLayoutedElements = ( elements: CustomElement, - direction = EntityLineageDirection.LEFT_RIGHT + direction = EntityLineageDirection.LEFT_RIGHT, + isExpanded = true ) => { const dagreGraph = new dagre.graphlib.Graph(); dagreGraph.setDefaultEdgeLabel(() => ({})); @@ -423,7 +424,6 @@ export const getLayoutedElements = ( const nodeIds = node.map((item) => item.id); node.forEach((el) => { - const isExpanded = el.data.isExpanded; dagreGraph.setNode(el.id, { width: NODE_WIDTH, height: isExpanded ? EXPANDED_NODE_HEIGHT : NODE_HEIGHT, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts new file mode 100644 index 000000000000..cc258bdff6d7 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts @@ -0,0 +1,178 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Edge, MarkerType, Position } from 'reactflow'; +import { + EntityLineageDirection, + EntityLineageNodeType, +} from '../enums/entity.enum'; +import { EntityReference } from '../generated/entity/type'; + +import dagre from 'dagre'; +import { isNil, isUndefined } from 'lodash'; +import { EdgeDetails } from '../components/Lineage/Lineage.interface'; +import { NODE_HEIGHT, NODE_WIDTH } from '../constants/Lineage.constants'; + +export const checkUpstreamDownstream = (id: string, data: EdgeDetails[]) => { + const hasUpstream = data.some((edge: EdgeDetails) => edge.toEntity.id === id); + + const hasDownstream = data.some( + (edge: EdgeDetails) => edge.fromEntity.id === id + ); + + return { hasUpstream, hasDownstream }; +}; + +const removeDuplicateNodes = (nodesData: EntityReference[]) => { + const uniqueNodesMap = new Map(); + nodesData.forEach((node) => { + uniqueNodesMap.set(node.fullyQualifiedName ?? '', node); + }); + + const uniqueNodesArray = Array.from(uniqueNodesMap.values()); + + return uniqueNodesArray; +}; + +const getNodeType = ( + edgesData: EdgeDetails[], + id: string +): EntityLineageNodeType => { + const hasDownStreamToEntity = edgesData.find( + (down) => down.toEntity.id === id + ); + const hasDownStreamFromEntity = edgesData.find( + (down) => down.fromEntity.id === id + ); + const hasUpstreamFromEntity = edgesData.find((up) => up.fromEntity.id === id); + const hasUpstreamToEntity = edgesData.find((up) => up.toEntity.id === id); + + if (hasDownStreamToEntity && !hasDownStreamFromEntity) { + return EntityLineageNodeType.OUTPUT; + } + if (hasUpstreamFromEntity && !hasUpstreamToEntity) { + return EntityLineageNodeType.INPUT; + } + + return EntityLineageNodeType.DEFAULT; +}; + +export const createNodes = ( + nodesData: EntityReference[], + edgesData: EdgeDetails[] +) => { + const uniqueNodesData = removeDuplicateNodes(nodesData); + + // Create a new dagre graph + const graph = new dagre.graphlib.Graph(); + + // Set an object for the graph label + graph.setGraph({ rankdir: EntityLineageDirection.LEFT_RIGHT }); + + // Default to assigning a new object as a label for each new edge. + graph.setDefaultEdgeLabel(() => ({})); + + // Add nodes to the graph + uniqueNodesData.forEach((node) => { + graph.setNode(node.id, { width: NODE_WIDTH, height: NODE_HEIGHT }); + }); + + // Add edges to the graph (if you have edge information) + edgesData.forEach((edge) => { + graph.setEdge(edge.fromEntity.id, edge.toEntity.id); + }); + + // Perform the layout + dagre.layout(graph); + + // Get the layout positions + const layoutPositions = graph.nodes().map((nodeId) => graph.node(nodeId)); + + return uniqueNodesData.map((node, index) => { + const position = layoutPositions[index]; + + const type = getNodeType(edgesData, node.id); + + return { + id: `${node.id}`, + sourcePosition: Position.Right, + targetPosition: Position.Left, + type: type, + className: '', + data: { + node, + }, + position: { + x: position.x, + y: position.y, + }, + }; + }); +}; + +export const createEdges = (nodes: EntityReference[], edges: EdgeDetails[]) => { + const lineageEdgesV1: Edge[] = []; + + edges.forEach((edge) => { + const sourceType = nodes.find((n) => edge.fromEntity.id === n.id); + const targetType = nodes.find((n) => edge.toEntity.id === n.id); + + if (isUndefined(sourceType) || isUndefined(targetType)) { + return; + } + + if (!isUndefined(edge.columns)) { + edge.columns?.forEach((e) => { + const toColumn = e.toColumn || ''; + if (toColumn && e.fromColumns && e.fromColumns.length > 0) { + e.fromColumns.forEach((fromColumn) => { + lineageEdgesV1.push({ + id: `column-${fromColumn}-${toColumn}-edge-${edge.fromEntity.id}-${edge.toEntity.id}`, + source: edge.fromEntity.id, + target: edge.toEntity.id, + targetHandle: toColumn, + sourceHandle: fromColumn, + type: 'buttonedge', + markerEnd: { + type: MarkerType.ArrowClosed, + }, + data: { + edge, + isColumnLineage: true, + targetHandle: toColumn, + sourceHandle: fromColumn, + }, + }); + }); + } + }); + } + + lineageEdgesV1.push({ + id: `edge-${edge.fromEntity.id}-${edge.toEntity.id}`, + source: `${edge.fromEntity.id}`, + target: `${edge.toEntity.id}`, + type: 'buttonedge', + animated: !isNil(edge.pipeline), + style: { strokeWidth: '2px' }, + markerEnd: { + type: MarkerType.ArrowClosed, + }, + data: { + edge, + isColumnLineage: false, + }, + }); + }); + + return lineageEdgesV1; +}; From 1dd5fa30e77db20778c4e1b1069eb534accdbeae Mon Sep 17 00:00:00 2001 From: karanh37 Date: Thu, 14 Dec 2023 18:05:53 +0530 Subject: [PATCH 09/44] fix: lineage add edit --- .../EntityLineage/CustomEdge.component.tsx | 2 - .../LineageProvider/LineageProvider.tsx | 90 ++++++++++--------- .../resources/ui/src/utils/LineageV1Utils.ts | 1 + 3 files changed, 48 insertions(+), 45 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx index f8ea34d52416..9e81a1b6488a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx @@ -146,8 +146,6 @@ export const CustomEdge = ({ return false; }, [isColumnLineage, data]); - // const hasLabel = isColumnLineage ?? getEntityName(data.edge?.pipeline); - const isSelectedEditMode = selected && data.isEditMode; const isSelected = selected; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index 4f71194d4a41..ad121d3f4e0b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -12,7 +12,7 @@ */ import { Button } from 'antd'; import { AxiosError } from 'axios'; -import { isEqual, isNil, isUndefined, uniqueId, uniqWith } from 'lodash'; +import { isEqual, isUndefined, uniqueId, uniqWith } from 'lodash'; import { LoadingState } from 'Models'; import React, { createContext, @@ -40,7 +40,11 @@ import { EntityLineageDirection, EntityLineageNodeType, } from '../../enums/entity.enum'; -import { EntityReference } from '../../generated/type/entityLineage'; +import { EntitiesEdge } from '../../generated/api/lineage/addLineage'; +import { + ColumnLineage, + EntityReference, +} from '../../generated/type/entityLineage'; import { getLineageDataByFQN } from '../../rest/lineageAPI'; import { addLineageHandler, @@ -60,11 +64,13 @@ import EntityInfoDrawer from '../Entity/EntityInfoDrawer/EntityInfoDrawer.compon import { EdgeData, LineageConfig, - SelectedEdge, } from '../Entity/EntityLineage/EntityLineage.interface'; import EntityLineageSidebar from '../Entity/EntityLineage/EntityLineageSidebar.component'; import NodeSuggestions from '../Entity/EntityLineage/NodeSuggestions.component'; -import { EntityLineageReponse } from '../Lineage/Lineage.interface'; +import { + EdgeDetails, + EntityLineageReponse, +} from '../Lineage/Lineage.interface'; import { SourceType } from '../SearchedData/SearchedData.interface'; import { useTourProvider } from '../TourProvider/TourProvider'; @@ -227,22 +233,19 @@ const LineageProvider = ({ children }: LineageProviderProps) => { [nodes, edges] ); - const removeEdgeHandler = ( - { source, target }: SelectedEdge, - confirmDelete: boolean - ) => { + const removeEdgeHandler = (edge: Edge, confirmDelete: boolean) => { if (confirmDelete && entityLineage) { + const { data, id } = edge; const edgeData: EdgeData = { - fromEntity: source.type, - fromId: source.id, - toEntity: target.type, - toId: target.id, + fromEntity: data.edge.fromEntity.type, + fromId: data.edge.fromEntity.id, + toEntity: data.edge.toEntity.type, + toId: data.edge.toEntity.id, }; removeLineageHandler(edgeData); setEdges((prevEdges) => { return prevEdges.filter((edge) => { - const isRemovedEdge = - edge.source === source.id && edge.target === target.id; + const isRemovedEdge = edge.id === id; return !isRemovedEdge; }); @@ -257,18 +260,9 @@ const LineageProvider = ({ children }: LineageProviderProps) => { } // Get edges connected to selected node const edgesToRemove = getConnectedEdges([node], edges); - - // edgesToRemove.forEach((edge) => { - // removeEdgeHandler( - // getRemovedNodeData( - // entityLineage.nodes || [], - // edge, - // entityLineage.entity, - // selectedEntity - // ), - // true - // ); - // }); + edgesToRemove.forEach((edge) => { + removeEdgeHandler(edge, true); + }); setNodes( (previousNodes) => @@ -432,26 +426,15 @@ const LineageProvider = ({ children }: LineageProviderProps) => { return; } - const columnConnection = !isNil(sourceHandle) && !isNil(targetHandle); + const columnConnection = + source !== sourceHandle && target !== targetHandle; setStatus('waiting'); setLoading(true); - // const nodes = [ - // ...(entityLineage.nodes as EntityReference[]), - // entityLineage.entity, - // ]; - const targetNode = nodes?.find((n) => target === n.id); const sourceNode = nodes?.find((n) => source === n.id); - // if (isUndefined(targetNode) && sourceNode?.id !== selectedNode?.id) { - // targetNode = getSourceOrTargetNode(target || ''); - // } - // if (isUndefined(sourceNode) && targetNode?.id !== selectedNode?.id) { - // sourceNode = getSourceOrTargetNode(source || ''); - // } - if (!isUndefined(sourceNode) && !isUndefined(targetNode)) { const { id: sourceId, @@ -464,21 +447,42 @@ const LineageProvider = ({ children }: LineageProviderProps) => { fullyQualifiedName: targetFqn, } = targetNode.data.node; - const newEdgeWithFqn = { + const currentEdge = (entityLineage.edges ?? []).find( + (edge) => edge.fromEntity.id === source && edge.toEntity.id === target + ); + + const newEdgeWithFqn: { edge: EdgeDetails } = { edge: { fromEntity: { id: sourceId, type: sourceType, fqn: sourceFqn }, toEntity: { id: targetId, type: targetType, fqn: targetFqn }, + sqlQuery: '', }, }; - // Create another variable without the fqn field - const newEdgeWithoutFqn = { + const newEdgeWithoutFqn: { edge: EntitiesEdge } = { edge: { fromEntity: { id: sourceId, type: sourceType }, toEntity: { id: targetId, type: targetType }, + lineageDetails: { + sqlQuery: '', + columnsLineage: [], + }, }, }; + if (columnConnection) { + if (!isUndefined(currentEdge)) { + const colsData = [ + ...(currentEdge.columns ?? []), + { fromColumns: [sourceHandle], toColumn: targetHandle }, + ] as ColumnLineage[]; + if (newEdgeWithoutFqn.edge.lineageDetails) { + newEdgeWithoutFqn.edge.lineageDetails.columnsLineage = colsData; + } + newEdgeWithFqn.edge.columns = colsData; + } + } + addLineageHandler(newEdgeWithoutFqn) .then(() => { if (!entityLineage) { @@ -562,7 +566,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { ...el, connectable: true, selectable: true, - id: nodeId, + id: selectedEntity.id, data: { saved: false, node: selectedEntity, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts index cc258bdff6d7..c74f30f7132f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts @@ -141,6 +141,7 @@ export const createEdges = (nodes: EntityReference[], edges: EdgeDetails[]) => { target: edge.toEntity.id, targetHandle: toColumn, sourceHandle: fromColumn, + style: { strokeWidth: '2px' }, type: 'buttonedge', markerEnd: { type: MarkerType.ArrowClosed, From dd6ee2a7273579162d3367c8cbde0a03ed7c6d9e Mon Sep 17 00:00:00 2001 From: karanh37 Date: Thu, 14 Dec 2023 19:10:00 +0530 Subject: [PATCH 10/44] fix: center on load --- .../Entity/EntityLineage/CustomNodeV1.component.tsx | 2 +- .../ui/src/components/LineageProvider/LineageProvider.tsx | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx index f1195c8ab4a6..613fbb758c48 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx @@ -59,7 +59,7 @@ const CustomNodeV1 = (props: NodeProps) => { } = useLineageProvider(); /* eslint-disable-next-line */ - const { label, isNewNode, saved, node = {} } = data; + const { label, isNewNode, node = {} } = data; const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type; const isSelected = selectedNode === node; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index ad121d3f4e0b..76d842a249e1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -359,7 +359,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }; const onInitReactFlow = (reactFlowInstance: ReactFlowInstance) => { - onLoad(reactFlowInstance); + setTimeout(() => { + onLoad(reactFlowInstance); + }, 500); + setReactFlowInstance(reactFlowInstance); }; From 79f5357f3ab722b953edb7a76860538f8ea517eb Mon Sep 17 00:00:00 2001 From: karanh37 Date: Mon, 18 Dec 2023 11:48:11 +0530 Subject: [PATCH 11/44] fix: add remove edges --- .../AppPipelineModel/AddPipeLineModal.tsx | 80 ++++-- .../EntityLineage/CustomEdge.component.tsx | 22 +- .../LineageProvider/LineageProvider.tsx | 236 ++++++++++++++++-- .../ui/src/utils/EntityLineageUtils.tsx | 54 ++-- .../resources/ui/src/utils/LineageV1Utils.ts | 27 ++ 5 files changed, 342 insertions(+), 77 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx index 843ce506cf6d..c22df4e72333 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx @@ -12,44 +12,78 @@ */ import { Button, Input, Modal } from 'antd'; +import { AxiosError } from 'axios'; import classNames from 'classnames'; import { t } from 'i18next'; -import { isEmpty, isUndefined } from 'lodash'; -import React, { useMemo } from 'react'; +import { isUndefined } from 'lodash'; +import React, { useEffect, useMemo, useState } from 'react'; +import { Edge } from 'reactflow'; +import { PAGE_SIZE } from '../../../../constants/constants'; import { ERROR_PLACEHOLDER_TYPE, SIZE } from '../../../../enums/common.enum'; +import { EntityType } from '../../../../enums/entity.enum'; +import { SearchIndex } from '../../../../enums/search.enum'; import { EntityReference } from '../../../../generated/entity/type'; -import { getEntityName } from '../../../../utils/EntityUtils'; +import { searchData } from '../../../../rest/miscAPI'; +import { + getEntityName, + getEntityReferenceFromEntity, +} from '../../../../utils/EntityUtils'; import Fqn from '../../../../utils/Fqn'; import { getEntityIcon } from '../../../../utils/TableUtils'; +import { showErrorToast } from '../../../../utils/ToastUtils'; import ErrorPlaceHolder from '../../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import '../../../FeedEditor/feed-editor.less'; import './add-pipeline-modal.less'; interface AddPipeLineModalType { showAddEdgeModal: boolean; - edgeSearchValue: string; - selectedEdgeId: string | undefined; - edgeOptions: EntityReference[]; + selectedEdge?: Edge; onModalCancel: () => void; - onSave: () => void; + onSave: (value?: EntityReference) => void; onRemoveEdgeClick: (evt: React.MouseEvent) => void; - onSearch: (value: string) => void; - onSelect: (value: string) => void; } const AddPipeLineModal = ({ showAddEdgeModal, - edgeOptions, - edgeSearchValue, - selectedEdgeId, + selectedEdge, onRemoveEdgeClick, onModalCancel, onSave, - onSearch, - onSelect, }: AddPipeLineModalType) => { + const currentPipeline = selectedEdge?.data.edge.pipeline; + const [edgeSearchValue, setEdgeSearchValue] = useState(''); + const [edgeSelection, setEdgeSelection] = useState( + currentPipeline ?? {} + ); + const [edgeOptions, setEdgeOptions] = useState([]); + + const getSearchResults = async (value = '*') => { + try { + const data = await searchData(value, 1, PAGE_SIZE, '', '', '', [ + SearchIndex.PIPELINE, + SearchIndex.STORED_PROCEDURE, + ]); + + const edgeOptions = data.data.hits.hits.map((hit) => + getEntityReferenceFromEntity( + hit._source, + hit._source.entityType as EntityType + ) + ); + + setEdgeOptions(edgeOptions); + } catch (error) { + showErrorToast( + error as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.suggestion-lowercase-plural'), + }) + ); + } + }; + const errorPlaceholderEdge = useMemo(() => { - if (isEmpty(edgeOptions)) { + if (isUndefined(selectedEdge)) { if (edgeSearchValue) { return ( { + getSearchResults(edgeSearchValue); + }, [edgeSearchValue]); return ( + onClick={() => onSave(edgeSelection)}> {t('label.save')} , ]} maskClosable={false} open={showAddEdgeModal} - title={t(`label.${isUndefined(selectedEdgeId) ? 'add' : 'edit'}-entity`, { + title={t(`label.${isUndefined(selectedEdge) ? 'add' : 'edit'}-entity`, { entity: t('label.edge'), })} onCancel={onModalCancel}> @@ -98,7 +136,7 @@ const AddPipeLineModal = ({ data-testid="field-input" placeholder={t('message.search-for-edge')} value={edgeSearchValue} - onChange={(e) => onSearch(e.target.value)} + onChange={(e) => setEdgeSearchValue(e.target.value)} />
@@ -109,10 +147,10 @@ const AddPipeLineModal = ({ return (
onSelect(item.id)}> + onClick={() => setEdgeSelection(item)}>
{icon}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx index 9e81a1b6488a..033a8bc4699a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx @@ -62,11 +62,16 @@ export const CustomEdge = ({ data, selected, }: EdgeProps) => { - const { onEdgeClick, addPipelineClick, edge, isColumnLineage, ...rest } = - data; + const { edge, isColumnLineage, ...rest } = data; const offset = 4; - const { tracedNodes, tracedColumns } = useLineageProvider(); + const { + tracedNodes, + tracedColumns, + isEditMode, + onAddPipelineClick, + onColumnEdgeRemove, + } = useLineageProvider(); const isColumnHighlighted = useMemo(() => { if (!isColumnLineage) { @@ -146,7 +151,7 @@ export const CustomEdge = ({ return false; }, [isColumnLineage, data]); - const isSelectedEditMode = selected && data.isEditMode; + const isSelectedEditMode = selected && isEditMode; const isSelected = selected; const getInvisiblePath = (path: string) => { @@ -170,10 +175,7 @@ export const CustomEdge = ({ className="flex-center custom-edge-pipeline-button" data-testid={dataTestId} icon={icon} - onClick={(event) => - data.isEditMode && - data.addPipelineClick?.(event, rest as CustomEdgeData) - } + onClick={() => isEditMode && onAddPipelineClick()} /> ); @@ -237,11 +239,11 @@ export const CustomEdge = ({ )} {isColumnLineageAllowed && isSelectedEditMode && - getEditLineageIcon('add-pipeline', true, addPipelineClick)} + getEditLineageIcon('add-pipeline', true, onAddPipelineClick)} {!isColumnLineageAllowed && isSelectedEditMode && isSelected && - getEditLineageIcon('delete-button', false, onEdgeClick)} + getEditLineageIcon('delete-button', false, onColumnEdgeRemove)} {!isColumnLineageAllowed && data.columnFunctionValue && data.isExpanded && diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index 76d842a249e1..b0a8ab2e2c20 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button } from 'antd'; +import { Button, Modal } from 'antd'; import { AxiosError } from 'axios'; import { isEqual, isUndefined, uniqueId, uniqWith } from 'lodash'; import { LoadingState } from 'Models'; @@ -34,13 +34,19 @@ import { useEdgesState, useNodesState, } from 'reactflow'; -import { ZOOM_VALUE } from '../../constants/Lineage.constants'; +import { + ELEMENT_DELETE_STATE, + ZOOM_VALUE, +} from '../../constants/Lineage.constants'; import { mockDatasetData } from '../../constants/mockTourData.constants'; import { EntityLineageDirection, EntityLineageNodeType, } from '../../enums/entity.enum'; -import { EntitiesEdge } from '../../generated/api/lineage/addLineage'; +import { + AddLineage, + EntitiesEdge, +} from '../../generated/api/lineage/addLineage'; import { ColumnLineage, EntityReference, @@ -52,17 +58,26 @@ import { getAllTracedNodes, getClassifiedEdge, getLayoutedElements, + getLoadingStatusValue, + getModalBodyText, + getNewLineageConnectionDetails, getUniqueFlowElements, onLoad, removeLineageHandler, } from '../../utils/EntityLineageUtils'; -import { createEdges, createNodes } from '../../utils/LineageV1Utils'; +import { + createEdges, + createNodes, + getColumnLineageData, +} from '../../utils/LineageV1Utils'; import SVGIcons from '../../utils/SvgUtils'; import { showErrorToast } from '../../utils/ToastUtils'; import EdgeInfoDrawer from '../Entity/EntityInfoDrawer/EdgeInfoDrawer.component'; import EntityInfoDrawer from '../Entity/EntityInfoDrawer/EntityInfoDrawer.component'; +import AddPipeLineModal from '../Entity/EntityLineage/AppPipelineModel/AddPipeLineModal'; import { EdgeData, + ElementLoadingState, LineageConfig, } from '../Entity/EntityLineage/EntityLineage.interface'; import EntityLineageSidebar from '../Entity/EntityLineage/EntityLineageSidebar.component'; @@ -91,6 +106,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { {} as SourceType ); const [selectedColumn, setSelectedColumn] = useState(''); + const [showAddEdgeModal, setShowAddEdgeModal] = useState(false); const [expandedNodes, setExpandedNodes] = useState([]); const [expandAllColumns, setExpandAllColumns] = useState(false); const [selectedEdge, setSelectedEdge] = useState(); @@ -99,6 +115,11 @@ const LineageProvider = ({ children }: LineageProviderProps) => { edges: [], entity: {} as EntityReference, }); + const [showDeleteModal, setShowDeleteModal] = useState(false); + const [deletionState, setDeletionState] = useState<{ + loading: boolean; + status: ElementLoadingState; + }>(ELEMENT_DELETE_STATE); const [nodes, setNodes, onNodesChange] = useNodesState([]); const [edges, setEdges, onEdgesChange] = useEdgesState([]); const [loading, setLoading] = useState(false); @@ -233,7 +254,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => { [nodes, edges] ); - const removeEdgeHandler = (edge: Edge, confirmDelete: boolean) => { + const removeEdgeHandler = async ( + edge: Edge, + confirmDelete: boolean + ): Promise => { if (confirmDelete && entityLineage) { const { data, id } = edge; const edgeData: EdgeData = { @@ -242,14 +266,56 @@ const LineageProvider = ({ children }: LineageProviderProps) => { toEntity: data.edge.toEntity.type, toId: data.edge.toEntity.id, }; - removeLineageHandler(edgeData); + + await removeLineageHandler(edgeData); setEdges((prevEdges) => { - return prevEdges.filter((edge) => { - const isRemovedEdge = edge.id === id; + return prevEdges.filter((e) => e.id !== id); + }); + } + }; - return !isRemovedEdge; - }); + const removeColumnEdge = async (edge: Edge, confirmDelete: boolean) => { + if (confirmDelete && entityLineage) { + const { data, id } = edge; + const selectedEdge: AddLineage = { + edge: { + fromEntity: { + id: data.edge.fromEntity.id, + type: data.edge.fromEntity.type, + }, + toEntity: { + id: data.edge.toEntity.id, + type: data.edge.toEntity.type, + }, + }, + }; + + const updatedCols = getColumnLineageData(data.edge.columns, edge); + + await addLineageHandler(selectedEdge); + setEntityLineage((prev) => { + return { + ...prev, + edges: (prev.edges ?? []).map((obj) => { + if ( + obj.fromEntity.id === data.edge.fromEntity.id && + obj.toEntity.id === data.edge.toEntity.id + ) { + return { + ...obj, + columns: updatedCols, + }; + } + + return obj; + }), + }; }); + + setEdges((pre) => { + return pre.filter((e) => e.id !== id); + }); + setShowDeleteModal(false); } }; @@ -418,6 +484,24 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }); }; + const onRemove = useCallback(async () => { + try { + setDeletionState({ ...ELEMENT_DELETE_STATE, loading: true }); + + if (selectedEdge?.data?.isColumnLineage) { + await removeColumnEdge(selectedEdge, true); + } else { + await removeEdgeHandler(selectedEdge as Edge, true); + } + + setShowDeleteModal(false); + } catch (err) { + showErrorToast(err as AxiosError); + } finally { + setDeletionState((pre) => ({ ...pre, status: 'initial' })); + } + }, [selectedEdge, setShowDeleteModal]); + const onConnect = useCallback( (params: Edge | Connection) => { if (!entityLineage) { @@ -541,6 +625,15 @@ const LineageProvider = ({ children }: LineageProviderProps) => { [selectedNode, entityLineage, nodes] ); + const onAddPipelineClick = useCallback(() => { + setShowAddEdgeModal(true); + }, []); + + const handleModalCancel = useCallback(() => { + setShowAddEdgeModal(false); + setSelectedEdge({} as Edge); + }, []); + const onEntitySelect = (selectedEntity: EntityReference, nodeId: string) => { const isExistingNode = nodes.some( (n) => @@ -583,16 +676,97 @@ const LineageProvider = ({ children }: LineageProviderProps) => { } }; + const onAddPipelineModalSave = useCallback( + async (pipelineData?: EntityReference) => { + if (!selectedEdge || !entityLineage) { + return; + } + + setStatus('waiting'); + setLoading(true); + + const { source, target } = selectedEdge.data; + const existingEdge = (entityLineage.edges ?? []).find( + (ed) => ed.fromEntity === source && ed.toEntity === target + ); + + let edgeIndex = -1; + if (existingEdge) { + edgeIndex = (entityLineage.edges ?? []).indexOf(existingEdge); + + if (pipelineData) { + existingEdge.pipeline = pipelineData; + } + } + + const { newEdge, updatedLineageDetails } = getNewLineageConnectionDetails( + selectedEdge, + pipelineData + ); + + try { + await addLineageHandler(newEdge); + + setStatus('success'); + setLoading(false); + + setEntityLineage((pre) => { + if (!selectedEdge.data || !pre) { + return pre; + } + + const newEdges = [...(pre.edges ?? [])]; + + if (newEdges[edgeIndex]) { + newEdges[edgeIndex] = existingEdge as EdgeDetails; + } + + const newData = { + ...pre, + edges: newEdges, + }; + + return newData; + }); + + setEdges((pre) => + pre.map((edge) => { + if (edge.id === selectedEdge.id) { + return { + ...edge, + animated: true, + data: { + edge: { + ...edge.data.edge, + pipeline: updatedLineageDetails.pipeline, + }, + }, + }; + } + + return edge; + }) + ); + } catch (error) { + setLoading(false); + } finally { + setStatus('initial'); + handleModalCancel(); + } + }, + [selectedEdge, entityLineage] + ); + + const onColumnEdgeRemove = useCallback(() => { + setShowDeleteModal(true); + }, []); + useEffect(() => { if (entityFqn) { fetchLineageData(entityFqn, lineageConfig); } }, [lineageConfig, entityFqn, queryFilter]); - useEffect(() => { - // filter out the unsaved nodes - }, []); - const activityFeedContextValues = useMemo(() => { return { isDrawerOpen, @@ -628,8 +802,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => { removeNodeHandler, onNodeClick, onEdgeClick, + onColumnEdgeRemove, onLineageConfigUpdate, onLineageEditClick, + onAddPipelineClick, }; }, [ isDrawerOpen, @@ -665,8 +841,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => { removeNodeHandler, onNodeClick, onEdgeClick, + onColumnEdgeRemove, onLineageConfigUpdate, onLineageEditClick, + onAddPipelineClick, ]); return ( @@ -695,6 +873,36 @@ const LineageProvider = ({ children }: LineageProviderProps) => { onCancel={() => setIsDrawerOpen(false)} /> ))} + + {showDeleteModal && ( + { + setShowDeleteModal(false); + }} + onOk={onRemove}> + {getModalBodyText(selectedEdge as Edge)} + + )} + {showAddEdgeModal && ( + { + setShowDeleteModal(true); + setShowAddEdgeModal(false); + }} + onSave={onAddPipelineModalSave} + /> + )} ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index 277d67d6b18a..5203389fa97a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -62,6 +62,7 @@ import { SelectedNode, } from '../components/Entity/EntityLineage/EntityLineage.interface'; import { ExploreSearchIndex } from '../components/Explore/ExplorePage.interface'; +import { EdgeDetails } from '../components/Lineage/Lineage.interface'; import Loader from '../components/Loader/Loader'; import { getContainerDetailPath, @@ -461,36 +462,34 @@ export const getLayoutedElements = ( return { node: uNode, edge: edgesRequired }; }; -export const getModalBodyText = (selectedEdge: SelectedEdge) => { - const { data, source, target } = selectedEdge; +export const getModalBodyText = (selectedEdge: Edge) => { + const { data } = selectedEdge; + const { fromEntity, toEntity } = data.edge as EdgeDetails; const { isColumnLineage } = data as CustomEdgeData; let sourceEntity = ''; let targetEntity = ''; - const sourceFQN = isColumnLineage - ? data?.sourceHandle - : source.fullyQualifiedName; - const targetFQN = isColumnLineage - ? data?.targetHandle - : target.fullyQualifiedName; + const sourceFQN = isColumnLineage ? data?.sourceHandle : fromEntity.fqn; + + const targetFQN = isColumnLineage ? data?.targetHandle : toEntity.fqn; const fqnPart = isColumnLineage ? FqnPart.Column : FqnPart.Table; - if (source.type === EntityType.TABLE) { + if (fromEntity.type === EntityType.TABLE) { sourceEntity = getPartialNameFromTableFQN(sourceFQN || '', [fqnPart]); } else { sourceEntity = getPartialNameFromFQN(sourceFQN || '', ['database']); } - if (target.type === EntityType.TABLE) { + if (toEntity.type === EntityType.TABLE) { targetEntity = getPartialNameFromTableFQN(targetFQN || '', [fqnPart]); } else { targetEntity = getPartialNameFromFQN(targetFQN || '', ['database']); } return t('message.remove-edge-between-source-and-target', { - sourceDisplayName: source.displayName ? source.displayName : sourceEntity, - targetDisplayName: target.displayName ? target.displayName : targetEntity, + sourceDisplayName: sourceEntity, + targetDisplayName: targetEntity, }); }; @@ -825,35 +824,26 @@ export const getUpdatedEdgeWithPipeline = ( }; export const getNewLineageConnectionDetails = ( - selectedEdgeValue: EntityLineageEdge | undefined, - selectedPipelineId: string | undefined, - customEdgeData: CustomEdgeData, - type: EntityType, - edgeDetails?: EntityReference + selectedEdgeValue: Edge | undefined, + selectedPipeline: EntityReference | undefined ) => { - const { source, sourceType, target, targetType } = customEdgeData; + const { fromEntity, toEntity, sqlQuery, columns } = + selectedEdgeValue?.data.edge; const updatedLineageDetails: LineageDetails = { - ...selectedEdgeValue?.lineageDetails, - sqlQuery: selectedEdgeValue?.lineageDetails?.sqlQuery || '', - columnsLineage: selectedEdgeValue?.lineageDetails?.columnsLineage || [], - pipeline: isUndefined(selectedPipelineId) - ? undefined - : { - id: selectedPipelineId, - type, - fullyQualifiedName: edgeDetails?.fullyQualifiedName ?? '', - }, + sqlQuery: sqlQuery ?? '', + columnsLineage: columns ?? [], + pipeline: selectedPipeline, }; const newEdge: AddLineage = { edge: { fromEntity: { - id: source, - type: sourceType, + id: fromEntity.id, + type: fromEntity.type, }, toEntity: { - id: target, - type: targetType, + id: toEntity.id, + type: toEntity.type, }, lineageDetails: updatedLineageDetails, }, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts index c74f30f7132f..9ef494bdd806 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts @@ -21,6 +21,7 @@ import dagre from 'dagre'; import { isNil, isUndefined } from 'lodash'; import { EdgeDetails } from '../components/Lineage/Lineage.interface'; import { NODE_HEIGHT, NODE_WIDTH } from '../constants/Lineage.constants'; +import { ColumnLineage } from '../generated/type/entityLineage'; export const checkUpstreamDownstream = (id: string, data: EdgeDetails[]) => { const hasUpstream = data.some((edge: EdgeDetails) => edge.toEntity.id === id); @@ -177,3 +178,29 @@ export const createEdges = (nodes: EntityReference[], edges: EdgeDetails[]) => { return lineageEdgesV1; }; + +export const getColumnLineageData = ( + columnsData: ColumnLineage[], + data: Edge +) => { + const columnsLineage = columnsData?.reduce((col, curr) => { + if (curr.toColumn === data.data?.targetHandle) { + const newCol = { + ...curr, + fromColumns: + curr.fromColumns?.filter( + (column) => column !== data.data?.sourceHandle + ) ?? [], + }; + if (newCol.fromColumns?.length) { + return [...col, newCol]; + } else { + return col; + } + } + + return [...col, curr]; + }, [] as ColumnLineage[]); + + return columnsLineage; +}; From 563c07e082c3ee063394d663c7b7de53ffdb65bf Mon Sep 17 00:00:00 2001 From: karanh37 Date: Mon, 18 Dec 2023 21:39:30 +0530 Subject: [PATCH 12/44] fix: column lineage issues --- .../Entity/EntityLineage/custom-node.less | 22 ++- .../EntityLineage/entity-lineage.style.less | 6 + .../LineageProvider/LineageProvider.tsx | 160 +++++++++--------- .../resources/ui/src/utils/LineageV1Utils.ts | 63 ++++++- 4 files changed, 166 insertions(+), 85 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less index b556b977bee3..22eead0b6695 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less @@ -21,8 +21,8 @@ } .lineage-expand-icon { - width: 12px; - height: 12px; + width: 16px; + height: 16px; color: inherit; } @@ -117,6 +117,9 @@ } .lineage-node-handle { border-color: @primary-color; + svg { + color: @primary-color; + } } .label-container { background: @primary-color-hover; @@ -133,21 +136,24 @@ .react-flow { .lineage-node-handle.react-flow__handle-left { - left: -10px; + left: -18px; } .lineage-node-handle.react-flow__handle-right { - right: -10px; + right: -18px; } } .react-flow .lineage-node-handle { - width: 20px; - min-width: 20px; - height: 20px; + width: 35px; + min-width: 35px; + height: 35px; border-radius: 50%; border-color: @lineage-border; background: @white; - top: 38px; // Need to show handles on top half + top: 43px; // Need to show handles on top half + svg { + color: @text-grey-muted; + } } .react-flow .lineage-column-node-handle { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less index f0d9cba72f85..3e5dda972d01 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less @@ -36,6 +36,12 @@ .react-flow__handle { border-color: @primary-color; } + .lineage-node-handle { + border: 1px solid @primary-color; + svg { + color: @primary-color; + } + } } .custom-node-header-active { border-color: @primary-color; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index b0a8ab2e2c20..a6e99f4a4d1d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -43,10 +43,7 @@ import { EntityLineageDirection, EntityLineageNodeType, } from '../../enums/entity.enum'; -import { - AddLineage, - EntitiesEdge, -} from '../../generated/api/lineage/addLineage'; +import { AddLineage } from '../../generated/api/lineage/addLineage'; import { ColumnLineage, EntityReference, @@ -69,6 +66,9 @@ import { createEdges, createNodes, getColumnLineageData, + getLineageDetailsObject, + getLineageEdge, + getLineageEdgeForAPI, } from '../../utils/LineageV1Utils'; import SVGIcons from '../../utils/SvgUtils'; import { showErrorToast } from '../../utils/ToastUtils'; @@ -276,7 +276,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const removeColumnEdge = async (edge: Edge, confirmDelete: boolean) => { if (confirmDelete && entityLineage) { - const { data, id } = edge; + const { data } = edge; const selectedEdge: AddLineage = { edge: { fromEntity: { @@ -291,30 +291,39 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }; const updatedCols = getColumnLineageData(data.edge.columns, edge); + selectedEdge.edge.lineageDetails = getLineageDetailsObject(edge); + selectedEdge.edge.lineageDetails.columnsLineage = updatedCols; await addLineageHandler(selectedEdge); + + const updatedEdgeWithColumns = (entityLineage.edges ?? []).map((obj) => { + if ( + obj.fromEntity.id === data.edge.fromEntity.id && + obj.toEntity.id === data.edge.toEntity.id + ) { + return { + ...obj, + columns: updatedCols, + }; + } + + return obj; + }); + setEntityLineage((prev) => { return { ...prev, - edges: (prev.edges ?? []).map((obj) => { - if ( - obj.fromEntity.id === data.edge.fromEntity.id && - obj.toEntity.id === data.edge.toEntity.id - ) { - return { - ...obj, - columns: updatedCols, - }; - } - - return obj; - }), + edges: updatedEdgeWithColumns, }; }); - setEdges((pre) => { - return pre.filter((e) => e.id !== id); - }); + const updatedEdges = createEdges( + entityLineage.nodes ?? [], + updatedEdgeWithColumns + ); + + setEdges(updatedEdges); + setShowDeleteModal(false); } }; @@ -396,14 +405,14 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setQueryFilter(query); }, []); - const onNodeClick = (node: Node) => { + const onNodeClick = useCallback((node: Node) => { if (node) { setSelectedEdge(undefined); setSelectedNode(node.data.node as SourceType); setIsDrawerOpen(true); handleLineageTracing(node); } - }; + }, []); const onPaneClick = useCallback(() => { setIsDrawerOpen(false); @@ -412,17 +421,17 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setSelectedNode({} as SourceType); }, []); - const onEdgeClick = (edge: Edge) => { + const onEdgeClick = useCallback((edge: Edge) => { setSelectedEdge(edge); setSelectedNode({} as SourceType); setIsDrawerOpen(true); - }; + }, []); - const onLineageEditClick = () => { + const onLineageEditClick = useCallback(() => { setIsEditMode((pre) => !pre); setSelectedNode({} as SourceType); setIsDrawerOpen(false); - }; + }, []); const onInitReactFlow = (reactFlowInstance: ReactFlowInstance) => { setTimeout(() => { @@ -448,12 +457,13 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setEntityFqn(value); }, []); - const toggleColumnView = () => { + const toggleColumnView = useCallback(() => { const updatedVal = !expandAllColumns; setExpandAllColumns(updatedVal); setNodes((prevNodes) => { const updatedNode = prevNodes.map((node) => { const nodeId = node.data.node.id; + // Update the expandedNodes state based on the toggle value if (updatedVal && !expandedNodes.includes(nodeId)) { setExpandedNodes((prevExpandedNodes) => [ @@ -482,7 +492,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { return node; }); - }; + }, [expandAllColumns, expandedNodes, edges]); const onRemove = useCallback(async () => { try { @@ -504,9 +514,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const onConnect = useCallback( (params: Edge | Connection) => { - if (!entityLineage) { - return; - } const { target, source, sourceHandle, targetHandle } = params; if (target === source) { @@ -523,50 +530,53 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const sourceNode = nodes?.find((n) => source === n.id); if (!isUndefined(sourceNode) && !isUndefined(targetNode)) { - const { - id: sourceId, - entityType: sourceType, - fullyQualifiedName: sourceFqn, - } = sourceNode.data.node; - const { - id: targetId, - entityType: targetType, - fullyQualifiedName: targetFqn, - } = targetNode.data.node; - const currentEdge = (entityLineage.edges ?? []).find( (edge) => edge.fromEntity.id === source && edge.toEntity.id === target ); - const newEdgeWithFqn: { edge: EdgeDetails } = { - edge: { - fromEntity: { id: sourceId, type: sourceType, fqn: sourceFqn }, - toEntity: { id: targetId, type: targetType, fqn: targetFqn }, - sqlQuery: '', - }, - }; + const newEdgeWithFqn = getLineageEdge( + sourceNode.data.node, + targetNode.data.node + ); - const newEdgeWithoutFqn: { edge: EntitiesEdge } = { - edge: { - fromEntity: { id: sourceId, type: sourceType }, - toEntity: { id: targetId, type: targetType }, - lineageDetails: { - sqlQuery: '', - columnsLineage: [], - }, - }, - }; + const newEdgeWithoutFqn = getLineageEdgeForAPI( + sourceNode.data.node, + targetNode.data.node + ); if (columnConnection) { if (!isUndefined(currentEdge)) { - const colsData = [ - ...(currentEdge.columns ?? []), - { fromColumns: [sourceHandle], toColumn: targetHandle }, - ] as ColumnLineage[]; + const updatedColumns: ColumnLineage[] = + currentEdge.columns?.map((lineage) => { + if (lineage.toColumn === targetHandle) { + return { + ...lineage, + fromColumns: [ + ...(lineage.fromColumns ?? []), + sourceHandle ?? '', + ], + }; + } + + return lineage; + }) ?? []; + + if ( + !updatedColumns.find( + (lineage) => lineage.toColumn === targetHandle + ) + ) { + updatedColumns.push({ + fromColumns: [sourceHandle ?? ''], + toColumn: targetHandle ?? '', + }); + } + if (newEdgeWithoutFqn.edge.lineageDetails) { - newEdgeWithoutFqn.edge.lineageDetails.columnsLineage = colsData; + newEdgeWithoutFqn.edge.lineageDetails.columnsLineage = + updatedColumns; } - newEdgeWithFqn.edge.columns = colsData; + currentEdge.columns = updatedColumns; // update current edge with new columns } } @@ -584,10 +594,9 @@ const LineageProvider = ({ children }: LineageProviderProps) => { targetNode?.data.node as EntityReference, ]; - const allEdges = [ - ...(entityLineage.edges ?? []), - newEdgeWithFqn.edge, - ]; + const allEdges = isUndefined(currentEdge) + ? [...(entityLineage.edges ?? []), newEdgeWithFqn.edge] + : entityLineage.edges ?? []; const updatedNodes = createNodes( allNodes as EntityReference[], @@ -610,19 +619,18 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setNodes(updatedNodes); setEdges(updatedEdges); - - setTimeout(() => { - setStatus('initial'); - }, 100); setNewAddedNode({} as Node); }) - .catch(() => { + .catch((err) => { + showErrorToast(err); + }) + .finally(() => { setStatus('initial'); setLoading(false); }); } }, - [selectedNode, entityLineage, nodes] + [selectedNode, entityLineage, nodes, edges] ); const onAddPipelineClick = useCallback(() => { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts index 9ef494bdd806..495b8ff6c9dc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts @@ -20,8 +20,10 @@ import { EntityReference } from '../generated/entity/type'; import dagre from 'dagre'; import { isNil, isUndefined } from 'lodash'; import { EdgeDetails } from '../components/Lineage/Lineage.interface'; +import { SourceType } from '../components/SearchedData/SearchedData.interface'; import { NODE_HEIGHT, NODE_WIDTH } from '../constants/Lineage.constants'; -import { ColumnLineage } from '../generated/type/entityLineage'; +import { EntitiesEdge } from '../generated/api/lineage/addLineage'; +import { ColumnLineage, LineageDetails } from '../generated/type/entityLineage'; export const checkUpstreamDownstream = (id: string, data: EdgeDetails[]) => { const hasUpstream = data.some((edge: EdgeDetails) => edge.toEntity.id === id); @@ -204,3 +206,62 @@ export const getColumnLineageData = ( return columnsLineage; }; + +export const getLineageEdge = ( + sourceNode: SourceType, + targetNode: SourceType +): { edge: EdgeDetails } => { + const { + id: sourceId, + entityType: sourceType, + fullyQualifiedName: sourceFqn, + } = sourceNode; + const { + id: targetId, + entityType: targetType, + fullyQualifiedName: targetFqn, + } = targetNode; + + return { + edge: { + fromEntity: { + id: sourceId, + type: sourceType ?? '', + fqn: sourceFqn ?? '', + }, + toEntity: { id: targetId, type: targetType ?? '', fqn: targetFqn ?? '' }, + sqlQuery: '', + }, + }; +}; + +export const getLineageEdgeForAPI = ( + sourceNode: SourceType, + targetNode: SourceType +): { edge: EntitiesEdge } => { + const { id: sourceId, entityType: sourceType } = sourceNode; + const { id: targetId, entityType: targetType } = targetNode; + + return { + edge: { + fromEntity: { id: sourceId, type: sourceType ?? '' }, + toEntity: { id: targetId, type: targetType ?? '' }, + lineageDetails: { + sqlQuery: '', + columnsLineage: [], + }, + }, + }; +}; + +export const getLineageDetailsObject = (edge: Edge): LineageDetails => { + const { data } = edge; + + return { + sqlQuery: data?.edge?.sqlQuery ?? '', + columnsLineage: data?.edge?.columns ?? [], + description: data?.edge?.description ?? '', + pipeline: data?.edge?.pipeline ?? undefined, + source: data?.edge?.source ?? '', + }; +}; From 6467ddb2471cd942525d4f8bfa514e87756c2097 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 19 Dec 2023 11:52:57 +0530 Subject: [PATCH 13/44] use reactflow controls --- .../EntityInfoDrawer/entity-info-drawer.less | 2 +- .../CustomControls.component.tsx | 110 +----------------- .../EntityLineage/EntityLineage.interface.ts | 5 +- .../Entity/EntityLineage/custom-node.less | 7 +- .../EntityLineage/entity-lineage.style.less | 9 ++ .../components/Lineage/Lineage.component.tsx | 9 +- .../LineageProvider/LineageProvider.tsx | 30 ++++- 7 files changed, 47 insertions(+), 125 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/entity-info-drawer.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/entity-info-drawer.less index fd23f7bb7bc5..cd87d66c1b5a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/entity-info-drawer.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/entity-info-drawer.less @@ -29,7 +29,7 @@ } .entity-panel-container { - margin-top: 60px; + margin-top: 54px; .ant-drawer-header { border-bottom: none; padding-bottom: 0 !important; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx index 5c949ac84690..3c58d36abe53 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx @@ -13,7 +13,6 @@ import { FilterOutlined, SettingOutlined } from '@ant-design/icons'; import { Button, Col, Dropdown, Row, Select, Space } from 'antd'; -import Input from 'antd/lib/input/Input'; import { ItemType } from 'antd/lib/menu/hooks/useItems'; import classNames from 'classnames'; import { isEmpty } from 'lodash'; @@ -26,20 +25,14 @@ import React, { useState, } from 'react'; import { useTranslation } from 'react-i18next'; -import { Node, useReactFlow } from 'reactflow'; +import { Node } from 'reactflow'; import { ReactComponent as ExitFullScreen } from '../../../assets/svg/exit-full-screen.svg'; import { ReactComponent as FullScreen } from '../../../assets/svg/full-screen.svg'; import { ReactComponent as EditIconColor } from '../../../assets/svg/ic-edit-lineage-colored.svg'; import { ReactComponent as EditIcon } from '../../../assets/svg/ic-edit-lineage.svg'; import { PRIMERY_COLOR } from '../../../constants/constants'; import { NO_PERMISSION_FOR_ACTION } from '../../../constants/HelperTextUtil'; -import { - MAX_ZOOM_VALUE, - MIN_ZOOM_VALUE, - ZOOM_BUTTON_STEP, - ZOOM_SLIDER_STEP, - ZOOM_TRANSITION_DURATION, -} from '../../../constants/Lineage.constants'; +import { ZOOM_TRANSITION_DURATION } from '../../../constants/Lineage.constants'; import { SearchIndex } from '../../../enums/search.enum'; import { QueryFieldInterface, @@ -50,7 +43,6 @@ import { handleSearchFilterOption } from '../../../utils/CommonUtils'; import { getLoadingStatusValue } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; import { getSelectedValuesFromQuickFilter } from '../../../utils/Explore.utils'; -import SVGIcons, { Icons } from '../../../utils/SvgUtils'; import { ExploreQuickFilterField } from '../../Explore/ExplorePage.interface'; import ExploreQuickFilters from '../../Explore/ExploreQuickFilters'; import { useLineageProvider } from '../../LineageProvider/LineageProvider'; @@ -59,9 +51,6 @@ import LineageConfigModal from './LineageConfigModal'; const CustomControls: FC = ({ style, - showFitView = true, - showZoom = true, - fitViewParams, className, deleted, hasEditAccess, @@ -69,7 +58,6 @@ const CustomControls: FC = ({ onExitFullScreenViewClick, }: ControlProps) => { const { t } = useTranslation(); - const { fitView, zoomTo } = useReactFlow(); const [dialogVisible, setDialogVisible] = useState(false); const { nodes, @@ -105,41 +93,6 @@ const CustomControls: FC = ({ })); }, [filters]); - const onZoomHandler = useCallback( - (zoomLevel: number) => { - zoomTo?.(zoomLevel, { duration: ZOOM_TRANSITION_DURATION }); - }, - [zoomTo] - ); - - const onZoomInHandler = useCallback(() => { - setZoom((pre) => { - const zoomInValue = pre < MAX_ZOOM_VALUE ? pre + ZOOM_BUTTON_STEP : pre; - onZoomHandler(zoomInValue); - - return zoomInValue; - }); - }, [onZoomHandler]); - - const onZoomOutHandler = useCallback(() => { - setZoom((pre) => { - const zoomOutValue = pre > MIN_ZOOM_VALUE ? pre - ZOOM_BUTTON_STEP : pre; - onZoomHandler(zoomOutValue); - - return zoomOutValue; - }); - }, [onZoomHandler]); - - const onFitViewHandler = useCallback(() => { - fitView?.(fitViewParams); - }, [fitView, fitViewParams]); - - const onRangeChange = (event: React.ChangeEvent) => { - const zoomValue = parseFloat(event.target.value); - onZoomHandler(zoomValue); - setZoom(zoomValue); - }; - useEffect(() => { if (zoomValue !== zoom) { setZoom(zoomValue); @@ -283,7 +236,7 @@ const CustomControls: FC = ({ className={classNames('z-10 w-full', className)} gutter={[8, 8]} style={style}> - + - -
- )} - {showFitView && ( -
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index a6e99f4a4d1d..1d2004f8df92 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -259,7 +259,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { confirmDelete: boolean ): Promise => { if (confirmDelete && entityLineage) { - const { data, id } = edge; + const { data } = edge; const edgeData: EdgeData = { fromEntity: data.edge.fromEntity.type, fromId: data.edge.fromEntity.id, @@ -268,9 +268,33 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }; await removeLineageHandler(edgeData); - setEdges((prevEdges) => { - return prevEdges.filter((e) => e.id !== id); + + const updatedEdges = entityLineage.edges?.filter( + (item) => + !( + item.fromEntity.id === edgeData.fromId && + item.toEntity.id === edgeData.toId + ) + ); + + setEntityLineage((prev) => { + return { + ...prev, + edges: updatedEdges, + }; }); + + const newNodes = createNodes( + entityLineage.nodes ?? [], + updatedEdges ?? [] + ); + const newEdges = createEdges( + entityLineage.nodes ?? [], + updatedEdges ?? [] + ); + + setNodes(newNodes); + setEdges(newEdges); } }; From 06451f2b1df15302119a66b500c9c587e5e59c98 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 19 Dec 2023 16:19:04 +0530 Subject: [PATCH 14/44] add collapse --- .../EntityLineage/CustomNodeV1.component.tsx | 91 ++++++++++++- .../Entity/EntityLineage/custom-node.less | 4 +- .../LineageProvider/LineageProvider.tsx | 25 ++++ .../resources/ui/src/utils/LineageV1Utils.ts | 127 +++++++++++++++++- 4 files changed, 233 insertions(+), 14 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx index 613fbb758c48..84b8a45b82b7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx @@ -25,6 +25,7 @@ import { Position, useUpdateNodeInternals, } from 'reactflow'; +import { ReactComponent as MinusIcon } from '../../../assets/svg/control-minus.svg'; import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-outlined.svg'; import { BORDER_COLOR } from '../../../constants/constants'; import { EntityLineageNodeType, EntityType } from '../../../enums/entity.enum'; @@ -37,7 +38,7 @@ import { useLineageProvider } from '../../LineageProvider/LineageProvider'; import './custom-node.less'; import { getColumnHandle } from './CustomNode.utils'; import './entity-lineage.style.less'; -import { ModifiedColumn } from './EntityLineage.interface'; +import { EdgeTypeEnum, ModifiedColumn } from './EntityLineage.interface'; import LineageNodeLabelV1 from './LineageNodeLabelV1'; const CustomNodeV1 = (props: NodeProps) => { @@ -54,6 +55,7 @@ const CustomNodeV1 = (props: NodeProps) => { nodes, edges, onColumnClick, + onNodeCollapse, removeNodeHandler, loadChildNodesHandler, } = useLineageProvider(); @@ -70,10 +72,9 @@ const CustomNodeV1 = (props: NodeProps) => { const [isExpanded, setIsExpanded] = useState(false); const [isTraced, setIsTraced] = useState(false); - const { hasDownstream, hasUpstream } = checkUpstreamDownstream( - id, - lineage ?? [] - ); + const { hasDownstream, hasUpstream } = useMemo(() => { + return checkUpstreamDownstream(id, lineage ?? []); + }, [id, lineage]); const isLeafNode = useMemo(() => { return ( @@ -82,6 +83,14 @@ const CustomNodeV1 = (props: NodeProps) => { ); }, [nodes, edges]); + // console.log( + // node.fullyQualifiedName, + // nodeType, + // isLeafNode, + // hasUpstream, + // hasDownstream + // ); + const supportsColumns = useMemo(() => { if (node && node.entityType === EntityType.TABLE) { return true; @@ -168,6 +177,18 @@ const CustomNodeV1 = (props: NodeProps) => { }} /> )} + {hasDownstream && !isLeafNode && !isEditMode && ( + { } catch (err) { showErrorToast(err as AxiosError); } finally { - setDeletionState((pre) => ({ ...pre, status: 'initial' })); + setDeletionState((pre) => ({ + ...pre, + status: 'initial', + loading: false, + })); } }, [selectedEdge, setShowDeleteModal]); diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts index aaef2b9da4d4..9f681e011a8f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts @@ -71,3 +71,9 @@ export const ELEMENT_DELETE_STATE = { loading: false, status: 'initial' as ElementLoadingState, }; + +export const LINEAGE_DEFAULT_QUICK_FILTERS = [ + 'domain.displayName.keyword', + 'owner.displayName.keyword', + 'tags.tagFQN', +]; From 6e515862aa0626f87446d8771b138b18b29d4d2e Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 19 Dec 2023 19:28:05 +0530 Subject: [PATCH 18/44] add includeDeleted in API --- .../src/main/resources/ui/src/rest/lineageAPI.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts index edd171548ebb..121f0d8339eb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts @@ -43,7 +43,16 @@ export const getLineageDataByFQN = async ( ) => { const { upstreamDepth = 1, downstreamDepth = 1 } = config ?? {}; const response = await APIClient.get( - `/search/getLineage?fqn=${fqn}&upstreamDepth=${upstreamDepth}&downstreamDepth=${downstreamDepth}&query_filter=${queryFilter}` + `/search/getLineage`, + { + params: { + fqn, + upstreamDepth, + downstreamDepth, + query_filter: queryFilter, + includeDeleted: false, + }, + } ); return response.data; From 00369519d0df850acb845a3844f6520e1e0f8852 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 20 Dec 2023 12:41:50 +0530 Subject: [PATCH 19/44] cleanup --- .../DashboardDetails.test.tsx | 9 +- .../AddPipeLineModal.test.tsx | 4 +- .../EntityLineage/CustomControls.test.tsx | 61 +- .../CustomEdge.component.test.tsx | 41 +- .../Entity/EntityLineage/CustomNode.utils.tsx | 112 -- .../EntityLineage/CustomNodeV1.component.tsx | 56 +- .../EntityLineage/EntityLineage.component.tsx | 1790 ----------------- .../EntityLineage/EntityLineage.interface.ts | 1 - .../Entitylineage.component.test.tsx | 182 -- .../LineageProvider/LineageProvider.tsx | 21 +- .../MlModelDetail.component.test.tsx | 9 +- .../PipelineDetails/PipelineDetails.test.tsx | 2 +- .../TasksDAGView/TasksDAGView.test.tsx | 4 - .../TopicDetails/TopicDetails.test.tsx | 9 +- .../ContainerPage/ContainerPage.test.tsx | 13 +- .../StoredProcedurePage.test.tsx | 20 +- .../TableDetailsPageV1.test.tsx | 9 +- .../ui/src/utils/EntityLineageUtils.test.tsx | 227 +-- .../ui/src/utils/EntityLineageUtils.tsx | 856 +------- 19 files changed, 110 insertions(+), 3316 deletions(-) delete mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.component.tsx delete mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/Entitylineage.component.test.tsx diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.test.tsx index 7fa6e9d762e1..3b6100629b0b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DashboardDetails/DashboardDetails.test.tsx @@ -124,12 +124,9 @@ jest.mock('../FeedEditor/FeedEditor', () => { return jest.fn().mockReturnValue(

FeedEditor

); }); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest.fn().mockReturnValue(

Lineage

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockReturnValue(

Lineage

); +}); jest.mock('../common/CustomPropertyTable/CustomPropertyTable', () => ({ CustomPropertyTable: jest .fn() diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.test.tsx index 7f4c5c9e1cb8..7c222411fe05 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.test.tsx @@ -52,9 +52,7 @@ describe('Test CustomEdge Component', () => { }); it('CTA should work properly', async () => { - render( - - ); + render(); const removeEdge = await screen.findByTestId('remove-edge-button'); const saveButton = await screen.findByTestId('save-button'); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.test.tsx index 7e430f675df6..a3564f4d9e08 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.test.tsx @@ -16,22 +16,20 @@ import { LOADING_STATE } from '../../../enums/common.enum'; import { MOCK_LINEAGE_DATA } from '../../../mocks/Lineage.mock'; import CustomControlsComponent from './CustomControls.component'; -const mockFitView = jest.fn(); -const mockZoomTo = jest.fn(); const mockOnOptionSelect = jest.fn(); const mockOnLineageConfigUpdate = jest.fn(); -const mockOnEditLinageClick = jest.fn(); +const mockOnEditLineageClick = jest.fn(); const mockOnExpandColumnClick = jest.fn(); const mockHandleFullScreenViewClick = jest.fn(); const mockOnExitFullScreenViewClick = jest.fn(); const mockOnZoomHandler = jest.fn(); const mockZoomValue = 1; +jest.mock('../../Explore/ExploreQuickFilters', () => + jest.fn().mockReturnValue(

ExploreQuickFilters

) +); + jest.mock('reactflow', () => ({ - useReactFlow: () => ({ - fitView: mockFitView, - zoomTo: mockZoomTo, - }), Position: () => ({ Left: 'left', Top: 'top', @@ -44,13 +42,16 @@ jest.mock('reactflow', () => ({ }), })); +jest.mock('../../LineageProvider/LineageProvider', () => ({ + useLineageProvider: jest.fn().mockImplementation(() => ({ + toggleColumnView: mockOnExpandColumnClick, + onLineageEditClick: mockOnEditLineageClick, + })), +})); + const customProps = { - fitView: mockFitView, - zoomTo: mockZoomTo, onOptionSelect: mockOnOptionSelect, onLineageConfigUpdate: mockOnLineageConfigUpdate, - onEditLinageClick: mockOnEditLinageClick, - onExpandColumnClick: mockOnExpandColumnClick, handleFullScreenViewClick: mockHandleFullScreenViewClick, onExitFullScreenViewClick: mockOnExitFullScreenViewClick, onZoomHandler: mockOnZoomHandler, @@ -74,42 +75,6 @@ describe('CustomControls', () => { jest.clearAllMocks(); }); - it('calls fitView on Fit View button click', () => { - const { getByTestId } = render( - - ); - const fitViewButton = getByTestId('fit-to-screen'); - fireEvent.click(fitViewButton); - - expect(mockFitView).toHaveBeenCalled(); - }); - - it('calls zoomTo with zoomInValue on Zoom In button click', () => { - const { getByTestId } = render( - - ); - const zoomInButton = getByTestId('zoom-in-button'); - fireEvent.click(zoomInButton); - const zoomRangeInput = getByTestId( - 'lineage-zoom-slider' - ) as HTMLInputElement; - - expect(zoomRangeInput.value).toBe('0.75'); - }); - - it('calls zoomTo with zoomOutValue on Zoom Out button click', () => { - const { getByTestId } = render( - - ); - const zoomOutButton = getByTestId('zoom-out-button'); - fireEvent.click(zoomOutButton); - const zoomRangeInput = getByTestId( - 'lineage-zoom-slider' - ) as HTMLInputElement; - - expect(zoomRangeInput.value).toBe('1.25'); - }); - it('calls onEditLinageClick on Edit Lineage button click', () => { const { getByTestId } = render( @@ -117,7 +82,7 @@ describe('CustomControls', () => { const editLineageButton = getByTestId('edit-lineage'); fireEvent.click(editLineageButton); - expect(mockOnEditLinageClick).toHaveBeenCalled(); + expect(mockOnEditLineageClick).toHaveBeenCalled(); }); it('calls onExpandColumnClick on Expand Column button click', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx index 6cd61d7f000b..96cdef6df258 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx @@ -38,32 +38,39 @@ const mockCustomEdgeProp = { sourceType: EntityType.TABLE, targetType: EntityType.DASHBOARD, onEdgeClick: jest.fn(), + isColumnLineage: false, selectedNode: { id: 'node1', }, - isColumnLineage: false, + edge: { + fromEntity: { + id: '1', + type: 'table', + }, + toEntity: { + id: '2', + type: 'table', + }, + pipeline: { + id: 'pipeline1', + fullyQualifiedName: 'pipeline1', + type: 'pipeline', + name: 'pipeline1', + }, + }, isEditMode: true, }, selected: true, } as EdgeProps; -describe('Test CustomEdge Component', () => { - it('Check if CustomEdge has all child elements', async () => { - render(, { - wrapper: MemoryRouter, - }); - - const deleteButton = await screen.findByTestId('delete-button'); - const edgePathElement = await screen.findAllByTestId( - 'react-flow-edge-path' - ); - const pipelineLabelAsEdge = screen.queryByTestId('pipeline-label'); - - expect(deleteButton).toBeInTheDocument(); - expect(pipelineLabelAsEdge).not.toBeInTheDocument(); - expect(edgePathElement).toHaveLength(edgePathElement.length); - }); +jest.mock('../../LineageProvider/LineageProvider', () => ({ + useLineageProvider: jest.fn().mockImplementation(() => ({ + tracedNodes: [], + tracedColumns: [], + })), +})); +describe('Test CustomEdge Component', () => { it('Check if CustomEdge has selected as false', async () => { render(, { wrapper: MemoryRouter, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx index 8a13f2d94676..91809c4f48d2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx @@ -10,121 +10,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button } from 'antd'; -import classNames from 'classnames'; import React, { Fragment } from 'react'; import { Handle, HandleProps, HandleType, Position } from 'reactflow'; -import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-outlined.svg'; import { EntityLineageNodeType } from '../../../enums/entity.enum'; -import { EntityReference } from '../../../generated/entity/type'; -import { getEncodedFqn } from '../../../utils/StringsUtils'; -import { - LineagePos, - LoadingNodeState, - SelectedNode, -} from './EntityLineage.interface'; - -export const getHandle = ( - node: EntityReference, - nodeType: string, - isConnectable: HandleProps['isConnectable'], - isLeafNode: boolean, - isNodeLoading: LoadingNodeState, - className?: string, - onSelect?: (state: boolean, value: SelectedNode) => void, - loadNodeHandler?: (node: EntityReference, pos: LineagePos) => void -) => { - if (nodeType === EntityLineageNodeType.OUTPUT) { - return ( - <> - {getHandleByType(isConnectable, Position.Left, 'target', className)} - {generateExpandButton( - isLeafNode, - node, - isNodeLoading, - 'to', - nodeType, - onSelect, - loadNodeHandler - )} - - ); - } else if (nodeType === EntityLineageNodeType.INPUT) { - return ( - <> - {generateExpandButton( - isLeafNode, - node, - isNodeLoading, - 'from', - nodeType, - onSelect, - loadNodeHandler - )} - {getHandleByType(isConnectable, Position.Right, 'source', className)} - - ); - } else if (nodeType === EntityLineageNodeType.NOT_CONNECTED) { - return null; - } else { - return ( - <> - {getHandleByType(isConnectable, Position.Left, 'target', className)} - {getHandleByType(isConnectable, Position.Right, 'source', className)} - - ); - } -}; - -const generateExpandButton = ( - isLeaf: boolean, - node: EntityReference, - isNodeLoading: LoadingNodeState, - direction: LineagePos, - nodeType: EntityLineageNodeType, - onSelect?: (state: boolean, value: SelectedNode) => void, - loadNodeHandler?: (node: EntityReference, pos: LineagePos) => void -) => { - // const isLoading = node.id.includes(isNodeLoading.id as string); - const isLoading = false; - - if (isLeaf && !isLoading) { - return ( -
- ), - removeNodeHandler, - onNodeExpand: handleNodeExpand, - isEditMode, - isNewNode: true, - }, - }; - setNewAddedNode(newNode as Node); - - setNodes( - (es) => getUniqueFlowElements(es.concat(newNode as Node)) as Node[] - ); - } - }; - - /** - * After dropping node to graph user will search and select entity - * and this method will take care of changing node information based on selected entity. - */ - const onEntitySelect = () => { - if (!isEmpty(selectedEntity)) { - const isExistingNode = nodes.some((n) => n.id === selectedEntity.id); - if (isExistingNode) { - setNodes((es) => - es - .map((n) => - n.id.includes(selectedEntity.id) - ? { - ...n, - selectable: true, - className: `${n.className} selected`, - } - : n - ) - .filter((es) => es.id !== newAddedNode.id) - ); - resetSelectedData(); - } else { - setNodes((es) => { - return es.map((el) => { - if (el.id === newAddedNode.id) { - return { - ...el, - connectable: true, - selectable: true, - id: selectedEntity.id, - data: { - ...el.data, - removeNodeHandler, - isEditMode, - node: selectedEntity, - }, - }; - } else { - return el; - } - }); - }); - } - } - }; - - /** - * This method will handle the delete edge modal confirmation - */ - const onRemove = useCallback(() => { - setDeletionState({ ...ELEMENT_DELETE_STATE, loading: true }); - setTimeout(() => { - setDeletionState({ ...ELEMENT_DELETE_STATE, status: 'success' }); - setTimeout(() => { - setShowDeleteModal(false); - setConfirmDelete(true); - setDeletionState((pre) => ({ ...pre, status: 'initial' })); - }, 500); - }, 500); - }, []); - - const handleEdgeClick = useCallback( - (_e: React.MouseEvent, edge: Edge) => { - setSelectedEdgeInfo(edge); - setIsDrawerOpen(true); - }, - [] - ); - - const toggleColumnView = (value: boolean) => { - setExpandAllColumns(value); - setEdges((prevEdges) => { - return prevEdges.map((edge) => { - edge.data.isExpanded = value; - - return edge; - }); - }); - setNodes((prevNodes) => { - const updatedNode = prevNodes.map((node) => { - node.data.isExpanded = value; - - return node; - }); - const { edge, node } = getLayoutedElements({ - node: updatedNode, - edge: edges, - }); - setEdges(edge); - - return node; - }); - }; - - const handleExpandColumnClick = () => { - if (!updatedLineageData) { - return; - } - if (expandAllColumns) { - toggleColumnView(false); - } else { - const allTableNodes = nodes - .map((item) => item.data.node) - .filter( - (node) => - [EntityType.TABLE, EntityType.DASHBOARD_DATA_MODEL].includes( - node.type as EntityType - ) && isUndefined(tableColumnsRef.current[node.id]) - ); - - allTableNodes.length && - allTableNodes.map(async (node) => await getTableColumns(node)); - toggleColumnView(true); - } - }; - - const getSearchResults = async (value = '*') => { - try { - const data = await searchData(value, 1, PAGE_SIZE, '', '', '', [ - SearchIndex.PIPELINE, - SearchIndex.STORED_PROCEDURE, - ]); - - const selectedPipeline = selectedEdge.data?.pipeline; - - const edgeOptions = data.data.hits.hits.map((hit) => - getEntityReferenceFromEntity( - hit._source, - hit._source.entityType as EntityType - ) - ); - - const optionContainItem = edgeOptions.find( - (item) => item.id === selectedEdgeId - ); - - setEdgeOptions([ - ...(selectedPipeline && - isEmpty(optionContainItem) && - isEmpty(edgeSearchValue) && - selectedPipeline.id === selectedEdgeId - ? [selectedPipeline] - : []), - ...edgeOptions, - ]); - } catch (error) { - showErrorToast( - error as AxiosError, - t('server.entity-fetch-error', { - entity: t('label.suggestion-lowercase-plural'), - }) - ); - } - }; - - const handleLineageConfigUpdate = useCallback((config: LineageConfig) => { - setLineageConfig(config); - fetchLineageData(config); - }, []); - const selectNode = (node: Node) => { - const { position } = node; - onNodeClick(node); - // moving selected node in center - reactFlowInstance && - reactFlowInstance.setCenter(position.x, position.y, { - duration: ZOOM_TRANSITION_DURATION, - zoom: zoomValue, - }); - }; - - const handleOptionSelect = (value?: string) => { - if (value) { - const selectedNode = nodes.find((node) => node.id === value); - - if (selectedNode) { - selectNode(selectedNode); - } else { - const path = findNodeById(value, childMap?.children, []) || []; - const lastNode = path[path?.length - 1]; - if (updatedLineageData) { - const { nodes, edges } = getPaginatedChildMap( - updatedLineageData, - childMap, - paginationData, - lineageConfig.nodesPerLayer - ); - const newNodes = union(nodes, path); - setElementsHandle( - { - ...updatedLineageData, - nodes: newNodes, - downstreamEdges: [ - ...(updatedLineageData.downstreamEdges || []), - ...edges, - ], - }, - lastNode.id - ); - } - } - } - }; - - const onEdgeDescriptionUpdate = useCallback( - async (updatedEdgeDetails: AddLineage) => { - try { - await updateLineageEdge(updatedEdgeDetails); - if (selectedEdgeInfo) { - const updatedSelectedEdgeInfo = { - ...selectedEdgeInfo, - data: { - ...selectedEdgeInfo.data, - edge: { - ...selectedEdgeInfo.data.edge, - lineageDetails: updatedEdgeDetails.edge.lineageDetails, - }, - }, - }; - - const updatedEdges = edges.map((edge) => - edge.id === selectedEdgeInfo.id ? updatedSelectedEdgeInfo : edge - ); - - setEdges(updatedEdges); - setSelectedEdgeInfo(updatedSelectedEdgeInfo); - - setUpdatedLineageData((pre) => { - if (!pre) { - return; - } - - const newData = { - ...pre, - downstreamEdges: updateEdgesWithLineageDetails( - pre.downstreamEdges ?? [], - updatedEdgeDetails - ), - upstreamEdges: updateEdgesWithLineageDetails( - pre.upstreamEdges ?? [], - updatedEdgeDetails - ), - }; - - return newData; - }); - } - } catch (err) { - showErrorToast(err as AxiosError); - } - }, - [edges, selectedEdgeInfo, updatedLineageData, setUpdatedLineageData] - ); - - /** - * Handle updated lineage nodes - * Change newly added node label based on entity:EntityReference - */ - const handleUpdatedLineageNode = () => { - const uNodes = updatedLineageData?.nodes; - const newlyAddedNodeElement = nodes.find((el) => el?.data?.isNewNode); - const newlyAddedNode = uNodes?.find( - (node) => node.id === newlyAddedNodeElement?.id - ); - - setNodes((els) => { - return (els || []).map((el) => { - if (el.id === newlyAddedNode?.id) { - return { - ...el, - data: { - ...el.data, - }, - }; - } else { - return el; - } - }); - }); - }; - - const handleZoomLevel = debounce((value: number) => { - setZoomValue(value); - }, 150); - - const initLineageChildMaps = ( - lineageData: EntityLineage, - childMapObj: EntityReferenceChild | undefined, - paginationObj: Record - ) => { - if (lineageData && childMapObj) { - const { nodes: newNodes, edges } = getPaginatedChildMap( - lineageData, - childMapObj, - paginationObj, - lineageConfig.nodesPerLayer - ); - setElementsHandle({ - ...lineageData, - nodes: newNodes, - downstreamEdges: [...(lineageData.downstreamEdges || []), ...edges], - }); - } - }; - - useEffect(() => { - if (!entity?.deleted) { - fetchLineageData(lineageConfig); - } - }, [entity?.deleted]); - - useEffect(() => { - if (!entityLineage) { - return; - } - if ( - !isEmpty(entityLineage) && - !isUndefined(entityLineage.entity) && - !deleted - ) { - const childMapObj: EntityReferenceChild = getChildMap(entityLineage); - setChildMap(childMapObj); - initLineageChildMaps(entityLineage, childMapObj, paginationData); - } - }, [entityLineage]); - - useEffect(() => { - if (!updatedLineageData) { - return; - } - setEntityLineage({ - ...updatedLineageData, - nodes: getNewNodes(updatedLineageData), - }); - }, [isEditMode]); - - useEffect(() => { - handleUpdatedLineageNode(); - }, [updatedLineageData]); - - useEffect(() => { - onEntitySelect(); - }, [selectedEntity]); - - useEffect(() => { - if (selectedEdge.data?.isColumnLineage) { - removeColumnEdge(selectedEdge, confirmDelete); - } else { - removeEdgeHandler(selectedEdge, confirmDelete); - } - }, [selectedEdge, confirmDelete]); - - useEffect(() => { - if (showAddEdgeModal) { - getSearchResults(edgeSearchValue); - } - }, [edgeSearchValue, showAddEdgeModal]); - - useEffect(() => { - edgesRef.current = edges; - }, [edges]); - - if (isLineageLoading || (nodes.length === 0 && !deleted)) { - return ; - } - - if (deleted) { - return getDeletedLineagePlaceholder(); - } - - return ( - - {isFullScreen && ( - - )} -
-
- - { - onLoad(reactFlowInstance); - setReactFlowInstance(reactFlowInstance); - }} - onMove={(_e, viewPort) => handleZoomLevel(viewPort.zoom)} - onNodeClick={(_e, node) => { - onNodeClick(node); - _e.stopPropagation(); - }} - onNodeContextMenu={onNodeContextMenu} - onNodeDrag={dragHandle} - onNodeDragStart={dragHandle} - onNodeDragStop={dragHandle} - onNodeMouseEnter={onNodeMouseEnter} - onNodeMouseLeave={onNodeMouseLeave} - onNodeMouseMove={onNodeMouseMove} - onNodesChange={onNodesChange} - onPaneClick={onPaneClick}> - {updatedLineageData && ( - - )} - - - -
- {isDrawerOpen && - !isEditMode && - (selectedEdgeInfo ? ( - { - setIsDrawerOpen(false); - setSelectedEdgeInfo(undefined); - }} - onEdgeDescriptionUpdate={onEdgeDescriptionUpdate} - /> - ) : ( - - ))} - - {showDeleteModal && ( - { - setShowDeleteModal(false); - }} - onOk={onRemove}> - {getModalBodyText(selectedEdge)} - - )} - - setEdgeSearchValue(value)} - onSelect={handleEdgeSelection} - /> -
- - ); -}; - -export default withLoader(EntityLineageComponent); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts index d051233d0867..25913d9a67b9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntityLineage.interface.ts @@ -107,7 +107,6 @@ export enum EdgeTypeEnum { } export interface ControlProps extends HTMLAttributes { - onFitView?: () => void; handleFullScreenViewClick?: () => void; onExitFullScreenViewClick?: () => void; deleted: boolean | undefined; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/Entitylineage.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/Entitylineage.component.test.tsx deleted file mode 100644 index bc66a51b879f..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/Entitylineage.component.test.tsx +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2022 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { fireEvent, render, screen } from '@testing-library/react'; -import React from 'react'; -import { act } from 'react-dom/test-utils'; -import { MemoryRouter } from 'react-router-dom'; -import { EntityType } from '../../../enums/entity.enum'; -import { MOCK_CHILD_MAP, MOCK_LINEAGE_DATA } from '../../../mocks/Lineage.mock'; -import EntityLineage from './EntityLineage.component'; - -const mockEntityLineageProp = { - deleted: false, - entityType: EntityType.TABLE, - hasEditAccess: true, -}; - -const mockFlowData = { - node: [ - { - id: 'a4b21449-b03b-4527-b482-148f52f92ff2', - sourcePosition: 'right', - targetPosition: 'left', - type: 'default', - className: 'leaf-node core', - data: { - label: 'dim_address etl', - isEditMode: false, - columns: {}, - isExpanded: false, - }, - position: { - x: 0, - y: 0, - }, - }, - ], - edge: [], -}; - -const mockPaginatedData = { - nodes: [...mockFlowData.node], - edges: [], -}; - -jest.mock('../../../utils/EntityLineageUtils', () => ({ - dragHandle: jest.fn(), - getDeletedLineagePlaceholder: jest - .fn() - .mockReturnValue( -

Lineage data is not available for deleted entities.

- ), - getHeaderLabel: jest.fn().mockReturnValue(

Header label

), - getLoadingStatusValue: jest.fn().mockReturnValue(

Confirm

), - getLayoutedElements: jest.fn().mockImplementation(() => mockFlowData), - getLineageData: jest.fn().mockImplementation(() => mockFlowData), - getPaginatedChildMap: jest.fn().mockImplementation(() => mockPaginatedData), - getChildMap: jest.fn().mockImplementation(() => MOCK_CHILD_MAP), - getModalBodyText: jest.fn(), - onLoad: jest.fn(), - onNodeContextMenu: jest.fn(), - onNodeMouseEnter: jest.fn(), - onNodeMouseLeave: jest.fn(), - onNodeMouseMove: jest.fn(), - getUniqueFlowElements: jest.fn().mockReturnValue([]), - getParamByEntityType: jest.fn().mockReturnValue('entityFQN'), -})); - -jest.mock('../../../rest/lineageAPI', () => ({ - getLineageByFQN: jest.fn().mockImplementation(() => - Promise.resolve({ - ...MOCK_LINEAGE_DATA, - }) - ), -})); - -jest.mock('../EntityInfoDrawer/EntityInfoDrawer.component', () => { - return jest.fn().mockReturnValue(

EntityInfoDrawerComponent

); -}); - -const mockPush = jest.fn(); - -jest.mock('react-router-dom', () => ({ - ...jest.requireActual('react-router-dom'), - useHistory: () => ({ - push: mockPush, - }), -})); - -describe('Test EntityLineage Component', () => { - it('Check if EntityLineage is rendering all the nodes', async () => { - act(() => { - render(, { - wrapper: MemoryRouter, - }); - }); - - const lineageContainer = await screen.findByTestId('lineage-container'); - const reactFlowElement = await screen.findByTestId('rf__wrapper'); - - expect(lineageContainer).toBeInTheDocument(); - expect(reactFlowElement).toBeInTheDocument(); - }); - - it('Check if EntityLineage has deleted as true', async () => { - act(() => { - render(, { - wrapper: MemoryRouter, - }); - }); - - const lineageContainer = screen.queryByTestId('lineage-container'); - const reactFlowElement = screen.queryByTestId('rf__wrapper'); - const deletedMessage = await screen.findByText( - /Lineage data is not available for deleted entities/i - ); - - expect(deletedMessage).toBeInTheDocument(); - - expect(reactFlowElement).not.toBeInTheDocument(); - - expect(lineageContainer).not.toBeInTheDocument(); - }); - - it('should add fullscreen true in url on fullscreen button click', async () => { - render(, { - wrapper: MemoryRouter, - }); - - const lineageContainer = await screen.findByTestId('lineage-container'); - const reactFlowElement = await screen.findByTestId('rf__wrapper'); - const fullscreenButton = await screen.getByTestId('full-screen'); - - expect(lineageContainer).toBeInTheDocument(); - expect(reactFlowElement).toBeInTheDocument(); - - act(() => { - fireEvent.click(fullscreenButton); - }); - - expect(mockPush).toHaveBeenCalledTimes(1); - expect(mockPush).toHaveBeenCalledWith({ - search: 'fullscreen=true', - }); - }); - - it('should show breadcrumbs when URL has fullscreen=true', async () => { - act(() => { - render( - - - - ); - }); - const lineageContainer = await screen.findByTestId('lineage-container'); - const reactFlowElement = await screen.findByTestId('rf__wrapper'); - - expect(lineageContainer).toBeInTheDocument(); - expect(reactFlowElement).toBeInTheDocument(); - - const breadcrumbs = await screen.getByTestId('breadcrumb'); - - expect(breadcrumbs).toBeInTheDocument(); - - const mainRootElement = await screen.getByTestId('lineage-details'); - - expect(mainRootElement).toHaveClass('full-screen-lineage'); - }); -}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index b5d877958751..ceba72d4cf19 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -243,7 +243,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setTracedNodes(connectedNodeIds); setTracedColumns([]); }, - [nodes, edges, selectedNode] + [nodes, edges] ); const onColumnClick = useCallback( @@ -435,14 +435,17 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setQueryFilter(query); }, []); - const onNodeClick = useCallback((node: Node) => { - if (node) { - setSelectedEdge(undefined); - setSelectedNode(node.data.node as SourceType); - setIsDrawerOpen(true); - handleLineageTracing(node); - } - }, []); + const onNodeClick = useCallback( + (node: Node) => { + if (node) { + setSelectedEdge(undefined); + setSelectedNode(node.data.node as SourceType); + setIsDrawerOpen(true); + handleLineageTracing(node); + } + }, + [handleLineageTracing] + ); const onPaneClick = useCallback(() => { setIsDrawerOpen(false); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.test.tsx index 63d8e2e6f51f..d2c1b487e4e4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MlModelDetail/MlModelDetail.component.test.tsx @@ -206,12 +206,9 @@ jest.mock('../common/RichTextEditor/RichTextEditorPreviewer', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviewer

); }); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest.fn().mockReturnValue(

EntityLineage.component

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockReturnValue(

EntityLineage.component

); +}); jest.mock('./MlModelFeaturesList', () => { return jest.fn().mockReturnValue(

MlModelFeaturesList

); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx index 97817294ddcc..759d7540cf5f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx @@ -143,7 +143,7 @@ jest.mock('../FeedEditor/FeedEditor', () => { return jest.fn().mockReturnValue(

FeedEditor

); }); -jest.mock('../Entity/EntityLineage/EntityLineage.component', () => { +jest.mock('../Lineage/Lineage.component', () => { return jest .fn() .mockReturnValue(

Lineage

); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx index 0fa1fdb1fcf9..d835f7459ca8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx @@ -64,15 +64,11 @@ const TasksDAGViewProps = { jest.mock('../../utils/EntityLineageUtils', () => ({ dragHandle: jest.fn(), - getDeletedLineagePlaceholder: jest - .fn() - .mockReturnValue(

Task data is not available for deleted entities.

), getHeaderLabel: jest.fn().mockReturnValue(

Header label

), getLayoutedElements: jest.fn().mockImplementation(() => ({ node: mockNodes, edge: mockEdges, })), - getLineageData: jest.fn().mockReturnValue([]), getModalBodyText: jest.fn(), onLoad: jest.fn(), onNodeContextMenu: jest.fn(), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.test.tsx index d717b53f1eb6..35464135daef 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicDetails.test.tsx @@ -73,12 +73,9 @@ jest.mock('react-router-dom', () => ({ useParams: jest.fn().mockImplementation(() => mockParams), })); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest.fn().mockReturnValue(

EntityLineage.component

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockReturnValue(

EntityLineage.component

); +}); jest.mock('../common/EntityDescription/Description', () => { return jest.fn().mockReturnValue(

Description Component

); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx index 20376fffa3e7..e09a9bb0632d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx @@ -106,14 +106,11 @@ jest.mock( } ); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest - .fn() - .mockReturnValue(
EntityLineage
); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest + .fn() + .mockReturnValue(
EntityLineage
); +}); jest.mock('../../components/Loader/Loader', () => { return jest.fn().mockReturnValue(
Loader
); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.test.tsx index 32e76365ec9f..c32be1d5e24d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/StoredProcedure/StoredProcedurePage.test.tsx @@ -69,14 +69,9 @@ jest.mock('../../components/common/EntityDescription/DescriptionV1', () => { return jest.fn().mockImplementation(() =>

testDescriptionV1

); }); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest - .fn() - .mockImplementation(() =>

testEntityLineageComponent

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockImplementation(() =>

testEntityLineageComponent

); +}); jest.mock( '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder', @@ -98,12 +93,9 @@ jest.mock( }) ); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest.fn().mockImplementation(() =>

testEntityLineage

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockImplementation(() =>

testEntityLineage

); +}); jest.mock('../../components/SchemaEditor/SchemaEditor', () => { return jest.fn().mockImplementation(() =>

testSchemaEditor

); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx index b43ff14a6ef3..e46bf64fc719 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx @@ -98,12 +98,9 @@ jest.mock( }) ); -jest.mock( - '../../components/Entity/EntityLineage/EntityLineage.component', - () => { - return jest.fn().mockImplementation(() =>

testEntityLineage

); - } -); +jest.mock('../../components/Lineage/Lineage.component', () => { + return jest.fn().mockImplementation(() =>

testEntityLineage

); +}); jest.mock('../../components/SampleDataTable/SampleDataTable.component', () => { return jest.fn().mockImplementation(() =>

testSampleDataTable

); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx index 49f7629371f5..ff9e9500a708 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx @@ -11,221 +11,18 @@ * limitations under the License. */ -import { Edge } from 'reactflow'; +import { MOCK_NODES_AND_EDGES } from '../mocks/Lineage.mock'; import { - CustomEdgeData, - EdgeTypeEnum, - SelectedEdge, -} from '../components/Entity/EntityLineage/EntityLineage.interface'; -import { - getDashboardDetailsPath, - getMlModelPath, - getPipelineDetailsPath, - getTableTabPath, - getTopicDetailsPath, -} from '../constants/constants'; -import { EntityType } from '../enums/entity.enum'; -import { LineageDetails } from '../generated/api/lineage/addLineage'; -import { EntityLineage } from '../generated/type/entityLineage'; -import { EntityReference } from '../generated/type/entityReference'; -import { - COLUMN_LINEAGE_DETAILS, - EDGE_TO_BE_REMOVED, - MOCK_CHILD_MAP, - MOCK_COLUMN_LINEAGE_EDGE, - MOCK_LINEAGE_DATA, - MOCK_NODES_AND_EDGES, - MOCK_NORMAL_LINEAGE_EDGE, - MOCK_PARAMS_FOR_DOWN_STREAM, - MOCK_PARAMS_FOR_UP_STREAM, - MOCK_PIPELINE, - MOCK_REMOVED_NODE, - SELECTED_EDGE, - UPDATED_COLUMN_LINEAGE, - UPDATED_EDGE_PARAM, - UPDATED_LINEAGE_EDGE, - UPDATED_NORMAL_LINEAGE, - UP_STREAM_EDGE, -} from '../mocks/Lineage.mock'; -import { - createNewEdge, - findUpstreamDownStreamEdge, getAllTracedColumnEdge, getAllTracedEdges, getAllTracedNodes, - getChildMap, getClassifiedEdge, getEdgeStyle, - getEdgeType, - getEntityLineagePath, - getRemovedNodeData, - getUpdatedEdge, - getUpdatedEdgeWithPipeline, - getUpdatedUpstreamDownStreamEdgeArr, - getUpStreamDownStreamColumnLineageArr, isColumnLineageTraced, isTracedEdge, } from './EntityLineageUtils'; describe('Test EntityLineageUtils utility', () => { - it('findUpstreamDownStreamEdge function should work properly', () => { - const upstreamData = findUpstreamDownStreamEdge( - MOCK_LINEAGE_DATA.upstreamEdges, - SELECTED_EDGE as SelectedEdge - ); - const nodata = findUpstreamDownStreamEdge( - undefined, - SELECTED_EDGE as SelectedEdge - ); - - expect(upstreamData).toStrictEqual(UP_STREAM_EDGE); - expect(nodata).toBeUndefined(); - }); - - it('getUpStreamDownStreamColumnLineageArr function should work properly', () => { - const columnLineageData = getUpStreamDownStreamColumnLineageArr( - MOCK_LINEAGE_DATA.upstreamEdges[0].lineageDetails as LineageDetails, - SELECTED_EDGE as SelectedEdge - ); - const nodata = getUpStreamDownStreamColumnLineageArr( - MOCK_LINEAGE_DATA.upstreamEdges[1].lineageDetails as LineageDetails, - SELECTED_EDGE as SelectedEdge - ); - - expect(columnLineageData).toStrictEqual(COLUMN_LINEAGE_DETAILS); - expect(nodata).toStrictEqual({ sqlQuery: '', columnsLineage: [] }); - }); - - it('getUpdatedUpstreamDownStreamEdgeArr function should work properly', () => { - const columnLineageData = getUpdatedUpstreamDownStreamEdgeArr( - MOCK_LINEAGE_DATA.upstreamEdges, - SELECTED_EDGE as SelectedEdge, - COLUMN_LINEAGE_DETAILS - ); - const nodata = getUpdatedUpstreamDownStreamEdgeArr( - [], - SELECTED_EDGE as SelectedEdge, - COLUMN_LINEAGE_DETAILS - ); - - expect(columnLineageData).toStrictEqual(UPDATED_LINEAGE_EDGE); - expect(nodata).toStrictEqual([]); - }); - - it('getRemovedNodeData function should work properly', () => { - const data = getRemovedNodeData( - MOCK_LINEAGE_DATA.nodes, - EDGE_TO_BE_REMOVED as Edge, - MOCK_LINEAGE_DATA.entity, - MOCK_LINEAGE_DATA.nodes[0] - ); - const nodata = getRemovedNodeData( - [], - SELECTED_EDGE.data, - MOCK_LINEAGE_DATA.entity, - {} as EntityReference - ); - - expect(data).toStrictEqual(MOCK_REMOVED_NODE); - expect(nodata).toStrictEqual({ - id: SELECTED_EDGE.data.id, - source: MOCK_LINEAGE_DATA.entity, - target: MOCK_LINEAGE_DATA.entity, - }); - }); - - it('getEdgeType function should work properly', () => { - const upStreamData = getEdgeType( - MOCK_LINEAGE_DATA, - MOCK_PARAMS_FOR_UP_STREAM - ); - const downStreamData = getEdgeType( - MOCK_LINEAGE_DATA, - MOCK_PARAMS_FOR_DOWN_STREAM - ); - - expect(upStreamData).toStrictEqual(EdgeTypeEnum.UP_STREAM); - expect(downStreamData).toStrictEqual(EdgeTypeEnum.DOWN_STREAM); - }); - - it('getUpdatedEdge function should work properly', () => { - const node = MOCK_LINEAGE_DATA.upstreamEdges[1]; - const data = getUpdatedEdge( - [node], - UPDATED_EDGE_PARAM, - UPDATED_COLUMN_LINEAGE - ); - - expect(data).toStrictEqual([ - { - ...node, - lineageDetails: UPDATED_COLUMN_LINEAGE, - }, - ]); - }); - - it('createNewEdge function should work properly', () => { - const columnLineageEdge = createNewEdge( - UPDATED_EDGE_PARAM, - true, - 'table', - 'table', - true, - jest.fn, - jest.fn - ); - const normalLineageEdge = createNewEdge( - UPDATED_EDGE_PARAM, - true, - 'table', - 'table', - false, - jest.fn, - jest.fn - ); - - const updatedColLineageEdge = MOCK_COLUMN_LINEAGE_EDGE as Edge; - updatedColLineageEdge.data.onEdgeClick = jest.fn; - - const updatedNormalLineageEdge = MOCK_NORMAL_LINEAGE_EDGE as Edge; - updatedNormalLineageEdge.data.onEdgeClick = jest.fn; - updatedNormalLineageEdge.data.addPipelineClick = jest.fn; - - expect(columnLineageEdge).toMatchObject(updatedColLineageEdge); - expect(normalLineageEdge).toMatchObject(updatedNormalLineageEdge); - }); - - it('getUpdatedEdgeWithPipeline function should work properly', () => { - const lineageEdge = getUpdatedEdgeWithPipeline( - MOCK_LINEAGE_DATA.upstreamEdges, - UPDATED_NORMAL_LINEAGE, - MOCK_NORMAL_LINEAGE_EDGE.data as CustomEdgeData, - MOCK_PIPELINE - ); - - const noData = getUpdatedEdgeWithPipeline( - undefined, - UPDATED_NORMAL_LINEAGE, - MOCK_NORMAL_LINEAGE_EDGE.data as CustomEdgeData, - MOCK_PIPELINE - ); - - const updatedData: EntityLineage['upstreamEdges'] = [ - ...MOCK_LINEAGE_DATA.upstreamEdges, - ]; - updatedData[1].lineageDetails = { - ...UPDATED_NORMAL_LINEAGE, - pipeline: { - ...UPDATED_NORMAL_LINEAGE.pipeline, - name: MOCK_PIPELINE.name, - displayName: MOCK_PIPELINE.displayName, - }, - }; - - expect(lineageEdge).toMatchObject(updatedData); - expect(noData).toMatchObject([]); - }); - it('getAllTracedNodes & isTracedEdge function should work properly', () => { const { nodes, edges } = MOCK_NODES_AND_EDGES; const incomerNode = getAllTracedNodes(nodes[1], nodes, edges, [], true); @@ -312,28 +109,6 @@ describe('Test EntityLineageUtils utility', () => { expect(isColumnTracedFalsy).toBeFalsy(); }); - it('should return the correct lineage path for the given entity type and FQN', () => { - expect(getEntityLineagePath(EntityType.TABLE, 'myTable')).toEqual( - getTableTabPath('myTable', 'lineage') - ); - expect(getEntityLineagePath(EntityType.TOPIC, 'myTopic')).toEqual( - getTopicDetailsPath('myTopic', 'lineage') - ); - expect(getEntityLineagePath(EntityType.DASHBOARD, 'myDashboard')).toEqual( - getDashboardDetailsPath('myDashboard', 'lineage') - ); - expect(getEntityLineagePath(EntityType.PIPELINE, 'myPipeline')).toEqual( - getPipelineDetailsPath('myPipeline', 'lineage') - ); - expect(getEntityLineagePath(EntityType.MLMODEL, 'myModel')).toEqual( - getMlModelPath('myModel', 'lineage') - ); - }); - - it('getChildMap should return valid map object', () => { - expect(getChildMap(MOCK_LINEAGE_DATA)).toEqual(MOCK_CHILD_MAP); - }); - it('getEdgeStyle should returns the expected edge style for a value', () => { const expectedStyle = { opacity: 1, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index 5203389fa97a..c5d12494e095 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -16,34 +16,16 @@ import { Button, Typography } from 'antd'; import { AxiosError } from 'axios'; import dagre from 'dagre'; import { t } from 'i18next'; -import { - cloneDeep, - isEmpty, - isEqual, - isNil, - isUndefined, - uniqueId, - uniqWith, - upperCase, -} from 'lodash'; +import { isEmpty, isUndefined, upperCase } from 'lodash'; import { LoadingState } from 'Models'; import React, { Fragment, MouseEvent as ReactMouseEvent } from 'react'; import { Link } from 'react-router-dom'; -import { - Connection, - Edge, - isNode, - MarkerType, - Node, - Position, - ReactFlowInstance, -} from 'reactflow'; +import { Edge, isNode, Node, Position, ReactFlowInstance } from 'reactflow'; import { ReactComponent as DashboardIcon } from '../assets/svg/dashboard-grey.svg'; import { ReactComponent as MlModelIcon } from '../assets/svg/mlmodal.svg'; import { ReactComponent as PipelineIcon } from '../assets/svg/pipeline-grey.svg'; import { ReactComponent as TableIcon } from '../assets/svg/table-grey.svg'; import { ReactComponent as TopicIcon } from '../assets/svg/topic-grey.svg'; -import ErrorPlaceHolder from '../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import { CustomEdge } from '../components/Entity/EntityLineage/CustomEdge.component'; import CustomNodeV1 from '../components/Entity/EntityLineage/CustomNodeV1.component'; import { @@ -51,51 +33,25 @@ import { CustomElement, CustomFlow, EdgeData, - EdgeTypeEnum, - EntityReferenceChild, - LeafNodes, - LineagePos, - LoadingNodeState, - ModifiedColumn, - NodeIndexMap, - SelectedEdge, - SelectedNode, } from '../components/Entity/EntityLineage/EntityLineage.interface'; import { ExploreSearchIndex } from '../components/Explore/ExplorePage.interface'; import { EdgeDetails } from '../components/Lineage/Lineage.interface'; import Loader from '../components/Loader/Loader'; -import { - getContainerDetailPath, - getDashboardDetailsPath, - getDataModelDetailsPath, - getMlModelPath, - getPipelineDetailsPath, - getTableTabPath, - getTopicDetailsPath, - INFO_COLOR, -} from '../constants/constants'; +import { INFO_COLOR } from '../constants/constants'; import { EXPANDED_NODE_HEIGHT, NODE_HEIGHT, NODE_WIDTH, ZOOM_VALUE, } from '../constants/Lineage.constants'; -import { ERROR_PLACEHOLDER_TYPE } from '../enums/common.enum'; import { EntityLineageDirection, - EntityLineageNodeType, EntityType, FqnPart, } from '../enums/entity.enum'; import { SearchIndex } from '../enums/search.enum'; import { AddLineage } from '../generated/api/lineage/addLineage'; -import { Column } from '../generated/entity/data/table'; -import { - ColumnLineage, - Edge as EntityLineageEdge, - EntityLineage, - LineageDetails, -} from '../generated/type/entityLineage'; +import { LineageDetails } from '../generated/type/entityLineage'; import { EntityReference } from '../generated/type/entityReference'; import { addLineage, deleteLineageEdge } from '../rest/miscAPI'; import { @@ -103,7 +59,6 @@ import { getPartialNameFromTableFQN, prepareLabel, } from './CommonUtils'; -import { getEntityName } from './EntityUtils'; import { getEntityLink } from './TableUtils'; import { showErrorToast } from './ToastUtils'; @@ -169,247 +124,6 @@ export const dragHandle = (event: ReactMouseEvent) => { event.stopPropagation(); }; -const getNodeType = ( - entityLineage: EntityLineage, - id: string -): EntityLineageNodeType => { - const upStreamEdges = entityLineage.upstreamEdges || []; - const downStreamEdges = entityLineage.downstreamEdges || []; - - const hasDownStreamToEntity = downStreamEdges.find( - (down) => down.toEntity === id - ); - const hasDownStreamFromEntity = downStreamEdges.find( - (down) => down.fromEntity === id - ); - const hasUpstreamFromEntity = upStreamEdges.find( - (up) => up.fromEntity === id - ); - const hasUpstreamToEntity = upStreamEdges.find((up) => up.toEntity === id); - - if (hasDownStreamToEntity && !hasDownStreamFromEntity) { - return EntityLineageNodeType.OUTPUT; - } - if (hasUpstreamFromEntity && !hasUpstreamToEntity) { - return EntityLineageNodeType.INPUT; - } - - return EntityLineageNodeType.DEFAULT; -}; - -export const getColumnType = (edges: Edge[], id: string) => { - const sourceEdge = edges.find((edge) => edge.sourceHandle === id); - const targetEdge = edges.find((edge) => edge.targetHandle === id); - - if (sourceEdge?.sourceHandle === id && targetEdge?.targetHandle === id) { - return EntityLineageNodeType.DEFAULT; - } - if (sourceEdge?.sourceHandle === id) { - return EntityLineageNodeType.INPUT; - } - if (targetEdge?.targetHandle === id) { - return EntityLineageNodeType.OUTPUT; - } - - return EntityLineageNodeType.NOT_CONNECTED; -}; - -export const getLineageData = ( - entityLineage: EntityLineage, - onSelect: (state: boolean, value: SelectedNode) => void, - loadNodeHandler: (node: EntityReference, pos: LineagePos) => void, - lineageLeafNodes: LeafNodes, - isNodeLoading: LoadingNodeState, - isEditMode: boolean, - edgeType: string, - onEdgeClick: ( - evt: React.MouseEvent, - data: CustomEdgeData - ) => void, - removeNodeHandler: (node: Node) => void, - columns: { [key: string]: Column[] }, - addPipelineClick?: ( - evt: React.MouseEvent, - data: CustomEdgeData - ) => void, - handleColumnClick?: (value: string) => void, - isExpanded?: boolean, - onNodeExpand?: (isExpanded: boolean, node: EntityReference) => void -) => { - const [x, y] = [0, 0]; - const nodes = [...(entityLineage['nodes'] || []), entityLineage['entity']]; - const edgesV1 = [ - ...(entityLineage.downstreamEdges || []), - ...(entityLineage.upstreamEdges || []), - ]; - const lineageEdgesV1: Edge[] = []; - const mainNode = entityLineage['entity']; - - edgesV1.forEach((edge) => { - const sourceType = nodes.find((n) => edge.fromEntity === n.id); - const targetType = nodes.find((n) => edge.toEntity === n.id); - - if (isUndefined(sourceType) || isUndefined(targetType)) { - return; - } - - if (!isUndefined(edge.lineageDetails)) { - edge.lineageDetails.columnsLineage?.forEach((e) => { - const toColumn = e.toColumn || ''; - if (toColumn && e.fromColumns && e.fromColumns.length > 0) { - e.fromColumns.forEach((fromColumn) => { - lineageEdgesV1.push({ - id: `column-${fromColumn}-${toColumn}-edge-${edge.fromEntity}-${edge.toEntity}`, - source: edge.fromEntity, - target: edge.toEntity, - targetHandle: toColumn, - sourceHandle: fromColumn, - type: edgeType, - markerEnd: { - type: MarkerType.ArrowClosed, - }, - data: { - id: `column-${fromColumn}-${toColumn}-edge-${edge.fromEntity}-${edge.toEntity}`, - source: edge.fromEntity, - target: edge.toEntity, - targetHandle: toColumn, - sourceHandle: fromColumn, - isEditMode, - onEdgeClick, - isColumnLineage: true, - isExpanded, - columnFunctionValue: e.function, - edge, - }, - }); - }); - } - }); - } - - lineageEdgesV1.push({ - id: `edge-${edge.fromEntity}-${edge.toEntity}`, - source: `${edge.fromEntity}`, - target: `${edge.toEntity}`, - type: edgeType, - animated: !isUndefined(edge.lineageDetails?.pipeline), - style: { strokeWidth: '2px' }, - markerEnd: { - type: MarkerType.ArrowClosed, - }, - data: { - id: `edge-${edge.fromEntity}-${edge.toEntity}`, - label: getEntityName(edge.lineageDetails?.pipeline), - pipeline: edge.lineageDetails?.pipeline, - source: `${edge.fromEntity}`, - target: `${edge.toEntity}`, - sourceType: sourceType?.type, - targetType: targetType?.type, - isEditMode, - onEdgeClick, - addPipelineClick, - isColumnLineage: false, - isExpanded, - edge, - }, - }); - }); - - const makeNode = (node: EntityReference) => { - let type = node.type as EntityLineageNodeType; - if (type !== EntityLineageNodeType.LOAD_MORE) { - type = getNodeType(entityLineage, node.id); - } - const cols: { [key: string]: ModifiedColumn } = {}; - columns[node.id]?.forEach((col) => { - cols[col.fullyQualifiedName || col.name] = { - ...col, - type: - type === EntityLineageNodeType.LOAD_MORE - ? type - : isEditMode - ? EntityLineageNodeType.DEFAULT - : getColumnType(lineageEdgesV1, col.fullyQualifiedName || col.name), - }; - }); - - return { - id: `${node.id}`, - sourcePosition: Position.Right, - targetPosition: Position.Left, - type: - type === EntityLineageNodeType.LOAD_MORE || !isEditMode - ? type - : EntityLineageNodeType.DEFAULT, - className: '', - data: { - entityType: node.type, - lineageLeafNodes: lineageLeafNodes, - removeNodeHandler, - isEditMode, - isExpanded, - columns: cols, - handleColumnClick, - onNodeExpand, - node, - isNodeLoading, - loadNodeHandler, - onSelect, - }, - position: { - x: x, - y: y, - }, - }; - }; - - const mainCols: { [key: string]: ModifiedColumn } = {}; - columns[mainNode.id]?.forEach((col) => { - mainCols[col.fullyQualifiedName || col.name] = { - ...col, - type: isEditMode - ? EntityLineageNodeType.DEFAULT - : getColumnType(lineageEdgesV1, col.fullyQualifiedName || col.name), - }; - }); - const mainNodeType = getNodeType(entityLineage, mainNode.id); - const lineageData = [ - { - id: `${mainNode.id}`, - sourcePosition: 'right', - targetPosition: 'left', - type: mainNodeType, - className: `core`, - data: { - isEditMode, - removeNodeHandler, - handleColumnClick, - onNodeExpand, - columns: mainCols, - isExpanded, - node: mainNode, - }, - position: { x, y }, - }, - ]; - - (entityLineage.nodes || []).forEach((n) => lineageData.push(makeNode(n))); - - return { node: lineageData, edge: lineageEdgesV1 }; -}; - -export const getDeletedLineagePlaceholder = () => { - return ( -
- - {t('message.field-data-is-not-available-for-deleted-entities', { - field: t('label.lineage'), - })} - -
- ); -}; - export const getLayoutedElements = ( elements: CustomElement, direction = EntityLineageDirection.LEFT_RIGHT, @@ -507,322 +221,6 @@ export const getUniqueFlowElements = (elements: CustomFlow[]) => { return uniqueElements; }; -export const getSelectedEdgeArr = ( - edgeArr: EntityLineageEdge[], - edgeData: EdgeData -) => { - return edgeArr.filter( - (edge) => - !edgeArr.find( - () => - edgeData.fromId === edge.fromEntity && edgeData.toId === edge.toEntity - ) - ); -}; - -/** - * Finds the upstream/downstream edge based on selected edge - * @param edgeArr edge[] - * @param data selected edge - * @returns edge - */ - -export const findUpstreamDownStreamEdge = ( - edgeArr: EntityLineageEdge[] | undefined, - data: SelectedEdge -) => { - return edgeArr?.find( - (edge) => - edge.fromEntity === data.source.id && edge.toEntity === data.target.id - ); -}; - -/** - * Get upstream/downstream column lineage array - * @param lineageDetails LineageDetails - * @param data SelectedEdge - * @returns Updated LineageDetails - */ - -export const getUpStreamDownStreamColumnLineageArr = ( - lineageDetails: LineageDetails, - data: SelectedEdge -) => { - const columnsLineage = lineageDetails.columnsLineage?.reduce((col, curr) => { - if (curr.toColumn === data.data?.targetHandle) { - const newCol = { - ...curr, - fromColumns: - curr.fromColumns?.filter( - (column) => column !== data.data?.sourceHandle - ) || [], - }; - if (newCol.fromColumns?.length) { - return [...col, newCol]; - } else { - return col; - } - } - - return [...col, curr]; - }, [] as ColumnLineage[]); - - return { - sqlQuery: lineageDetails.sqlQuery || '', - columnsLineage: columnsLineage, - }; -}; - -/** - * Get updated EntityLineageEdge Array based on selected data - * @param edge EntityLineageEdge[] - * @param data SelectedEdge - * @param lineageDetails updated LineageDetails - * @returns updated EntityLineageEdge[] - */ -export const getUpdatedUpstreamDownStreamEdgeArr = ( - edge: EntityLineageEdge[], - data: SelectedEdge, - lineageDetails: LineageDetails -) => { - return edge.map((down) => { - if ( - down.fromEntity === data.source.id && - down.toEntity === data.target.id - ) { - return { - ...down, - lineageDetails: lineageDetails, - }; - } - - return down; - }); -}; - -/** - * Get array of the removed node - * @param nodes All the node - * @param edge selected edge - * @param entity main entity - * @param selectedEntity selected entity - * @returns details of removed node - */ -export const getRemovedNodeData = ( - nodes: EntityReference[], - edge: Edge, - entity: EntityReference, - selectedEntity: EntityReference -) => { - let targetNode = nodes.find((node) => edge.target?.includes(node.id)); - let sourceNode = nodes.find((node) => edge.source?.includes(node.id)); - const selectedNode = isEmpty(selectedEntity) ? entity : selectedEntity; - - if (isUndefined(targetNode)) { - targetNode = selectedNode; - } - if (isUndefined(sourceNode)) { - sourceNode = selectedNode; - } - - return { - id: edge.id, - source: sourceNode, - target: targetNode, - }; -}; - -/** - * Get source/target edge based on query string - * @param edge upstream/downstream edge array - * @param queryStr source/target string - * @param id main entity id - * @returns source/target edge - */ -const getSourceTargetNode = ( - edge: EntityLineageEdge[], - queryStr: string | null, - id: string -) => { - return edge.find( - (d) => - (queryStr?.includes(d.fromEntity) || queryStr?.includes(d.toEntity)) && - queryStr !== id - ); -}; - -export const getEdgeType = ( - updatedLineageData: EntityLineage, - params: Edge | Connection -) => { - const { entity } = updatedLineageData; - const { target, source } = params; - const sourceDownstreamNode = getSourceTargetNode( - updatedLineageData.downstreamEdges || [], - source, - entity.id - ); - - const sourceUpStreamNode = getSourceTargetNode( - updatedLineageData.upstreamEdges || [], - source, - entity.id - ); - - const targetDownStreamNode = getSourceTargetNode( - updatedLineageData.downstreamEdges || [], - target, - entity.id - ); - - const targetUpStreamNode = getSourceTargetNode( - updatedLineageData.upstreamEdges || [], - target, - entity.id - ); - - const isUpstream = - (!isNil(sourceUpStreamNode) && !isNil(targetDownStreamNode)) || - !isNil(sourceUpStreamNode) || - !isNil(targetUpStreamNode) || - target?.includes(entity.id); - - const isDownstream = - (!isNil(sourceDownstreamNode) && !isNil(targetUpStreamNode)) || - !isNil(sourceDownstreamNode) || - !isNil(targetDownStreamNode) || - source?.includes(entity.id); - - if (isUpstream) { - return EdgeTypeEnum.UP_STREAM; - } else if (isDownstream) { - return EdgeTypeEnum.DOWN_STREAM; - } - - return EdgeTypeEnum.NO_STREAM; -}; - -/** - * Get updated Edge with lineageDetails - * @param edges Array of Edge - * @param params new connected edge - * @param lineageDetails updated lineage details - * @returns updated edge array - */ -export const getUpdatedEdge = ( - edges: EntityLineageEdge[], - params: Edge | Connection, - lineageDetails: LineageDetails | undefined -) => { - const updatedEdge: EntityLineageEdge[] = []; - const { target, source } = params; - edges.forEach((edge) => { - if (edge.fromEntity === source && edge.toEntity === target) { - updatedEdge.push({ - ...edge, - lineageDetails: lineageDetails, - }); - } else { - updatedEdge.push(edge); - } - }); - - return updatedEdge; -}; - -// create new edge -export const createNewEdge = ( - params: Edge | Connection, - isEditMode: boolean, - sourceNodeType: string, - targetNodeType: string, - isColumnLineage: boolean, - onEdgeClick: ( - evt: React.MouseEvent, - data: CustomEdgeData - ) => void, - addPipelineClick: ( - evt: React.MouseEvent, - data: CustomEdgeData - ) => void -) => { - const { target, source, sourceHandle, targetHandle } = params; - let data: Edge = { - id: `edge-${source}-${target}`, - source: `${source}`, - target: `${target}`, - type: isEditMode ? 'buttonedge' : 'default', - style: { strokeWidth: '2px' }, - markerEnd: { - type: MarkerType.ArrowClosed, - }, - data: { - id: `edge-${source}-${target}`, - source: source, - target: target, - sourceType: sourceNodeType, - targetType: targetNodeType, - isColumnLineage: isColumnLineage, - onEdgeClick, - isEditMode, - addPipelineClick, - }, - }; - - if (isColumnLineage) { - data = { - ...data, - id: `column-${sourceHandle}-${targetHandle}-edge-${source}-${target}`, - sourceHandle: sourceHandle, - targetHandle: targetHandle, - style: undefined, - data: { - ...data.data, - id: `column-${sourceHandle}-${targetHandle}-edge-${source}-${target}`, - sourceHandle: sourceHandle, - targetHandle: targetHandle, - addPipelineClick: undefined, - }, - }; - } - - return data; -}; - -export const getUpdatedEdgeWithPipeline = ( - edges: EntityLineage['downstreamEdges'], - updatedLineageDetails: LineageDetails, - selectedEdge: CustomEdgeData, - edgeDetails: EntityReference | undefined -) => { - if (isUndefined(edges)) { - return []; - } - - const { source, target } = selectedEdge; - - return edges.map((edge) => { - if (edge.fromEntity === source && edge.toEntity === target) { - return { - ...edge, - lineageDetails: { - ...updatedLineageDetails, - pipeline: !isUndefined(updatedLineageDetails.pipeline) - ? { - displayName: edgeDetails?.displayName, - name: edgeDetails?.name, - fullyQualifiedName: edgeDetails?.fullyQualifiedName, - ...updatedLineageDetails.pipeline, - } - : undefined, - }, - }; - } - - return edge; - }); -}; - export const getNewLineageConnectionDetails = ( selectedEdgeValue: Edge | undefined, selectedPipeline: EntityReference | undefined @@ -1063,176 +461,6 @@ export const getEdgeStyle = (value: boolean) => { }; }; -export const getChildMap = (obj: EntityLineage) => { - const nodeSet = new Set(); - nodeSet.add(obj.entity.id); - const newData = cloneDeep(obj); - newData.downstreamEdges = removeDuplicates(newData.downstreamEdges || []); - newData.upstreamEdges = removeDuplicates(newData.upstreamEdges || []); - - const childMap: EntityReferenceChild[] = getLineageChildParents( - newData, - nodeSet, - obj.entity.id, - false - ); - - const parentsMap: EntityReferenceChild[] = getLineageChildParents( - newData, - nodeSet, - obj.entity.id, - true - ); - - const map: EntityReferenceChild = { - ...obj.entity, - children: childMap, - parents: parentsMap, - }; - - return map; -}; - -export const getPaginatedChildMap = ( - obj: EntityLineage, - map: EntityReferenceChild | undefined, - pagination_data: Record, - maxLineageLength: number -) => { - const nodes = []; - const edges: EntityLineageEdge[] = []; - nodes.push(obj.entity); - if (map) { - flattenObj( - obj, - map, - true, - obj.entity.id, - nodes, - edges, - pagination_data, - maxLineageLength - ); - flattenObj( - obj, - map, - false, - obj.entity.id, - nodes, - edges, - pagination_data, - maxLineageLength - ); - } - - return { nodes, edges }; -}; - -export const flattenObj = ( - entityObj: EntityLineage, - childMapObj: EntityReferenceChild, - downwards: boolean, - id: string, - nodes: EntityReference[], - edges: EntityLineageEdge[], - pagination_data: Record, - maxLineageLength = 50 -) => { - const children = downwards ? childMapObj.children : childMapObj.parents; - if (!children) { - return; - } - const startIndex = - pagination_data[id]?.[downwards ? 'downstream' : 'upstream'][0] ?? 0; - const hasMoreThanLimit = children.length > startIndex + maxLineageLength; - const endIndex = startIndex + maxLineageLength; - - children.slice(0, endIndex).forEach((item) => { - if (item) { - flattenObj( - entityObj, - item, - downwards, - item.id, - nodes, - edges, - pagination_data, - maxLineageLength - ); - nodes.push(item); - } - }); - - if (hasMoreThanLimit) { - const newNodeId = `loadmore_${uniqueId('node_')}_${id}_${startIndex}`; - const childrenLength = children.length - endIndex; - - const newNode = { - description: 'Demo description', - displayName: 'Load More', - id: newNodeId, - type: EntityLineageNodeType.LOAD_MORE, - pagination_data: { - index: endIndex, - parentId: id, - childrenLength, - }, - edgeType: downwards ? EdgeTypeEnum.DOWN_STREAM : EdgeTypeEnum.UP_STREAM, - }; - nodes.push(newNode); - const newEdge: EntityLineageEdge = { - fromEntity: downwards ? id : newNodeId, - toEntity: downwards ? newNodeId : id, - }; - edges.push(newEdge); - } -}; - -export const getLineageChildParents = ( - obj: EntityLineage, - nodeSet: Set, - id: string, - isParent = false, - index = 0 -) => { - const edges = isParent ? obj.upstreamEdges || [] : obj.downstreamEdges || []; - const filtered = edges.filter((edge) => { - return isParent ? edge.toEntity === id : edge.fromEntity === id; - }); - - return filtered.reduce((childMap: EntityReferenceChild[], edge, i) => { - const node = obj.nodes?.find((node) => { - return isParent ? node.id === edge.fromEntity : node.id === edge.toEntity; - }); - - if (node && !nodeSet.has(node.id)) { - nodeSet.add(node.id); - const childNodes = getLineageChildParents( - obj, - nodeSet, - node.id, - isParent, - i - ); - const lineage: EntityReferenceChild = { ...node, pageIndex: index + i }; - - if (isParent) { - lineage.parents = childNodes; - } else { - lineage.children = childNodes; - } - - childMap.push(lineage); - } - - return childMap; - }, []); -}; - -export const removeDuplicates = (arr: EntityLineageEdge[]) => { - return uniqWith(arr, isEqual); -}; - export const nodeTypes = { output: CustomNodeV1, input: CustomNodeV1, @@ -1242,38 +470,6 @@ export const nodeTypes = { export const customEdges = { buttonedge: CustomEdge }; -export const getNewNodes = ({ - nodes, - downstreamEdges, - upstreamEdges, -}: EntityLineage) => { - return nodes?.filter( - (n) => - !isUndefined(downstreamEdges?.find((d) => d.toEntity === n.id)) || - !isUndefined(upstreamEdges?.find((u) => u.fromEntity === n.id)) - ); -}; - -export const findNodeById = ( - id: string, - items: EntityReferenceChild[] = [], - path: EntityReferenceChild[] = [] -): EntityReferenceChild[] | undefined => { - for (const [index, item] of items.entries()) { - item.pageIndex = index; - if (item.id === id) { - // Return the path to the item, including the item itself - return [...path, item]; - } - const found = findNodeById(id, item.children, [...path, item]); - if (found) { - return found; - } - } - - return undefined; -}; - export const addLineageHandler = async (edge: AddLineage): Promise => { try { await addLineage(edge); @@ -1309,37 +505,6 @@ export const removeLineageHandler = async (data: EdgeData): Promise => { } }; -export const getEntityLineagePath = ( - entityType: EntityType, - entityFQN: string -): string => { - switch (entityType) { - case EntityType.TABLE: - return getTableTabPath(entityFQN, 'lineage'); - - case EntityType.TOPIC: - return getTopicDetailsPath(entityFQN, 'lineage'); - - case EntityType.DASHBOARD: - return getDashboardDetailsPath(entityFQN, 'lineage'); - - case EntityType.PIPELINE: - return getPipelineDetailsPath(entityFQN, 'lineage'); - - case EntityType.MLMODEL: - return getMlModelPath(entityFQN, 'lineage'); - - case EntityType.DASHBOARD_DATA_MODEL: - return getDataModelDetailsPath(entityFQN, 'lineage'); - - case EntityType.CONTAINER: - return getContainerDetailPath(entityFQN, 'lineage'); - - default: - return ''; - } -}; - // Nodes Icons export const getEntityNodeIcon = (label: string) => { switch (label) { @@ -1368,16 +533,3 @@ export const getSearchIndexFromNodeType = (entityType: string) => { return SearchIndex[searchIndexKey] as ExploreSearchIndex; }; - -export const updateEdgesWithLineageDetails = ( - edgesArray: EntityLineageEdge[], - updatedEdgeDetails: AddLineage -) => { - const { fromEntity, toEntity, lineageDetails } = updatedEdgeDetails.edge; - - return edgesArray.map((item) => - item.toEntity === toEntity.id && item.fromEntity === fromEntity.id - ? { ...item, lineageDetails: lineageDetails } - : item - ); -}; From 4c9a75dd05f6fa9c32316f38ce2ef1b574d51afd Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 20 Dec 2023 13:12:41 +0530 Subject: [PATCH 20/44] cleanup --- .../EntityLineage/CustomNodeV1.component.tsx | 2 +- .../LineageProvider/LineageProvider.tsx | 17 +- .../ui/src/utils/EntityLineageUtils.tsx | 367 ++++++++++++++++- .../resources/ui/src/utils/LineageV1Utils.ts | 380 ------------------ 4 files changed, 372 insertions(+), 394 deletions(-) delete mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx index 5ad628a5a391..5719bae97744 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx @@ -30,8 +30,8 @@ import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-outlined.sv import { BORDER_COLOR } from '../../../constants/constants'; import { EntityLineageNodeType, EntityType } from '../../../enums/entity.enum'; import { formTwoDigitNumber } from '../../../utils/CommonUtils'; +import { checkUpstreamDownstream } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; -import { checkUpstreamDownstream } from '../../../utils/LineageV1Utils'; import SVGIcons from '../../../utils/SvgUtils'; import { getConstraintIcon, getEntityIcon } from '../../../utils/TableUtils'; import { useLineageProvider } from '../../LineageProvider/LineageProvider'; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index ceba72d4cf19..c06fb54c6f1c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -51,10 +51,17 @@ import { import { getLineageDataByFQN } from '../../rest/lineageAPI'; import { addLineageHandler, + createEdges, + createNodes, getAllTracedColumnEdge, getAllTracedNodes, getClassifiedEdge, + getColumnLineageData, + getConnectedNodesEdges, getLayoutedElements, + getLineageDetailsObject, + getLineageEdge, + getLineageEdgeForAPI, getLoadingStatusValue, getModalBodyText, getNewLineageConnectionDetails, @@ -62,15 +69,7 @@ import { onLoad, removeLineageHandler, } from '../../utils/EntityLineageUtils'; -import { - createEdges, - createNodes, - getColumnLineageData, - getConnectedNodesEdges, - getLineageDetailsObject, - getLineageEdge, - getLineageEdgeForAPI, -} from '../../utils/LineageV1Utils'; + import SVGIcons from '../../utils/SvgUtils'; import { showErrorToast } from '../../utils/ToastUtils'; import EdgeInfoDrawer from '../Entity/EntityInfoDrawer/EdgeInfoDrawer.component'; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index c5d12494e095..e70f9f74d9b7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -16,11 +16,21 @@ import { Button, Typography } from 'antd'; import { AxiosError } from 'axios'; import dagre from 'dagre'; import { t } from 'i18next'; -import { isEmpty, isUndefined, upperCase } from 'lodash'; +import { isEmpty, isNil, isUndefined, upperCase } from 'lodash'; import { LoadingState } from 'Models'; import React, { Fragment, MouseEvent as ReactMouseEvent } from 'react'; import { Link } from 'react-router-dom'; -import { Edge, isNode, Node, Position, ReactFlowInstance } from 'reactflow'; +import { + Edge, + getConnectedEdges, + getIncomers, + getOutgoers, + isNode, + MarkerType, + Node, + Position, + ReactFlowInstance, +} from 'reactflow'; import { ReactComponent as DashboardIcon } from '../assets/svg/dashboard-grey.svg'; import { ReactComponent as MlModelIcon } from '../assets/svg/mlmodal.svg'; import { ReactComponent as PipelineIcon } from '../assets/svg/pipeline-grey.svg'; @@ -33,10 +43,12 @@ import { CustomElement, CustomFlow, EdgeData, + EdgeTypeEnum, } from '../components/Entity/EntityLineage/EntityLineage.interface'; import { ExploreSearchIndex } from '../components/Explore/ExplorePage.interface'; import { EdgeDetails } from '../components/Lineage/Lineage.interface'; import Loader from '../components/Loader/Loader'; +import { SourceType } from '../components/SearchedData/SearchedData.interface'; import { INFO_COLOR } from '../constants/constants'; import { EXPANDED_NODE_HEIGHT, @@ -46,12 +58,13 @@ import { } from '../constants/Lineage.constants'; import { EntityLineageDirection, + EntityLineageNodeType, EntityType, FqnPart, } from '../enums/entity.enum'; import { SearchIndex } from '../enums/search.enum'; -import { AddLineage } from '../generated/api/lineage/addLineage'; -import { LineageDetails } from '../generated/type/entityLineage'; +import { AddLineage, EntitiesEdge } from '../generated/api/lineage/addLineage'; +import { ColumnLineage, LineageDetails } from '../generated/type/entityLineage'; import { EntityReference } from '../generated/type/entityReference'; import { addLineage, deleteLineageEdge } from '../rest/miscAPI'; import { @@ -533,3 +546,349 @@ export const getSearchIndexFromNodeType = (entityType: string) => { return SearchIndex[searchIndexKey] as ExploreSearchIndex; }; + +export const checkUpstreamDownstream = (id: string, data: EdgeDetails[]) => { + const hasUpstream = data.some((edge: EdgeDetails) => edge.toEntity.id === id); + + const hasDownstream = data.some( + (edge: EdgeDetails) => edge.fromEntity.id === id + ); + + return { hasUpstream, hasDownstream }; +}; + +const removeDuplicateNodes = (nodesData: EntityReference[]) => { + const uniqueNodesMap = new Map(); + nodesData.forEach((node) => { + uniqueNodesMap.set(node.fullyQualifiedName ?? '', node); + }); + + const uniqueNodesArray = Array.from(uniqueNodesMap.values()); + + return uniqueNodesArray; +}; + +const getNodeType = ( + edgesData: EdgeDetails[], + id: string +): EntityLineageNodeType => { + const hasDownStreamToEntity = edgesData.find( + (down) => down.toEntity.id === id + ); + const hasDownStreamFromEntity = edgesData.find( + (down) => down.fromEntity.id === id + ); + const hasUpstreamFromEntity = edgesData.find((up) => up.fromEntity.id === id); + const hasUpstreamToEntity = edgesData.find((up) => up.toEntity.id === id); + + if (hasDownStreamToEntity && !hasDownStreamFromEntity) { + return EntityLineageNodeType.OUTPUT; + } + if (hasUpstreamFromEntity && !hasUpstreamToEntity) { + return EntityLineageNodeType.INPUT; + } + + return EntityLineageNodeType.DEFAULT; +}; + +export const createNodes = ( + nodesData: EntityReference[], + edgesData: EdgeDetails[] +) => { + const uniqueNodesData = removeDuplicateNodes(nodesData); + + // Create a new dagre graph + const graph = new dagre.graphlib.Graph(); + + // Set an object for the graph label + graph.setGraph({ rankdir: EntityLineageDirection.LEFT_RIGHT }); + + // Default to assigning a new object as a label for each new edge. + graph.setDefaultEdgeLabel(() => ({})); + + // Add nodes to the graph + uniqueNodesData.forEach((node) => { + graph.setNode(node.id, { width: NODE_WIDTH, height: NODE_HEIGHT }); + }); + + // Add edges to the graph (if you have edge information) + edgesData.forEach((edge) => { + graph.setEdge(edge.fromEntity.id, edge.toEntity.id); + }); + + // Perform the layout + dagre.layout(graph); + + // Get the layout positions + const layoutPositions = graph.nodes().map((nodeId) => graph.node(nodeId)); + + return uniqueNodesData.map((node, index) => { + const position = layoutPositions[index]; + + const type = getNodeType(edgesData, node.id); + + return { + id: `${node.id}`, + sourcePosition: Position.Right, + targetPosition: Position.Left, + type: type, + className: '', + data: { + node, + }, + position: { + x: position.x, + y: position.y, + }, + }; + }); +}; + +export const createEdges = (nodes: EntityReference[], edges: EdgeDetails[]) => { + const lineageEdgesV1: Edge[] = []; + + edges.forEach((edge) => { + const sourceType = nodes.find((n) => edge.fromEntity.id === n.id); + const targetType = nodes.find((n) => edge.toEntity.id === n.id); + + if (isUndefined(sourceType) || isUndefined(targetType)) { + return; + } + + if (!isUndefined(edge.columns)) { + edge.columns?.forEach((e) => { + const toColumn = e.toColumn ?? ''; + if (toColumn && e.fromColumns && e.fromColumns.length > 0) { + e.fromColumns.forEach((fromColumn) => { + lineageEdgesV1.push({ + id: `column-${fromColumn}-${toColumn}-edge-${edge.fromEntity.id}-${edge.toEntity.id}`, + source: edge.fromEntity.id, + target: edge.toEntity.id, + targetHandle: toColumn, + sourceHandle: fromColumn, + style: { strokeWidth: '2px' }, + type: 'buttonedge', + markerEnd: { + type: MarkerType.ArrowClosed, + }, + data: { + edge, + isColumnLineage: true, + targetHandle: toColumn, + sourceHandle: fromColumn, + }, + }); + }); + } + }); + } + + lineageEdgesV1.push({ + id: `edge-${edge.fromEntity.id}-${edge.toEntity.id}`, + source: `${edge.fromEntity.id}`, + target: `${edge.toEntity.id}`, + type: 'buttonedge', + animated: !isNil(edge.pipeline), + style: { strokeWidth: '2px' }, + markerEnd: { + type: MarkerType.ArrowClosed, + }, + data: { + edge, + isColumnLineage: false, + }, + }); + }); + + return lineageEdgesV1; +}; + +export const getColumnLineageData = ( + columnsData: ColumnLineage[], + data: Edge +) => { + const columnsLineage = columnsData?.reduce((col, curr) => { + if (curr.toColumn === data.data?.targetHandle) { + const newCol = { + ...curr, + fromColumns: + curr.fromColumns?.filter( + (column) => column !== data.data?.sourceHandle + ) ?? [], + }; + if (newCol.fromColumns?.length) { + return [...col, newCol]; + } else { + return col; + } + } + + return [...col, curr]; + }, [] as ColumnLineage[]); + + return columnsLineage; +}; + +export const getLineageEdge = ( + sourceNode: SourceType, + targetNode: SourceType +): { edge: EdgeDetails } => { + const { + id: sourceId, + entityType: sourceType, + fullyQualifiedName: sourceFqn, + } = sourceNode; + const { + id: targetId, + entityType: targetType, + fullyQualifiedName: targetFqn, + } = targetNode; + + return { + edge: { + fromEntity: { + id: sourceId, + type: sourceType ?? '', + fqn: sourceFqn ?? '', + }, + toEntity: { id: targetId, type: targetType ?? '', fqn: targetFqn ?? '' }, + sqlQuery: '', + }, + }; +}; + +export const getLineageEdgeForAPI = ( + sourceNode: SourceType, + targetNode: SourceType +): { edge: EntitiesEdge } => { + const { id: sourceId, entityType: sourceType } = sourceNode; + const { id: targetId, entityType: targetType } = targetNode; + + return { + edge: { + fromEntity: { id: sourceId, type: sourceType ?? '' }, + toEntity: { id: targetId, type: targetType ?? '' }, + lineageDetails: { + sqlQuery: '', + columnsLineage: [], + }, + }, + }; +}; + +export const getLineageDetailsObject = (edge: Edge): LineageDetails => { + const { data } = edge; + + return { + sqlQuery: data?.edge?.sqlQuery ?? '', + columnsLineage: data?.edge?.columns ?? [], + description: data?.edge?.description ?? '', + pipeline: data?.edge?.pipeline ?? undefined, + source: data?.edge?.source ?? '', + }; +}; + +const checkTarget = (edgesObj: Edge[], id: string) => { + const edges = edgesObj.filter((ed) => { + return ed.target !== id; + }); + + return edges; +}; + +const checkSource = (edgesObj: Edge[], id: string) => { + const edges = edgesObj.filter((ed) => { + return ed.source !== id; + }); + + return edges; +}; + +const getOutgoersAndConnectedEdges = ( + node: Node, + allNodes: Node[], + allEdges: Edge[], + currentNodeID: string +) => { + const outgoers = getOutgoers(node, allNodes, allEdges); + const connectedEdges = checkTarget( + getConnectedEdges([node], allEdges), + currentNodeID + ); + + return { outgoers, connectedEdges }; +}; + +const getIncomersAndConnectedEdges = ( + node: Node, + allNodes: Node[], + allEdges: Edge[], + currentNodeID: string +) => { + const outgoers = getIncomers(node, allNodes, allEdges); + const connectedEdges = checkSource( + getConnectedEdges([node], allEdges), + currentNodeID + ); + + return { outgoers, connectedEdges }; +}; + +/** + * This function returns all downstream nodes and edges of given node. + * The output of this method is further passed to collapse downstream nodes and edges. + * + * @param {Node} selectedNode - The node for which to retrieve the downstream nodes and edges. + * @param {Node[]} nodes - All nodes in the lineage. + * @param {Edge[]} edges - All edges in the lineage. + * @return {{ nodes: Node[]; edges: Edge[], nodeIds: string[], edgeIds: string[] }} - + * An object containing the downstream nodes and edges. + */ +export const getConnectedNodesEdges = ( + selectedNode: Node, + nodes: Node[], + edges: Edge[], + direction: EdgeTypeEnum +): { nodes: Node[]; edges: Edge[]; nodeFqn: string[] } => { + const visitedNodes = new Set(); + const outgoers: Node[] = []; + const connectedEdges: Edge[] = []; + const stack: Node[] = [selectedNode]; + const currentNodeID = selectedNode.id; + + while (stack.length > 0) { + const currentNode = stack.pop(); + if (currentNode && !visitedNodes.has(currentNode.id)) { + visitedNodes.add(currentNode.id); + + const { outgoers: childNodes, connectedEdges: childEdges } = + direction === EdgeTypeEnum.DOWN_STREAM + ? getOutgoersAndConnectedEdges( + currentNode, + nodes, + edges, + currentNodeID + ) + : getIncomersAndConnectedEdges( + currentNode, + nodes, + edges, + currentNodeID + ); + + stack.push(...childNodes); + outgoers.push(...childNodes); + connectedEdges.push(...childEdges); + } + } + + const childNodeFqn = outgoers.map( + (node) => node.data.node.fullyQualifiedName + ); + + return { + nodes: outgoers, + edges: connectedEdges, + nodeFqn: childNodeFqn, + }; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts deleted file mode 100644 index a85da75bc6fb..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/utils/LineageV1Utils.ts +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Copyright 2023 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import dagre from 'dagre'; -import { isNil, isUndefined } from 'lodash'; -import { - Edge, - getConnectedEdges, - getIncomers, - getOutgoers, - MarkerType, - Node, - Position, -} from 'reactflow'; -import { EdgeTypeEnum } from '../components/Entity/EntityLineage/EntityLineage.interface'; -import { EdgeDetails } from '../components/Lineage/Lineage.interface'; -import { SourceType } from '../components/SearchedData/SearchedData.interface'; -import { NODE_HEIGHT, NODE_WIDTH } from '../constants/Lineage.constants'; -import { - EntityLineageDirection, - EntityLineageNodeType, -} from '../enums/entity.enum'; -import { EntitiesEdge } from '../generated/api/lineage/addLineage'; -import { EntityReference } from '../generated/entity/type'; -import { ColumnLineage, LineageDetails } from '../generated/type/entityLineage'; - -export const checkUpstreamDownstream = (id: string, data: EdgeDetails[]) => { - const hasUpstream = data.some((edge: EdgeDetails) => edge.toEntity.id === id); - - const hasDownstream = data.some( - (edge: EdgeDetails) => edge.fromEntity.id === id - ); - - return { hasUpstream, hasDownstream }; -}; - -const removeDuplicateNodes = (nodesData: EntityReference[]) => { - const uniqueNodesMap = new Map(); - nodesData.forEach((node) => { - uniqueNodesMap.set(node.fullyQualifiedName ?? '', node); - }); - - const uniqueNodesArray = Array.from(uniqueNodesMap.values()); - - return uniqueNodesArray; -}; - -const getNodeType = ( - edgesData: EdgeDetails[], - id: string -): EntityLineageNodeType => { - const hasDownStreamToEntity = edgesData.find( - (down) => down.toEntity.id === id - ); - const hasDownStreamFromEntity = edgesData.find( - (down) => down.fromEntity.id === id - ); - const hasUpstreamFromEntity = edgesData.find((up) => up.fromEntity.id === id); - const hasUpstreamToEntity = edgesData.find((up) => up.toEntity.id === id); - - if (hasDownStreamToEntity && !hasDownStreamFromEntity) { - return EntityLineageNodeType.OUTPUT; - } - if (hasUpstreamFromEntity && !hasUpstreamToEntity) { - return EntityLineageNodeType.INPUT; - } - - return EntityLineageNodeType.DEFAULT; -}; - -export const createNodes = ( - nodesData: EntityReference[], - edgesData: EdgeDetails[] -) => { - const uniqueNodesData = removeDuplicateNodes(nodesData); - - // Create a new dagre graph - const graph = new dagre.graphlib.Graph(); - - // Set an object for the graph label - graph.setGraph({ rankdir: EntityLineageDirection.LEFT_RIGHT }); - - // Default to assigning a new object as a label for each new edge. - graph.setDefaultEdgeLabel(() => ({})); - - // Add nodes to the graph - uniqueNodesData.forEach((node) => { - graph.setNode(node.id, { width: NODE_WIDTH, height: NODE_HEIGHT }); - }); - - // Add edges to the graph (if you have edge information) - edgesData.forEach((edge) => { - graph.setEdge(edge.fromEntity.id, edge.toEntity.id); - }); - - // Perform the layout - dagre.layout(graph); - - // Get the layout positions - const layoutPositions = graph.nodes().map((nodeId) => graph.node(nodeId)); - - return uniqueNodesData.map((node, index) => { - const position = layoutPositions[index]; - - const type = getNodeType(edgesData, node.id); - - return { - id: `${node.id}`, - sourcePosition: Position.Right, - targetPosition: Position.Left, - type: type, - className: '', - data: { - node, - }, - position: { - x: position.x, - y: position.y, - }, - }; - }); -}; - -export const createEdges = (nodes: EntityReference[], edges: EdgeDetails[]) => { - const lineageEdgesV1: Edge[] = []; - - edges.forEach((edge) => { - const sourceType = nodes.find((n) => edge.fromEntity.id === n.id); - const targetType = nodes.find((n) => edge.toEntity.id === n.id); - - if (isUndefined(sourceType) || isUndefined(targetType)) { - return; - } - - if (!isUndefined(edge.columns)) { - edge.columns?.forEach((e) => { - const toColumn = e.toColumn || ''; - if (toColumn && e.fromColumns && e.fromColumns.length > 0) { - e.fromColumns.forEach((fromColumn) => { - lineageEdgesV1.push({ - id: `column-${fromColumn}-${toColumn}-edge-${edge.fromEntity.id}-${edge.toEntity.id}`, - source: edge.fromEntity.id, - target: edge.toEntity.id, - targetHandle: toColumn, - sourceHandle: fromColumn, - style: { strokeWidth: '2px' }, - type: 'buttonedge', - markerEnd: { - type: MarkerType.ArrowClosed, - }, - data: { - edge, - isColumnLineage: true, - targetHandle: toColumn, - sourceHandle: fromColumn, - }, - }); - }); - } - }); - } - - lineageEdgesV1.push({ - id: `edge-${edge.fromEntity.id}-${edge.toEntity.id}`, - source: `${edge.fromEntity.id}`, - target: `${edge.toEntity.id}`, - type: 'buttonedge', - animated: !isNil(edge.pipeline), - style: { strokeWidth: '2px' }, - markerEnd: { - type: MarkerType.ArrowClosed, - }, - data: { - edge, - isColumnLineage: false, - }, - }); - }); - - return lineageEdgesV1; -}; - -export const getColumnLineageData = ( - columnsData: ColumnLineage[], - data: Edge -) => { - const columnsLineage = columnsData?.reduce((col, curr) => { - if (curr.toColumn === data.data?.targetHandle) { - const newCol = { - ...curr, - fromColumns: - curr.fromColumns?.filter( - (column) => column !== data.data?.sourceHandle - ) ?? [], - }; - if (newCol.fromColumns?.length) { - return [...col, newCol]; - } else { - return col; - } - } - - return [...col, curr]; - }, [] as ColumnLineage[]); - - return columnsLineage; -}; - -export const getLineageEdge = ( - sourceNode: SourceType, - targetNode: SourceType -): { edge: EdgeDetails } => { - const { - id: sourceId, - entityType: sourceType, - fullyQualifiedName: sourceFqn, - } = sourceNode; - const { - id: targetId, - entityType: targetType, - fullyQualifiedName: targetFqn, - } = targetNode; - - return { - edge: { - fromEntity: { - id: sourceId, - type: sourceType ?? '', - fqn: sourceFqn ?? '', - }, - toEntity: { id: targetId, type: targetType ?? '', fqn: targetFqn ?? '' }, - sqlQuery: '', - }, - }; -}; - -export const getLineageEdgeForAPI = ( - sourceNode: SourceType, - targetNode: SourceType -): { edge: EntitiesEdge } => { - const { id: sourceId, entityType: sourceType } = sourceNode; - const { id: targetId, entityType: targetType } = targetNode; - - return { - edge: { - fromEntity: { id: sourceId, type: sourceType ?? '' }, - toEntity: { id: targetId, type: targetType ?? '' }, - lineageDetails: { - sqlQuery: '', - columnsLineage: [], - }, - }, - }; -}; - -export const getLineageDetailsObject = (edge: Edge): LineageDetails => { - const { data } = edge; - - return { - sqlQuery: data?.edge?.sqlQuery ?? '', - columnsLineage: data?.edge?.columns ?? [], - description: data?.edge?.description ?? '', - pipeline: data?.edge?.pipeline ?? undefined, - source: data?.edge?.source ?? '', - }; -}; - -const checkTarget = (edgesObj: Edge[], id: string) => { - const edges = edgesObj.filter((ed) => { - return ed.target !== id; - }); - - return edges; -}; - -const checkSource = (edgesObj: Edge[], id: string) => { - const edges = edgesObj.filter((ed) => { - return ed.source !== id; - }); - - return edges; -}; - -const getOutgoersAndConnectedEdges = ( - node: Node, - allNodes: Node[], - allEdges: Edge[], - currentNodeID: string -) => { - const outgoers = getOutgoers(node, allNodes, allEdges); - const connectedEdges = checkTarget( - getConnectedEdges([node], allEdges), - currentNodeID - ); - - return { outgoers, connectedEdges }; -}; - -const getIncomersAndConnectedEdges = ( - node: Node, - allNodes: Node[], - allEdges: Edge[], - currentNodeID: string -) => { - const outgoers = getIncomers(node, allNodes, allEdges); - const connectedEdges = checkSource( - getConnectedEdges([node], allEdges), - currentNodeID - ); - - return { outgoers, connectedEdges }; -}; - -/** - * This function returns all downstream nodes and edges of given node. - * The output of this method is further passed to collapse downstream nodes and edges. - * - * @param {Node} selectedNode - The node for which to retrieve the downstream nodes and edges. - * @param {Node[]} nodes - All nodes in the lineage. - * @param {Edge[]} edges - All edges in the lineage. - * @return {{ nodes: Node[]; edges: Edge[], nodeIds: string[], edgeIds: string[] }} - - * An object containing the downstream nodes and edges. - */ -export const getConnectedNodesEdges = ( - selectedNode: Node, - nodes: Node[], - edges: Edge[], - direction: EdgeTypeEnum -): { nodes: Node[]; edges: Edge[]; nodeFqn: string[] } => { - const visitedNodes = new Set(); - const outgoers: Node[] = []; - const connectedEdges: Edge[] = []; - const stack: Node[] = [selectedNode]; - const currentNodeID = selectedNode.id; - - while (stack.length > 0) { - const currentNode = stack.pop(); - if (currentNode && !visitedNodes.has(currentNode.id)) { - visitedNodes.add(currentNode.id); - - const { outgoers: childNodes, connectedEdges: childEdges } = - direction === EdgeTypeEnum.DOWN_STREAM - ? getOutgoersAndConnectedEdges( - currentNode, - nodes, - edges, - currentNodeID - ) - : getIncomersAndConnectedEdges( - currentNode, - nodes, - edges, - currentNodeID - ); - - stack.push(...childNodes); - outgoers.push(...childNodes); - connectedEdges.push(...childEdges); - } - } - - const childNodeFqn = outgoers.map( - (node) => node.data.node.fullyQualifiedName - ); - - return { - nodes: outgoers, - edges: connectedEdges, - nodeFqn: childNodeFqn, - }; -}; From 68b3a7f72d7402fc56ef652c2503381588b2d782 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 20 Dec 2023 17:37:11 +0530 Subject: [PATCH 21/44] cleanup --- .../Entity/EntityLineage/CustomNode.utils.tsx | 51 +++++++ .../EntityLineage/CustomNodeV1.component.tsx | 121 ++++++--------- .../components/Lineage/Lineage.component.tsx | 5 +- .../src/components/Lineage/Lineage.test.tsx | 139 ++++++++++++++++++ .../LineageProvider.interface.tsx | 74 ++++++++++ .../LineageProvider/LineageProvider.tsx | 12 +- .../ui/src/utils/EntityLineageUtils.tsx | 18 ++- 7 files changed, 330 insertions(+), 90 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.test.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.interface.tsx diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx index 91809c4f48d2..2cea7732cc73 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx @@ -10,9 +10,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Button } from 'antd'; +import classNames from 'classnames'; import React, { Fragment } from 'react'; import { Handle, HandleProps, HandleType, Position } from 'reactflow'; +import { ReactComponent as MinusIcon } from '../../../assets/svg/control-minus.svg'; +import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-outlined.svg'; import { EntityLineageNodeType } from '../../../enums/entity.enum'; +import { EdgeTypeEnum } from './EntityLineage.interface'; export const getColumnHandle = ( nodeType: string, @@ -55,3 +60,49 @@ export const getHandleByType = ( /> ); }; + +export const getExpandHandle = ( + direction: EdgeTypeEnum, + onClickHandler: () => void +) => { + return ( + + +
+ ); +}; + jest.mock('react-router-dom', () => ({ useHistory: jest.fn().mockReturnValue({ push: jest.fn(), listen: jest.fn() }), useLocation: jest.fn().mockImplementation(() => mockLocation), @@ -48,4 +82,32 @@ describe('LineageProvider', () => { expect(getLineageDataByFQN).toHaveBeenCalled(); }); + + it('should call loadChildNodesHandler', async () => { + await act(async () => { + render( + + + + ); + }); + + const loadButton = await screen.getByTestId('load-nodes'); + fireEvent.click(loadButton); + + expect(getLineageDataByFQN).toHaveBeenCalled(); + }); + + it.skip('should call onNodeClick', async () => { + await act(async () => { + render( + + + + ); + }); + + const nodeClick = await screen.getByTestId('node1'); + fireEvent.click(nodeClick); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx index edc9efe63aeb..e17f75fdbf80 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx @@ -226,7 +226,7 @@ describe('Test EntityLineageUtils utility', () => { expect(result.edge.toEntity.type).toBe('table'); }); - it.skip('should handle different scenarios for getColumnLineageData', () => { + it('should handle different scenarios for getColumnLineageData', () => { const mockEdge = { data: { targetHandle: 'target', sourceHandle: 'source' }, } as Edge; @@ -254,14 +254,17 @@ describe('Test EntityLineageUtils utility', () => { mockEdge ); - // Assert expect(resultUndefined).toEqual([]); expect(resultNoMatch).toEqual(columnsDataNoMatch); expect(resultRemoveSource).toEqual([ - { toColumn: 'column1', fromColumns: ['column2', 'column3'] }, + { toColumn: 'column1', fromColumns: ['column2', 'column3', 'source'] }, { toColumn: 'column4', fromColumns: ['column5', 'column6'] }, ]); expect(resultEmptyResult).toEqual([ + { + fromColumns: ['source'], + toColumn: 'column1', + }, { toColumn: 'column4', fromColumns: ['column5', 'column6'] }, ]); }); From be25483e951d0c39cfe1e5be8f7ab465c87497e8 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Fri, 22 Dec 2023 16:41:19 +0530 Subject: [PATCH 24/44] cleanup --- .../LineageProvider/LineageProvider.test.tsx | 48 ++- .../LineageProvider/LineageProvider.tsx | 1 + .../resources/ui/src/mocks/Lineage.mock.ts | 363 ------------------ .../ui/src/utils/EntityLineageUtils.test.tsx | 112 ++++++ 4 files changed, 151 insertions(+), 373 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.test.tsx index a950a47739dd..f95e9f771459 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.test.tsx @@ -12,6 +12,7 @@ */ import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; +import { Edge } from 'reactflow'; import { getLineageDataByFQN } from '../../rest/lineageAPI'; import { EdgeTypeEnum } from '../Entity/EntityLineage/EntityLineage.interface'; import LineageProvider, { useLineageProvider } from './LineageProvider'; @@ -22,7 +23,7 @@ const mockLocation = { }; const DummyChildrenComponent = () => { - const { loadChildNodesHandler, onNodeClick } = useLineageProvider(); + const { loadChildNodesHandler, onEdgeClick } = useLineageProvider(); const nodeData = { name: 'table1', @@ -30,11 +31,23 @@ const DummyChildrenComponent = () => { fullyQualifiedName: 'table1', id: 'table1', }; - const nodeObj = { - id: 'dummyId', - position: { x: 100, y: 100 }, + + const MOCK_EDGE = { + id: 'test', + source: 'test', + target: 'test', + type: 'test', data: { - ...nodeData, + edge: { + fromEntity: { + id: 'test', + type: 'test', + }, + toEntity: { + id: 'test', + type: 'test', + }, + }, }, }; const handleButtonClick = () => { @@ -47,8 +60,13 @@ const DummyChildrenComponent = () => { - +
); @@ -62,6 +80,12 @@ jest.mock('react-router-dom', () => ({ }), })); +jest.mock('../Entity/EntityInfoDrawer/EdgeInfoDrawer.component', () => { + return jest.fn().mockImplementation(() => { + return

Edge Info Drawer

; + }); +}); + jest.mock('../../rest/lineageAPI', () => ({ getLineageDataByFQN: jest.fn(), })); @@ -98,7 +122,7 @@ describe('LineageProvider', () => { expect(getLineageDataByFQN).toHaveBeenCalled(); }); - it.skip('should call onNodeClick', async () => { + it('should show delete modal', async () => { await act(async () => { render( @@ -107,7 +131,11 @@ describe('LineageProvider', () => { ); }); - const nodeClick = await screen.getByTestId('node1'); - fireEvent.click(nodeClick); + const edgeClick = await screen.getByTestId('edge-click'); + fireEvent.click(edgeClick); + + const edgeDrawer = screen.getByText('Edge Info Drawer'); + + expect(edgeDrawer).toBeInTheDocument(); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index 99bf912d5436..c5a3d33a839d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -955,6 +955,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { {showDeleteModal && ( ({ + addLineage: jest.fn(), +})); + describe('Test EntityLineageUtils utility', () => { it('getAllTracedNodes & isTracedEdge function should work properly', () => { const { nodes, edges } = MOCK_NODES_AND_EDGES; @@ -268,4 +277,107 @@ describe('Test EntityLineageUtils utility', () => { { toColumn: 'column4', fromColumns: ['column5', 'column6'] }, ]); }); + + it('getConnectedNodesEdges should return an object with nodes, edges, and nodeFqn properties for downstream', () => { + const selectedNode = { id: '1', position: { x: 0, y: 0 }, data: {} }; + const nodes = [ + { + id: '1', + position: { x: 0, y: 0 }, + data: { node: { fullyQualifiedName: '1' } }, + }, + { + id: '2', + position: { x: 0, y: 0 }, + data: { node: { fullyQualifiedName: '2' } }, + }, + { + id: '3', + position: { x: 0, y: 0 }, + data: { node: { fullyQualifiedName: '3' } }, + }, + ]; + const edges = [ + { id: '1', source: '1', target: '2' }, + { id: '2', source: '1', target: '3' }, + { id: '3', source: '2', target: '3' }, + ]; + const direction = EdgeTypeEnum.DOWN_STREAM; + + const result = getConnectedNodesEdges( + selectedNode, + nodes, + edges, + direction + ); + + expect(result).toHaveProperty('nodes'); + expect(result).toHaveProperty('edges'); + expect(result).toHaveProperty('nodeFqn'); + + expect(result.nodes).toContainEqual(nodes[1]); + expect(result.nodes).toContainEqual(nodes[2]); + expect(result.edges).toContainEqual(edges[1]); + expect(result.edges).toContainEqual(edges[2]); + + const emptyResult = getConnectedNodesEdges(selectedNode, [], [], direction); + + expect(emptyResult.nodes).toEqual([]); + expect(emptyResult.edges).toEqual([]); + }); + + it('getConnectedNodesEdges should return an object with nodes, edges, and nodeFqn properties for upstream', () => { + const selectedNode = { id: '1', position: { x: 0, y: 0 }, data: {} }; + const nodes = [ + { + id: '1', + position: { x: 0, y: 0 }, + data: { node: { fullyQualifiedName: '1' } }, + }, + { + id: '2', + position: { x: 0, y: 0 }, + data: { node: { fullyQualifiedName: '2' } }, + }, + { + id: '3', + position: { x: 0, y: 0 }, + data: { node: { fullyQualifiedName: '3' } }, + }, + ]; + const edges = [ + { id: '1', source: '1', target: '2' }, + { id: '2', source: '1', target: '3' }, + { id: '3', source: '2', target: '3' }, + ]; + const direction = EdgeTypeEnum.UP_STREAM; + + const result = getConnectedNodesEdges( + selectedNode, + nodes, + edges, + direction + ); + + expect(result).toHaveProperty('nodes'); + expect(result).toHaveProperty('edges'); + expect(result).toHaveProperty('nodeFqn'); + + expect(result.nodes).toEqual([]); + expect(result.edges).toEqual([]); + + const emptyResult = getConnectedNodesEdges(selectedNode, [], [], direction); + + expect(emptyResult.nodes).toEqual([]); + expect(emptyResult.edges).toEqual([]); + }); + + it('should call addLineage with the provided edge', async () => { + const edge = { + edge: { fromEntity: {}, toEntity: {} }, + } as AddLineage; + await addLineageHandler(edge); + + expect(addLineage).toHaveBeenCalledWith(edge); + }); }); From f24859829907919a40cdc9200c5679b7ef819fac Mon Sep 17 00:00:00 2001 From: karanh37 Date: Mon, 1 Jan 2024 12:33:56 +0530 Subject: [PATCH 25/44] update filter query --- .../AssetSelectionModal.tsx | 33 ++--------------- .../EdgeInfoDrawer.component.tsx | 12 ++++--- .../CustomControls.component.tsx | 35 +++--------------- .../tabs/AssetsTabs.component.tsx | 30 ++-------------- .../LineageProvider/LineageProvider.tsx | 16 +++++++-- .../src/constants/AdvancedSearch.constants.ts | 4 +++ .../ui/src/utils/EntityLineageUtils.tsx | 4 +-- .../resources/ui/src/utils/Explore.utils.ts | 36 ++++++++++++++++++- 8 files changed, 72 insertions(+), 98 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx index 1404129cff04..402a7b519d69 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx @@ -30,7 +30,6 @@ import { import { ItemType } from 'antd/lib/menu/hooks/useItems'; import { AxiosError } from 'axios'; import classNames from 'classnames'; -import { isEmpty } from 'lodash'; import { EntityDetailUnion } from 'Models'; import VirtualList from 'rc-virtual-list'; import { @@ -52,11 +51,7 @@ import { Status, } from '../../../generated/type/bulkOperationResult'; import { Aggregations } from '../../../interface/search.interface'; -import { - QueryFieldInterface, - QueryFieldValueInterface, - QueryFilterInterface, -} from '../../../pages/ExplorePage/ExplorePage.interface'; +import { QueryFilterInterface } from '../../../pages/ExplorePage/ExplorePage.interface'; import { addAssetsToDataProduct, getDataProductByName, @@ -71,6 +66,7 @@ import { getAssetsPageQuickFilters } from '../../../utils/AdvancedSearchUtils'; import { getEntityReferenceFromEntity } from '../../../utils/EntityUtils'; import { getAggregations, + getQuickFilterQuery, getSelectedValuesFromQuickFilter, } from '../../../utils/Explore.utils'; import { getCombinedQueryFilterObject } from '../../../utils/ExplorePage/ExplorePageUtils'; @@ -422,30 +418,7 @@ export const AssetSelectionModal = ({ ); const handleQuickFiltersChange = (data: ExploreQuickFilterField[]) => { - const must: QueryFieldInterface[] = []; - data.forEach((filter) => { - if (!isEmpty(filter.value)) { - const should: QueryFieldValueInterface[] = []; - if (filter.value) { - filter.value.forEach((filterValue) => { - const term: Record = {}; - term[filter.key] = filterValue.key; - should.push({ term }); - }); - } - - must.push({ - bool: { should }, - }); - } - }); - - const quickFilterQuery = isEmpty(must) - ? undefined - : { - query: { bool: { must } }, - }; - + const quickFilterQuery = getQuickFilterQuery(data); setQuickFilterQuery(quickFilterQuery); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx index 2d30e1a0a1d1..c20dd1283c06 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx @@ -22,6 +22,7 @@ import DescriptionV1 from '../../../components/common/EntityDescription/Descript import { CSMode } from '../../../enums/codemirror.enum'; import { EntityType } from '../../../enums/entity.enum'; import { getNameFromFQN } from '../../../utils/CommonUtils'; +import { getLineageDetailsObject } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; import { getEncodedFqn } from '../../../utils/StringsUtils'; import { getEntityLink } from '../../../utils/TableUtils'; @@ -121,18 +122,19 @@ const EdgeInfoDrawer = ({ async (updatedHTML: string) => { if (edgeDescription !== updatedHTML && edgeEntity) { const lineageDetails = { - ...edgeEntity.lineageDetails, + ...getLineageDetailsObject(edgeEntity), description: updatedHTML, }; + const updatedEdgeDetails = { edge: { fromEntity: { - id: edgeEntity.fromEntity, - type: edge.data.sourceType, + id: edgeEntity.fromEntity.id, + type: edgeEntity.fromEntity.type, }, toEntity: { - id: edgeEntity.toEntity, - type: edge.data.sourceType, + id: edgeEntity.toEntity.id, + type: edgeEntity.toEntity.type, }, lineageDetails, }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx index bdc2721cbfc4..aa8e801bdc05 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx @@ -15,7 +15,6 @@ import { RightOutlined, SettingOutlined } from '@ant-design/icons'; import { Button, Col, Dropdown, Row, Select, Space } from 'antd'; import { ItemType } from 'antd/lib/menu/hooks/useItems'; import classNames from 'classnames'; -import { isEmpty } from 'lodash'; import React, { FC, memo, @@ -37,15 +36,14 @@ import { ZOOM_TRANSITION_DURATION, } from '../../../constants/Lineage.constants'; import { SearchIndex } from '../../../enums/search.enum'; -import { - QueryFieldInterface, - QueryFieldValueInterface, -} from '../../../pages/ExplorePage/ExplorePage.interface'; import { getAssetsPageQuickFilters } from '../../../utils/AdvancedSearchUtils'; import { handleSearchFilterOption } from '../../../utils/CommonUtils'; import { getLoadingStatusValue } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; -import { getSelectedValuesFromQuickFilter } from '../../../utils/Explore.utils'; +import { + getQuickFilterQuery, + getSelectedValuesFromQuickFilter, +} from '../../../utils/Explore.utils'; import { ExploreQuickFilterField } from '../../Explore/ExplorePage.interface'; import ExploreQuickFilters from '../../Explore/ExploreQuickFilters'; import { useLineageProvider } from '../../LineageProvider/LineageProvider'; @@ -170,30 +168,7 @@ const CustomControls: FC = ({ ); const handleQuickFiltersChange = (data: ExploreQuickFilterField[]) => { - const must: QueryFieldInterface[] = []; - data.forEach((filter) => { - if (!isEmpty(filter.value)) { - const should: QueryFieldValueInterface[] = []; - if (filter.value) { - filter.value.forEach((filterValue) => { - const term: Record = {}; - term[filter.key] = filterValue.key; - should.push({ term }); - }); - } - - must.push({ - bool: { should }, - }); - } - }); - - const quickFilterQuery = isEmpty(must) - ? undefined - : { - query: { bool: { must } }, - }; - + const quickFilterQuery = getQuickFilterQuery(data); onQueryFilterUpdate(JSON.stringify(quickFilterQuery)); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx index fcec4499c510..f41fb5b89ca8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx @@ -61,10 +61,6 @@ import { DataProduct } from '../../../../generated/entity/domains/dataProduct'; import { Domain } from '../../../../generated/entity/domains/domain'; import { usePaging } from '../../../../hooks/paging/usePaging'; import { Aggregations } from '../../../../interface/search.interface'; -import { - QueryFieldInterface, - QueryFieldValueInterface, -} from '../../../../pages/ExplorePage/ExplorePage.interface'; import { getDataProductByName, removeAssetsFromDataProduct, @@ -86,6 +82,7 @@ import { } from '../../../../utils/EntityUtils'; import { getAggregations, + getQuickFilterQuery, getSelectedValuesFromQuickFilter, } from '../../../../utils/Explore.utils'; import { @@ -516,30 +513,7 @@ const AssetsTabs = forwardRef( }, []); const handleQuickFiltersChange = (data: ExploreQuickFilterField[]) => { - const must: QueryFieldInterface[] = []; - data.forEach((filter) => { - if (!isEmpty(filter.value)) { - const should: QueryFieldValueInterface[] = []; - if (filter.value) { - filter.value.forEach((filterValue) => { - const term: Record = {}; - term[filter.key] = filterValue.key; - should.push({ term }); - }); - } - - must.push({ - bool: { should }, - }); - } - }); - - const quickFilterQuery = isEmpty(must) - ? undefined - : { - query: { bool: { must } }, - }; - + const quickFilterQuery = getQuickFilterQuery(data); setQuickFilterQuery(quickFilterQuery); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index c5a3d33a839d..c6432f66688b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -49,7 +49,7 @@ import { ColumnLineage, EntityReference, } from '../../generated/type/entityLineage'; -import { getLineageDataByFQN } from '../../rest/lineageAPI'; +import { getLineageDataByFQN, updateLineageEdge } from '../../rest/lineageAPI'; import { addLineageHandler, createEdges, @@ -131,7 +131,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const [zoomValue, setZoomValue] = useState(ZOOM_VALUE); const [tracedNodes, setTracedNodes] = useState([]); const [tracedColumns, setTracedColumns] = useState([]); - // const [entityFqn, setEntityFqn] = useState(''); const [status, setStatus] = useState('initial'); const [newAddedNode, setNewAddedNode] = useState({} as Node); const [lineageConfig, setLineageConfig] = useState({ @@ -801,6 +800,18 @@ const LineageProvider = ({ children }: LineageProviderProps) => { [selectedEdge, entityLineage] ); + const onEdgeDescriptionUpdate = useCallback( + async (updatedEdgeDetails: AddLineage) => { + try { + await updateLineageEdge(updatedEdgeDetails); + // Refresh Edge + } catch (err) { + showErrorToast(err as AxiosError); + } + }, + [edges, entityLineage, selectedEdge] + ); + const onColumnEdgeRemove = useCallback(() => { setShowDeleteModal(true); }, []); @@ -943,6 +954,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setIsDrawerOpen(false); setSelectedEdge(undefined); }} + onEdgeDescriptionUpdate={onEdgeDescriptionUpdate} /> ) : ( { const { fromEntity, toEntity, sqlQuery, columns } = - selectedEdgeValue?.data.edge; + selectedEdgeValue?.data.edge ?? {}; const updatedLineageDetails: LineageDetails = { sqlQuery: sqlQuery ?? '', columnsLineage: columns ?? [], @@ -782,7 +782,7 @@ export const getLineageDetailsObject = (edge: Edge): LineageDetails => { columns = [], description = '', pipeline, - source = '', + source, } = edge.data?.edge || {}; return { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts index 817d012074a2..7adcfcb5a474 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts @@ -11,7 +11,7 @@ * limitations under the License. */ -import { get } from 'lodash'; +import { get, isEmpty } from 'lodash'; import { ExploreQuickFilterField, ExploreSearchIndex, @@ -113,3 +113,37 @@ export const getAggregations = (data: Aggregations) => { ]) ) as Aggregations; }; + +/** + * Generates a ElasticSearch Query filter based on the given data. + * + * @param {ExploreQuickFilterField[]} data - An array of ExploreQuickFilterField objects representing the filter data. + * @return {object} - The generated quick filter query. + */ +export const getQuickFilterQuery = (data: ExploreQuickFilterField[]) => { + const must: QueryFieldInterface[] = []; + data.forEach((filter) => { + if (!isEmpty(filter.value)) { + const should: QueryFieldValueInterface[] = []; + if (filter.value) { + filter.value.forEach((filterValue) => { + const term: Record = {}; + term[filter.key] = filterValue.key; + should.push({ term }); + }); + } + + must.push({ + bool: { should }, + }); + } + }); + + const quickFilterQuery = isEmpty(must) + ? undefined + : { + query: { bool: { must } }, + }; + + return quickFilterQuery; +}; From d98bee196a962d95118bda6e34f920f4a1702c56 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 2 Jan 2024 17:38:52 +0530 Subject: [PATCH 26/44] fix: allow sql query to update --- .../EdgeInfoDrawer.component.tsx | 214 +++++++++++------- .../EntityInfoDrawer.interface.ts | 2 +- .../CustomControls.component.tsx | 2 +- .../EntityLineage/CustomNodeV1.component.tsx | 2 +- .../LineageProvider/LineageProvider.tsx | 46 +++- .../ModalWithQueryEditor.interface.ts | 19 ++ .../ModalWithQueryEditor.tsx | 112 +++++++++ .../ui/src/utils/EntityLineageUtils.test.tsx | 2 +- 8 files changed, 315 insertions(+), 84 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.tsx diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx index c20dd1283c06..56840c2fb7fe 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx @@ -12,13 +12,15 @@ */ import { CloseOutlined } from '@ant-design/icons'; -import { Col, Divider, Drawer, Row, Typography } from 'antd'; +import { Button, Col, Divider, Drawer, Row, Typography } from 'antd'; import { isUndefined } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; import { Node } from 'reactflow'; +import { ReactComponent as EditIcon } from '../../../assets/svg/edit-new.svg'; import DescriptionV1 from '../../../components/common/EntityDescription/DescriptionV1'; +import { DE_ACTIVE_COLOR } from '../../../constants/constants'; import { CSMode } from '../../../enums/codemirror.enum'; import { EntityType } from '../../../enums/entity.enum'; import { getNameFromFQN } from '../../../utils/CommonUtils'; @@ -27,6 +29,7 @@ import { getEntityName } from '../../../utils/EntityUtils'; import { getEncodedFqn } from '../../../utils/StringsUtils'; import { getEntityLink } from '../../../utils/TableUtils'; import Loader from '../../Loader/Loader'; +import { ModalWithQueryEditor } from '../../Modals/ModalWithQueryEditor/ModalWithQueryEditor'; import SchemaEditor from '../../SchemaEditor/SchemaEditor'; import './entity-info-drawer.less'; import { @@ -40,13 +43,14 @@ const EdgeInfoDrawer = ({ onClose, nodes, hasEditAccess, - onEdgeDescriptionUpdate, + onEdgeDetailsUpdate, }: EdgeInfoDrawerInfo) => { const [edgeData, setEdgeData] = useState(); const [mysqlQuery, setMysqlQuery] = useState(''); const [isLoading, setIsLoading] = useState(false); const [isDescriptionEditable, setIsDescriptionEditable] = useState(false); + const [showSqlQueryModal, setShowSqlQueryModal] = useState(false); const { t } = useTranslation(); @@ -115,14 +119,14 @@ const EdgeInfoDrawer = ({ }; const edgeDescription = useMemo(() => { - return edgeEntity?.lineageDetails?.description ?? ''; + return edgeEntity?.description ?? ''; }, [edgeEntity]); const onDescriptionUpdate = useCallback( async (updatedHTML: string) => { - if (edgeDescription !== updatedHTML && edgeEntity) { + if (edgeDescription !== updatedHTML && edge) { const lineageDetails = { - ...getLineageDetailsObject(edgeEntity), + ...getLineageDetailsObject(edge), description: updatedHTML, }; @@ -139,91 +143,147 @@ const EdgeInfoDrawer = ({ lineageDetails, }, }; - await onEdgeDescriptionUpdate?.(updatedEdgeDetails); - setIsDescriptionEditable(false); - } else { - setIsDescriptionEditable(false); + await onEdgeDetailsUpdate?.(updatedEdgeDetails); } + setIsDescriptionEditable(false); }, - [edgeDescription, edgeEntity, edge.data] + [edgeDescription, edgeEntity, edge] + ); + + const onSqlQueryUpdate = useCallback( + async (updatedQuery: string) => { + if (mysqlQuery !== updatedQuery && edge) { + const lineageDetails = { + ...getLineageDetailsObject(edge), + sqlQuery: updatedQuery, + }; + + const updatedEdgeDetails = { + edge: { + fromEntity: { + id: edgeEntity.fromEntity.id, + type: edgeEntity.fromEntity.type, + }, + toEntity: { + id: edgeEntity.toEntity.id, + type: edgeEntity.toEntity.type, + }, + lineageDetails, + }, + }; + await onEdgeDetailsUpdate?.(updatedEdgeDetails); + setMysqlQuery(updatedQuery); + } + setShowSqlQueryModal(false); + }, + [edgeEntity, edge, mysqlQuery] ); useEffect(() => { setIsLoading(true); getEdgeInfo(); - setMysqlQuery(edge.data.edge?.lineageDetails?.sqlQuery); + setMysqlQuery(edge.data.edge?.sqlQuery); }, [edge, visible]); return ( - } - getContainer={false} - headerStyle={{ padding: 16 }} - mask={false} - open={visible} - style={{ position: 'absolute' }} - title={t('label.edge-information')}> - {isLoading ? ( - - ) : ( - - {edgeData && - Object.values(edgeData).map( - (data) => - data.value && ( - - - {`${data.key}:`} - + <> + } + getContainer={false} + headerStyle={{ padding: 16 }} + mask={false} + open={visible} + style={{ position: 'absolute' }} + title={t('label.edge-information')}> + {isLoading ? ( + + ) : ( + + {edgeData && + Object.values(edgeData).map( + (data) => + data.value && ( + + + {`${data.key}:`} + - {isUndefined(data.link) ? ( - {data.value} - ) : ( - - {data.value} - - )} - - ) - )} - - - setIsDescriptionEditable(false)} - onDescriptionEdit={() => setIsDescriptionEditable(true)} - onDescriptionUpdate={onDescriptionUpdate} - /> - - - - - {`${t('label.sql-uppercase-query')}:`} - - {mysqlQuery ? ( - {data.value} + ) : ( + + {data.value} + + )} + + ) + )} + + + setIsDescriptionEditable(false)} + onDescriptionEdit={() => setIsDescriptionEditable(true)} + onDescriptionUpdate={onDescriptionUpdate} /> - ) : ( - - {t('server.no-query-available')} - - )} - - + + + +
+ + {`${t('label.sql-uppercase-query')}`} + + {hasEditAccess && ( +
+ {mysqlQuery ? ( + + ) : ( + + {t('server.no-query-available')} + + )} + + + )} + + {showSqlQueryModal && ( + setShowSqlQueryModal(false)} + onSave={onSqlQueryUpdate} + /> )} - + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts index 471272daf5f4..e56e9e13bc91 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts @@ -28,7 +28,7 @@ export interface EdgeInfoDrawerInfo { visible: boolean; hasEditAccess: boolean; onClose: () => void; - onEdgeDescriptionUpdate?: (updatedEdgeDetails: AddLineage) => Promise; + onEdgeDetailsUpdate?: (updatedEdgeDetails: AddLineage) => Promise; } type InfoType = { key: string; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx index aa8e801bdc05..5c5a917a05e9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx @@ -242,7 +242,7 @@ const CustomControls: FC = ({ selectedKeys: selectedFilter, }} trigger={['click']}> - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx index fc4f2de5ca87..f090a9dec6b9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx @@ -243,7 +243,7 @@ const CustomNodeV1 = (props: NodeProps) => { {isDownstreamLeafNode && !isEditMode && getExpandHandle(EdgeTypeEnum.DOWN_STREAM, () => - onExpand(EdgeTypeEnum.UP_STREAM) + onExpand(EdgeTypeEnum.DOWN_STREAM) )} ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index c6432f66688b..b813ca91743b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -800,11 +800,51 @@ const LineageProvider = ({ children }: LineageProviderProps) => { [selectedEdge, entityLineage] ); - const onEdgeDescriptionUpdate = useCallback( + const onEdgeDetailsUpdate = useCallback( async (updatedEdgeDetails: AddLineage) => { + const { description, sqlQuery } = + updatedEdgeDetails.edge.lineageDetails ?? {}; + try { await updateLineageEdge(updatedEdgeDetails); - // Refresh Edge + const updatedEdges = (entityLineage.edges ?? []).map((edge) => { + if ( + edge.fromEntity.id === updatedEdgeDetails.edge.fromEntity.id && + edge.toEntity.id === updatedEdgeDetails.edge.toEntity.id + ) { + return { + ...edge, + description, + sqlQuery, + }; + } + + return edge; + }); + setEntityLineage((prev) => { + return { + ...prev, + edges: updatedEdges, + }; + }); + + const edgesObj = edges.map((edge) => { + if (selectedEdge && edge.id === selectedEdge.id) { + return { + ...selectedEdge, + data: { + edge: { + ...selectedEdge.data.edge, + description, + sqlQuery, + }, + }, + }; + } + + return edge; + }); + setEdges(edgesObj); } catch (err) { showErrorToast(err as AxiosError); } @@ -954,7 +994,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setIsDrawerOpen(false); setSelectedEdge(undefined); }} - onEdgeDescriptionUpdate={onEdgeDescriptionUpdate} + onEdgeDetailsUpdate={onEdgeDetailsUpdate} /> ) : ( Promise; + onCancel?: () => void; + visible: boolean; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.tsx new file mode 100644 index 000000000000..803f6b42044c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.tsx @@ -0,0 +1,112 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Form, Modal, Typography } from 'antd'; +import { FormProps, useForm } from 'antd/lib/form/Form'; +import { AxiosError } from 'axios'; +import React, { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { CSMode } from '../../../enums/codemirror.enum'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import Loader from '../../Loader/Loader'; +import SchemaEditor from '../../SchemaEditor/SchemaEditor'; +import { ModalWithQueryEditorProps } from './ModalWithQueryEditor.interface'; + +export const ModalWithQueryEditor = ({ + header, + value, + onSave, + onCancel, + visible, +}: ModalWithQueryEditorProps) => { + const { t } = useTranslation(); + const [form] = useForm(); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + + const onFinish: FormProps['onFinish'] = async (value) => { + setIsSaving(true); + try { + await onSave?.(value.query ?? ''); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsSaving(false); + } + }; + + useEffect(() => { + if (visible) { + form.setFieldsValue({ query: value }); + setIsLoading(false); + } + }, [form, visible]); + + return ( + + {t('label.cancel')} + , + , + ]} + maskClosable={false} + open={visible} + title={{header}} + width="90%" + onCancel={onCancel}> + {isLoading ? ( + + ) : ( +
+ + + +
+ )} + + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx index 76a38d37d1d6..3c1a72a3937e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx @@ -173,7 +173,7 @@ describe('Test EntityLineageUtils utility', () => { columnsLineage: [], description: '', pipeline: undefined, - source: '', + source: undefined, }); }); From ce56981209db2bb30480894671b301842c8cdf04 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 2 Jan 2024 17:48:25 +0530 Subject: [PATCH 27/44] use input instead of select --- .../EntityLineage/LineageConfigModal.tsx | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx index 7eec343b6a90..3e5b205dfa0d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Form, InputNumber, Modal, Select } from 'antd'; +import { Form, Input, Modal } from 'antd'; import React, { useState } from 'react'; import { useTranslation } from 'react-i18next'; import { @@ -18,12 +18,6 @@ import { LineageConfigModalProps, } from './EntityLineage.interface'; -const SELECT_OPTIONS = [1, 2, 3].map((value) => ( - - {value} - -)); - const LineageConfigModal: React.FC = ({ visible, config, @@ -54,8 +48,8 @@ const LineageConfigModal: React.FC = ({ return (
= ({ }, ]} tooltip={t('message.upstream-depth-tooltip')}> - + type="number" + value={upstreamDepth} + onChange={(e) => setUpstreamDepth(Number(e.target.value))} + /> = ({ }, ]} tooltip={t('message.downstream-depth-tooltip')}> - + type="number" + value={downstreamDepth} + onChange={(e) => setDownstreamDepth(Number(e.target.value))} + /> = ({ }, ]} tooltip={t('message.nodes-per-layer-tooltip')}> - setNodesPerLayer(value as number)} + type="number" + onChange={(e) => setNodesPerLayer(Number(e.target.value))} />
From cc34ca652dbcaf7fdafa68083feb2d1273d03dcf Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 2 Jan 2024 19:33:58 +0530 Subject: [PATCH 28/44] unit tests --- .../CustomControls.component.tsx | 15 ++-- .../EntityLineage/LineageConfigModal.tsx | 2 +- .../LineageProvider/LineageProvider.tsx | 50 +++--------- .../ModalWithQueryEditor.interface.ts | 2 +- .../ModalWithQueryEditor.test.tsx | 77 +++++++++++++++++++ .../ContainerPage/ContainerPage.test.tsx | 4 - .../PipelineDetailsPage.test.tsx | 4 - .../main/resources/ui/src/rest/lineageAPI.ts | 14 ---- .../ui/src/utils/EntityLineageUtils.test.tsx | 52 +++++++++++++ .../ui/src/utils/EntityLineageUtils.tsx | 34 ++++++++ 10 files changed, 179 insertions(+), 75 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.test.tsx diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx index 5c5a917a05e9..54add983c5cc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx @@ -106,11 +106,7 @@ const CustomControls: FC = ({ setFilters( dropdownItems.map((item) => ({ ...item, - value: getSelectedValuesFromQuickFilter( - item, - dropdownItems, - undefined // pass in state variable - ), + value: getSelectedValuesFromQuickFilter(item, dropdownItems), })) ); @@ -157,11 +153,10 @@ const CustomControls: FC = ({ const { position } = selectedNode; onNodeClick(selectedNode); // moving selected node in center - reactFlowInstance && - reactFlowInstance.setCenter(position.x, position.y, { - duration: ZOOM_TRANSITION_DURATION, - zoom: zoomValue, - }); + reactFlowInstance?.setCenter(position.x, position.y, { + duration: ZOOM_TRANSITION_DURATION, + zoom: zoomValue, + }); } }, [onNodeClick, reactFlowInstance] diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx index 3e5b205dfa0d..83ec104af80b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageConfigModal.tsx @@ -48,8 +48,8 @@ const LineageConfigModal: React.FC = ({ return (
{ targetNode.data.node ); - if (columnConnection) { - if (!isUndefined(currentEdge)) { - const updatedColumns: ColumnLineage[] = - currentEdge.columns?.map((lineage) => { - if (lineage.toColumn === targetHandle) { - return { - ...lineage, - fromColumns: [ - ...(lineage.fromColumns ?? []), - sourceHandle ?? '', - ], - }; - } - - return lineage; - }) ?? []; - - if ( - !updatedColumns.find( - (lineage) => lineage.toColumn === targetHandle - ) - ) { - updatedColumns.push({ - fromColumns: [sourceHandle ?? ''], - toColumn: targetHandle ?? '', - }); - } - - if (newEdgeWithoutFqn.edge.lineageDetails) { - newEdgeWithoutFqn.edge.lineageDetails.columnsLineage = - updatedColumns; - } - currentEdge.columns = updatedColumns; // update current edge with new columns + if (columnConnection && currentEdge) { + const updatedColumns = getUpdatedColumnsFromEdge(params, currentEdge); + if (newEdgeWithoutFqn.edge.lineageDetails) { + newEdgeWithoutFqn.edge.lineageDetails.columnsLineage = + updatedColumns; } + currentEdge.columns = updatedColumns; // update current edge with new columns } addLineageHandler(newEdgeWithoutFqn) @@ -764,12 +735,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => { newEdges[edgeIndex] = existingEdge as EdgeDetails; } - const newData = { + return { ...pre, edges: newEdges, }; - - return newData; }); setEdges((pre) => @@ -966,7 +935,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { onDrawerClose, loadChildNodesHandler, fetchLineageData, - toggleColumnView, removeNodeHandler, onNodeClick, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.interface.ts index 24498c40736d..462ce9ce2d01 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.interface.ts @@ -13,7 +13,7 @@ export type ModalWithQueryEditorProps = { header: string; value: string; - onSave?: (text: string) => Promise; + onSave?: (text: string) => void; onCancel?: () => void; visible: boolean; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.test.tsx new file mode 100644 index 000000000000..2a9d4ead22bc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithQueryEditor/ModalWithQueryEditor.test.tsx @@ -0,0 +1,77 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, fireEvent, render } from '@testing-library/react'; +import React from 'react'; +import { ModalWithQueryEditor } from './ModalWithQueryEditor'; + +describe('ModalWithQueryEditor', () => { + const onSaveMock = jest.fn(); + const onCancelMock = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('renders the modal with the correct header', () => { + const header = 'Test Header'; + const { getByTestId } = render( + + ); + const headerElement = getByTestId('header'); + + expect(headerElement).toBeInTheDocument(); + expect(headerElement).toHaveTextContent(header); + }); + + it('calls onSave with the correct value when save button is clicked', async () => { + const value = 'Test Query'; + const { getByTestId } = render( + + ); + const saveButton = getByTestId('save'); + + await act(async () => { + fireEvent.click(saveButton); + }); + + expect(onSaveMock).toHaveBeenCalledWith(value); + }); + + it('calls onCancel when cancel button is clicked', () => { + const { getByTestId } = render( + + ); + const cancelButton = getByTestId('cancel'); + fireEvent.click(cancelButton); + + expect(onCancelMock).toHaveBeenCalled(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx index e09a9bb0632d..f6fcf37ac25b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ContainerPage/ContainerPage.test.tsx @@ -116,10 +116,6 @@ jest.mock('../../components/Loader/Loader', () => { return jest.fn().mockReturnValue(
Loader
); }); -jest.mock('../../rest/lineageAPI', () => ({ - getLineageByFQN: jest.fn().mockImplementation(() => Promise.resolve()), -})); - jest.mock('../../rest/miscAPI', () => ({ deleteLineageEdge: jest.fn().mockImplementation(() => Promise.resolve()), addLineage: jest.fn().mockImplementation(() => Promise.resolve()), diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/PipelineDetails/PipelineDetailsPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/PipelineDetails/PipelineDetailsPage.test.tsx index 5942078c2558..903a0ff4a42c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/PipelineDetails/PipelineDetailsPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/PipelineDetails/PipelineDetailsPage.test.tsx @@ -24,10 +24,6 @@ jest.mock('react-router-dom', () => ({ }), })); -jest.mock('../../rest/lineageAPI', () => ({ - getLineageByFQN: jest.fn().mockImplementation(() => Promise.resolve()), -})); - jest.mock('../../rest/miscAPI', () => ({ addLineage: jest.fn(), deleteLineageEdge: jest.fn(), diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts index 121f0d8339eb..0b00758957b2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts @@ -14,22 +14,8 @@ import { LineageConfig } from '../components/Entity/EntityLineage/EntityLineage.interface'; import { EntityLineageReponse } from '../components/Lineage/Lineage.interface'; import { AddLineage } from '../generated/api/lineage/addLineage'; -import { EntityLineage } from '../generated/type/entityLineage'; import APIClient from './index'; -export const getLineageByFQN = async ( - fqn: string, - type: string, - upstreamDepth = 3, - downstreamDepth = 3 -) => { - const response = await APIClient.get( - `/lineage/${type}/name/${fqn}?upstreamDepth=${upstreamDepth}&downstreamDepth=${downstreamDepth}` - ); - - return response.data; -}; - export const updateLineageEdge = async (edge: AddLineage) => { const response = await APIClient.put(`/lineage`, edge); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx index 3c1a72a3937e..1cd5237771d2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx @@ -13,6 +13,7 @@ import { Edge } from 'reactflow'; import { EdgeTypeEnum } from '../components/Entity/EntityLineage/EntityLineage.interface'; +import { EdgeDetails } from '../components/Lineage/Lineage.interface'; import { SourceType } from '../components/SearchedData/SearchedData.interface'; import { AddLineage } from '../generated/api/lineage/addLineage'; import { MOCK_NODES_AND_EDGES } from '../mocks/Lineage.mock'; @@ -29,6 +30,7 @@ import { getLineageDetailsObject, getLineageEdge, getLineageEdgeForAPI, + getUpdatedColumnsFromEdge, isColumnLineageTraced, isTracedEdge, } from './EntityLineageUtils'; @@ -380,4 +382,54 @@ describe('Test EntityLineageUtils utility', () => { expect(addLineage).toHaveBeenCalledWith(edge); }); + + it('getUpdatedColumnsFromEdge should appropriate columns', () => { + const edgeToConnect = { + source: 'dim_customer', + sourceHandle: 'shopId', + target: 'dim_client', + targetHandle: 'shopId', + }; + const currentEdge: EdgeDetails = { + fromEntity: { + id: 'source', + type: 'table', + fqn: 'sourceFqn', + }, + toEntity: { + id: 'target', + type: 'table', + fqn: 'targetFqn', + }, + columns: [], + }; + + const result = getUpdatedColumnsFromEdge(edgeToConnect, currentEdge); + + expect(result).toEqual([ + { + fromColumns: ['shopId'], + toColumn: 'shopId', + }, + ]); + + currentEdge.columns = [ + { + fromColumns: ['customerId'], + toColumn: 'customerId', + }, + ]; + const result1 = getUpdatedColumnsFromEdge(edgeToConnect, currentEdge); + + expect(result1).toEqual([ + { + fromColumns: ['customerId'], + toColumn: 'customerId', + }, + { + fromColumns: ['shopId'], + toColumn: 'shopId', + }, + ]); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index 0117098c6f12..fbfd40e07fb3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -21,6 +21,7 @@ import { LoadingState } from 'Models'; import React, { Fragment, MouseEvent as ReactMouseEvent } from 'react'; import { Link } from 'react-router-dom'; import { + Connection, Edge, getConnectedEdges, getIncomers, @@ -898,3 +899,36 @@ export const getConnectedNodesEdges = ( nodeFqn: childNodeFqn, }; }; + +export const getUpdatedColumnsFromEdge = ( + edgeToConnect: Edge | Connection, + currentEdge: EdgeDetails +) => { + const { target, source, sourceHandle, targetHandle } = edgeToConnect; + const columnConnection = source !== sourceHandle && target !== targetHandle; + + if (columnConnection) { + const updatedColumns: ColumnLineage[] = + currentEdge.columns?.map((lineage) => { + if (lineage.toColumn === targetHandle) { + return { + ...lineage, + fromColumns: [...(lineage.fromColumns ?? []), sourceHandle ?? ''], + }; + } + + return lineage; + }) ?? []; + + if (!updatedColumns.find((lineage) => lineage.toColumn === targetHandle)) { + updatedColumns.push({ + fromColumns: [sourceHandle ?? ''], + toColumn: targetHandle ?? '', + }); + } + + return updatedColumns; + } + + return []; +}; From ccbdbb3351c4d3273ed28e9e357ec04fbc03e99d Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 2 Jan 2024 20:48:28 +0530 Subject: [PATCH 29/44] unit tests --- .../EntityLineage/CustomNode.utils.test.tsx | 108 ++++++++++++++ .../Entity/EntityLineage/CustomNode.utils.tsx | 41 +++--- .../LineageProvider/LineageProvider.tsx | 137 ++++++++---------- .../ui/src/utils/EntityLineageUtils.test.tsx | 73 ++++++++++ .../ui/src/utils/EntityLineageUtils.tsx | 22 +++ .../ui/src/utils/Explore.utils.test.ts | 71 +++++++++ 6 files changed, 355 insertions(+), 97 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.test.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.test.ts diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.test.tsx new file mode 100644 index 000000000000..2a9c98652e1e --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.test.tsx @@ -0,0 +1,108 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { fireEvent, render } from '@testing-library/react'; +import React from 'react'; +import { ReactFlowProvider } from 'reactflow'; +import { EntityLineageNodeType } from '../../../enums/entity.enum'; +import { + getCollapseHandle, + getColumnHandle, + getExpandHandle, +} from './CustomNode.utils'; +import { EdgeTypeEnum } from './EntityLineage.interface'; + +describe('Custom Node Utils', () => { + it('getColumnHandle should return null when nodeType is NOT_CONNECTED', () => { + const result = getColumnHandle( + EntityLineageNodeType.NOT_CONNECTED, + true, + 'test', + '123' + ); + + expect(result).toBeNull(); + }); + + it('getColumnHandle should render handles when nodeType is not NOT_CONNECTED', () => { + const { getByTestId } = render( + +
+ {getColumnHandle('CONNECTED', true)} +
+ + ); + + expect(getByTestId('column-handle')).toBeInTheDocument(); + }); + + describe('getExpandHandle', () => { + it('renders a Button component', () => { + const { getByRole } = render( + getExpandHandle(EdgeTypeEnum.DOWN_STREAM, jest.fn()) + ); + + expect(getByRole('button')).toBeInTheDocument(); + expect(getByRole('button')).toHaveClass('react-flow__handle-right'); + }); + + it('applies the correct class name for non-DOWN_STREAM direction', () => { + const { getByRole } = render( + getExpandHandle(EdgeTypeEnum.UP_STREAM, jest.fn()) + ); + + expect(getByRole('button')).toHaveClass('react-flow__handle-left'); + }); + + it('calls the onClickHandler when clicked', () => { + const onClickHandler = jest.fn(); + const { getByRole } = render( + getExpandHandle(EdgeTypeEnum.DOWN_STREAM, onClickHandler) + ); + + fireEvent.click(getByRole('button')); + + expect(onClickHandler).toHaveBeenCalled(); + }); + }); + + describe('getCollapseHandle', () => { + it('renders the collapse handle component correctly', () => { + const onClickHandler = jest.fn(); + + const { getByTestId } = render( + getCollapseHandle(EdgeTypeEnum.DOWN_STREAM, onClickHandler) + ); + + const collapseHandle = getByTestId('downstream-collapse-handle'); + + expect(collapseHandle).toBeInTheDocument(); + expect(collapseHandle).toHaveClass('react-flow__handle-right'); + }); + + it('calls the onClickHandler when the collapse handle is clicked', () => { + const onClickHandler = jest.fn(); + + const { getByTestId } = render( + getCollapseHandle(EdgeTypeEnum.UP_STREAM, onClickHandler) + ); + + const collapseHandle = getByTestId('upstream-collapse-handle'); + + expect(collapseHandle).toHaveClass('react-flow__handle-left'); + + fireEvent.click(collapseHandle); + + expect(onClickHandler).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx index 2cea7732cc73..a2a607a5f95d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx @@ -19,6 +19,24 @@ import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-outlined.sv import { EntityLineageNodeType } from '../../../enums/entity.enum'; import { EdgeTypeEnum } from './EntityLineage.interface'; +export const getHandleByType = ( + isConnectable: HandleProps['isConnectable'], + position: Position, + type: HandleType, + className?: string, + id?: string +) => { + return ( + + ); +}; + export const getColumnHandle = ( nodeType: string, isConnectable: HandleProps['isConnectable'], @@ -43,24 +61,6 @@ export const getColumnHandle = ( } }; -export const getHandleByType = ( - isConnectable: HandleProps['isConnectable'], - position: Position, - type: HandleType, - className?: string, - id?: string -) => { - return ( - - ); -}; - export const getExpandHandle = ( direction: EdgeTypeEnum, onClickHandler: () => void @@ -96,6 +96,11 @@ export const getCollapseHandle = ( ? 'react-flow__handle-right' : 'react-flow__handle-left' )} + data-testid={ + direction === EdgeTypeEnum.DOWN_STREAM + ? 'downstream-collapse-handle' + : 'upstream-collapse-handle' + } icon={} shape="circle" size="small" diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index 3fe7cbefb015..0135437cb2e0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -50,14 +50,13 @@ import { getLineageDataByFQN, updateLineageEdge } from '../../rest/lineageAPI'; import { addLineageHandler, createEdges, + createNewEdge, createNodes, getAllTracedColumnEdge, getAllTracedNodes, getClassifiedEdge, - getColumnLineageData, getConnectedNodesEdges, getLayoutedElements, - getLineageDetailsObject, getLineageEdge, getLineageEdgeForAPI, getLoadingStatusValue, @@ -264,98 +263,78 @@ const LineageProvider = ({ children }: LineageProviderProps) => { edge: Edge, confirmDelete: boolean ): Promise => { - if (confirmDelete && entityLineage) { - const { data } = edge; - const edgeData: EdgeData = { - fromEntity: data.edge.fromEntity.type, - fromId: data.edge.fromEntity.id, - toEntity: data.edge.toEntity.type, - toId: data.edge.toEntity.id, - }; + if (!confirmDelete || !entityLineage) { + return; + } - await removeLineageHandler(edgeData); + const { data } = edge; + const edgeData: EdgeData = { + fromEntity: data.edge.fromEntity.type, + fromId: data.edge.fromEntity.id, + toEntity: data.edge.toEntity.type, + toId: data.edge.toEntity.id, + }; - const updatedEdges = entityLineage.edges?.filter( - (item) => - !( - item.fromEntity.id === edgeData.fromId && - item.toEntity.id === edgeData.toId - ) - ); + await removeLineageHandler(edgeData); + const updatedEdges = entityLineage.edges?.filter( + (item) => + !( + item.fromEntity.id === edgeData.fromId && + item.toEntity.id === edgeData.toId + ) + ); - setEntityLineage((prev) => { - return { - ...prev, - edges: updatedEdges, - }; - }); + setEntityLineage((prev) => ({ + ...prev, + edges: updatedEdges, + })); - const newNodes = createNodes( - entityLineage.nodes ?? [], - updatedEdges ?? [] - ); - const newEdges = createEdges( - entityLineage.nodes ?? [], - updatedEdges ?? [] - ); + const newNodes = createNodes(entityLineage.nodes ?? [], updatedEdges ?? []); + const newEdges = createEdges(entityLineage.nodes ?? [], updatedEdges ?? []); - setNodes(newNodes); - setEdges(newEdges); - } + setNodes(newNodes); + setEdges(newEdges); }; const removeColumnEdge = async (edge: Edge, confirmDelete: boolean) => { - if (confirmDelete && entityLineage) { - const { data } = edge; - const selectedEdge: AddLineage = { - edge: { - fromEntity: { - id: data.edge.fromEntity.id, - type: data.edge.fromEntity.type, - }, - toEntity: { - id: data.edge.toEntity.id, - type: data.edge.toEntity.type, - }, - }, - }; - - const updatedCols = getColumnLineageData(data.edge.columns, edge); - selectedEdge.edge.lineageDetails = getLineageDetailsObject(edge); - selectedEdge.edge.lineageDetails.columnsLineage = updatedCols; - - await addLineageHandler(selectedEdge); + if (!confirmDelete || !entityLineage) { + return; + } - const updatedEdgeWithColumns = (entityLineage.edges ?? []).map((obj) => { - if ( - obj.fromEntity.id === data.edge.fromEntity.id && - obj.toEntity.id === data.edge.toEntity.id - ) { - return { - ...obj, - columns: updatedCols, - }; - } + const { data } = edge; + const selectedEdge = createNewEdge(edge); + const updatedCols = selectedEdge.edge.lineageDetails?.columnsLineage ?? []; + await addLineageHandler(selectedEdge); - return obj; - }); - - setEntityLineage((prev) => { + const updatedEdgeWithColumns = (entityLineage.edges ?? []).map((obj) => { + if ( + obj.fromEntity.id === data.edge.fromEntity.id && + obj.toEntity.id === data.edge.toEntity.id + ) { return { - ...prev, - edges: updatedEdgeWithColumns, + ...obj, + columns: updatedCols, }; - }); + } - const updatedEdges = createEdges( - entityLineage.nodes ?? [], - updatedEdgeWithColumns - ); + return obj; + }); - setEdges(updatedEdges); + setEntityLineage((prev) => { + return { + ...prev, + edges: updatedEdgeWithColumns, + }; + }); - setShowDeleteModal(false); - } + const updatedEdges = createEdges( + entityLineage.nodes ?? [], + updatedEdgeWithColumns + ); + + setEdges(updatedEdges); + + setShowDeleteModal(false); }; const removeNodeHandler = useCallback( diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx index 1cd5237771d2..7d736bb1ddfd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx @@ -20,6 +20,7 @@ import { MOCK_NODES_AND_EDGES } from '../mocks/Lineage.mock'; import { addLineage } from '../rest/miscAPI'; import { addLineageHandler, + createNewEdge, getAllTracedColumnEdge, getAllTracedEdges, getAllTracedNodes, @@ -432,4 +433,76 @@ describe('Test EntityLineageUtils utility', () => { }, ]); }); + + describe('createNewEdge', () => { + it('should create a new edge with the correct properties', () => { + const edge = { + data: { + edge: { + fromEntity: { + id: 'fromEntityId', + type: 'fromEntityType', + }, + toEntity: { + id: 'toEntityId', + type: 'toEntityType', + }, + columns: [], + }, + }, + }; + + const result = createNewEdge(edge as Edge); + + expect(result.edge.fromEntity.id).toEqual('fromEntityId'); + expect(result.edge.fromEntity.type).toEqual('fromEntityType'); + expect(result.edge.toEntity.id).toEqual('toEntityId'); + expect(result.edge.toEntity.type).toEqual('toEntityType'); + expect(result.edge.lineageDetails).toBeDefined(); + expect(result.edge.lineageDetails?.columnsLineage).toBeDefined(); + }); + + it('should update the columns lineage details correctly', () => { + const edge = { + data: { + edge: { + fromEntity: { + id: 'table1', + type: 'table', + }, + toEntity: { + id: 'table2', + type: 'table', + }, + columns: [ + { name: 'column1', type: 'type1' }, + { name: 'column2', type: 'type2' }, + ], + }, + }, + }; + + const result = createNewEdge(edge as Edge); + + expect(result).toEqual({ + edge: { + fromEntity: { + id: 'table1', + type: 'table', + }, + toEntity: { + id: 'table2', + type: 'table', + }, + lineageDetails: { + columnsLineage: [], + description: '', + pipeline: undefined, + source: undefined, + sqlQuery: '', + }, + }, + }); + }); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index fbfd40e07fb3..5b8a8c4738fd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -932,3 +932,25 @@ export const getUpdatedColumnsFromEdge = ( return []; }; + +export const createNewEdge = (edge: Edge) => { + const { data } = edge; + const selectedEdge: AddLineage = { + edge: { + fromEntity: { + id: data.edge.fromEntity.id, + type: data.edge.fromEntity.type, + }, + toEntity: { + id: data.edge.toEntity.id, + type: data.edge.toEntity.type, + }, + }, + }; + + const updatedCols = getColumnLineageData(data.edge.columns, edge); + selectedEdge.edge.lineageDetails = getLineageDetailsObject(edge); + selectedEdge.edge.lineageDetails.columnsLineage = updatedCols; + + return selectedEdge; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.test.ts new file mode 100644 index 000000000000..9da17e658262 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ExploreQuickFilterField } from '../components/Explore/ExplorePage.interface'; +import { getQuickFilterQuery } from './Explore.utils'; + +describe('Explore Utils', () => { + it('should return undefined if data is empty', () => { + const data: ExploreQuickFilterField[] = []; + const result = getQuickFilterQuery(data); + + expect(result).toBeUndefined(); + }); + + it('should generate quick filter query correctly', () => { + const data = [ + { + label: 'Domain', + key: 'domain.displayName.keyword', + value: [], + }, + { + label: 'Owner', + key: 'owner.displayName.keyword', + value: [], + }, + { + label: 'Tag', + key: 'tags.tagFQN', + value: [ + { + key: 'personaldata.personal', + label: 'personaldata.personal', + count: 1, + }, + ], + }, + ]; + const result = getQuickFilterQuery(data); + const expectedQuery = { + query: { + bool: { + must: [ + { + bool: { + should: [ + { + term: { + 'tags.tagFQN': 'personaldata.personal', + }, + }, + ], + }, + }, + ], + }, + }, + }; + + expect(result).toEqual(expectedQuery); + }); +}); From 098aee46e1701cac6c435da6e2c6b40a4bfb71aa Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 3 Jan 2024 15:27:21 +0530 Subject: [PATCH 30/44] fix edge removal --- .../LineageProvider/LineageProvider.tsx | 55 +++++++++++++------ .../ui/src/utils/EntityLineageUtils.tsx | 19 ++++++- .../resources/ui/src/utils/EntityUtils.tsx | 21 +++++-- 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index 0135437cb2e0..d3eb7d55f96e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -267,7 +267,8 @@ const LineageProvider = ({ children }: LineageProviderProps) => { return; } - const { data } = edge; + const { data, target } = edge; + const edgeData: EdgeData = { fromEntity: data.edge.fromEntity.type, fromId: data.edge.fromEntity.id, @@ -276,24 +277,46 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }; await removeLineageHandler(edgeData); - const updatedEdges = entityLineage.edges?.filter( - (item) => - !( - item.fromEntity.id === edgeData.fromId && - item.toEntity.id === edgeData.toId - ) - ); - setEntityLineage((prev) => ({ - ...prev, - edges: updatedEdges, - })); + const outgoingNode = nodes.find((n) => n.id === target); + if (outgoingNode) { + const { nodeFqn, edges: connectedEdges } = getConnectedNodesEdges( + outgoingNode, + nodes, + edges, + EdgeTypeEnum.DOWN_STREAM + ); - const newNodes = createNodes(entityLineage.nodes ?? [], updatedEdges ?? []); - const newEdges = createEdges(entityLineage.nodes ?? [], updatedEdges ?? []); + const updatedNodes = (entityLineage.nodes ?? []).filter( + (item) => !nodeFqn.includes(item.fullyQualifiedName ?? '') + ); + const updatedEdges = (entityLineage.edges ?? []).filter((val) => { + return !connectedEdges.some( + (connectedEdge) => connectedEdge.data.edge === val + ); + }); + + const filteredEdges = + updatedEdges.filter( + (item) => + !( + item.fromEntity.id === edgeData.fromId && + item.toEntity.id === edgeData.toId + ) + ) ?? []; - setNodes(newNodes); - setEdges(newEdges); + setEntityLineage((prev) => ({ + ...prev, + nodes: updatedNodes, + edges: filteredEdges, + })); + + const newNodes = createNodes(updatedNodes ?? [], filteredEdges); + const newEdges = createEdges(updatedNodes ?? [], filteredEdges); + + setNodes(newNodes); + setEdges(newEdges); + } }; const removeColumnEdge = async (edge: Edge, confirmDelete: boolean) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index 5b8a8c4738fd..4265f0b56665 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -592,11 +592,25 @@ const getNodeType = ( return EntityLineageNodeType.DEFAULT; }; +const filterNodesWithoutEdges = ( + nodesData: EntityReference[], + edges: EdgeDetails[] +) => { + const nodesWithEdges = nodesData.filter((node) => { + return edges.some((edge) => { + return edge.fromEntity.id === node.id || edge.toEntity.id === node.id; + }); + }); + + return nodesWithEdges; +}; + export const createNodes = ( nodesData: EntityReference[], edgesData: EdgeDetails[] ) => { const uniqueNodesData = removeDuplicateNodes(nodesData); + const filteredNodesData = filterNodesWithoutEdges(uniqueNodesData, edgesData); // Create a new dagre graph const graph = new dagre.graphlib.Graph(); @@ -608,7 +622,7 @@ export const createNodes = ( graph.setDefaultEdgeLabel(() => ({})); // Add nodes to the graph - uniqueNodesData.forEach((node) => { + filteredNodesData.forEach((node) => { graph.setNode(node.id, { width: NODE_WIDTH, height: NODE_HEIGHT }); }); @@ -623,9 +637,8 @@ export const createNodes = ( // Get the layout positions const layoutPositions = graph.nodes().map((nodeId) => graph.node(nodeId)); - return uniqueNodesData.map((node, index) => { + return filteredNodesData.map((node, index) => { const position = layoutPositions[index]; - const type = getNodeType(edgesData, node.id); return { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx index 9a63cd2fa441..26edcaa1a820 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx @@ -204,21 +204,28 @@ const getTableFieldsFromTableDetails = (tableDetails: Table) => { profile, columns, tableType, + service, + database, + databaseSchema, } = tableDetails; - const [service, database, schema] = getPartialNameFromTableFQN( + const [serviceName, databaseName, schemaName] = getPartialNameFromTableFQN( fullyQualifiedName ?? '', [FqnPart.Service, FqnPart.Database, FqnPart.Schema], FQN_SEPARATOR_CHAR ).split(FQN_SEPARATOR_CHAR); + const serviceDisplayName = getEntityName(service) || serviceName; + const databaseDisplayName = getEntityName(database) || databaseName; + const schemaDisplayName = getEntityName(databaseSchema) || schemaName; + const tier = getTierFromTableTags(tags ?? []); return { fullyQualifiedName, owner, - service, - database, - schema, + service: serviceDisplayName, + database: databaseDisplayName, + schema: schemaDisplayName, tier, usage: getUsageData(usageSummary), profile, @@ -336,6 +343,7 @@ const getTableOverview = (tableDetails: Table) => { const getPipelineOverview = (pipelineDetails: Pipeline) => { const { owner, tags, sourceUrl, service, displayName } = pipelineDetails; const tier = getTierFromTableTags(tags ?? []); + const serviceDisplayName = getEntityName(service); const overview = [ { @@ -365,7 +373,7 @@ const getPipelineOverview = (pipelineDetails: Pipeline) => { }, { name: i18next.t('label.service'), - value: (service?.name as string) || NO_DATA, + value: serviceDisplayName || NO_DATA, url: getServiceDetailsPath( service?.name as string, ServiceCategory.PIPELINE_SERVICES @@ -388,6 +396,7 @@ const getPipelineOverview = (pipelineDetails: Pipeline) => { const getDashboardOverview = (dashboardDetails: Dashboard) => { const { owner, tags, sourceUrl, service, displayName } = dashboardDetails; const tier = getTierFromTableTags(tags ?? []); + const serviceDisplayName = getEntityName(service); const overview = [ { @@ -416,7 +425,7 @@ const getDashboardOverview = (dashboardDetails: Dashboard) => { }, { name: i18next.t('label.service'), - value: (service?.fullyQualifiedName as string) || NO_DATA, + value: serviceDisplayName || NO_DATA, url: getServiceDetailsPath( service?.name as string, ServiceCategory.DASHBOARD_SERVICES From ade1383ea2c7aa250e15eb7c363d2143282d7ce1 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 3 Jan 2024 17:59:31 +0530 Subject: [PATCH 31/44] cleanup --- .../LineageProvider/LineageProvider.tsx | 129 +++++------------- .../ui/src/utils/EntityLineageUtils.tsx | 18 +-- 2 files changed, 36 insertions(+), 111 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index d3eb7d55f96e..a909426bc1d4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -146,12 +146,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const res = await getLineageDataByFQN(fqn, config, queryFilter); if (res) { setEntityLineage(res); - const allNodes = [res.entity, ...(res.nodes ?? [])]; - const updatedNodes = createNodes(allNodes, res.edges ?? []); - const updatedEdges = createEdges(allNodes, res.edges ?? []); - - setNodes(updatedNodes); - setEdges(updatedEdges); } else { showErrorToast( t('server.entity-fetch-error', { @@ -194,9 +188,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { isEqual ); - const newNodes = createNodes(allNodes, allEdges); - const newEdges = createEdges(allNodes, allEdges); - setEntityLineage((prev) => { return { ...prev, @@ -204,9 +195,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { edges: allEdges, }; }); - - setNodes(newNodes); - setEdges(newEdges); } catch (err) { showErrorToast( err as AxiosError, @@ -279,6 +267,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { await removeLineageHandler(edgeData); const outgoingNode = nodes.find((n) => n.id === target); + if (outgoingNode) { const { nodeFqn, edges: connectedEdges } = getConnectedNodesEdges( outgoingNode, @@ -287,35 +276,31 @@ const LineageProvider = ({ children }: LineageProviderProps) => { EdgeTypeEnum.DOWN_STREAM ); + const nodesToRemove = [ + ...nodeFqn, + outgoingNode.data.node.fullyQualifiedName ?? '', + ]; + const updatedNodes = (entityLineage.nodes ?? []).filter( - (item) => !nodeFqn.includes(item.fullyQualifiedName ?? '') + (item) => !nodesToRemove.includes(item.fullyQualifiedName ?? '') ); - const updatedEdges = (entityLineage.edges ?? []).filter((val) => { - return !connectedEdges.some( - (connectedEdge) => connectedEdge.data.edge === val - ); - }); - const filteredEdges = - updatedEdges.filter( - (item) => - !( - item.fromEntity.id === edgeData.fromId && - item.toEntity.id === edgeData.toId - ) - ) ?? []; + const filteredEdges = (entityLineage.edges ?? []).filter( + (item) => + !( + item.fromEntity.id === edgeData.fromId && + item.toEntity.id === edgeData.toId + ) && + !connectedEdges.some( + (connectedEdge) => connectedEdge.data.edge === item + ) + ); setEntityLineage((prev) => ({ ...prev, nodes: updatedNodes, edges: filteredEdges, })); - - const newNodes = createNodes(updatedNodes ?? [], filteredEdges); - const newEdges = createEdges(updatedNodes ?? [], filteredEdges); - - setNodes(newNodes); - setEdges(newEdges); } }; @@ -350,13 +335,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }; }); - const updatedEdges = createEdges( - entityLineage.nodes ?? [], - updatedEdgeWithColumns - ); - - setEdges(updatedEdges); - setShowDeleteModal(false); }; @@ -606,15 +584,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { ? [...(entityLineage.edges ?? []), newEdgeWithFqn.edge] : entityLineage.edges ?? []; - const updatedNodes = createNodes( - allNodes as EntityReference[], - allEdges - ); - const updatedEdges = createEdges( - allNodes as EntityReference[], - allEdges - ); - setEntityLineage((pre) => { const newData = { ...pre, @@ -625,8 +594,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { return newData; }); - setNodes(updatedNodes); - setEdges(updatedEdges); setNewAddedNode({} as Node); }) .catch((err) => { @@ -701,9 +668,9 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setStatus('waiting'); setLoading(true); - const { source, target } = selectedEdge.data; + const { source, target } = selectedEdge; const existingEdge = (entityLineage.edges ?? []).find( - (ed) => ed.fromEntity === source && ed.toEntity === target + (ed) => ed.fromEntity.id === source && ed.toEntity.id === target ); let edgeIndex = -1; @@ -715,7 +682,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { } } - const { newEdge, updatedLineageDetails } = getNewLineageConnectionDetails( + const { newEdge } = getNewLineageConnectionDetails( selectedEdge, pipelineData ); @@ -742,25 +709,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { edges: newEdges, }; }); - - setEdges((pre) => - pre.map((edge) => { - if (edge.id === selectedEdge.id) { - return { - ...edge, - animated: true, - data: { - edge: { - ...edge.data.edge, - pipeline: updatedLineageDetails.pipeline, - }, - }, - }; - } - - return edge; - }) - ); } catch (error) { setLoading(false); } finally { @@ -798,24 +746,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { edges: updatedEdges, }; }); - - const edgesObj = edges.map((edge) => { - if (selectedEdge && edge.id === selectedEdge.id) { - return { - ...selectedEdge, - data: { - edge: { - ...selectedEdge.data.edge, - description, - sqlQuery, - }, - }, - }; - } - - return edge; - }); - setEdges(edgesObj); } catch (err) { showErrorToast(err as AxiosError); } @@ -852,11 +782,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { edges: updatedEdges, }; }); - - const allNodes = createNodes(updatedNodes, updatedEdges); - const allEdges = createEdges(updatedNodes, updatedEdges); - setNodes(allNodes); - setEdges(allEdges); }, [nodes, edges, entityLineage] ); @@ -867,6 +792,20 @@ const LineageProvider = ({ children }: LineageProviderProps) => { } }, [lineageConfig, entityFqn, queryFilter]); + useEffect(() => { + const updatedNodes = createNodes( + entityLineage.nodes ?? [], + entityLineage.edges ?? [] + ); + const updatedEdges = createEdges( + entityLineage.nodes ?? [], + entityLineage.edges ?? [] + ); + + setNodes(updatedNodes); + setEdges(updatedEdges); + }, [entityLineage]); + const activityFeedContextValues = useMemo(() => { return { isDrawerOpen, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index 4265f0b56665..8f67dde7f833 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -592,25 +592,11 @@ const getNodeType = ( return EntityLineageNodeType.DEFAULT; }; -const filterNodesWithoutEdges = ( - nodesData: EntityReference[], - edges: EdgeDetails[] -) => { - const nodesWithEdges = nodesData.filter((node) => { - return edges.some((edge) => { - return edge.fromEntity.id === node.id || edge.toEntity.id === node.id; - }); - }); - - return nodesWithEdges; -}; - export const createNodes = ( nodesData: EntityReference[], edgesData: EdgeDetails[] ) => { const uniqueNodesData = removeDuplicateNodes(nodesData); - const filteredNodesData = filterNodesWithoutEdges(uniqueNodesData, edgesData); // Create a new dagre graph const graph = new dagre.graphlib.Graph(); @@ -622,7 +608,7 @@ export const createNodes = ( graph.setDefaultEdgeLabel(() => ({})); // Add nodes to the graph - filteredNodesData.forEach((node) => { + uniqueNodesData.forEach((node) => { graph.setNode(node.id, { width: NODE_WIDTH, height: NODE_HEIGHT }); }); @@ -637,7 +623,7 @@ export const createNodes = ( // Get the layout positions const layoutPositions = graph.nodes().map((nodeId) => graph.node(nodeId)); - return filteredNodesData.map((node, index) => { + return uniqueNodesData.map((node, index) => { const position = layoutPositions[index]; const type = getNodeType(edgesData, node.id); From 3c0ff5729ff4487eb55c98f8f081de52b08898bd Mon Sep 17 00:00:00 2001 From: 07Himank Date: Thu, 4 Jan 2024 14:22:21 +0530 Subject: [PATCH 32/44] description and deleted --- .../service/jdbi3/LineageRepository.java | 20 +++-- .../resources/search/SearchResource.java | 34 ++++---- .../service/search/SearchClient.java | 25 +++--- .../service/search/SearchRepository.java | 73 ++++++++-------- .../elasticsearch/ElasticSearchClient.java | 87 ++++++++++--------- .../service/search/indexes/SearchIndex.java | 34 ++++---- .../search/opensearch/OpenSearchClient.java | 2 +- 7 files changed, 145 insertions(+), 130 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index d2d7a2ad58d2..e13ef7c8afd1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -13,15 +13,6 @@ package org.openmetadata.service.jdbi3; -import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; -import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -43,6 +34,16 @@ import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; +import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT; + @Repository public class LineageRepository { private final CollectionDAO dao; @@ -130,6 +131,7 @@ private void processLineageData( relationshipDetails.put( "pipeline", JsonUtils.getMap(CommonUtil.nullOrEmpty(lineageDetails.getPipeline()) ? null : lineageDetails.getPipeline())); + relationshipDetails.put("edgeDescription", CommonUtil.nullOrEmpty(lineageDetails.getDescription()) ? null : lineageDetails.getDescription()); if (!CommonUtil.nullOrEmpty(lineageDetails.getColumnsLineage())) { List> colummnLineageList = new ArrayList<>(); for (ColumnLineage columnLineage : lineageDetails.getColumnsLineage()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java index 037d25fd61b6..2c33c574faeb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java @@ -13,9 +13,6 @@ package org.openmetadata.service.resources.search; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.service.search.SearchRepository.ELASTIC_SEARCH_EXTENSION; - import es.org.elasticsearch.action.search.SearchResponse; import es.org.elasticsearch.search.suggest.Suggest; import io.swagger.v3.oas.annotations.Operation; @@ -24,8 +21,15 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; -import java.io.IOException; -import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.system.EventPublisherJob; +import org.openmetadata.service.Entity; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.search.SearchRequest; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.JsonUtils; + import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -36,14 +40,11 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; -import lombok.extern.slf4j.Slf4j; -import org.openmetadata.schema.system.EventPublisherJob; -import org.openmetadata.service.Entity; -import org.openmetadata.service.resources.Collection; -import org.openmetadata.service.search.SearchRepository; -import org.openmetadata.service.search.SearchRequest; -import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.util.JsonUtils; +import java.io.IOException; +import java.util.List; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.search.SearchRepository.ELASTIC_SEARCH_EXTENSION; @Slf4j @Path("/v1/search") @@ -244,10 +245,13 @@ public Response searchLineage( description = "Elasticsearch query that will be combined with the query_string query generator from the `query` argument") @QueryParam("query_filter") - String queryFilter) + String queryFilter, + @Parameter(description = "Filter documents by deleted param. By default deleted is false") + @QueryParam("deleted") + boolean deleted) throws IOException { - return searchRepository.searchLineage(fqn, upstreamDepth, downstreamDepth, queryFilter); + return searchRepository.searchLineage(fqn, upstreamDepth, downstreamDepth, queryFilter, deleted); } @GET diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index 7d1591e558d1..0c93fa9fca9d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -1,16 +1,5 @@ package org.openmetadata.service.search; -import static org.openmetadata.service.exception.CatalogExceptionMessage.NOT_IMPLEMENTED_METHOD; - -import java.io.IOException; -import java.security.KeyStoreException; -import java.text.ParseException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import javax.net.ssl.SSLContext; -import javax.ws.rs.core.Response; import org.apache.commons.lang3.tuple.Pair; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; @@ -21,6 +10,18 @@ import os.org.opensearch.action.bulk.BulkResponse; import os.org.opensearch.client.RequestOptions; +import javax.net.ssl.SSLContext; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.security.KeyStoreException; +import java.text.ParseException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.openmetadata.service.exception.CatalogExceptionMessage.NOT_IMPLEMENTED_METHOD; + public interface SearchClient { String UPDATE = "update"; @@ -73,7 +74,7 @@ public interface SearchClient { Response searchBySourceUrl(String sourceUrl) throws IOException; - Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth, String queryFilter) throws IOException; + Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) throws IOException; Response searchByField(String fieldName, String fieldValue, String index) throws IOException; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index c820edb85980..0abb80178cb8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -1,40 +1,6 @@ package org.openmetadata.service.search; -import static org.openmetadata.service.Entity.AGGREGATED_COST_ANALYSIS_REPORT_DATA; -import static org.openmetadata.service.Entity.ENTITY_REPORT_DATA; -import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; -import static org.openmetadata.service.Entity.FIELD_USAGE_SUMMARY; -import static org.openmetadata.service.Entity.QUERY; -import static org.openmetadata.service.Entity.RAW_COST_ANALYSIS_REPORT_DATA; -import static org.openmetadata.service.Entity.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA; -import static org.openmetadata.service.Entity.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA; -import static org.openmetadata.service.search.SearchClient.DEFAULT_UPDATE_SCRIPT; -import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; -import static org.openmetadata.service.search.SearchClient.PROPAGATE_ENTITY_REFERENCE_FIELD_SCRIPT; -import static org.openmetadata.service.search.SearchClient.PROPAGATE_FIELD_SCRIPT; -import static org.openmetadata.service.search.SearchClient.REMOVE_DOMAINS_CHILDREN_SCRIPT; -import static org.openmetadata.service.search.SearchClient.REMOVE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT; -import static org.openmetadata.service.search.SearchClient.REMOVE_PROPAGATED_FIELD_SCRIPT; -import static org.openmetadata.service.search.SearchClient.REMOVE_TAGS_CHILDREN_SCRIPT; -import static org.openmetadata.service.search.SearchClient.REMOVE_TEST_SUITE_CHILDREN_SCRIPT; -import static org.openmetadata.service.search.SearchClient.SOFT_DELETE_RESTORE_SCRIPT; -import static org.openmetadata.service.search.SearchClient.UPDATE_ADDED_DELETE_GLOSSARY_TAGS; -import static org.openmetadata.service.search.SearchClient.UPDATE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT; - import com.fasterxml.jackson.core.type.TypeReference; -import java.io.IOException; -import java.io.InputStream; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.SortedMap; -import javax.json.JsonObject; -import javax.ws.rs.core.Response; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -61,6 +27,41 @@ import org.openmetadata.service.search.opensearch.OpenSearchClient; import org.openmetadata.service.util.JsonUtils; +import javax.json.JsonObject; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.io.InputStream; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.SortedMap; + +import static org.openmetadata.service.Entity.AGGREGATED_COST_ANALYSIS_REPORT_DATA; +import static org.openmetadata.service.Entity.ENTITY_REPORT_DATA; +import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; +import static org.openmetadata.service.Entity.FIELD_USAGE_SUMMARY; +import static org.openmetadata.service.Entity.QUERY; +import static org.openmetadata.service.Entity.RAW_COST_ANALYSIS_REPORT_DATA; +import static org.openmetadata.service.Entity.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA; +import static org.openmetadata.service.Entity.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA; +import static org.openmetadata.service.search.SearchClient.DEFAULT_UPDATE_SCRIPT; +import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; +import static org.openmetadata.service.search.SearchClient.PROPAGATE_ENTITY_REFERENCE_FIELD_SCRIPT; +import static org.openmetadata.service.search.SearchClient.PROPAGATE_FIELD_SCRIPT; +import static org.openmetadata.service.search.SearchClient.REMOVE_DOMAINS_CHILDREN_SCRIPT; +import static org.openmetadata.service.search.SearchClient.REMOVE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT; +import static org.openmetadata.service.search.SearchClient.REMOVE_PROPAGATED_FIELD_SCRIPT; +import static org.openmetadata.service.search.SearchClient.REMOVE_TAGS_CHILDREN_SCRIPT; +import static org.openmetadata.service.search.SearchClient.REMOVE_TEST_SUITE_CHILDREN_SCRIPT; +import static org.openmetadata.service.search.SearchClient.SOFT_DELETE_RESTORE_SCRIPT; +import static org.openmetadata.service.search.SearchClient.UPDATE_ADDED_DELETE_GLOSSARY_TAGS; +import static org.openmetadata.service.search.SearchClient.UPDATE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT; + @Slf4j public class SearchRepository { @@ -620,9 +621,9 @@ public Response searchBySourceUrl(String sourceUrl) throws IOException { return searchClient.searchBySourceUrl(sourceUrl); } - public Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth, String queryFilter) + public Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) throws IOException { - return searchClient.searchLineage(fqn, upstreamDepth, downstreamDepth, queryFilter); + return searchClient.searchLineage(fqn, upstreamDepth, downstreamDepth, queryFilter, deleted); } public Response searchByField(String fieldName, String fieldValue, String index) throws IOException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index d61a4cae469d..e45515e5923d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -1,26 +1,5 @@ package org.openmetadata.service.search.elasticsearch; -import static javax.ws.rs.core.Response.Status.OK; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; -import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; -import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.Entity.QUERY; -import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; -import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES; -import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; -import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; -import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; -import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; -import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES; -import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; -import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; - import com.fasterxml.jackson.databind.JsonNode; import es.org.elasticsearch.action.ActionListener; import es.org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -89,20 +68,6 @@ import es.org.elasticsearch.xcontent.NamedXContentRegistry; import es.org.elasticsearch.xcontent.XContentParser; import es.org.elasticsearch.xcontent.XContentType; -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; -import javax.net.ssl.SSLContext; -import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -111,6 +76,7 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; +import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.DataInsightInterface; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; @@ -155,6 +121,42 @@ import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.JsonUtils; +import javax.net.ssl.SSLContext; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + +import static javax.ws.rs.core.Response.Status.OK; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; +import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_NAME; +import static org.openmetadata.service.Entity.QUERY; +import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES; +import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE; +import static org.openmetadata.service.search.EntityBuilderConstant.MAX_RESULT_HITS; +import static org.openmetadata.service.search.EntityBuilderConstant.OWNER_DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.POST_TAG; +import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG; +import static org.openmetadata.service.search.EntityBuilderConstant.SCHEMA_FIELD_NAMES; +import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; +import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; + @Slf4j // Not tagged with Repository annotation as it is programmatically initialized public class ElasticSearchClient implements SearchClient { @@ -400,7 +402,7 @@ public Response search(SearchRequest request) throws IOException { } @Override - public Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth, String queryFilter) + public Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) throws IOException { Map responseMap = new HashMap<>(); List> edges = new ArrayList<>(); @@ -414,8 +416,8 @@ public Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth for (var hit : searchResponse.getHits().getHits()) { responseMap.put("entity", hit.getSourceAsMap()); } - getLineage(fqn, downstreamDepth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword"); - getLineage(fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword"); + getLineage(fqn, downstreamDepth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword", deleted); + getLineage(fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword", deleted); responseMap.put("edges", edges); responseMap.put("nodes", nodes); return Response.status(OK).entity(responseMap).build(); @@ -427,7 +429,7 @@ private void getLineage( List> edges, Set> nodes, String queryFilter, - String direction) + String direction, boolean deleted) throws IOException { if (depth <= 0) { return; @@ -436,6 +438,9 @@ private void getLineage( new es.org.elasticsearch.action.search.SearchRequest("all"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(direction, fqn))); + if (CommonUtil.nullOrEmpty(deleted)){ + searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(direction, fqn)).must(QueryBuilders.termQuery("deleted", deleted))); + } if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) { try { XContentParser filterParser = @@ -461,12 +466,12 @@ private void getLineage( if (!edges.contains(lin) && fromEntity.get("fqn").equals(fqn)) { edges.add(lin); } - getLineage(toEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction); + getLineage(toEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction, deleted); } else { if (!edges.contains(lin) && toEntity.get("fqn").equals(fqn)) { edges.add(lin); } - getLineage(fromEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction); + getLineage(fromEntity.get("fqn"), depth - 1, edges, nodes, queryFilter, direction, deleted); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index e44efc3e6a38..b1d0f447583d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -1,21 +1,5 @@ package org.openmetadata.service.search.indexes; -import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; -import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; -import static org.openmetadata.service.Entity.FIELD_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; -import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; -import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM; -import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME; -import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; -import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.type.ColumnLineage; import org.openmetadata.schema.type.EntityReference; @@ -27,6 +11,23 @@ import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; +import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_NAME; +import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME; +import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS; +import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; + public interface SearchIndex { Map buildESDoc(); @@ -99,6 +100,7 @@ static void getLineageDataDirection( relationshipDetails.put( "pipeline", JsonUtils.getMap(CommonUtil.nullOrEmpty(lineageDetails.getPipeline()) ? null : lineageDetails.getPipeline())); + relationshipDetails.put("edgeDescription", CommonUtil.nullOrEmpty(lineageDetails.getDescription()) ? null : lineageDetails.getDescription()); if (!CommonUtil.nullOrEmpty(lineageDetails.getColumnsLineage())) { List> colummnLineageList = new ArrayList<>(); for (ColumnLineage columnLineage : lineageDetails.getColumnsLineage()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 5a5016dc77a6..663784ad33b6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -408,7 +408,7 @@ public Response searchBySourceUrl(String sourceUrl) throws IOException { } @Override - public Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth, String queryFilter) + public Response searchLineage(String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) throws IOException { return Response.status(OK).build(); } From 9365069cddfb308b26150b8664bdef2f90c18ab2 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Thu, 4 Jan 2024 17:23:58 +0530 Subject: [PATCH 33/44] fix: initial lineage tests --- .../ui/cypress/constants/lineage.constants.js | 54 ++++++ .../ui/cypress/e2e/Flow/Lineage.spec.js | 161 +++++++++++------- .../EdgeInfoDrawer.component.tsx | 2 +- .../EntityLineage/CustomEdge.component.tsx | 18 +- .../EntityLineage/CustomNodeV1.component.tsx | 6 +- .../EntityLineageSidebar.component.tsx | 5 +- .../NodeSuggestions.component.tsx | 10 +- .../EntityLineage/entity-lineage.style.less | 5 +- .../components/Lineage/Lineage.component.tsx | 34 +++- .../LineageProvider.interface.tsx | 6 +- .../LineageProvider/LineageProvider.tsx | 31 ++-- .../ui/src/constants/Lineage.constants.ts | 18 +- .../resources/ui/src/interface/types.d.ts | 1 + .../main/resources/ui/src/utils/APIUtils.ts | 4 + 14 files changed, 254 insertions(+), 101 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js new file mode 100644 index 000000000000..6e0dd1a4c99d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js @@ -0,0 +1,54 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DATA_ASSETS, SEARCH_INDEX } from './constants'; + +export const PIPELINE_SUPPORTED_TYPES = ['Table', 'Topic']; + +export const LINEAGE_ITEMS = [ + { + term: 'raw_customer', + displayName: 'raw_customer', + entity: DATA_ASSETS.tables, + serviceName: 'sample_data', + entityType: 'Table', + fqn: 'sample_data.ecommerce_db.shopify.raw_customer', + searchIndex: SEARCH_INDEX.tables, + }, + { + term: 'fact_session', + displayName: 'fact_session', + entity: DATA_ASSETS.tables, + serviceName: 'sample_data', + schemaName: 'shopify', + entityType: 'Table', + fqn: 'sample_data.ecommerce_db.shopify.fact_session', + searchIndex: SEARCH_INDEX.tables, + }, + { + term: 'shop_products', + displayName: 'shop_products', + entity: DATA_ASSETS.topics, + serviceName: 'sample_kafka', + fqn: 'sample_kafka.shop_products', + entityType: 'Topic', + searchIndex: SEARCH_INDEX.topics, + }, + { + term: 'forecast_sales', + entity: DATA_ASSETS.mlmodels, + serviceName: 'mlflow_svc', + entityType: 'ML Model', + fqn: 'mlflow_svc.forecast_sales', + searchIndex: SEARCH_INDEX.mlmodels, + }, +]; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js index 378d21dd0475..2b6a80b6c96d 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js @@ -11,79 +11,126 @@ * limitations under the License. */ -import { visitEntityDetailsPage } from '../../common/common'; import { - SEARCH_ENTITY_PIPELINE, - SEARCH_ENTITY_TABLE, - SEARCH_ENTITY_TOPIC, -} from '../../constants/constants'; - -const tableEntity = SEARCH_ENTITY_TABLE.table_1; -const topicEntity = SEARCH_ENTITY_TOPIC.topic_1; -const pipelineEntity = SEARCH_ENTITY_PIPELINE.pipeline_1; -// Todo:- skipping flaky test -// const dashboardEntity = SEARCH_ENTITY_DASHBOARD.dashboard_1; - -const ENTITIES_LIST = [ - tableEntity, - topicEntity, - pipelineEntity, - // dashboardEntity, -]; + interceptURL, + verifyResponseStatusCode, + visitEntityDetailsPage, +} from '../../common/common'; +import { LINEAGE_ITEMS } from '../../constants/lineage.constants'; + +const dataTransfer = new DataTransfer(); + +const dragConnection = (sourceFqn, targetFqn) => { + cy.get( + `[data-testid="lineage-node-${sourceFqn}"] .react-flow__handle-right` + ).click({ force: true }); // Adding force true for handles because it can be hidden behind the node + + return cy + .get(`[data-testid="lineage-node-${targetFqn}"] .react-flow__handle-left`) + .click({ force: true }); // Adding force true for handles because it can be hidden behind the node +}; + +const connectEdgeBetweenNodes = (fromNode, toNode) => { + interceptURL('PUT', '/api/v1/lineage', 'lineageApi'); + const type = toNode.searchIndex; + + cy.get(`[data-testid="${type}-draggable-icon"]`) + .invoke('attr', 'draggable') + .should('contain', 'true'); + + cy.get(`[data-testid="${type}-draggable-icon"]`).trigger('dragstart', { + dataTransfer, + }); + + cy.get('[data-testid="lineage-details"]') + .trigger('drop', { dataTransfer }) + .trigger('dragend'); + + cy.get(`[data-testid="${type}-draggable-icon"]`) + .invoke('attr', 'draggable') + .should('contain', 'false'); + + cy.get('[data-testid="suggestion-node"]').click(); + cy.get('[data-testid="suggestion-node"] input').click().type(toNode.term); + cy.get('.ant-select-dropdown .ant-select-item').eq(0).click(); + + dragConnection(fromNode.fqn, toNode.fqn); + verifyResponseStatusCode('@lineageApi', 200); +}; + +const verifyNodePresent = (node) => { + cy.get('.react-flow__controls-fitview').click(); + cy.get(`[data-testid="lineage-node-${node.fqn}"]`).should('be.visible'); + cy.get( + `[data-testid="lineage-node-${node.fqn}"] [data-testid="entity-header-name"]` + ).should('have.text', node.term); +}; + +const deleteNode = (node) => { + interceptURL('DELETE', '/api/v1/lineage/**', 'lineageDeleteApi'); + cy.get(`[data-testid="lineage-node-${node.fqn}"]`).click(); + // Adding force true for handles because it can be hidden behind the node + cy.get('[data-testid="lineage-node-remove-btn"]').click({ force: true }); + verifyResponseStatusCode('@lineageDeleteApi', 200); +}; describe('Entity Details Page', () => { beforeEach(() => { cy.login(); }); - ENTITIES_LIST.map((entity) => { - it(`Edit lineage should work for ${entity.entity} entity`, () => { + LINEAGE_ITEMS.forEach((entity, index) => { + it(`Lineage Add Node for entity ${entity.entityType}`, () => { visitEntityDetailsPage({ term: entity.term, serviceName: entity.serviceName, entity: entity.entity, }); - cy.get('[data-testid="lineage"]').should('be.visible').click(); - // Check edit button should not be disabled - cy.get('[data-testid="edit-lineage"]') - .should('be.visible') - .should('not.be.disabled'); + cy.get('[data-testid="lineage"]').click(); + cy.get('[data-testid="edit-lineage"]').click(); + + // Connect the current entity to all others in the array except itself + for (let i = 0; i < LINEAGE_ITEMS.length; i++) { + if (i !== index) { + connectEdgeBetweenNodes(entity, LINEAGE_ITEMS[i]); + } + } + + cy.get('[data-testid="edit-lineage"]').click(); + cy.reload(); + + // Verify Added Nodes + for (let i = 0; i < LINEAGE_ITEMS.length; i++) { + if (i !== index) { + verifyNodePresent(LINEAGE_ITEMS[i]); + } + } + + cy.get('[data-testid="edit-lineage"]').click(); }); - }); -}); -describe('Lineage functionality', () => { - beforeEach(() => { - cy.login(); - }); + it(`Lineage Remove Node between ${entity.entityType}`, () => { + visitEntityDetailsPage({ + term: entity.term, + serviceName: entity.serviceName, + entity: entity.entity, + }); + + cy.get('[data-testid="lineage"]').click(); + cy.get('[data-testid="edit-lineage"]').click(); + + // Delete Nodes + for (let i = 0; i < LINEAGE_ITEMS.length; i++) { + if (i !== index) { + deleteNode(LINEAGE_ITEMS[i]); + cy.get(`[data-testid="lineage-node-${LINEAGE_ITEMS[i].fqn}"]`).should( + 'not.exist' + ); + } + } - it('toggle fullscreen mode', () => { - visitEntityDetailsPage({ - term: tableEntity.term, - serviceName: tableEntity.serviceName, - entity: tableEntity.entity, + cy.get('[data-testid="edit-lineage"]').click(); }); - cy.get('[data-testid="lineage"]').click(); - - // Enable fullscreen - cy.get('[data-testid="full-screen"]').click(); - cy.url().should('include', 'fullscreen=true'); - cy.get('[data-testid="breadcrumb"]') - .should('be.visible') - .and('contain', 'Lineage'); - cy.get('[data-testid="lineage-details"]').should( - 'have.class', - 'full-screen-lineage' - ); - - // Exit fullscreen - cy.get('[data-testid="exit-full-screen"]').click(); - cy.url().should('not.include', 'fullscreen=true'); - cy.get('[data-testid="breadcrumb"]').should('not.contain', 'Lineage'); - cy.get('[data-testid="lineage-details"]').should( - 'not.have.class', - 'full-screen-lineage' - ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx index 56840c2fb7fe..2367013cd2a8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx @@ -277,7 +277,7 @@ const EdgeInfoDrawer = ({ header={t('label.edit-entity', { entity: t('label.sql-uppercase-query'), })} - value={mysqlQuery} + value={mysqlQuery ?? ''} visible={showSqlQueryModal} onCancel={() => setShowSqlQueryModal(false)} onSave={onSqlQueryUpdate} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx index 033a8bc4699a..a78a91ba9916 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx @@ -62,7 +62,7 @@ export const CustomEdge = ({ data, selected, }: EdgeProps) => { - const { edge, isColumnLineage, ...rest } = data; + const { edge, isColumnLineage, sourceHandle, targetHandle, ...rest } = data; const offset = 4; const { @@ -79,10 +79,10 @@ export const CustomEdge = ({ } return ( - tracedColumns.includes(data.sourceHandle) && - tracedColumns.includes(data.targetHandle) + tracedColumns.includes(sourceHandle) && + tracedColumns.includes(targetHandle) ); - }, [isColumnLineage, tracedColumns]); + }, [isColumnLineage, tracedColumns, sourceHandle, targetHandle]); const [edgePath, edgeCenterX, edgeCenterY] = getBezierPath({ sourceX, @@ -218,12 +218,20 @@ export const CustomEdge = ({ [offset, edgeCenterX, edgeCenterY, rest, data] ); + const dataTestId = useMemo(() => { + if (!isColumnLineage) { + return `edge-${edge.fromEntity.fqn}-${edge.toEntity.fqn}`; + } else { + return `column-edge-${sourceHandle}-${targetHandle}`; + } + }, [edge, isColumnLineage, sourceHandle, targetHandle]); + return ( { const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type; const isSelected = selectedNode === node; - const { columns, id, testSuite, lineage } = node; + const { columns, id, testSuite, lineage, fullyQualifiedName } = node; const [searchValue, setSearchValue] = useState(''); const [filteredColumns, setFilteredColumns] = useState([]); const [showAllColumns, setShowAllColumns] = useState(false); @@ -156,6 +156,7 @@ const CustomNodeV1 = (props: NodeProps) => { {isSelected && isEditMode ? (
diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts index 9f681e011a8f..8b844670b948 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts @@ -13,7 +13,7 @@ import { t } from 'i18next'; import { ElementLoadingState } from '../components/Entity/EntityLineage/EntityLineage.interface'; -import { EntityType } from '../enums/entity.enum'; +import { SearchIndex } from '../enums/search.enum'; export const FOREIGN_OBJECT_SIZE = 40; export const ZOOM_VALUE = 0.75; @@ -27,35 +27,35 @@ export const PIPELINE_EDGE_WIDTH = 200; export const entityData = [ { - type: EntityType.TABLE, + type: SearchIndex.TABLE, label: t('label.table-plural'), }, { - type: EntityType.DASHBOARD, + type: SearchIndex.DASHBOARD, label: t('label.dashboard-plural'), }, { - type: EntityType.TOPIC, + type: SearchIndex.TOPIC, label: t('label.topic-plural'), }, { - type: EntityType.MLMODEL, + type: SearchIndex.MLMODEL, label: t('label.ml-model-plural'), }, { - type: EntityType.CONTAINER, + type: SearchIndex.CONTAINER, label: t('label.container-plural'), }, { - type: EntityType.PIPELINE, + type: SearchIndex.PIPELINE, label: t('label.pipeline-plural'), }, { - type: EntityType.SEARCH_INDEX, + type: SearchIndex.SEARCH_INDEX, label: t('label.search-index-plural'), }, { - type: EntityType.DASHBOARD_DATA_MODEL, + type: SearchIndex.DASHBOARD_DATA_MODEL, label: t('label.data-model-plural'), }, ]; diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/types.d.ts b/openmetadata-ui/src/main/resources/ui/src/interface/types.d.ts index 47328f892790..19803e00f918 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/types.d.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/types.d.ts @@ -93,6 +93,7 @@ declare module 'Models' { deleted?: boolean; entityType?: string; changeDescription?: ChangeDescription; + columns?: TableColumn[]; }; export type SearchedUsersAndTeams = { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/APIUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/APIUtils.ts index 88093d1f110f..39e463fb73cf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/APIUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/APIUtils.ts @@ -80,6 +80,10 @@ export const formatDataResponse = ( newData.databaseSchema = source.databaseSchema?.name; } + if ('columns' in source) { + newData.columns = source.columns; + } + newData.changeDescription = source.changeDescription; return newData; From c7289d3c49b37475d45482dcb951d22e63205267 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Thu, 4 Jan 2024 18:10:38 +0530 Subject: [PATCH 34/44] fix unit tests --- .../EntityLineage/NodeSuggestions.test.tsx | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.test.tsx index b7994ec36750..1b64c47f6738 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.test.tsx @@ -19,10 +19,20 @@ import NodeSuggestions from './NodeSuggestions.component'; const mockProps = { onSelectHandler: jest.fn(), - entityType: 'TABLE', + entityType: SearchIndex.TABLE, }; -const entityType = ['TABLE', 'TOPIC', 'DASHBOARD', 'MLMODEL']; +const entityType = [ + SearchIndex.TABLE, + SearchIndex.TOPIC, + SearchIndex.DASHBOARD, + SearchIndex.PIPELINE, + SearchIndex.MLMODEL, + SearchIndex.CONTAINER, + SearchIndex.PIPELINE, + SearchIndex.SEARCH_INDEX, + SearchIndex.DASHBOARD_DATA_MODEL, +]; jest.mock('../../../rest/miscAPI', () => ({ searchData: jest.fn().mockImplementation(() => Promise.resolve()), @@ -50,9 +60,7 @@ describe('Test NodeSuggestions Component', () => { // 1st call on page load with empty search string and respective searchIndex expect(mockSearchData.mock.calls[0][0]).toBe(''); - expect(mockSearchData.mock.calls[0][6]).toEqual( - SearchIndex[value as keyof typeof SearchIndex] - ); + expect(mockSearchData.mock.calls[0][6]).toEqual(value); const suggestionNode = await screen.findByTestId('suggestion-node'); const searchInput = await screen.findByRole('combobox'); From 083b0f3dadc2e64570eeb6472700952b0421da6f Mon Sep 17 00:00:00 2001 From: karanh37 Date: Sun, 7 Jan 2024 20:57:52 +0530 Subject: [PATCH 35/44] fix: review comments --- .../EntityInfoDrawer.component.tsx | 11 +- .../EntityInfoDrawer.interface.ts | 1 - .../CustomControls.component.tsx | 9 +- .../EntityLineage/CustomNodeV1.component.tsx | 10 +- .../LineageProvider/LineageProvider.tsx | 133 +++++++++++------- .../ui/src/utils/EntityLineageUtils.tsx | 27 ++++ 6 files changed, 120 insertions(+), 71 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx index 665666a3777d..7ae2cdff0307 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx @@ -49,7 +49,6 @@ const EntityInfoDrawer = ({ show, onCancel, selectedNode, - isMainNode = false, }: LineageDrawerProps) => { const [entityDetail, setEntityDetail] = useState( {} as EntityDetailUnion @@ -169,7 +168,7 @@ const EntityInfoDrawer = ({ open={show} style={{ position: 'absolute' }} title={ - + {'databaseSchema' in entityDetail && 'database' in entityDetail && ( + className={classNames( + 'flex items-center text-base entity-info-header-link' + )}> {getEntityIcon(selectedNode.entityType as string)} @@ -189,7 +188,7 @@ const EntityInfoDrawer = ({ selectedNode.displayName ?? selectedNode.name, selectedNode.fullyQualifiedName, selectedNode.entityType as string, - isMainNode + false )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts index e56e9e13bc91..454f3b7c4d42 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.interface.ts @@ -19,7 +19,6 @@ export interface LineageDrawerProps { show: boolean; onCancel: () => void; selectedNode: SourceType; - isMainNode: boolean; } export interface EdgeInfoDrawerInfo { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx index 54add983c5cc..3bc65210196b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx @@ -75,7 +75,6 @@ const CustomControls: FC = ({ onQueryFilterUpdate, onNodeClick, } = useLineageProvider(); - const [zoom, setZoom] = useState(zoomValue); const [selectedFilter, setSelectedFilter] = useState([]); const [selectedQuickFilters, setSelectedQuickFilters] = useState< ExploreQuickFilterField[] @@ -94,12 +93,6 @@ const CustomControls: FC = ({ })); }, [filters]); - useEffect(() => { - if (zoomValue !== zoom) { - setZoom(zoomValue); - } - }, [zoomValue]); - useEffect(() => { const dropdownItems = getAssetsPageQuickFilters(); @@ -258,7 +251,7 @@ const CustomControls: FC = ({ className="expand-btn" data-testid="expand-column" type="primary" - onClick={() => toggleColumnView()}> + onClick={toggleColumnView}> {expandAllColumns ? t('label.collapse-all') : t('label.expand-all')} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx index 5d707dadb7b1..249966b3ea6c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx @@ -17,6 +17,7 @@ import classNames from 'classnames'; import { isEmpty } from 'lodash'; import React, { memo, useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; +import { useParams } from 'react-router-dom'; import { getIncomers, getOutgoers, @@ -30,6 +31,7 @@ import { EntityLineageNodeType, EntityType } from '../../../enums/entity.enum'; import { formTwoDigitNumber } from '../../../utils/CommonUtils'; import { checkUpstreamDownstream } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; +import { getDecodedFqn } from '../../../utils/StringsUtils'; import SVGIcons from '../../../utils/SvgUtils'; import { getConstraintIcon, getEntityIcon } from '../../../utils/TableUtils'; import { useLineageProvider } from '../../LineageProvider/LineageProvider'; @@ -45,6 +47,7 @@ import LineageNodeLabelV1 from './LineageNodeLabelV1'; const CustomNodeV1 = (props: NodeProps) => { const { t } = useTranslation(); + const { fqn: entityFqn } = useParams<{ fqn: string }>(); const updateNodeInternals = useUpdateNodeInternals(); const { data, type, isConnectable } = props; @@ -62,7 +65,6 @@ const CustomNodeV1 = (props: NodeProps) => { loadChildNodesHandler, } = useLineageProvider(); - /* eslint-disable-next-line */ const { label, isNewNode, node = {} } = data; const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type; @@ -74,6 +76,10 @@ const CustomNodeV1 = (props: NodeProps) => { const [isExpanded, setIsExpanded] = useState(false); const [isTraced, setIsTraced] = useState(false); + const isRootNode = useMemo(() => { + return getDecodedFqn(entityFqn) === fullyQualifiedName; + }, [fullyQualifiedName, entityFqn]); + const { hasDownstream, hasUpstream } = useMemo(() => { return checkUpstreamDownstream(id, lineage ?? []); }, [id, lineage]); @@ -153,7 +159,7 @@ const CustomNodeV1 = (props: NodeProps) => { return ( <> - {isSelected && isEditMode ? ( + {isSelected && isEditMode && !isRootNode ? (