diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java index 260ebc0a8b..6bdc02691a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java @@ -26,11 +26,13 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerPrincipal.PrincipalType; +import org.apache.ranger.plugin.store.PList; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -167,8 +169,8 @@ public StringBuilder toString(StringBuilder sb) { .append("acl={").append(acl).append("} ") .append("validitySchedule={").append(validitySchedule).append("} ") .append("termsOfUse={").append(termsOfUse).append("} ") - .append("labels={").append(validitySchedule).append("} ") - .append("keywords={").append(termsOfUse).append("} ") + .append("labels={").append(labels).append("} ") + .append("keywords={").append(keywords).append("} ") .append("}"); return sb; @@ -860,9 +862,48 @@ public StringBuilder toString(StringBuilder sb) { .append("aclPrincipalsCount={").append(aclPrincipalsCount).append("} ") .append("totalResourceCount={").append(totalResourceCount).append("} ") .append("dataShares={").append(dataShares).append("} ") - .append("validitySchedule={").append(totalResourceCount).append("} ") - .append("labels={").append(totalResourceCount).append("} ") - .append("keywords={").append(totalResourceCount).append("} ") + .append("validitySchedule={").append(validitySchedule).append("} ") + .append("labels={").append(labels).append("} ") + .append("keywords={").append(keywords).append("} ") + .append("}"); + + return sb; + } + } + + @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JsonIgnoreProperties(ignoreUnknown = true) + public static class DatasetsSummary extends PList { + private static final long serialVersionUID = 1L; + private Map> additionalInfo; + + public DatasetsSummary() { + super(); + } + + public DatasetsSummary(PList datasetSummary, Map> additionalInfo) { + super(datasetSummary); + this.additionalInfo = (additionalInfo != null) ? additionalInfo : Collections.emptyMap(); + } + + public Map> getAdditionalInfo() { + return additionalInfo; + } + + public void setAdditionalInfo(Map> additionalInfo) { + this.additionalInfo = additionalInfo; + } + + @Override + public String toString() { + return toString(new StringBuilder()).toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("DatasetsSummary={") + .append("list={").append(this.list).append("} ") + .append("additionalInfo={").append(additionalInfo).append("} ") .append("}"); return sb; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java index 4bf2ee02fe..1a1b16a5bf 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java @@ -19,6 +19,8 @@ package org.apache.ranger.plugin.store; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class PList implements java.io.Serializable { @@ -75,6 +77,26 @@ public PList(List list, int startIndex, int pageSize, long totalCount, int re this.sortBy = sortBy; } + public PList(PList other) { + if (other != null) { + this.list = other.getList() != null ? new ArrayList<>(other.getList()) : Collections.emptyList(); + this.startIndex = other.getStartIndex(); + this.pageSize = other.getPageSize(); + this.totalCount = other.getTotalCount(); + this.resultSize = other.getResultSize(); + this.sortType = other.getSortType(); + this.sortBy = other.getSortBy(); + } else { + this.list = Collections.emptyList(); + this.startIndex = 0; + this.pageSize = 0; + this.totalCount = 0; + this.resultSize = 0; + this.sortType = null; + this.sortBy = null; + } + } + public int getListSize() { return list == null ? 0 : list.size(); } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java index 7916f08188..312a67a28f 100755 --- a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java @@ -48,6 +48,7 @@ import org.apache.ranger.plugin.model.RangerGds.DataShareInDatasetSummary; import org.apache.ranger.plugin.model.RangerGds.DataShareSummary; import org.apache.ranger.plugin.model.RangerGds.DatasetSummary; +import org.apache.ranger.plugin.model.RangerGds.DatasetsSummary; import org.apache.ranger.plugin.model.RangerGds.GdsPermission; import org.apache.ranger.plugin.model.RangerGds.GdsShareStatus; import org.apache.ranger.plugin.model.RangerGds.RangerDataShare; @@ -120,6 +121,9 @@ public class GdsDBStore extends AbstractGdsStore { public static final String NOT_AUTHORIZED_TO_VIEW_PROJECT_POLICIES = "User is not authorized to view policies for this dataset"; public static final String GDS_POLICY_NAME_TIMESTAMP_SEP = "@"; + public static final String LABELS = "labelCounts"; + public static final String KEYWORDS = "keywordCounts"; + private static final Set SHARE_STATUS_AGR = new HashSet<>(Arrays.asList(GdsShareStatus.ACTIVE.ordinal(), GdsShareStatus.GRANTED.ordinal(), GdsShareStatus.REQUESTED.ordinal())); @Autowired @@ -1370,19 +1374,68 @@ public ServiceGdsInfo getGdsInfoIfUpdated(String serviceName, Long lastKnownVers } public PList getDatasetSummary(SearchFilter filter) { - LOG.debug("==> getDatasetSummary({})", filter); + return getDatasetSummary(filter, false); + } - PList datasets = getUnscrubbedDatasets(filter); - List datasetSummary = toDatasetSummary(datasets.getList(), getGdsPermissionFromFilter(filter)); - PList ret = new PList<>(datasetSummary, datasets.getStartIndex(), datasets.getPageSize(), datasets.getTotalCount(), datasets.getResultSize(), datasets.getSortType(), datasets.getSortBy()); + public DatasetsSummary getEnhancedDatasetSummary(SearchFilter filter) { + return getDatasetSummary(filter, true); + } - ret.setQueryTimeMS(datasets.getQueryTimeMS()); + public DatasetsSummary getDatasetSummary(SearchFilter filter, boolean includeAdditionalInfo) { + LOG.debug("==> getDatasetSummary({}, {})", filter, includeAdditionalInfo); - LOG.debug("<== getDatasetSummary({}): ret={}", filter, ret); + PList datasets; + Map> additionalInfo = null; + + if (includeAdditionalInfo) { + List datasetsMatchingCriteria = fetchDatasetsBySearchCriteria(filter); + additionalInfo = buildAdditionalInfoForDatasets(datasetsMatchingCriteria); + datasets = applyPaginataionAndSorting(datasetsMatchingCriteria, filter); + } else { + datasets = getUnscrubbedDatasets(filter); + } + + List datasetSummary = toDatasetSummary(datasets.getList(), getGdsPermissionFromFilter(filter)); + PList paginatedDatasetSummary = createdPaginatedDatasetSummary(datasets, datasetSummary); + DatasetsSummary ret = new DatasetsSummary(paginatedDatasetSummary, additionalInfo); + + LOG.debug("<== getDatasetSummary({}, {}): ret={}", filter, includeAdditionalInfo, ret); return ret; } + private Map> buildAdditionalInfoForDatasets(List datasets) { + Map> additionalInfo = new HashMap<>(); + for (RangerDataset dataset : datasets) { + updateAdditionalInfo(LABELS, dataset.getLabels(), additionalInfo); + updateAdditionalInfo(KEYWORDS, dataset.getKeywords(), additionalInfo); + } + return additionalInfo; + } + + private void updateAdditionalInfo(String field, List fieldValues, Map> additionalInfo) { + if (CollectionUtils.isNotEmpty(fieldValues)) { + Map aggregatedFieldMap = additionalInfo.computeIfAbsent(field, key -> new HashMap<>()); + for (String value : fieldValues) { + aggregatedFieldMap.put(value, aggregatedFieldMap.getOrDefault(value, 0) + 1); + } + } + } + + private PList createdPaginatedDatasetSummary(PList datasets, List datasetSummary) { + PList paginatedDatasetSummary = new PList<>( + datasetSummary, + datasets.getStartIndex(), + datasets.getPageSize(), + datasets.getTotalCount(), + datasets.getResultSize(), + datasets.getSortType(), + datasets.getSortBy()); + + paginatedDatasetSummary.setQueryTimeMS(datasets.getQueryTimeMS()); + return paginatedDatasetSummary; + } + public PList getDataShareSummary(SearchFilter filter) { LOG.debug("==> getDataShareSummary({})", filter); @@ -1698,6 +1751,12 @@ private PList getUnscrubbedProjects(SearchFilter filter) { } private PList getUnscrubbedDatasets(SearchFilter filter) { + List datasets = fetchDatasetsBySearchCriteria(filter); + + return applyPaginataionAndSorting(datasets, filter); + } + + private List fetchDatasetsBySearchCriteria(SearchFilter filter) { filter.setParam(SearchFilter.RETRIEVE_ALL_PAGES, "true"); GdsPermission gdsPermission = getGdsPermissionFromFilter(filter); @@ -1727,10 +1786,14 @@ private PList getUnscrubbedDatasets(SearchFilter filter) { } } + return datasets; + } + + private PList applyPaginataionAndSorting(List datasets, SearchFilter filter) { int maxRows = filter.getMaxRows(); int startIndex = filter.getStartIndex(); - return getPList(datasets, startIndex, maxRows, result.getSortBy(), result.getSortType()); + return getPList(datasets, startIndex, maxRows, filter.getSortBy(), filter.getSortType()); } private PList getUnscrubbedDataShares(SearchFilter filter) { diff --git a/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java b/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java index 0d3ef3d767..05bc4fd17b 100755 --- a/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java @@ -36,6 +36,7 @@ import org.apache.ranger.plugin.model.RangerGds.DataShareInDatasetSummary; import org.apache.ranger.plugin.model.RangerGds.DataShareSummary; import org.apache.ranger.plugin.model.RangerGds.DatasetSummary; +import org.apache.ranger.plugin.model.RangerGds.DatasetsSummary; import org.apache.ranger.plugin.model.RangerGds.RangerDataShare; import org.apache.ranger.plugin.model.RangerGds.RangerDataShareInDataset; import org.apache.ranger.plugin.model.RangerGds.RangerDataset; @@ -449,13 +450,13 @@ public PList getDatasetSummary(@Context HttpServletRequest reque LOG.debug("==> GdsREST.getDatasetSummary()"); PList ret; - RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getDatasetSummary()"); - SearchFilter filter = null; + RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getDatasetSummary()"); + SearchFilter filter = null; try { filter = searchUtil.getSearchFilter(request, datasetService.sortFields); - ret = gdsStore.getDatasetSummary(filter); + ret = gdsStore.getDatasetSummary(filter); } catch (WebApplicationException we) { throw we; } catch (Throwable ex) { @@ -466,7 +467,37 @@ public PList getDatasetSummary(@Context HttpServletRequest reque RangerPerfTracer.log(perf); } - LOG.debug("<== GdsREST.getDatasetSummary(): {}", ret); + LOG.debug("<== GdsREST.getDatasetSummary()"); + + return ret; + } + + @GET + @Path("/dataset/enhancedsummary") + @Produces("application/json") + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_DATASET_SUMMARY + "\")") + public DatasetsSummary getEnhancedDatasetSummary(@Context HttpServletRequest request) { + LOG.debug("==> GdsREST.getEnhancedDatasetSummary()"); + + DatasetsSummary ret; + RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getEnhancedDatasetSummary()"); + SearchFilter filter = null; + + try { + filter = searchUtil.getSearchFilter(request, datasetService.sortFields); + + ret = gdsStore.getEnhancedDatasetSummary(filter); + } catch (WebApplicationException we) { + throw we; + } catch (Throwable ex) { + LOG.error("getEnhancedDatasetSummary({}) failed", filter, ex); + + throw restErrorUtil.createRESTException(ex.getMessage()); + } finally { + RangerPerfTracer.log(perf); + } + + LOG.debug("<== GdsREST.getEnhancedDatasetSummary()"); return ret; }