Skip to content

Commit

Permalink
RANGER-5111: Enhance Datashare Summary API to Include Labels, Keyword…
Browse files Browse the repository at this point in the history 
…s, and Share Counts for Actionable Filters

Signed-off-by: Ramesh Mani <[email protected]>
  • Loading branch information
@rkundam
rkundam authored and Ramesh Mani committed Feb 10, 2025
1 parent 852d894 commit c334ba8
Show file tree
Hide file tree
Showing 4 changed files with 173 additions and 16 deletions.
51 changes: 46 additions & 5 deletions agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerPrincipal.PrincipalType;
import org.apache.ranger.plugin.store.PList;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;

import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -167,8 +169,8 @@ public StringBuilder toString(StringBuilder sb) {
.append("acl={").append(acl).append("} ")
.append("validitySchedule={").append(validitySchedule).append("} ")
.append("termsOfUse={").append(termsOfUse).append("} ")
.append("labels={").append(validitySchedule).append("} ")
.append("keywords={").append(termsOfUse).append("} ")
.append("labels={").append(labels).append("} ")
.append("keywords={").append(keywords).append("} ")
.append("}");

return sb;
Expand Down Expand Up @@ -860,9 +862,48 @@ public StringBuilder toString(StringBuilder sb) {
.append("aclPrincipalsCount={").append(aclPrincipalsCount).append("} ")
.append("totalResourceCount={").append(totalResourceCount).append("} ")
.append("dataShares={").append(dataShares).append("} ")
.append("validitySchedule={").append(totalResourceCount).append("} ")
.append("labels={").append(totalResourceCount).append("} ")
.append("keywords={").append(totalResourceCount).append("} ")
.append("validitySchedule={").append(validitySchedule).append("} ")
.append("labels={").append(labels).append("} ")
.append("keywords={").append(keywords).append("} ")
.append("}");

return sb;
}
}

@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
@JsonIgnoreProperties(ignoreUnknown = true)
public static class DatasetsSummary extends PList<DatasetSummary> {
private static final long serialVersionUID = 1L;
private Map<String, Map<String, Integer>> additionalInfo;

public DatasetsSummary() {
super();
}

public DatasetsSummary(PList<DatasetSummary> datasetSummary, Map<String, Map<String, Integer>> additionalInfo) {
super(datasetSummary);
this.additionalInfo = (additionalInfo != null) ? additionalInfo : Collections.emptyMap();
}

public Map<String, Map<String, Integer>> getAdditionalInfo() {
return additionalInfo;
}

public void setAdditionalInfo(Map<String, Map<String, Integer>> additionalInfo) {
this.additionalInfo = additionalInfo;
}

@Override
public String toString() {
return toString(new StringBuilder()).toString();
}

public StringBuilder toString(StringBuilder sb) {
sb.append("DatasetsSummary={")
.append("list={").append(this.list).append("} ")
.append("additionalInfo={").append(additionalInfo).append("} ")
.append("}");

return sb;
Expand Down
22 changes: 22 additions & 0 deletions agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.apache.ranger.plugin.store;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

public class PList<T> implements java.io.Serializable {
Expand Down Expand Up @@ -75,6 +77,26 @@ public PList(List<T> list, int startIndex, int pageSize, long totalCount, int re
this.sortBy = sortBy;
}

public PList(PList<T> other) {
if (other != null) {
this.list = other.getList() != null ? new ArrayList<>(other.getList()) : Collections.emptyList();
this.startIndex = other.getStartIndex();
this.pageSize = other.getPageSize();
this.totalCount = other.getTotalCount();
this.resultSize = other.getResultSize();
this.sortType = other.getSortType();
this.sortBy = other.getSortBy();
} else {
this.list = Collections.emptyList();
this.startIndex = 0;
this.pageSize = 0;
this.totalCount = 0;
this.resultSize = 0;
this.sortType = null;
this.sortBy = null;
}
}

public int getListSize() {
return list == null ? 0 : list.size();
}
Expand Down
77 changes: 70 additions & 7 deletions security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.apache.ranger.plugin.model.RangerGds.DataShareInDatasetSummary;
import org.apache.ranger.plugin.model.RangerGds.DataShareSummary;
import org.apache.ranger.plugin.model.RangerGds.DatasetSummary;
import org.apache.ranger.plugin.model.RangerGds.DatasetsSummary;
import org.apache.ranger.plugin.model.RangerGds.GdsPermission;
import org.apache.ranger.plugin.model.RangerGds.GdsShareStatus;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShare;
Expand Down Expand Up @@ -120,6 +121,9 @@ public class GdsDBStore extends AbstractGdsStore {
public static final String NOT_AUTHORIZED_TO_VIEW_PROJECT_POLICIES = "User is not authorized to view policies for this dataset";
public static final String GDS_POLICY_NAME_TIMESTAMP_SEP = "@";

public static final String LABELS = "labelCounts";
public static final String KEYWORDS = "keywordCounts";

private static final Set<Integer> SHARE_STATUS_AGR = new HashSet<>(Arrays.asList(GdsShareStatus.ACTIVE.ordinal(), GdsShareStatus.GRANTED.ordinal(), GdsShareStatus.REQUESTED.ordinal()));

@Autowired
Expand Down Expand Up @@ -1370,19 +1374,68 @@ public ServiceGdsInfo getGdsInfoIfUpdated(String serviceName, Long lastKnownVers
}

public PList<DatasetSummary> getDatasetSummary(SearchFilter filter) {
LOG.debug("==> getDatasetSummary({})", filter);
return getDatasetSummary(filter, false);
}

PList<RangerDataset> datasets = getUnscrubbedDatasets(filter);
List<DatasetSummary> datasetSummary = toDatasetSummary(datasets.getList(), getGdsPermissionFromFilter(filter));
PList<DatasetSummary> ret = new PList<>(datasetSummary, datasets.getStartIndex(), datasets.getPageSize(), datasets.getTotalCount(), datasets.getResultSize(), datasets.getSortType(), datasets.getSortBy());
public DatasetsSummary getEnhancedDatasetSummary(SearchFilter filter) {
return getDatasetSummary(filter, true);
}

ret.setQueryTimeMS(datasets.getQueryTimeMS());
public DatasetsSummary getDatasetSummary(SearchFilter filter, boolean includeAdditionalInfo) {
LOG.debug("==> getDatasetSummary({}, {})", filter, includeAdditionalInfo);

LOG.debug("<== getDatasetSummary({}): ret={}", filter, ret);
PList<RangerDataset> datasets;
Map<String, Map<String, Integer>> additionalInfo = null;

if (includeAdditionalInfo) {
List<RangerDataset> datasetsMatchingCriteria = fetchDatasetsBySearchCriteria(filter);
additionalInfo = buildAdditionalInfoForDatasets(datasetsMatchingCriteria);
datasets = applyPaginataionAndSorting(datasetsMatchingCriteria, filter);
} else {
datasets = getUnscrubbedDatasets(filter);
}

List<DatasetSummary> datasetSummary = toDatasetSummary(datasets.getList(), getGdsPermissionFromFilter(filter));
PList<DatasetSummary> paginatedDatasetSummary = createdPaginatedDatasetSummary(datasets, datasetSummary);
DatasetsSummary ret = new DatasetsSummary(paginatedDatasetSummary, additionalInfo);

LOG.debug("<== getDatasetSummary({}, {}): ret={}", filter, includeAdditionalInfo, ret);

return ret;
}

private Map<String, Map<String, Integer>> buildAdditionalInfoForDatasets(List<RangerDataset> datasets) {
Map<String, Map<String, Integer>> additionalInfo = new HashMap<>();
for (RangerDataset dataset : datasets) {
updateAdditionalInfo(LABELS, dataset.getLabels(), additionalInfo);
updateAdditionalInfo(KEYWORDS, dataset.getKeywords(), additionalInfo);
}
return additionalInfo;
}

private void updateAdditionalInfo(String field, List<String> fieldValues, Map<String, Map<String, Integer>> additionalInfo) {
if (CollectionUtils.isNotEmpty(fieldValues)) {
Map<String, Integer> aggregatedFieldMap = additionalInfo.computeIfAbsent(field, key -> new HashMap<>());
for (String value : fieldValues) {
aggregatedFieldMap.put(value, aggregatedFieldMap.getOrDefault(value, 0) + 1);
}
}
}

private PList<DatasetSummary> createdPaginatedDatasetSummary(PList<RangerDataset> datasets, List<DatasetSummary> datasetSummary) {
PList<DatasetSummary> paginatedDatasetSummary = new PList<>(
datasetSummary,
datasets.getStartIndex(),
datasets.getPageSize(),
datasets.getTotalCount(),
datasets.getResultSize(),
datasets.getSortType(),
datasets.getSortBy());

paginatedDatasetSummary.setQueryTimeMS(datasets.getQueryTimeMS());
return paginatedDatasetSummary;
}

public PList<DataShareSummary> getDataShareSummary(SearchFilter filter) {
LOG.debug("==> getDataShareSummary({})", filter);

Expand Down Expand Up @@ -1698,6 +1751,12 @@ private PList<RangerProject> getUnscrubbedProjects(SearchFilter filter) {
}

private PList<RangerDataset> getUnscrubbedDatasets(SearchFilter filter) {
List<RangerDataset> datasets = fetchDatasetsBySearchCriteria(filter);

return applyPaginataionAndSorting(datasets, filter);
}

private List<RangerDataset> fetchDatasetsBySearchCriteria(SearchFilter filter) {
filter.setParam(SearchFilter.RETRIEVE_ALL_PAGES, "true");

GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
Expand Down Expand Up @@ -1727,10 +1786,14 @@ private PList<RangerDataset> getUnscrubbedDatasets(SearchFilter filter) {
}
}

return datasets;
}

private PList<RangerDataset> applyPaginataionAndSorting(List<RangerDataset> datasets, SearchFilter filter) {
int maxRows = filter.getMaxRows();
int startIndex = filter.getStartIndex();

return getPList(datasets, startIndex, maxRows, result.getSortBy(), result.getSortType());
return getPList(datasets, startIndex, maxRows, filter.getSortBy(), filter.getSortType());
}

private PList<RangerDataShare> getUnscrubbedDataShares(SearchFilter filter) {
Expand Down
39 changes: 35 additions & 4 deletions security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.ranger.plugin.model.RangerGds.DataShareInDatasetSummary;
import org.apache.ranger.plugin.model.RangerGds.DataShareSummary;
import org.apache.ranger.plugin.model.RangerGds.DatasetSummary;
import org.apache.ranger.plugin.model.RangerGds.DatasetsSummary;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShare;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShareInDataset;
import org.apache.ranger.plugin.model.RangerGds.RangerDataset;
Expand Down Expand Up @@ -449,13 +450,13 @@ public PList<DatasetSummary> getDatasetSummary(@Context HttpServletRequest reque
LOG.debug("==> GdsREST.getDatasetSummary()");

PList<DatasetSummary> ret;
RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getDatasetSummary()");
SearchFilter filter = null;
RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getDatasetSummary()");
SearchFilter filter = null;

try {
filter = searchUtil.getSearchFilter(request, datasetService.sortFields);

ret = gdsStore.getDatasetSummary(filter);
ret = gdsStore.getDatasetSummary(filter);
} catch (WebApplicationException we) {
throw we;
} catch (Throwable ex) {
Expand All @@ -466,7 +467,37 @@ public PList<DatasetSummary> getDatasetSummary(@Context HttpServletRequest reque
RangerPerfTracer.log(perf);
}

LOG.debug("<== GdsREST.getDatasetSummary(): {}", ret);
LOG.debug("<== GdsREST.getDatasetSummary()");

return ret;
}

@GET
@Path("/dataset/enhancedsummary")
@Produces("application/json")
@PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_DATASET_SUMMARY + "\")")
public DatasetsSummary getEnhancedDatasetSummary(@Context HttpServletRequest request) {
LOG.debug("==> GdsREST.getEnhancedDatasetSummary()");

DatasetsSummary ret;
RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getEnhancedDatasetSummary()");
SearchFilter filter = null;

try {
filter = searchUtil.getSearchFilter(request, datasetService.sortFields);

ret = gdsStore.getEnhancedDatasetSummary(filter);
} catch (WebApplicationException we) {
throw we;
} catch (Throwable ex) {
LOG.error("getEnhancedDatasetSummary({}) failed", filter, ex);

throw restErrorUtil.createRESTException(ex.getMessage());
} finally {
RangerPerfTracer.log(perf);
}

LOG.debug("<== GdsREST.getEnhancedDatasetSummary()");

return ret;
}
Expand Down

0 comments on commit c334ba8

Please sign in to comment.