Skip to content

Commit

Permalink
Add column sort and filters in dependency list view #823 (#830)
Browse files Browse the repository at this point in the history 
Signed-off-by: Thomas Druez <[email protected]>
  • Loading branch information
@tdruez
tdruez authored Jul 27, 2023
1 parent afda178 commit 28805d1
Show file tree
Hide file tree
Showing 5 changed files with 207 additions and 76 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ to be updated for the new ``app`` user, using:
require to be updated for the new ``app`` user.
https://github.com/nexB/scancode.io/issues/399

- Add column sort and filters in dependency list view.
https://github.com/nexB/scancode.io/issues/823

- Add a new ``ScanCodebasePackage`` pipeline to scan a codebase for packages only.
https://github.com/nexB/scancode.io/issues/815

Expand Down
201 changes: 132 additions & 69 deletions scanpipe/filters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,91 @@
OTHER_VAR = "_OTHER_"


class ParentAllValuesFilter(django_filters.ChoiceFilter):
"""
Similar to ``django_filters.AllValuesFilter`` but using the queryset of the parent
``FilterSet``.
"""

@property
def field(self):
qs = self.parent.queryset.distinct()
qs = qs.order_by(self.field_name).values_list(self.field_name, flat=True)
self.extra["choices"] = [(o, o) for o in qs]
return super().field


class StrictBooleanFilter(django_filters.ChoiceFilter):
def __init__(self, *args, **kwargs):
kwargs["choices"] = (
(True, _("Yes")),
(False, _("No")),
)
super().__init__(*args, **kwargs)


class BulmaLinkWidget(LinkWidget):
"""Replace LinkWidget rendering with Bulma CSS classes."""

extra_css_class = ""

def render_option(self, name, selected_choices, option_value, option_label):
option_value = str(option_value)
if option_label == BLANK_CHOICE_DASH[0][1]:
option_label = _("All")

data = self.data.copy()
data[name] = option_value
selected = data == self.data or option_value in selected_choices

# Do not include the pagination in the filter query string.
data.pop(PAGE_VAR, None)

css_class = str(self.extra_css_class)
if selected:
css_class += " is-active"

try:
url = data.urlencode()
except AttributeError:
url = urlencode(data, doseq=True)

return self.option_string().format(
css_class=css_class,
query_string=url,
label=str(option_label),
)

def option_string(self):
return '<li><a href="?{query_string}" class="{css_class}">{label}</a></li>'


class BulmaDropdownWidget(BulmaLinkWidget):
extra_css_class = "dropdown-item"


class HasValueDropdownWidget(BulmaDropdownWidget):
def __init__(self, attrs=None, choices=()):
super().__init__(attrs)
self.choices = (
("", "All"),
(EMPTY_VAR, "None"),
(ANY_VAR, "Any"),
)


class FilterSetUtilsMixin:
empty_value = EMPTY_VAR
any_value = ANY_VAR
other_value = OTHER_VAR
dropdown_widget_class = BulmaDropdownWidget
dropdown_widget_fields = []

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Set the widget class for defined ``dropdown_widget_fields``.
for field_name in self.dropdown_widget_fields:
self.filters[field_name].extra["widget"] = self.dropdown_widget_class

@staticmethod
def remove_field_from_query_dict(query_dict, field_name, remove_value=None):
Expand Down Expand Up @@ -137,57 +218,13 @@ def filter_queryset(self, queryset):
return queryset


class BulmaLinkWidget(LinkWidget):
"""Replace LinkWidget rendering with Bulma CSS classes."""

extra_css_class = ""

def render_option(self, name, selected_choices, option_value, option_label):
option_value = str(option_value)
if option_label == BLANK_CHOICE_DASH[0][1]:
option_label = _("All")

data = self.data.copy()
data[name] = option_value
selected = data == self.data or option_value in selected_choices

# Do not include the pagination in the filter query string.
data.pop(PAGE_VAR, None)

css_class = str(self.extra_css_class)
if selected:
css_class += " is-active"

try:
url = data.urlencode()
except AttributeError:
url = urlencode(data, doseq=True)

return self.option_string().format(
css_class=css_class,
query_string=url,
label=str(option_label),
)

def option_string(self):
return '<li><a href="?{query_string}" class="{css_class}">{label}</a></li>'


class BulmaDropdownWidget(BulmaLinkWidget):
extra_css_class = "dropdown-item"


class HasValueDropdownWidget(BulmaDropdownWidget):
def __init__(self, attrs=None, choices=()):
super().__init__(attrs)
self.choices = (
("", "All"),
(EMPTY_VAR, "None"),
(ANY_VAR, "Any"),
)


class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
dropdown_widget_fields = [
"sort",
"pipeline",
"status",
]

search = django_filters.CharFilter(
label="Search", field_name="name", lookup_expr="icontains"
)
Expand Down Expand Up @@ -215,13 +252,11 @@ class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
("-projecterrors_count", "Errors (+)"),
("projecterrors_count", "Errors (-)"),
),
widget=BulmaDropdownWidget,
)
pipeline = django_filters.ChoiceFilter(
label="Pipeline",
field_name="runs__pipeline_name",
choices=scanpipe_app.get_pipeline_choices(include_blank=False),
widget=BulmaDropdownWidget,
distinct=True,
)
status = django_filters.ChoiceFilter(
Expand All @@ -234,7 +269,6 @@ class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
("succeed", "Success"),
("failed", "Failure"),
],
widget=BulmaDropdownWidget,
distinct=True,
)

Expand Down Expand Up @@ -334,7 +368,7 @@ def filter(self, qs, value):


class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
dropdown_widget = [
dropdown_widget_fields = [
"status",
"type",
"compliance_alert",
Expand Down Expand Up @@ -407,17 +441,18 @@ class Meta:
"urls",
"in_package",
"relation_map_type",
"is_binary",
"is_text",
"is_archive",
"is_key_file",
"is_media",
]

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if status_filter := self.filters.get("status"):
status_filter.extra.update({"choices": self.get_status_choices()})

# Set the `BulmaDropdownWidget`` widget for defined ``dropdown_widget``.
for field_name in self.dropdown_widget:
self.filters[field_name].extra["widget"] = BulmaDropdownWidget()

license_expression_filer = self.filters["detected_license_expression"]
license_expression_filer.extra["widget"] = HasValueDropdownWidget()

Expand Down Expand Up @@ -458,6 +493,11 @@ def filter(self, qs, value):


class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
dropdown_widget_fields = [
"is_vulnerable",
"compliance_alert",
]

search = django_filters.CharFilter(
label="Search", field_name="name", lookup_expr="icontains"
)
Expand All @@ -472,13 +512,13 @@ class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
],
)
purl = PackageURLFilter(label="Package URL")
is_vulnerable = IsVulnerable(
field_name="affected_by_vulnerabilities",
widget=BulmaDropdownWidget,
)
is_vulnerable = IsVulnerable(field_name="affected_by_vulnerabilities")
compliance_alert = django_filters.ChoiceFilter(
choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
widget=BulmaDropdownWidget,
)
copyright = django_filters.filters.CharFilter(widget=HasValueDropdownWidget)
declared_license_expression = django_filters.filters.CharFilter(
widget=HasValueDropdownWidget
)

class Meta:
Expand Down Expand Up @@ -514,18 +554,41 @@ class Meta:
"compliance_alert",
]

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
license_expression_filer = self.filters["declared_license_expression"]
license_expression_filer.extra["widget"] = HasValueDropdownWidget()
self.filters["copyright"].extra["widget"] = HasValueDropdownWidget()


class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
dropdown_widget_fields = [
"type",
"scope",
"is_runtime",
"is_optional",
"is_resolved",
"datasource_id",
]

search = django_filters.CharFilter(
label="Search", field_name="name", lookup_expr="icontains"
)
sort = django_filters.OrderingFilter(
label="Sort",
fields=[
"type",
"extracted_requirement",
"scope",
"is_runtime",
"is_optional",
"is_resolved",
"for_package",
"datafile_resource",
"datasource_id",
],
)
purl = PackageURLFilter(label="Package URL")
type = ParentAllValuesFilter()
scope = ParentAllValuesFilter()
datasource_id = ParentAllValuesFilter()
is_runtime = StrictBooleanFilter()
is_optional = StrictBooleanFilter()
is_resolved = StrictBooleanFilter()

class Meta:
model = DiscoveredDependency
Expand Down
2 changes: 1 addition & 1 deletion scanpipe/templates/scanpipe/dependency_list.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
{% endif %}
</td>
<td>
{{ dependency.datasource_id }}
<a href="?datasource_id={{ dependency.datasource_id }}" class="is-black-link">{{ dependency.datasource_id }}</a>
</td>
</tr>
{% empty %}
Expand Down
45 changes: 45 additions & 0 deletions scanpipe/tests/test_filters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,18 @@
from django.test import TestCase
from django.utils import timezone

from scanpipe.filters import DependencyFilterSet
from scanpipe.filters import FilterSetUtilsMixin
from scanpipe.filters import PackageFilterSet
from scanpipe.filters import ProjectFilterSet
from scanpipe.filters import ResourceFilterSet
from scanpipe.models import CodebaseResource
from scanpipe.models import DiscoveredDependency
from scanpipe.models import DiscoveredPackage
from scanpipe.models import Project
from scanpipe.models import Run
from scanpipe.tests import dependency_data1
from scanpipe.tests import dependency_data2
from scanpipe.tests import package_data1
from scanpipe.tests import package_data2

Expand Down Expand Up @@ -158,3 +162,44 @@ def test_scanpipe_filters_package_filterset_is_vulnerable(self):

filterset = PackageFilterSet(data={"is_vulnerable": "yes"})
self.assertEqual([p2], list(filterset.qs))

def test_scanpipe_filters_dependency_filterset(self):
DiscoveredPackage.create_from_data(self.project1, package_data1)
CodebaseResource.objects.create(
project=self.project1,
path="daglib-0.3.2.tar.gz-extract/daglib-0.3.2/PKG-INFO",
)
CodebaseResource.objects.create(
project=self.project1,
path="data.tar.gz-extract/Gemfile.lock",
)
d1 = DiscoveredDependency.create_from_data(self.project1, dependency_data1)
d2 = DiscoveredDependency.create_from_data(self.project1, dependency_data2)

filterset = DependencyFilterSet(data={"is_resolved": ""})
self.assertEqual(2, len(filterset.qs))
filterset = DependencyFilterSet(data={"is_resolved": True})
self.assertEqual([d2], list(filterset.qs))
filterset = DependencyFilterSet(data={"is_resolved": False})
self.assertEqual([d1], list(filterset.qs))

filterset = DependencyFilterSet(data={"type": ""})
self.assertEqual(2, len(filterset.qs))
filterset = DependencyFilterSet(data={"type": "pypi"})
self.assertEqual([d1], list(filterset.qs))
filterset = DependencyFilterSet(data={"type": "gem"})
self.assertEqual([d2], list(filterset.qs))

filterset = DependencyFilterSet(data={"scope": ""})
self.assertEqual(2, len(filterset.qs))
filterset = DependencyFilterSet(data={"scope": "install"})
self.assertEqual([d1], list(filterset.qs))
filterset = DependencyFilterSet(data={"scope": "dependencies"})
self.assertEqual([d2], list(filterset.qs))

filterset = DependencyFilterSet(data={"datasource_id": ""})
self.assertEqual(2, len(filterset.qs))
filterset = DependencyFilterSet(data={"datasource_id": "pypi_sdist_pkginfo"})
self.assertEqual([d1], list(filterset.qs))
filterset = DependencyFilterSet(data={"datasource_id": "gemfile_lock"})
self.assertEqual([d2], list(filterset.qs))
Loading

0 comments on commit 28805d1

Please sign in to comment.