fix: Fetch all lineitems using pagination (#328)

src/illumidesk/grades/senders.py

@@ -3,6 +3,7 @@
 import logging
 import time
 import os
+import re
 
 from datetime import datetime
 
@@ -164,16 +165,51 @@ def __init__(self, course_id: str, assignment_name: str):
         nbgrader_service = NbGraderServiceHelper(course_id)
         course = nbgrader_service.get_course()
         self.course = course
+        self.all_lineitems = []
+        self.headers = {}
 
-    async def _get_line_item_info_by_assignment_name(self) -> str:
+    def _find_next_url(self, link_header: str) -> str:
+        """
+        Extract the url value from link header value
+        """
+        # split the paths
+        next_url = [n for n in link_header.split(',') if 'next' in n]
+        if next_url:
+            # get only one
+            next_url = next_url[0]
+            print('There are more lineitems in:', next_url)
+            link_regex = re.compile(
+                r"((https?):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&](#!)?)*)", re.DOTALL
+            )  # noqa W605
+            links = re.findall(link_regex, next_url)
+            if links:
+                return links[0][0]
+
+    async def _get_lineitems_from_url(self, url: str) -> None:
+        """
+        Fetch the lineitems from specific url and add them to general list
+        """
+        items = []
+        if not url:
+            return
         client = AsyncHTTPClient()
-        resp = await client.fetch(self.course.lms_lineitems_endpoint, headers=self.headers)
+        resp = await client.fetch(url, method='GET', headers=self.headers)
         items = json.loads(resp.body)
-        logger.debug(f'LineItems retrieved: {items}')
-        if not items or isinstance(items, list) is False:
+        if items:
+            self.all_lineitems.extend(items)
+            headers = resp.headers
+            # check if there is more items/pages
+            if 'Link' in headers and 'next' in headers['Link']:
+                next_url = self._find_next_url(headers['link'])
+                await self._get_lineitems_from_url(next_url)
+
+    async def _get_line_item_info_by_assignment_name(self) -> str:
+        await self._get_lineitems_from_url(self.course.lms_lineitems_endpoint)
+        if not self.all_lineitems:
             raise GradesSenderMissingInfoError(f'No line-items were detected for this course: {self.course_id}')
+        logger.debug(f'LineItems retrieved: {self.all_lineitems}')
         lineitem_matched = None
-        for item in items:
+        for item in self.all_lineitems:
             item_label = item['label']
             if self.assignment_name.lower() == item_label.lower() or self.assignment_name.lower() == LTIUtils().normalize_string(
                 item_label
@@ -184,6 +220,7 @@ async def _get_line_item_info_by_assignment_name(self) -> str:
         if lineitem_matched is None:
             raise GradesSenderMissingInfoError(f'No lineitem matched with the assignment name: {self.assignment_name}')
 
+        client = AsyncHTTPClient()
         resp = await client.fetch(lineitem_matched, headers=self.headers)
         lineitem_info = json.loads(resp.body)
         logger.debug(f'Fetched lineitem info from lms {lineitem_info}')

src/tests/illumidesk/grades/test_senders.py

@@ -2,6 +2,8 @@
 
 from unittest.mock import patch
 
+from tornado.httputil import HTTPHeaders
+
 from illumidesk.grades.senders import LTIGradeSender
 from illumidesk.grades.senders import LTI13GradeSender
 from illumidesk.grades.exceptions import GradesSenderCriticalError
@@ -98,10 +100,70 @@ async def test_sender_raises_an_error_if_no_line_items_were_found(
         sut = LTI13GradeSender('course-id', 'lab')
 
         access_token_result = {'token_type': '', 'access_token': ''}
-        with patch('illumidesk.grades.senders.get_lms_access_token', return_value=access_token_result) as mock_method:
+        with patch('illumidesk.grades.senders.get_lms_access_token', return_value=access_token_result):
 
             with patch.object(
                 LTI13GradeSender, '_retrieve_grades_from_db', return_value=(lambda: 10, [{'score': 10}])
             ):
                 with pytest.raises(GradesSenderMissingInfoError):
                     await sut.send_grades()
+
+    @pytest.mark.asyncio
+    async def test_get_lineitems_from_url_method_does_fetch_lineitems_from_url(
+        self, lti_config_environ, mock_nbhelper, make_http_response, make_mock_request_handler
+    ):
+        local_handler = make_mock_request_handler(RequestHandler)
+        sut = LTI13GradeSender('course-id', 'lab')
+        lineitems_url = 'https://example.canvas.com/api/lti/courses/111/line_items'
+        with patch.object(
+            AsyncHTTPClient, 'fetch', return_value=make_http_response(handler=local_handler.request)
+        ) as mock_client:
+            await sut._get_lineitems_from_url(lineitems_url)
+            assert mock_client.called
+            mock_client.assert_called_with(lineitems_url, method='GET', headers={})
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "http_async_httpclient_with_simple_response",
+        [[{"id": "value", "scoreMaximum": 0.0, "label": "label", "resourceLinkId": "abc"}]],
+        indirect=True,
+    )
+    async def test_get_lineitems_from_url_method_sets_all_lineitems_property(
+        self, lti_config_environ, mock_nbhelper, http_async_httpclient_with_simple_response
+    ):
+        sut = LTI13GradeSender('course-id', 'lab')
+
+        await sut._get_lineitems_from_url('https://example.canvas.com/api/lti/courses/111/line_items')
+        assert len(sut.all_lineitems) == 1
+
+    @pytest.mark.asyncio
+    async def test_get_lineitems_from_url_method_calls_itself_recursively(
+        self, lti_config_environ, mock_nbhelper, make_http_response, make_mock_request_handler
+    ):
+        local_handler = make_mock_request_handler(RequestHandler)
+        sut = LTI13GradeSender('course-id', 'lab')
+
+        lineitems_body_result = {
+            'body': [{"id": "value", "scoreMaximum": 0.0, "label": "label", "resourceLinkId": "abc"}]
+        }
+        lineitems_body_result['headers'] = HTTPHeaders(
+            {
+                'content-type': 'application/vnd.ims.lis.v2.lineitemcontainer+json',
+                'link': '<https://learning.flatironschool.com/api/lti/courses/691/line_items?page=2&per_page=10>; rel="next"',
+            }
+        )
+
+        with patch.object(
+            AsyncHTTPClient,
+            'fetch',
+            side_effect=[
+                make_http_response(handler=local_handler.request, **lineitems_body_result),
+                make_http_response(handler=local_handler.request, body=lineitems_body_result['body']),
+            ],
+        ) as mock_fetch:
+            # initial call then the method will detect the Link header to get the next items
+            await sut._get_lineitems_from_url('https://example.canvas.com/api/lti/courses/111/line_items')
+            # assert the lineitems number
+            assert len(sut.all_lineitems) == 2
+            # assert the number of calls
+            assert mock_fetch.call_count == 2