comments and minor refactoring

tests/test_scrape.py

@@ -87,3 +87,78 @@ class ScrapeTestCase(IsolatedAsyncioTestCase):
             mock_open.assert_called_once()
             mock_open.return_value.__enter__.return_value.write.assert_called_once_with(test_soup.prettify())
         logging.disable(logging.NOTSET)
+
+    @patch.object(scrape, 'soup_from_url')
+    @patch.object(scrape, 'trs_from_page')
+    @patch.object(scrape, 'extract_row_data')
+    @patch.object(scrape, 'ClientSession')
+    async def test_get_data_from_category(self, mock_session_cls, mock_extract_row_data,
+                                          mock_trs_from_page, mock_soup_from_url):
+        # We do not pass a session object into the tested function,
+        # so we mock the ClientSession class for the first two tests.
+        mock_session = MagicMock()
+        mock_session_cls.return_value = mock_session
+        mock_soup = MagicMock()
+        mock_soup_from_url.return_value = mock_soup
+        category = 'ßßß'
+        url = f'{scrape.BASE_URL}{category}'
+
+        # First test with no TRs returned, thus not entering the loop.
+        mock_trs = []
+        mock_trs_from_page.return_value = mock_trs
+        mock_extract_row_data.return_value = expected_output = []
+        output = await scrape.get_data_from_category(category)
+        self.assertListEqual(expected_output, output)
+        mock_soup_from_url.assert_called_once_with(url, mock_session)
+        mock_trs_from_page.assert_called_once_with(mock_soup)
+        mock_extract_row_data.assert_not_called()
+
+        mock_soup_from_url.reset_mock()
+        mock_trs_from_page.reset_mock()
+
+        # Second test with (fake) TRs returned, thus entering the loop.
+        # We pass a last_page argument to stop after the first iteration.
+        mock_trs = ['foo', 'bar']
+        mock_trs_from_page.return_value = mock_trs
+        mock_extract_row_data.return_value = expected_output = ['a', 'b']
+
+        # Factored out checks because the only difference in the next two tests is the presence or absence of a
+        # real session instance.
+        def check_assertions(test_output, session_obj):
+            self.assertListEqual(expected_output, test_output)
+            mock_soup_from_url.assert_has_calls([call(url, session_obj), call(f'{url}/{2}', session_obj)])
+            mock_trs_from_page.assert_has_calls([call(mock_soup), call(mock_soup)])
+            mock_extract_row_data.assert_called_once_with(*mock_trs)
+
+        output = await scrape.get_data_from_category(category, last_page=1)
+        check_assertions(output, mock_session)
+
+        mock_soup_from_url.reset_mock()
+        mock_trs_from_page.reset_mock()
+        mock_extract_row_data.reset_mock()
+
+        # Third test with (fake) TRs returned and explicitly passing a real session object.
+        async with scrape.ClientSession() as session:
+            output = await scrape.get_data_from_category(category, session, last_page=1)
+            check_assertions(output, session)
+
+    @patch.object(scrape, 'get_data_from_category')
+    @patch.object(scrape, 'ClientSession')
+    async def test_get_all_data(self, mock_session_cls, mock_get_data_from_category):
+        mock_session = MagicMock()
+        mock_session_cls.return_value.__aenter__.return_value = mock_session
+        mock_result = ['foo']
+        expected_output = len(scrape.CATEGORIES) * mock_result
+        mock_get_data_from_category.return_value = mock_result
+
+        output = await scrape.get_all_data(sequential=True)
+        self.assertListEqual(expected_output, output)
+        mock_get_data_from_category.assert_has_calls([
+            call(category, mock_session) for category in scrape.CATEGORIES
+        ])
+
+        output = await scrape.get_all_data(sequential=False)
+        self.assertListEqual(expected_output, output)
+        mock_get_data_from_category.assert_has_calls([
+            call(category, mock_session) for category in scrape.CATEGORIES
+        ])