daniil-berg/stocksymbolscraper
2
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

comments and minor refactoring

Browse Source
This commit is contained in:
Daniil Fajnberg 2021-11-15 16:06:58 +01:00
parent 661a1a98da
commit 7dbfcf568b
1 changed files with 24 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
tests/test_scrape.py
View File

@ -94,47 +94,53 @@ class ScrapeTestCase(IsolatedAsyncioTestCase):
@patch.object(scrape, 'ClientSession')
async def test_get_data_from_category(self, mock_session_cls, mock_extract_row_data,
mock_trs_from_page, mock_soup_from_url):
# We do not pass a session object into the tested function,
# so we mock the ClientSession class for the first two tests.
mock_session = MagicMock()
mock_session_cls.return_value = mock_session
mock_soup = MagicMock()
mock_soup_from_url.return_value = mock_soup
category = 'ßßß'
url = f'{scrape.BASE_URL}{category}'
# First test with no TRs returned, thus not entering the loop.
mock_trs = []
mock_trs_from_page.return_value = mock_trs
mock_extract_row_data.return_value = expected_output = []
output = await scrape.get_data_from_category(category, last_page=1)
output = await scrape.get_data_from_category(category)
self.assertListEqual(expected_output, output)
mock_soup_from_url.assert_called_once_with(f'{scrape.BASE_URL}{category}', mock_session)
mock_soup_from_url.assert_called_once_with(url, mock_session)
mock_trs_from_page.assert_called_once_with(mock_soup)
mock_extract_row_data.assert_not_called()
mock_soup_from_url.reset_mock()
mock_trs_from_page.reset_mock()
# Second test with (fake) TRs returned, thus entering the loop.
# We pass a last_page argument to stop after the first iteration.
mock_trs = ['foo', 'bar']
mock_trs_from_page.return_value = mock_trs
mock_extract_row_data.return_value = expected_output = ['a', 'b']
# Factored out checks because the only difference in the next two tests is the presence or absence of a
# real session instance.
def check_assertions(test_output, session_obj):
self.assertListEqual(expected_output, test_output)
mock_soup_from_url.assert_has_calls([call(url, session_obj), call(f'{url}/{2}', session_obj)])
mock_trs_from_page.assert_has_calls([call(mock_soup), call(mock_soup)])
mock_extract_row_data.assert_called_once_with(*mock_trs)
output = await scrape.get_data_from_category(category, last_page=1)
self.assertListEqual(expected_output, output)
mock_soup_from_url.assert_has_calls([
call(f'{scrape.BASE_URL}{category}', mock_session),
call(f'{scrape.BASE_URL}{category}/{2}', mock_session)
])
mock_trs_from_page.assert_has_calls([call(mock_soup), call(mock_soup)])
mock_extract_row_data.assert_called_once_with(*mock_trs)
check_assertions(output, mock_session)
mock_soup_from_url.reset_mock()
mock_trs_from_page.reset_mock()
mock_extract_row_data.reset_mock()
# Third test with (fake) TRs returned and explicitly passing a real session object.
async with scrape.ClientSession() as session:
output = await scrape.get_data_from_category(category, session, last_page=1)
self.assertListEqual(expected_output, output)
mock_soup_from_url.assert_has_calls([
call(f'{scrape.BASE_URL}{category}', session),
call(f'{scrape.BASE_URL}{category}/{2}', session)
])
mock_trs_from_page.assert_has_calls([call(mock_soup), call(mock_soup)])
mock_extract_row_data.assert_called_once_with(*mock_trs)
check_assertions(output, session)
@patch.object(scrape, 'get_data_from_category')
@patch.object(scrape, 'ClientSession')
@ -144,11 +150,13 @@ class ScrapeTestCase(IsolatedAsyncioTestCase):
mock_result = ['foo']
expected_output = len(scrape.CATEGORIES) * mock_result
mock_get_data_from_category.return_value = mock_result
output = await scrape.get_all_data(sequential=True)
self.assertListEqual(expected_output, output)
mock_get_data_from_category.assert_has_calls([
call(category, mock_session) for category in scrape.CATEGORIES
])
output = await scrape.get_all_data(sequential=False)
self.assertListEqual(expected_output, output)
mock_get_data_from_category.assert_has_calls([