more tests; coverage script and settings

This commit is contained in:
Daniil Fajnberg 2021-11-13 21:56:42 +01:00
parent 83281f3625
commit d9344b1b4b
5 changed files with 53 additions and 9 deletions

10
.coveragerc Normal file
View File

@ -0,0 +1,10 @@
[run]
source = .
branch = true
omit =
.venv/*
[report]
fail_under = 100
show_missing = True
skip_covered = True

4
.gitignore vendored
View File

@ -5,4 +5,6 @@
# Distribution / packaging:
*.egg-info/
# Python cache:
__pycache__/
__pycache__/
# Tests:
.coverage

3
coverage.sh Executable file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env sh
coverage erase && coverage run -m unittest discover && coverage report

View File

@ -19,7 +19,7 @@ keywords = webscraping, html, markup, dom, scraper, attributes, tags, stocks, fi
package_dir =
= src
packages = find:
python_requires = >=3.7
python_requires = >=3.8
install_requires =
beautifulsoup4
aiohttp

View File

@ -1,12 +1,13 @@
from unittest import TestCase
from unittest.mock import patch, MagicMock, call
import logging
from unittest import IsolatedAsyncioTestCase
from unittest.mock import patch, MagicMock, AsyncMock, call
from bs4 import BeautifulSoup
from stocksymbolscraper import scrape
class ScrapeTestCase(TestCase):
class ScrapeTestCase(IsolatedAsyncioTestCase):
@patch.object(scrape, 'get_single_tr_data')
def test_extract_row_data(self, mock_get_single_tr_data: MagicMock):
@ -53,8 +54,36 @@ class ScrapeTestCase(TestCase):
output = scrape.get_str_from_td(test_td)
self.assertEqual(expected_output, output)
@patch.object(scrape, 'ClientSession')
async def test_soup_from_url(self, mock_session_cls):
test_html = '<b>foo</b>'
mock_response = MagicMock()
mock_response.text = AsyncMock(return_value=test_html)
mock_get_return = MagicMock()
mock_get_return.__aenter__ = AsyncMock(return_value=mock_response)
mock_session_obj = MagicMock()
mock_session_obj.get = MagicMock(return_value=mock_get_return)
mock_session_cls.return_value = mock_session_obj
output = await scrape.soup_from_url('foo')
expected_output = BeautifulSoup(test_html, scrape.HTML_PARSER)
self.assertEqual(expected_output, output)
output = await scrape.soup_from_url('foo', mock_session_obj)
self.assertEqual(expected_output, output)
def test_trs_from_page(self):
# Tested function takes URL as argument (GET request is issued)
# HTML to be parsed could be substituted
#
pass
tr1_text, tr2_text = '<tr>foo</tr>', '<tr>bar</tr>'
test_html = f'<div id="marketsindex"><table><tbody>{tr1_text}{tr2_text}</tbody></table></div>'
test_soup = BeautifulSoup(test_html, scrape.HTML_PARSER)
output = scrape.trs_from_page(test_soup)
expected_output = test_soup.find_all('tr')
self.assertSequenceEqual(expected_output, output)
logging.disable(logging.CRITICAL)
test_html = f'<div id="marketsindex"><table>garbage</table></div>'
test_soup = BeautifulSoup(test_html, scrape.HTML_PARSER)
with patch.object(scrape, 'open') as mock_open:
self.assertRaises(scrape.UnexpectedMarkupError, scrape.trs_from_page, test_soup)
mock_open.assert_called_once()
mock_open.return_value.__enter__.return_value.write.assert_called_once_with(test_soup.prettify())
logging.disable(logging.NOTSET)