more tests; coverage script and settings
This commit is contained in:
parent
83281f3625
commit
d9344b1b4b
10
.coveragerc
Normal file
10
.coveragerc
Normal file
@ -0,0 +1,10 @@
|
||||
[run]
|
||||
source = .
|
||||
branch = true
|
||||
omit =
|
||||
.venv/*
|
||||
|
||||
[report]
|
||||
fail_under = 100
|
||||
show_missing = True
|
||||
skip_covered = True
|
4
.gitignore
vendored
4
.gitignore
vendored
@ -5,4 +5,6 @@
|
||||
# Distribution / packaging:
|
||||
*.egg-info/
|
||||
# Python cache:
|
||||
__pycache__/
|
||||
__pycache__/
|
||||
# Tests:
|
||||
.coverage
|
3
coverage.sh
Executable file
3
coverage.sh
Executable file
@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
coverage erase && coverage run -m unittest discover && coverage report
|
@ -19,7 +19,7 @@ keywords = webscraping, html, markup, dom, scraper, attributes, tags, stocks, fi
|
||||
package_dir =
|
||||
= src
|
||||
packages = find:
|
||||
python_requires = >=3.7
|
||||
python_requires = >=3.8
|
||||
install_requires =
|
||||
beautifulsoup4
|
||||
aiohttp
|
||||
|
@ -1,12 +1,13 @@
|
||||
from unittest import TestCase
|
||||
from unittest.mock import patch, MagicMock, call
|
||||
import logging
|
||||
from unittest import IsolatedAsyncioTestCase
|
||||
from unittest.mock import patch, MagicMock, AsyncMock, call
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from stocksymbolscraper import scrape
|
||||
|
||||
|
||||
class ScrapeTestCase(TestCase):
|
||||
class ScrapeTestCase(IsolatedAsyncioTestCase):
|
||||
|
||||
@patch.object(scrape, 'get_single_tr_data')
|
||||
def test_extract_row_data(self, mock_get_single_tr_data: MagicMock):
|
||||
@ -53,8 +54,36 @@ class ScrapeTestCase(TestCase):
|
||||
output = scrape.get_str_from_td(test_td)
|
||||
self.assertEqual(expected_output, output)
|
||||
|
||||
@patch.object(scrape, 'ClientSession')
|
||||
async def test_soup_from_url(self, mock_session_cls):
|
||||
test_html = '<b>foo</b>'
|
||||
mock_response = MagicMock()
|
||||
mock_response.text = AsyncMock(return_value=test_html)
|
||||
mock_get_return = MagicMock()
|
||||
mock_get_return.__aenter__ = AsyncMock(return_value=mock_response)
|
||||
mock_session_obj = MagicMock()
|
||||
mock_session_obj.get = MagicMock(return_value=mock_get_return)
|
||||
mock_session_cls.return_value = mock_session_obj
|
||||
output = await scrape.soup_from_url('foo')
|
||||
expected_output = BeautifulSoup(test_html, scrape.HTML_PARSER)
|
||||
self.assertEqual(expected_output, output)
|
||||
|
||||
output = await scrape.soup_from_url('foo', mock_session_obj)
|
||||
self.assertEqual(expected_output, output)
|
||||
|
||||
def test_trs_from_page(self):
|
||||
# Tested function takes URL as argument (GET request is issued)
|
||||
# HTML to be parsed could be substituted
|
||||
#
|
||||
pass
|
||||
tr1_text, tr2_text = '<tr>foo</tr>', '<tr>bar</tr>'
|
||||
test_html = f'<div id="marketsindex"><table><tbody>{tr1_text}{tr2_text}</tbody></table></div>'
|
||||
test_soup = BeautifulSoup(test_html, scrape.HTML_PARSER)
|
||||
output = scrape.trs_from_page(test_soup)
|
||||
expected_output = test_soup.find_all('tr')
|
||||
self.assertSequenceEqual(expected_output, output)
|
||||
|
||||
logging.disable(logging.CRITICAL)
|
||||
test_html = f'<div id="marketsindex"><table>garbage</table></div>'
|
||||
test_soup = BeautifulSoup(test_html, scrape.HTML_PARSER)
|
||||
with patch.object(scrape, 'open') as mock_open:
|
||||
self.assertRaises(scrape.UnexpectedMarkupError, scrape.trs_from_page, test_soup)
|
||||
mock_open.assert_called_once()
|
||||
mock_open.return_value.__enter__.return_value.write.assert_called_once_with(test_soup.prettify())
|
||||
logging.disable(logging.NOTSET)
|
||||
|
Loading…
Reference in New Issue
Block a user