import logging from pathlib import Path from unittest import IsolatedAsyncioTestCase from unittest.mock import patch, MagicMock, AsyncMock, call from bs4 import BeautifulSoup from mwfin import functions from mwfin.constants import HTML_PARSER, BASE_URL, FIN_STMT_URL_SUFFIX, IS, BS, CF, END_DATE THIS_DIR = Path(__file__).parent class FunctionsTestCase(IsolatedAsyncioTestCase): # boiled down & accurate structure of a relevant data table # https://www.marketwatch.com/investing/stock/aapl/financials/cash-flow # view page source @ line 2055 TEST_HTML_FILE_PATH = Path(THIS_DIR, 'test_structure.html') log_lvl: int test_html: str @staticmethod def get_mock_session(response_text: str = None) -> MagicMock: mock_response = MagicMock() mock_response.text = AsyncMock(return_value=response_text) mock_get_return = MagicMock() mock_get_return.__aenter__ = AsyncMock(return_value=mock_response) mock_session_obj = MagicMock() mock_session_obj.get = MagicMock(return_value=mock_get_return) return mock_session_obj @classmethod def setUpClass(cls) -> None: with open(cls.TEST_HTML_FILE_PATH, 'r') as f: cls.test_html = f.read() cls.test_soup = BeautifulSoup(cls.test_html, HTML_PARSER) cls.log_lvl = functions.log.level functions.log.setLevel(logging.CRITICAL) @classmethod def tearDownClass(cls) -> None: functions.log.setLevel(cls.log_lvl) @patch.object(functions, 'ClientSession') async def test_soup_from_url(self, mock_session_cls): test_html = 'foo' mock_session_cls.return_value = mock_session_obj = self.get_mock_session(test_html) expected_output = BeautifulSoup(test_html, 'html.parser') output = await functions.soup_from_url('baz', mock_session_obj) self.assertEqual(expected_output, output) def test_extract_end_dates(self): expected_output = ('End_Date_1', 'End_Date_2') output = functions.extract_end_dates(self.test_soup) self.assertTupleEqual(expected_output, output) def test_is_relevant_table_row(self): test_soup = BeautifulSoup('
Cash & Short Term Investments
', HTML_PARSER) self.assertTrue(functions.is_relevant_table_row(test_soup.tr)) test_soup = BeautifulSoup('
Cash & Short Term Investments Growth
', HTML_PARSER) self.assertFalse(functions.is_relevant_table_row(test_soup.tr)) test_soup = BeautifulSoup('
baz
', HTML_PARSER) self.assertFalse(functions.is_relevant_table_row(test_soup.tr)) @patch.object(functions, 'is_relevant_table_row') def test_find_relevant_table_rows(self, mock_is_relevant_table_row): mock_is_relevant_table_row.return_value = True expected_output = self.test_soup.find('div', attrs={'class': 'financials'}).tbody.find_all('tr') output = functions.find_relevant_table_rows(self.test_soup) self.assertListEqual(expected_output, output) mock_is_relevant_table_row.assert_has_calls([call(expected_output[0]), call(expected_output[1])]) def test_extract_row_data(self): test_row = self.test_soup.find('div', attrs={'class': 'financials'}).tbody.tr expected_output = ('Cash & Short Term Investments', (11000000, -22000000)) output = functions.extract_row_data(test_row) self.assertTupleEqual(expected_output, output) @patch.object(functions, 'extract_row_data') @patch.object(functions, 'find_relevant_table_rows') @patch.object(functions, 'extract_end_dates') def test_extract_all_data(self, mock_extract_end_dates, mock_find_relevant_table_rows, mock_extract_row_data): test_end_dates = ('foo', 'bar') mock_extract_end_dates.return_value = test_end_dates test_relevant_rows = ['tr1', 'tr2'] mock_find_relevant_table_rows.return_value = test_relevant_rows test_row_data = ('item_name', (123, 456)) mock_extract_row_data.return_value = test_row_data expected_output = { END_DATE: test_end_dates, test_row_data[0]: test_row_data[1], test_row_data[0]: test_row_data[1], } output = functions.extract_all_data(self.test_soup) self.assertDictEqual(expected_output, output) mock_extract_end_dates.assert_called_once_with(self.test_soup) mock_find_relevant_table_rows.assert_called_once_with(self.test_soup) mock_extract_row_data.assert_has_calls([call(test_relevant_rows[0]), call(test_relevant_rows[1])]) @patch.object(functions, 'extract_all_data') @patch.object(functions, 'soup_from_url') async def test__get_financial_statement(self, mock_soup_from_url, mock_extract_all_data): mock_session = MagicMock() test_ticker, statement = 'bar', BS test_url = f'{BASE_URL}/{test_ticker}/financials{FIN_STMT_URL_SUFFIX[statement]}' mock_soup_from_url.return_value = mock_soup = MagicMock() mock_extract_all_data.return_value = expected_output = {'foo': 'bar'} quarterly = False output = await functions._get_financial_statement(statement, test_ticker, quarterly, mock_session) self.assertDictEqual(expected_output, output) mock_soup_from_url.assert_called_once_with(test_url, mock_session) mock_extract_all_data.assert_called_once_with(mock_soup) mock_soup_from_url.reset_mock() mock_extract_all_data.reset_mock() quarterly = True output = await functions._get_financial_statement(statement, test_ticker, quarterly, mock_session) self.assertDictEqual(expected_output, output) mock_soup_from_url.assert_called_once_with(test_url + '/quarter', mock_session) mock_extract_all_data.assert_called_once_with(mock_soup) async def _helper_test_get_any_statement(self, statement: str, mock__get_financial_statement): symbol, quarterly, mock_session = 'foo', False, MagicMock() mock__get_financial_statement.return_value = expected_output = 'bar' if statement == BS: function = functions.get_balance_sheet elif statement == IS: function = functions.get_income_statement elif statement == CF: function = functions.get_cash_flow_statement else: raise ValueError output = await function(symbol, quarterly=quarterly, session=mock_session) self.assertEqual(expected_output, output) mock__get_financial_statement.assert_called_once_with(statement, symbol, quarterly, mock_session) mock__get_financial_statement.reset_mock() symbol1, symbol2 = 'x', 'y' expected_output = {symbol1: expected_output, symbol2: expected_output} output = await function(symbol1, symbol2, quarterly=quarterly, session=mock_session) self.assertDictEqual(expected_output, output) mock__get_financial_statement.assert_has_calls([ call(statement, symbol1, quarterly, mock_session), call(statement, symbol2, quarterly, mock_session), ]) @patch.object(functions, '_get_financial_statement') async def test_get_balance_sheet(self, mock__get_financial_statement): await self._helper_test_get_any_statement(BS, mock__get_financial_statement) @patch.object(functions, '_get_financial_statement') async def test_get_income_statement(self, mock__get_financial_statement): await self._helper_test_get_any_statement(IS, mock__get_financial_statement) @patch.object(functions, '_get_financial_statement') async def test_get_cash_flow_statement(self, mock__get_financial_statement): await self._helper_test_get_any_statement(CF, mock__get_financial_statement) @patch.object(functions, 'get_cash_flow_statement') @patch.object(functions, 'get_income_statement') @patch.object(functions, 'get_balance_sheet') async def test_get_company_financials(self, mock_get_bs, mock_get_is, mock_get_cf): mock_end_dates = ('bar', 'baz') mock_get_bs.return_value = {END_DATE: mock_end_dates, 'a': (1, 2)} mock_get_is.return_value = {END_DATE: mock_end_dates, 'b': (2, 3)} mock_get_cf.return_value = {END_DATE: mock_end_dates, 'c': (3, 4)} expected_output = { BS: {END_DATE: mock_end_dates, 'a': (1, 2)}, IS: {END_DATE: mock_end_dates, 'b': (2, 3)}, CF: {END_DATE: mock_end_dates, 'c': (3, 4)} } symbol, quarterly, mock_session = 'foo', False, MagicMock() output = await functions.get_company_financials(symbol, quarterly=quarterly, session=mock_session) self.assertDictEqual(expected_output, output) mock_get_bs.assert_called_once_with(symbol, quarterly, mock_session) mock_get_is.assert_called_once_with(symbol, quarterly, mock_session) mock_get_cf.assert_called_once_with(symbol, quarterly, mock_session) mock_get_bs.reset_mock() mock_get_is.reset_mock() mock_get_cf.reset_mock() test_symbol1, test_symbol2 = 'x', 'y' expected_output = {test_symbol1: expected_output, test_symbol2: expected_output} output = await functions.get_company_financials(test_symbol1, test_symbol2, quarterly=quarterly, session=mock_session) self.assertDictEqual(expected_output, output) mock_get_bs.assert_has_calls([ call(test_symbol1, quarterly, mock_session), call(test_symbol2, quarterly, mock_session) ]) mock_get_is.assert_has_calls([ call(test_symbol1, quarterly, mock_session), call(test_symbol2, quarterly, mock_session) ]) mock_get_cf.assert_has_calls([ call(test_symbol1, quarterly, mock_session), call(test_symbol2, quarterly, mock_session) ]) @patch.object(functions, 'ClientSession') async def test_integration_get_company_financials(self, mock_session_cls): mock_session_cls.return_value = mock_session_obj = self.get_mock_session(self.test_html) symbol = 'foo' # Since the web request is mocked we always receive the same HTML markup. expected_output = { BS: {END_DATE: ('End_Date_1', 'End_Date_2'), 'Cash & Short Term Investments': (11000000, -22000000)}, IS: {END_DATE: ('End_Date_1', 'End_Date_2'), 'Cash & Short Term Investments': (11000000, -22000000)}, CF: {END_DATE: ('End_Date_1', 'End_Date_2'), 'Cash & Short Term Investments': (11000000, -22000000)} } output = await functions.get_company_financials(symbol, session=mock_session_obj) self.assertDictEqual(expected_output, output) mock_session_obj.get.assert_has_calls([ call(f'{BASE_URL}/{symbol}/financials{FIN_STMT_URL_SUFFIX[BS]}'), call(f'{BASE_URL}/{symbol}/financials{FIN_STMT_URL_SUFFIX[IS]}'), call(f'{BASE_URL}/{symbol}/financials{FIN_STMT_URL_SUFFIX[CF]}'), ])