diff --git a/README.md b/README.md index 6f558c19..cbc656f4 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Yahoo! finance API is intended for personal use only.** - `Ticker`: single ticker data - `Tickers`: multiple tickers' data - `download`: download market data for multiple tickers +- `Market`: get infomation about a market - `Search`: quotes and news from search - `Sector` and `Industry`: sector and industry information - `EquityQuery` and `Screener`: build query to screen market diff --git a/doc/source/reference/examples/market.py b/doc/source/reference/examples/market.py new file mode 100644 index 00000000..07cec661 --- /dev/null +++ b/doc/source/reference/examples/market.py @@ -0,0 +1,6 @@ +import yfinance as yf + +EUROPE = yf.Market("EUROPE") + +status = EUROPE.status +summary = EUROPE.summary diff --git a/doc/source/reference/examples/search.py b/doc/source/reference/examples/search.py index 20ca1fd1..022d86ad 100644 --- a/doc/source/reference/examples/search.py +++ b/doc/source/reference/examples/search.py @@ -4,4 +4,7 @@ quotes = yf.Search("AAPL", max_results=10).quotes # get list of news -news = yf.Search("Google", news_count=10).news \ No newline at end of file +news = yf.Search("Google", news_count=10).news + +# get list of related research +research = yf.Search("apple", include_research=True).research \ No newline at end of file diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index bbfa5455..51237a79 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -15,11 +15,13 @@ The following are the publicly available classes, and functions exposed by the ` - :attr:`Ticker `: Class for accessing single ticker data. - :attr:`Tickers `: Class for handling multiple tickers. +- :attr:`MarketSummary `: Class for accessing market summary. - :attr:`Search `: Class for accessing search results. - :attr:`Sector `: Domain class for accessing sector information. - :attr:`Industry `: Domain class for accessing industry information. - :attr:`download `: Function to download market data for multiple tickers. -- :attr:`EquityQuery `: Class to build equity market query. +- :attr:`EquityOperation `: Class to build equity market operation. +- :attr:`Query `: Class to build query. - :attr:`Screener `: Class to screen the market using defined query. - :attr:`enable_debug_mode `: Function to enable debug mode for logging. - :attr:`set_tz_cache_location `: Function to set the timezone cache location. @@ -33,8 +35,10 @@ The following are the publicly available classes, and functions exposed by the ` yfinance.stock yfinance.financials yfinance.analysis + yfinance.marketsummary yfinance.search yfinance.sector_industry + yfinance.screener yfinance.functions yfinance.funds_data diff --git a/doc/source/reference/yfinance.functions.rst b/doc/source/reference/yfinance.functions.rst index ef83454b..5206e441 100644 --- a/doc/source/reference/yfinance.functions.rst +++ b/doc/source/reference/yfinance.functions.rst @@ -13,25 +13,6 @@ The `download` function allows you to retrieve market data for multiple tickers download -Query Market Data -~~~~~~~~~~~~~~~~~~~~~ -The `Sector` and `Industry` modules allow you to access the sector and industry information. - -.. autosummary:: - :toctree: api/ - - EquityQuery - Screener - -.. seealso:: - :attr:`EquityQuery.valid_operand_fields ` - supported operand values for query - :attr:`EquityQuery.valid_eq_operand_map ` - supported `EQ query operand parameters` - :attr:`Screener.predefined_bodies ` - supported predefined screens - - Enable Debug Mode ~~~~~~~~~~~~~~~~~ Enables logging of debug information for the `yfinance` package. diff --git a/doc/source/reference/yfinance.market.rst b/doc/source/reference/yfinance.market.rst new file mode 100644 index 00000000..a11a8e82 --- /dev/null +++ b/doc/source/reference/yfinance.market.rst @@ -0,0 +1,16 @@ +===================== +Market Summary +===================== +.. currentmodule:: yfinance +Class +------------ +The `Market` class, allows you to access market data in a Pythonic way. +.. autosummary:: + :toctree: api/ + Market + +Market Sample Code +-------------------------- +The `Market` class, allows you to access market summary data in a Pythonic way. +.. literalinclude:: examples/market.py + :language: python \ No newline at end of file diff --git a/doc/source/reference/yfinance.screener.rst b/doc/source/reference/yfinance.screener.rst new file mode 100644 index 00000000..74e8f2be --- /dev/null +++ b/doc/source/reference/yfinance.screener.rst @@ -0,0 +1,27 @@ +========================= +Screener & Query +========================= + +.. currentmodule:: yfinance + +Query Market Data +~~~~~~~~~~~~~~~~~~~~~ +The `Sector` and `Industry` modules allow you to access the sector and industry information. + +.. autosummary:: + :toctree: api/ + + EquityQuery + FundQuery + screen + +.. seealso:: + :attr:`EquityQuery.valid_fields ` + supported operand values for query + :attr:`EquityQuery.valid_values ` + supported `EQ query operand parameters` + :attr:`FundQuery.valid_fields ` + supported operand values for query + :attr:`FundQuery.valid_values ` + supported `EQ query operand parameters` + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 55fea8d8..e90d14c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,7 @@ pytz>=2022.5 frozendict>=2.3.4 beautifulsoup4>=4.11.1 html5lib>=1.1 -peewee>=3.16.2 \ No newline at end of file +peewee>=3.16.2 +requests_cache>=1.0 +requests_ratelimiter>=0.3.1 +scipy>=1.6.3 \ No newline at end of file diff --git a/tests/test_screener.py b/tests/test_screener.py index 0ec6c201..89412433 100644 --- a/tests/test_screener.py +++ b/tests/test_screener.py @@ -1,153 +1,38 @@ import unittest from unittest.mock import patch, MagicMock -from yfinance.const import PREDEFINED_SCREENER_BODY_MAP -from yfinance.screener.screener import Screener -from yfinance.screener.screener_query import EquityQuery +from yfinance.screener.screener import screen +from yfinance.screener.query import EquityQuery class TestScreener(unittest.TestCase): @classmethod def setUpClass(self): - self.screener = Screener() self.query = EquityQuery('gt',['eodprice',3]) - - def test_set_default_body(self): - result = self.screener.set_default_body(self.query) - - self.assertEqual(self.screener.body['offset'], 0) - self.assertEqual(self.screener.body['size'], 100) - self.assertEqual(self.screener.body['sortField'], 'ticker') - self.assertEqual(self.screener.body['sortType'], 'desc') - self.assertEqual(self.screener.body['quoteType'], 'equity') - self.assertEqual(self.screener.body['query'], self.query.to_dict()) - self.assertEqual(self.screener.body['userId'], '') - self.assertEqual(self.screener.body['userIdType'], 'guid') - self.assertEqual(self.screener, result) - - def test_set_predefined_body(self): - k = 'most_actives' - result = self.screener.set_predefined_body(k) - self.assertEqual(self.screener.body, PREDEFINED_SCREENER_BODY_MAP[k]) - self.assertEqual(self.screener, result) - - def test_set_predefined_body_invalid_key(self): - with self.assertRaises(ValueError): - self.screener.set_predefined_body('invalid_key') - - def test_set_body(self): - body = { - "offset": 0, - "size": 100, - "sortField": "ticker", - "sortType": "desc", - "quoteType": "equity", - "query": self.query.to_dict(), - "userId": "", - "userIdType": "guid" - } - result = self.screener.set_body(body) - - self.assertEqual(self.screener.body, body) - self.assertEqual(self.screener, result) - - def test_set_body_missing_keys(self): - body = { - "offset": 0, - "size": 100, - "sortField": "ticker", - "sortType": "desc", - "quoteType": "equity" - } - with self.assertRaises(ValueError): - self.screener.set_body(body) - - def test_set_body_extra_keys(self): - body = { - "offset": 0, - "size": 100, - "sortField": "ticker", - "sortType": "desc", - "quoteType": "equity", - "query": self.query.to_dict(), - "userId": "", - "userIdType": "guid", - "extraKey": "extraValue" - } - with self.assertRaises(ValueError): - self.screener.set_body(body) - - def test_patch_body(self): - initial_body = { - "offset": 0, - "size": 100, - "sortField": "ticker", - "sortType": "desc", - "quoteType": "equity", - "query": self.query.to_dict(), - "userId": "", - "userIdType": "guid" - } - self.screener.set_body(initial_body) - patch_values = {"size": 50} - result = self.screener.patch_body(patch_values) - - self.assertEqual(self.screener.body['size'], 50) - self.assertEqual(self.screener.body['query'], self.query.to_dict()) - self.assertEqual(self.screener, result) - - def test_patch_body_extra_keys(self): - initial_body = { - "offset": 0, - "size": 100, - "sortField": "ticker", - "sortType": "desc", - "quoteType": "equity", - "query": self.query.to_dict(), - "userId": "", - "userIdType": "guid" - } - self.screener.set_body(initial_body) - patch_values = {"extraKey": "extraValue"} - with self.assertRaises(ValueError): - self.screener.patch_body(patch_values) + self.predefined = 'aggressive_small_caps' @patch('yfinance.screener.screener.YfData.post') def test_set_large_size_in_body(self, mock_post): - body = { - "offset": 0, - "size": 251, # yahoo limits at 250 - "sortField": "ticker", - "sortType": "desc", - "quoteType": "equity", - "query": self.query.to_dict(), - "userId": "", - "userIdType": "guid" - } - with self.assertRaises(ValueError): - self.screener.set_body(body).response + screen(self.query, size=251) - @patch('yfinance.screener.screener.YfData.post') - def test_fetch(self, mock_post): + @patch('yfinance.data.YfData.post') + def test_fetch_query(self, mock_post): mock_response = MagicMock() - mock_response.json.return_value = {'finance': {'result': [{}]}} + mock_response.json.return_value = {'finance': {'result': [{'key': 'value'}]}} mock_post.return_value = mock_response - self.screener.set_default_body(self.query) - response = self.screener._fetch() - - self.assertEqual(response, {'finance': {'result': [{}]}}) + response = screen(self.query) + self.assertEqual(response, {'key': 'value'}) - @patch('yfinance.screener.screener.YfData.post') - def test_fetch_and_parse(self, mock_post): + @patch('yfinance.data.YfData.get') + def test_fetch_predefined(self, mock_get): mock_response = MagicMock() mock_response.json.return_value = {'finance': {'result': [{'key': 'value'}]}} - mock_post.return_value = mock_response + mock_get.return_value = mock_response - self.screener.set_default_body(self.query) - self.screener._fetch_and_parse() - self.assertEqual(self.screener.response, {'key': 'value'}) + response = screen(self.predefined) + self.assertEqual(response, {'key': 'value'}) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/test_search.py b/tests/test_search.py index 9e074e5c..b8007bde 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -4,18 +4,14 @@ class TestSearch(unittest.TestCase): - def test_valid_query(self): - search = yf.Search(query="AAPL", max_results=5, news_count=3) - - self.assertEqual(len(search.quotes), 5) - self.assertEqual(len(search.news), 3) - self.assertIn("AAPL", search.quotes[0]['symbol']) - def test_invalid_query(self): search = yf.Search(query="XYZXYZ") self.assertEqual(len(search.quotes), 0) self.assertEqual(len(search.news), 0) + self.assertEqual(len(search.lists), 0) + self.assertEqual(len(search.nav), 0) + self.assertEqual(len(search.research), 0) def test_empty_query(self): search = yf.Search(query="") @@ -29,3 +25,18 @@ def test_fuzzy_query(self): # Check if the fuzzy search retrieves relevant results despite the typo self.assertGreater(len(search.quotes), 0) self.assertIn("AAPL", search.quotes[0]['symbol']) + + def test_quotes(self): + search = yf.Search(query="AAPL", max_results=5) + + self.assertEqual(len(search.quotes), 5) + self.assertIn("AAPL", search.quotes[0]['symbol']) + + def test_news(self): + search = yf.Search(query="AAPL", news_count=3) + + self.assertEqual(len(search.news), 3) + + def test_research_reports(self): + search = yf.Search(query="AAPL", include_research=True) + self.assertEqual(len(search.research), 3) diff --git a/tests/test_ticker.py b/tests/test_ticker.py index ea95f89d..c3af7c66 100644 --- a/tests/test_ticker.py +++ b/tests/test_ticker.py @@ -8,6 +8,8 @@ python -m unittest tests.ticker.TestTicker """ +from datetime import datetime, timedelta + import pandas as pd from tests.context import yfinance as yf @@ -133,8 +135,55 @@ def test_invalid_period(self): with self.assertRaises(YFInvalidPeriodError): dat.history(period="2wks", interval="1d", raise_errors=True) with self.assertRaises(YFInvalidPeriodError): - dat.history(period="2mo", interval="1d", raise_errors=True) + dat.history(period="2mos", interval="1d", raise_errors=True) + + def test_valid_custom_periods(self): + valid_periods = [ + # Yahoo provided periods + ("1d", "1m"), ("5d", "15m"), ("1mo", "1d"), ("3mo", "1wk"), + ("6mo", "1d"), ("1y", "1mo"), ("5y", "1wk"), ("max", "1mo"), + # Custom periods + ("2d", "30m"), ("10mo", "1d"), ("1y", "1d"), ("3y", "1d"), + ("2wk", "15m"), ("6mo", "5d"), ("10y", "1wk") + ] + + tkr = "AAPL" + dat = yf.Ticker(tkr, session=self.session) + + for period, interval in valid_periods: + with self.subTest(period=period, interval=interval): + df = dat.history(period=period, interval=interval, raise_errors=True) + self.assertIsInstance(df, pd.DataFrame) + self.assertFalse(df.empty, f"No data returned for period={period}, interval={interval}") + self.assertIn("Close", df.columns, f"'Close' column missing for period={period}, interval={interval}") + + # Validate date range + now = datetime.now() + if period != "max": # Difficult to assert for "max", therefore we skip + if period.endswith("d"): + days = int(period[:-1]) + expected_start = now - timedelta(days=days) + elif period.endswith("mo"): + months = int(period[:-2]) + expected_start = now - timedelta(days=30 * months) + elif period.endswith("y"): + years = int(period[:-1]) + expected_start = now - timedelta(days=365 * years) + elif period.endswith("wk"): + weeks = int(period[:-2]) + expected_start = now - timedelta(weeks=weeks) + else: + continue + + actual_start = df.index[0].to_pydatetime().replace(tzinfo=None) + expected_start = expected_start.replace(hour=0, minute=0, second=0, microsecond=0) + + # leeway added because of weekends + self.assertGreaterEqual(actual_start, expected_start - timedelta(days=7), + f"Start date {actual_start} out of range for period={period}") + self.assertLessEqual(df.index[-1].to_pydatetime().replace(tzinfo=None), now, + f"End date {df.index[-1]} out of range for period={period}") def test_prices_missing(self): # this test will need to be updated every time someone wants to run a test @@ -760,7 +809,7 @@ def test_upgrades_downgrades(self): self.assertIsInstance(data, pd.DataFrame, "data has wrong type") self.assertFalse(data.empty, "data is empty") self.assertTrue(len(data.columns) == 4, "data has wrong number of columns") - self.assertEqual(data.columns.values.tolist(), ['Firm', 'ToGrade', 'FromGrade', 'Action'], "data has wrong column names") + self.assertCountEqual(data.columns.values.tolist(), ['Firm', 'ToGrade', 'FromGrade', 'Action'], "data has wrong column names") self.assertIsInstance(data.index, pd.DatetimeIndex, "data has wrong index type") data_cached = self.ticker.upgrades_downgrades @@ -771,7 +820,7 @@ def test_analyst_price_targets(self): self.assertIsInstance(data, dict, "data has wrong type") keys = {'current', 'low', 'high', 'mean', 'median'} - self.assertEqual(data.keys(), keys, "data has wrong keys") + self.assertCountEqual(data.keys(), keys, "data has wrong keys") data_cached = self.ticker.analyst_price_targets self.assertIs(data, data_cached, "data not cached") @@ -782,10 +831,10 @@ def test_earnings_estimate(self): self.assertFalse(data.empty, "data is empty") columns = ['numberOfAnalysts', 'avg', 'low', 'high', 'yearAgoEps', 'growth'] - self.assertEqual(data.columns.values.tolist(), columns, "data has wrong column names") + self.assertCountEqual(data.columns.values.tolist(), columns, "data has wrong column names") index = ['0q', '+1q', '0y', '+1y'] - self.assertEqual(data.index.values.tolist(), index, "data has wrong row names") + self.assertCountEqual(data.index.values.tolist(), index, "data has wrong row names") data_cached = self.ticker.earnings_estimate self.assertIs(data, data_cached, "data not cached") @@ -796,10 +845,10 @@ def test_revenue_estimate(self): self.assertFalse(data.empty, "data is empty") columns = ['numberOfAnalysts', 'avg', 'low', 'high', 'yearAgoRevenue', 'growth'] - self.assertEqual(data.columns.values.tolist(), columns, "data has wrong column names") + self.assertCountEqual(data.columns.values.tolist(), columns, "data has wrong column names") index = ['0q', '+1q', '0y', '+1y'] - self.assertEqual(data.index.values.tolist(), index, "data has wrong row names") + self.assertCountEqual(data.index.values.tolist(), index, "data has wrong row names") data_cached = self.ticker.revenue_estimate self.assertIs(data, data_cached, "data not cached") @@ -810,7 +859,7 @@ def test_earnings_history(self): self.assertFalse(data.empty, "data is empty") columns = ['epsEstimate', 'epsActual', 'epsDifference', 'surprisePercent'] - self.assertEqual(data.columns.values.tolist(), columns, "data has wrong column names") + self.assertCountEqual(data.columns.values.tolist(), columns, "data has wrong column names") self.assertIsInstance(data.index, pd.DatetimeIndex, "data has wrong index type") data_cached = self.ticker.earnings_history @@ -822,10 +871,10 @@ def test_eps_trend(self): self.assertFalse(data.empty, "data is empty") columns = ['current', '7daysAgo', '30daysAgo', '60daysAgo', '90daysAgo'] - self.assertEqual(data.columns.values.tolist(), columns, "data has wrong column names") + self.assertCountEqual(data.columns.values.tolist(), columns, "data has wrong column names") index = ['0q', '+1q', '0y', '+1y'] - self.assertEqual(data.index.values.tolist(), index, "data has wrong row names") + self.assertCountEqual(data.index.values.tolist(), index, "data has wrong row names") data_cached = self.ticker.eps_trend self.assertIs(data, data_cached, "data not cached") @@ -835,11 +884,11 @@ def test_growth_estimates(self): self.assertIsInstance(data, pd.DataFrame, "data has wrong type") self.assertFalse(data.empty, "data is empty") - columns = ['stock', 'industry', 'sector', 'index'] - self.assertEqual(data.columns.values.tolist(), columns, "data has wrong column names") + columns = ['stockTrend', 'indexTrend'] + self.assertCountEqual(data.columns.values.tolist(), columns, "data has wrong column names") - index = ['0q', '+1q', '0y', '+1y'] - self.assertEqual(data.index.values.tolist(), index, "data has wrong row names") + index = ['0q', '+1q', '0y', '+1y', '+5y'] + self.assertCountEqual(data.index.values.tolist(), index, "data has wrong row names") data_cached = self.ticker.growth_estimates self.assertIs(data, data_cached, "data not cached") diff --git a/tests/test_utils.py b/tests/test_utils.py index 655daf10..bcaac8de 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -21,6 +21,8 @@ import tempfile import os +from yfinance.utils import is_valid_period_format + class TestCache(unittest.TestCase): @classmethod @@ -105,11 +107,32 @@ def test_mixed_timezones_to_datetime(self): i += 1 +class TestUtils(unittest.TestCase): + def test_is_valid_period_format_valid(self): + self.assertTrue(is_valid_period_format("1d")) + self.assertTrue(is_valid_period_format("5wk")) + self.assertTrue(is_valid_period_format("12mo")) + self.assertTrue(is_valid_period_format("2y")) + + def test_is_valid_period_format_invalid(self): + self.assertFalse(is_valid_period_format("1m")) # Incorrect suffix + self.assertFalse(is_valid_period_format("2wks")) # Incorrect suffix + self.assertFalse(is_valid_period_format("10")) # Missing suffix + self.assertFalse(is_valid_period_format("abc")) # Invalid string + self.assertFalse(is_valid_period_format("")) # Empty string + + def test_is_valid_period_format_edge_cases(self): + self.assertFalse(is_valid_period_format(None)) # None input + self.assertFalse(is_valid_period_format("0d")) # Zero is invalid + self.assertTrue(is_valid_period_format("999mo")) # Large number valid + + def suite(): ts: TestSuite = unittest.TestSuite() ts.addTest(TestCache('Test cache')) ts.addTest(TestCacheNoPermission('Test cache no permission')) ts.addTest(TestPandas("Test pandas")) + ts.addTest(TestUtils("Test utils")) return ts diff --git a/yfinance/__init__.py b/yfinance/__init__.py index bb79ca98..2f78eb46 100644 --- a/yfinance/__init__.py +++ b/yfinance/__init__.py @@ -28,8 +28,10 @@ from .cache import set_tz_cache_location from .domain.sector import Sector from .domain.industry import Industry -from .screener.screener import Screener -from .screener.screener_query import EquityQuery +from .domain.market import Market + +from .screener.query import EquityQuery, FundQuery +from .screener.screener import screen, PREDEFINED_SCREENER_QUERIES __version__ = version.version __author__ = "Ran Aroussi" @@ -37,5 +39,6 @@ import warnings warnings.filterwarnings('default', category=DeprecationWarning, module='^yfinance') -__all__ = ['download', 'Search', 'Ticker', 'Tickers', 'enable_debug_mode', 'set_tz_cache_location', 'Sector', - 'Industry', 'EquityQuery', 'Screener'] +__all__ = ['download', 'Market', 'Search', 'Ticker', 'Tickers', 'enable_debug_mode', 'set_tz_cache_location', 'Sector', 'Industry'] +# screener stuff: +__all__ += ['EquityQuery', 'FundQuery', 'screen', 'PREDEFINED_SCREENER_QUERIES'] \ No newline at end of file diff --git a/yfinance/base.py b/yfinance/base.py index b2eda5e3..dc773ff9 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -33,7 +33,7 @@ from . import utils, cache from .data import YfData -from .exceptions import YFEarningsDateMissing +from .exceptions import YFEarningsDateMissing, YFRateLimitError from .scrapers.analysis import Analysis from .scrapers.fundamentals import Fundamentals from .scrapers.holders import Holders @@ -125,6 +125,9 @@ def _fetch_ticker_tz(self, proxy, timeout): try: data = self._data.cache_get(url=url, params=params, proxy=proxy, timeout=timeout) data = data.json() + except YFRateLimitError: + # Must propagate this + raise except Exception as e: logger.error(f"Failed to get ticker '{self.ticker}' reason: {e}") return None diff --git a/yfinance/const.py b/yfinance/const.py index 2735f254..b4a1fd3c 100644 --- a/yfinance/const.py +++ b/yfinance/const.py @@ -305,21 +305,92 @@ 'utilities-independent-power-producers', 'utilities-regulated-water'} } + +def merge_two_level_dicts(dict1, dict2): + result = dict1.copy() + for key, value in dict2.items(): + if key in result: + # If both are sets, merge them + if isinstance(value, set) and isinstance(result[key], set): + result[key] = result[key] | value + # If both are dicts, merge their contents + elif isinstance(value, dict) and isinstance(result[key], dict): + result[key] = { + k: (result[key].get(k, set()) | v if isinstance(v, set) + else v) if k in result[key] + else v + for k, v in value.items() + } + else: + result[key] = value + return result + EQUITY_SCREENER_EQ_MAP = { - "region": { - "za", "ve", "vn", "us", "tw", "th", "tr", "sr", "sg", "sa", "se", "ru", "ro", "qa", "pt", "pk", "pl", - "ph", "nz", "nl", "mx", "pe", "no", "my", "lv", "lt", "kw", "jp", "is", "il", "lk", "kr", "it", "in", - "ie", "hu", "id", "hk", "gb", "fi", "eg", "dk", "gr", "fr", "es", "ee", "de", "cz", "cl", "ca", "be", - "at", "cn", "br", "au", "ar", "ch" + "exchange": { + 'ar': {'BUE'}, + 'at': {'VIE'}, + 'au': {'ASX'}, + 'be': {'BRU'}, + 'br': {'SAO'}, + 'ca': {'CNQ', 'NEO', 'TOR', 'VAN'}, + 'ch': {'EBS'}, + 'cl': {'SGO'}, + 'cn': {'SHH', 'SHZ'}, + 'co': {'BVC'}, + 'cz': {'PRA'}, + 'de': {'BER', 'DUS', 'FRA', 'HAM', 'GER', 'MUN', 'STU'}, + 'dk': {'CPH'}, + 'ee': {'TAL'}, + 'eg': {'CAI'}, + 'es': {'MCE'}, + 'fi': {'HEL'}, + 'fr': {'PAR'}, + 'gb': {'AQS', 'IOB', 'LSE'}, + 'gr': {'ATH'}, + 'hk': {'HKG'}, + 'hu': {'BUD'}, + 'id': {'JKT'}, + 'ie': {'ISE'}, + 'il': {'TLV'}, + 'in': {'BSE', 'NSI'}, + 'is': {'ICE'}, + 'it': {'MIL'}, + 'jp': {'FKA', 'JPX', 'SAP'}, + 'kr': {'KOE', 'KSC'}, + 'kw': {'KUW'}, + 'lk': {}, + 'lt': {'LIT'}, + 'lv': {'RIS'}, + 'mx': {'MEX'}, + 'my': {'KLS'}, + 'nl': {'AMS'}, + 'no': {'OSL'}, + 'nz': {'NZE'}, + 'pe': {}, + 'ph': {'PHP', 'PHS'}, + 'pk': {}, + 'pl': {'WSE'}, + 'pt': {'LIS'}, + 'qa': {'DOH'}, + 'ro': {'BVB'}, + 'ru': {}, + 'sa': {'SAU'}, + 'se': {'STO'}, + 'sg': {'SES'}, + 'sr': {}, + 'th': {'SET'}, + 'tr': {'IST'}, + 'tw': {'TAI', 'TWO'}, + 'us': {'ASE', 'BTS', 'CXI', 'NCM', 'NGM', 'NMS', 'NYQ', 'OEM', 'OQB', 'OQX', 'PCX', 'PNK', 'YHD'}, + 've': {'CCS'}, + 'vn': {}, + 'za': {'JNB'} }, "sector": { "Basic Materials", "Industrials", "Communication Services", "Healthcare", "Real Estate", "Technology", "Energy", "Utilities", "Financial Services", "Consumer Defensive", "Consumer Cyclical" }, - "exchanges": { - "NMS", "NAS", "YHD", "NYQ", "NGM", "NCM", "BSE" - }, "peer_group": { "US Fund Equity Energy", "US CE Convertibles", @@ -426,20 +497,42 @@ "Banks" } } +EQUITY_SCREENER_EQ_MAP['region'] = EQUITY_SCREENER_EQ_MAP['exchange'].keys() +ordered_keys = ['region'] + [k for k in EQUITY_SCREENER_EQ_MAP.keys() if k != 'region'] +EQUITY_SCREENER_EQ_MAP = {k:EQUITY_SCREENER_EQ_MAP[k] for k in ordered_keys} +FUND_SCREENER_EQ_MAP = { + "exchange": { + 'us': {'NAS'} + } +} +COMMON_SCREENER_FIELDS = { + "price":{ + "eodprice", + "intradaypricechange", + "intradayprice" + }, + "eq_fields": { + "exchange"}, +} +FUND_SCREENER_FIELDS = { + "eq_fields": { + "categoryname", + "performanceratingoverall", + "initialinvestment", + "annualreturnnavy1categoryrank", + "riskratingoverall"} +} +FUND_SCREENER_FIELDS = merge_two_level_dicts(FUND_SCREENER_FIELDS, COMMON_SCREENER_FIELDS) EQUITY_SCREENER_FIELDS = { "eq_fields": { "region", "sector", - "peer_group", - "exchanges"}, + "peer_group"}, "price":{ - "eodprice", - "intradaypricechange", "lastclosemarketcap.lasttwelvemonths", "percentchange", "lastclose52weekhigh.lasttwelvemonths", "fiftytwowkpercentchange", - "intradayprice", "lastclose52weeklow.lasttwelvemonths", "intradaymarketcap"}, "trading":{ @@ -530,21 +623,4 @@ "social_score", "highest_controversy"} } - -PREDEFINED_SCREENER_BODY_MAP = { - 'aggressive_small_caps': {"offset":0,"size":25,"sortField":"eodvolume","sortType":"desc","quoteType":"equity","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NMS"]},{"operator":"eq","operands":["exchange","NYQ"]}]},{"operator":"or","operands":[{"operator":"LT","operands":["epsgrowth.lasttwelvemonths",15]}]}]},"userId":"","userIdType":"guid"}, - 'day_gainers': {"offset":0,"size":25,"sortField":"percentchange","sortType":"DESC","quoteType":"EQUITY","query":{"operator":"AND","operands":[{"operator":"gt","operands":["percentchange",3]},{"operator":"eq","operands":["region","us"]},{"operator":"or","operands":[{"operator":"BTWN","operands":["intradaymarketcap",2000000000,10000000000]},{"operator":"BTWN","operands":["intradaymarketcap",10000000000,100000000000]},{"operator":"GT","operands":["intradaymarketcap",100000000000]}]},{"operator":"gte","operands":["intradayprice",5]},{"operator":"gt","operands":["dayvolume",15000]}]},"userId":"","userIdType":"guid"}, - 'day_losers': {"offset":0,"size":25,"sortField":"percentchange","sortType":"ASC","quoteType":"EQUITY","query":{"operator":"AND","operands":[{"operator":"lt","operands":["percentchange",-2.5]},{"operator":"eq","operands":["region","us"]},{"operator":"or","operands":[{"operator":"BTWN","operands":["intradaymarketcap",2000000000,10000000000]},{"operator":"BTWN","operands":["intradaymarketcap",10000000000,100000000000]},{"operator":"GT","operands":["intradaymarketcap",100000000000]}]},{"operator":"gte","operands":["intradayprice",5]},{"operator":"gt","operands":["dayvolume",20000]}]},"userId":"","userIdType":"guid"}, - 'growth_technology_stocks': {"offset":0,"size":25,"sortField":"eodvolume","sortType":"desc","quoteType":"equity","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"BTWN","operands":["quarterlyrevenuegrowth.quarterly",50,100]},{"operator":"GT","operands":["quarterlyrevenuegrowth.quarterly",100]},{"operator":"BTWN","operands":["quarterlyrevenuegrowth.quarterly",25,50]}]},{"operator":"or","operands":[{"operator":"BTWN","operands":["epsgrowth.lasttwelvemonths",25,50]},{"operator":"BTWN","operands":["epsgrowth.lasttwelvemonths",50,100]},{"operator":"GT","operands":["epsgrowth.lasttwelvemonths",100]}]},{"operator":"eq","operands":["sector","Technology"]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NMS"]},{"operator":"eq","operands":["exchange","NYQ"]}]}]},"userId":"","userIdType":"guid"}, - 'most_actives': {"offset":0,"size":25,"sortField":"dayvolume","sortType":"DESC","quoteType":"EQUITY","query":{"operator":"AND","operands":[{"operator":"eq","operands":["region","us"]},{"operator":"or","operands":[{"operator":"BTWN","operands":["intradaymarketcap",10000000000,100000000000]},{"operator":"GT","operands":["intradaymarketcap",100000000000]},{"operator":"BTWN","operands":["intradaymarketcap",2000000000,10000000000]}]},{"operator":"gt","operands":["dayvolume",5000000]}]},"userId":"","userIdType":"guid"}, - 'most_shorted_stocks': {"size":25,"offset":0,"sortField":"short_percentage_of_shares_outstanding.value","sortType":"DESC","quoteType":"EQUITY","topOperator":"AND","query":{"operator":"AND","operands":[{"operator":"or","operands":[{"operator":"EQ","operands":["region","us"]}]},{"operator":"gt","operands":["intradayprice",1]},{"operator":"gt","operands":["avgdailyvol3m",200000]}]},"userId":"","userIdType":"guid"}, - 'small_cap_gainers': {"offset":0,"size":25,"sortField":"eodvolume","sortType":"desc","quoteType":"equity","query":{"operator":"and","operands":[{"operator":"lt","operands":["intradaymarketcap",2000000000]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NMS"]},{"operator":"eq","operands":["exchange","NYQ"]}]}]},"userId":"","userIdType":"guid"}, - 'undervalued_growth_stocks': {"offset":0,"size":25,"sortType":"DESC","sortField":"eodvolume","quoteType":"EQUITY","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"BTWN","operands":["peratio.lasttwelvemonths",0,20]}]},{"operator":"or","operands":[{"operator":"LT","operands":["pegratio_5y",1]}]},{"operator":"or","operands":[{"operator":"BTWN","operands":["epsgrowth.lasttwelvemonths",25,50]},{"operator":"BTWN","operands":["epsgrowth.lasttwelvemonths",50,100]},{"operator":"GT","operands":["epsgrowth.lasttwelvemonths",100]}]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NMS"]},{"operator":"eq","operands":["exchange","NYQ"]}]}]},"userId":"","userIdType":"guid"}, - 'undervalued_large_caps': {"offset":0,"size":25,"sortField":"eodvolume","sortType":"desc","quoteType":"equity","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"BTWN","operands":["peratio.lasttwelvemonths",0,20]}]},{"operator":"lt","operands":["pegratio_5y",1]},{"operator":"btwn","operands":["intradaymarketcap",10000000000,100000000000]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NMS"]},{"operator":"eq","operands":["exchange","NYQ"]}]}]},"userId":"","userIdType":"guid"}, - 'conservative_foreign_funds': {"offset":0,"size":25,"sortType":"DESC","sortField":"fundnetassets","quoteType":"MUTUALFUND","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"EQ","operands":["categoryname","Foreign Large Value"]},{"operator":"EQ","operands":["categoryname","Foreign Large Blend"]},{"operator":"EQ","operands":["categoryname","Foreign Large Growth"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Growth"]},{"operator":"EQ","operands":["categoryname","Foreign Large Blend"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Blend"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Value"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Blend"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Value"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Blend"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Value"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Blend"]},{"operator":"EQ","operands":["categoryname","Foreign Small/Mid Value"]}]},{"operator":"or","operands":[{"operator":"EQ","operands":["performanceratingoverall",4]},{"operator":"EQ","operands":["performanceratingoverall",5]}]},{"operator":"lt","operands":["initialinvestment",100001]},{"operator":"lt","operands":["annualreturnnavy1categoryrank",50]},{"operator":"or","operands":[{"operator":"EQ","operands":["riskratingoverall",1]},{"operator":"EQ","operands":["riskratingoverall",3]},{"operator":"EQ","operands":["riskratingoverall",2]}]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NAS"]}]}]},"userId":"","userIdType":"guid"}, - 'high_yield_bond': {"offset":0,"size":25,"sortType":"DESC","sortField":"fundnetassets","quoteType":"MUTUALFUND","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"EQ","operands":["performanceratingoverall",4]},{"operator":"EQ","operands":["performanceratingoverall",5]}]},{"operator":"lt","operands":["initialinvestment",100001]},{"operator":"lt","operands":["annualreturnnavy1categoryrank",50]},{"operator":"or","operands":[{"operator":"EQ","operands":["riskratingoverall",1]},{"operator":"EQ","operands":["riskratingoverall",3]},{"operator":"EQ","operands":["riskratingoverall",2]}]},{"operator":"or","operands":[{"operator":"EQ","operands":["categoryname","High Yield Bond"]}]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NAS"]}]}]},"userId":"","userIdType":"guid"}, - 'portfolio_anchors': {"offset":0,"size":25,"sortType":"DESC","sortField":"fundnetassets","quoteType":"MUTUALFUND","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"EQ","operands":["categoryname","Large Blend"]}]},{"operator":"or","operands":[{"operator":"EQ","operands":["performanceratingoverall",4]},{"operator":"EQ","operands":["performanceratingoverall",5]}]},{"operator":"lt","operands":["initialinvestment",100001]},{"operator":"lt","operands":["annualreturnnavy1categoryrank",50]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NAS"]}]}]},"userId":"","userIdType":"guid"}, - 'solid_large_growth_funds': {"offset":0,"size":25,"sortType":"DESC","sortField":"fundnetassets","quoteType":"MUTUALFUND","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"EQ","operands":["categoryname","Large Growth"]}]},{"operator":"or","operands":[{"operator":"EQ","operands":["performanceratingoverall",5]},{"operator":"EQ","operands":["performanceratingoverall",4]}]},{"operator":"lt","operands":["initialinvestment",100001]},{"operator":"lt","operands":["annualreturnnavy1categoryrank",50]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NAS"]}]}]},"userId":"","userIdType":"guid"}, - 'solid_midcap_growth_funds': {"offset":0,"size":25,"sortType":"DESC","sortField":"fundnetassets","quoteType":"MUTUALFUND","query":{"operator":"and","operands":[{"operator":"or","operands":[{"operator":"EQ","operands":["categoryname","Mid-Cap Growth"]}]},{"operator":"or","operands":[{"operator":"EQ","operands":["performanceratingoverall",5]},{"operator":"EQ","operands":["performanceratingoverall",4]}]},{"operator":"lt","operands":["initialinvestment",100001]},{"operator":"lt","operands":["annualreturnnavy1categoryrank",50]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NAS"]}]}]},"userId":"","userIdType":"guid"}, - 'top_mutual_funds': {"offset":0,"size":25,"sortType":"DESC","sortField":"percentchange","quoteType":"MUTUALFUND","query":{"operator":"and","operands":[{"operator":"gt","operands":["intradayprice",15]},{"operator":"or","operands":[{"operator":"EQ","operands":["performanceratingoverall",5]},{"operator":"EQ","operands":["performanceratingoverall",4]}]},{"operator":"gt","operands":["initialinvestment",1000]},{"operator":"or","operands":[{"operator":"eq","operands":["exchange","NAS"]}]}]},"userId":"","userIdType":"guid"} -} \ No newline at end of file +EQUITY_SCREENER_FIELDS = merge_two_level_dicts(EQUITY_SCREENER_FIELDS, COMMON_SCREENER_FIELDS) diff --git a/yfinance/data.py b/yfinance/data.py index 87b63b5f..48af3cef 100644 --- a/yfinance/data.py +++ b/yfinance/data.py @@ -10,6 +10,8 @@ from . import utils, cache import threading +from .exceptions import YFRateLimitError + cache_maxsize = 64 @@ -390,6 +392,10 @@ def _make_request(self, url, request_method, user_agent_headers=None, body=None, response = request_method(**request_args) utils.get_yf_logger().debug(f'response code={response.status_code}') + # Raise exception if rate limited + if response.status_code == 429: + raise YFRateLimitError() + return response @lru_cache_freezeargs diff --git a/yfinance/domain/market.py b/yfinance/domain/market.py new file mode 100644 index 00000000..e4c7d8b2 --- /dev/null +++ b/yfinance/domain/market.py @@ -0,0 +1,100 @@ +import datetime as dt + +from ..data import YfData +from ..data import utils +from ..const import _QUERY1_URL_ +import json as _json + +class Market(): + def __init__(self, market:'str', session=None, proxy=None, timeout=30): + self.market = market + self.session = session + self.proxy = proxy + self.timeout = timeout + + self._data = YfData(session=self.session) + self._logger = utils.get_yf_logger() + + self._status = None + self._summary = None + + def _fetch_json(self, url, params): + data = self._data.cache_get(url=url, params=params, proxy=self.proxy, timeout=self.timeout) + if data is None or "Will be right back" in data.text: + raise RuntimeError("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\n" + "Our engineers are working quickly to resolve " + "the issue. Thank you for your patience.") + try: + return data.json() + except _json.JSONDecodeError: + self._logger.error(f"{self.market}: Failed to retrieve market data and recieved faulty data.") + return {} + + def _parse_data(self): + # Fetch both to ensure they are at the same time + if (self._status is not None) and (self._summary is not None): + return + + self._logger.debug(f"{self.market}: Parsing market data") + + # Summary + + summary_url = f"{_QUERY1_URL_}/v6/finance/quote/marketSummary" + summary_fields = ["shortName", "regularMarketPrice", "regularMarketChange", "regularMarketChangePercent"] + summary_params = { + "fields": ",".join(summary_fields), + "formatted": False, + "lang": "en-US", + "market": self.market + } + + status_url = f"{_QUERY1_URL_}/v6/finance/markettime" + status_params = { + "formatted": True, + "key": "finance", + "lang": "en-GB", + "market": self.market + } + + self._summary = self._fetch_json(summary_url, summary_params) + self._status = self._fetch_json(status_url, status_params) + + try: + self._summary = self._summary['marketSummaryResponse']['result'] + self._summary = {x['exchange']:x for x in self._summary} + except Exception as e: + self._logger.error(f"{self.market}: Failed to parse market summary") + self._logger.debug(f"{type(e)}: {e}") + + + try: + # Unpack + self._status = self._status['finance']['marketTimes'][0]['marketTime'][0] + self._status['timezone'] = self._status['timezone'][0] + del self._status['time'] # redundant + try: + self._status.update( + open = dt.datetime.fromisoformat(self._status["open"]), + close = dt.datetime.fromisoformat(self._status["close"]), + tz = dt.timezone(self._status["timezone"]["gmtoffset"], self._status["timezone"]["short"]) + ) + except Exception as e: + self._logger.error(f"{self.market}: Failed to update market status") + self._logger.debug(f"{type(e)}: {e}") + except Exception as e: + self._logger.error(f"{self.market}: Failed to parse market status") + self._logger.debug(f"{type(e)}: {e}") + + + + + @property + def status(self): + self._parse_data() + return self._status + + + @property + def summary(self): + self._parse_data() + return self._summary diff --git a/yfinance/exceptions.py b/yfinance/exceptions.py index 46ef4fd4..f7be6ffd 100644 --- a/yfinance/exceptions.py +++ b/yfinance/exceptions.py @@ -44,4 +44,10 @@ def __init__(self, ticker, invalid_period, valid_ranges): self.ticker = ticker self.invalid_period = invalid_period self.valid_ranges = valid_ranges - super().__init__(f"{self.ticker}: Period '{invalid_period}' is invalid, must be one of {valid_ranges}") + super().__init__(f"{self.ticker}: Period '{invalid_period}' is invalid, " + f"must be of the format {valid_ranges}, etc.") + + +class YFRateLimitError(YFException): + def __init__(self): + super().__init__("Too Many Requests. Rate limited. Try after a while.") diff --git a/yfinance/scrapers/analysis.py b/yfinance/scrapers/analysis.py index 8bdc9429..9bd39b55 100644 --- a/yfinance/scrapers/analysis.py +++ b/yfinance/scrapers/analysis.py @@ -28,84 +28,72 @@ def __init__(self, data: YfData, symbol: str, proxy=None): self._eps_revisions = None self._growth_estimates = None - @property - def analyst_price_targets(self) -> dict: - if self._analyst_price_targets is not None: - return self._analyst_price_targets - - try: - data = self._fetch(['financialData']) - data = data['quoteSummary']['result'][0]['financialData'] - except (TypeError, KeyError): - self._analyst_price_targets = {} - return self._analyst_price_targets - - keys = [ - ('currentPrice', 'current'), - ('targetLowPrice', 'low'), - ('targetHighPrice', 'high'), - ('targetMeanPrice', 'mean'), - ('targetMedianPrice', 'median'), - ] + def _get_periodic_df(self, key) -> pd.DataFrame: + if self._earnings_trend is None: + self._fetch_earnings_trend() - self._analyst_price_targets = {newKey: data.get(oldKey, None) for oldKey, newKey in keys} - return self._analyst_price_targets + data = [] + for item in self._earnings_trend[:4]: + row = {'period': item['period']} + for k, v in item[key].items(): + if not isinstance(v, dict) or len(v) == 0: + continue + row[k] = v['raw'] + data.append(row) + if len(data) == 0: + return pd.DataFrame() + return pd.DataFrame(data).set_index('period') @property def earnings_estimate(self) -> pd.DataFrame: if self._earnings_estimate is not None: return self._earnings_estimate - - if self._earnings_trend is None: - self._fetch_earnings_trend() - - data_dict = { - 'numberOfAnalysts': [], - 'avg': [], - 'low': [], - 'high': [], - 'yearAgoEps': [], - 'growth': [] - } - periods = [] - - for item in self._earnings_trend[:4]: - periods.append(item['period']) - earnings_estimate = item.get('earningsEstimate', {}) - - for key in data_dict.keys(): - data_dict[key].append(earnings_estimate.get(key, {}).get('raw', None)) - - self._earnings_estimate = pd.DataFrame(data_dict, index=periods) + self._earnings_estimate = self._get_periodic_df('earningsEstimate') return self._earnings_estimate @property def revenue_estimate(self) -> pd.DataFrame: if self._revenue_estimate is not None: return self._revenue_estimate + self._revenue_estimate = self._get_periodic_df('revenueEstimate') + return self._revenue_estimate - if self._earnings_trend is None: - self._fetch_earnings_trend() + @property + def eps_trend(self) -> pd.DataFrame: + if self._eps_trend is not None: + return self._eps_trend + self._eps_trend = self._get_periodic_df('epsTrend') + return self._eps_trend + + @property + def eps_revisions(self) -> pd.DataFrame: + if self._eps_revisions is not None: + return self._eps_revisions + self._eps_revisions = self._get_periodic_df('epsRevisions') + return self._eps_revisions - data_dict = { - 'numberOfAnalysts': [], - 'avg': [], - 'low': [], - 'high': [], - 'yearAgoRevenue': [], - 'growth': [] - } - periods = [] + @property + def analyst_price_targets(self) -> dict: + if self._analyst_price_targets is not None: + return self._analyst_price_targets - for item in self._earnings_trend[:4]: - periods.append(item['period']) - revenue_estimate = item.get('revenueEstimate', {}) + try: + data = self._fetch(['financialData']) + data = data['quoteSummary']['result'][0]['financialData'] + except (TypeError, KeyError): + self._analyst_price_targets = {} + return self._analyst_price_targets - for key in data_dict.keys(): - data_dict[key].append(revenue_estimate.get(key, {}).get('raw', None)) + result = {} + for key, value in data.items(): + if key.startswith('target'): + new_key = key.replace('target', '').lower().replace('price', '').strip() + result[new_key] = value + elif key == 'currentPrice': + result['current'] = value - self._revenue_estimate = pd.DataFrame(data_dict, index=periods) - return self._revenue_estimate + self._analyst_price_targets = result + return self._analyst_price_targets @property def earnings_history(self) -> pd.DataFrame: @@ -119,77 +107,27 @@ def earnings_history(self) -> pd.DataFrame: self._earnings_history = pd.DataFrame() return self._earnings_history - data_dict = { - 'epsEstimate': [], - 'epsActual': [], - 'epsDifference': [], - 'surprisePercent': [] - } - quarters = [] - + rows = [] for item in data: - quarters.append(item.get('quarter', {}).get('fmt', None)) - - for key in data_dict.keys(): - data_dict[key].append(item.get(key, {}).get('raw', None)) - - datetime_index = pd.to_datetime(quarters, format='%Y-%m-%d') - self._earnings_history = pd.DataFrame(data_dict, index=datetime_index) + row = {'quarter': item.get('quarter', {}).get('fmt', None)} + for k, v in item.items(): + if k == 'quarter': + continue + if not isinstance(v, dict) or len(v) == 0: + continue + row[k] = v.get('raw', None) + rows.append(row) + if len(data) == 0: + return pd.DataFrame() + + df = pd.DataFrame(rows) + if 'quarter' in df.columns: + df['quarter'] = pd.to_datetime(df['quarter'], format='%Y-%m-%d') + df.set_index('quarter', inplace=True) + + self._earnings_history = df return self._earnings_history - @property - def eps_trend(self) -> pd.DataFrame: - if self._eps_trend is not None: - return self._eps_trend - - if self._earnings_trend is None: - self._fetch_earnings_trend() - - data_dict = { - 'current': [], - '7daysAgo': [], - '30daysAgo': [], - '60daysAgo': [], - '90daysAgo': [] - } - periods = [] - - for item in self._earnings_trend[:4]: - periods.append(item['period']) - eps_trend = item.get('epsTrend', {}) - - for key in data_dict.keys(): - data_dict[key].append(eps_trend.get(key, {}).get('raw', None)) - - self._eps_trend = pd.DataFrame(data_dict, index=periods) - return self._eps_trend - - @property - def eps_revisions(self) -> pd.DataFrame: - if self._eps_revisions is not None: - return self._eps_revisions - - if self._earnings_trend is None: - self._fetch_earnings_trend() - - data_dict = { - 'upLast7days': [], - 'upLast30days': [], - 'downLast7days': [], - 'downLast30days': [] - } - periods = [] - - for item in self._earnings_trend[:4]: - periods.append(item['period']) - eps_revisions = item.get('epsRevisions', {}) - - for key in data_dict.keys(): - data_dict[key].append(eps_revisions.get(key, {}).get('raw', None)) - - self._eps_revisions = pd.DataFrame(data_dict, index=periods) - return self._eps_revisions - @property def growth_estimates(self) -> pd.DataFrame: if self._growth_estimates is not None: @@ -205,48 +143,26 @@ def growth_estimates(self) -> pd.DataFrame: self._growth_estimates = pd.DataFrame() return self._growth_estimates - # LTG is not defined in yahoo finance front-end as at 2024-11-14. - # But its addition is breaking the retrieval of growth estimates. - # Also, support for 5 year seem to have dropped. - # TODO: Revisit this change and consider permanently removing these keys. - data_dict = { - '0q': [], - '+1q': [], - '0y': [], - '+1y': [], - # 'LTG': [], - # '+5y': [], - # '-5y': [] - } - - # make sure no column is empty - dummy_trend = [{'period': key, 'growth': None} for key in data_dict.keys()] - industry_trend = trends['industryTrend']['estimates'] or dummy_trend - sector_trend = trends['sectorTrend']['estimates'] or dummy_trend - index_trend = trends['indexTrend']['estimates'] or dummy_trend - + data = [] for item in self._earnings_trend: period = item['period'] - if period in data_dict: - data_dict[period].append(item.get('growth', {}).get('raw', None)) - - for item in industry_trend: - period = item['period'] - if period in data_dict: - data_dict[period].append(item.get('growth', None)) - - for item in sector_trend: - period = item['period'] - if period in data_dict: - data_dict[period].append(item.get('growth', None)) - - for item in index_trend: - period = item['period'] - if period in data_dict: - data_dict[period].append(item.get('growth', None)) - - cols = ['stock', 'industry', 'sector', 'index'] - self._growth_estimates = pd.DataFrame(data_dict, index=cols).T + row = {'period': period, 'stockTrend': item.get('growth', {}).get('raw', None)} + data.append(row) + + for trend_name, trend_info in trends.items(): + if trend_info.get('estimates'): + for estimate in trend_info['estimates']: + period = estimate['period'] + existing_row = next((row for row in data if row['period'] == period), None) + if existing_row: + existing_row[trend_name] = estimate.get('growth') + else: + row = {'period': period, trend_name: estimate.get('growth')} + data.append(row) + if len(data) == 0: + return pd.DataFrame() + + self._growth_estimates = pd.DataFrame(data).set_index('period').dropna(how='all') return self._growth_estimates # modified version from quote.py diff --git a/yfinance/scrapers/history.py b/yfinance/scrapers/history.py index e4bdca2b..7692208e 100644 --- a/yfinance/scrapers/history.py +++ b/yfinance/scrapers/history.py @@ -9,7 +9,7 @@ from yfinance import shared, utils from yfinance.const import _BASE_URL_, _PRICE_COLNAMES_ -from yfinance.exceptions import YFInvalidPeriodError, YFPricesMissingError, YFTzMissingError +from yfinance.exceptions import YFInvalidPeriodError, YFPricesMissingError, YFTzMissingError, YFRateLimitError class PriceHistory: def __init__(self, data, ticker, tz, session=None, proxy=None): @@ -78,7 +78,7 @@ def history(self, period="1mo", interval="1d", interval_user = interval period_user = period - if repair and interval in ['5d', '1wk', '1mo', '3mo']: + if repair and interval in ["5d", "1wk", "1mo", "3mo"]: # Yahoo's way of adjusting mutiday intervals is fundamentally broken. # Have to fetch 1d, adjust, then resample. if interval == '5d': @@ -184,6 +184,9 @@ def history(self, period="1mo", interval="1d", "the issue. Thank you for your patience.") data = data.json() + # Special case for rate limits + except YFRateLimitError: + raise except Exception: if raise_errors: raise @@ -229,10 +232,9 @@ def history(self, period="1mo", interval="1d", elif "chart" not in data or data["chart"]["result"] is None or not data["chart"]["result"] or not data["chart"]["result"][0]["indicators"]["quote"][0]: _exception = YFPricesMissingError(self.ticker, _price_data_debug) fail = True - elif period is not None and period not in self._history_metadata["validRanges"]: - # even if timestamp is in the data, the data doesn't encompass the period requested - # User provided a bad period. The minimum should be '1d', but sometimes Yahoo accepts '1h'. - _exception = YFInvalidPeriodError(self.ticker, period, self._history_metadata['validRanges']) + elif period and period not in self._history_metadata['validRanges'] and not utils.is_valid_period_format(period): + # User provided a bad period + _exception = YFInvalidPeriodError(self.ticker, period, ", ".join(self._history_metadata['validRanges'])) fail = True if fail: @@ -247,6 +249,13 @@ def history(self, period="1mo", interval="1d", self._reconstruct_start_interval = None return utils.empty_df() + # Process custom periods + if period and period not in self._history_metadata.get("validRanges", []): + end = int(_time.time()) + start = _datetime.date.fromtimestamp(end) + start -= utils._interval_to_timedelta(period) + start -= _datetime.timedelta(days=4) + # parse quotes quotes = utils.parse_quotes(data["chart"]["result"][0]) # Yahoo bug fix - it often appends latest price even if after end date diff --git a/yfinance/screener/__init__.py b/yfinance/screener/__init__.py index 3254bdcc..b3e31224 100644 --- a/yfinance/screener/__init__.py +++ b/yfinance/screener/__init__.py @@ -1,4 +1,4 @@ -from .screener import Screener -from .screener_query import EquityQuery +from .query import EquityQuery +from .screener import screen, PREDEFINED_SCREENER_QUERIES -__all__ = ['EquityQuery', 'Screener'] \ No newline at end of file +__all__ = ['EquityQuery', 'FundQuery', 'screen', 'PREDEFINED_SCREENER_QUERIES'] diff --git a/yfinance/screener/query.py b/yfinance/screener/query.py new file mode 100644 index 00000000..81214a7d --- /dev/null +++ b/yfinance/screener/query.py @@ -0,0 +1,218 @@ +from abc import ABC, abstractmethod +import numbers +from typing import List, Union, Dict, TypeVar, Tuple + +from yfinance.const import EQUITY_SCREENER_EQ_MAP, EQUITY_SCREENER_FIELDS +from yfinance.const import FUND_SCREENER_EQ_MAP, FUND_SCREENER_FIELDS +from yfinance.exceptions import YFNotImplementedError +from ..utils import dynamic_docstring, generate_list_table_from_dict_universal + +T = TypeVar('T', bound=Union[str, numbers.Real]) + +class QueryBase(ABC): + def __init__(self, operator: str, operand: Union[ List['QueryBase'], Tuple[str, Tuple[Union[str, numbers.Real], ...]] ]): + operator = operator.upper() + + if not isinstance(operand, list): + raise TypeError('Invalid operand type') + if len(operand) <= 0: + raise ValueError('Invalid field for EquityQuery') + + if operator == 'IS-IN': + self._validate_isin_operand(operand) + elif operator in {'OR','AND'}: + self._validate_or_and_operand(operand) + elif operator == 'EQ': + self._validate_eq_operand(operand) + elif operator == 'BTWN': + self._validate_btwn_operand(operand) + elif operator in {'GT','LT','GTE','LTE'}: + self._validate_gt_lt(operand) + else: + raise ValueError('Invalid Operator Value') + + self.operator = operator + self.operands = operand + + @property + @abstractmethod + def valid_fields(self) -> List: + raise YFNotImplementedError('valid_fields() needs to be implemented by child') + + @property + @abstractmethod + def valid_values(self) -> Dict: + raise YFNotImplementedError('valid_values() needs to be implemented by child') + + def _validate_or_and_operand(self, operand: List['QueryBase']) -> None: + if len(operand) <= 1: + raise ValueError('Operand must be length longer than 1') + if all(isinstance(e, QueryBase) for e in operand) is False: + raise TypeError(f'Operand must be type {type(self)} for OR/AND') + + def _validate_eq_operand(self, operand: List[Union[str, numbers.Real]]) -> None: + if len(operand) != 2: + raise ValueError('Operand must be length 2 for EQ') + + if not any(operand[0] in fields_by_type for fields_by_type in self.valid_fields.values()): + raise ValueError(f'Invalid field for {type(self)} "{operand[0]}"') + if operand[0] in self.valid_values: + vv = self.valid_values[operand[0]] + if isinstance(vv, dict): + # this data structure is slightly different to generate better docs, + # need to unpack here. + vv = set().union(*[e for e in vv.values()]) + if operand[1] not in vv: + raise ValueError(f'Invalid EQ value "{operand[1]}"') + + def _validate_btwn_operand(self, operand: List[Union[str, numbers.Real]]) -> None: + if len(operand) != 3: + raise ValueError('Operand must be length 3 for BTWN') + if not any(operand[0] in fields_by_type for fields_by_type in self.valid_fields.values()): + raise ValueError(f'Invalid field for {type(self)}') + if isinstance(operand[1], numbers.Real) is False: + raise TypeError('Invalid comparison type for BTWN') + if isinstance(operand[2], numbers.Real) is False: + raise TypeError('Invalid comparison type for BTWN') + + def _validate_gt_lt(self, operand: List[Union[str, numbers.Real]]) -> None: + if len(operand) != 2: + raise ValueError('Operand must be length 2 for GT/LT') + if not any(operand[0] in fields_by_type for fields_by_type in self.valid_fields.values()): + raise ValueError(f'Invalid field for {type(self)} "{operand[0]}"') + if isinstance(operand[1], numbers.Real) is False: + raise TypeError('Invalid comparison type for GT/LT') + + def _validate_isin_operand(self, operand: List['QueryBase']) -> None: + if len(operand) < 2: + raise ValueError('Operand must be length 2+ for IS-IN') + + if not any(operand[0] in fields_by_type for fields_by_type in self.valid_fields.values()): + raise ValueError(f'Invalid field for {type(self)} "{operand[0]}"') + if operand[0] in self.valid_values: + vv = self.valid_values[operand[0]] + if isinstance(vv, dict): + # this data structure is slightly different to generate better docs, + # need to unpack here. + vv = set().union(*[e for e in vv.values()]) + for i in range(1, len(operand)): + if operand[i] not in vv: + raise ValueError(f'Invalid EQ value "{operand[i]}"') + + def to_dict(self) -> Dict: + op = self.operator + ops = self.operands + if self.operator == 'IS-IN': + # Expand to OR of EQ queries + op = 'OR' + ops = [type(self)('EQ', [self.operands[0], v]) for v in self.operands[1:]] + return { + "operator": op, + "operands": [o.to_dict() if isinstance(o, QueryBase) else o for o in ops] + } + + def __repr__(self, indent=0) -> str: + indent_str = " " * indent + class_name = self.__class__.__name__ + + if isinstance(self.operands, list): + # For list operands, check if they contain any QueryBase objects + if any(isinstance(op, QueryBase) for op in self.operands): + # If there are nested queries, format them with newlines + operands_str = ",\n".join( + f"{indent_str} {op.__repr__(indent + 1) if isinstance(op, QueryBase) else repr(op)}" + for op in self.operands + ) + return f"{class_name}({self.operator}, [\n{operands_str}\n{indent_str}])" + else: + # For lists of simple types, keep them on one line + return f"{class_name}({self.operator}, {repr(self.operands)})" + else: + # Handle single operand + return f"{class_name}({self.operator}, {repr(self.operands)})" + + def __str__(self) -> str: + return self.__repr__() + + +class EquityQuery(QueryBase): + """ + The `EquityQuery` class constructs filters for stocks based on specific criteria such as region, sector, exchange, and peer group. + + Start with value operations: `EQ` (equals), `IS-IN` (is in), `BTWN` (between), `GT` (greater than), `LT` (less than), `GTE` (greater or equal), `LTE` (less or equal). + + Combine them with logical operations: `AND`, `OR`. + + Example: + Predefined Yahoo query `aggressive_small_caps`: + + .. code-block:: python + + from yfinance import EquityQuery + + EquityQuery('and', [ + EquityQuery('is-in', ['exchange', 'NMS', 'NYQ']), + EquityQuery('lt', ["epsgrowth.lasttwelvemonths", 15]) + ]) + """ + + @dynamic_docstring({"valid_operand_fields_table": generate_list_table_from_dict_universal(EQUITY_SCREENER_FIELDS)}) + @property + def valid_fields(self) -> Dict: + """ + Valid operands, grouped by category. + {valid_operand_fields_table} + """ + return EQUITY_SCREENER_FIELDS + + @dynamic_docstring({"valid_values_table": generate_list_table_from_dict_universal(EQUITY_SCREENER_EQ_MAP, concat_keys=['exchange'])}) + @property + def valid_values(self) -> Dict: + """ + Most operands take number values, but some have a restricted set of valid values. + {valid_values_table} + """ + return EQUITY_SCREENER_EQ_MAP + + +class FundQuery(QueryBase): + """ + The `FundQuery` class constructs filters for mutual funds based on specific criteria such as region, sector, exchange, and peer group. + + Start with value operations: `EQ` (equals), `IS-IN` (is in), `BTWN` (between), `GT` (greater than), `LT` (less than), `GTE` (greater or equal), `LTE` (less or equal). + + Combine them with logical operations: `AND`, `OR`. + + Example: + Predefined Yahoo query `solid_large_growth_funds`: + + .. code-block:: python + + from yfinance import FundQuery + + FundQuery('and', [ + FundQuery('eq', ['categoryname', 'Large Growth']), + FundQuery('is-in', ['performanceratingoverall', 4, 5]), + FundQuery('lt', ['initialinvestment', 100001]), + FundQuery('lt', ['annualreturnnavy1categoryrank', 50]), + FundQuery('eq', ['exchange', 'NAS']) + ]) + """ + @dynamic_docstring({"valid_operand_fields_table": generate_list_table_from_dict_universal(FUND_SCREENER_FIELDS)}) + @property + def valid_fields(self) -> Dict: + """ + Valid operands, grouped by category. + {valid_operand_fields_table} + """ + return FUND_SCREENER_FIELDS + + @dynamic_docstring({"valid_values_table": generate_list_table_from_dict_universal(FUND_SCREENER_EQ_MAP)}) + @property + def valid_values(self) -> Dict: + """ + Most operands take number values, but some have a restricted set of valid values. + {valid_values_table} + """ + return FUND_SCREENER_EQ_MAP + diff --git a/yfinance/screener/screener.py b/yfinance/screener/screener.py index 01ff667b..60d84c31 100644 --- a/yfinance/screener/screener.py +++ b/yfinance/screener/screener.py @@ -1,225 +1,180 @@ -from typing import Dict +from .query import EquityQuery as EqyQy +from .query import FundQuery as FndQy +from .query import QueryBase, EquityQuery, FundQuery -from yfinance import utils +from yfinance.const import _BASE_URL_ from yfinance.data import YfData -from yfinance.const import _BASE_URL_, PREDEFINED_SCREENER_BODY_MAP -from .screener_query import Query -from ..utils import dynamic_docstring, generate_list_table_from_dict_of_dict -_SCREENER_URL_ = f"{_BASE_URL_}/v1/finance/screener" +from ..utils import dynamic_docstring, generate_list_table_from_dict_universal -class Screener: - """ - The `Screener` class is used to execute the queries and return the filtered results. +from typing import Union +import requests - The Screener class provides methods to set and manipulate the body of a screener request, - fetch and parse the screener results, and access predefined screener bodies. +_SCREENER_URL_ = f"{_BASE_URL_}/v1/finance/screener" +_PREDEFINED_URL_ = f"{_SCREENER_URL_}/predefined/saved" + +PREDEFINED_SCREENER_BODY_DEFAULTS = { + "offset":0, "size":25, "userId":"","userIdType":"guid" +} + +PREDEFINED_SCREENER_QUERIES = { + 'aggressive_small_caps': {"sortField":"eodvolume", "sortType":"desc", + "query": EqyQy('and', [EqyQy('is-in', ['exchange', 'NMS', 'NYQ']), EqyQy('lt', ["epsgrowth.lasttwelvemonths", 15])])}, + 'day_gainers': {"sortField":"percentchange", "sortType":"DESC", + "query": EqyQy('and', [EqyQy('gt', ['percentchange', 3]), EqyQy('eq', ['region', 'us']), EqyQy('gte', ['intradaymarketcap', 2000000000]), EqyQy('gte', ['intradayprice', 5]), EqyQy('gt', ['dayvolume', 15000])])}, + 'day_losers': {"sortField":"percentchange", "sortType":"ASC", + "query": EqyQy('and', [EqyQy('lt', ['percentchange', -2.5]), EqyQy('eq', ['region', 'us']), EqyQy('gte', ['intradaymarketcap', 2000000000]), EqyQy('gte', ['intradayprice', 5]), EqyQy('gt', ['dayvolume', 20000])])}, + 'growth_technology_stocks': {"sortField":"eodvolume", "sortType":"desc", + "query": EqyQy('and', [EqyQy('gte', ['quarterlyrevenuegrowth.quarterly', 25]), EqyQy('gte', ['epsgrowth.lasttwelvemonths', 25]), EqyQy('eq', ['sector', 'Technology']), EqyQy('is-in', ['exchange', 'NMS', 'NYQ'])])}, + 'most_actives': {"sortField":"dayvolume", "sortType":"DESC", + "query": EqyQy('and', [EqyQy('eq', ['region', 'us']), EqyQy('gte', ['intradaymarketcap', 2000000000]), EqyQy('gt', ['dayvolume', 5000000])])}, + 'most_shorted_stocks': {"size":25, "offset":0, "sortField":"short_percentage_of_shares_outstanding.value", "sortType":"DESC", + "query": EqyQy('and', [EqyQy('eq', ['region', 'us']), EqyQy('gt', ['intradayprice', 1]), EqyQy('gt', ['avgdailyvol3m', 200000])])}, + 'small_cap_gainers': {"sortField":"eodvolume", "sortType":"desc", + "query": EqyQy("and", [EqyQy("lt", ["intradaymarketcap",2000000000]), EqyQy("is-in", ["exchange", "NMS", "NYQ"])])}, + 'undervalued_growth_stocks': {"sortType":"DESC", "sortField":"eodvolume", + "query": EqyQy('and', [EqyQy('btwn', ['peratio.lasttwelvemonths', 0, 20]), EqyQy('lt', ['pegratio_5y', 1]), EqyQy('gte', ['epsgrowth.lasttwelvemonths', 25]), EqyQy('is-in', ['exchange', 'NMS', 'NYQ'])])}, + 'undervalued_large_caps': {"sortField":"eodvolume", "sortType":"desc", + "query": EqyQy('and', [EqyQy('btwn', ['peratio.lasttwelvemonths', 0, 20]), EqyQy('lt', ['pegratio_5y', 1]), EqyQy('btwn', ['intradaymarketcap', 10000000000, 100000000000]), EqyQy('is-in', ['exchange', 'NMS', 'NYQ'])])}, + 'conservative_foreign_funds': {"sortType":"DESC", "sortField":"fundnetassets", + "query": FndQy('and', [FndQy('is-in', ['categoryname', 'Foreign Large Value', 'Foreign Large Blend', 'Foreign Large Growth', 'Foreign Small/Mid Growth', 'Foreign Small/Mid Blend', 'Foreign Small/Mid Value']), FndQy('is-in', ['performanceratingoverall', 4, 5]), FndQy('lt', ['initialinvestment', 100001]), FndQy('lt', ['annualreturnnavy1categoryrank', 50]), FndQy('is-in', ['riskratingoverall', 1, 2, 3]), FndQy('eq', ['exchange', 'NAS'])])}, + 'high_yield_bond': {"sortType":"DESC", "sortField":"fundnetassets", + "query": FndQy('and', [FndQy('is-in', ['performanceratingoverall', 4, 5]), FndQy('lt', ['initialinvestment', 100001]), FndQy('lt', ['annualreturnnavy1categoryrank', 50]), FndQy('is-in', ['riskratingoverall', 1, 2, 3]), FndQy('eq', ['categoryname', 'High Yield Bond']), FndQy('eq', ['exchange', 'NAS'])])}, + 'portfolio_anchors': {"sortType":"DESC", "sortField":"fundnetassets", + "query": FndQy('and', [FndQy('eq', ['categoryname', 'Large Blend']), FndQy('is-in', ['performanceratingoverall', 4, 5]), FndQy('lt', ['initialinvestment', 100001]), FndQy('lt', ['annualreturnnavy1categoryrank', 50]), FndQy('eq', ['exchange', 'NAS'])])}, + 'solid_large_growth_funds': {"sortType":"DESC", "sortField":"fundnetassets", + "query": FndQy('and', [FndQy('eq', ['categoryname', 'Large Growth']), FndQy('is-in', ['performanceratingoverall', 4, 5]), FndQy('lt', ['initialinvestment', 100001]), FndQy('lt', ['annualreturnnavy1categoryrank', 50]), FndQy('eq', ['exchange', 'NAS'])])}, + 'solid_midcap_growth_funds': {"sortType":"DESC", "sortField":"fundnetassets", + "query": FndQy('and', [FndQy('eq', ['categoryname', 'Mid-Cap Growth']), FndQy('is-in', ['performanceratingoverall', 4, 5]), FndQy('lt', ['initialinvestment', 100001]), FndQy('lt', ['annualreturnnavy1categoryrank', 50]), FndQy('eq', ['exchange', 'NAS'])])}, + 'top_mutual_funds': {"sortType":"DESC", "sortField":"percentchange", + "query": FndQy('and', [FndQy('gt', ['intradayprice', 15]), FndQy('is-in', ['performanceratingoverall', 4, 5]), FndQy('gt', ['initialinvestment', 1000]), FndQy('eq', ['exchange', 'NAS'])])} +} + +@dynamic_docstring({"predefined_screeners": generate_list_table_from_dict_universal(PREDEFINED_SCREENER_QUERIES, bullets=True, title='Predefined queries (Dec-2024)')}) +def screen(query: Union[str, EquityQuery, FundQuery], + offset: int = None, + size: int = None, + sortField: str = None, + sortAsc: bool = None, + userId: str = None, + userIdType: str = None, + session = None, proxy = None): """ - def __init__(self, session=None, proxy=None): - """ - Args: - session (requests.Session, optional): A requests session object to be used for making HTTP requests. Defaults to None. - proxy (str, optional): A proxy URL to be used for making HTTP requests. Defaults to None. - - .. seealso:: - - :attr:`Screener.predefined_bodies ` - supported predefined screens - """ - self.proxy = proxy - self.session = session - - self._data: YfData = YfData(session=session) - self._body: Dict = {} - self._response: Dict = {} - self._body_updated = False - self._accepted_body_keys = {"offset","size","sortField","sortType","quoteType","query","userId","userIdType"} - self._predefined_bodies = PREDEFINED_SCREENER_BODY_MAP.keys() - - @property - def body(self) -> Dict: - return self._body - - @property - def response(self) -> Dict: - """ - Fetch screen result - - Example: - - .. code-block:: python - - result = screener.response - symbols = [quote['symbol'] for quote in result['quotes']] - """ - if self._body_updated or self._response is None: - self._fetch_and_parse() - - self._body_updated = False - return self._response - - @dynamic_docstring({"predefined_screeners": generate_list_table_from_dict_of_dict(PREDEFINED_SCREENER_BODY_MAP,bullets=False)}) - @property - def predefined_bodies(self) -> Dict: - """ - Predefined Screeners - {predefined_screeners} - """ - return self._predefined_bodies - - def set_default_body(self, query: Query, offset: int = 0, size: int = 100, sortField: str = "ticker", sortType: str = "desc", quoteType: str = "equity", userId: str = "", userIdType: str = "guid") -> 'Screener': - """ - Set the default body using a custom query. - - Args: - query (Query): The Query object to set as the body. - offset (Optional[int]): The offset for the results. Defaults to 0. - size (Optional[int]): The number of results to return. Defaults to 100. Maximum is 250 as set by Yahoo. - sortField (Optional[str]): The field to sort the results by. Defaults to "ticker". - sortType (Optional[str]): The type of sorting (e.g., "asc" or "desc"). Defaults to "desc". - quoteType (Optional[str]): The type of quote (e.g., "equity"). Defaults to "equity". - userId (Optional[str]): The user ID. Defaults to an empty string. - userIdType (Optional[str]): The type of user ID (e.g., "guid"). Defaults to "guid". - - Returns: - Screener: self - - Example: - - .. code-block:: python - - screener.set_default_body(qf) - """ - self._body_updated = True - - self._body = { - "offset": offset, - "size": size, - "sortField": sortField, - "sortType": sortType, - "quoteType": quoteType, - "query": query.to_dict(), - "userId": userId, - "userIdType": userIdType - } - return self - - def set_predefined_body(self, predefined_key: str) -> 'Screener': - """ - Set a predefined body - - Args: - predefined_key (str): key to one of predefined screens - - Returns: - Screener: self - - Example: - - .. code-block:: python - - screener.set_predefined_body('day_gainers') - - - .. seealso:: - - :attr:`Screener.predefined_bodies ` - supported predefined screens - """ - body = PREDEFINED_SCREENER_BODY_MAP.get(predefined_key, None) - if not body: - raise ValueError(f'Invalid key {predefined_key} provided for predefined screener') - - self._body_updated = True - self._body = body - return self - - def set_body(self, body: Dict) -> 'Screener': - """ - Set the fully custom body using dictionary input - - Args: - body (Dict): full query body - - Returns: - Screener: self - - Example: - - .. code-block:: python - - screener.set_body({ - "offset": 0, - "size": 100, - "sortField": "ticker", - "sortType": "desc", - "quoteType": "equity", - "query": qf.to_dict(), - "userId": "", - "userIdType": "guid" - }) - """ - missing_keys = [key for key in self._accepted_body_keys if key not in body] - if missing_keys: - raise ValueError(f"Missing required keys in body: {missing_keys}") - - extra_keys = [key for key in body if key not in self._accepted_body_keys] - if extra_keys: - raise ValueError(f"Body contains extra keys: {extra_keys}") - - self._body_updated = True - self._body = body - return self - - def patch_body(self, values: Dict) -> 'Screener': - """ - Patch parts of the body using dictionary input - - Args: - body (Dict): partial query body - - Returns: - Screener: self - - Example: - - .. code-block:: python - - screener.patch_body({"offset": 100}) - """ - extra_keys = [key for key in values if key not in self._accepted_body_keys] - if extra_keys: - raise ValueError(f"Body contains extra keys: {extra_keys}") - - self._body_updated = True - for k in values: - self._body[k] = values[k] - return self - - def _validate_body(self) -> None: - if not all(k in self._body for k in self._accepted_body_keys): - raise ValueError("Missing required keys in body") - - if self._body["size"] > 250: - raise ValueError("Yahoo limits query size to 250. Please decrease the size of the query.") - - def _fetch(self) -> Dict: - params_dict = {"corsDomain": "finance.yahoo.com", "formatted": "false", "lang": "en-US", "region": "US"} - response = self._data.post(_SCREENER_URL_, body=self.body, user_agent_headers=self._data.user_agent_headers, params=params_dict, proxy=self.proxy) - response.raise_for_status() - return response.json() - - def _fetch_and_parse(self) -> None: - response = None - self._validate_body() - + Run a screen: predefined query, or custom query. + + :Parameters: + * Defaults only apply if query = EquityQuery or FundQuery + query : str | Query: + The query to execute, either name of predefined or custom query. + For predefined list run yf.PREDEFINED_SCREENER_QUERIES.keys() + offset : int + The offset for the results. Default 0. + size : int + number of results to return. Default 100, maximum 250 (Yahoo) + sortField : str + field to sort by. Default "ticker" + sortAsc : bool + Sort ascending? Default False + userId : str + The user ID. Default empty. + userIdType : str + Type of user ID (e.g., "guid"). Default "guid". + + Example: predefined query + .. code-block:: python + + import yfinance as yf + response = yf.screen("aggressive_small_caps") + + Example: custom query + .. code-block:: python + + import yfinance as yf + from yfinance import EquityQuery + q = EquityQuery('and', [ + EquityQuery('gt', ['percentchange', 3]), + EquityQuery('eq', ['region', 'us']) + ]) + response = yf.screen(q, sortField = 'percentchange', sortAsc = True) + + To access predefineds query code + .. code-block:: python + + import yfinance as yf + query = yf.PREDEFINED_SCREENER_QUERIES['aggressive_small_caps'] + + {predefined_screeners} + """ + + # Only use defaults when user NOT give a predefined, because + # Yahoo's predefined endpoint auto-applies defaults. Also, + # that endpoint might be ignoring these fields. + defaults = { + 'offset': 0, + 'size': 25, + 'sortField': 'ticker', + 'sortAsc': False, + 'userId': "", + 'userIdType': "guid" + } + + if size is not None and size > 250: + raise ValueError("Yahoo limits query size to 250, reduce size.") + + fields = dict(locals()) + for k in ['query', 'session', 'proxy']: + if k in fields: + del fields[k] + + params_dict = {"corsDomain": "finance.yahoo.com", "formatted": "false", "lang": "en-US", "region": "US"} + + post_query = None + if isinstance(query, str): + # post_query = PREDEFINED_SCREENER_QUERIES[query] + # Switch to Yahoo's predefined endpoint + _data = YfData(session=session) + params_dict['scrIds'] = query + for k,v in fields.items(): + if v is not None: + params_dict[k] = v + resp = _data.get(url=_PREDEFINED_URL_, params=params_dict, proxy=proxy) try: - response = self._fetch() - self._response = response['finance']['result'][0] - except Exception as e: - logger = utils.get_yf_logger() - logger.error(f"Failed to get screener data for '{self._body.get('query', 'query not set')}' reason: {e}") - logger.debug("Got response: ") - logger.debug("-------------") - logger.debug(f" {response}") - logger.debug("-------------") + resp.raise_for_status() + except requests.exceptions.HTTPError: + if query not in PREDEFINED_SCREENER_QUERIES: + print(f"yfinance.screen: '{query}' is probably not a predefined query.") + raise + return resp.json()["finance"]["result"][0] + + elif isinstance(query, QueryBase): + # Prepare other fields + for k in defaults: + if k not in fields or fields[k] is None: + fields[k] = defaults[k] + fields['sortType'] = 'ASC' if fields['sortAsc'] else 'DESC' + del fields['sortAsc'] + + post_query = fields + post_query['query'] = query + + else: + raise ValueError(f'Query must be type str or QueryBase, not "{type(query)}"') + + if query is None: + raise ValueError('No query provided') + + if isinstance(post_query['query'], EqyQy): + post_query['quoteType'] = 'EQUITY' + elif isinstance(post_query['query'], FndQy): + post_query['quoteType'] = 'MUTUALFUND' + post_query['query'] = post_query['query'].to_dict() + + # Fetch + _data = YfData(session=session) + response = _data.post(_SCREENER_URL_, + body=post_query, + user_agent_headers=_data.user_agent_headers, + params=params_dict, + proxy=proxy) + response.raise_for_status() + return response.json()['finance']['result'][0] diff --git a/yfinance/screener/screener_query.py b/yfinance/screener/screener_query.py deleted file mode 100644 index 65c93759..00000000 --- a/yfinance/screener/screener_query.py +++ /dev/null @@ -1,145 +0,0 @@ -from abc import ABC, abstractmethod -import numbers -from typing import List, Union, Dict - -from yfinance.const import EQUITY_SCREENER_EQ_MAP, EQUITY_SCREENER_FIELDS -from yfinance.exceptions import YFNotImplementedError -from ..utils import dynamic_docstring, generate_list_table_from_dict - -class Query(ABC): - def __init__(self, operator: str, operand: Union[numbers.Real, str, List['Query']]): - self.operator = operator - self.operands = operand - - @abstractmethod - def to_dict(self) -> Dict: - raise YFNotImplementedError('to_dict() needs to be implemented by children classes') - -class EquityQuery(Query): - """ - The `EquityQuery` class constructs filters for stocks based on specific criteria such as region, sector, exchange, and peer group. - - The queries support operators: `GT` (greater than), `LT` (less than), `BTWN` (between), `EQ` (equals), and logical operators `AND` and `OR` for combining multiple conditions. - - Example: - Screen for stocks where the end-of-day price is greater than 3. - - .. code-block:: python - - gt = yf.EquityQuery('gt', ['eodprice', 3]) - - Screen for stocks where the average daily volume over the last 3 months is less than a very large number. - - .. code-block:: python - - lt = yf.EquityQuery('lt', ['avgdailyvol3m', 99999999999]) - - Screen for stocks where the intraday market cap is between 0 and 100 million. - - .. code-block:: python - - btwn = yf.EquityQuery('btwn', ['intradaymarketcap', 0, 100000000]) - - Screen for stocks in the Technology sector. - - .. code-block:: python - - eq = yf.EquityQuery('eq', ['sector', 'Technology']) - - Combine queries using AND/OR. - - .. code-block:: python - - qt = yf.EquityQuery('and', [gt, lt]) - qf = yf.EquityQuery('or', [qt, btwn, eq]) - """ - def __init__(self, operator: str, operand: Union[numbers.Real, str, List['EquityQuery']]): - """ - .. seealso:: - - :attr:`EquityQuery.valid_operand_fields ` - supported operand values for query - :attr:`EquityQuery.valid_eq_operand_map ` - supported `EQ query operand parameters` - """ - operator = operator.upper() - - if not isinstance(operand, list): - raise TypeError('Invalid operand type') - if len(operand) <= 0: - raise ValueError('Invalid field for Screener') - - if operator in {'OR','AND'}: - self._validate_or_and_operand(operand) - elif operator == 'EQ': - self._validate_eq_operand(operand) - elif operator == 'BTWN': - self._validate_btwn_operand(operand) - elif operator in {'GT','LT'}: - self._validate_gt_lt(operand) - else: - raise ValueError('Invalid Operator Value') - - self.operator = operator - self.operands = operand - self._valid_eq_operand_map = EQUITY_SCREENER_EQ_MAP - self._valid_operand_fields = EQUITY_SCREENER_FIELDS - - @dynamic_docstring({"valid_eq_operand_map_table": generate_list_table_from_dict(EQUITY_SCREENER_EQ_MAP)}) - @property - def valid_eq_operand_map(self) -> Dict: - """ - Valid Operand Map for Operator "EQ" - {valid_eq_operand_map_table} - """ - return self._valid_eq_operand_map - - @dynamic_docstring({"valid_operand_fields_table": generate_list_table_from_dict(EQUITY_SCREENER_FIELDS)}) - @property - def valid_operand_fields(self) -> Dict: - """ - Valid Operand Fields - {valid_operand_fields_table} - """ - return self._valid_operand_fields - - def _validate_or_and_operand(self, operand: List['EquityQuery']) -> None: - if len(operand) <= 1: - raise ValueError('Operand must be length longer than 1') - if all(isinstance(e, EquityQuery) for e in operand) is False: - raise TypeError('Operand must be type EquityQuery for OR/AND') - - def _validate_eq_operand(self, operand: List[Union[str, numbers.Real]]) -> None: - if len(operand) != 2: - raise ValueError('Operand must be length 2 for EQ') - - if not any(operand[0] in fields_by_type for fields_by_type in EQUITY_SCREENER_FIELDS.values()): - raise ValueError('Invalid field for Screener') - if operand[0] not in EQUITY_SCREENER_EQ_MAP: - raise ValueError('Invalid EQ key') - if operand[1] not in EQUITY_SCREENER_EQ_MAP[operand[0]]: - raise ValueError('Invalid EQ value') - - def _validate_btwn_operand(self, operand: List[Union[str, numbers.Real]]) -> None: - if len(operand) != 3: - raise ValueError('Operand must be length 3 for BTWN') - if not any(operand[0] in fields_by_type for fields_by_type in EQUITY_SCREENER_FIELDS.values()): - raise ValueError('Invalid field for Screener') - if isinstance(operand[1], numbers.Real) is False: - raise TypeError('Invalid comparison type for BTWN') - if isinstance(operand[2], numbers.Real) is False: - raise TypeError('Invalid comparison type for BTWN') - - def _validate_gt_lt(self, operand: List[Union[str, numbers.Real]]) -> None: - if len(operand) != 2: - raise ValueError('Operand must be length 2 for GT/LT') - if not any(operand[0] in fields_by_type for fields_by_type in EQUITY_SCREENER_FIELDS.values()): - raise ValueError('Invalid field for Screener') - if isinstance(operand[1], numbers.Real) is False: - raise TypeError('Invalid comparison type for GT/LT') - - def to_dict(self) -> Dict: - return { - "operator": self.operator, - "operands": [operand.to_dict() if isinstance(operand, EquityQuery) else operand for operand in self.operands] - } \ No newline at end of file diff --git a/yfinance/search.py b/yfinance/search.py index 7dbafc93..7e834233 100644 --- a/yfinance/search.py +++ b/yfinance/search.py @@ -27,7 +27,8 @@ class Search: - def __init__(self, query, max_results=8, news_count=8, enable_fuzzy_query=False, + def __init__(self, query, max_results=8, news_count=8, lists_count=8, include_cb=True, include_nav_links=False, + include_research=False, include_cultural_assets=False, enable_fuzzy_query=False, recommended=8, session=None, proxy=None, timeout=30, raise_errors=True): """ Fetches and organizes search results from Yahoo Finance, including stock quotes and news articles. @@ -36,7 +37,13 @@ def __init__(self, query, max_results=8, news_count=8, enable_fuzzy_query=False, query: The search query (ticker symbol or company name). max_results: Maximum number of stock quotes to return (default 8). news_count: Number of news articles to include (default 8). + lists_count: Number of lists to include (default 8). + include_cb: Include the company breakdown (default True). + include_nav_links: Include the navigation links (default False). + include_research: Include the research reports (default False). + include_cultural_assets: Include the cultural assets (default False). enable_fuzzy_query: Enable fuzzy search for typos (default False). + recommended: Recommended number of results to return (default 8). session: Custom HTTP session for requests (default None). proxy: Proxy settings for requests (default None). timeout: Request timeout in seconds (default 30). @@ -51,14 +58,28 @@ def __init__(self, query, max_results=8, news_count=8, enable_fuzzy_query=False, self.timeout = timeout self.raise_errors = raise_errors + self.lists_count = lists_count + self.include_cb = include_cb + self.nav_links = include_nav_links + self.enable_research = include_research + self.enable_cultural_assets = include_cultural_assets + self.recommended = recommended + self._data = YfData(session=self.session) self._logger = utils.get_yf_logger() - self._response = self._fetch_results() - self._quotes = self._response.get("quotes", []) - self._news = self._response.get("news", []) + self._response = {} + self._all = {} + self._quotes = [] + self._news = [] + self._lists = [] + self._research = [] + self._nav = [] + + self.search() - def _fetch_results(self): + def search(self) -> 'Search': + """Search using the query parameters defined in the constructor.""" url = f"{_BASE_URL_}/v1/finance/search" params = { "q": self.query, @@ -66,7 +87,13 @@ def _fetch_results(self): "enableFuzzyQuery": self.enable_fuzzy_query, "newsCount": self.news_count, "quotesQueryId": "tss_match_phrase_query", - "newsQueryId": "news_cie_vespa" + "newsQueryId": "news_cie_vespa", + "listsCount": self.lists_count, + "enableCb": self.include_cb, + "enableNavLinks": self.nav_links, + "enableResearchReports": self.enable_research, + "enableCulturalAssets": self.enable_cultural_assets, + "recommendedCount": self.recommended } self._logger.debug(f'{self.query}: Yahoo GET parameters: {str(dict(params))}') @@ -79,17 +106,53 @@ def _fetch_results(self): try: data = data.json() except _json.JSONDecodeError: - self._logger.error(f"{self.query}: Failed to retrieve the news and received faulty response instead.") + self._logger.error(f"{self.query}: Failed to retrieve search results and received faulty response instead.") data = {} - return data + self._response = data + # Filter quotes to only include symbols + self._quotes = [quote for quote in data.get("quotes", []) if "symbol" in quote] + self._news = data.get("news", []) + self._lists = data.get("lists", []) + self._research = data.get("researchReports", []) + self._nav = data.get("nav", []) + + self._all = {"quotes": self._quotes, "news": self._news, "lists": self._lists, "research": self._research, + "nav": self._nav} + + return self @property - def quotes(self): + def quotes(self) -> 'list': """Get the quotes from the search results.""" return self._quotes @property - def news(self): + def news(self) -> 'list': """Get the news from the search results.""" return self._news + + @property + def lists(self) -> 'list': + """Get the lists from the search results.""" + return self._lists + + @property + def research(self) -> 'list': + """Get the research reports from the search results.""" + return self._research + + @property + def nav(self) -> 'list': + """Get the navigation links from the search results.""" + return self._nav + + @property + def all(self) -> 'dict[str,list]': + """Get all the results from the search results: filtered down version of response.""" + return self._all + + @property + def response(self) -> 'dict': + """Get the raw response from the search results.""" + return self._response diff --git a/yfinance/utils.py b/yfinance/utils.py index 54820d1a..201e9878 100644 --- a/yfinance/utils.py +++ b/yfinance/utils.py @@ -23,6 +23,7 @@ import datetime as _datetime import logging +import re import re as _re import sys as _sys import threading @@ -429,26 +430,28 @@ def _parse_user_dt(dt, exchange_tz): def _interval_to_timedelta(interval): - if interval == "1mo": - return relativedelta(months=1) - elif interval == "3mo": - return relativedelta(months=3) - elif interval == "6mo": - return relativedelta(months=6) - elif interval == "1y": - return relativedelta(years=1) - elif interval == "2y": - return relativedelta(years=2) - elif interval == "5y": - return relativedelta(years=5) - elif interval == "10y": - return relativedelta(years=10) - elif interval == "1wk": - return _pd.Timedelta(days=7) + if interval[-1] == "d": + return relativedelta(days=int(interval[:-1])) + elif interval[-2:] == "wk": + return relativedelta(weeks=int(interval[:-2])) + elif interval[-2:] == "mo": + return relativedelta(months=int(interval[:-2])) + elif interval[-1] == "y": + return relativedelta(years=int(interval[:-1])) else: return _pd.Timedelta(interval) +def is_valid_period_format(period): + """Check if the provided period has a valid format.""" + if period is None: + return False + + # Regex pattern to match valid period formats like '1d', '2wk', '3mo', '1y' + valid_pattern = r"^[1-9]\d*(d|wk|mo|y)$" + return bool(re.match(valid_pattern, period)) + + def auto_adjust(data): col_order = data.columns df = data.copy() @@ -951,10 +954,12 @@ def decorator(func): return func return decorator -def _generate_table_configurations() -> str: +def _generate_table_configurations(title = None) -> str: import textwrap - table = textwrap.dedent(""" - .. list-table:: Permitted Keys/Values + if title is None: + title = "Permitted Keys/Values" + table = textwrap.dedent(f""" + .. list-table:: {title} :widths: 25 75 :header-rows: 1 @@ -964,34 +969,134 @@ def _generate_table_configurations() -> str: return table -def generate_list_table_from_dict(data: dict, bullets: bool=True) -> str: +def generate_list_table_from_dict(data: dict, bullets: bool=True, title: str=None) -> str: """ Generate a list-table for the docstring showing permitted keys/values. """ - table = _generate_table_configurations() - for key, values in data.items(): - value_str = ', '.join(sorted(values)) - table += f" * - {key}\n" - if bullets: - table += " -\n" + table = _generate_table_configurations(title) + for k in sorted(data.keys()): + values = data[k] + table += ' '*3 + f"* - {k}\n" + lengths = [len(str(v)) for v in values] + if bullets and max(lengths) > 5: + table += ' '*5 + "-\n" for value in sorted(values): - table += f" - {value}\n" + table += ' '*7 + f"- {value}\n" else: - table += f" - {value_str}\n" + value_str = ', '.join(sorted(values)) + table += ' '*5 + f"- {value_str}\n" return table -def generate_list_table_from_dict_of_dict(data: dict, bullets: bool=True) -> str: +# def generate_list_table_from_dict_of_dict(data: dict, bullets: bool=True, title: str=None) -> str: +# """ +# Generate a list-table for the docstring showing permitted keys/values. +# """ +# table = _generate_table_configurations(title) +# for k in sorted(data.keys()): +# values = data[k] +# table += ' '*3 + f"* - {k}\n" +# if bullets: +# table += ' '*5 + "-\n" +# for value in sorted(values): +# table += ' '*7 + f"- {value}\n" +# else: +# table += ' '*5 + f"- {values}\n" +# return table + + +def generate_list_table_from_dict_universal(data: dict, bullets: bool=True, title: str=None, concat_keys=[]) -> str: """ Generate a list-table for the docstring showing permitted keys/values. """ - table = _generate_table_configurations() - for key, values in data.items(): - value_str = values - table += f" * - {key}\n" - if bullets: - table += " -\n" - for value in sorted(values): - table += f" - {value}\n" + table = _generate_table_configurations(title) + for k in data.keys(): + values = data[k] + + table += ' '*3 + f"* - {k}\n" + if isinstance(values, dict): + table_add = '' + + concat_short_lines = k in concat_keys + + if bullets: + k_keys = sorted(list(values.keys())) + current_line = '' + block_format = 'query' in k_keys + for i in range(len(k_keys)): + k2 = k_keys[i] + k2_values = values[k2] + k2_values_str = None + if isinstance(k2_values, set): + k2_values = list(k2_values) + elif isinstance(k2_values, dict) and len(k2_values) == 0: + k2_values = [] + if isinstance(k2_values, list): + k2_values = sorted(k2_values) + all_scalar = all(isinstance(k2v, (int, float, str)) for k2v in k2_values) + if all_scalar: + k2_values_str = _re.sub(r"[{}\[\]']", "", str(k2_values)) + + if k2_values_str is None: + k2_values_str = str(k2_values) + + if len(current_line) > 0 and (len(current_line) + len(k2_values_str) > 40): + # new line + table_add += current_line + '\n' + current_line = '' + + if concat_short_lines: + if current_line == '': + current_line += ' '*5 + if i == 0: + # Only add dash to first + current_line += "- " + else: + current_line += " " + # Don't draw bullet points: + current_line += '| ' + else: + current_line += '. ' + current_line += f"{k2}: " + k2_values_str + else: + table_add += ' '*5 + if i == 0: + # Only add dash to first + table_add += "- " + else: + table_add += " " + + if '\n' in k2_values_str: + # Block format multiple lines + table_add += '| ' + f"{k2}: " + "\n" + k2_values_str_lines = k2_values_str.split('\n') + for j in range(len(k2_values_str_lines)): + line = k2_values_str_lines[j] + table_add += ' '*7 + '|' + ' '*5 + line + if j < len(k2_values_str_lines)-1: + table_add += "\n" + else: + if block_format: + table_add += '| ' + else: + table_add += '* ' + table_add += f"{k2}: " + k2_values_str + + table_add += "\n" + if current_line != '': + table_add += current_line + '\n' + else: + table_add += ' '*5 + f"- {values}\n" + + table += table_add + else: - table += f" - {value_str}\n" - return table \ No newline at end of file + lengths = [len(str(v)) for v in values] + if bullets and max(lengths) > 5: + table += ' '*5 + "-\n" + for value in sorted(values): + table += ' '*7 + f"- {value}\n" + else: + value_str = ', '.join(sorted(values)) + table += ' '*5 + f"- {value_str}\n" + + return table