From 03a2cfb7a15d7d82b80303bcd542f85002ac996c Mon Sep 17 00:00:00 2001 From: yancong <32220263+ice-tong@users.noreply.github.com> Date: Wed, 10 Feb 2021 21:10:50 +0800 Subject: [PATCH 1/5] Fix `get_data_yahoo()` Bug in windows when satrt date is earlier than 1970-1-1 8:00 Bug reproduce: ``` python3 from pandas_datareader import data as web web.get_data_yahoo("IBM", start='1968-02-28', end='2020-02-08', interval="d") ``` ERROR `Overflow error: mktime argument out of range` --- pandas_datareader/yahoo/daily.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas_datareader/yahoo/daily.py b/pandas_datareader/yahoo/daily.py index e1578b09..19321fdf 100644 --- a/pandas_datareader/yahoo/daily.py +++ b/pandas_datareader/yahoo/daily.py @@ -2,7 +2,7 @@ import json import re -import time +import time, datetime from pandas import DataFrame, isnull, notnull, to_datetime @@ -127,7 +127,11 @@ def url(self): def _get_params(self, symbol): # This needed because yahoo returns data shifted by 4 hours ago. four_hours_in_seconds = 14400 - unix_start = int(time.mktime(self.start.timetuple())) + unix_zero = datetime.datetime(1970, 1, 1, 8) + if self.start < unix_zero: + unix_start = int((self.start - unix_zero).total_seconds()) + else: + unix_start = int(time.mktime(self.start.timetuple())) unix_start += four_hours_in_seconds day_end = self.end.replace(hour=23, minute=59, second=59) unix_end = int(time.mktime(day_end.timetuple())) From f5a864deee0856f321ab0eada5512cf350220377 Mon Sep 17 00:00:00 2001 From: yancong <32220263+ice-tong@users.noreply.github.com> Date: Wed, 10 Feb 2021 21:25:04 +0800 Subject: [PATCH 2/5] Use the difference between the unix_zero and start to prevent the bug occurrence of `OverflowError: mktime argument out of range`. --- pandas_datareader/yahoo/daily.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas_datareader/yahoo/daily.py b/pandas_datareader/yahoo/daily.py index 19321fdf..416e67ea 100644 --- a/pandas_datareader/yahoo/daily.py +++ b/pandas_datareader/yahoo/daily.py @@ -127,11 +127,9 @@ def url(self): def _get_params(self, symbol): # This needed because yahoo returns data shifted by 4 hours ago. four_hours_in_seconds = 14400 + # Use the difference between the unix_zero and start to prevent the bug occurrence of `OverflowError: mktime argument out of range`. unix_zero = datetime.datetime(1970, 1, 1, 8) - if self.start < unix_zero: - unix_start = int((self.start - unix_zero).total_seconds()) - else: - unix_start = int(time.mktime(self.start.timetuple())) + unix_start = int((self.start - unix_zero).total_seconds()) unix_start += four_hours_in_seconds day_end = self.end.replace(hour=23, minute=59, second=59) unix_end = int(time.mktime(day_end.timetuple())) From d22667cf5be21dd3ec5358341358dbd43d367415 Mon Sep 17 00:00:00 2001 From: yancong <32220263+ice-tong@users.noreply.github.com> Date: Wed, 10 Feb 2021 22:15:22 +0800 Subject: [PATCH 3/5] meet flake8 formart --- pandas_datareader/yahoo/daily.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas_datareader/yahoo/daily.py b/pandas_datareader/yahoo/daily.py index 416e67ea..dc28815f 100644 --- a/pandas_datareader/yahoo/daily.py +++ b/pandas_datareader/yahoo/daily.py @@ -2,7 +2,8 @@ import json import re -import time, datetime +import time +import datetime from pandas import DataFrame, isnull, notnull, to_datetime @@ -127,7 +128,8 @@ def url(self): def _get_params(self, symbol): # This needed because yahoo returns data shifted by 4 hours ago. four_hours_in_seconds = 14400 - # Use the difference between the unix_zero and start to prevent the bug occurrence of `OverflowError: mktime argument out of range`. + # Use the difference between the unix_zero and start to prevent the bug occurrence of + # `OverflowError: mktime argument out of range`. unix_zero = datetime.datetime(1970, 1, 1, 8) unix_start = int((self.start - unix_zero).total_seconds()) unix_start += four_hours_in_seconds From 2e99cea9cb9bb967b1664c653339107e5f78d15d Mon Sep 17 00:00:00 2001 From: yancong <32220263+ice-tong@users.noreply.github.com> Date: Wed, 10 Feb 2021 23:04:47 +0800 Subject: [PATCH 4/5] meet flake8 formart --- pandas_datareader/yahoo/daily.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas_datareader/yahoo/daily.py b/pandas_datareader/yahoo/daily.py index dc28815f..26120642 100644 --- a/pandas_datareader/yahoo/daily.py +++ b/pandas_datareader/yahoo/daily.py @@ -128,8 +128,6 @@ def url(self): def _get_params(self, symbol): # This needed because yahoo returns data shifted by 4 hours ago. four_hours_in_seconds = 14400 - # Use the difference between the unix_zero and start to prevent the bug occurrence of - # `OverflowError: mktime argument out of range`. unix_zero = datetime.datetime(1970, 1, 1, 8) unix_start = int((self.start - unix_zero).total_seconds()) unix_start += four_hours_in_seconds From a7c02b9d563019503674f7ee6a52ccd924e27748 Mon Sep 17 00:00:00 2001 From: yancong <32220263+ice-tong@users.noreply.github.com> Date: Wed, 10 Feb 2021 23:10:12 +0800 Subject: [PATCH 5/5] Use the difference between the unix_zero and start to prevent the bug occurrence of `OverflowError: mktime argument out of range`. --- pandas_datareader/yahoo/fx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas_datareader/yahoo/fx.py b/pandas_datareader/yahoo/fx.py index 81a0f0ec..69e15f1c 100644 --- a/pandas_datareader/yahoo/fx.py +++ b/pandas_datareader/yahoo/fx.py @@ -1,5 +1,6 @@ import json import time +import datetime import warnings from pandas import DataFrame, Series, concat, to_datetime @@ -39,7 +40,8 @@ class YahooFXReader(YahooDailyReader): """ def _get_params(self, symbol): - unix_start = int(time.mktime(self.start.timetuple())) + unix_zero = datetime.datetime(1970, 1, 1, 8) + unix_start = int((self.start - unix_zero).total_seconds()) day_end = self.end.replace(hour=23, minute=59, second=59) unix_end = int(time.mktime(day_end.timetuple()))