funcs.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532
  1. # Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from typing import Callable, Dict, List, Optional, Union
  15. import chinese_calendar
  16. import numpy as np
  17. import pandas as pd
  18. from pandas.tseries import holiday as hd
  19. from pandas.tseries.offsets import DateOffset, Day, Easter
  20. from sklearn.preprocessing import StandardScaler
  21. MAX_WINDOW = 183 + 17
  22. EasterSunday = hd.Holiday("Easter Sunday", month=1, day=1, offset=[Easter(), Day(0)])
  23. NewYearsDay = hd.Holiday("New Years Day", month=1, day=1)
  24. SuperBowl = hd.Holiday("Superbowl", month=2, day=1, offset=DateOffset(weekday=hd.SU(1)))
  25. MothersDay = hd.Holiday(
  26. "Mothers Day", month=5, day=1, offset=DateOffset(weekday=hd.SU(2))
  27. )
  28. IndependenceDay = hd.Holiday("Independence Day", month=7, day=4)
  29. ChristmasEve = hd.Holiday("Christmas", month=12, day=24)
  30. ChristmasDay = hd.Holiday("Christmas", month=12, day=25)
  31. NewYearsEve = hd.Holiday("New Years Eve", month=12, day=31)
  32. BlackFriday = hd.Holiday(
  33. "Black Friday",
  34. month=11,
  35. day=1,
  36. offset=[pd.DateOffset(weekday=hd.TH(4)), Day(1)],
  37. )
  38. CyberMonday = hd.Holiday(
  39. "Cyber Monday",
  40. month=11,
  41. day=1,
  42. offset=[pd.DateOffset(weekday=hd.TH(4)), Day(4)],
  43. )
  44. HOLIDAYS = [
  45. hd.EasterMonday,
  46. hd.GoodFriday,
  47. hd.USColumbusDay,
  48. hd.USLaborDay,
  49. hd.USMartinLutherKingJr,
  50. hd.USMemorialDay,
  51. hd.USPresidentsDay,
  52. hd.USThanksgivingDay,
  53. EasterSunday,
  54. NewYearsDay,
  55. SuperBowl,
  56. MothersDay,
  57. IndependenceDay,
  58. ChristmasEve,
  59. ChristmasDay,
  60. NewYearsEve,
  61. BlackFriday,
  62. CyberMonday,
  63. ]
  64. def _cal_year(
  65. x: np.datetime64,
  66. ):
  67. return x.year
  68. def _cal_month(
  69. x: np.datetime64,
  70. ):
  71. return x.month
  72. def _cal_day(
  73. x: np.datetime64,
  74. ):
  75. return x.day
  76. def _cal_hour(
  77. x: np.datetime64,
  78. ):
  79. return x.hour
  80. def _cal_weekday(
  81. x: np.datetime64,
  82. ):
  83. return x.dayofweek
  84. def _cal_quarter(
  85. x: np.datetime64,
  86. ):
  87. return x.quarter
  88. def _cal_hourofday(
  89. x: np.datetime64,
  90. ):
  91. return x.hour / 23.0 - 0.5
  92. def _cal_dayofweek(
  93. x: np.datetime64,
  94. ):
  95. return x.dayofweek / 6.0 - 0.5
  96. def _cal_dayofmonth(
  97. x: np.datetime64,
  98. ):
  99. return x.day / 30.0 - 0.5
  100. def _cal_dayofyear(
  101. x: np.datetime64,
  102. ):
  103. return x.dayofyear / 364.0 - 0.5
  104. def _cal_weekofyear(
  105. x: np.datetime64,
  106. ):
  107. return x.weekofyear / 51.0 - 0.5
  108. def _cal_holiday(
  109. x: np.datetime64,
  110. ):
  111. return float(chinese_calendar.is_holiday(x))
  112. def _cal_workday(
  113. x: np.datetime64,
  114. ):
  115. return float(chinese_calendar.is_workday(x))
  116. def _cal_minuteofhour(
  117. x: np.datetime64,
  118. ):
  119. return x.minute / 59 - 0.5
  120. def _cal_monthofyear(
  121. x: np.datetime64,
  122. ):
  123. return x.month / 11.0 - 0.5
  124. CAL_DATE_METHOD = {
  125. "year": _cal_year,
  126. "month": _cal_month,
  127. "day": _cal_day,
  128. "hour": _cal_hour,
  129. "weekday": _cal_weekday,
  130. "quarter": _cal_quarter,
  131. "minuteofhour": _cal_minuteofhour,
  132. "monthofyear": _cal_monthofyear,
  133. "hourofday": _cal_hourofday,
  134. "dayofweek": _cal_dayofweek,
  135. "dayofmonth": _cal_dayofmonth,
  136. "dayofyear": _cal_dayofyear,
  137. "weekofyear": _cal_weekofyear,
  138. "is_holiday": _cal_holiday,
  139. "is_workday": _cal_workday,
  140. }
  141. def load_from_one_dataframe(
  142. data: Union[pd.DataFrame, pd.Series],
  143. time_col: Optional[str] = None,
  144. value_cols: Optional[Union[List[str], str]] = None,
  145. freq: Optional[Union[str, int]] = None,
  146. drop_tail_nan: bool = False,
  147. dtype: Optional[Union[type, Dict[str, type]]] = None,
  148. ) -> pd.DataFrame:
  149. """Transforms a DataFrame or Series into a time-indexed DataFrame.
  150. Args:
  151. data (Union[pd.DataFrame, pd.Series]): The input data containing time series information.
  152. time_col (Optional[str]): The column name representing time information. If None, uses the index.
  153. value_cols (Optional[Union[List[str], str]]): Columns to extract as values. If None, uses all except time_col.
  154. freq (Optional[Union[str, int]]): The frequency of the time series data.
  155. drop_tail_nan (bool): If True, drop trailing NaN values from the data.
  156. dtype (Optional[Union[type, Dict[str, type]]]): Enforce a specific data type on the resulting DataFrame.
  157. Returns:
  158. pd.DataFrame: A DataFrame with time as the index and specified value columns.
  159. Raises:
  160. ValueError: If the time column doesn't exist, or if frequency cannot be inferred.
  161. """
  162. # Initialize series_data with specified value columns or all except time_col
  163. series_data = None
  164. if value_cols is None:
  165. if isinstance(data, pd.Series):
  166. series_data = data.copy()
  167. else:
  168. series_data = data.loc[:, data.columns != time_col].copy()
  169. else:
  170. series_data = data.loc[:, value_cols].copy()
  171. # Determine the time column values
  172. if time_col:
  173. if time_col not in data.columns:
  174. raise ValueError(
  175. "The time column: {} doesn't exist in the `data`!".format(time_col)
  176. )
  177. time_col_vals = data.loc[:, time_col]
  178. else:
  179. time_col_vals = data.index
  180. # Handle integer-based time column values when frequency is a string
  181. if np.issubdtype(time_col_vals.dtype, np.integer) and isinstance(freq, str):
  182. time_col_vals = time_col_vals.astype(str)
  183. # Process integer-based time column values
  184. if np.issubdtype(time_col_vals.dtype, np.integer):
  185. if freq:
  186. if not isinstance(freq, int) or freq < 1:
  187. raise ValueError(
  188. "The type of `freq` should be `int` when the type of `time_col` is `RangeIndex`."
  189. )
  190. else:
  191. freq = 1 # Default frequency for integer index
  192. start_idx, stop_idx = min(time_col_vals), max(time_col_vals) + freq
  193. if (stop_idx - start_idx) / freq != len(data):
  194. raise ValueError("The number of rows doesn't match with the RangeIndex!")
  195. time_index = pd.RangeIndex(start=start_idx, stop=stop_idx, step=freq)
  196. # Process datetime-like time column values
  197. elif np.issubdtype(time_col_vals.dtype, np.object_) or np.issubdtype(
  198. time_col_vals.dtype, np.datetime64
  199. ):
  200. time_col_vals = pd.to_datetime(time_col_vals, infer_datetime_format=True)
  201. time_index = pd.DatetimeIndex(time_col_vals)
  202. if freq:
  203. if not isinstance(freq, str):
  204. raise ValueError(
  205. "The type of `freq` should be `str` when the type of `time_col` is `DatetimeIndex`."
  206. )
  207. else:
  208. # Attempt to infer frequency if not provided
  209. freq = pd.infer_freq(time_index)
  210. if freq is None:
  211. raise ValueError(
  212. "Failed to infer the `freq`. A valid `freq` is required."
  213. )
  214. if freq[0] == "-":
  215. freq = freq[1:]
  216. # Raise error for unsupported time column types
  217. else:
  218. raise ValueError("The type of `time_col` is invalid.")
  219. # Ensure series_data is a DataFrame
  220. if isinstance(series_data, pd.Series):
  221. series_data = series_data.to_frame()
  222. # Set time index and sort data
  223. series_data.set_index(time_index, inplace=True)
  224. series_data.sort_index(inplace=True)
  225. return series_data
  226. def load_from_dataframe(
  227. df: pd.DataFrame,
  228. group_id: Optional[str] = None,
  229. time_col: Optional[str] = None,
  230. target_cols: Optional[Union[List[str], str]] = None,
  231. label_col: Optional[Union[List[str], str]] = None,
  232. observed_cov_cols: Optional[Union[List[str], str]] = None,
  233. feature_cols: Optional[Union[List[str], str]] = None,
  234. known_cov_cols: Optional[Union[List[str], str]] = None,
  235. static_cov_cols: Optional[Union[List[str], str]] = None,
  236. freq: Optional[Union[str, int]] = None,
  237. fill_missing_dates: bool = False,
  238. fillna_method: str = "pre",
  239. fillna_window_size: int = 10,
  240. **kwargs,
  241. ) -> Dict[str, Optional[Union[pd.DataFrame, Dict[str, any]]]]:
  242. """Loads and processes time series data from a DataFrame.
  243. This function extracts and organizes time series data from a given DataFrame.
  244. It supports optional grouping and extraction of specific columns as features.
  245. Args:
  246. df (pd.DataFrame): The input DataFrame containing time series data.
  247. group_id (Optional[str]): Column name used for grouping the data.
  248. time_col (Optional[str]): Name of the time column.
  249. target_cols (Optional[Union[List[str], str]]): Columns to be used as target.
  250. label_col (Optional[Union[List[str], str]]): Columns to be used as label.
  251. observed_cov_cols (Optional[Union[List[str], str]]): Columns for observed covariates.
  252. feature_cols (Optional[Union[List[str], str]]): Columns to be used as features.
  253. known_cov_cols (Optional[Union[List[str], str]]): Columns for known covariates.
  254. static_cov_cols (Optional[Union[List[str], str]]): Columns for static covariates.
  255. freq (Optional[Union[str, int]]): Frequency of the time series data.
  256. fill_missing_dates (bool): Whether to fill missing dates in the time series.
  257. fillna_method (str): Method to fill missing values ('pre' or 'post').
  258. fillna_window_size (int): Window size for filling missing values.
  259. **kwargs: Additional keyword arguments.
  260. Returns:
  261. Dict[str, Optional[Union[pd.DataFrame, Dict[str, any]]]]: A dictionary containing processed time series data.
  262. """
  263. # List to store DataFrames if grouping is applied
  264. dfs = []
  265. # Separate the DataFrame into groups if group_id is provided
  266. if group_id is not None:
  267. group_unique = df[group_id].unique()
  268. for column in group_unique:
  269. dfs.append(df[df[group_id].isin([column])])
  270. else:
  271. dfs = [df]
  272. # Result list to store processed data from each group
  273. res = []
  274. # If label_col is provided, ensure it is a single column
  275. if label_col:
  276. if isinstance(label_col, str) and len(label_col) > 1:
  277. raise ValueError("The length of label_col must be 1.")
  278. target_cols = label_col
  279. # If feature_cols is provided, treat it as observed_cov_cols
  280. if feature_cols:
  281. observed_cov_cols = feature_cols
  282. # Process each DataFrame in the list
  283. for df in dfs:
  284. target = None
  285. observed_cov = None
  286. known_cov = None
  287. static_cov = dict()
  288. # If no specific columns are provided, use all columns except time_col
  289. if not any([target_cols, observed_cov_cols, known_cov_cols, static_cov_cols]):
  290. target = load_from_one_dataframe(
  291. df,
  292. time_col,
  293. [a for a in df.columns if a != time_col],
  294. freq,
  295. )
  296. else:
  297. if target_cols:
  298. target = load_from_one_dataframe(
  299. df,
  300. time_col,
  301. target_cols,
  302. freq,
  303. )
  304. if observed_cov_cols:
  305. observed_cov = load_from_one_dataframe(
  306. df,
  307. time_col,
  308. observed_cov_cols,
  309. freq,
  310. )
  311. if known_cov_cols:
  312. known_cov = load_from_one_dataframe(
  313. df,
  314. time_col,
  315. known_cov_cols,
  316. freq,
  317. )
  318. if static_cov_cols:
  319. if isinstance(static_cov_cols, str):
  320. static_cov_cols = [static_cov_cols]
  321. for col in static_cov_cols:
  322. if col not in df.columns or len(np.unique(df[col])) != 1:
  323. raise ValueError(
  324. "Static covariate columns data is not in columns or schema is not correct!"
  325. )
  326. static_cov[col] = df[col].iloc[0]
  327. # Append the processed data into the results list
  328. res.append(
  329. {
  330. "past_target": target,
  331. "observed_cov_numeric": observed_cov,
  332. "known_cov_numeric": known_cov,
  333. "static_cov_numeric": static_cov,
  334. }
  335. )
  336. # Return the first processed result
  337. return res[0]
  338. def _distance_to_holiday(holiday) -> Callable[[pd.Timestamp], float]:
  339. """Creates a function to calculate the distance in days to the nearest holiday.
  340. This function generates a closure that computes the number of days from
  341. a given date index to the nearest holiday within a defined window.
  342. Args:
  343. holiday: An object that provides a `dates` method, which returns the
  344. dates of holidays within a specified range.
  345. Returns:
  346. Callable[[pd.Timestamp], float]: A function that takes a date index
  347. as input and returns the distance in days to the nearest holiday.
  348. """
  349. def _distance_to_day(index: pd.Timestamp) -> float:
  350. """Calculates the distance in days from a given date index to the nearest holiday.
  351. Args:
  352. index (pd.Timestamp): The date index for which the distance to the
  353. nearest holiday should be calculated.
  354. Returns:
  355. float: The number of days to the nearest holiday.
  356. Raises:
  357. AssertionError: If no holiday is found within the specified window.
  358. """
  359. holiday_date = holiday.dates(
  360. index - pd.Timedelta(days=MAX_WINDOW),
  361. index + pd.Timedelta(days=MAX_WINDOW),
  362. )
  363. assert (
  364. len(holiday_date) != 0
  365. ), f"No closest holiday for the date index {index} found."
  366. # It sometimes returns two dates if it is exactly half a year after the
  367. # holiday. In this case, the smaller distance (182 days) is returned.
  368. return float((index - holiday_date[0]).days)
  369. return _distance_to_day
  370. def time_feature(
  371. dataset: Dict,
  372. freq: Optional[Union[str, int]],
  373. feature_cols: List[str],
  374. extend_points: int,
  375. inplace: bool = False,
  376. ) -> Dict:
  377. """Transforms the time column of a dataset into time features.
  378. This function extracts time-related features from the time column in a
  379. dataset, optionally extending the time series for future points and
  380. normalizing holiday distances.
  381. Args:
  382. dataset (Dict): Dataset to be transformed.
  383. freq: Optional[Union[str, int]]: Frequency of the time series data. If not provided,
  384. the frequency will be inferred.
  385. feature_cols (List[str]): List of feature columns to be extracted.
  386. extend_points (int): Number of future points to extend the time series.
  387. inplace (bool): Whether to perform the transformation inplace. Default is False.
  388. Returns:
  389. Dict: The transformed dataset with time features added.
  390. Raises:
  391. ValueError: If the time column is of an integer type instead of datetime.
  392. """
  393. new_ts = dataset
  394. if not inplace:
  395. new_ts = dataset.copy()
  396. # Get known_cov_numeric or initialize with past target index
  397. kcov = new_ts["known_cov_numeric"]
  398. if not kcov:
  399. tf_kcov = new_ts["past_target"].index.to_frame()
  400. else:
  401. tf_kcov = kcov.index.to_frame()
  402. time_col = tf_kcov.columns[0]
  403. # Check if time column is of datetime type
  404. if np.issubdtype(tf_kcov[time_col].dtype, np.integer):
  405. raise ValueError(
  406. "The time_col can't be the type of numpy.integer, and it must be the type of numpy.datetime64"
  407. )
  408. # Extend the time series if no known_cov_numeric
  409. if not kcov:
  410. freq = freq if freq is not None else pd.infer_freq(tf_kcov[time_col])
  411. extend_time = pd.date_range(
  412. start=tf_kcov[time_col][-1],
  413. freq=freq,
  414. periods=extend_points + 1,
  415. closed="right",
  416. name=time_col,
  417. ).to_frame()
  418. tf_kcov = pd.concat([tf_kcov, extend_time])
  419. # Extract and add time features to known_cov_numeric
  420. for k in feature_cols:
  421. if k != "holidays":
  422. v = tf_kcov[time_col].apply(lambda x: CAL_DATE_METHOD[k](x))
  423. v.index = tf_kcov[time_col]
  424. if new_ts["known_cov_numeric"] is None:
  425. new_ts["known_cov_numeric"] = pd.DataFrame(v.rename(k), index=v.index)
  426. else:
  427. new_ts["known_cov_numeric"][k] = v.rename(k).reindex(
  428. new_ts["known_cov_numeric"].index
  429. )
  430. else:
  431. holidays_col = []
  432. for i, H in enumerate(HOLIDAYS):
  433. v = tf_kcov[time_col].apply(_distance_to_holiday(H))
  434. v.index = tf_kcov[time_col]
  435. holidays_col.append(k + "_" + str(i))
  436. if new_ts["known_cov_numeric"] is None:
  437. new_ts["known_cov_numeric"] = pd.DataFrame(
  438. v.rename(k + "_" + str(i)), index=v.index
  439. )
  440. else:
  441. new_ts["known_cov_numeric"][k + "_" + str(i)] = v.rename(k).reindex(
  442. new_ts["known_cov_numeric"].index
  443. )
  444. scaler = StandardScaler()
  445. scaler.fit(new_ts["known_cov_numeric"][holidays_col])
  446. new_ts["known_cov_numeric"][holidays_col] = scaler.transform(
  447. new_ts["known_cov_numeric"][holidays_col]
  448. )
  449. return new_ts