forecaster.py 37 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055
  1. # Copyright (c) 2017-present, Facebook, Inc.
  2. # All rights reserved.
  3. #
  4. # This source code is licensed under the BSD-style license found in the
  5. # LICENSE file in the root directory of this source tree. An additional grant
  6. # of patent rights can be found in the PATENTS file in the same directory.
  7. from __future__ import absolute_import
  8. from __future__ import division
  9. from __future__ import print_function
  10. from __future__ import unicode_literals
  11. from collections import defaultdict
  12. from datetime import timedelta
  13. import pickle
  14. from matplotlib import pyplot as plt
  15. from matplotlib.dates import DateFormatter, MonthLocator, num2date
  16. from matplotlib.ticker import FuncFormatter
  17. import numpy as np
  18. import pandas as pd
  19. # fb-block 1 start
  20. import pkg_resources
  21. # fb-block 1 end
  22. try:
  23. import pystan
  24. except ImportError:
  25. print('You cannot run prophet without pystan installed')
  26. raise
  27. # fb-block 2
  28. class Prophet(object):
  29. """Prophet forecaster.
  30. Parameters
  31. ----------
  32. growth: String 'linear' or 'logistic' to specify a linear or logistic
  33. trend.
  34. changepoints: List of dates at which to include potential changepoints. If
  35. not specified, potential changepoints are selected automatically.
  36. n_changepoints: Number of potential changepoints to include. Not used
  37. if input `changepoints` is supplied. If `changepoints` is not supplied,
  38. then n.changepoints potential changepoints are selected uniformly from
  39. the first 80 percent of the history.
  40. yearly_seasonality: Boolean, fit yearly seasonality.
  41. weekly_seasonality: Boolean, fit weekly seasonality.
  42. holidays: pd.DataFrame with columns holiday (string) and ds (date type)
  43. and optionally columns lower_window and upper_window which specify a
  44. range of days around the date to be included as holidays.
  45. lower_window=-2 will include 2 days prior to the date as holidays.
  46. seasonality_prior_scale: Parameter modulating the strength of the
  47. seasonality model. Larger values allow the model to fit larger seasonal
  48. fluctuations, smaller values dampen the seasonality.
  49. holidays_prior_scale: Parameter modulating the strength of the holiday
  50. components model.
  51. changepoint_prior_scale: Parameter modulating the flexibility of the
  52. automatic changepoint selection. Large values will allow many
  53. changepoints, small values will allow few changepoints.
  54. mcmc_samples: Integer, if great than 0, will do full Bayesian inference
  55. with the specified number of MCMC samples. If 0, will do MAP
  56. estimation.
  57. interval_width: Float, width of the uncertainty intervals provided
  58. for the forecast. If mcmc_samples=0, this will be only the uncertainty
  59. in the trend using the MAP estimate of the extrapolated generative
  60. model. If mcmc.samples>0, this will be integrated over all model
  61. parameters, which will include uncertainty in seasonality.
  62. uncertainty_samples: Number of simulated draws used to estimate
  63. uncertainty intervals.
  64. """
  65. def __init__(
  66. self,
  67. growth='linear',
  68. changepoints=None,
  69. n_changepoints=25,
  70. yearly_seasonality=True,
  71. weekly_seasonality=True,
  72. holidays=None,
  73. seasonality_prior_scale=10.0,
  74. holidays_prior_scale=10.0,
  75. changepoint_prior_scale=0.05,
  76. mcmc_samples=0,
  77. interval_width=0.80,
  78. uncertainty_samples=1000,
  79. ):
  80. self.growth = growth
  81. self.changepoints = pd.to_datetime(changepoints)
  82. if self.changepoints is not None:
  83. self.n_changepoints = len(self.changepoints)
  84. else:
  85. self.n_changepoints = n_changepoints
  86. self.yearly_seasonality = yearly_seasonality
  87. self.weekly_seasonality = weekly_seasonality
  88. if holidays is not None:
  89. if not (
  90. isinstance(holidays, pd.DataFrame)
  91. and 'ds' in holidays
  92. and 'holiday' in holidays
  93. ):
  94. raise ValueError("holidays must be a DataFrame with 'ds' and "
  95. "'holiday' columns.")
  96. holidays['ds'] = pd.to_datetime(holidays['ds'])
  97. self.holidays = holidays
  98. self.seasonality_prior_scale = float(seasonality_prior_scale)
  99. self.changepoint_prior_scale = float(changepoint_prior_scale)
  100. self.holidays_prior_scale = float(holidays_prior_scale)
  101. self.mcmc_samples = mcmc_samples
  102. self.interval_width = interval_width
  103. self.uncertainty_samples = uncertainty_samples
  104. # Set during fitting
  105. self.start = None
  106. self.y_scale = None
  107. self.t_scale = None
  108. self.changepoints_t = None
  109. self.stan_fit = None
  110. self.params = {}
  111. self.history = None
  112. self.history_dates = None
  113. self.validate_inputs()
  114. def validate_inputs(self):
  115. """Validates the inputs to Prophet."""
  116. if self.growth not in ('linear', 'logistic'):
  117. raise ValueError(
  118. "Parameter 'growth' should be 'linear' or 'logistic'.")
  119. if self.holidays is not None:
  120. has_lower = 'lower_window' in self.holidays
  121. has_upper = 'upper_window' in self.holidays
  122. if has_lower + has_upper == 1:
  123. raise ValueError('Holidays must have both lower_window and ' +
  124. 'upper_window, or neither')
  125. if has_lower:
  126. if max(self.holidays['lower_window']) > 0:
  127. raise ValueError('Holiday lower_window should be <= 0')
  128. if min(self.holidays['upper_window']) < 0:
  129. raise ValueError('Holiday upper_window should be >= 0')
  130. for h in self.holidays['holiday'].unique():
  131. if '_delim_' in h:
  132. raise ValueError('Holiday name cannot contain "_delim_"')
  133. if h in ['zeros', 'yearly', 'weekly', 'yhat', 'seasonal',
  134. 'trend']:
  135. raise ValueError('Holiday name {} reserved.'.format(h))
  136. @classmethod
  137. def get_linear_model(cls):
  138. """Load compiled linear trend Stan model"""
  139. # fb-block 3
  140. # fb-block 4 start
  141. model_file = pkg_resources.resource_filename(
  142. 'fbprophet',
  143. 'stan_models/linear_growth.pkl'
  144. )
  145. # fb-block 4 end
  146. with open(model_file, 'rb') as f:
  147. return pickle.load(f)
  148. @classmethod
  149. def get_logistic_model(cls):
  150. """Load compiled logistic trend Stan model"""
  151. # fb-block 5
  152. # fb-block 6 start
  153. model_file = pkg_resources.resource_filename(
  154. 'fbprophet',
  155. 'stan_models/logistic_growth.pkl'
  156. )
  157. # fb-block 6 end
  158. with open(model_file, 'rb') as f:
  159. return pickle.load(f)
  160. def setup_dataframe(self, df, initialize_scales=False):
  161. """Prepare dataframe for fitting or predicting.
  162. Adds a time index and scales y. Creates auxillary columns 't', 't_ix',
  163. 'y_scaled', and 'cap_scaled'. These columns are used during both
  164. fitting and predicting.
  165. Parameters
  166. ----------
  167. df: pd.DataFrame with columns ds, y, and cap if logistic growth.
  168. initialize_scales: Boolean set scaling factors in self from df.
  169. Returns
  170. -------
  171. pd.DataFrame prepared for fitting or predicting.
  172. """
  173. if 'y' in df:
  174. df['y'] = pd.to_numeric(df['y'])
  175. df['ds'] = pd.to_datetime(df['ds'])
  176. df = df.sort_values('ds')
  177. df.reset_index(inplace=True, drop=True)
  178. if initialize_scales:
  179. self.y_scale = df['y'].max()
  180. self.start = df['ds'].min()
  181. self.t_scale = df['ds'].max() - self.start
  182. df['t'] = (df['ds'] - self.start) / self.t_scale
  183. if 'y' in df:
  184. df['y_scaled'] = df['y'] / self.y_scale
  185. if self.growth == 'logistic':
  186. assert 'cap' in df
  187. df['cap_scaled'] = df['cap'] / self.y_scale
  188. return df
  189. def set_changepoints(self):
  190. """Set changepoints
  191. Sets m$changepoints to the dates of changepoints. Either:
  192. 1) The changepoints were passed in explicitly.
  193. A) They are empty.
  194. B) They are not empty, and need validation.
  195. 2) We are generating a grid of them.
  196. 3) The user prefers no changepoints be used.
  197. """
  198. if self.changepoints is not None:
  199. if len(self.changepoints) == 0:
  200. pass
  201. else:
  202. too_low = min(self.changepoints) < self.history['ds'].min()
  203. too_high = max(self.changepoints) > self.history['ds'].max()
  204. if too_low or too_high:
  205. raise ValueError('Changepoints must fall within training data.')
  206. elif self.n_changepoints > 0:
  207. # Place potential changepoints evenly throuh first 80% of history
  208. max_ix = np.floor(self.history.shape[0] * 0.8)
  209. cp_indexes = (
  210. np.linspace(0, max_ix, self.n_changepoints + 1)
  211. .round()
  212. .astype(np.int)
  213. )
  214. self.changepoints = self.history.ix[cp_indexes]['ds'].tail(-1)
  215. else:
  216. # set empty changepoints
  217. self.changepoints = []
  218. if len(self.changepoints) > 0:
  219. self.changepoints_t = np.sort(np.array(
  220. (self.changepoints - self.start) / self.t_scale))
  221. else:
  222. self.changepoints_t = np.array([0]) # dummy changepoint
  223. def get_changepoint_matrix(self):
  224. """Gets changepoint matrix for history dataframe."""
  225. A = np.zeros((self.history.shape[0], len(self.changepoints_t)))
  226. for i, t_i in enumerate(self.changepoints_t):
  227. A[self.history['t'].values >= t_i, i] = 1
  228. return A
  229. @staticmethod
  230. def fourier_series(dates, period, series_order):
  231. """Provides Fourier series components with the specified frequency
  232. and order.
  233. Parameters
  234. ----------
  235. dates: pd.Series containing timestamps.
  236. period: Number of days of the period.
  237. series_order: Number of components.
  238. Returns
  239. -------
  240. Matrix with seasonality features.
  241. """
  242. # convert to days since epoch
  243. t = np.array(
  244. (dates - pd.datetime(1970, 1, 1))
  245. .dt.days
  246. .astype(np.float)
  247. )
  248. return np.column_stack([
  249. fun((2.0 * (i + 1) * np.pi * t / period))
  250. for i in range(series_order)
  251. for fun in (np.sin, np.cos)
  252. ])
  253. @classmethod
  254. def make_seasonality_features(cls, dates, period, series_order, prefix):
  255. """Data frame with seasonality features.
  256. Parameters
  257. ----------
  258. cls: Prophet class.
  259. dates: pd.Series containing timestamps.
  260. period: Number of days of the period.
  261. series_order: Number of components.
  262. prefix: Column name prefix.
  263. Returns
  264. -------
  265. pd.DataFrame with seasonality features.
  266. """
  267. features = cls.fourier_series(dates, period, series_order)
  268. columns = [
  269. '{}_delim_{}'.format(prefix, i + 1)
  270. for i in range(features.shape[1])
  271. ]
  272. return pd.DataFrame(features, columns=columns)
  273. def make_holiday_features(self, dates):
  274. """Construct a dataframe of holiday features.
  275. Parameters
  276. ----------
  277. dates: pd.Series containing timestamps used for computing seasonality.
  278. Returns
  279. -------
  280. pd.DataFrame with a column for each holiday.
  281. """
  282. # A smaller prior scale will shrink holiday estimates more
  283. scale_ratio = self.holidays_prior_scale / self.seasonality_prior_scale
  284. # Holds columns of our future matrix.
  285. expanded_holidays = defaultdict(lambda: np.zeros(dates.shape[0]))
  286. # Makes an index so we can perform `get_loc` below.
  287. row_index = pd.DatetimeIndex(dates)
  288. for ix, row in self.holidays.iterrows():
  289. dt = row.ds.date()
  290. try:
  291. lw = int(row.get('lower_window', 0))
  292. uw = int(row.get('upper_window', 0))
  293. except ValueError:
  294. lw = 0
  295. uw = 0
  296. for offset in range(lw, uw + 1):
  297. occurrence = dt + timedelta(days=offset)
  298. try:
  299. loc = row_index.get_loc(occurrence)
  300. except KeyError:
  301. loc = None
  302. key = '{}_delim_{}{}'.format(
  303. row.holiday,
  304. '+' if offset >= 0 else '-',
  305. abs(offset)
  306. )
  307. if loc is not None:
  308. expanded_holidays[key][loc] = scale_ratio
  309. else:
  310. # Access key to generate value
  311. expanded_holidays[key]
  312. # This relies pretty importantly on pandas keeping the columns in order.
  313. return pd.DataFrame(expanded_holidays)
  314. def make_all_seasonality_features(self, df):
  315. """Dataframe with seasonality features.
  316. Parameters
  317. ----------
  318. df: pd.DataFrame with dates for computing seasonality features.
  319. Returns
  320. -------
  321. pd.DataFrame with seasonality.
  322. """
  323. seasonal_features = [
  324. # Add a column of zeros in case no seasonality is used.
  325. pd.DataFrame({'zeros': np.zeros(df.shape[0])})
  326. ]
  327. # Seasonality features
  328. if self.yearly_seasonality:
  329. seasonal_features.append(self.make_seasonality_features(
  330. df['ds'],
  331. 365.25,
  332. 10,
  333. 'yearly',
  334. ))
  335. if self.weekly_seasonality:
  336. seasonal_features.append(self.make_seasonality_features(
  337. df['ds'],
  338. 7,
  339. 3,
  340. 'weekly',
  341. ))
  342. if self.holidays is not None:
  343. seasonal_features.append(self.make_holiday_features(df['ds']))
  344. return pd.concat(seasonal_features, axis=1)
  345. @staticmethod
  346. def linear_growth_init(df):
  347. """Initialize linear growth.
  348. Provides a strong initialization for linear growth by calculating the
  349. growth and offset parameters that pass the function through the first
  350. and last points in the time series.
  351. Parameters
  352. ----------
  353. df: pd.DataFrame with columns ds (date), y_scaled (scaled time series),
  354. and t (scaled time).
  355. Returns
  356. -------
  357. A tuple (k, m) with the rate (k) and offset (m) of the linear growth
  358. function.
  359. """
  360. i0, i1 = df['ds'].idxmin(), df['ds'].idxmax()
  361. T = df['t'].ix[i1] - df['t'].ix[i0]
  362. k = (df['y_scaled'].ix[i1] - df['y_scaled'].ix[i0]) / T
  363. m = df['y_scaled'].ix[i0] - k * df['t'].ix[i0]
  364. return (k, m)
  365. @staticmethod
  366. def logistic_growth_init(df):
  367. """Initialize logistic growth.
  368. Provides a strong initialization for logistic growth by calculating the
  369. growth and offset parameters that pass the function through the first
  370. and last points in the time series.
  371. Parameters
  372. ----------
  373. df: pd.DataFrame with columns ds (date), cap_scaled (scaled capacity),
  374. y_scaled (scaled time series), and t (scaled time).
  375. Returns
  376. -------
  377. A tuple (k, m) with the rate (k) and offset (m) of the logistic growth
  378. function.
  379. """
  380. i0, i1 = df['ds'].idxmin(), df['ds'].idxmax()
  381. T = df['t'].ix[i1] - df['t'].ix[i0]
  382. # Force valid values, in case y > cap.
  383. r0 = max(1.01, df['cap_scaled'].ix[i0] / df['y_scaled'].ix[i0])
  384. r1 = max(1.01, df['cap_scaled'].ix[i1] / df['y_scaled'].ix[i1])
  385. if abs(r0 - r1) <= 0.01:
  386. r0 = 1.05 * r0
  387. L0 = np.log(r0 - 1)
  388. L1 = np.log(r1 - 1)
  389. # Initialize the offset
  390. m = L0 * T / (L0 - L1)
  391. # And the rate
  392. k = L0 / m
  393. return (k, m)
  394. # fb-block 7
  395. def fit(self, df, **kwargs):
  396. """Fit the Prophet model.
  397. Parameters
  398. ----------
  399. df: pd.DataFrame containing the history. Must have columns ds (date
  400. type) and y, the time series. If self.growth is 'logistic', then
  401. df must also have a column cap that specifies the capacity at
  402. each ds.
  403. kwargs: Additional arguments passed to the optimizing or sampling
  404. functions in Stan.
  405. Returns
  406. -------
  407. The fitted Prophet object.
  408. """
  409. history = df[df['y'].notnull()].copy()
  410. self.history_dates = pd.to_datetime(df['ds']).sort_values()
  411. history = self.setup_dataframe(history, initialize_scales=True)
  412. self.history = history
  413. seasonal_features = self.make_all_seasonality_features(history)
  414. self.set_changepoints()
  415. A = self.get_changepoint_matrix()
  416. dat = {
  417. 'T': history.shape[0],
  418. 'K': seasonal_features.shape[1],
  419. 'S': len(self.changepoints_t),
  420. 'y': history['y_scaled'],
  421. 't': history['t'],
  422. 'A': A,
  423. 't_change': self.changepoints_t,
  424. 'X': seasonal_features,
  425. 'sigma': self.seasonality_prior_scale,
  426. 'tau': self.changepoint_prior_scale,
  427. }
  428. if self.growth == 'linear':
  429. kinit = self.linear_growth_init(history)
  430. model = self.get_linear_model()
  431. else:
  432. dat['cap'] = history['cap_scaled']
  433. kinit = self.logistic_growth_init(history)
  434. model = self.get_logistic_model()
  435. def stan_init():
  436. return {
  437. 'k': kinit[0],
  438. 'm': kinit[1],
  439. 'delta': np.zeros(len(self.changepoints_t)),
  440. 'beta': np.zeros(seasonal_features.shape[1]),
  441. 'sigma_obs': 1,
  442. }
  443. if self.mcmc_samples > 0:
  444. stan_fit = model.sampling(
  445. dat,
  446. init=stan_init,
  447. iter=self.mcmc_samples,
  448. **kwargs
  449. )
  450. for par in stan_fit.model_pars:
  451. self.params[par] = stan_fit[par]
  452. else:
  453. params = model.optimizing(dat, init=stan_init, iter=1e4, **kwargs)
  454. for par in params:
  455. self.params[par] = params[par].reshape((1, -1))
  456. # If no changepoints were requested, replace delta with 0s
  457. if len(self.changepoints) == 0:
  458. # Fold delta into the base rate k
  459. params['k'] = params['k'] + params['delta']
  460. params['delta'] = np.zeros(params['delta'].shape)
  461. return self
  462. # fb-block 8
  463. def predict(self, df=None):
  464. """Predict using the prophet model.
  465. Parameters
  466. ----------
  467. df: pd.DataFrame with dates for predictions (column ds), and capacity
  468. (column cap) if logistic growth. If not provided, predictions are
  469. made on the history.
  470. Returns
  471. -------
  472. A pd.DataFrame with the forecast components.
  473. """
  474. if df is None:
  475. df = self.history.copy()
  476. else:
  477. df = self.setup_dataframe(df)
  478. df['trend'] = self.predict_trend(df)
  479. seasonal_components = self.predict_seasonal_components(df)
  480. intervals = self.predict_uncertainty(df)
  481. df2 = pd.concat((df, intervals, seasonal_components), axis=1)
  482. df2['yhat'] = df2['trend'] + df2['seasonal']
  483. return df2
  484. @staticmethod
  485. def piecewise_linear(t, deltas, k, m, changepoint_ts):
  486. """Evaluate the piecewise linear function.
  487. Parameters
  488. ----------
  489. t: np.array of times on which the function is evaluated.
  490. deltas: np.array of rate changes at each changepoint.
  491. k: Float initial rate.
  492. m: Float initial offset.
  493. changepoint_ts: np.array of changepoint times.
  494. Returns
  495. -------
  496. Vector y(t).
  497. """
  498. # Intercept changes
  499. gammas = -changepoint_ts * deltas
  500. # Get cumulative slope and intercept at each t
  501. k_t = k * np.ones_like(t)
  502. m_t = m * np.ones_like(t)
  503. for s, t_s in enumerate(changepoint_ts):
  504. indx = t >= t_s
  505. k_t[indx] += deltas[s]
  506. m_t[indx] += gammas[s]
  507. return k_t * t + m_t
  508. @staticmethod
  509. def piecewise_logistic(t, cap, deltas, k, m, changepoint_ts):
  510. """Evaluate the piecewise logistic function.
  511. Parameters
  512. ----------
  513. t: np.array of times on which the function is evaluated.
  514. cap: np.array of capacities at each t.
  515. deltas: np.array of rate changes at each changepoint.
  516. k: Float initial rate.
  517. m: Float initial offset.
  518. changepoint_ts: np.array of changepoint times.
  519. Returns
  520. -------
  521. Vector y(t).
  522. """
  523. # Compute offset changes
  524. k_cum = np.concatenate((np.atleast_1d(k), np.cumsum(deltas) + k))
  525. gammas = np.zeros(len(changepoint_ts))
  526. for i, t_s in enumerate(changepoint_ts):
  527. gammas[i] = (
  528. (t_s - m - np.sum(gammas))
  529. * (1 - k_cum[i] / k_cum[i + 1])
  530. )
  531. # Get cumulative rate and offset at each t
  532. k_t = k * np.ones_like(t)
  533. m_t = m * np.ones_like(t)
  534. for s, t_s in enumerate(changepoint_ts):
  535. indx = t >= t_s
  536. k_t[indx] += deltas[s]
  537. m_t[indx] += gammas[s]
  538. return cap / (1 + np.exp(-k_t * (t - m_t)))
  539. def predict_trend(self, df):
  540. """Predict trend using the prophet model.
  541. Parameters
  542. ----------
  543. df: Prediction dataframe.
  544. Returns
  545. -------
  546. Vector with trend on prediction dates.
  547. """
  548. k = np.nanmean(self.params['k'])
  549. m = np.nanmean(self.params['m'])
  550. deltas = np.nanmean(self.params['delta'], axis=0)
  551. t = np.array(df['t'])
  552. if self.growth == 'linear':
  553. trend = self.piecewise_linear(t, deltas, k, m, self.changepoints_t)
  554. else:
  555. cap = df['cap_scaled']
  556. trend = self.piecewise_logistic(
  557. t, cap, deltas, k, m, self.changepoints_t)
  558. return trend * self.y_scale
  559. def predict_seasonal_components(self, df):
  560. """Predict seasonality broken down into components.
  561. Parameters
  562. ----------
  563. df: Prediction dataframe.
  564. Returns
  565. -------
  566. Dataframe with seasonal components.
  567. """
  568. seasonal_features = self.make_all_seasonality_features(df)
  569. lower_p = 100 * (1.0 - self.interval_width) / 2
  570. upper_p = 100 * (1.0 + self.interval_width) / 2
  571. components = pd.DataFrame({
  572. 'col': np.arange(seasonal_features.shape[1]),
  573. 'component': [x.split('_delim_')[0] for x in seasonal_features.columns],
  574. })
  575. # Remove the placeholder
  576. components = components[components['component'] != 'zeros']
  577. if components.shape[0] > 0:
  578. X = seasonal_features.as_matrix()
  579. data = {}
  580. for component, features in components.groupby('component'):
  581. cols = features.col.tolist()
  582. comp_beta = self.params['beta'][:, cols]
  583. comp_features = X[:, cols]
  584. comp = (
  585. np.matmul(comp_features, comp_beta.transpose())
  586. * self.y_scale
  587. )
  588. data[component] = np.nanmean(comp, axis=1)
  589. data[component + '_lower'] = np.nanpercentile(comp, lower_p,
  590. axis=1)
  591. data[component + '_upper'] = np.nanpercentile(comp, upper_p,
  592. axis=1)
  593. component_predictions = pd.DataFrame(data)
  594. component_predictions['seasonal'] = (
  595. component_predictions[components['component'].unique()].sum(1))
  596. else:
  597. component_predictions = pd.DataFrame(
  598. {'seasonal': np.zeros(df.shape[0])})
  599. return component_predictions
  600. def predict_uncertainty(self, df):
  601. """Predict seasonality broken down into components.
  602. Parameters
  603. ----------
  604. df: Prediction dataframe.
  605. Returns
  606. -------
  607. Dataframe with uncertainty intervals.
  608. """
  609. n_iterations = self.params['k'].shape[0]
  610. samp_per_iter = max(1, int(np.ceil(
  611. self.uncertainty_samples / float(n_iterations)
  612. )))
  613. # Generate seasonality features once so we can re-use them.
  614. seasonal_features = self.make_all_seasonality_features(df)
  615. sim_values = {'yhat': [], 'trend': [], 'seasonal': []}
  616. for i in range(n_iterations):
  617. for j in range(samp_per_iter):
  618. sim = self.sample_model(df, seasonal_features, i)
  619. for key in sim_values:
  620. sim_values[key].append(sim[key])
  621. lower_p = 100 * (1.0 - self.interval_width) / 2
  622. upper_p = 100 * (1.0 + self.interval_width) / 2
  623. series = {}
  624. for key, value in sim_values.items():
  625. mat = np.column_stack(value)
  626. series['{}_lower'.format(key)] = np.nanpercentile(mat, lower_p,
  627. axis=1)
  628. series['{}_upper'.format(key)] = np.nanpercentile(mat, upper_p,
  629. axis=1)
  630. return pd.DataFrame(series)
  631. def sample_model(self, df, seasonal_features, iteration):
  632. """Simulate observations from the extrapolated generative model.
  633. Parameters
  634. ----------
  635. df: Prediction dataframe.
  636. seasonal_features: pd.DataFrame of seasonal features.
  637. iteration: Int sampling iteration to use parameters from.
  638. Returns
  639. -------
  640. Dataframe with trend, seasonality, and yhat, each like df['t'].
  641. """
  642. trend = self.sample_predictive_trend(df, iteration)
  643. beta = self.params['beta'][iteration]
  644. seasonal = np.matmul(seasonal_features.as_matrix(), beta) * self.y_scale
  645. sigma = self.params['sigma_obs'][iteration]
  646. noise = np.random.normal(0, sigma, df.shape[0]) * self.y_scale
  647. return pd.DataFrame({
  648. 'yhat': trend + seasonal + noise,
  649. 'trend': trend,
  650. 'seasonal': seasonal,
  651. })
  652. def sample_predictive_trend(self, df, iteration):
  653. """Simulate the trend using the extrapolated generative model.
  654. Parameters
  655. ----------
  656. df: Prediction dataframe.
  657. seasonal_features: pd.DataFrame of seasonal features.
  658. iteration: Int sampling iteration to use parameters from.
  659. Returns
  660. -------
  661. np.array of simulated trend over df['t'].
  662. """
  663. k = self.params['k'][iteration]
  664. m = self.params['m'][iteration]
  665. deltas = self.params['delta'][iteration]
  666. t = np.array(df['t'])
  667. T = t.max()
  668. if T > 1:
  669. # Get the time discretization of the history
  670. dt = np.diff(self.history['t'])
  671. dt = np.min(dt[dt > 0])
  672. # Number of time periods in the future
  673. N = np.ceil((T - 1) / float(dt))
  674. S = len(self.changepoints_t)
  675. prob_change = min(1, (S * (T - 1)) / N)
  676. n_changes = np.random.binomial(N, prob_change)
  677. # Sample ts
  678. changepoint_ts_new = sorted(np.random.uniform(1, T, n_changes))
  679. else:
  680. # Case where we're not extrapolating.
  681. changepoint_ts_new = []
  682. n_changes = 0
  683. # Get the empirical scale of the deltas, plus epsilon to avoid NaNs.
  684. lambda_ = np.mean(np.abs(deltas)) + 1e-8
  685. # Sample deltas
  686. deltas_new = np.random.laplace(0, lambda_, n_changes)
  687. # Prepend the times and deltas from the history
  688. changepoint_ts = np.concatenate((self.changepoints_t,
  689. changepoint_ts_new))
  690. deltas = np.concatenate((deltas, deltas_new))
  691. if self.growth == 'linear':
  692. trend = self.piecewise_linear(t, deltas, k, m, changepoint_ts)
  693. else:
  694. cap = df['cap_scaled']
  695. trend = self.piecewise_logistic(t, cap, deltas, k, m,
  696. changepoint_ts)
  697. return trend * self.y_scale
  698. def make_future_dataframe(self, periods, freq='D', include_history=True):
  699. """Simulate the trend using the extrapolated generative model.
  700. Parameters
  701. ----------
  702. periods: Int number of periods to forecast forward.
  703. freq: Any valid frequency for pd.date_range, such as 'D' or 'M'.
  704. include_history: Boolean to include the historical dates in the data
  705. frame for predictions.
  706. Returns
  707. -------
  708. pd.Dataframe that extends forward from the end of self.history for the
  709. requested number of periods.
  710. """
  711. last_date = self.history_dates.max()
  712. dates = pd.date_range(
  713. start=last_date,
  714. periods=periods + 1, # An extra in case we include start
  715. freq=freq)
  716. dates = dates[dates > last_date] # Drop start if equals last_date
  717. dates = dates[:periods] # Return correct number of periods
  718. if include_history:
  719. dates = np.concatenate((np.array(self.history_dates), dates))
  720. return pd.DataFrame({'ds': dates})
  721. def plot(self, fcst, uncertainty=True, xlabel='ds', ylabel='y'):
  722. """Plot the Prophet forecast.
  723. Parameters
  724. ----------
  725. fcst: pd.DataFrame output of self.predict.
  726. uncertainty: Optional boolean to plot uncertainty intervals.
  727. xlabel: Optional label name on X-axis
  728. ylabel: Optional label name on Y-axis
  729. Returns
  730. -------
  731. a matplotlib figure.
  732. """
  733. fig = plt.figure(facecolor='w', figsize=(10, 6))
  734. ax = fig.add_subplot(111)
  735. ax.plot(self.history['ds'].values, self.history['y'], 'k.')
  736. ax.plot(fcst['ds'].values, fcst['yhat'], ls='-', c='#0072B2')
  737. if 'cap' in fcst:
  738. ax.plot(fcst['ds'].values, fcst['cap'], ls='--', c='k')
  739. if uncertainty:
  740. ax.fill_between(fcst['ds'].values, fcst['yhat_lower'],
  741. fcst['yhat_upper'], color='#0072B2',
  742. alpha=0.2)
  743. ax.grid(True, which='major', c='gray', ls='-', lw=1, alpha=0.2)
  744. ax.set_xlabel(xlabel)
  745. ax.set_ylabel(ylabel)
  746. fig.tight_layout()
  747. return fig
  748. def plot_components(self, fcst, uncertainty=True):
  749. """Plot the Prophet forecast components.
  750. Will plot whichever are available of: trend, holidays, weekly
  751. seasonality, and yearly seasonality.
  752. Parameters
  753. ----------
  754. fcst: pd.DataFrame output of self.predict.
  755. uncertainty: Optional boolean to plot uncertainty intervals.
  756. Returns
  757. -------
  758. a matplotlib figure.
  759. """
  760. # Identify components to be plotted
  761. components = [('plot_trend', True),
  762. ('plot_holidays', self.holidays is not None),
  763. ('plot_weekly', 'weekly' in fcst),
  764. ('plot_yearly', 'yearly' in fcst)]
  765. components = [(plot, cond) for plot, cond in components if cond]
  766. npanel = len(components)
  767. fig, axes = plt.subplots(npanel, 1, facecolor='w',
  768. figsize=(9, 3 * npanel))
  769. artists = []
  770. for ax, plot in zip(axes,
  771. [getattr(self, plot) for plot, _ in components]):
  772. artists += plot(fcst, ax=ax, uncertainty=uncertainty)
  773. fig.tight_layout()
  774. return artists
  775. def plot_trend(self, fcst, ax=None, uncertainty=True):
  776. """Plot the trend component of the forecast.
  777. Parameters
  778. ----------
  779. fcst: pd.DataFrame output of self.predict.
  780. ax: Optional matplotlib Axes to plot on.
  781. uncertainty: Optional boolean to plot uncertainty intervals.
  782. Returns
  783. -------
  784. a list of matplotlib artists
  785. """
  786. artists = []
  787. if not ax:
  788. fig = plt.figure(facecolor='w', figsize=(10, 6))
  789. ax = fig.add_subplot(111)
  790. artists += ax.plot(fcst['ds'].values, fcst['trend'], ls='-',
  791. c='#0072B2')
  792. if 'cap' in fcst:
  793. artists += ax.plot(fcst['ds'].values, fcst['cap'], ls='--', c='k')
  794. if uncertainty:
  795. artists += [ax.fill_between(
  796. fcst['ds'].values, fcst['trend_lower'], fcst['trend_upper'],
  797. color='#0072B2', alpha=0.2)]
  798. ax.grid(True, which='major', c='gray', ls='-', lw=1, alpha=0.2)
  799. ax.set_xlabel('ds')
  800. ax.set_ylabel('trend')
  801. return artists
  802. def plot_holidays(self, fcst, ax=None, uncertainty=True):
  803. """Plot the holidays component of the forecast.
  804. Parameters
  805. ----------
  806. fcst: pd.DataFrame output of self.predict.
  807. ax: Optional matplotlib Axes to plot on. One will be created if this
  808. is not provided.
  809. uncertainty: Optional boolean to plot uncertainty intervals.
  810. Returns
  811. -------
  812. a list of matplotlib artists
  813. """
  814. artists = []
  815. if not ax:
  816. fig = plt.figure(facecolor='w', figsize=(10, 6))
  817. ax = fig.add_subplot(111)
  818. holiday_comps = self.holidays['holiday'].unique()
  819. y_holiday = fcst[holiday_comps].sum(1)
  820. y_holiday_l = fcst[[h + '_lower' for h in holiday_comps]].sum(1)
  821. y_holiday_u = fcst[[h + '_upper' for h in holiday_comps]].sum(1)
  822. # NOTE the above CI calculation is incorrect if holidays overlap
  823. # in time. Since it is just for the visualization we will not
  824. # worry about it now.
  825. artists += ax.plot(fcst['ds'].values, y_holiday, ls='-',
  826. c='#0072B2')
  827. if uncertainty:
  828. artists += [ax.fill_between(fcst['ds'].values,
  829. y_holiday_l, y_holiday_u,
  830. color='#0072B2', alpha=0.2)]
  831. ax.grid(True, which='major', c='gray', ls='-', lw=1, alpha=0.2)
  832. ax.set_xlabel('ds')
  833. ax.set_ylabel('holidays')
  834. return artists
  835. def plot_weekly(self, fcst, ax=None, uncertainty=True):
  836. """Plot the weekly component of the forecast.
  837. Parameters
  838. ----------
  839. fcst: pd.DataFrame output of self.predict.
  840. ax: Optional matplotlib Axes to plot on. One will be created if this
  841. is not provided.
  842. uncertainty: Optional boolean to plot uncertainty intervals.
  843. Returns
  844. -------
  845. a list of matplotlib artists
  846. """
  847. artists = []
  848. if not ax:
  849. fig = plt.figure(facecolor='w', figsize=(10, 6))
  850. ax = fig.add_subplot(111)
  851. df_s = fcst.copy()
  852. df_s['dow'] = df_s['ds'].dt.weekday_name
  853. df_s = df_s.groupby('dow').first()
  854. days = pd.date_range(start='2017-01-01', periods=7).weekday_name
  855. y_weekly = [df_s.loc[d]['weekly'] for d in days]
  856. y_weekly_l = [df_s.loc[d]['weekly_lower'] for d in days]
  857. y_weekly_u = [df_s.loc[d]['weekly_upper'] for d in days]
  858. artists += ax.plot(range(len(days)), y_weekly, ls='-',
  859. c='#0072B2')
  860. if uncertainty:
  861. artists += [ax.fill_between(range(len(days)),
  862. y_weekly_l, y_weekly_u,
  863. color='#0072B2', alpha=0.2)]
  864. ax.grid(True, which='major', c='gray', ls='-', lw=1, alpha=0.2)
  865. ax.set_xticks(range(len(days)))
  866. ax.set_xticklabels(days)
  867. ax.set_xlabel('Day of week')
  868. ax.set_ylabel('weekly')
  869. return artists
  870. def plot_yearly(self, fcst, ax=None, uncertainty=True):
  871. """Plot the yearly component of the forecast.
  872. Parameters
  873. ----------
  874. fcst: pd.DataFrame output of self.predict.
  875. ax: Optional matplotlib Axes to plot on. One will be created if
  876. this is not provided.
  877. uncertainty: Optional boolean to plot uncertainty intervals.
  878. Returns
  879. -------
  880. a list of matplotlib artists
  881. """
  882. artists = []
  883. if not ax:
  884. fig = plt.figure(facecolor='w', figsize=(10, 6))
  885. ax = fig.add_subplot(111)
  886. df_s = fcst.copy()
  887. df_s['doy'] = df_s['ds'].map(lambda x: x.strftime('2000-%m-%d'))
  888. df_s = df_s.groupby('doy').first().sort_index()
  889. artists += ax.plot(pd.to_datetime(df_s.index), df_s['yearly'], ls='-',
  890. c='#0072B2')
  891. if uncertainty:
  892. artists += [ax.fill_between(
  893. pd.to_datetime(df_s.index), df_s['yearly_lower'],
  894. df_s['yearly_upper'], color='#0072B2', alpha=0.2)]
  895. ax.grid(True, which='major', c='gray', ls='-', lw=1, alpha=0.2)
  896. months = MonthLocator(range(1, 13), bymonthday=1, interval=2)
  897. ax.xaxis.set_major_formatter(FuncFormatter(
  898. lambda x, pos=None: '{dt:%B} {dt.day}'.format(dt=num2date(x))))
  899. ax.xaxis.set_major_locator(months)
  900. ax.set_xlabel('Day of year')
  901. ax.set_ylabel('yearly')
  902. return artists
  903. # fb-block 9