Split Samples in Time Series

Split Samples in Time Series#

  1. Simulated Dataset:

  • We generate 100 customers, each with data over 24 months.

  • The dataset includes two random features (feature_1 and feature_2), and a binary default outcome, with a default rate of 20%.

  1. Rolling-Window Cross-Validation (recommended for R&D):

  • We use TimeSeriesSplit from sklearn.model_selection, which ensures that earlier months are used for training and later months for testing.

  • The training and testing sets “roll” forward as you move through the data.

  1. Expanding Window Cross-Validation (for production purpose):

  • In the expanding window approach, the training set grows as more data becomes available.

  • We define a function expanding_window_split that expands the training set window while testing on the next available step.

  1. Model:

  • We use a RandomForestClassifier for demonstration, though you can replace this with any model.

  1. Accuracy Calculation:

  • The accuracy of the model for each fold is printed out for both rolling-window and expanding window cross-validation.

import pandas as pd
import numpy as np
from sklearn.model_selection import TimeSeriesSplit
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score

# Simulate example dataset
np.random.seed(42)

# Parameters for the simulation
n_customers = 100
n_months = 24

# Create a DataFrame for customers across months
customer_ids = np.repeat(np.arange(1, n_customers + 1), n_months)
months = np.tile(np.arange(1, n_months + 1), n_customers)
default = np.random.binomial(1, 0.2, n_customers * n_months)  # 20% default rate

# Simulate some features (you can add more complex features)
feature_1 = np.random.randn(n_customers * n_months)  # Random feature
feature_2 = np.random.randn(n_customers * n_months)  # Another random feature

# Create the DataFrame
df = pd.DataFrame({
    'customer_id': customer_ids,
    'month': months,
    'feature_1': feature_1,
    'feature_2': feature_2,
    'default': default
})

# Sort by customer_id and month to maintain temporal order
df = df.sort_values(by=['customer_id', 'month'])

# Prepare features (X) and target (y)
X = df[['feature_1', 'feature_2']]
y = df['default']

# ---------------------------------
# Rolling-Window Cross-Validation
# ---------------------------------
print("Rolling-Window Cross-Validation:")
tscv = TimeSeriesSplit(n_splits=5)
model = RandomForestClassifier()  # Example model

# Cross-validation loop for rolling-window
for fold, (train_index, test_index) in enumerate(tscv.split(X), 1):
    X_train, X_test = X.iloc[train_index], X.iloc[test_index]
    y_train, y_test = y.iloc[train_index], y.iloc[test_index]

    # Fit the model on the training set
    model.fit(X_train, y_train)

    # Predict on the testing set
    y_pred = model.predict(X_test)

    # Evaluate the model
    accuracy = accuracy_score(y_test, y_pred)
    print(f'Fold {fold} Accuracy: {accuracy:.4f}')
print('End')

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[1], line 1
----> 1 import pandas as pd
      2 import numpy as np
      3 from sklearn.model_selection import TimeSeriesSplit
      4 from sklearn.ensemble import RandomForestClassifier

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/__init__.py:22
     19 del _hard_dependencies, _dependency, _missing_dependencies
     21 # numpy compat
---> 22 from pandas.compat import is_numpy_dev as _is_numpy_dev  # pyright: ignore # noqa:F401
     24 try:
     25     from pandas._libs import hashtable as _hashtable, lib as _lib, tslib as _tslib

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/compat/__init__.py:18
     15 from typing import TYPE_CHECKING
     17 from pandas._typing import F
---> 18 from pandas.compat.numpy import (
     19     is_numpy_dev,
     20     np_version_under1p21,
     21 )
     22 from pandas.compat.pyarrow import (
     23     pa_version_under1p01,
     24     pa_version_under2p0,
   (...)     31     pa_version_under9p0,
     32 )
     34 if TYPE_CHECKING:

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/compat/numpy/__init__.py:4
      1 """ support numpy compatibility across versions """
      2 import numpy as np
----> 4 from pandas.util.version import Version
      6 # numpy versioning
      7 _np_version = np.__version__

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/util/__init__.py:2
      1 # pyright: reportUnusedImport = false
----> 2 from pandas.util._decorators import (  # noqa:F401
      3     Appender,
      4     Substitution,
      5     cache_readonly,
      6 )
      8 from pandas.core.util.hashing import (  # noqa:F401
      9     hash_array,
     10     hash_pandas_object,
     11 )
     14 def __getattr__(name):

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/util/_decorators.py:14
      6 from typing import (
      7     Any,
      8     Callable,
      9     Mapping,
     10     cast,
     11 )
     12 import warnings
---> 14 from pandas._libs.properties import cache_readonly
     15 from pandas._typing import (
     16     F,
     17     T,
     18 )
     19 from pandas.util._exceptions import find_stack_level

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/_libs/__init__.py:13
      1 __all__ = [
      2     "NaT",
      3     "NaTType",
   (...)      9     "Interval",
     10 ]
---> 13 from pandas._libs.interval import Interval
     14 from pandas._libs.tslibs import (
     15     NaT,
     16     NaTType,
   (...)     21     iNaT,
     22 )

File pandas/_libs/interval.pyx:1, in init pandas._libs.interval()
----> 1 'Could not get source, probably due dynamically evaluated source code.'

ValueError: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject