Multi-Armed Bandits

Multi-Armed Bandits#

Step 1: Simulate the Dataset

We’ll create a dataset where each potential client has a set of features (context) and can receive one of several possible messages.

The goal is to learn which message works best for different segments based on their context.

Key Components:

  • Contextual features: These could be behavioral data such as the number of SMSs read, the average time taken to respond, etc.

  • Actions: The different messages that can be sent.

  • Rewards: The reply rate, indicating whether the client responded to the message.

import numpy as np
import pandas as pd

# Set seed for reproducibility
np.random.seed(42)

# Parameters
n_customers = 1000  # Number of customers
n_messages = 5  # Number of different messages
n_features = 4  # Number of contextual features

# Simulate customer features (contexts)
X = np.random.rand(n_customers, n_features)

# Simulate rewards for each message (action)
# We assume that different contexts have different optimal messages
true_coefficients = np.random.rand(n_messages, n_features)
noise = np.random.randn(n_customers, n_messages) * 0.1
rewards = X @ true_coefficients.T + noise

# Convert rewards to probabilities (between 0 and 1)
reply_probabilities = 1 / (1 + np.exp(-rewards))

# Generate actual replies (binary rewards) based on probabilities
y = np.random.binomial(1, reply_probabilities)

# Create a DataFrame to store the dataset
columns = [f'feature_{i+1}' for i in range(n_features)] + [f'message_{i+1}' for i in range(n_messages)]
data = np.hstack((X, y))
df = pd.DataFrame(data, columns=columns)

# Display the first few rows of the dataset
df.head()

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[1], line 2
      1 import numpy as np
----> 2 import pandas as pd
      3 
      4 # Set seed for reproducibility
      5 np.random.seed(42)

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/__init__.py:22
     19 del _hard_dependencies, _dependency, _missing_dependencies
     21 # numpy compat
---> 22 from pandas.compat import is_numpy_dev as _is_numpy_dev  # pyright: ignore # noqa:F401
     24 try:
     25     from pandas._libs import hashtable as _hashtable, lib as _lib, tslib as _tslib

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/compat/__init__.py:18
     15 from typing import TYPE_CHECKING
     17 from pandas._typing import F
---> 18 from pandas.compat.numpy import (
     19     is_numpy_dev,
     20     np_version_under1p21,
     21 )
     22 from pandas.compat.pyarrow import (
     23     pa_version_under1p01,
     24     pa_version_under2p0,
   (...)     31     pa_version_under9p0,
     32 )
     34 if TYPE_CHECKING:

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/compat/numpy/__init__.py:4
      1 """ support numpy compatibility across versions """
      2 import numpy as np
----> 4 from pandas.util.version import Version
      6 # numpy versioning
      7 _np_version = np.__version__

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/util/__init__.py:2
      1 # pyright: reportUnusedImport = false
----> 2 from pandas.util._decorators import (  # noqa:F401
      3     Appender,
      4     Substitution,
      5     cache_readonly,
      6 )
      8 from pandas.core.util.hashing import (  # noqa:F401
      9     hash_array,
     10     hash_pandas_object,
     11 )
     14 def __getattr__(name):

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/util/_decorators.py:14
      6 from typing import (
      7     Any,
      8     Callable,
      9     Mapping,
     10     cast,
     11 )
     12 import warnings
---> 14 from pandas._libs.properties import cache_readonly
     15 from pandas._typing import (
     16     F,
     17     T,
     18 )
     19 from pandas.util._exceptions import find_stack_level

File /opt/hostedtoolcache/Python/3.11.15/x64/lib/python3.11/site-packages/pandas/_libs/__init__.py:13
      1 __all__ = [
      2     "NaT",
      3     "NaTType",
   (...)      9     "Interval",
     10 ]
---> 13 from pandas._libs.interval import Interval
     14 from pandas._libs.tslibs import (
     15     NaT,
     16     NaTType,
   (...)     21     iNaT,
     22 )

File pandas/_libs/interval.pyx:1, in init pandas._libs.interval()
----> 1 'Could not get source, probably due dynamically evaluated source code.'

ValueError: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject

Step 2: Test the Contextual Bandit Model Assuming your team’s model is implemented as a function contextual_bandit_predict(X), which takes customer features and predicts the best message, we can simulate running the model on this dataset.

def contextual_bandit_predict(X):
    """
    Dummy implementation for the sake of testing.
    Replace this with your team's actual model.
    """
    # For simplicity, let's assume it picks the message with the highest predicted probability
    predicted_rewards = X @ true_coefficients.T
    return np.argmax(predicted_rewards, axis=1)

# Simulate running the bandit model
predicted_messages = contextual_bandit_predict(X)

# Calculate the actual rewards for the predicted messages
actual_rewards = [y[i, predicted_messages[i]] for i in range(n_customers)]

# Evaluate the performance: average reward
average_reward = np.mean(actual_rewards)
print(f'Average Reward: {average_reward:.4f}')

Average Reward: 0.7440

The higher the average reward (at least above random = 0.5) the better