Latent Class MNL, another example

# Install necessary requirements

# If you run this notebook on Google Colab, or in standalone mode, you need to install the required packages.
# Uncomment the following lines:

# !pip install choice-learn

# If you run the notebook within the GitHub repository, you need to run the following lines, that can skipped otherwise:
import os
import sys

sys.path.append("../../")

import os

os.environ["CUDA_VISIBLE_DEVICES"] = ""

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

import tensorflow as tf

Let's use the cars preferences dataset in this example.

from choice_learn.datasets import load_car_preferences

dataset = load_car_preferences(as_frame=False, one_hot_features=True)

Load the right classes from choice-learn

from choice_learn.models.simple_mnl import SimpleMNL
from choice_learn.models.latent_class_mnl import LatentClassSimpleMNL

First, we estimate a two classes model with a simple 'Maximum Likelihood Estimation' (MLE) that is performed with a gradient descent (adam optimizer).

lc_model = LatentClassSimpleMNL(
    n_latent_classes=2,
    fit_method="mle",
    optimizer="adam",
    epochs=2_000,
    batch_size=-1,
    lr=5e-3
)
hist = lc_model.fit(dataset, verbose=1)

Let's check that the optimization went well.

plt.plot(hist["train_loss"])
plt.xlabel("Epochs")
plt.ylabel("Training Loss")
plt.title("Negative Log-Likelihood over Epochs")
plt.show()

We observe the difference between the two classes's estimated models.

colors = ["#E69F00", "#56B4E9", "#009E73", "#0072B2", "#D55E00", "#CC79A7", "#F0E442"]

plt.figure(figsize=(12, 12))
plt.subplot(3, 1, 1)
for i in range(lc_model.n_latent_classes):
    plt.bar(i, lc_model.get_latent_classes_weights()[i], color=colors[i % len(colors)])
plt.ylabel("Class Probability")
plt.xticks(range(lc_model.n_latent_classes), [f"Class {i}" for i in range(lc_model.n_latent_classes)])
plt.title("Class Probabilities")

plt.subplot(3, 1, 2)
div = 1 / (lc_model.n_latent_classes + 1)
for i in range(lc_model.n_latent_classes):
    for j in range(dataset.get_n_shared_features()):
        plt.bar(j + i * div, lc_model.models[i].trainable_weights[0][j], color=colors[i % len(colors)], width=div)
plt.ylabel("Feature Weight")
plt.xticks(range(dataset.get_n_shared_features()), [f"{j}" for j in dataset.shared_features_by_choice_names[0]])
plt.title("Shared Feature Weights")

plt.subplot(3, 1, 3)
div = 1 / (lc_model.n_latent_classes + 1)
for i in range(lc_model.n_latent_classes):
    for j in range(dataset.get_n_items_features()):
        plt.bar(j + i * div, lc_model.models[i].trainable_weights[1][j], color=colors[i % len(colors)], width=div)
plt.ylabel("Feature Weight")
plt.xticks(range(dataset.get_n_items_features()), [f"{j}" for j in dataset.items_features_by_choice_names[0]])
plt.title("Item Feature Weights")

plt.tight_layout()
plt.show()

Let's do the same with a training that is performed with the 'Expectation-Maximization' (EM) algorithm. For the maximization step, the weights are computed using the lbfgs algorithm.

em_model = LatentClassSimpleMNL(n_latent_classes=2, fit_method="em", optimizer="lbfgs", epochs=400, lbfgs_tolerance=1e-6)
em_model.instantiate(
                n_items=dataset.get_n_items(),
                n_shared_features=dataset.get_n_shared_features(),
                n_items_features=dataset.get_n_items_features(),
                base_seed=1024
            )
em_hist = em_model.fit(dataset, verbose=0)

plt.plot(em_hist[0])

colors = ["#E69F00", "#56B4E9", "#009E73", "#0072B2", "#D55E00", "#CC79A7", "#F0E442"]

plt.figure(figsize=(12, 12))
plt.subplot(3, 1, 1)
for i in range(em_model.n_latent_classes):
    plt.bar(i, em_model.get_latent_classes_weights()[i], color=colors[i % len(colors)])
plt.ylabel("Class Probability")
plt.xticks(range(em_model.n_latent_classes), [f"Class {i}" for i in range(em_model.n_latent_classes)])
plt.title("Class Probabilities")

plt.subplot(3, 1, 2)
div = 1 / (em_model.n_latent_classes + 1)
for i in range(em_model.n_latent_classes):
    for j in range(dataset.get_n_shared_features()):
        plt.bar(j + i * div, em_model.models[i].trainable_weights[0][j], color=colors[i % len(colors)], width=div)
plt.ylabel("Feature Weight")
plt.xticks(range(dataset.get_n_shared_features()), [f"{j}" for j in dataset.shared_features_by_choice_names[0]])
plt.title("Shared Feature Weights")

plt.subplot(3, 1, 3)
div = 1 / (em_model.n_latent_classes + 1)
for i in range(em_model.n_latent_classes):
    for j in range(dataset.get_n_items_features()):
        plt.bar(j + i * div, em_model.models[i].trainable_weights[1][j], color=colors[i % len(colors)], width=div)
plt.ylabel("Feature Weight")
plt.xticks(range(dataset.get_n_items_features()), [f"{j}" for j in dataset.items_features_by_choice_names[0]])
plt.title("Item Feature Weights")

plt.tight_layout()
plt.show()

Now let's estimate a latent class model for different number of classes. We use the MLE and lbfgs to get faster results (that are slightly less precise here).

nlls = []
for n_classes in range(2, 8):
    lc_model = LatentClassSimpleMNL(
        n_latent_classes=n_classes,
        fit_method="mle",
        optimizer="lbfgs",
        epochs=2_000,
        lbfgs_tolerance=1e-20
        )
    hist = lc_model.fit(dataset, verbose=0)

    nll = lc_model.evaluate(dataset)
    nlls.append(nll)

plt.plot(list(range(2, 8)), nlls, marker="o", color=colors[3])
plt.xlabel("Number of Latent Classes")
plt.ylabel("Negative Log-Likelihood")
plt.title("NLL for Different Numbers of Latent Classes")
plt.show()