import torch
from torchmetrics.nominal import FleissKappa
metric = FleissKappa(mode="probs")
values = [ ]
for _ in range(10):
    values.append(metric(torch.randn(100, 5, 10).softmax(dim=1)))
fig_, ax_ = metric.plot(values)