| import numpy as np | |
| from numpy.typing import NDArray | |
| def fleiss_kappa(ratings: NDArray[np.int_]) -> float: | |
| n, k = ratings.shape # n is number of items, k is number of raters | |
| N = np.sum( | |
| ratings[0] | |
| ) # Total number of ratings per item (assumed to be the same for all items) | |
| # Calculating p_i | |
| p_i = np.sum(ratings * (ratings - 1), axis=1) / (N * (N - 1)) | |
| P_o = np.mean(p_i) # Mean of p_i | |
| # Calculating p_e | |
| p = np.sum(ratings, axis=0) / ( | |
| n * N | |
| ) # Sum over items, divide by total number of ratings | |
| P_e = np.sum(p**2) | |
| # Calculating kappa | |
| kappa = float((P_o - P_e) / (1 - P_e)) | |
| return kappa | |