Skip to content

Instantly share code, notes, and snippets.

@kmundnic
Last active May 9, 2019 15:24
Show Gist options
  • Save kmundnic/5adb34b2337f3a24df3b534d1d684965 to your computer and use it in GitHub Desktop.
Save kmundnic/5adb34b2337f3a24df3b534d1d684965 to your computer and use it in GitHub Desktop.
Krippendorff's alpha in MATLAB
function alpha=kriAlpha(data,scale)
% alpha=kriAlpha(data,scale)
% calculates Krippendorff's Alpha as a measure of inter-rater agreement
% data: rate matrix, each row is a rater or coder, each column is a case
% scale: level of measurement, supported are 'nominal', 'ordinal', 'interval'
% missing values have to be coded as NaN or inf
% For details about Krippendorff's Alpha see:
% http://en.wikipedia.org/wiki/Krippendorff%27s_Alpha
% Hayes, Andrew F. & Krippendorff, Klaus (2007). Answering the call for a
% standard reliability measure for coding data. Communication Methods and
% Measures, 1, 77-89
if nargin~=2
help kriAlpha
error('Wrong number of input arguments.')
end
allVals=unique(data(:));
allVals=allVals(isfinite(allVals));
% coincidence matrix
coinMatr=nan(length(allVals));
for r=1:length(allVals)
for c=r:length(allVals)
val=0;
for d=1:size(data,2)
%find number of pairs
thisEx=data(:,d);
thisEx=thisEx(isfinite(thisEx));
numEntr=length(thisEx);
numP=0;
for p1=1:numEntr
for p2=1:numEntr
if p1==p2
continue
end
if (thisEx(p1)==allVals(r) && thisEx(p2)==allVals(c))
numP=numP+1;
end
end
end
if numP
val=val+numP/(numEntr-1);
end
end
coinMatr(r,c)=val;
coinMatr(c,r)=val;
end
end
nc=sum(coinMatr,2);
n=sum(nc);
% expected agreement
expMatr=nan(length(allVals));
for i=1:length(allVals)
for j=1:length(allVals)
if i==j
val=nc(i)*(nc(j)-1)/(n-1);
else
val=nc(i)*nc(j)/(n-1);
end
expMatr(i,j)=val;
end
end
% difference matrix
diffMatr=zeros(length(allVals));
for i=1:length(allVals)
for j=i+1:length(allVals)
if i~=j
if strcmp(scale, 'nominal')
val=1;
elseif strcmp(scale, 'ordinal')
val=sum(nc(i:j))-nc(i)/2-nc(j)/2;
val=val.^2;
elseif strcmp(scale, 'interval')
val=(allVals(j)-allVals(i)).^2;
else
error('unknown scale: %s', scale);
end
else
val=0;
end
diffMatr(i,j)=val;
diffMatr(j,i)=val;
end
end
% observed - expected agreement
do=0; de=0;
for c=1:length(allVals)
for k=c+1:length(allVals)
if strcmp(scale, 'nominal')
do=do+coinMatr(c,k);
de=de+nc(c)*nc(k);
elseif strcmp(scale, 'ordinal')
do=do+coinMatr(c,k)*diffMatr(c,k);
de=de+nc(c)*nc(k)*diffMatr(c,k);
elseif strcmp(scale, 'interval')
do=do+coinMatr(c,k)*(allVals(c)-allVals(k)).^2;
de=de+nc(c)*nc(k)*(allVals(c)-allVals(k)).^2;
else
error('unknown scale: %s', scale);
end
end
end
de=1/(n-1)*de;
alpha=1-do/de;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment