MKAPPA For different numbers of raters

[simkinm2]

Is anyone familiar with how Chen's MKAPPA macro accounts for cases of different rater numbers?  I am calculating the Fleiss kappa for patient charts that were reviewed, and some charts were reviewed by 2 raters while some were reviewed by 3.  My kappas seems too low, and I am wondering if has to do with the way it is treating the "missing" rater observations. For example, I have a variable with 85.7% agreement, 11 charts were reviewed by 2 raters and 10 were reviewed by 3.  The output I get from the Mkappa macro is a kappa of 0.49716, SE of 0.050526, and prob > z of 0.  

Any thoughts or suggestions are greatly appreciated! I've included my code below:

/*CHANGE THE DATASET NAME AND CATEGORY NUMBER TO MATCH VARIABLE*/

%let dloc = H:\WorkinginProgress\kappa ;
%let dset = kappa_first_admit ;
%let Nrater = 3 ;
%let Ncat = 9 ;
%let outfile = output.rtf ;

%display setmacro ;

%macro mkappa;

data temp; set kap.&dset;
subj=_n_;
data temp; set temp;

%do i=1 %to &nrater;
rate=rater&i; rater=&i; output; %end;
run;
proc freq data=temp noprint; tables subj*rate / out=xij ;
run;
data subj; set xij; keep subj;
proc sort data=subj nodupkey out=subj; by subj; run;
proc sql noprint; select count(*) into :tsub from subj;

data temp2; do subj=1 to &tsub; do rate=1 to &ncat; output; end; end; run;
data xij; set xij; drop percent;
proc sort data=xij; by subj rate;
proc sort data=temp2; by subj rate;
data xij; merge xij temp2(in=a); by subj rate; if a;
data xij; set xij; if count=. then count=0; run ;
data xij; set xij; xmx=count*(&nrater-count); run;

proc sort data=xij; by rate;
data xij; set xij; by rate;
retain x xmx2;
if first.rate then do; x=0; xmx2=0; end;
x=x+count; xmx2=xmx2+xmx;
if last.rate then output;
run;
data xij; set xij; keep rate x xmx2 ; run;

data xij; set xij;
p=x/(&tsub*&nrater);
pq=p*(1-P);
pqp=p*(1-p)*(1-2*p);
kj=abs(1 - xmx2/(&tsub*&nrater*(&nrater-1)*pq));
numj=kj*pq;
run;
data xij; set xij; if p=0 then do;
kj=0; numj=0; end; run;

data final; set xij end=end; retain num den pqpsum;
if _n_=1 then do; num=0; den=0; pqpsum=0; end;
num=num+numj;
den=den+pq;
pqpsum=pqpsum+pqp;
if end then output;
run;

data kap.koutput; set final; keep Kappa SE pvalue;
Kappa=num/den;
SE=sqrt(2*(den*den-pqpsum))/(den*sqrt(&tsub*&nrater*(&nrater-1)));
pvalue=1-probnorm(kappa/se);
run;

ods rtf file="&dloc&outfile" ;
options nonumber ;
goptions dev=png target=png htitle=1.5 htext=1.2 ftext=swissb ftitle=swissb
xmax=6 ymax=5 ;
proc print data=kap.koutput noobs label;
label kappa="Kappa" SE="Standard Error" pvalue="Prob>Z";
title "Kappa statistics for &Ncat-category ratings by &Nrater raters";
run;
ods rtf close;

%mend mkappa;

%mkappa;