%macro MultAUC(version, data=_last_, response=, prefix=);
%let time = %sysfunc(datetime());
%if &data=_last_ %then %let data=&syslast;

/*  Define macro for checking inputs
/---------------------------------------------------------------------*/
%macro existchk(data=, var=, dmsg=e, vmsg=e);
   %global status; %let status=ok;
   %if &dmsg=e %then %let dmsg=ERROR;
   %else %if &dmsg=w %then %let dmsg=WARNING;
   %else %let dmsg=NOTE;
   %if &vmsg=e %then %let vmsg=ERROR;
   %else %if &vmsg=w %then %let vmsg=WARNING;
   %else %let vmsg=NOTE;
   %if &data ne %then %do;
     %if %sysfunc(exist(&data)) ne 1 %then %do;
       %put &dmsg: Data set %upcase(&data) not found.;
       %let status=nodata;
     %end;
     %else %if &var ne %then %do;
       %let dsid=%sysfunc(open(&data));
       %if &dsid %then %do;
         %let i=1;
         %do %while (%scan(&var,&i) ne %str() );
            %let var&i=%scan(&var,&i);
            %if %sysfunc(varnum(&dsid,&&var&i))=0 %then %do;
     %put &vmsg: Variable %upcase(&&var&i) not found in data %upcase(&data).;
              %let status=novar;
            %end;
            %let i=%eval(&i+1);
         %end;
         %let rc=%sysfunc(close(&dsid));
       %end;
       %else %put ERROR: Could not open data set &data.;
     %end;
   %end;
   %else %do;
     %put &dmsg: Data set not specified.;
     %let status=nodata;
   %end;   
%mend;

/*  Version and debug options
/---------------------------------------------------------------------*/
%let _version = 1.3;
%if &version ne %then %put NOTE: &sysmacroname macro Version &_version..;
%let _opts = %sysfunc(getoption(notes));
%if %index(%upcase(&version),DEBUG) %then %do;
  options notes mprint
    %if %index(%upcase(&version),DEBUG2) %then mlogic symbolgen;
  ;
  ods select all;
  %put _user_;
%end;
%else %do;
  options nonotes nomprint nomlogic nosymbolgen;
  ods exclude all;
%end;

/* Check for newer version 
/---------------------------------------------------------------------*/
%let _notfound=0;
filename _ver url 'http://ftp.sas.com/techsup/download/stat/versions.dat' 
         termstr=crlf;
data _null_;
  infile _ver end=_eof;
  input name:$15. ver;
  if upcase(name)="&sysmacroname" then do;
    call symput("_newver",ver); stop;
  end;
  if _eof then call symput("_notfound",1);
  run;
options notes;
%if &syserr ne 0 or &_notfound=1 %then
  %put NOTE: Unable to check for newer version of &sysmacroname macro.;
%else %if %sysevalf(&_newver > &_version) %then %do;
  %put NOTE: A newer version of the &sysmacroname macro is available at;
  %put NOTE- this location: http://support.sas.com/ ;
%end;
%if %index(%upcase(&version),DEBUG)=0 %then options nonotes;;

/* Check required data= 
/---------------------------------------------------------------------*/
%if &response= %then %let response=_FROM_;
%if %upcase(&response)=_FROM_ %then %let prefix=IP_;

%existchk(data=&data, var=&response)
%if &status=nodata or &status=novar %then %goto exit;

%let needpfx=0;
%let dsid=%sysfunc(open(&data));
%if &dsid %then %do;
  %let varnum=%sysfunc(varnum(&dsid,&response));
  %if %sysfunc(vartype(&dsid,&varnum))=N and &prefix= %then %do;
    %put ERROR: PREFIX= is required when RESPONSE= is numeric.;
    %let needpfx=1;
  %end;
  %let rc=%sysfunc(close(&dsid));
  %if &needpfx %then %goto exit;
%end;

/* Create character response, get levels, and generate all pairs 
/---------------------------------------------------------------------*/
proc datasets nolist nowarn;
  delete _Aij; 
  run; quit;
data _data;
  set &data;
  length _response $ 29;
  if not missing(&response);
  _response=cats("&prefix",translate(catt(&response),"______________________________"," &%()*-=+^$#@!`~[]{}\|/?,<.>;:")); 
  run;
proc freq data=_data; 
  table _response / out=_rvals; 
  run;
  
/* Check that predicted probability variables exist with inferred names  
/---------------------------------------------------------------------*/
%if &response ne _from_ %then %do; 
   %let missppvar=0;
   data _null_;
     set _rvals;
     dsid=open("_data");
     if varnum(dsid,_response)=0 then do;
       put "ERROR: Predicted probability variable " _response "not found."; 
       call symput('missppvar','1');
     end;
     rc=close(dsid);
     run;
   %if &missppvar %then %goto exit;
%end;

/* Generate all pairs 
/---------------------------------------------------------------------*/
proc transpose data=_rvals out=_rvals; 
  var _response; 
  run;
data _pairs;
  set _rvals;
  length L1 L2 $ 29;
  array col (*) col:;
  nlvls=dim(col);
  cmb=comb(nlvls,2);
  call symput('cmb',cats(cmb));
  do j=1 to cmb;
     call allcomb(j, 2, of col[*]);
     L1=cats(col1||''); L2=cats(col2||'');
     if L1>L2 then do; 
       L1tmp=L1; L1=L2; L2=L1tmp;
     end;
     keep L1 L2; output;
  end;
  run;

/* Define & run macro to compute pairwise Aij
/---------------------------------------------------------------------*/
%macro Aij;
%do k=1 %to &cmb;
data _null_; 
  cmb=&k;
  set _pairs point=cmb;
  call symput("i",cats(L1));
  call symput("j",cats(L2));
  stop;
  run;
proc npar1way data=_data wilcoxon; 
  where _response in ("&i","&j");
  class _response; 
  var &i &j;
  ods output wilcoxonscores=_sumranks; 
  run;
%if &syserr %then %let status=np1err;  
data _pairAij; 
  set _sumranks end=eof;
  length Level1 Level2 $29;
  retain num logn;
  by variable;
  if first.variable then logn=0;
  logn=logn+log(n);
  if upcase(class)=upcase(variable) then num=sumofscores-n*(n+1)/2;
  if last.variable then Aij+num/(2*exp(logn));
  keep Level: Aij;
  if eof then do;
    Level1="&i"; Level2="&j";
    output;
  end;
  run;
%if &syserr %then %let status=aijerr;  
proc append base=_Aij data=_pairAij; 
  run;
%end;
%mend Aij;

%Aij
%if &status=np1err or &status=aijerr %then %goto exit;  

/* Compute & display multi-level AUC as mean of the Aij 
/---------------------------------------------------------------------*/
proc summary data=_Aij;
  var Aij;
  output out=MultAUC(keep=AUC) mean=AUC;
  run;
ods select all;
proc print data=MultAUC noobs;
  title "Multi-level AUC";
  run;
data PairAUC;
  set _Aij(rename=(Aij=AUC));
  Level1=tranwrd(Level1,"&prefix","");
  Level2=tranwrd(Level2,"&prefix","");
  run;
proc print data=PairAUC;
  id Level:; var AUC;
  title "Pairwise AUC";
  run;

/* Clean up
/---------------------------------------------------------------------*/
%exit:
  %if %index(%upcase(&version),DEBUG)=0 %then %do;  
     proc datasets nolist nowarn;
        delete _data _rvals _sumranks _pairs _pairaij _Aij;
        run; quit;
  %end;
  %if %index(%upcase(&version),DEBUG) %then %do;
    options nomprint nomlogic nosymbolgen;
    %put _user_;
  %end;
  title;
  %let status=;
  ods select all;
  options &_opts;
  %let time = %sysfunc(round(%sysevalf(%sysfunc(datetime()) - &time), 0.01));
  %put NOTE: The &sysmacroname macro used &time seconds.;
%mend MultAUC;