Hello,
I have the following code which runs well on a small sample and I get the cases and controls dataset (cacoset). However, I am now running it using two very large datasets (cases = 65000 cases) and (controls > 60 million controls) and it is taking forever to run. For example for the past 20 hours it has only read 500 observations from the created "Controls" dataset as part of the matching process.
Any advice?
Thanks,
Shtroumpfette
Code:
options ls=78 ps=60 obs=max;
%Let results = %STR(M:\Case Control\Results);
/*IMPORT REQUIRED DATABASE*/
proc import datafile = 'C:\Users\Shtroump\Final Analysis\controls.csv'
out = pat_sk2;
run;
*copy dataset to keep intact copy;
data pat_sk3;
set pat_sk2;
run;
data Pat_sk3_plusminus;
set pat_sk3;
*figure out start/end of each interval;
index_start = index_date - 30;
index_end = index_date + 30;
run;
/*keep variables of interest for now-*/
data pat_sk4;
set Pat_sk3_plusminus;
keep encounter_id hospital_id age patient_sk sex diabetes_event index_date index_start index_end;
run;
/*SPLIT pat_sk4 elligible cases and & possible controls*/
data case_diab control_diab;
set pat_sk4;
if diabetes_event = 1 then output case_diab;
else output control_diab;
run;
data out1.case_diab (compress=yes);
set case_diab;
run;
data out1.contol_diab (compress=yes);
set control_diab;
run;
data caseset (compress=yes);
set case_diab;
run;
data cerner.controloriginal (compress=yes);
set control_diab;
run; /*save this dataset for future use*/
/*create a copy of controls dataset to use n iterative steps below*/
data contset (compress=yes);
set control_diab;
run;
/*Choose 2 matched controls for each case. Matches subject on gender and age hospital_id and index date.
Unique controls (sampling without replacement) are chosen for each case.
Cases cannot be their own controls*/
proc contents data=caseset out = contents noprint; run;
/*Creates macro variable for total # of cases in case data set*/
%global num_obs;
data _null_;
set contents (obs=1);
call symput('num_obs', trim(left(put(nobs,7.))));
run;
%put _global_;
/*assign case observation number to SETID that will be a unique number defiining each case-control risk set. CACO =1 for all cases*/
data caseset_temp;
set caseset;
setid=_N_;
caco=1;
run;
/*options mprint symbolgen mtrace*/
/*Begin macro MATCHED*/
options mprint;
%macro MATCHED;
%do i=1 %to &num_obs;
%let obsno=&i;
/*Loops interatively through case dataset to select one case at a time, then
evaluates all observations in control datset to select potential controls*/
data contset1;
if _N_=1 then set caseset_temp(firstobs=&obsno obs=&obsno
keep = patient_sk encounter_id diabetes_event age sex hospital_id index_start index_end setid
rename = (patient_sk = case_patient_sk
encounter_id = case_encounter_id
diabetes_event = case_diabetes_event
index_start=case_index_start
index_end = case_index_end
age=case_age
sex=case_sex
hospital_id = case_hospital_id
));
set contset;
if abs((<=case_index_end<=index_date <=case_index_end)
and hospital_id=case_hospital_id
and (case_age-2<=age<=case_age+2)
and sex=case_sex
and patient_sk ne case_patient_sk
)
then do;
choose = ranuni(0);
caco=0; /*CACO = 0 for all controls*/
output contset1; /*output controls meeting matching criteria*/
end;
label case_patient_sk = 'Case: Person ID (DUID+PID)';
run;
proc sort data = contset1; by choose; run;
data contset2(drop=choose);
set contset1(obs=2);
run;
%if &obsno=1 %then %do;
data controls;
set contset2;
run;
%end;
%else %do;
data controls;
set controls contset2;
run;
%end;
%end;
%mend matched;
%matched;
/*appends all cases and selected control datasets together*/
data matchset;
set caseset_temp controls;
by setid;
run;
proc sort data = matchset;
by setid descending caco;
run;
/*assigns unique IDNUM to each ase and control in all risk sets*/
data cacoset;
set matchset;
by setid descending caco;
if first.setid then count = 0;
if caco=1 then idnum =setid*10000;
else if caco ne 1 then do;
count+1;
idnum = (setid*10000)+count;
end;
run;
data less2(keep = setid count);
set cacoset;
by setid;
if last.setid and count<2;
run;
proc freq data =less2;
tables count;
title2 'cases with less than 2 controls after matching';
title3 'Frequency of # of controls';
run;
data less2cont;
merge less2(in=in1 drop=count) cacoset;
by setid;
if ini;
run;
proc print data = less2cont noobs n double;
var case_patient_sk diabetes_event patient_sk caco setid idnum sex age;
title2 'Cases with less than 2 controls after matching';
title3 '';
run;
proc sort data = cacoset out = byidentifier nodupkey;
by patient_sk;
run;
/*creating a final permanent dataset with all case-control risk sets*/
data out1.cacoset;
set cacoset(drop=count);
run;
