@Reeza wrote: 1GB isn't a large data set for SAS in most cases. What makes you think it is?
A hundred files of 1GB is a lot, especially if the application is to work more or less in real time (10 seconds is a lot, 1 min is forever).
@Reeza wrote: Are you trying to get 40 observations or a window of time (30 minutes it seems) relative to the time in table 1? I think its the latter but that isn't clear to me.
40 observation.
data A;
informat datetime datetime21.;
input info $7. datetime Nr ;
format datetime datetime21.;
cards;
GC_5_0 30APR2020:00:30:00 1
GC_5_2 30APR2020:00:40:00 2
GC_50_4 24NOV2010:21:00:00 3
GC_5_1 28OCT2003:13:35:00 4
GC_8_1 01AUG2016:11:20:00 5
GC_5_3 09AUG2016:18:10:00 6
GC_10_2 06SEP2019:06:40:00 7
GC_5_0 30APR2020:00:39:30 8
GC_15_2 27NOV2006:09:45:00 9
GC_5_2 30APR2020:00:17:00 10
run;
data GC_5_0;
set GC_5_0;
nrdb = _n_;
run;
data GC_5_2;
set GC_5_2;
nrdb = _n_;
run;
%macro test();
* for simplicity;
data AA;
set A;
if info = 'GC_5_0' | info = 'GC_5_2';
run;
proc sort data = AA out = AA_count nodupkey;
key info;
run;
* Obs_count variable needed for the loop;
data _NULL_;
if 0 then set AA_count nobs = n;
call symputx('Obs_count',n);
stop;
run;
%put no. of observations = &Obs_count;
* I need to search by number of observations, not by datetime;
data _null_;
if _n_ = 1 then do;
declare hash H (multidata:"y");
H.definekey("info");
H.defineData( "datetime","Nr");
H.definedone();
end;
do until (last.info);
set AA;
by info;
H.add();
end;
H.output (dataset: catx("_", "AA_", info));
H.clear();
run;
%do i = 1 %to &Obs_count %by 1;
* extracting the source file name;
data _NULL_;
set AA_count;
if _n_ = &i then do;
call symputx('info',info);
end;
run;
%put &info;
data aa__&info;
set aa__&info;
if _n_ = 1 then do;
declare hash H1 (dataset:"&info");
H1.definekey("datetime");
H1.definedata("nrdb");
H1.definedone();
end;
if H1.find() = 0 then nrdb = nrdb;
run;
* for simplicity, remove undiscovered data;
data aa__&info;
set aa__&info;
if nrdb ne .;
run;
data wont_&i;
set aa__&info;
if _n_ = 1
then do;
declare hash H (dataset:"&info");
H.definekey("nrdb");
H.definedata("datetime","high_temp","low_temp","end_temp");
H.definedone();
end;
do nrdb = nrdb - 20 to nrdb + 19 by 1;
if H.find() = 0 then output;
high_temp = high_temp;
low_temp = low_temp;
end_temp = end_temp;
end;
run;
* merging partial tables into the final table;
%if &i = 1 %then %do;
data wont;
set wont_&i (drop = nrdb);
run;
%end;
%else %do;
data wont;
set wont wont_&i (drop = nrdb);
run;
%end;
%end;
proc sort data = wont out = wont;
key nr;
run;
%mend test;
%test;
Maybe it can be done faster and smarter.
Best regards
... View more
