Hi,
I am running a batch of Proc SSM for each stock on each day.
I would like to set upper limit to one of the variable. The limit varies from stock-day to stock-day.
I tried to reference a column that changes values depending on the stock-day.
Did not work. Can only put in numerical value.
My instinct is that some sort of macro can load up and reference these values (Upper_limit_Var1 below).
Much appreciated.
Hao
Data:
Code Date Time Log_Prices Upper_limit_Var1
A 1 9:00 0.1 0.9
A 1 9:30 0.12 0.9
......
A 1 4:00 0.11 0.9
A 2 9:00 0.111 0.81
.............
B 1 9:00 0.51 0.6
..............
SAS Code:
ods listing close;
ods output ConvergenceStatus = Convergence;
ods output SmoothedComponent = Smooth;
ods output NamedParameterEstimates = pe;
proc ssm data=SSM1 optimizer(maxiter=300);
by code date;
parms var1 / lower=(1.e-8) upper = (Upper_limit_Var1);
parms var2 / lower=(1.e-8) ;
parms var3 / lower = (-1) upper = (1);
..........
quit;
ods listing;
Hello @Hao3 ,
I read your topic title again and see there's only one limit per by-group.
In that case, it's really easy.
You can use a macro or rather data-driven code generation (i.e. data step with 'file' and 'put' , then %INCLUDE).
The execution might be a little slower since the generated code will do one PROC SSM per by-group (instead of one PROC SSM for all by-groups), but the results will be the results you want to achieve.
I will post an example later today (no time right now) !
Cheers,
Koen
Here's an example :
title; footnote;
title1 'Gas Furnace Data';
title2 '(Box and Jenkins, Series J)';
data seriesj;
input x y @@;
label x = 'Input Gas Rate'
y = 'Output CO2';
x3 = lag3(x);
x4 = lag4(x);
x5 = lag5(x);
obsIndex = _n_;
label obsIndex = 'Observation Index';
cards;
-0.109 53.8 0.000 53.6 0.178 53.5 0.339 53.5
0.373 53.4 0.441 53.1 0.461 52.7 0.348 52.4
0.127 52.2 -0.180 52.0 -0.588 52.0 -1.055 52.4
-1.421 53.0 -1.520 54.0 -1.302 54.9 -0.814 56.0
-0.475 56.8 -0.193 56.8 0.088 56.4 0.435 55.7
0.771 55.0 0.866 54.3 0.875 53.2 0.891 52.3
0.987 51.6 1.263 51.2 1.775 50.8 1.976 50.5
1.934 50.0 1.866 49.2 1.832 48.4 1.767 47.9
1.608 47.6 1.265 47.5 0.790 47.5 0.360 47.6
0.115 48.1 0.088 49.0 0.331 50.0 0.645 51.1
0.960 51.8 1.409 51.9 2.670 51.7 2.834 51.2
2.812 50.0 2.483 48.3 1.929 47.0 1.485 45.8
1.214 45.6 1.239 46.0 1.608 46.9 1.905 47.8
2.023 48.2 1.815 48.3 0.535 47.9 0.122 47.2
0.009 47.2 0.164 48.1 0.671 49.4 1.019 50.6
1.146 51.5 1.155 51.6 1.112 51.2 1.121 50.5
1.223 50.1 1.257 49.8 1.157 49.6 0.913 49.4
0.620 49.3 0.255 49.2 -0.280 49.3 -1.080 49.7
-1.551 50.3 -1.799 51.3 -1.825 52.8 -1.456 54.4
-0.944 56.0 -0.570 56.9 -0.431 57.5 -0.577 57.3
-0.960 56.6 -1.616 56.0 -1.875 55.4 -1.891 55.4
-1.746 56.4 -1.474 57.2 -1.201 58.0 -0.927 58.4
-0.524 58.4 0.040 58.1 0.788 57.7 0.943 57.0
0.930 56.0 1.006 54.7 1.137 53.2 1.198 52.1
1.054 51.6 0.595 51.0 -0.080 50.5 -0.314 50.4
-0.288 51.0 -0.153 51.8 -0.109 52.4 -0.187 53.0
-0.255 53.4 -0.229 53.6 -0.007 53.7 0.254 53.8
0.330 53.8 0.102 53.8 -0.423 53.3 -1.139 53.0
-2.275 52.9 -2.594 53.4 -2.716 54.6 -2.510 56.4
-1.790 58.0 -1.346 59.4 -1.081 60.2 -0.910 60.0
-0.876 59.4 -0.885 58.4 -0.800 57.6 -0.544 56.9
-0.416 56.4 -0.271 56.0 0.000 55.7 0.403 55.3
0.841 55.0 1.285 54.4 1.607 53.7 1.746 52.8
1.683 51.6 1.485 50.6 0.993 49.4 0.648 48.8
0.577 48.5 0.577 48.7 0.632 49.2 0.747 49.8
0.900 50.4 0.993 50.7 0.968 50.9 0.790 50.7
0.399 50.5 -0.161 50.4 -0.553 50.2 -0.603 50.4
-0.424 51.2 -0.194 52.3 -0.049 53.2 0.060 53.9
0.161 54.1 0.301 54.0 0.517 53.6 0.566 53.2
0.560 53.0 0.573 52.8 0.592 52.3 0.671 51.9
0.933 51.6 1.337 51.6 1.460 51.4 1.353 51.2
0.772 50.7 0.218 50.0 -0.237 49.4 -0.714 49.3
-1.099 49.7 -1.269 50.6 -1.175 51.8 -0.676 53.0
0.033 54.0 0.556 55.3 0.643 55.9 0.484 55.9
0.109 54.6 -0.310 53.5 -0.697 52.4 -1.047 52.1
-1.218 52.3 -1.183 53.0 -0.873 53.8 -0.336 54.6
0.063 55.4 0.084 55.9 0.000 55.9 0.001 55.2
0.209 54.4 0.556 53.7 0.782 53.6 0.858 53.6
0.918 53.2 0.862 52.5 0.416 52.0 -0.336 51.4
-0.959 51.0 -1.813 50.9 -2.378 52.4 -2.499 53.5
-2.473 55.6 -2.330 58.0 -2.053 59.5 -1.739 60.0
-1.261 60.4 -0.569 60.5 -0.137 60.2 -0.024 59.7
-0.050 59.0 -0.135 57.6 -0.276 56.4 -0.534 55.2
-0.871 54.5 -1.243 54.1 -1.439 54.1 -1.422 54.4
-1.175 55.5 -0.813 56.2 -0.634 57.0 -0.582 57.3
-0.625 57.4 -0.713 57.0 -0.848 56.4 -1.039 55.9
-1.346 55.5 -1.628 55.3 -1.619 55.2 -1.149 55.4
-0.488 56.0 -0.160 56.5 -0.007 57.1 -0.092 57.3
-0.620 56.8 -1.086 55.6 -1.525 55.0 -1.858 54.1
-2.029 54.3 -2.024 55.3 -1.961 56.4 -1.952 57.2
-1.794 57.8 -1.302 58.3 -1.030 58.6 -0.918 58.8
-0.798 58.8 -0.867 58.6 -1.047 58.0 -1.123 57.4
-0.876 57.0 -0.395 56.4 0.185 56.3 0.662 56.4
0.709 56.4 0.605 56.0 0.501 55.2 0.603 54.0
0.943 53.0 1.223 52.0 1.249 51.6 0.824 51.6
0.102 51.1 0.025 50.4 0.382 50.0 0.922 50.0
1.032 52.0 0.866 54.0 0.527 55.1 0.093 54.5
-0.458 52.8 -0.748 51.4 -0.947 50.8 -1.029 51.2
-0.928 52.0 -0.645 52.8 -0.424 53.8 -0.276 54.5
-0.158 54.9 -0.033 54.9 0.102 54.8 0.251 54.4
0.280 53.7 0.000 53.3 -0.493 52.8 -0.759 52.6
-0.824 52.6 -0.740 53.0 -0.528 54.3 -0.204 56.0
0.034 57.0 0.204 58.0 0.253 58.6 0.195 58.5
0.131 58.3 0.017 57.8 -0.182 57.3 -0.262 57.0
;
run;
data Seriesj_exp;
set Seriesj(firstobs=6);
if mod(_N_,2) =0 then do; grpby = 'grp1'; delta_upper_limit=0.40; end;
else if mod(_N_,2)^=0 then do; grpby = 'grp2'; delta_upper_limit=0.30; end;
else do; grpby = 'grp3'; delta_upper_limit=0.20; end;
run;
proc sort data=Seriesj_exp; by grpby obsIndex; run;
PROC FREQ data=Seriesj_exp;
tables grpby * delta_upper_limit / list out=work.delta_upper_limit;
run;
filename deltaul clear;
filename deltaul temp;
data _NULL_;
set work.delta_upper_limit;
file deltaul;
PUT 'title "A Transfer Function Model for the Gas Furnace Data (group=' grpby +(-1) ')";';
PUT 'proc ssm data=Seriesj_exp;';
PUT ' where grpby="' grpby +(-1) '";';
PUT ' id obsIndex;';
PUT ' parms delta /lower=-0.9999 upper=' delta_upper_limit ';';
PUT ' state tfstate(1) T(g)=(delta) W(g)=(x3 x4 x5) a1(1) checkbreak;';
PUT ' comp tfinput = tfstate[1];';
PUT ' trend ar2(arma(p=2)) ;';
PUT ' intercept = 1;';
PUT ' model y = intercept tfinput ar2 ;';
PUT ' eval modelCurve = intercept + tfinput;';
PUT ' forecast out=For;';
PUT 'run;';
run;
title; footnote;
%INCLUDE deltaul / source2;
title; footnote;
filename deltaul clear;
/* end of program */
Koen
@Hao3 wrote:
1. I rely on the ODS output function to collect the fitted estimates such as the state time series and parameter estimates. The HTML output crashes always with that many by groups.
I wonder whether/where those were incorporate in your code?
2. The by group seems to be your 'grpby' variable. Does that mean I need to flatten my double by group (stock and day) into just one?
1. Your HTML becomes too big.
Use ODS EXCLUDE or ODS SELECT statement to ban / select the specified output objects.
Size of output is reduced that way!
2. No need to flatten / collapse into one variable!
In the WHERE clause , you can specify 2nd group-var (and value) after 'AND' operator.
Or you can work with 'WHERE ALSO'.
You need an example?
Koen
Sorry.
I forgot about this.
Will post some code tomorrow.
Br,
Koen
Hello,
I updated my previous program such that it works with
I want to emphasize I have NOT done it through a macro.
It is using data driven code generation with :
The code is generated in a data _NULL_ step , based on the incoming observations from the dataset on the SET statement. After generation of the temporary file in work space , the file is %INCLUDE-d (submitted).
See the LOG for the generated code.
There are as many PROC SSM 's generated as there are grpby1 * grpby2 combinations ( = # observations in the dataset on the SET statement)!
PROC DATASETS LIBRARY=WORK NoList NoDetails memtype=DATA;
delete seriesj: ; run;
delete count_grpby1_grpby2; run;
delete for ; run;
delete DELTA_UPPER_LIMIT ; run;
QUIT;
title; footnote;
title1 'Gas Furnace Data';
title2 '(Box and Jenkins, Series J)';
data seriesj;
input x y @@;
label x = 'Input Gas Rate'
y = 'Output CO2';
x3 = lag3(x);
x4 = lag4(x);
x5 = lag5(x);
obsIndex = _n_;
label obsIndex = 'Observation Index';
cards;
-0.109 53.8 0.000 53.6 0.178 53.5 0.339 53.5
0.373 53.4 0.441 53.1 0.461 52.7 0.348 52.4
0.127 52.2 -0.180 52.0 -0.588 52.0 -1.055 52.4
-1.421 53.0 -1.520 54.0 -1.302 54.9 -0.814 56.0
-0.475 56.8 -0.193 56.8 0.088 56.4 0.435 55.7
0.771 55.0 0.866 54.3 0.875 53.2 0.891 52.3
0.987 51.6 1.263 51.2 1.775 50.8 1.976 50.5
1.934 50.0 1.866 49.2 1.832 48.4 1.767 47.9
1.608 47.6 1.265 47.5 0.790 47.5 0.360 47.6
0.115 48.1 0.088 49.0 0.331 50.0 0.645 51.1
0.960 51.8 1.409 51.9 2.670 51.7 2.834 51.2
2.812 50.0 2.483 48.3 1.929 47.0 1.485 45.8
1.214 45.6 1.239 46.0 1.608 46.9 1.905 47.8
2.023 48.2 1.815 48.3 0.535 47.9 0.122 47.2
0.009 47.2 0.164 48.1 0.671 49.4 1.019 50.6
1.146 51.5 1.155 51.6 1.112 51.2 1.121 50.5
1.223 50.1 1.257 49.8 1.157 49.6 0.913 49.4
0.620 49.3 0.255 49.2 -0.280 49.3 -1.080 49.7
-1.551 50.3 -1.799 51.3 -1.825 52.8 -1.456 54.4
-0.944 56.0 -0.570 56.9 -0.431 57.5 -0.577 57.3
-0.960 56.6 -1.616 56.0 -1.875 55.4 -1.891 55.4
-1.746 56.4 -1.474 57.2 -1.201 58.0 -0.927 58.4
-0.524 58.4 0.040 58.1 0.788 57.7 0.943 57.0
0.930 56.0 1.006 54.7 1.137 53.2 1.198 52.1
1.054 51.6 0.595 51.0 -0.080 50.5 -0.314 50.4
-0.288 51.0 -0.153 51.8 -0.109 52.4 -0.187 53.0
-0.255 53.4 -0.229 53.6 -0.007 53.7 0.254 53.8
0.330 53.8 0.102 53.8 -0.423 53.3 -1.139 53.0
-2.275 52.9 -2.594 53.4 -2.716 54.6 -2.510 56.4
-1.790 58.0 -1.346 59.4 -1.081 60.2 -0.910 60.0
-0.876 59.4 -0.885 58.4 -0.800 57.6 -0.544 56.9
-0.416 56.4 -0.271 56.0 0.000 55.7 0.403 55.3
0.841 55.0 1.285 54.4 1.607 53.7 1.746 52.8
1.683 51.6 1.485 50.6 0.993 49.4 0.648 48.8
0.577 48.5 0.577 48.7 0.632 49.2 0.747 49.8
0.900 50.4 0.993 50.7 0.968 50.9 0.790 50.7
0.399 50.5 -0.161 50.4 -0.553 50.2 -0.603 50.4
-0.424 51.2 -0.194 52.3 -0.049 53.2 0.060 53.9
0.161 54.1 0.301 54.0 0.517 53.6 0.566 53.2
0.560 53.0 0.573 52.8 0.592 52.3 0.671 51.9
0.933 51.6 1.337 51.6 1.460 51.4 1.353 51.2
0.772 50.7 0.218 50.0 -0.237 49.4 -0.714 49.3
-1.099 49.7 -1.269 50.6 -1.175 51.8 -0.676 53.0
0.033 54.0 0.556 55.3 0.643 55.9 0.484 55.9
0.109 54.6 -0.310 53.5 -0.697 52.4 -1.047 52.1
-1.218 52.3 -1.183 53.0 -0.873 53.8 -0.336 54.6
0.063 55.4 0.084 55.9 0.000 55.9 0.001 55.2
0.209 54.4 0.556 53.7 0.782 53.6 0.858 53.6
0.918 53.2 0.862 52.5 0.416 52.0 -0.336 51.4
-0.959 51.0 -1.813 50.9 -2.378 52.4 -2.499 53.5
-2.473 55.6 -2.330 58.0 -2.053 59.5 -1.739 60.0
-1.261 60.4 -0.569 60.5 -0.137 60.2 -0.024 59.7
-0.050 59.0 -0.135 57.6 -0.276 56.4 -0.534 55.2
-0.871 54.5 -1.243 54.1 -1.439 54.1 -1.422 54.4
-1.175 55.5 -0.813 56.2 -0.634 57.0 -0.582 57.3
-0.625 57.4 -0.713 57.0 -0.848 56.4 -1.039 55.9
-1.346 55.5 -1.628 55.3 -1.619 55.2 -1.149 55.4
-0.488 56.0 -0.160 56.5 -0.007 57.1 -0.092 57.3
-0.620 56.8 -1.086 55.6 -1.525 55.0 -1.858 54.1
-2.029 54.3 -2.024 55.3 -1.961 56.4 -1.952 57.2
-1.794 57.8 -1.302 58.3 -1.030 58.6 -0.918 58.8
-0.798 58.8 -0.867 58.6 -1.047 58.0 -1.123 57.4
-0.876 57.0 -0.395 56.4 0.185 56.3 0.662 56.4
0.709 56.4 0.605 56.0 0.501 55.2 0.603 54.0
0.943 53.0 1.223 52.0 1.249 51.6 0.824 51.6
0.102 51.1 0.025 50.4 0.382 50.0 0.922 50.0
1.032 52.0 0.866 54.0 0.527 55.1 0.093 54.5
-0.458 52.8 -0.748 51.4 -0.947 50.8 -1.029 51.2
-0.928 52.0 -0.645 52.8 -0.424 53.8 -0.276 54.5
-0.158 54.9 -0.033 54.9 0.102 54.8 0.251 54.4
0.280 53.7 0.000 53.3 -0.493 52.8 -0.759 52.6
-0.824 52.6 -0.740 53.0 -0.528 54.3 -0.204 56.0
0.034 57.0 0.204 58.0 0.253 58.6 0.195 58.5
0.131 58.3 0.017 57.8 -0.182 57.3 -0.262 57.0
;
run;
/* As nothing can be assumed about the initial condition of this state equation */
/* , it is taken to be diffuse (as signified by the A1 option). */
data Seriesj_exp;
set Seriesj(firstobs=6);
if mod(_N_,2) =0 then do; grpby1 = 'grpby1_1'; end;
else if mod(_N_,2)^=0 then do; grpby1 = 'grpby1_2'; end;
else do; grpby1 = 'grpby1_3'; end;
if mod(_N_,3) =0 then do; grpby2 = 'grpby2_1'; end;
else if mod(_N_,3) =1 then do; grpby2 = 'grpby2_2'; end;
else if mod(_N_,3) =2 then do; grpby2 = 'grpby2_3'; end;
else do; grpby2 = 'grpby2_4'; end;
if (grpby1 = 'grpby1_1' AND grpby2 = 'grpby2_1') then do; delta_upper_limit=0.40; initial_condition_state='a1(1)'; end;
else if (grpby1 = 'grpby1_1' AND grpby2 = 'grpby2_2') then do; delta_upper_limit=0.30; initial_condition_state='a1(1)'; end;
else if (grpby1 = 'grpby1_1' AND grpby2 = 'grpby2_3') then do; delta_upper_limit=0.20; initial_condition_state='a1(1)'; end;
else if (grpby1 = 'grpby1_2' AND grpby2 = 'grpby2_1') then do; delta_upper_limit=0.40; initial_condition_state='a1(1)'; end;
else if (grpby1 = 'grpby1_2' AND grpby2 = 'grpby2_2') then do; delta_upper_limit=0.30; initial_condition_state='a1(1)'; end;
else if (grpby1 = 'grpby1_2' AND grpby2 = 'grpby2_3') then do; delta_upper_limit=0.30; initial_condition_state='a1(1)'; end;
else do; delta_upper_limit=0.20; initial_condition_state='a1(1)'; end;
run;
PROC FREQ data=Seriesj_exp;
tables grpby1 * grpby2 / list out=work.count_grpby1_grpby2;
run;
proc sort data=Seriesj_exp; by grpby1 grpby2 obsIndex; run;
PROC FREQ data=Seriesj_exp;
tables grpby1 * grpby2 * delta_upper_limit * initial_condition_state / list out=work.delta_upper_limit;
run;
filename deltaul clear;
filename deltaul temp;
data _NULL_;
set work.delta_upper_limit;
file deltaul;
PUT 'title1 "A Transfer Function Model for the Gas Furnace Data";';
PUT 'title2 "Gas Furnace Data (grpby1 group_1 =' grpby2 +(-1) ')";';
PUT 'title3 "Gas Furnace Data (grpby2 group_2 =' grpby2 +(-1) ')";';
PUT 'proc ssm data=Seriesj_exp;';
PUT ' where grpby1="' grpby1 +(-1) '";';
PUT ' where ALSO grpby2="' grpby2 +(-1) '";';
PUT ' id obsIndex;';
PUT ' parms delta /lower=-0.9999 upper=' delta_upper_limit ';';
PUT ' state tfstate(1) T(g)=(delta) W(g)=(x3 x4 x5) ' initial_condition_state ' checkbreak;';
PUT ' comp tfinput = tfstate[1];';
PUT ' trend ar2(arma(p=2)) ;';
PUT ' intercept = 1;';
PUT ' model y = intercept tfinput ar2 ;';
PUT ' eval modelCurve = intercept + tfinput;';
PUT ' forecast out=For;';
PUT 'run;';
run;
title; footnote;
%INCLUDE deltaul / source2;
title; footnote;
filename deltaul clear;
/* end of program */
Cheers,
Koen
Many thanks Koen,
Unfortunately, after some digging I still am missing a final step: collecting and collating the parameter estimates and convergence statistics.
I did some research but each answer comes with three more questions. I am writing Greek without understanding a single word in Greek.
The closest I get is with the following modification to your code:
I managed to generate parameter estimates with names indexed by the two by groups.
But the collating steps misses two features:
1. The bygroup variable cannot be populated from the PE dataset.
2. The length of PE dataset is having varying length. Never the correct length (6 = 2grpby1*3grpby2).
/*Mod: turning off ODS output to not crash*/
ods graphics off;
ods exclude all;
ods noresults;
/*Koen's main code modified lines are marked as 'mod'*/
filename deltaul clear;
filename deltaul temp;
data _NULL_;
set work.delta_upper_limit;
file deltaul;
PUT 'title1 "A Transfer Function Model for the Gas Furnace Data";';
PUT 'title2 "Gas Furnace Data (grpby1 group_1 =' grpby1 +(-1) ')";'; /*Mod: I think you mean 'grpby1' here*/
PUT 'title3 "Gas Furnace Data (grpby2 group_2 =' grpby2 +(-1) ')";';
PUT 'ods output ConvergenceStatus = Convergence' grpby1 +(-1)grpby2 +(-1) ';'; /*Mod: collecting convergence statistics from each by group*/
PUT 'ods output NamedParameterEstimates = pe' grpby1 +(-1)grpby2 +(-1) ';'; /*Mod: collecting Parameter estimates from each by group*/
PUT 'proc ssm data=Seriesj_exp;';
PUT ' where grpby1="' grpby1 +(-1) '";';
PUT ' where ALSO grpby2="' grpby2 +(-1) '";';
PUT ' id obsIndex;';
PUT ' parms delta /lower=-0.9999 upper=' delta_upper_limit ';';
PUT ' state tfstate(1) T(g)=(delta) W(g)=(x3 x4 x5) ' initial_condition_state ' checkbreak;';
PUT ' comp tfinput = tfstate[1];';
PUT ' trend ar2(arma(p=2)) ;';
PUT ' intercept = 1;';
PUT ' model y = intercept tfinput ar2 ;';
PUT ' eval modelCurve = intercept + tfinput;';
PUT ' forecast out=For;';
PUT 'run;';
/*Mod: attempting to collect parameter estimates (PE) from the double by group and add a flag (bygroup)*/
PUT 'data pe;';
PUT 'set pe pe' grpby1 +(-1)grpby2 +(-1) ';';
PUT 'bygroup = by' grpby1 +(-1)grpby2 +(-1) ';';
PUT 'run;';
run;
title; footnote;
%INCLUDE deltaul / source2;
title; footnote;
filename deltaul clear;
/*Mod: turning ODS output back on*/
ods graphics on;
ods exclude none;
ods results;
/* end of program */
I have added a 'mod' to wherever I made the modfiication.
Would you be able to nudge it into place?
Thanks again.
Hao
Hi,
Easy to do ...
I will nudge it into place tomorrow (25-FEB).
I would not say you are writing Greek without understanding a single word in Greek.
You are rather writing Latin without understanding a single word in Latin.
(SAS is using the Latin alphabet , not the Greek one) 😉
BR,
Koen
Registration is now open for SAS Innovate 2025 , our biggest and most exciting global event of the year! Join us in Orlando, FL, May 6-9.
Sign up by Dec. 31 to get the 2024 rate of just $495.
Register now!
Learn how to run multiple linear regression models with and without interactions, presented by SAS user Alex Chaplin.
Find more tutorials on the SAS Users YouTube channel.