Now that PROC OPTMODEL is officially FCMP-enabled in SAS/OR 13.1, you can access the matrix functions and subroutines that are supplied in PROC FCMP. See Base SAS(R) 9.4 Procedures Guide, Second Edition for a complete list. For example:
proc fcmp outlib=work.myfuncs.test;
function mydet(x[*,*]);
call det(x, result);
return (result);
endsub;
subroutine mytranspose(x[*,*],y[*,*]);
outargs y;
call transpose(x,y);
endsub;
subroutine myinv(x[*,*],y[*,*]);
outargs y;
call inv(x,y);
endsub;
subroutine mymult(x[*,*],y[*,*],z[*,*]);
outargs z;
call mult(x,y,z);
endsub;
quit;
options cmplib = work.myfuncs;
proc optmodel;
/* matrix declaration */
num mat1 {1..3,1..3} = [0.3, -0.78, -0.82, 0.54, 1.74, 1.2, -1.3, 0.25, 1.49];
print mat1;
/* determinant */
print (mydet(mat1));
/* matrix transpose */
num trans {1..3,1..3};
call mytranspose(mat1, trans);
print trans;
/* matrix inversion */
num inv {1..3,1..3};
call myinv(mat1, inv);
print inv;
/* matrix multiplication */
num mat1_dot_inv {1..3,1..3};
call mymult(mat1, inv, mat1_dot_inv);
print mat1_dot_inv;
quit;
/* maximize determinant */
proc optmodel;
num n = 5;
var x {1..n, 1..n} >= -1 <= 1;
max z = mydet(x);
solve with NLP / ms;
print x;
quit;