Now that PROC OPTMODEL is officially FCMP-enabled in SAS/OR 13.1, you can access the matrix functions and subroutines that are supplied in PROC FCMP.  See Base SAS(R) 9.4 Procedures Guide, Second Edition for a complete list.  For example:
proc fcmp outlib=work.myfuncs.test;
     function mydet(x[*,*]);
          call det(x, result);
          return (result);
      endsub;
      subroutine mytranspose(x[*,*],y[*,*]);
          outargs y;
          call transpose(x,y);
      endsub;
      subroutine myinv(x[*,*],y[*,*]);
          outargs y;
          call inv(x,y);
      endsub;
      subroutine mymult(x[*,*],y[*,*],z[*,*]);
          outargs z;
          call mult(x,y,z);
      endsub;
quit;
options cmplib = work.myfuncs;
proc optmodel;
     /* matrix declaration */
     num mat1 {1..3,1..3} = [0.3, -0.78, -0.82, 0.54, 1.74, 1.2, -1.3, 0.25, 1.49];
     print mat1;
     /* determinant */
     print (mydet(mat1));
     /* matrix transpose */
     num trans {1..3,1..3};
     call mytranspose(mat1, trans);
     print trans;
     /* matrix inversion */
     num inv {1..3,1..3};
     call myinv(mat1, inv);
     print inv;
     /* matrix multiplication */
     num mat1_dot_inv {1..3,1..3};
     call mymult(mat1, inv, mat1_dot_inv);
     print mat1_dot_inv;
quit;
/* maximize determinant */
proc optmodel;
     num n = 5;
     var x {1..n, 1..n} >= -1 <= 1;
     max z = mydet(x);
     solve with NLP / ms;
     print x;
quit;