Copy or Move the file based on file name

[Kurt_Bremser]

Run this:

filename source "&inpath";

data files ;
  length msg filename $256 ;
  did=dopen("source");
  if did<=0 then do;
      msg=sysmsg();
      put msg;
      stop;
  end;
  do id=1 to dnum(did);
    filename=dread(did,id);
    output;
  end;
run;

and look at the contents of dataset FILES. You will find that the value is not caught by any of your conditions, so SOURCE and TARGET stay empty.

[David_Billa]

@Kurt_Bremser - I've corrected the condition in 'else if' statements but still source and target value is empty in 'files' dataset. Any idea why still it's empty as the filename and macro variable value is statisfying the below else if?

File Name is IFR_ITM_TRN_2_6010_1_20201009T075212_copied.csv

Macro variable value which I passed in the program is,

%let function=NL;

/*added*/
      else if  index(lowcase(filename),'_copied.csv ') and "&function" EQ 'NL' then do;
      source="&inpath" ||"/"|| filename;
      target="&path2"||"/"||filename;
      end;
      /*added*/
      else if  index(lowcase(filename),'_copied.csv ') and "&function" EQ 'LH' then do;
      source="&inpath" ||"/"|| filename;
      target="&path1"||"/"||filename;
      end;

[Kurt_Bremser]

If your filename is actually that, source and target CANNOT be empty:

%let function=NL;

%let inpath=/var/sasdata/GTM;
%let path1=/var/sasdata/LH;
%let path2=/var/sasdata/NL_GTM;

data test;
filename = "IFR_ITM_TRN_2_6010_1_20201009T075212_copied.csv";
    if scan(lowcase(filename),-1,'.')='csv' and not index(lowcase(filename),'_copied.csv ') then do;
      source="&inpath" ||"/"|| filename;
      if indexw(filename,'5601','_') or indexw(filename,'6010','_') or indexw(filename,'6020','_') then do;
        target="&path1"||substr(filename,1,length(filename)-4)||'_copied.csv';
      end;
      else if indexw(filename,'0169','_') or indexw(filename,'1070','_') or indexw(filename,'0417','_') then do;
        target="&path2"||"/"||filename;
      end;
      /*added*/
      else if scan(lowcase(filename),-1,'.')='csv' and  index(lowcase(filename),'_copied.csv ') and "&function" EQ 'NL'
then do;
      source="&inpath" ||"/"|| filename;
      target="&path2"||"/"||filename;
      end;
                                               
      /*added*/
      else if scan(lowcase(filename),-1,'.')='csv' and  index(lowcase(filename),'_copied.csv ') and "&function" EQ 'LH'
then do;
      source="&inpath" ||"/"|| filename;
      target="&path1"||"/"||filename;
      end;
      else target="&path1"||"/"||filename;
    end;
    output;
put source= target=;
run;

Log excerpt:

 105        put source= target=;
 106        run;
 
 source=/var/sasdata/GTM/IFR_ITM_TRN_2_6010_1_20201009T075212_copied.csv
 target=/var/sasdata/LHIFR_ITM_TRN_2_6010_1_20201009T075212_copied_copied.csv

Run the data step I gave you earlier. Then take the dataset, and apply your conditions. If that does not do it, create your dataset as a data step with datalines (you can use the macro from my footnotes for that), and rerun the code. If that still does result in empty variables, post both steps here.

[David_Billa]

@Kurt_Bremser @Tom What will be best idea to  remove the string '_copied' from the filename in the target folder?

[David_Billa]

@Kurt_Bremser Still SOURCE and TARGET value is empty even after simplying the last 'else if'. My code is,

%let function=NL;   

%let inpath=/var/sasdata/GTM;
%let path1=/var/sasdata/LH;
%let path2=/var/sasdata/NL_GTM;

filename source "&inpath";

data files ;
  length msg filename $256 ;
  did=dopen("source");
  if did<=0 then do;
      msg=sysmsg();
      put msg;
      stop;
  end;
  do id=1 to dnum(did);
    filename=dread(did,id);
    output;
  end;
  did=dclose(did);
run;

data files_;
  set files;
    if scan(lowcase(filename),-1,'.')='csv' and not index(lowcase(filename),'_copied.csv ') then do;
      source="&inpath" ||"/"|| filename;
      if indexw(filename,'5601','_') or indexw(filename,'6010','_') or indexw(filename,'6020','_') then do;
        target="&path1"||substr(filename,1,length(filename)-4)||'_copied.csv';
      end;
      else if indexw(filename,'0169','_') or indexw(filename,'1070','_') or indexw(filename,'0417','_') then do;
        target="&path2"||"/"||filename;
      end;
      /*added*/
      else if  index(lowcase(filename),'_copied.csv ') ne 0 then do;
      source="&inpath" ||"/"|| filename;
      target="&path2"||"/"||filename;
      end;

      else target="&path1"||"/"||filename;
    end;
    output;
    put source= target=;
run;

Last else if should have executed for my file as it has the string '_copied'.

Log:

26         %let function=NL;
27         
28         %let inpath=/var/sasdata/GTM;
29         %let path1=/var/sasdata/LH;
30         %let path2=/var/sasdata/NL_GTM;
31         
32         filename source "&inpath";
33         
34         
35         
36         
37         data files ;
38           length msg filename $256 ;
39           did=dopen("source");
40           if did<=0 then do;
41               msg=sysmsg();
42               put msg;
43               stop;
44           end;
45           do id=1 to dnum(did);
46             filename=dread(did,id);
47             output;
48           end;
49           did=dclose(did);
50         run;

NOTE: The data set WORK.FILES has 1 observations and 4 variables.
51         
52         data files_;
53           set files;
54             if scan(lowcase(filename),-1,'.')='csv' and not index(lowcase(filename),'_copied.csv ') then do;
55               source="&inpath" ||"/"|| filename;
56               if indexw(filename,'5601','_') or indexw(filename,'6010','_') or indexw(filename,'6020','_') then do;
57                 target="&path1"||substr(filename,1,length(filename)-4)||'_copied.csv';
58               end;
59               else if indexw(filename,'0169','_') or indexw(filename,'1070','_') or indexw(filename,'0417','_') then do;
60                 target="&path2"||"/"||filename;
61               end;
62               /*added*/
63               else if  index(lowcase(filename),'_copied.csv ') ne 0 then do;
64               source="&inpath" ||"/"|| filename;
65               target="&path2"||"/"||filename;
66               end;
67               /*added*/
68         /*      else if  index(lowcase(filename),'_copied.csv ')=0  then do;*/
69         /*      source="&inpath" ||"/"|| filename;*/
70         /*      target="&path1"||"/"||filename;*/
71         /*      end;*/
72               else target="&path1"||"/"||filename;
73             end;
74             output;
75             put source= target=;
76         /*  end;*/
77         /*  did=dclose(did);*/
78         /*  drop did msg;*/
79         run;

SOURCE=  TARGET= 
NOTE: There were 1 observations read from the data set WORK.FILES.
NOTE: The data set WORK.FILES_ has 1 observations and 6 variables.

[Kurt_Bremser]

Look at the contents of dataset files. If in doubt, use the data2datastep macro to convert the dataset to a data step, and post that here; or post the dataset as attachment.

[Tom]

Fix the logic of the step.  

If you are getting empty values for SOURCE and TARGET then non of the conditions in your logic was met.

The original program was going to ignore the filenames with _copied.csv at the end.

It might be easier to see the issues if you separate the steps to the testing and the logic to generate the different target names.

So create a series of variables that indicate the value of the various tests. For example something like this:

csvfile = lowcase(scan(filename,-1,'.')) = 'csv';
copied = index(lowcase(filename),'_copied') ;
list1 = indexw(filename,'5901','_') or .... ;
list2 = indexw(filename,'6201','_') or ... ;

Then you can build your logic using those flags. Plus you can print the dataset and make sure the flags are right and the logic using the flags is right.

[David_Billa]

Let me give a try. May I know how to rename the file in target folder after
moving it?

If the filename has the string '_copied' in the source folder, then I want
to move and remove the string '_copied' in the respective destination
folder.

[Tom]

The program I posted was based on building the name you wanted to use for the NEW file you creating by copying the file.  So just adjust the code that creates the name of the file so that there is no need to rename it after it is created.

To remove _copied from the filename just remove it from the value you use to create the target filename. So instead of using FILENAME to build the target use a verison of FILENAME that does not include that text.  The TRANSTRN() function will allow you replace it with nothing.

transtrn(filename,'_copied',trimn(' '))

[David_Billa]

@Tom Thanks. Your proposed solution is working fine. But I'm unable to correct your program to copy the file instead of move, if the file name has '_<some four digits>'. 

[Tom]

If you mean any four digit string the perhaps you will need to use regular expressions. 

 prxchange() 

 https://support.sas.com/rnd/base/datastep/perl_regexp/regexp-tip-sheet.pdf

[David_Billa]

No alternatives other than using regular expressions?

[Tom]

Depends on the details.  If you want to test if there are 4 digits right before the period in a filename then SCAN(), LENGTH() and NOTDIGIT() could do it.

[David_Billa]

@Tom I'm unable to adjust your code below to copy the files to target folder. If the filename has 1234, 5678 and 9012 then I want 'target="&path3"||filename;'. 

Digits are coming the same position in the filename as coded below.

%let inpath=/var/source/ ;
%let path1=/var/life/;
%let path2=/var/nonlife/;

filename source "&inpath";

data files ;
  length id 8 msg filename source target $256 ;
  did=dopen("source");
  if did<=0 then do;
      msg=sysmsg(); 
      put msg; 
      stop;
  end;
  do id=1 to dnum(did);
    filename=dread(did,id);
    if scan(lowcase(filename),-1,'.')='csv' and not index(lowcase(filename),'_copied.csv ') then do;
      source="&inpath" || filename;
      if indexw(filename,'5601','_') or indexw(filename,'6010','_') or indexw(filename,'6020','_') then do;
        target="&path1"||substr(filename,1,length(filename)-4)||'_copied.csv';
      end;
      else if indexw(filename,'0169','_') or indexw(filename,'1070','_') or indexw(filename,'0417','_') then do;
        target="&path2"||filename ;
      end;
      else target="&path1"||filename;
    end;
    output;
  end;
  did=dclose(did);
  drop did msg;
run;

data files_moved;
  set files ;
  length rc1-rc4 8 msg $256 ;
  rc1=filename('from',source);
  rc2=filename('to',target);
  rc3=fcopy('from','to');
  if rc3 then do;
    msg=sysmsg(); 
    put 'ERROR: Unable to copy. ' source= target= rc3= msg=;
  end;
  else do;
    rc4=fdelete('from');
  end;
  output;
  rc1=filename('from');
  rc2=filename('to');
run;