Re: ODS PACKAGE with CALL EXECUTE - not "releasing" created zip file

GBL__ · Posted 02-22-2021 10:26 AM

Hello,

I am creating a zip file using ODS PACKAGE and CALL EXECUTE, and it appears that everything is working as expected EXCEPT that when I go to open the zip file, I get the error "The process cannot access the file because it is being used by another process" i.e. it is still being used by SAS. If I copy the "Data Files.zip" to another folder, then I can open it.

I am unsure exactly what the issue could be as there are no errors in SAS. Please see below for the SAS code used to create the zip files:

%MACRO create_zip_files ;

*******************************************************************************************************************************************
*******************************************************************************************************************************************
** PLACES ALL RAW DATA FILES IN THE CLIENTS FOLDER INTO A TEMP SAS DATA SET ;


FILENAME filelist PIPE "dir /a:-d /b %bquote(""&file_path.\&client_name.\"")" ;

DATA dir_contents(KEEP=_file_names) ;
	LENGTH _file_names $ 500. ;
	INFILE filelist TRUNCOVER ;
	INPUT _file_names $500. ;

		/*  ^   matches at the start of the string (unless used at the start of a character class, which then means negation i.e. [^abc]) */
		/*  ()  defines a capture group, groups pieces of a regular expression together and treats them as a single unit called a subpattern */
		/*  \w  matches any word character (any lowercase or uppercase letter, an underscore (_) character, or any digit) */
		/*  +   quantifier that matches preceding item/value one or more times */
		/*  \   is an escape character (\. matches the actual period character) */
		/*  |   specifies the OR condition (the construct x|y matches either x or y) */ 
		/*  $   matches at the end of the string */
		_perl_string = PRXPARSE("/^(\w+_log\.txt)$|^(\w+_results\.html)$|^(\w+\.sas7bdat)$|^(.+\.zip)$|^(\w+\.gpg)$|^(\w+\.ps1)$/") ;

		/* sends the names of the files in the clients directory if they are NOT log files, results files, SAS data sets, */
		/* zip files, or gpg files (i.e. the clients raw data files) to the temporary SAS data set 'dir_contents' */
		IF NOT PRXMATCH(_perl_string, STRIP(_file_names)) THEN OUTPUT dir_contents ;
RUN ;

FILENAME filelist CLEAR ;


*******************************************************************************************************************************************
*******************************************************************************************************************************************
** SENDS ALL RAW DATA FILES TO Data Files.zip ;


ODS PACKAGE(datazip) OPEN NOPF ;

DATA _NULL_ ;
SYSECHO "%sysfunc(PROPCASE(%sysfunc(TRANSLATE(&client_name., ' ', '_')))) - Creating the Data Files Zip File" ;
	SET dir_contents ;
	/* reads in the raw data file names from the temporary SAS data set created from the PRXPARSE/PRXMATCH functions in the preceding step/process */
	CALL EXECUTE('ODS PACKAGE(datazip) ADD FILE="&file_path.\&client_name.\'||STRIP(_file_names)||'";') ;
RUN ;

ODS PACKAGE(datazip) PUBLISH ARCHIVE PROPERTIES(archive_name="Data Files.zip" archive_path="%bquote(&file_path.\&client_name.\)") ;

ODS PACKAGE(datazip) CLOSE ;

PROC DELETE LIBRARY=work DATA=dir_contents (MEMTYPE=data) ;
SYSECHO "%sysfunc(PROPCASE(%sysfunc(TRANSLATE(&client_name., ' ', '_')))) - Creating the Data Files Zip File" ;
RUN ;


*******************************************************************************************************************************************
*******************************************************************************************************************************************
** SENDS RESULTS, LOG, AND ANALYSIS DATA TO CAATs.zip ;


SYSECHO "%sysfunc(PROPCASE(%sysfunc(TRANSLATE(&client_name., ' ', '_')))) - Creating the CAATs Zip File" ;

ODS PACKAGE(caatszip) OPEN NOPF ;

%if %sysfunc(fileexist("&file_path.\&client_name.\&client_name._&medte._results.html")) %then %do ;

	ODS PACKAGE(caatszip) ADD FILE="&file_path.\&client_name.\&client_name._&medte._results.html" ;

%end ;

%else %do ;

	%put ERROR: The results.html file was not included in the CAATS.zip file ;

%end ;

%if %sysfunc(fileexist("&file_path.\&client_name.\_&client_number._&medte._log.txt")) %then %do ;

	ODS PACKAGE(caatszip) ADD FILE="&file_path.\&client_name.\_&client_number._&medte._log.txt" ;

%end ;

%else %do ;

	%put ERROR: The log.txt file was not included in the CAATS.zip file ;

%end ;

%if %sysfunc(fileexist("&file_path.\&client_name.\&client_number._&medte._analysis.sas7bdat")) %then %do ;

	ODS PACKAGE(caatszip) ADD FILE="&file_path.\&client_name.\&client_number._&medte._analysis.sas7bdat" ;

%end ;

%else %do ;

	%put ERROR: The clients analysis data set was not included in the CAATS.zip file ;

%end ;

%if %sysfunc(fileexist("&file_path.\&client_name.\&client_number._&medte._insurance.sas7bdat")) %then %do ;

	ODS PACKAGE(caatszip) ADD FILE="&file_path.\&client_name.\&client_number._&medte._insurance.sas7bdat" ;

%end ;

%else %do ;

	%put WARNING: Insurance data set was not included in the CAATS.zip file, confirm client does not include insurance information ;

%end ;

ODS PACKAGE(caatszip) PUBLISH ARCHIVE PROPERTIES(archive_name="CAATs.zip" archive_path="&file_path.\&client_name.\") ;

ODS PACKAGE(caatszip) CLOSE ;


*******************************************************************************************************************************************
*******************************************************************************************************************************************
** SENDS PCART DATA TO pcart.zip ;


SYSECHO "%sysfunc(PROPCASE(%sysfunc(TRANSLATE(&client_name., ' ', '_')))) - Creating the PCART Zip File" ;

ODS PACKAGE(pcartzip) OPEN NOPF ;

%if %sysfunc(fileexist("&file_path.\&client_name.\&client_number._&medte._pcart.sas7bdat")) %then %do ;

	ODS PACKAGE(pcartzip) ADD FILE="&file_path.\&client_name.\&client_number._&medte._pcart.sas7bdat" ;

%end ;

ODS PACKAGE(pcartzip) PUBLISH ARCHIVE PROPERTIES(archive_name="&client_number._&medte._pcart.zip" archive_path="&file_path.\&client_name.\") ;

ODS PACKAGE(pcartzip) CLOSE ;


*******************************************************************************************************************************************
*******************************************************************************************************************************************
** END OF PROGRAM ;

%MEND create_zip_files ;

The above code includes only the raw data files included in the path, as the other file types go into separate zip files (and those can be opened with no issues).

I am curious if this is all related to the FILENAME PIPE? I updated that part so as to not to try and include any folders in a zip that may be inside the path.

Log for review (sensitive data removed):

31         %create_zip_files ;

NOTE: The infile FILELIST is:
      Unnamed Pipe Access Device,
      PROCESS=dir /a:-d /b "C:\........\SAS_TESTING\........\",
      RECFM=V,LRECL=32767

NOTE: 7 records were read from the infile FILELIST.
      The minimum record length was 9.
      The maximum record length was 56.
NOTE: The data set WORK.DIR_CONTENTS has 2 observations and 1 variables.
NOTE: DATA statement used (Total process time):
      real time           0.56 seconds
      user cpu time       0.01 seconds
      system cpu time     0.04 seconds
      memory              819.03k
      OS Memory           23908.00k
      Timestamp           02/22/2021 09:22:43 AM
      Step Count                        363  Switch Count  36
      

NOTE: Fileref FILELIST has been deassigned.

NOTE: There were 2 observations read from the data set WORK.DIR_CONTENTS.
NOTE: DATA statement used (Total process time):
      real time           0.01 seconds
      user cpu time       0.01 seconds
      system cpu time     0.00 seconds
      memory              946.00k
      OS Memory           23908.00k
      Timestamp           02/22/2021 09:22:43 AM
      Step Count                        364  Switch Count  21
      

NOTE: CALL EXECUTE generated line.
1         + ODS PACKAGE(datazip) ADD FILE="C:\........\SAS_TESTING\........\Loan Portfolio Details - Dec 2020 updated 17Feb2021.xlsx";
2         + ODS PACKAGE(datazip) ADD FILE="........\SAS_TESTING\........\_goez_DEC2020_converted_file.csv";
NOTE: Writing DATAZIP file: C:\........\SAS_TESTING\........\Data Files.zip

NOTE: Deleting WORK.DIR_CONTENTS (memtype=DATA).
NOTE: PROCEDURE DELETE used (Total process time):
      real time           0.00 seconds
      user cpu time       0.00 seconds
      system cpu time     0.00 seconds
      memory              21.46k
      OS Memory           23908.00k
      Timestamp           02/22/2021 09:22:57 AM
      Step Count                        365  Switch Count  23
      

ERROR: The results.html file was not included in the CAATS.zip file
ERROR: The log.txt file was not included in the CAATS.zip file
WARNING: Insurance data set was not included in the CAATS.zip file, confirm client does not include insurance information
NOTE: Writing CAATSZIP file: C:\........\SAS_TESTING\........\CAATs.zip
NOTE: Writing PCARTZIP file: C:\........\SAS_TESTING\........\goez_DEC2020_pcart.zip

Thank you very much for any assistance, help, or direction!

ballardw · Posted 02-22-2021 10:42 AM

Does the code you submit with call execute properly close the document file before the next code AFTER your data step using call execute finishes?

GBL__ · Posted 02-22-2021 10:47 AM

Currently, I am only using CALL EXECUTE to "Add" the files to the zip. Then I PUBLISH ARCHIVE and CLOSE after that DATA Step.

ODS PACKAGE(datazip) OPEN NOPF ;

DATA _NULL_ ;
SYSECHO "%sysfunc(PROPCASE(%sysfunc(TRANSLATE(&client_name., ' ', '_')))) - Creating the Data Files Zip File" ;
	SET dir_contents ;
	/* reads in the raw data file names from the temporary SAS data set created from the PRXPARSE/PRXMATCH functions in the preceding step/process */
	CALL EXECUTE('ODS PACKAGE(datazip) ADD FILE="&file_path.\&client_name.\'||STRIP(_file_names)||'";') ;
RUN ;

ODS PACKAGE(datazip) PUBLISH ARCHIVE PROPERTIES(archive_name="Data Files.zip" archive_path="%bquote(&file_path.\&client_name.\)") ;

ODS PACKAGE(datazip) CLOSE ;

Should I include the ODS PACKAGE PUBLISH ARCHIVE and ODS PACKAGE CLOSE statements inside the DATA Step that uses CALL EXECUTE?

GBL__ · Posted 02-22-2021 10:58 AM

@ballardw

Like, should I include in that DATA Step an END=last option and then do IF last THEN DO?

i.e. something like this?

ODS PACKAGE(datazip) OPEN NOPF ;

DATA _NULL_ ;
SYSECHO "%sysfunc(PROPCASE(%sysfunc(TRANSLATE(&client_name., ' ', '_')))) - Creating the Data Files Zip File" ;
	SET dir_contents END=last ;
	/* reads in the raw data file names from the temporary SAS data set created from the PRXPARSE/PRXMATCH functions in the preceding step/process */
	CALL EXECUTE('ODS PACKAGE(datazip) ADD FILE="&file_path.\&client_name.\'||STRIP(_file_names)||'";') ;

	IF last THEN DO ;
		CALL EXECUTE('ODS PACKAGE(datazip) PUBLISH ARCHIVE PROPERTIES(archive_name="Data Files.zip" archive_path="%bquote(&file_path.\&client_name.\)") ; ') ;.
		CALL EXECUTE('ODS PACKAGE(datazip) CLOSE ; ') ;
	END ;
RUN ;

OK I just tried the above code and the result was the same.

GBL__ · Posted 02-22-2021 11:14 AM

@ballardw

Reason why I ask about the FILENAME PIPE probably being the issue is because if I change the beginning code to this:

FILENAME filelist "%bquote(&file_path.\&client_name.\)" ;

DATA dir_contents(KEEP=_file_names) ;
	LENGTH _file_names $ 500. ;
	handle=DOPEN("filelist") ;
	IF handle > 0 THEN DO ;
		_count=DNUM(handle) ;
		DO i=1 TO _count ;
			_file_names=STRIP(DREAD(handle, i)) ;
			/*  ^   matches at the start of the string (unless used at the start of a character class, which then means negation i.e. [^abc]) */
			/*  ()  defines a capture group, groups pieces of a regular expression together and treats them as a single unit called a subpattern */
			/*  \w  matches any word character (any lowercase or uppercase letter, an underscore (_) character, or any digit) */
			/*  +   quantifier that matches preceding item/value one or more times */
			/*  \   is an escape character (\. matches the actual period character) */
			/*  |   specifies the OR condition (the construct x|y matches either x or y) */ 
			/*  $   matches at the end of the string */
			_perl_string = PRXPARSE("/^(\w+_log\.txt)$|^(\w+_results\.html)$|^(\w+\.sas7bdat)$|^(.+\.zip)$|^(\w+\.gpg)$|^(\w+\.ps1)$/") ;

			/* sends the names of the files in the clients directory if they are NOT log files, results files, SAS data sets, */
			/* zip files, or gpg files (i.e. the clients raw data files) to the temporary SAS data set 'dir_contents' */
			IF NOT PRXMATCH(_perl_string, STRIP(_file_names)) THEN OUTPUT dir_contents ;
		END ;
	END ;
	rc=DCLOSE(handle) ;
RUN ;

FILENAME filelist CLEAR ;

Everything works correctly, as expected and there are no issues with opening the zip file after program completion.

ODS PACKAGE with CALL EXECUTE - not "releasing" created zip file