/*--------------------------------------------------------------------------------------\
| PROGRAM NAME: |
| scdm_control_flow.sas |
| |
|---------------------------------------------------------------------------------------|
| PURPOSE: |
| The purpose of this program is to define selective and sequential execution |
| of QA modules in the program 00.0_scdm_data_qa_review_master_file.sas. |
|---------------------------------------------------------------------------------------|
| MAJOR STEPS PERFORMED BY THE PROGRAM: |
| |
| 0- Perform setup for this macro |
| 1- Clean up control flow dataset to enable safe and consistent querying |
| 2- Retrieve modules explicitly chosen to run (i.e. where execute_flag=y) |
| 3- Ensure that all the necessary modules are run given dependencies |
| a) Modules for optional tables are only selected if defined in CC |
| b) If any utilization modules are selected, all utilization modules |
| will be run |
| 4- Set up macro variable lists for processing each module |
| 5- Loop through and execute each module |
| 6- Call request-level "signature file consolidation" and "log checker" macros |
| 7- Run scdm_snapshot |
| 8- If specified, run mother-infant identification |
| 9- If specified, run qa_common_components |
| |
|---------------------------------------------------------------------------------------|
| DEPENDENCIES/CONSTRAINTS/CAUTIONS |
| This macro is to be called by 00.0_scdm_data_qa_review_master_file.sas. |
| Please refer to the master program for details. |
| |
|---------------------------------------------------------------------------------------|
| PROGRAM INPUT: |
| see scdm_data_qa_review_master_file.sas |
| |
| PROGRAM OUTPUT: |
| see Workplan PDF |
|---------------------------------------------------------------------------------------|
| CONTACT: |
| Sentinel Coordinating Center |
| info@sentinelsystem.org |
\*-------------------------------------------------------------------------------------*/
*-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-;
* PLEASE DO NOT EDIT BELOW WITHOUT CONTACTING THE SENTINEL OPERATIONS CENTER ;
*-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-;
%global end_qa module table;
/***************************************************************************************************/
/* QA LOCAL SET-UP AND PROCESSING */
/***************************************************************************************************/
%macro run_qa_package;
proc printto log="&msoc.qa_package.log" new;
run;
%* overall processing macros;
%let end_qa=0;
%* Include standard and utility macros;
%include "&INFOLDER.macros/scdm_standard_macros.sas" /nosource2;
%include "&INFOLDER.macros/qa_create_l2_xtable_ds.sas"/nosource2;
%include "&INFOLDER.macros/_customize_input.sas" /nosource2;
%include "&INFOLDER.scdm_sas_log_checker_directory_cc.sas" /nosource2;
%include "&INFOLDER.scdm_qasignaturerequest.sas" /nosource2;
%include "&INFOLDER.scdm_formats.sas" /nosource2;
%include "&INFOLDER.scdm_data_qa_review-partition.sas" /nosource2;
%soc_lib(DPLOCALO,&DPLOCAL) ; /* always point to main dplocal libary - needed for partitioned data*/
%* validate patid to partition crosswalk file;
%partition_meta_chks;
%partition_expected_scdmtab_chks;
%partition_expected_scdmpart_chks;
%partition_appropriate_table_chks;
%partition_dup_chks;
%partition_consistent_fmt_chks;
/* Initialize global macro variables */
/* list of tabid/tablenames without patid column - in lower case*/
%global/readonly tabid_exc=fac pvd prs ;
%global/readonly sql_tabid_exc="fac", "pvd", "prs" ;
/* key variable not partitioned (in lower case) */
%global/readonly varname_exc=facilityid providerid surveyid questionid questionverid prm_code ;
%global/readonly sql_varname_exc="facilityid", "providerid", "surveyid", "questionid", "questionverid", "prm_code";
%**** list modules that should not run in parallel(in upcase) ;
%global/readonly module_nospawn_sql=%str("L1", "L2", "L3", "DATES", "OVLAP") ;
%global/readonly module_nospawn=%str(L1 L2 L3 DATES OVLAP) ;
%global/readonly do_partitions= 1 ;
%global sql_tabidlist tabidlist tablename_sorted sql_tablename_sorted;
%global prefix_length pnum_length ;
%let tabidlist = ;
%let sql_tabidlist = ;
%let tablename_sorted = ;
%let sql_tablename_sorted = ;
%let prefix_length = 11 ;
%let pnum_length=%sysfunc(length(&numpartitions)) ;
%put |&=pnum_length| ;
%global SPAWN SPAWNED dplocalo ;
%if &NumSession ge 1 and &SASConnect eq Y
%then %do; /* macro vars if spawning */
%let SPAWN=1;
%include "&INFOLDER.macros/remote_macros.sas" /nosource ;
%let dplocalo=dplocalo ;
%end ;
%else %do; /* macro vars if not spawning */
%let SPAWN=0;
%let SPAWNED=0 ;
%let dplocalo=dplocal ;
%end ;
%put |&=SPAWN| |&=dplocalo|;
%* read-in control flow input file ;
data control_flow;
infile "&INFOLDER.control_flow.csv" dlm=',' dsd truncover firstobs=2;
informat module $5. ;
informat execute_flag $1. ;
informat sascode $45. ;
informat cc_table $12.;
informat seqno best.;
format module $5. ;
format execute_flag $1. ;
format sascode $45. ;
format cc_table $12.;
format seqno best. ;
format unique_id $15. ;
input module :$5. execute_flag :$1. sascode :$45. cc_table :$12.
seqno :8. unique_id :$15.;
/* standardize capitalization */
module = lowcase(module) ;
execute_flag = lowcase(execute_flag) ;
unique_id = lowcase(unique_id);
run;
proc sort data=control_flow;
by seqno;
run;
proc sql noprint;
select strip(module), quote(strip(module))
into :tabidlist separated by ' ', :sql_tabidlist separated by ', '
from work.control_flow
where upcase(execute_flag) = 'Y' and cc_table ^= 'X';
select symget(cc_table), quote(symget(cc_table))
, max(length(symget(cc_table)))
into :tablename_sorted separated by ' '
, :sql_tablename_sorted separated by ', '
, :prefix_length trimmed
from work.control_flow
where upcase(execute_flag) = 'Y' and cc_table ^= 'X'
order by 1 ;
;quit;
**** cross_varID ;
%global cross_varID sql_cross_varID retain_ds ;
%let cross_varID= ;
proc sql noprint;
select distinct(strip(lowcase(variable))), quote(strip(lowcase(variable)))
into :cross_varID separated by ' ', :sql_cross_varID separated by ', '
from infolder.lkp_all_l1
where upcase(crossvar)="X"
;
;quit;
%let retain_ds = patid encounterid providerid facilityid questionid questionverid prm_code surveyid;
%* customize input files as necessary;
%kill_directory (kill_list=custom); %* defensive coding - remove content from customs folder to avoid issues with reruns of package;
%customize_input;
%* defensive pre-check: abort program if NumPartitions is not ge 1 *;
%if (%isBlank(&NumPartitions)=1 or &NumPartitions <1) %then %do;
data _null_;
put 90*'!';
put ' ';
put 'Defensive Pre-Check: NumPartitions must be >=1. RUN_QA_PACKAGE macro is aborting.';
put ' ';
put '==> The NumPartitions parameter must be populated with whole number >=1';
put '==> Please go back to the master program and populate NumPartitions with valid value.';
put ' ';
put 90*'!';
put ' ';
run;
%abort cancel 99 ;
%end ;
%* defensive pre-check: abort program if NumPartitions>1 and ParTable is null *;
%if %isBlank(&ParTable.)=1 and (&NumPartitions.>1) %then %do;
data _null_;
put 90*'!';
put ' ';
put 'Defensive Pre-Check: ParTable parameter not populated. RUN_QA_PACKAGE macro is aborting.';
put ' ';
put '==> The ParTable parameter cannot be blank';
put '==> Please go back to the master program and populate ParTable.';
put ' ';
put 90*'!';
put ' ';
run;
%abort cancel 99 ;
%end ;
%* defensive pre-check: abort program if _SasGrid=Y and _GridSrv is null *;
%if &SasGrid=Y and %isBlank(&GridSrv)=1 %then %do;
data _null_;
put 90*'!';
put ' ';
put 'Defensive Pre-Check: GridSrv parameter not populated. RUN_QA_PACKAGE macro is aborting.';
put ' ';
put '==> The GridSrv parameter cannot be blank if SasGrid=Y.';
put '==> Please go back to the master program and populate GridSrv.';
put ' ';
put 90*'!';
put ' ';
run;
%abort cancel 99 ;
%end ;
%* defensive pre-check: abort program if _SasGrid=Y and _SASConnect ne Y *;
%if &SasGrid=Y and &SASConnect ne Y %then %do;
data _null_;
put 90*'!';
put ' ';
put 'Defensive Pre-Check: SASConnect parameter has an unexpected value. RUN_QA_PACKAGE macro is aborting.';
put ' ';
put '==> The SASConnect parameter must be Y if SasGrid=Y.';
put '==> Please go back to the master program and populate SASConnect as Y.';
put ' ';
put 90*'!';
put ' ';
run;
%abort cancel 99 ;
%end ;
%* defensive pre-check: abort program if NumSession is not a positive integer or 0 *;
%if &NumSession ne %sysfunc(int(&NumSession)) %then %do;
data _null_;
put 90*'!';
put ' ';
put 'Defensive Pre-Check: NumSession parameter has an unexpected value. RUN_QA_PACKAGE macro is aborting.';
put ' ';
put '==> The NumSession parameter must be a positive integer or 0.';
put '==> Please go back to the master program and populate NumSession correctly.';
put ' ';
put 90*'!';
put ' ';
run;
%abort cancel 99 ;
%end ;
%* defensive pre-check: abort program if Execute_CC=Y and _ROOT_DPLOCAL variable not populated *;
%if &Execute_CC=Y and %isBlank(&_ROOT_DPLOCAL)=1 %then %do;
data _null_;
put 90*'!';
put ' ';
put 'Defensive Pre-Check: _ROOT_DPLOCAL parameter not populated. RUN_QA_PACKAGE macro is aborting.';
put ' ';
put '==> The _ROOT_DPLOCAL parameter cannot be blank if Execute_CC=Y.';
put '==> Please go back to the master program and populate _ROOT_DPLOCAL.';
put ' ';
put 90*'!';
put ' ';
run;
%abort cancel 99 ;
%end ;
%* confirm ETL value is valid *;
%* identify ETL values as non-digit (includes missing);
%if %sysfunc(notdigit(&ETL)) %then %do;
data _null_;
put 80*'!';
put ' ';
put 'Pre-Module Check 2: ETL er'"ror detected. RUN_QA_PACKAGE macro is aborting. ";
put ' ';
put "==> The ETL parameter cannot be blank or include any non-numeric value(s). Please contact SOC DMQA for next steps.";
put ' ';
put 80*'!';
put ' ';
run;
%abort cancel 99 ;
%end ;
%* Create ETL Version table *;
%if %sysfunc(exist(msoc.etl_version,data)) %then %do; /* begin if etl_version ds exists */
%put ;
%put ==> An ETL_version table already exists and will be reviewed to;
%put ==> ensure that the ETL in the ETL_version table is equal to the expected ETL;
%put ;
proc sql noprint;
select etl into :ETL_VersionTable trimmed
from msoc.etl_version
;
quit;
%put ;
%put ====> CURRENT ETL: &ETL. ;
%put ====> ETL from ETL_Version table in this QA package MSOC folder: &ETL_VersionTable.;
%put ;
%if &ETL.=&ETL_VersionTable. %then %do;
%put ;
%put ==> ETL Versions match and the program will continue;
%put ==> HOWEVER, unless otherwise specified in the master program, all ;
%put ==> existing SAS datasets in the local output directories ('dplocal','msoc');
%put ==> will be deleted for data quality assurance purposes ;
%put ;
%end;
%else %do;
data _null_;
put 70*'!';
put ' ';
put "Pre-Module Check 3: Different ETL content detected in MSOC library. RUN_QA_PACKAGE macro is aborting.";
put ' ';
put ' ';
put 70*'!';
run;
%abort cancel 99;
%end;
%end; /* end if etl_version ds exists */
%kill_directory (kill_list=dplocal msoc);
data msoc.etl_version;
length DP $6;
dp=upcase("&dpid.");
ETL=&ETL.;
run;
%if %sysevalf(&syscc.>4) %then %do;
data _null_;
putlog 80*'!';
putlog ' ';
putlog 'Pre-Module Check 4: System er'"ror detected. RUN_QA_PACKAGE macro is aborting.";
putlog ' ';
putlog ' ';
putlog 80*'!';
run;
%abort cancel 99 ;
%end ;
%* ensure that the site has populated all expected table names in the master file *;
%local t tabct abortct tablelist;
%let tabct=0;
%let abortct=0;
proc sql noprint;
select cc_table
, count(*)
into :tablelist separated by "*"
, :tabct trimmed
from control_flow (where=(lowcase(cc_table) ne 'x' and execute_flag='y'))
;
quit;
%do t=1 %to &tabct.;
%let table=%scan(&tablelist.,&t.,%str(*));
%let table=%qleft(%qtrim(&table));
%if %length(%superq(&table)) = 0 %then %do;
%let abortct=%eval(&abortct. + 1);
data _null_;
put 70*'!';
put "SCDM &table. parameter not specified in master progam";
put "but defined by SOC inputfiles/control flow.csv file ";
put ' ';
put 70*'!';
run;
%end;
%end;
%if "&abortct." ne "0" %then %do;
data _null_;
put 70*'!';
put ' ';
put "Pre-Module Check 5: RUN_QA_PACKAGE macro is aborting because an expected SCDM table ";
put "name was not defined by your site for the ETL under review. ";
put ' ';
put 70*'!';
run;
%abort cancel 99;
%end;
%* ensure that the site has NOT populated an unexpected table names in the master file *;
%local t tabct abortct tablelist;
%let tabct=0;
%let abortct=0;
proc sql noprint;
select cc_table
, count(*)
into :tablelist separated by "*"
, :tabct trimmed
from control_flow (where=(lowcase(cc_table) ne 'x' and execute_flag='n'))
;
quit;
%if %eval(&tabct. ge 1) %then %do;
%do t=1 %to &tabct.;
%let table=%scan(&tablelist.,&t.,%str(*));
%let table=%qleft(%qtrim(&table));
%if %length(%superq(&table)) ne 0 %then %do;
%let abortct=%eval(&abortct. + 1);
data _null_;
put 70*'!';
put ' ';
put "SCDM &table. name is defined by DP, but not expected";
put "at the site. Only the SCDM tables defined by SOC in ";
put "inputfiles/control_flow.csv file should be present in the ";
put "directory and named in the master program. ";
put ' ';
put 70*'!';
run;
%end;
%end;
%if "&abortct." ne "0" %then %do;
data _null_;
put 70*'!';
put ' ';
put "Pre-Module Check 6: RUN_QA_PACKAGE macro is aborting because an unexpected SCDM table";
put "name was defined by your site for the ETL under review.";
put ' ';
put 70*'!';
run;
%abort cancel 99;
%end;
%end;
%* ensure ZIP3 populated with valid value *;
%if %isBlank(&demtable)=0 %then %do;
proc sql noprint;
select length
into :ZIPLEN trimmed
from dictionary.columns
where upcase ( libname ) = 'QADATA' and upcase ( name ) = 'POSTALCODE'
and PRXCHANGE ('s/\d+$//', 1, trim(lowcase(memname)) ) = lowcase( "&demtable" ) ;
quit;
%if &ZIP3 = Y %then %do ;
%if &ZIPLEN gt 3 %then %do;
data _null_;
put 70*'!';
put ' ';
put "Pre-Module Check 7: RUN_QA_PACKAGE macro is aborting because ZIP code length";
put "of three was indicated, but longer ZIP codes detected in DEMOGRAPHIC table.";
put ' ';
put 70*'!';
run;
%abort cancel 99;
%end;
%end ;
%if &ZIP3 eq %then %do ;
%if &ZIPLEN le 3 %then %do;
data _null_;
put 70*'!';
put ' ';
put "Pre-Module Check 7: RUN_QA_PACKAGE macro is aborting because ZIP code length";
put "of three was detected, but not indicated for DEMOGRAPHIC table.";
put ' ';
put 70*'!';
run;
%abort cancel 99;
%end;
%end;
%end;
/* prepare file that includes list of modules and accompanying SAS code to be executed and */
/* verify all required logic will be executed with respect to dependencies in QA package */
/* Create separate control_flow dataset for CC*/
data msoc.cc_control_flow;
set control_flow;
run;
data msoc.control_flow_3;
set control_flow(where=(execute_flag = 'y'));
by seqno;
run;
/* Conditionally execute module, aborting processing and reviewing errors when error(s) detected */
%let module_list=;
%let sascode_list=;
%* load modules and code to run into macro parameters to be used in do loop;
proc sql noprint;
select module
, sascode
, seqno
into :module_list separated by ' '
, :sascode_list separated by '~'
, :seqno_list
from msoc.control_flow_3
order by seqno ;
quit;
%put |&=module_list| ;
%let total_starttime = %sysfunc(datetime());
%if &SPAWN %then %do ;
*Initialize connections to remote SAS Grid or SAS/CONNECT sessions*;
%signon ;
%end;
%*Loop through and execute each module in sequence ;
%do z = 1 %to %sysfunc(countw(&module_list));
%kill_directory (kill_list=work);
%if %length(&module_list) > 0 %then %let module = %scan(&module_list., &z.);
%else %let module = ;
%if %length(&sascode_list) > 0 %then %let sascode = %scan(&sascode_list., &z.,~);
%else %let sascode = ;
%let tabid = %sysfunc(strip(&module));
%put |&=tabid| |&=z| |&=end_qa| ;
%if &end_qa.=0 %then %do;
data _null_;
put 75*'-';
put ' ';
put "==> Begin Execution of module: &module., sascode: &sascode..sas";
put ' ';
put 75*'-';
run;
%if &SPAWN %then %do ;
%* some module do NOT need to be run in parallel ;
%if not %eval(%upcase(&module) in &module_nospawn) %then %do ;
%waitForAvailableSession ;
%let SPAWNED=1 ;
%put |&=SPAWNED| |&=module| |&=&openSessID| |&=z| |&=end_qa| ;
%syslput _USER_ /remote=mySess&openSessID.; %* pass all user macro var to spawned process ;
rsubmit mySess&openSessID. wait=no cmacvar=myRsubmitVar&openSessID.
inheritlib=(dplocal=dplocalo msoc qadata infolder)
;;
%nrstr(%put ..Start of rsubmitted code for &tabid --- vvvvvvvvvvvvvvvvvvvv ;) ;
%include "&INFOLDER.macros/do_module.sas" ;
/* pass macro variable from spawned to original dplocal folder */
%nrstr(%let &module._end_qa=&end_qa) ;;
proc sql noprint;
create table dplocalo.macvalue_&module. as
select name, value, offset
from dictionary.macros
where substr(upcase(name),1,length("&module")+1) in (%upcase("&module._"))
or upcase(name) in ("END_QA")
order by name, offset
;quit;
%nrstr(%put ..End of rsubmitted code for &tabid --- ^^^^^^^^^^^^^^^^^^^^^^ ;) ;
endrsubmit;
%end ;
%else %do ; /* no-spawn module(except L1). L1 is run in the module and do not need to run here */
%if not %eval(%upcase(&module) in L1 ) %then %do ;
* Wait for all remote sessions to complete;
WAITFOR _all_
%do s = 1 %to &numSession.;
mySess&s.
%end; ;
%let SPAWNED=0 ;
%include "&INFOLDER.macros/do_module.sas" ;
%end ;
%end;
%end; /* if spawn */
%else %do ; /* if not spawning - run modules one by one in sequence */
%let SPAWNED=0 ;
%include "&INFOLDER.macros/do_module.sas" ;
%end; %* not %eval(&SPAWN and %upcase(&module) in L1 ) ;
%if &end_qa. ne 0 %then %do;
%let z=99;
%end;
%end; %* %if &end_qa.=0 ;
%end; %* end loop z - loop thru modules;
%if &SPAWN %then %do;
*Close connections to remote SAS Grid or SAS/CONNECT sessions*;
%signoff ;
%end;
%let total_stoptime = %sysfunc(datetime());
/* Stack runtimes datasets, sum them for total runtime */
data msoc.runtimes(keep=module starttime stoptime runtime);
set dplocal.runtimes_: end=last;
output;
if last then do;
module = "Total RunTime";
StartTime=put(&total_starttime,e8601dz.);
StopTime=put(&total_stoptime,e8601dz.);
diff_times = &total_stoptime - &total_starttime;
hour = hour(diff_times);
minute = minute(diff_times);
second = floor(second(diff_times));
runtime = catx(" ", hour , "h", minute, "m", second, "s");
output;
end;
run;
proc datasets lib=dplocal nolist nodetails;
delete runtimes_:;
run;
%* If qa did not abort and masking specified in the master program file, call the results masking program *;
%if &z.<99
%then %do;
%if %lowcase(&execute_mask.)=y
and %length(&masktabid_list) > 0
%then %do;
%inc "&infolder.scdm_data_qa_review-maskresults.sas";
%end;
%end;
%soc_qasignaturerequest;
proc printto;
run;
%* Run scdm_snapshot program *;
%snapshot_run
%* If specified in the master program file, call the Mother-Infant Identification program *;
%if %lowcase(&execute_MI.)=y
%then %do;
proc printto log="&msoc.mir_package.log" new;
run;
%if &do_partitions eq 1 %then %do;
%str(%inc "&infolder./mir/mir_master_program_pd.sas") ;
%end;
%else %do;
%str(%inc "&infolder./mir/mir_master_program.sas") ;
%end;
*redirect libraries back to QA package request id root folder;
%redirect_libs(&_root.)
proc printto;
run;
%end;
%* If specified in the master program file, call the qa_common_components program *;
%if %lowcase(&execute_CC.)=y %then %do;
%cc_run
%end;
%mend run_qa_package;
%run_qa_package;
proc printto;
run;
%logcheck(logdir=&MSOC, logdir_out=msoc);
*-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-;
* End scdm_control_flow.sas ;
*-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-;