****************************************************************************************************
*                                           PROGRAM OVERVIEW
****************************************************************************************************
*
* PROGRAM: runqrp.sas  
*
* Created (mm/dd/yyyy): 01/26/2016
* Last modified: 
* Version: See qrp_master_header.sas
*
*--------------------------------------------------------------------------------------------------
* PURPOSE:
*   Macro to run QRP analytic code                                    
*   
*  Program inputs:                                                                                   
*                                     
*  Program outputs:                                                                                                                                       
*                                                                                                  
*  PARAMETERS:    
*       All the parameters are set by the run_programs program. 
*            
*  Programming Notes:                                                                                
*
*
*--------------------------------------------------------------------------------------------------
* CONTACT INFO: 
*  Sentinel Coordinating Center
*  info@sentinelsystem.org
*
*--------------------------------------------------------------------------------------------------
*  CHANGE LOG: 
*
*   Version   Date       Initials      Comment (reference external documentation when available)
*   -------   --------   --------   ---------------------------------------------------------------
*             mm/dd/yy
*
***************************************************************************************************;

%macro runqrp ;
  
    *Enter version for any custom code written on top of QRP;
    %let CustomCodeVer =distributed_cder_mpl2p_wp033_1.0.0;

    /*option required to ensure successful multiple runs of CIDA*/
    options nosyntaxcheck;

/*************************************************************************************************/
/* Loop through each Run and execute QRP                                                         */
/*************************************************************************************************/

    /*Determine number of runs*/
    %isdata(dataset=infolder.qrp_parameters);
    %if %eval(&nobs.>0) %then %do;
        proc contents data=infolder.qrp_parameters noprint out=qrp_parameters;
        run;
       
        %global numruns numparms;
        proc sql noprint;
            select count(*) into: numruns
            from qrp_parameters
            where substr(upcase(name),1,3) = 'RUN';

            select count(*) into: numparms
            from infolder.qrp_parameters
        quit;

        %do qrprun = 1 %to %eval(&numruns.);

            /*Create temporary dplocal/msoc subfolders*/
            options DLCREATEDIR ;
            libname worktemp "&worktemp" ;
            libname wrktemp2 "&wrktemp2";
            libname freedir "&freedir";
            libname dptemp "&dptemp" ;
            libname msoctemp "&msoctemp" ;
            libname sasenv "&sasenv" ;
            libname medstemp "&medstemp";
            options NODLCREATEDIR;

            *Reset all parameters;
            %let RUNID=;
            %let PERIODIDSTART=; 
            %let PERIODIDEND=;   
            %let ANALYSIS=;
            %let MONITORINGFILE=; 
            %let USERSTRATA=;
            %let COMBOFILE=;
            %let COHORTFILE=;
            %let COHORTCODES=; 
            %let INCLUSIONCODES=;   
            %let STOCKPILINGFILE=;
            %let RUN_ENVELOPE =; 
            %let FREEZEDATA=N;
            %let ZIPFILE=;
            %let LABSCODEMAP=; /*must be specified if lab data are used*/
            %let DISTINDEX =;

            *Type files;
            %let TYPE1FILE=;
            %let TYPE2FILE=;
            %let TYPE3FILE=;
            %let TYPE4FILE=;
            %let MODPREGDUR=;
            %let TYPE5FILE=;
            %let TYPE6FILE=;

            ** Sequential Analysis parameters **;
            %let METADATAFILE=;
            %let SURVEILLANCEMODE=;

            *Baseline Table files;
            %let COVARIATECODES=;
            %let UTILFILE=;     
            %let COMORBFILE=; 
            %let DRUGCLASSFILE=; 
            %let PROFILE = ; 
            %let MFUFILE = ;

            *Additional analyses files: multiple events, overlap, adherence, ITS, switching, concomittant episodes;
            %let MULTEVENTFILE=;
            %let MULTEVENTFILE_ADHERE=;
            %let OVERLAPFILE=;
            %let OVERLAPFILE_ADHERE=;
            %let ITSFILE = ;
            %let CONCFILE=;
            %let TREATMENTPATHWAYS=;

            *Mother-Infant Cohort file;
            %let MICOHORTFILE=;

            ** Treescan parameters **;
            %let TREEFILE = ;
            %let TREELOOKUP=;
            %let ICD10ICD9MAP=;

            ** Propensity Score and Covariate Stratification parameters ** ;
            %let PSESTIMATIONFILE = ; 
            %let PSMATCHFILE = ;
            %let STRATIFICATIONFILE = ;
            %let COVSTRATFILE = ;
            %let IPTWFILE = ;
            %let PSCSSUBGROUPFILE = ;
            %let DIAGNOSTICS=;
            %let INDLEVEL=; 

            /*plus*/
            %let ormodelfile =;

            /*Assign all parameters to macro variables*/
            %do qrpparameter = 1 %to %eval(&numparms.);
                data _null_;
                    set infolder.qrp_parameters;
                    if _n_ = &qrpparameter. then do;
                        call symputx("parameter", parameter);
                        call symputx("value", run&qrprun.);
                    end;
                run;
                %let &parameter. = &value.;
            %end;

            %let runid=%lowcase(&runid.);

            /* Clean out dplocal and msoc folder for every run */
            %isdata(dataset=dplocal.&runid._checkpoints);
            %if &restartqrp = N or (&restartqrp = Y and %eval(&nobs.<1)) %then %do;

                /*Replace dplocal, msoc, and medstemp folder contents*/
                proc datasets nowarn noprint lib=dplocal; 
                    delete &runid._: ;
                quit;
                proc datasets nowarn noprint lib=msoc; 
                    delete &runid._: log_checker: ;
                quit;
                proc datasets nowarn noprint lib=medstemp kill; 
                quit;
            %end;

            /* Determine whether:
                1. Start at beginning of Run
                2. Start at recently completed checkpoint
                3. to restart at beginnging of QRP 
               or if a successful checkpoint has been passed */

            %if &restartqrp = Y %then %do;
                %isdata(dataset=dplocal.&runid._checkpoints);
                %if %eval(&nobs.<1) %then %do;
                    /*File doesn't exist - need to restart run at beginning*/
                    %let restartrun = N;

                    /*Replace dplocal and msoc folder contents*/
                    proc datasets nowarn noprint lib=dplocal; 
                        delete &runid: ;
                    quit;
                    proc datasets nowarn noprint lib=msoc; 
                        delete &runid: ;
                    quit;
                
                    proc datasets nowarn noprint lib=worktemp kill; quit;
                %end;
                %else %do;
                    data _null_;
                        set dplocal.&runid._checkpoints nobs=nobs;
                        if _n_ = nobs then do;
                            call symputx('lastcheck', macrocalled);
                        end;
                    run;

                    %if &lastcheck = complete %then %do;
                        /*run is complete - do not run*/
                        %let restartrun = F;
                        %goto nextrun;
                    %end;
                    %else %do;
                        %let restartrun = Y;
                    %end;            
                %end;
            %end;
            %else %do;
                %let restartrun = N;
                proc datasets nowarn noprint lib=worktemp kill; quit;
                proc datasets nowarn noprint lib=wrktemp2 kill; quit;
                proc datasets nowarn noprint lib=dptemp kill; quit;
                proc datasets nowarn noprint lib=msoctemp kill; quit;
                proc datasets nowarn noprint lib=sasenv kill; quit;
            %end;

            proc printto log="&MSOC.&RUNID._cida.log" %if &restartrun = N %then %do; new %end; ;
            run;

            %put =====> MACRO CALLED: runqrp v&QRPVer.;

            /**********/
            /** CIDA **/
            /**********/
            %ms_cidanum(RUNID=&RUNID.,
                        PROJID=&PROJID.,  
                        WPTYPE=&WPTYPE.,
                        WPID=&WPID.,
                        DPID=&DPID.,
                        VERID=&VERID.,
                        PERIODIDSTART=&PERIODIDSTART.,
                        PERIODIDEND=&PERIODIDEND.,
                        ANALYSIS=&ANALYSIS.,
                        MONITORINGFILE=&MONITORINGFILE.,
                        USERSTRATA=&USERSTRATA.,
                        COMBOFILE=&COMBOFILE.,
                        COHORTFILE=&COHORTFILE.,
                        COHORTCODES=&COHORTCODES.,
                        INCLUSIONCODES=&INCLUSIONCODES.,
                        STOCKPILINGFILE=&STOCKPILINGFILE.,
                        RUN_ENVELOPE=&RUN_ENVELOPE.,
                        FREEZEDATA=&FREEZEDATA.,
                        ZIPFILE=&ZIPFILE.,
                        LABSCODEMAP=&LABSCODEMAP.,
                        DISTINDEX=&DISTINDEX.,
                        TYPE1FILE=&TYPE1FILE.,
                        TYPE2FILE=&TYPE2FILE.,
                        TYPE3FILE=&TYPE3FILE.,
                        TYPE4FILE=&TYPE4FILE.,
                        MODPREGDUR=&MODPREGDUR.,    
                        TYPE5FILE=&TYPE5FILE.,
                        TYPE6FILE=&TYPE6FILE.,
                        METADATAFILE=&METADATAFILE.,
                        SURVEILLANCEMODE=&SURVEILLANCEMODE.,
                        COVARIATECODES=&COVARIATECODES.,
                        UTILFILE=&UTILFILE.,
                        COMORBFILE=&COMORBFILE.,
                        DRUGCLASSFILE=&DRUGCLASSFILE.,
                        PROFILE=&PROFILE.,
                        MFUFILE=&MFUFILE.,
                        MULTEVENTFILE=&MULTEVENTFILE.,
                        MULTEVENTFILE_ADHERE=&MULTEVENTFILE_ADHERE.,
                        CONCFILE=&CONCFILE.,
                        OVERLAPFILE=&OVERLAPFILE.,
                        OVERLAPFILE_ADHERE=&OVERLAPFILE_ADHERE.,
                        ITSFILE=&ITSFILE.,
                        TREATMENTPATHWAYS=&TREATMENTPATHWAYS.,
                        MICOHORTFILE=&MICOHORTFILE.,
                        TREEFILE=&TREEFILE.,
                        TREELOOKUP=&TREELOOKUP.,
                        ICD10ICD9MAP=&ICD10ICD9MAP.,
                        HDVARSEL_INPUT=&RUNID._claims_ICDDX09 &RUNID._claims_ICDDX10 &RUNID._claims_DRUGCLASS &RUNID._claims_CPT &RUNID._claims_ICDPX09 &RUNID._claims_ICDPX10 &RUNID._claims_HCPCS,                          
                        PSESTIMATIONFILE=&PSESTIMATIONFILE.,
                        PSMATCHFILE=&PSMATCHFILE.,
                        STRATIFICATIONFILE=&STRATIFICATIONFILE.,
                        COVSTRATFILE=&COVSTRATFILE.,
                        IPTWFILE=&IPTWFILE.,
                        PSCSSUBGROUPFILE=&PSCSSUBGROUPFILE.,
                        INDLEVEL=&INDLEVEL.,
                        DIAGNOSTICS=&DIAGNOSTICS.
                        );  

            *Clean work library;
            proc datasets lib=work nolist kill; quit;

            
            %if %sysevalf(&sysver >= 9.4) %then %do; /*SAS 9.4 only*/

            	/*Evaluate if a previous error has occured - if &SYSCC >4 then abort*/
                %if %eval(&syscc.>4) %then %do;
				 /* Redirect log and call log checker */
			        proc printto log="&MSOC.log_checker.log" new;
                    run;
			        %ms_logchecker(logdir =&MSOC.);
			        
			     /* Redirect back to main log */
                    proc printto log="&MSOC.&RUNID._cida.log";
                    run;
					
                    %put ERROR: (Sentinel) An error has occured so processing will not proceed. Please contact SOC;
                    %abort;
                %end;

                *Add row to checkpoint file indicating complete run;
                data dplocal.&runid._checkpoints;
                    set dplocal.&runid._checkpoints end=eof;
                    output;
                    if eof then do;
                        macrocalled = "complete";
                        date = date();
                        time = time();
                        groupnum = .;
                        output;
                    end;
                run;

            %end;

            %put NOTE: ******** END OF MACRO: runqrp v&QRPVer. ********;

            proc printto;
            run;
			
            /*Delete temporary dplocal and msoc subfolders*/
            %soc_delete_dir(dir=msoctemp);
            %soc_delete_dir(dir=worktemp);
            %soc_delete_dir(dir=wrktemp2);
            %soc_delete_dir(dir=dptemp);
            %soc_delete_dir(dir=medstemp);

            /*Delete freezedata only if empty*/
            %macro isemptyfolder(folder);
            	%let filrf=mydir;
            	%let rc=%sysfunc(filename(filrf, "&folder"));
            	%let did=%sysfunc(dopen(&filrf));
            	%let memcount=%sysfunc(dnum(&did));
            	%let rc=%sysfunc(dclose(&did));
                %if %eval(&memcount.<1) %then %do;
                %soc_delete_dir(dir=freedir);
                %end;
            %mend isemptyfolder;
            %isemptyfolder(&freedir.);

            *delete file sas_environment.sas;
            %macro soc_delete_file(file);
            %if %sysfunc(fileexist(&file)) ge 1 %then %do;
            %let rc=%sysfunc(filename(temp,&file.));
            %let rc=%sysfunc(fdelete(&temp));
            %end; 
            %mend;
            %soc_delete_file(&sasenv.sas_environment.sas);
            %soc_delete_dir(dir=sasenv);

            %nextrun:

        %end; /*loop through each runID*/
    %end;
    %else %do;
        %put ERROR: (Sentinel) Input file qrp_parameters does not exist. QRP will abort;
        %abort;
    %end;

/*************************************************************************************************/
/* Loop through each Report and create report                                                    */
/*************************************************************************************************/
    /*Determine number of reports*/
    %isdata(dataset=infolder.report_parameters);
    %if %eval(&nobs.>0) %then %do;
        proc contents data=infolder.report_parameters noprint out=report_parameters;
        run;
       
        %global numreports numreportparams;
        proc sql noprint;
            select count(*) into: numreports
            from report_parameters
            where substr(upcase(name),1,6) = 'REPORT';

            select count(*) into: numreportparams
            from infolder.report_parameters
        quit;

        %do reportrun = 1 %to %eval(&numreports.);

            *Reset all parameters;
			%global createreportfile reportid reportdata database logofile leavebehindreport;
            %let createreportfile =;
			%let reportid =&reqid.;
			%let dpname =;
            %let logofile = ;
			%let reportdata = N;
			%let database = ;
			%let leavebehindreport = Y;

            /*Assign all parameters to macro variables*/
            %do reportparameter = 1 %to %eval(&numreportparams.);
                data _null_;
                    set infolder.report_parameters;
                    if _n_ = &reportparameter. then do;
                        call symputx("parameter", parameter);
                        call symputx("value", report&reportrun.);
                    end;
                run;
                %let &parameter. = &value.;
            %end;
			
			/* Add underscore for report id */
			%Let reportid = _&reportid.;
			
			/* Create dpinfo file */
			data dpinfofile;
              length dp $10 dpname $100 path $250 database $250 includedp $1;
			  dp = "&dp.";
			  path = "&msoc.";
			  %if %str(&dpname.) = %str() %then %do; dpname = "&dp."; %end;
			  %else %do; dpname = "&dpname."; %end;
			  %if %str(&database.) = %str() %then %do; database = "Sentinel Distributed Database"; %end;
			  %else %do; database = "&database."; %end;
			  includedp = "Y";
			run;
            
			/* Include qrp report for leave behind report */
            %include "&sasmacr.qrp_report.sas";
            
			/* Delete dpinfo file */
			proc datasets nowarn noprint lib=msoc;
              delete dpinfo;
            quit;
        %end; /*loop through each report*/
    %end;

/*************************************************************************************************/
/* Delete checkpoint file                                                                        */
/*************************************************************************************************/

    %do qrprun = 1 %to %eval(&numruns.);
        data _null_;
            set infolder.qrp_parameters(where=(lowcase(parameter)='runid'));
            call symputx("value", run&qrprun.);
        run; 
        %let runid = &value.;

        proc datasets nowarn noprint lib=dplocal;
            delete &runid._checkpoints;
        quit;
    %end;

/*************************************************************************************************/
/* Run log checker                                                                               */
/*************************************************************************************************/
    /* Redirect log and call log checker */
	   proc printto log="&MSOC.log_checker.log" new;
       run;
	   %ms_logchecker(logdir =&MSOC.);
	   
	   proc printto;
       run;
%mend runqrp ;

%runqrp ;