/**
   @file combo_processinputfiles.sas
   @brief This macro imports the Combo and Combo Codes input files, processes them by assigning default values, and if executing as part of QRP combines codes to extract from combo and QRP.

   @details 
	1. Import CombFile:
		- Set default values.		
		- Convert special characters to underscores.		
			- Upcase added to prevent group deduplication caused by uppercase/lowercase typos in the input file.			
	2. CodesFile:		
		- Temporality creating codetype for MS_ProcessWildcards.	
		- Process wildcards.	
		- Put non diagnosis codes back in the CodesFile.		
		- Sort by RawOrder to make sure the number of RawGroups to process by the Two-By-Two Method will be calculated correctly.		
		- Create lookup tables for raw record extraction.	
  
   @par Program inputs
	- infolder.&CODESFILE. (Dataset defining the combo codes input file.)
	- infolder.&COMBFILE. (Dataset defining the combo input file.)
	- work._diag (Lookup dataset from QRP containing diagnosis codes to extract (if running as part of QRP).
	- work._lab (Lookup dataset from QRP containing raw lab result codes to extract (if running as part of QRP).
	- work._ndc (Lookup dataset from QRP containing dispensing codes to extract (if running as part of QRP).
	- work._proc (Lookup dataset from QRP containing procedure codes to extract (if running as part of QRP).

   @par Program outputs
	- work._diag (Lookup dataset containing diagnosis codes to extract (if running as part of QRP).
	- work._lab (Lookup dataset containing raw lab result codes to extract (if running as part of QRP).
	- work._ndc (Lookup dataset containing dispensing codes to extract (if running as part of QRP).
	- work._proc (Lookup dataset containing procedure codes to extract (if running as part of QRP).
	- work._clab (Lookup dataset containing lab result codes to extract (if running as part of QRP).
	- work._cenr (Lookup dataset containing enrollment records to extract (if running as part of QRP).
	- work._cenc (Lookup dataset containing encounter records to extract (if running as part of QRP).
	- work._cdem (Lookup dataset containing demographic records to extract (if running as part of QRP).

* **Usage**

	  %combo_processinputfiles(COMBFILE=&COMBOFILE.,
					           CODESFILE=&COMBOFILE.Codes)

   @param [in] COMBFILE Name of the SAS dataset defining the Combo Input File.
   @param [in] CODESFILE Name of the SAS dataset defining the Combo Codes Input File. 

<h4> SAS Macros Dependencies </h4>
   @li ms_processwildcards.sas
   @li isdata.sas

   @author Sentinel Coordinating Center (info@sentinelsystem.org)
**/

%macro combo_processinputfiles(COMBFILE=,
                   			   CODESFILE=);

%PUT =====> MACRO CALLED: combo_processinputfiles;   

	/*utility macro*/
	*Convert special characters to underscores;
    %macro trans(var=);
        *Upcase added to prevent group deduplication caused by uppercase/lowercase typos in the input file;
        &var. = upcase(compress(trim(left(&var.))));
        &var. = translate(&var.,"_","-",
                                "_",".",
                                "_",",",
                                "_","%",
                                "_","$",
                                "_","!",
                                "_","*",
                                "_","&",
                                "_","#",
                                "_","@");
    %mend trans;

	/*Combo file import*/
    %IF %INDEX(%UPCASE("&COMBFILE."),CPORT) %THEN %DO;
        proc cimport infile="&infolder.&COMBFILE." library=infolder memtype=data;
        run;
    %END;
    

	/* Because the combo input file variables will be written to disk for many observations it is more efficient to reduce their size to the possible minimum */
	%global rawgroup_len;
	proc sql noprint;
	select max(length(combgroup)),		   
		   max(length(combenctype)),  
		   max(length(combpdx)),
		   max(length(rawgroup)),
		   max(length(RawDaysFromStartDt)),
		   max(length(RawDaysFromEndDt)),
		   max(length(RawADateType)),
		   max(length(RawEDateType))
	into :combgroup_len trimmed, 		 
		 :combenctype_len trimmed,
		 :combpdx_len trimmed,
		 :rawgroup_len trimmed,
		 :RawDaysFromStartDt_len trimmed,
		 :RawDaysFromEndDt_len trimmed,
		 :RawADateType_len trimmed,
		 :RawEDateType_len trimmed
	from infolder.&COMBFILE.;
	quit;

    data _COMBFILE;
		length combcode $&codelength. NewRawEpiType NewRawEpiGap NewRawExclusion 3;			
		informat combcode $&codelength..;			
		format combcode $&codelength..;	
	    set infolder.&COMBFILE.;
	    %trans(var=CombGroup);
	    %trans(var=RawGroup);
	    CombPDX = upcase(CombPDX);
	    CombEncType=upcase(CombEncType);
		if missing(RawEpiType) then RawEpiGap=0;
		if missing(RawEpiGap) then RawEpiGap=0;
		if missing(RawExclusion) then RawExclusion=0;
		
		format NewCombGroup $&combgroup_len.. NewCombEncType $&combenctype_len.. NewCombPDX $&combpdx_len.. NewRawGroup $&rawgroup_len.. 
			   NewRawDaysFromStartDt $&RawDaysFromStartDt_len.. NewRawDaysFromEndDt $&RawDaysFromEndDt_len.. NewRawADateType $&RawADateType_len.. NewRawEDateType $&RawEDateType_len..;
		NewCombGroup=CombGroup;	
		NewCombEncType=CombEncType; 
		NewCombPDX=CombPDX;
		NewRawGroup=RawGroup;
		NewRawEpiType=RawEpiType;
		NewRawEpiGap=RawEpiGap;
		NewRawExclusion=RawExclusion;
		NewRawDaysFromStartDt=RawDaysFromStartDt;
		NewRawDaysFromEndDt=RawDaysFromEndDt;
		NewRawADateType=RawADateType;
		NewRawEDateType=RawEDateType;

		rename NewCombGroup=CombGroup;	
		rename NewCombEncType=CombEncType; 
		rename NewCombPDX=CombPDX;
		rename NewRawGroup=RawGroup;
		rename NewRawEpiType=RawEpiType;
		rename NewRawEpiGap=RawEpiGap;
		rename NewRawExclusion=RawExclusion;
		rename NewRawDaysFromStartDt=RawDaysFromStartDt;
		rename NewRawDaysFromEndDt=RawDaysFromEndDt;
		rename NewRawADateType=RawADateType;
		rename NewRawEDateType=RawEDateType;	
		keep NewCombGroup CombOrder CombCode CombBehavior NewCombEncType NewCombPDX NewRawGroup RawOrder NewRawEpiType 
		     NewRawEpiGap NewRawExclusion NewRawDaysFromStartDt NewRawDaysFromEndDt NewRawADateType NewRawEDateType RawDurat;
    run;    

	/*Combocodes file import*/
	%IF %INDEX(%UPCASE("&CODESFILE."),CPORT) %THEN %DO;
        proc cimport infile="&infolder.&CODESFILE." library=infolder memtype=data;
        run;
    %END;

	proc sql noprint;
		select max(length(rawgroup)),
			   max(length(rawstockgroup)),
			   max(length(rawcaresetting)),  
			   max(length(rawpdx))
		into :rawgroup_len trimmed, 
			 :rawstockgroup_len trimmed,
			 :rawcaresetting_len trimmed,
			 :rawpdx_len trimmed
		from infolder.&CODESFILE.;
	quit;

    data _CODESFILE _OTHCODE;
		length rawcode $&codelength.;			
		informat rawcode $&codelength..;			
		format rawcode $&codelength..;	
	    set infolder.&CODESFILE.;       
	    %trans(var=RawGroup);
	    RawCode=compress(RawCode, '.' );

		format NewRawGroup $&rawgroup_len.. NewRawStockgroup $&rawstockgroup_len.. NewRawCaresetting $&rawcaresetting_len.. NewRawPDX $&rawpdx_len..;
		NewRawGroup=RawGroup;
		NewRawStockgroup=RawStockgroup;
		NewRawCaresetting=RawCaresetting; 
		NewRawPDX=RawPDX;
		rename NewRawGroup=RawGroup;
		rename NewRawStockgroup=RawStockgroup;
		rename NewRawCaresetting=RawCaresetting; 
		rename NewRawPDX=RawPDX;
		drop RawGroup RawStockgroup RawCaresetting RawPDX;

	    if RawCodeType in:("DX") then do;
	        *temporality creating codetype for MS_ProcessWildcards;
	        codetype=substr(RawCodeType,3,2);
	        output _CODESFILE;
	    end;
	    else output _OTHCODE;
    run;

    %ISDATA(dataset=_CODESFILE);
    %IF %eval(&NOBS.>0) %THEN %DO;   
        *Process wildcards;
        %MS_ProcessWildcards(InFile=_CODESFILE, 
                             CodeVar=RawCode,
                             codetype=CodeType, 
                             OutFile=_CODESFILE);
    %END;

    *put non diag codes back in the codesfile;
    data _CODESFILE;
	    set  _OTHCODE _CODESFILE ;
		length RawLength 3;
	    RawLength=length(RawCode); 
	    drop codetype;
    run;            

	*Sort by RawOrder to make sure the number of RawGroups to process by the Two-By-Two Method will be calculated correctly;
	proc sort data=_COMBFILE;
		by CombOrder CombGroup RawOrder;
	run;

	proc sort nodupkey data = _COMBFILE out = _CombGroupList(keep=CombGroup);
		by CombOrder CombGroup; 
	run;

	*Create lookup tables for raw record extraction;
	data _cdiag
	     _cproc
	     _cndc (keep=RawCode RawCodeType Codetype)
	     _cenc
	     _cdem
	     _cenr
	     _clab;
		set _CODESFILE;
		    length=length(RawCode);
		    RawPDX = upcase(RawPDX);               
		    RawCaresetting = upcase(RawCaresetting);         

		    codetype='   ';
		    if RawCodeType in:("DX") then do; codetype=compress(RawCodeType,"DX"); output _cdiag; end;
		    else if RawCodeType in:("RX") then do; codetype=compress(RawCodeType,"RX"); output _cndc; end;
		    else if RawCodeType in:("PX") then do; codetype=compress(RawCodeType,"PX"); output _cproc; end;   
		    else if RawCodeType in:("EN") then do; codetype=compress(RawCodeType,"EN"); output _cenc; end;
		    else if RawCodeType in:("DM") then do; codetype=compress(RawCodeType,"DM"); output _cdem; end; 
		    else if RawCodeType in:("EL") then do; codetype=compress(RawCodeType,"EL"); output _cenr; end;   
		    else if RawCodeType in:("LAB") then do; codetype=compress(RawCodeType,"LAB"); output _clab; end;   
	run;

	%MACRO CREATELOOKUP(file=);
	    *To verify if the _ndc dataset exists and has some observations. 
	     This dataset will come from the Modular program (if combo tool is not used as standalone);
	    %ISDATA(dataset=_&file.);
	    %IF %eval(&NOBS.>0) %THEN %DO;   *diags were also defined in cida;
	        data _&file.;
	        set _&file.(in=a) _c&file.(in=b);
			length cida comb 3;
	            if a then do;
					cida =1;
					comb=0;
					%if (&file.) = lab %then %do;
						rawlabresult=labresult;
						rawlabdatetype=labdatetype;
						drop labresult labdatetype;
					%end;
				end;
	            if b then do;
	               comb =1;
				   cida=0;
	               Code=RawCode;
	               CODECAT=substr(RawCodeType,1,2);
	            end;
	        run;
	    %END;
	    %ELSE %DO;
	        data _&file.;
	        set _c&file.;
				length cida comb 3;
	            Comb =1;
	            cida =0;
	            Code=RawCode;
	            CODECAT=substr(compress(RawCodeType,'Aa'),1,2);
	        run;
	    %END;

        *cleaning lookup file in case the code is used both in combo and in QRP;
		%if &file. ne lab %then %do; /*lab codes have other variables and deduplication is managed in combo*/
			proc means data=_&file. nway noprint;
		        var Comb cida ;
		        class Code Codetype;
		        output out=_&file.(keep=code Codetype comb cida) max= / keeplen;
        	run;

			/*collapse cida/comb to 1 variable*/
			data _&file.(drop=comb cida);
				set _&file.;
				if comb = 1 and cida ne 1 then qrpcombocode='C'; /*used in combo, not in QRP*/
				else if comb ne 1 and cida = 1 then qrpcombocode='Q'; /*used in QRP, not in combo*/
				else if comb = 1 and cida = 1 then qrpcombocode='B'; /*used in combo and QRP*/
			run;
		%end;

	%MEND CREATELOOKUP;
	%CREATELOOKUP(file=ndc);
	%CREATELOOKUP(file=diag);
	%CREATELOOKUP(file=proc);
	%CREATELOOKUP(file=lab);

 	*clean up;
    proc datasets library=WORK nowarn nolist;
    delete _OTHCODE _cdiag _cProc _cndc; /*_cenr _cenc _cdem are combo only datasets and need to be available for claim extraction*/
    quit;

%PUT =====> END MACRO: combo_processinputfiles;   

%mend combo_processinputfiles;