--MARS-770

set verify off
SET serveroutput ON
--SET TIMING ON
whenever sqlerror exit sql.sqlcode rollback;

column owner format a15
column object_name format a30
column spoolname new_value spoolname 

select 'install_package_MarsGeneralRelease_MARS770_packages_4_' || name || '_' ||to_char(sysdate,'YYYYMMDD_HH24MISS')||'.log' spoolname from v$pdbs;
spool '&spoolname'


prompt ##### started at time #####
select systimestamp from dual;
prompt ##### database name #####
select name pdb_name from v$pdbs;

--set echo on
--=============================================================================================================================
-- start install --MARS-770 payload --CT_MRDS.FILE_ARCHIVER package
--=============================================================================================================================

CREATE OR REPLACE EDITIONABLE PACKAGE "CT_MRDS"."FILE_ARCHIVER" 
AUTHID CURRENT_USER
AS
   /**
      * General comment for package: Please put comments for functions and procedures as shown in below example.
      * It is a standard.
      * The structure of comment is used by GET_PACKAGE_DOCUMENTATION function 
      * which returns documentation text for confluence page (to Copy-Paste it).
   **/

   -- Example comment:
         /**
            * @name     EX_PROCEDURE_NAME
            * @desc     Procedure description
            * @example  select LOGGING_AND_ERROR_MANAGER.EX_PROCEDURE_NAME(pParameter => 129) from dual;
            * @ex_rslt  Example Result
         **/

   cgBL       CONSTANT  VARCHAR2(2)   := ENV_MANAGER.cgBL;

   /**
      * @name     HISTORISE_TABLE_DATA
      * @desc     Wrapper procedure for DBMS_CLOUD.EXPORT_DATA.
      *           Exports data from table specified by pSourceFileConfigKey(A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY) into PARQUET file on OCI infrustructure.
      *           Each YEAR_MONTH pair goes to seperate file (implicit partitioning).
   **/
   PROCEDURE HISTORISE_TABLE_DATA (
      pSourceFileConfigKey   IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
   );



   /**
      * @name     GATHER_TABLE_STAT
      * @desc     Gather info about EXTERNAL TABLE specified by pSourceFileConfigKey parameter (A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY).
      *           Data is inserted into A_TABLE_STAT and A_TABLE_STAT_HIST.
   **/
   PROCEDURE GATHER_TABLE_STAT (
      pSourceFileConfigKey   IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
   );

END;
/

CREATE OR REPLACE EDITIONABLE PACKAGE BODY "CT_MRDS"."FILE_ARCHIVER" 
AS
  
   ----------------------------------------------------------------------------------------------------

   FUNCTION GET_TABLE_STAT(pSourceFileConfigKey   IN NUMBER)
   RETURN CT_MRDS.A_TABLE_STAT%ROWTYPE
   IS
      vTableStat        CT_MRDS.A_TABLE_STAT%ROWTYPE;
      vParameters       CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
      vCount            PLS_INTEGER;
      vSourceFileType   CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE%TYPE;

   BEGIN
      vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey),NULL)));
      ENV_MANAGER.LOG_PROCESS_EVENT('Start','DEBUG', vParameters);
      SELECT count(*) , min(SOURCE_FILE_TYPE)
        INTO vCount, vSourceFileType
        FROM CT_MRDS.A_TABLE_STAT s
        JOIN CT_MRDS.A_SOURCE_FILE_CONFIG c
          ON s.A_SOURCE_FILE_CONFIG_KEY = c.A_SOURCE_FILE_CONFIG_KEY
       WHERE s.A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;

      IF vCount=0 and vSourceFileType='INPUT' THEN
         GATHER_TABLE_STAT(pSourceFileConfigKey);
      END IF;

      BEGIN
         SELECT *
           INTO vTableStat
           FROM CT_MRDS.A_TABLE_STAT
          WHERE A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
--      EXCEPTION
--         WHEN NO_DATA_FOUND THEN
--            
      END;

      ENV_MANAGER.LOG_PROCESS_EVENT('End','DEBUG',vParameters);
      RETURN vTableStat;

   END GET_TABLE_STAT;

   ----------------------------------------------------------------------------------------------------

   PROCEDURE HISTORISE_TABLE_DATA (
      pSourceFileConfigKey   IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
   )
   IS
      vSourceFileConfig          CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
      vTableStat                 CT_MRDS.A_TABLE_STAT%ROWTYPE;
      vQuery                     VARCHAR2(4000);
      vTableName                 VARCHAR2(200);
      vUri                       VARCHAR2(1000);
      vfiles                     T_FILENAMES;
      vFilename                  VARCHAR2(300);
      vOperationId               NUMBER := -1;
      vParameters                CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
      vHistorisationTriggeredBy  VARCHAR2(60); -- Possible values: FILES_COUNT, ROWS_COUNT, BYTES_SUM
      vUserLoadOperations        USER_LOAD_OPERATIONS%ROWTYPE;
      vProcessControlStatus      VARCHAR2(60) := 'OK';

   BEGIN
      vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
      ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);

      vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);
      vTableStat        := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);

      if vSourceFileConfig.SOURCE_FILE_TYPE <> 'INPUT' then
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE, 'ERROR', vParameters);
         RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE);
      end if;

      if vTableStat.created < sysdate-(vSourceFileConfig.HOURS_TO_EXPIRE_STATISTICS/24) then
         GATHER_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
         vTableStat        := GET_TABLE_STAT(pSourceFileConfigKey => pSourceFileConfigKey);
      end if;

      if    vTableStat.OVER_HIST_THRESOLD_FILE_COUNT  >= vSourceFileConfig.FILES_COUNT_OVER_HISTORY_THRESHOLD then vHistorisationTriggeredBy := 'FILES_COUNT';
      elsif vTableStat.OVER_HIST_THRESOLD_ROW_COUNT   >= vSourceFileConfig.ROWS_COUNT_OVER_HISTORY_THRESHOLD  then vHistorisationTriggeredBy := vHistorisationTriggeredBy||', ROWS_COUNT';
      elsif vTableStat.OVER_HIST_THRESOLD_SIZE        >= vSourceFileConfig.BYTES_SUM_OVER_HISTORY_THRESHOLD   then vHistorisationTriggeredBy := vHistorisationTriggeredBy||', BYTES_SUM';
      else ENV_MANAGER.LOG_PROCESS_EVENT('Non of historisation triggers reached','INFO');
      end if;

      if LENGTH(vHistorisationTriggeredBy)>0 THEN 
         ENV_MANAGER.LOG_PROCESS_EVENT('Historisation Triggered By: '||vHistorisationTriggeredBy,'INFO');
         vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
         vQuery := '
         select t_filename(
             file$name
            ,file$path
            , to_char(h.workflow_start,''yyyy'')
            , to_char(h.workflow_start,''mm'')
            )

         from '||vTableName||' s
         join CT_MRDS.a_workflow_history h 
           on s.a_workflow_history_key = h.a_workflow_history_key
         where extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD
         ;

         -- Get all files that will be historised into  "vfiles" collection ("regular data files")
         execute immediate vQuery bulk collect into vfiles;

         -- Start EXPORT "regular data files" to parquet and DROP "csv"
         FOR ym_loop IN (select distinct year, month from table(vfiles) order by 1,2) LOOP
            dbms_output.put_line('year: '||ym_loop.year||' - '||'month: '||ym_loop.month);
            vQuery:=
            'select 
                s.*
--                  ,r.partition_year
--                  ,r.partition_month
            from '||  vTableName   ||' s
            join CT_MRDS.A_SOURCE_FILE_RECEIVED r
              on s.file$name = r.source_file_name
             and r.a_source_file_config_key = '||pSourceFileConfigKey||'
             and r.partition_year='''||ym_loop.year||'''
             and r.partition_month='''||ym_loop.month||'''
             and r.PROCESSING_STATUS = ''INGESTED''
             '
             ;
            vUri := FILE_MANAGER.GET_BUCKET_URI('HISTORY')||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/YEAR='||ym_loop.year||'/MONTH='||ym_loop.month||'/';

            ENV_MANAGER.LOG_PROCESS_EVENT('Start Historization for YEAR_MONTH:  '||ym_loop.year||'_'||ym_loop.month ,'INFO');
            ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => file_uri_list'  ,'DEBUG',vUri);
            ENV_MANAGER.LOG_PROCESS_EVENT('Parameter for DBMS_CLOUD.EXPORT_DATA => query'          ,'DEBUG',vQuery);



            BEGIN
               DBMS_CLOUD.EXPORT_DATA(
                  credential_name => ENV_MANAGER.gvCredentialName,
                  file_uri_list => vUri||'d' ,
                  format => json_object('type' value 'parquet'),
                  query => vQuery,
                  operation_id => vOperationId
                 );
            EXCEPTION 
               WHEN OTHERS THEN
               vProcessControlStatus :='EXPORT_FAILURE';
               ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
               RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_HIST_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED);

            END;

            ENV_MANAGER.LOG_PROCESS_EVENT('vOperationId of export: '||vOperationId,'DEBUG');

            -- Get USER_LOAD_OPERATIONS info
            select *
              into vUserLoadOperations
              from USER_LOAD_OPERATIONS 
             where id = vOperationId;

            IF     vUserLoadOperations.STATUS <>'COMPLETED' THEN
               ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED := ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED 
                          ||cgBL|| '           Export ended with status '||vUserLoadOperations.STATUS;
               RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_HIST_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED);
            ELSIF  vUserLoadOperations.STATUS = 'COMPLETED' and vUserLoadOperations.ROWS_LOADED = 0 THEN
               ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED := ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED 
                          ||cgBL|| '           Zero rows were exported.';
               RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_HIST_FAILED, ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED);
            ELSE
               ENV_MANAGER.LOG_PROCESS_EVENT('Data exported to historization file for YEAR_MONTH:  2025_01','INFO', vParameters);
               ENV_MANAGER.LOG_PROCESS_EVENT(FILE_MANAGER.GET_DET_USER_LOAD_OPERATIONS (pOperationId => vOperationId),'DEBUG', vParameters);
            END IF;

            SELECT 
               object_name
            into vFilename
            from DBMS_CLOUD.LIST_OBJECTS(
                credential_name => 'OCI$RESOURCE_PRINCIPAL',
                location_uri => vUri)
            where TO_UTC_TIMESTAMP_TZ(REGEXP_REPLACE(object_name,  '.*(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{6})Z\.parquet$', '\1-\2-\3T\4:\5:\6.\7'))
            between vUserLoadOperations.START_TIME
               and vUserLoadOperations.UPDATE_TIME
                ;

            -- Try to drop EXPORTED FILES ("regular data files")
            BEGIN
               FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) loop

                  -- first change of status
                  BEGIN
                     UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
                        SET PROCESSING_STATUS = 'HISTORISED'
                           ,HIST_FILE_NAME = vUri||vFilename
                     WHERE r.a_source_file_config_key= pSourceFileConfigKey
                       AND r.source_file_name = f.filename
                       AND r.processing_status = 'INGESTED'
                     ;
                  EXCEPTION
                     WHEN OTHERS THEN
                        ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
                        vProcessControlStatus := 'CHANGE_STATUS_TO_HISTORISED_FAILURE';
                  END;
                  EXIT WHEN vProcessControlStatus = 'CHANGE_STATUS_TO_HISTORISED_FAILURE';

                  -- move file to trash before dropping
                  BEGIN
                     DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, 
                          source_object_uri      => f.pathname||'/'||f.filename,
                          target_object_uri      => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
                          target_credential_name => ENV_MANAGER.gvCredentialName
                         );
                     ENV_MANAGER.LOG_PROCESS_EVENT('File moved to TRASH.','DEBUG', f.pathname||'/'||f.filename);
                  EXCEPTION
                     WHEN OTHERS THEN
                        ENV_MANAGER.LOG_PROCESS_EVENT('Failed to move file to TRASH.','ERROR', f.pathname||'/'||f.filename);
                        ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
                        rollback;
                        vProcessControlStatus := 'MOVE_FILE_TO_TRASH_FAILURE';
                  END;
                  EXIT WHEN vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE';
                  commit;
               END LOOP;

               --------------------------------------------------------------------
               -- IF All goes fine till this point, we drop files from TRASH (if not then ROLLBACK PART)
               IF vProcessControlStatus = 'OK' THEN
                  FOR f in (select filename, pathname from table(vfiles) where year = ym_loop.year and month = ym_loop.month) LOOP
                     --Drop file from TRASH
                     DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
                                                   object_uri => replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
                     ENV_MANAGER.LOG_PROCESS_EVENT('File dropped from TRASH.','DEBUG', f.pathname||'/'||f.filename);
                  END LOOP;

               --ROLLBACK PART
               --ROLLBACK PROCESS in case of FAILURE (restore files from TRASH)
               ELSIF vProcessControlStatus = 'MOVE_FILE_TO_TRASH_FAILURE' THEN
                  FOR f in ( SELECT vf.filename, vf.pathname 
                              FROM TABLE(vfiles) vf
                              JOIN CT_MRDS.A_SOURCE_FILE_RECEIVED r
                                ON r.source_file_name         = vf.filename
                               AND r.a_source_file_config_key = pSourceFileConfigKey
                               AND r.PROCESSING_STATUS = 'HISTORISED'
                               AND vf.year  = ym_loop.year 
                               AND vf.month = ym_loop.month
                           ) LOOP
                     BEGIN
                        DBMS_CLOUD.MOVE_OBJECT(source_credential_name => ENV_MANAGER.gvCredentialName, 
                             source_object_uri      => replace(f.pathname,'ODS','TRASH')||'/'||f.filename,
                             target_object_uri      => f.pathname||'/'||f.filename,
                             target_credential_name => ENV_MANAGER.gvCredentialName
                            );
                        ENV_MANAGER.LOG_PROCESS_EVENT('File restored from TRASH.','DEBUG', f.pathname||'/'||f.filename);

                        UPDATE CT_MRDS.A_SOURCE_FILE_RECEIVED r
                           SET PROCESSING_STATUS = 'INGESTED'
                              ,HIST_FILE_NAME = NULL
                        WHERE r.a_source_file_config_key = pSourceFileConfigKey
                          AND r.source_file_name = f.filename
                        ;

                     EXCEPTION
                        WHEN OTHERS THEN
                           ENV_MANAGER.LOG_PROCESS_EVENT('Failed to restore file from TRASH.','ERROR', replace(f.pathname,'ODS','TRASH')||'/'||f.filename);
                           ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
                           vProcessControlStatus := 'RESTORE_FILE_FROM_TRASH_FAILURE';
                     END;
                  END LOOP;

                  DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
                                                      object_uri => vUri||vFilename);                                                   
                  ENV_MANAGER.LOG_PROCESS_EVENT('ROLLBACK operation: Historization PARQUET file dropped.','DEBUG', vUri||vFilename);
                  RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);

               ELSIF vProcessControlStatus = 'CHANGE_STATUS_TO_HISTORISED_FAILURE' THEN
                  ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_CHANGE_STAT_TO_HISTORISED_FAILED, 'ERROR', vParameters);
                  DBMS_CLOUD.DELETE_OBJECT(credential_name => ENV_MANAGER.gvCredentialName,
                                                      object_uri => vUri||vFilename);                                                   
                  ENV_MANAGER.LOG_PROCESS_EVENT('Historization PARQUET file dropped.','DEBUG', vUri||vFilename);
                  RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_HISTORISED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_HISTORISED_FAILED);

               ELSIF vProcessControlStatus = 'RESTORE_FILE_FROM_TRASH_FAILURE' THEN
                  ENV_MANAGER.LOG_PROCESS_EVENT('Some files were not restored from TRASH. Check A_PROCESS_LOG table for details','ERROR');
                  RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);
               END IF;

            EXCEPTION
               WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_HISTORISED_FAILED THEN
                  RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_HISTORISED_FAILED, ENV_MANAGER.MSG_CHANGE_STAT_TO_HISTORISED_FAILED);

               WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
                  RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.MSG_MOVE_FILE_TO_TRASH_FAILED);

               WHEN ENV_MANAGER.ERR_RESTORE_FILE_FROM_TRASH THEN
                  ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
                  RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_RESTORE_FILE_FROM_TRASH, ENV_MANAGER.MSG_RESTORE_FILE_FROM_TRASH);

               WHEN OTHERS THEN
                  ENV_MANAGER.LOG_PROCESS_EVENT('Error during historization process','ERROR');
                  ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
                  RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_DROP_EXPORTED_FILES_FAILED, ENV_MANAGER.MSG_DROP_EXPORTED_FILES_FAILED);
            END; 
            -- END of "Try to drop EXPORTED FILES"

            ENV_MANAGER.LOG_PROCESS_EVENT('All historized files had been dropped.','INFO');            
            ENV_MANAGER.LOG_PROCESS_EVENT('End Historization for YEAR_MONTH:  '||ym_loop.year||'_'||ym_loop.month ,'INFO');   
         END LOOP; --ym_loop end (YEAR_MONTH)

         COMMIT;

      ELSE
         ENV_MANAGER.LOG_PROCESS_EVENT('Non of historization thresholds reached. Skip historization.'||vHistorisationTriggeredBy,'INFO');
      END IF;      

      ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
   EXCEPTION
      WHEN ENV_MANAGER.ERR_NOT_INPUT_SOURCE_FILE_TYPE THEN
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_NOT_INPUT_SOURCE_FILE_TYPE , 'ERROR', vParameters);
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
         RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_NOT_INPUT_SOURCE_FILE_TYPE, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));

      WHEN ENV_MANAGER.ERR_EXP_DATA_FOR_HIST_FAILED THEN
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_EXP_DATA_FOR_HIST_FAILED , 'ERROR', vParameters);
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
         RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_EXP_DATA_FOR_HIST_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));

      WHEN ENV_MANAGER.ERR_CHANGE_STAT_TO_HISTORISED_FAILED THEN
         RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_CHANGE_STAT_TO_HISTORISED_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));

      WHEN ENV_MANAGER.ERR_MOVE_FILE_TO_TRASH_FAILED THEN
         RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_MOVE_FILE_TO_TRASH_FAILED, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));         

      WHEN OTHERS THEN
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
         RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));

   END HISTORISE_TABLE_DATA;

   ----------------------------------------------------------------------------------------------------

   PROCEDURE GATHER_TABLE_STAT (
      pSourceFileConfigKey   IN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY%TYPE
   ) IS
      vParameters       CT_MRDS.A_PROCESS_LOG.PROCEDURE_PARAMETERS%TYPE;
      vStats            CT_MRDS.A_TABLE_STAT%ROWTYPE;
      vSourceFileConfig CT_MRDS.A_SOURCE_FILE_CONFIG%ROWTYPE;
      vTableName        VARCHAR2(200);
      vQuery            VARCHAR2(32000);
   BEGIN
      vParameters := ENV_MANAGER.FORMAT_PARAMETERS(SYS.ODCIVARCHAR2LIST('pSourceFileConfigKey => '||nvl(to_char(pSourceFileConfigKey), 'NULL')));
      ENV_MANAGER.LOG_PROCESS_EVENT('Start','INFO', vParameters);
      vSourceFileConfig := FILE_MANAGER.GET_SOURCE_FILE_CONFIG(pSourceFileConfigKey => pSourceFileConfigKey);

      vTableName := DBMS_ASSERT.SCHEMA_NAME(vSourceFileConfig.ODS_SCHEMA_NAME) || '.'||vSourceFileConfig.A_SOURCE_KEY||'_'||DBMS_ASSERT.simple_sql_name(vSourceFileConfig.TABLE_ID)||'_ODS';
      ENV_MANAGER.LOG_PROCESS_EVENT('vTableName','DEBUG',vTableName);
      vQuery := 
      'with tmp as (
         select 
             s.*
            ,file$name as filename
            ,h.workflow_start
            , to_char(h.workflow_start,''yyyy'') as year
            , to_char(h.workflow_start,''mm'') as month
         from '||vTableName||' s
         join CT_MRDS.a_workflow_history h 
           on s.a_workflow_history_key = h.a_workflow_history_key
         )
      , tmp_gr as (
         select 
            filename, count(*) as row_count_per_file, min(workflow_start) as workflow_start
         from tmp
         group by filename
         )
      select 
          NULL                   as A_TABLE_STAT_KEY
         ,'||pSourceFileConfigKey||'     as A_SOURCE_FILE_CONFIG_KEY
         ,'''||vTableName||''' as TABLE_NAME
         ,count(*) as FILE_COUNT
         ,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD||' then 1 else 0 end) as OLD_FILE_COUNT
         ,sum (row_count_per_file) as ROW_COUNT
         ,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD||' then row_count_per_file else 0 end) as OLD_ROW_COUNT
         ,sum(r.bytes) as BYTES
         ,sum(case when extract(day from (systimestamp - workflow_start)) > '||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD||' then r.bytes else 0 end) as OLD_BYTES
         ,'||vSourceFileConfig.DAYS_FOR_HISTORY_THRESHOLD||' as DAYS_FOR_HISTORY_THRESHOLD
         ,systimestamp as CREATED
      from tmp_gr t
      join (SELECT * from DBMS_CLOUD.LIST_OBJECTS(
                      credential_name => '''||ENV_MANAGER.gvCredentialName||''',
                      location_uri => FILE_MANAGER.GET_BUCKET_URI(''ODS'')||''ODS/'||vSourceFileConfig.A_SOURCE_KEY||'/'||vSourceFileConfig.TABLE_ID||'/''
                     )
            ) r
      on t.filename = r.object_name'
      ;
      ENV_MANAGER.LOG_PROCESS_EVENT('vQuery','DEBUG',vQuery);
      execute immediate vQuery into vStats;

      vStats.A_TABLE_STAT_KEY := CT_MRDS.A_TABLE_STAT_KEY_SEQ.NEXTVAL;
      insert into A_TABLE_STAT_HIST values vStats;
      delete from A_TABLE_STAT where A_SOURCE_FILE_CONFIG_KEY = pSourceFileConfigKey;
      insert into A_TABLE_STAT values vStats;
      COMMIT;

      ENV_MANAGER.LOG_PROCESS_EVENT('End','INFO',vParameters);
   EXCEPTION
      WHEN OTHERS THEN
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.MSG_UNKNOWN , 'ERROR', vParameters);
         ENV_MANAGER.LOG_PROCESS_EVENT(ENV_MANAGER.GET_ERROR_STACK(pFormat => 'TABLE', pCode=> SQLCODE), 'ERROR', vParameters);
         RAISE_APPLICATION_ERROR(ENV_MANAGER.CODE_UNKNOWN, ENV_MANAGER.GET_ERROR_STACK(pFormat => 'OUTPUT', pCode=> SQLCODE));
   END GATHER_TABLE_STAT;

   ----------------------------------------------------------------------------------------------------

END;