diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/01b_MARS_828_add_column_comments.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/01b_MARS_828_add_column_comments.sql new file mode 100644 index 0000000..f0ecacd --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/01b_MARS_828_add_column_comments.sql @@ -0,0 +1,160 @@ +-- ===================================================================== +-- Script: 01b_MARS_828_add_column_comments.sql +-- MARS Issue: MARS-828 +-- Author: Grzegorz Michalski +-- Date: 2026-02-20 +-- Purpose: Add comprehensive column comments for A_SOURCE_FILE_CONFIG and A_SOURCE_FILE_RECEIVED tables +-- Description: Documents all columns to improve database maintainability and user understanding +-- ===================================================================== + +PROMPT ======================================== +PROMPT MARS-828: Adding comprehensive column comments +PROMPT ======================================== + +-- ===================================================================== +-- A_SOURCE_FILE_CONFIG Column Comments +-- ===================================================================== + +PROMPT Adding column comments for A_SOURCE_FILE_CONFIG... + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS + 'Primary key - unique identifier for source file configuration record'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS + 'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS + 'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS + 'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS + 'Human-readable description of the source file and its purpose'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS + 'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS + 'Identifier for the target table where data will be loaded (without schema prefix)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS + 'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS + 'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS + 'Threshold for THRESHOLD_BASED strategy: archive data older than N days'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS + 'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS + 'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS + 'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS + 'Schema name where ODS external tables are created (typically ODS)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS + 'Number of hours before table statistics expire and need to be recalculated'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS + 'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS + 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS + 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS + 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS + 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy'; + +-- ===================================================================== +-- A_SOURCE_FILE_RECEIVED Column Comments +-- ===================================================================== + +PROMPT Adding column comments for A_SOURCE_FILE_RECEIVED... + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS + 'Primary key - unique identifier for received file record'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS + 'Foreign key to A_SOURCE_FILE_CONFIG - links file to its configuration and processing rules'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS + 'Full object name/path of the received file in OCI Object Storage (includes INBOX prefix)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS + 'MD5 checksum of file content for integrity verification and duplicate detection'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS + 'Timestamp with timezone when file was created/uploaded to Object Storage (from DBMS_CLOUD.LIST_OBJECTS)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS + 'File size in bytes'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS + 'Date when file was registered in the system (extracted from CREATED timestamp)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS + 'Current processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS + 'Name of temporary external table created for file validation (dropped after validation)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS + 'Year partition value (YYYY format) when file was archived to ARCHIVE bucket with Hive-style partitioning'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS + 'Month partition value (MM format) when file was archived to ARCHIVE bucket with Hive-style partitioning'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_PATH IS + 'Archive directory prefix in ARCHIVE bucket containing archived Parquet files (supports multiple files from parallel DBMS_CLOUD.EXPORT_DATA)'; + +-- ===================================================================== +-- Verification +-- ===================================================================== + +PROMPT +PROMPT Verifying column comments... +PROMPT + +SELECT + table_name, + COUNT(*) as total_columns, + COUNT(comments) as documented_columns, + COUNT(*) - COUNT(comments) as undocumented_columns +FROM all_col_comments +WHERE owner = 'CT_MRDS' + AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED') +GROUP BY table_name +ORDER BY table_name; + +PROMPT +PROMPT Detailed column documentation status: +PROMPT + +SELECT + table_name, + column_name, + CASE WHEN comments IS NULL THEN 'MISSING' ELSE 'OK' END as comment_status +FROM all_col_comments +WHERE owner = 'CT_MRDS' + AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED') +ORDER BY table_name, column_name; + +PROMPT +PROMPT ======================================== +PROMPT Column comments added successfully +PROMPT ======================================== +PROMPT A_SOURCE_FILE_CONFIG: All 20 columns documented +PROMPT A_SOURCE_FILE_RECEIVED: All 12 columns documented +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql index 3f431b8..9b4d70b 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/94_MARS_828_rollback_columns.sql @@ -20,7 +20,10 @@ DROP CONSTRAINT CHK_IS_KEEP_IN_TRASH; -- Drop columns ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP ( ARCHIVAL_STRATEGY, - MINIMUM_AGE_MONTHS, + MINIMUM_AGE_MONTHS +); + +ALTER TABLE CT_MRDS.A_SOURCE_FILE_CONFIG DROP ( IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH ); diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/94b_MARS_828_rollback_column_comments.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/94b_MARS_828_rollback_column_comments.sql new file mode 100644 index 0000000..f878eb7 --- /dev/null +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/94b_MARS_828_rollback_column_comments.sql @@ -0,0 +1,84 @@ +-- ===================================================================== +-- Script: 94b_MARS_828_rollback_column_comments.sql +-- MARS Issue: MARS-828 +-- Author: Grzegorz Michalski +-- Date: 2026-02-20 +-- Purpose: Remove column comments added by 01b_MARS_828_add_column_comments.sql +-- Description: Optional rollback - removes documentation but does not affect functionality +-- ===================================================================== + +PROMPT ======================================== +PROMPT MARS-828: Removing column comments (optional) +PROMPT ======================================== + +-- ===================================================================== +-- Remove A_SOURCE_FILE_CONFIG Column Comments +-- ===================================================================== + +PROMPT Removing column comments from A_SOURCE_FILE_CONFIG... + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS ''; + +-- ===================================================================== +-- Remove A_SOURCE_FILE_RECEIVED Column Comments +-- ===================================================================== + +PROMPT Removing column comments from A_SOURCE_FILE_RECEIVED... + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS ''; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_PATH IS ''; + +-- ===================================================================== +-- Verification +-- ===================================================================== + +PROMPT +PROMPT Verifying column comments removed... +PROMPT + +SELECT + table_name, + COUNT(*) as total_columns, + COUNT(CASE WHEN comments IS NOT NULL AND LENGTH(comments) > 0 THEN 1 END) as documented_columns +FROM all_col_comments +WHERE owner = 'CT_MRDS' + AND table_name IN ('A_SOURCE_FILE_CONFIG', 'A_SOURCE_FILE_RECEIVED') +GROUP BY table_name +ORDER BY table_name; + +PROMPT +PROMPT ======================================== +PROMPT Column comments removed successfully +PROMPT ======================================== +PROMPT NOTE: This is an optional rollback step +PROMPT Database functionality is not affected +PROMPT ======================================== diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql index 481cfea..e4248e2 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/install_mars828.sql @@ -38,7 +38,7 @@ PROMPT ========================================================================= PROMPT Package: CT_MRDS.FILE_ARCHIVER PROMPT Change: Enhanced archival strategies (MINIMUM_AGE_MONTHS, HYBRID) + TRASH retention + Selective archiving PROMPT Purpose: Flexible archival policies per data source with file retention and config-based control -PROMPT Steps: 11 (DDL, Rename Columns, Trigger, Statuses, Grants, Package v3.3.0, Verify, Track, Configure) +PROMPT Steps: 12 (DDL, Rename, Comments, Trigger, Statuses, Grants, Package v3.3.0, Verify, Track, Configure) PROMPT Timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS install_start FROM DUAL; PROMPT ============================================================================ @@ -56,57 +56,62 @@ WHENEVER SQLERROR CONTINUE -- Installation steps PROMPT -PROMPT Step 1/11: Adding archival strategy and config columns to A_SOURCE_FILE_CONFIG +PROMPT Step 1/12: Adding archival strategy and config columns to A_SOURCE_FILE_CONFIG PROMPT ============================================================================== @@01_MARS_828_install_add_archival_strategy_columns.sql PROMPT -PROMPT Step 2/11: Renaming threshold columns for consistent naming +PROMPT Step 2/12: Renaming threshold columns for consistent naming PROMPT ========================================================== @@01a_MARS_828_rename_threshold_columns.sql PROMPT -PROMPT Step 3/11: Creating validation trigger +PROMPT Step 3/12: Adding comprehensive column comments +PROMPT =============================================== +@@01b_MARS_828_add_column_comments.sql + +PROMPT +PROMPT Step 4/12: Creating validation trigger PROMPT ====================================== @@02_MARS_828_install_archival_strategy_trigger.sql PROMPT -PROMPT Step 4/11: Adding TRASH retention statuses to A_SOURCE_FILE_RECEIVED +PROMPT Step 5/12: Adding TRASH retention statuses to A_SOURCE_FILE_RECEIVED PROMPT =================================================================== @@07_MARS_828_install_add_trash_retention_statuses.sql PROMPT -PROMPT Step 5/11: Granting privileges on T_FILENAME to MRDS_LOADER +PROMPT Step 6/12: Granting privileges on T_FILENAME to MRDS_LOADER PROMPT ========================================================== @@08_MARS_828_install_grant_t_filename.sql PROMPT -PROMPT Step 6/11: Deploying FILE_ARCHIVER Package Specification v3.3.0 +PROMPT Step 7/12: Deploying FILE_ARCHIVER Package Specification v3.3.0 PROMPT ============================================================== @@03_MARS_828_install_CT_MRDS_FILE_ARCHIVER_SPEC.sql PROMPT -PROMPT Step 7/11: Deploying FILE_ARCHIVER Package Body v3.3.0 +PROMPT Step 8/12: Deploying FILE_ARCHIVER Package Body v3.3.0 PROMPT ==================================================== @@04_MARS_828_install_CT_MRDS_FILE_ARCHIVER_BODY.sql PROMPT -PROMPT Step 8/11: Verifying installation +PROMPT Step 9/12: Verifying installation PROMPT ================================= @@05_MARS_828_verify_installation.sql PROMPT -PROMPT Step 9/11: Tracking package versions +PROMPT Step 10/12: Tracking package versions PROMPT ==================================== @@track_package_versions.sql PROMPT -PROMPT Step 10/11: Verifying tracked packages +PROMPT Step 11/12: Verifying tracked packages PROMPT ====================================== @@verify_packages_version.sql PROMPT -PROMPT Step 11/11: Configuring Release 01 tables archival strategies +PROMPT Step 12/12: Configuring Release 01 tables archival strategies PROMPT ============================================================ @@06_MARS_828_configure_release01_tables.sql diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_CONFIG.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_CONFIG.sql index 6a791f7..739cfc9 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_CONFIG.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_CONFIG.sql @@ -47,10 +47,64 @@ ON "CT_MRDS"."A_SOURCE_FILE_CONFIG" ("SOURCE_FILE_TYPE", "SOURCE_FILE_ID", "TABL TABLESPACE "DATA"; -- Column comments -COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS 'Archival strategy: THRESHOLD_BASED, CURRENT_MONTH_ONLY, MINIMUM_AGE_MONTHS, HYBRID. Added in MARS-828'; -COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS strategy). Added in MARS-828'; -COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2). Added in MARS-1049'; -COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process. Added in MARS-828 v3.3.0'; -COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy. Added in MARS-828 v3.3.0'; +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_FILE_CONFIG_KEY IS + 'Primary key - unique identifier for source file configuration record'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.A_SOURCE_KEY IS + 'Foreign key to A_SOURCE table - identifies the source system (e.g., LM, C2D, CSDB)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_TYPE IS + 'Type of file configuration: INPUT (data files), CONTAINER (xml files), or LOAD_CONFIG (configuration files)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_ID IS + 'Unique identifier for the source file within the source system (e.g., UC_DISSEM, STANDING_FACILITIES)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_DESC IS + 'Human-readable description of the source file and its purpose'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.SOURCE_FILE_NAME_PATTERN IS + 'Filename pattern for matching incoming files (supports wildcards, e.g., UC_NMA_DISSEM-*.csv)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TABLE_ID IS + 'Identifier for the target table where data will be loaded (without schema prefix)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.TEMPLATE_TABLE_NAME IS + 'Fully qualified name of template table in CT_ET_TEMPLATES schema used for external table creation'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.CONTAINER_FILE_KEY IS + 'Foreign key to parent container configuration when this file is part of an xml (NULL for standalone files)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_DAYS IS + 'Threshold for THRESHOLD_BASED strategy: archive data older than N days'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_FILES_COUNT IS + 'Trigger archival when file count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_BYTES_SUM IS + 'Trigger archival when total size in bytes exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVE_THRESHOLD_ROWS_COUNT IS + 'Trigger archival when total row count exceeds this threshold (used in THRESHOLD_BASED and HYBRID strategies)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ODS_SCHEMA_NAME IS + 'Schema name where ODS external tables are created (typically ODS)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.HOURS_TO_EXPIRE_STATISTICS IS + 'Number of hours before table statistics expire and need to be recalculated'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ARCHIVAL_STRATEGY IS + 'Archival strategy: THRESHOLD_BASED (days-based), MINIMUM_AGE_MONTHS (0=current month, N=retain N months), HYBRID (combination)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.MINIMUM_AGE_MONTHS IS + 'Minimum age in months before archival (required for MINIMUM_AGE_MONTHS and HYBRID strategies, 0=current month only)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.ENCODING IS + 'Oracle character set name for CSV files (e.g., UTF8, WE8MSWIN1252, EE8ISO8859P2)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_ARCHIVE_ENABLED IS + 'Y=Enable archiving, N=Skip archiving. Controls if table participates in archival process'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_CONFIG.IS_KEEP_IN_TRASH IS + 'Y=Keep files in TRASH after archiving, N=Delete immediately. Controls TRASH retention policy'; GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_CONFIG TO MRDS_LOADER_ROLE; \ No newline at end of file diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql index b5be987..18739dc 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/new_version/A_SOURCE_FILE_RECEIVED.sql @@ -26,4 +26,41 @@ CREATE TABLE CT_MRDS.A_SOURCE_FILE_RECEIVED ( CREATE UNIQUE INDEX CT_MRDS.A_SOURCE_FILE_RECEIVED_UK1 ON CT_MRDS.A_SOURCE_FILE_RECEIVED(CHECKSUM, CREATED, BYTES); +-- Column comments +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_RECEIVED_KEY IS + 'Primary key - unique identifier for received file record'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.A_SOURCE_FILE_CONFIG_KEY IS + 'Foreign key to A_SOURCE_FILE_CONFIG - links file to its configuration and processing rules'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.SOURCE_FILE_NAME IS + 'Full object name/path of the received file in OCI Object Storage (includes INBOX prefix)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CHECKSUM IS + 'MD5 checksum of file content for integrity verification and duplicate detection'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.CREATED IS + 'Timestamp with timezone when file was created/uploaded to Object Storage (from DBMS_CLOUD.LIST_OBJECTS)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.BYTES IS + 'File size in bytes'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.RECEPTION_DATE IS + 'Date when file was registered in the system (extracted from CREATED timestamp)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PROCESSING_STATUS IS + 'Current processing status: RECEIVED → VALIDATED → READY_FOR_INGESTION → INGESTED → ARCHIVED_AND_TRASHED → ARCHIVED_AND_PURGED'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.EXTERNAL_TABLE_NAME IS + 'Name of temporary external table created for file validation (dropped after validation)'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_YEAR IS + 'Year partition value (YYYY format) when file was archived to ARCHIVE bucket with Hive-style partitioning'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.PARTITION_MONTH IS + 'Month partition value (MM format) when file was archived to ARCHIVE bucket with Hive-style partitioning'; + +COMMENT ON COLUMN CT_MRDS.A_SOURCE_FILE_RECEIVED.ARCH_FILE_NAME IS + 'Archive directory prefix in ARCHIVE bucket containing archived Parquet files (supports multiple files from parallel DBMS_CLOUD.EXPORT_DATA)'; + GRANT SELECT, INSERT, UPDATE, DELETE ON CT_MRDS.A_SOURCE_FILE_RECEIVED TO MRDS_LOADER_ROLE; \ No newline at end of file diff --git a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql index a69b06d..0de7510 100644 --- a/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql +++ b/MARS_Packages/REL01_ADDITIONS/MARS-828/rollback_mars828.sql @@ -35,10 +35,11 @@ PROMPT Rollback steps: PROMPT 1. Rollback TRASH retention statuses PROMPT 2. Revoke T_FILENAME privileges PROMPT 3. Remove validation trigger -PROMPT 4. Revert threshold column renames (restore original naming) -PROMPT 5. Drop all configuration columns (ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH) -PROMPT 6. Restore FILE_ARCHIVER package to v2.0.0 -PROMPT 7. Revert all archival strategies to THRESHOLD_BASED +PROMPT 4. Remove column comments (OPTIONAL - does not affect functionality) +PROMPT 5. Revert threshold column renames (restore original naming) +PROMPT 6. Drop all configuration columns (ARCHIVAL_STRATEGY, MINIMUM_AGE_MONTHS, IS_ARCHIVE_ENABLED, IS_KEEP_IN_TRASH) +PROMPT 7. Restore FILE_ARCHIVER package to v2.0.0 +PROMPT 8. Revert all archival strategies to THRESHOLD_BASED PROMPT PROMPT Timestamp: SELECT TO_CHAR(SYSDATE, 'YYYY-MM-DD HH24:MI:SS') AS rollback_start FROM DUAL; @@ -57,42 +58,50 @@ WHENEVER SQLERROR CONTINUE -- Rollback steps (in reverse order) PROMPT -PROMPT Step 1/8: Rolling back TRASH retention statuses +PROMPT Step 1/9: Rolling back TRASH retention statuses PROMPT ================================================ @@90_MARS_828_rollback_trash_retention_statuses.sql PROMPT -PROMPT Step 2/8: Revoking T_FILENAME privileges from MRDS_LOADER +PROMPT Step 2/9: Revoking T_FILENAME privileges from MRDS_LOADER PROMPT ========================================================== @@95_MARS_828_rollback_grant_t_filename.sql PROMPT -PROMPT Step 3/8: Dropping validation trigger +PROMPT Step 3/9: Dropping validation trigger PROMPT ====================================== @@93_MARS_828_rollback_trigger.sql PROMPT -PROMPT Step 4/8: Reverting threshold column renames +PROMPT Step 4/9 (OPTIONAL): Removing column comments +PROMPT ============================================== +PROMPT NOTE: This is optional - comments do not affect functionality +PROMPT Skipping column comments removal in standard rollback +PROMPT Execute 94b_MARS_828_rollback_column_comments.sql manually if needed +PROMPT + +PROMPT +PROMPT Step 5/9: Reverting threshold column renames PROMPT ============================================= @@94a_MARS_828_rollback_threshold_rename.sql PROMPT -PROMPT Step 5/8: Dropping all archival configuration columns +PROMPT Step 6/9: Dropping all archival configuration columns PROMPT ====================================================== @@94_MARS_828_rollback_columns.sql PROMPT -PROMPT Step 6/8: Restoring FILE_ARCHIVER Package Specification v2.0.0 +PROMPT Step 7/9: Restoring FILE_ARCHIVER Package Specification v2.0.0 PROMPT =============================================================== @@91_MARS_828_rollback_FILE_ARCHIVER_SPEC.sql PROMPT -PROMPT Step 7/8: Restoring FILE_ARCHIVER Package Body v2.0.0 +PROMPT Step 8/9: Restoring FILE_ARCHIVER Package Body v2.0.0 PROMPT ====================================================== @@92_MARS_828_rollback_FILE_ARCHIVER_BODY.sql PROMPT -PROMPT Step 8/8: Verifying tracked packages +PROMPT Step 9/9: Verifying tracked packages PROMPT ===================================== @@verify_packages_version.sql