USE DATABASE SCD_DB;

USE SCHEMA SCD_SC;

CREATE OR REPLACE TABLE CUSTOMER (

CUSTOMER_ID NUMBER,
FIRST_NAME VARCHAR,
LAST_NAME VARCHAR,
EMAIL VARCHAR,
STREET VARCHAR,
CITY VARCHAR,
STATE VARCHAR,
COUNTRY VARCHAR,
UPDATE_TIMESTAMP TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP()
);

CREATE OR REPLACE TABLE CUSTOMER_HISTORY (

CUSTOMER_ID NUMBER,
FIRST_NAME VARCHAR,
LAST_NAME VARCHAR,
EMAIL VARCHAR,
STREET VARCHAR,
CITY VARCHAR,
STATE VARCHAR,
COUNTRY VARCHAR,
START_TIME TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP(),
END_TIME TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP(),
IS_CURRENT BOOLEAN
);

CREATE OR REPLACE TABLE CUSTOMER_RAW (

CUSTOMER_ID NUMBER,
FIRST_NAME VARCHAR,
LAST_NAME VARCHAR,
EMAIL VARCHAR,
STREET VARCHAR,
CITY VARCHAR,
STATE VARCHAR,
COUNTRY VARCHAR
);

CREATE OR REPLACE STREAM CUSTOMER_TABLE_CHANGES ON TABLE CUSTOMER;

CREATE OR REPLACE STORAGE INTEGRATION S3_CONN

TYPE = EXTERNAL_STAGE
STORAGE_PROVIDER = S3
ENABLED = TRUE
STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::837934442683:role/snowflake_s3_conn'
STORAGE_ALLOWED_LOCATIONS = ('s3://snowflake-project-shrey')
COMMENT = 'Creating connection to s3'

DESC INTEGRATION S3_CONN

CREATE OR REPLACE FILE FORMAT SCD_DB.SCD_SC.CSV_FILE_FORMAT

TYPE = 'CSV'
FIELD_DELIMITER = ','
SKIP_HEADER = 1
NULL_IF = ('NULL', 'null')
EMPTY_FIELD_AS_NULL = TRUE;
CREATE OR REPLACE STAGE SCD_DB.SCD_SC.CUSTOMER_EXT_STAGE
URL = ''
STORAGE_INTEGRATION = S3_CONN
FILE_FORMAT = SCD_DB.SCD_SC.CSV_FILE_FORMAT;

LIST @SCD_DB.SCD_SC.CUSTOMER_EXT_STAGE;

CREATE OR REPLACE PIPE CUSTOMER_RAW_S3_PIPE

AUTO_INGEST = TRUE
AS
COPY INTO CUSTOMER_RAW
FROM @SCD_DB.SCD_SC.CUSTOMER_EXT_STAGE
FILE_FORMAT = SCD_DB.SCD_SC.CSV_FILE_FORMAT;

SHOW PIPES; --TO GET THE SQS NOTIFICATION_CHANNEL

SELECT SYSTEM$PIPE_STATUS('CUSTOMER_RAW_S3_PIPE');

MERGE INTO CUSTOMER C

USING CUSTOMER_RAW CR
ON C.CUSTOMER_ID = CR.CUSTOMER_ID
WHEN MATCHED AND ( C.CUSTOMER_ID <> CR.CUSTOMER_ID OR
C.FIRST_NAME <> CR.FIRST_NAME OR
C.LAST_NAME <> CR.LAST_NAME OR
C.EMAIL <> CR.EMAIL OR
C.STREET <> CR.STREET OR
C.CITY <> CR.CITY OR
C.STATE <> CR.STATE OR
C.COUNTRY <> CR.COUNTRY) THEN UPDATE
SET C.CUSTOMER_ID = CR.CUSTOMER_ID,
C.FIRST_NAME = CR.FIRST_NAME,
C.LAST_NAME = CR.LAST_NAME,
C.EMAIL = CR.EMAIL,
C.STREET = CR.STREET,
C.CITY = CR.CITY,
C.STATE = CR.STATE,
C.COUNTRY = CR.COUNTRY,
UPDATE_TIMESTAMP = CURRENT_TIMESTAMP()
WHEN NOT MATCHED THEN
INSERT (C.CUSTOMER_ID, C.FIRST_NAME, C.LAST_NAME, C.EMAIL, C.STREET, C.CITY,
C.STATE, C.COUNTRY)
VALUES (CR.CUSTOMER_ID, CR.FIRST_NAME, CR.LAST_NAME, CR.EMAIL, CR.STREET,
CR.CITY, CR.STATE, CR.COUNTRY);

CREATE OR REPLACE PROCEDURE PDR_SCD()

RETURNS STRING NOT NULL
LANGUAGE JAVASCRIPT
AS
$$
var cmd = `MERGE INTO CUSTOMER C
USING CUSTOMER_RAW CR
ON C.CUSTOMER_ID = CR.CUSTOMER_ID
WHEN MATCHED AND ( C.CUSTOMER_ID <> CR.CUSTOMER_ID OR
C.FIRST_NAME <> CR.FIRST_NAME OR
C.LAST_NAME <> CR.LAST_NAME OR
C.EMAIL <> CR.EMAIL OR
 C.STREET <> CR.STREET OR
C.CITY <> CR.CITY OR
C.STATE <> CR.STATE OR
C.COUNTRY <> CR.COUNTRY) THEN
UPDATE
SET C.CUSTOMER_ID = CR.CUSTOMER_ID,
C.FIRST_NAME = CR.FIRST_NAME,
C.LAST_NAME = CR.LAST_NAME,
C.EMAIL = CR.EMAIL,
C.STREET = CR.STREET,
C.CITY = CR.CITY,
C.STATE = CR.STATE,
C.COUNTRY = CR.COUNTRY,
UPDATE_TIMESTAMP = CURRENT_TIMESTAMP()
WHEN NOT MATCHED THEN
INSERT (C.CUSTOMER_ID, C.FIRST_NAME, C.LAST_NAME, C.EMAIL, C.STREET, C.CITY,
C.STATE, C.COUNTRY)
VALUES (CR.CUSTOMER_ID, CR.FIRST_NAME, CR.LAST_NAME, CR.EMAIL, CR.STREET,
CR.CITY, CR.STATE, CR.COUNTRY);`

var cmd1 = "truncate table SCD_DEMO.SCD2.customer_raw;"

var sql = snowflake.createStatement({sqlText: cmd});
var sql1 = snowflake.createStatement({sqlText: cmd1});
var result = sql.execute();
var result1 = sql1.execute();
return cmd+'\n'+cmd1;
$$;

CALL PROCEDURE PDR_SCD();

-- SET UP TASKADMIN ROLE

USE ROLE SECURITYADMIN;
CREATE OR REPLACE ROLE TASKADMIN;

-- SET THE ACTIVE ROLE TO ACCOUNTADMIN BEFORE GRANTING THE EXECUTE TASK PRIVILEGE
TO TASKADMIN
USE ROLE ACCOUNTADMIN;
GRANT EXECUTE TASK ON ACCOUNT TO ROLE TASKADMIN;

-- SET THE ACTIVE ROLE TO SECURITYADMIN TO SHOW THAT THIS ROLE CAN GRANT A ROLE TO
ANOTHER ROLE
USE ROLE SECURITYADMIN;
GRANT ROLE TASKADMIN TO ROLE SYSADMIN;

CREARTE OR REPLACE TASK TSK_SCD_RAW WAREHOUSE = COMPUTE_WH SCHEDULE = '1 MINUTE'

ERROR_ON_NONDETERMINISTIC_MERGE=FALSE
as
CALL PROCEDURE PDR_SCD();

SHOW TASKS;

ALTER TASK TSK_SCD_RAW SUSPEND; -- RESUME -- SUSPEND

SELECT TIMESTAMPDIFF(SECOND, CURRENT_TIMESTAMP, SCHEDULED_TIME) AS NEXT_RUN,

SCHEDULED_TIME, CURRENT_TIMESTAMP, NAME, STATE
FROM TABLE(INFORMATION_SCHEMA.TASK_HISTORY())
WHERE STATE = 'SCHEDULED'
ORDER BY COMPLETED_TIME DESC;

--STREAM

SELECT * FROM CUSTOMER_TABLE_CHANGES;

INSERT INTO CUSTOMER VALUES (223136, 'JESSICA', 'ARNOLD', '[email protected]',

'595 BENJAMIN FORGE SUITE 124', 'MICHAELSTAD', 'CONNECTICUT', 'CAPE VERDE',
CURRENT_TIMESTAMP());

UPDATE CUSTOMER
SET FIRST_NAME = 'JESSICA', UPDATE_TIMESTAMP = CURRENT_TIMESTAMP()::TIMESTAMP_NTZ
WHERE CUSTOMER_ID = 72;

DELETE FROM CUSTOMER WHERE CUSTOMER_ID = 73;

SELECT * FROM CUSTOMER_HISTORY WHERE CUSTOMER_ID IN (72, 73, 223136);

SELECT * FROM CUSTOMER_TABLE_CHANGES;

SELECT * FROM CUSTOMER WHERE CUSTOMER_ID IN (72, 73, 223136);

--VIEW CREATION--

CREATE OR REPLACE VIEW V_CUSTOMER_CHANGE_DATA AS

-- This subquery figures out what to do when data is inserted into the customer
table
-- An insert to the customer table results in an INSERT to the customer_HISTORY
table
SELECT CUSTOMER_ID, FIRST_NAME, LAST_NAME, EMAIL, STREET,
CITY,STATE,COUNTRY,START_TIME,END_TIME,IS_CURRENT,
'I' AS DML_TYPE
FROM
(SELECT CUSTOMER_ID, FIRST_NAME, LAST_NAME, EMAIL, STREET,
CITY,STATE,COUNTRY,UPDATE_TIMESTAMP AS START_TIME,
LAG(UPDATE_TIMESTAMP) OVER (PARTITION BY CUSTOMER_ID ORDER BY UPDATE_TIMESTAMP
DESC) AS END_TIME_RAW,
CASE WHEN END_TIME_RAW IS NULL THEN '9999-12-31'::TIMESTAMP_NTZ ELSE
END_TIME_RAW END AS END_TIME,
CASE WHEN END_TIME_RAW IS NULL THEN TRUE ELSE FALSE END AS IS_CURRENT
FROM
(SELECT CUSTOMER_ID, FIRST_NAME, LAST_NAME, EMAIL, STREET,
CITY,STATE,COUNTRY,UPDATE_TIMESTAMP
FROM SCD_DB.SCD_SC.CUSTOMER_TABLE_CHANGES
WHERE metadata$action = 'INSERT'AND metadata$isupdate = 'FALSE'))
UNION

-- This subquery figures out what to do when data is updated in the customer table
-- An update to the customer table results in an update AND an insert to the
customer_HISTORY table
-- The subquery below generates two records, each with a different dml_type
SELECT CUSTOMER_ID, FIRST_NAME, LAST_NAME, EMAIL, STREET, CITY, STATE, COUNTRY,
UPDATE_TIMESTAMP AS START_TIME,
LAG(UPDATE_TIMESTAMP) OVER (PARTITION BY CUSTOMER_ID ORDER BY UPDATE_TIMESTAMP
DESC) AS END_TIME_RAW,
CASE WHEN END_TIME_RAW IS NULL THEN '9999-12-31'::TIMESTAMP_NTZ ELSE END_TIME_RAW
END AS END_TIME,
CASE WHEN END_TIME_RAW IS NULL THEN TRUE ELSE FALSE END AS IS_CURRENT, DML_TYPE
FROM(-- Identify data to insert into customer_history table
SELECT CUSTOMER_ID, FIRST_NAME, LAST_NAME, EMAIL, STREET,
CITY,STATE,COUNTRY, update_timestamp, 'I' as DML_TYPE
FROM customer_table_changes
WHERE metadata$action = 'INSERT'
AND metadata$isupdate = 'TRUE'
UNION
-- Identify data in customer_HISTORY table that needs to be updated
SELECT CUSTOMER_ID, NULL, NULL, NULL, NULL, NULL, NULL, NULL, START_TIME,
'U' AS DML_TYPE
FROM CUSTOMER_HISTORY
WHERE CUSTOMER_ID IN
(SELECT DISTINCT CUSTOMER_ID
FROM CUSTOMER_TABLE_CHANGES
WHERE metadata$action = 'DELETE'
AND metadata$isupdate = 'TRUE')
AND IS_CURRENT = TRUE))
UNION
-- This subquery figures out what to do when data is deleted from the customer
table
-- A deletion from the customer table results in an update to the customer_HISTORY
table
SELECT CTC.CUSTOMER_ID, NULL, NULL, NULL, NULL, NULL, NULL, NULL, CH.START_TIME,
CURRENT_TIMESTAMP()::TIMESTAMP_NTZ,
NULL, 'D' AS DML_TYPE
FROM
CUSTOMER_HISTORY CH
INNER JOIN CUSTOMER_TABLE_CHANGES CTC
ON CH.CUSTOMER_ID = CTC.CUSTOMER_ID
WHERE CTC.metadata$action = 'DELETE'
AND CTC.metadata$isupdate = 'FALSE'
AND CH.IS_CURRENT = TRUE;

SELECT * FROM V_CUSTOMER_CHANGE_DATA;

CREATE OR REPLACE TASK TSK_SCD_HIST WAREHOUSE = COMPUTE_WH SCHEDULE = '1 MINUTE'

ERROR_ON_NONDETERMINISTIC_MERGE=FALSE
AS
MERGE INTO CUSTOMER_HISTORY CH --Target table to merge changes from NATION into
USING V_CUSTOMER_CHANGE_DATA CCD --v_customer_change_data is a view that holds the
logic that determines what to insert/update into the customer_history table.
ON CH.CUSTOMER_ID = CCD.CUSTOMER_ID
AND CH.START_TIME = CCDE.START_TIME
WHEN MATCHED AND CCD.DML_TYPE = 'U'
THEN UPDATE
SET CH.END_TIME = CCD.END_TIME,
CH.IS_CURRENT = FALSE
WHEN MATCHED AND CCD.DML_TYPE = 'D'
THEN UPDATE
SET CH.END_TIME = CCD.END_TIME,
CH.IS_CURRENT = FALSE
WHEN NOT MATCHED AND CCD.DML_TYPE = 'I'
THEN INSERT (CUSTOMER_ID, FIRST_NAME, LAST_NAME, EMAIL, STREET, CITY, STATE,
COUNTRY, START_TIME, END_TIME, IS_CURRENT)
VALUES (CCD.CUSTOMER_ID, CCD.FIRST_NAME, CCD.LAST_NAME, CCD.EMAIL, CCD.STREET,
CCD.CITY, CCD.STATE, CCD.COUNTRY, CCD.START_TIME, CCD.END_TIME, CCD.IS_CURRENT);
SHOW TASKS;

ALTER TASK TSK_SCD_HIST SUSPEND; --RESUME

INSERT INTO CUSTOMER VALUES(223136,'Jessica','Arnold','[email protected]','595

Benjamin Forge Suite 124','Michaelstad','Connecticut','Cape
Verde',CURRENT_TIMESTAMP());

UPDATE CUSTOMER SET FIRST_NAME='Jessica' WHERE CUSTOMER_ID=7523;

DELETE FROM CUSTOMER WHERE CUSTOMER_ID =136 AND FIRST_NAME = 'Kim';

SELECT COUNT(*),CUSTOMER_ID FROM CUSTOMER GROUP BY CUSTOMER_ID HAVING COUNT(*)=1;

SELECT * FROM CUSTOMER_HISTORY WHERE CUSTOMER_ID =223136;

SELECT * FROM CUSTOMER_HISTORY WHERE IS_CURRENT=TRUE;

--ALTER WAREHOUSE SUSPEND;

SELECT TIMESTAMPDIFF(SECOND, CURRENT_TIMESTAMP, SCHEDULED_TIME) AS NEXT_RUN,
SCHEDULED_TIME, CURRENT_TIMESTAMP, NAME, STATE
FROM TABLE(INFORMATION_SCHEMA.TASK_HISTORY()) WHERE STATE = 'SCHEDULED' ORDER BY
COMPLETED_TIME DESC;

SELECT * FROM CUSTOMER_HISTORY WHERE IS_CURRENT=FALSE;

SHOW TASKS;