Migrate sync objects into ProductionDataBaseSync schema + add permanent audit archive
- Add sql/00_schema.sql: create dedicated ProductionDataBaseSync schema (idempotent) - Move SyncQueue and usp_SyncApply from dbo into ProductionDataBaseSync - Add sql/03_sync_log_archive.sql: permanent, append-only SyncLogArchive that records both OriginalOperateType and ProcessedOperateType plus the Access log OriginalTime, so pipeline divergences (e.g. Insert applied as Delete) stay reconstructible forever (SyncQueue is transient and only keeps processed type) - config.py: inject sync_queue_table / archive_table / apply_proc (default to the new schema); SqlWriter takes these names instead of hardcoding dbo - sql_writer.py: add ArchiveRow + insert_archive_row (dedup on source keys), parametrize queue/archive/proc names throughout - capture.py: archive every consumed log row before enqueue (preserves evidence before cleanup deletes the Access log) - service.py: pass the three names into SqlWriter - tests: read queue/proc names from config instead of hardcoding dbo.SyncQueue
This commit is contained in:
@@ -11,7 +11,7 @@
|
||||
-- and silently drop a row whose true last op was an Insert. Routing off the one
|
||||
-- winning row's OperateType guarantees only the genuine last op wins.
|
||||
|
||||
CREATE OR ALTER PROCEDURE dbo.usp_SyncApply
|
||||
CREATE OR ALTER PROCEDURE ProductionDataBaseSync.usp_SyncApply
|
||||
@MaxRetries INT = 5
|
||||
AS
|
||||
BEGIN
|
||||
@@ -21,13 +21,13 @@ BEGIN
|
||||
DECLARE @sql NVARCHAR(MAX), @cols NVARCHAR(MAX), @upd NVARCHAR(MAX), @ins NVARCHAR(MAX);
|
||||
|
||||
-- Re-queue rows still under the retry budget.
|
||||
UPDATE dbo.SyncQueue
|
||||
UPDATE ProductionDataBaseSync.SyncQueue
|
||||
SET Status = 'pending'
|
||||
WHERE Status = 'error' AND RetryCount < @MaxRetries;
|
||||
|
||||
DECLARE cur CURSOR LOCAL FAST_FORWARD FOR
|
||||
SELECT DISTINCT TargetSchema, TargetTable
|
||||
FROM dbo.SyncQueue
|
||||
FROM ProductionDataBaseSync.SyncQueue
|
||||
WHERE Status = 'pending';
|
||||
|
||||
OPEN cur;
|
||||
@@ -70,7 +70,7 @@ BEGIN
|
||||
+ N';WITH ranked AS ('
|
||||
+ N' SELECT RecordID, OperateType, RowData,'
|
||||
+ N' ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn'
|
||||
+ N' FROM dbo.SyncQueue'
|
||||
+ N' FROM ProductionDataBaseSync.SyncQueue'
|
||||
+ N' WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'''
|
||||
+ N')'
|
||||
+ N'MERGE ' + @FullName + N' WITH (HOLDLOCK) AS tgt'
|
||||
@@ -90,7 +90,7 @@ BEGIN
|
||||
SET @sql = N';WITH ranked AS ('
|
||||
+ N' SELECT RecordID, OperateType,'
|
||||
+ N' ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn'
|
||||
+ N' FROM dbo.SyncQueue'
|
||||
+ N' FROM ProductionDataBaseSync.SyncQueue'
|
||||
+ N' WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'''
|
||||
+ N')'
|
||||
+ N'DELETE t FROM ' + @FullName + N' t'
|
||||
@@ -100,7 +100,7 @@ BEGIN
|
||||
N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
|
||||
END
|
||||
|
||||
UPDATE dbo.SyncQueue
|
||||
UPDATE ProductionDataBaseSync.SyncQueue
|
||||
SET Status = 'applied', AppliedAt = SYSDATETIME()
|
||||
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
|
||||
|
||||
@@ -113,7 +113,7 @@ BEGIN
|
||||
-- issues its own rollback. We roll back here so the partial per-table
|
||||
-- work is discarded before marking rows as 'error'/'dead'.
|
||||
IF @@TRANCOUNT > 0 ROLLBACK;
|
||||
UPDATE dbo.SyncQueue
|
||||
UPDATE ProductionDataBaseSync.SyncQueue
|
||||
SET Status = CASE WHEN RetryCount + 1 >= @MaxRetries THEN 'dead' ELSE 'error' END,
|
||||
RetryCount = RetryCount + 1,
|
||||
ErrorMsg = ERROR_MESSAGE()
|
||||
|
||||
Reference in New Issue
Block a user