Refine incremental sync audit observability
- Add per-cycle correlation ID ([cyc:xxxxxxxx]) threaded through Python logs and SQL audit tables for end-to-end traceability of any divergence. - New ProductionDataBaseSync.SyncApplyRunLog table + @CycleID on usp_SyncApply for per-table apply auditing (pending/merged/deleted/applied, dead/error counts, duration). Audit write isolated in its own TRY/CATCH outside txn. - Capture/cleanup phases now emit full detail: Insert->Delete downgrade warning with record_id/log_id, dedup_skipped as apply-stall signal, per-file summary, and access log-id ranges on cleanup. - Queue health check surfaces error/dead rows with recent samples instead of silent accumulation (previously the top cause of data divergence). - sql_writer uses INSERT...SELECT...WHERE NOT EXISTS for observable dedup; idle cycles lowered to DEBUG with periodic heartbeat. - Backward compatible: old proc callers still work (CycleID nullable; legacy coarse-grained logging with one-time notice). Excluded from this commit: CODE.md, build_code_doc.py (doc generation).
This commit is contained in:
Misaka_Company
@@ -10,15 +10,33 @@
|
||||
-- Delete), which let a stale Delete outrank a newer Insert for the same RecordID
|
||||
-- and silently drop a row whose true last op was an Insert. Routing off the one
|
||||
-- winning row's OperateType guarantees only the genuine last op wins.
|
||||
--
|
||||
-- AUDIT NOTE: when ProductionDataBaseSync.SyncApplyRunLog exists (see
|
||||
-- sql/04_sync_apply_runlog.sql), one audit row is written per target table and
|
||||
-- per invocation: pending/distinct counts going in, the MERGE/DELETE rowcounts,
|
||||
-- how many queue rows were flipped to applied (or to error/dead on failure),
|
||||
-- the error message, the duration, and the caller-supplied @CycleID that ties
|
||||
-- the row to the Python service log's [cyc:...] lines. The audit insert is
|
||||
-- best-effort (wrapped in its own TRY/CATCH, outside the data transaction) and
|
||||
-- can never fail the apply itself. @CycleID defaults to NULL so the legacy
|
||||
-- single-parameter EXEC keeps working.
|
||||
|
||||
CREATE OR ALTER PROCEDURE ProductionDataBaseSync.usp_SyncApply
|
||||
@MaxRetries INT = 5
|
||||
@MaxRetries INT = 5,
|
||||
@CycleID NVARCHAR(40) = NULL
|
||||
AS
|
||||
BEGIN
|
||||
SET NOCOUNT ON;
|
||||
|
||||
DECLARE @sch NVARCHAR(128), @tbl NVARCHAR(255), @FullName NVARCHAR(514);
|
||||
DECLARE @sql NVARCHAR(MAX), @cols NVARCHAR(MAX), @upd NVARCHAR(MAX), @ins NVARCHAR(MAX);
|
||||
DECLARE @hasRunLog bit =
|
||||
CASE WHEN OBJECT_ID('ProductionDataBaseSync.SyncApplyRunLog', 'U') IS NOT NULL
|
||||
THEN 1 ELSE 0 END;
|
||||
DECLARE @t0 DATETIME2, @pending INT, @distinctRecs INT,
|
||||
@merged INT, @deleted INT, @applied INT,
|
||||
@errCnt INT, @deadCnt INT,
|
||||
@errMsg NVARCHAR(MAX), @outcome VARCHAR(10);
|
||||
|
||||
-- Re-queue rows still under the retry budget.
|
||||
UPDATE ProductionDataBaseSync.SyncQueue
|
||||
@@ -37,6 +55,19 @@ BEGIN
|
||||
BEGIN
|
||||
SET @FullName = QUOTENAME(@sch) + N'.' + QUOTENAME(@tbl);
|
||||
|
||||
-- Per-table audit state. Explicit reset every iteration: local
|
||||
-- variables keep their previous value across cursor loops.
|
||||
SELECT @t0 = SYSDATETIME(),
|
||||
@merged = NULL, @deleted = NULL, @applied = NULL,
|
||||
@errCnt = NULL, @deadCnt = NULL,
|
||||
@errMsg = NULL, @outcome = 'ok',
|
||||
@cols = NULL, @upd = NULL, @ins = NULL;
|
||||
|
||||
SELECT @pending = COUNT(*),
|
||||
@distinctRecs = COUNT(DISTINCT RecordID)
|
||||
FROM ProductionDataBaseSync.SyncQueue
|
||||
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
|
||||
|
||||
BEGIN TRY
|
||||
BEGIN TRAN;
|
||||
-- @cols: comma-quoted column names (for INSERT target list)
|
||||
@@ -66,6 +97,9 @@ BEGIN
|
||||
-- pending ops so the rn=1 row is the true last op for that
|
||||
-- RecordID; an earlier Delete can no longer outrank a newer
|
||||
-- Insert for the same RecordID.
|
||||
-- AUDIT: @@ROWCOUNT is captured into @MergedOut IMMEDIATELY
|
||||
-- after the MERGE -- SET IDENTITY_INSERT (like any SET option)
|
||||
-- resets @@ROWCOUNT, so the order below is load-bearing.
|
||||
SET @sql = N'SET IDENTITY_INSERT ' + @FullName + N' ON;'
|
||||
+ N';WITH ranked AS ('
|
||||
+ N' SELECT RecordID, OperateType, RowData,'
|
||||
@@ -81,9 +115,11 @@ BEGIN
|
||||
+ N' WHEN MATCHED THEN UPDATE SET ' + @upd
|
||||
+ N' WHEN NOT MATCHED THEN INSERT (ID,' + @cols + N')'
|
||||
+ N' VALUES (TRY_CAST(src.RecordID AS int),' + @ins + N');'
|
||||
+ N'SET @MergedOut = @@ROWCOUNT;'
|
||||
+ N'SET IDENTITY_INSERT ' + @FullName + N' OFF;';
|
||||
EXEC sp_executesql @sql,
|
||||
N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
|
||||
N'@sch NVARCHAR(128),@tbl NVARCHAR(255),@MergedOut INT OUTPUT',
|
||||
@sch, @tbl, @MergedOut = @merged OUTPUT;
|
||||
|
||||
-- Delete: winners (rn=1) whose winning OperateType is Delete.
|
||||
-- Same ranked CTE — only the genuine last op can be a delete.
|
||||
@@ -95,14 +131,17 @@ BEGIN
|
||||
+ N')'
|
||||
+ N'DELETE t FROM ' + @FullName + N' t'
|
||||
+ N' JOIN (SELECT RecordID FROM ranked WHERE rn=1 AND OperateType=''Delete'') d'
|
||||
+ N' ON t.ID = TRY_CAST(d.RecordID AS int);';
|
||||
+ N' ON t.ID = TRY_CAST(d.RecordID AS int);'
|
||||
+ N'SET @DeletedOut = @@ROWCOUNT;';
|
||||
EXEC sp_executesql @sql,
|
||||
N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
|
||||
N'@sch NVARCHAR(128),@tbl NVARCHAR(255),@DeletedOut INT OUTPUT',
|
||||
@sch, @tbl, @DeletedOut = @deleted OUTPUT;
|
||||
END
|
||||
|
||||
UPDATE ProductionDataBaseSync.SyncQueue
|
||||
SET Status = 'applied', AppliedAt = SYSDATETIME()
|
||||
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
|
||||
SET @applied = @@ROWCOUNT;
|
||||
|
||||
COMMIT;
|
||||
END TRY
|
||||
@@ -113,13 +152,48 @@ BEGIN
|
||||
-- issues its own rollback. We roll back here so the partial per-table
|
||||
-- work is discarded before marking rows as 'error'/'dead'.
|
||||
IF @@TRANCOUNT > 0 ROLLBACK;
|
||||
SELECT @errMsg = ERROR_MESSAGE(), @outcome = 'error';
|
||||
|
||||
-- Split of the previous single CASE update so the audit row can
|
||||
-- report exactly how many rows died vs. how many will be retried.
|
||||
-- Net effect on SyncQueue is identical.
|
||||
UPDATE ProductionDataBaseSync.SyncQueue
|
||||
SET Status = CASE WHEN RetryCount + 1 >= @MaxRetries THEN 'dead' ELSE 'error' END,
|
||||
RetryCount = RetryCount + 1,
|
||||
ErrorMsg = ERROR_MESSAGE()
|
||||
SET Status = 'dead', RetryCount = RetryCount + 1, ErrorMsg = @errMsg
|
||||
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending'
|
||||
AND RetryCount + 1 >= @MaxRetries;
|
||||
SET @deadCnt = @@ROWCOUNT;
|
||||
|
||||
UPDATE ProductionDataBaseSync.SyncQueue
|
||||
SET Status = 'error', RetryCount = RetryCount + 1, ErrorMsg = @errMsg
|
||||
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
|
||||
SET @errCnt = @@ROWCOUNT;
|
||||
END CATCH
|
||||
|
||||
-- Best-effort audit row: sits outside the data transaction (after
|
||||
-- COMMIT/ROLLBACK) so it survives either outcome, and its own failure
|
||||
-- can never break the apply loop.
|
||||
IF @hasRunLog = 1
|
||||
BEGIN
|
||||
BEGIN TRY
|
||||
INSERT ProductionDataBaseSync.SyncApplyRunLog
|
||||
(CycleID, TargetSchema, TargetTable,
|
||||
PendingCount, DistinctRecords,
|
||||
MergedCount, DeletedCount, AppliedCount,
|
||||
ErrorCount, DeadCount,
|
||||
Outcome, ErrorMsg, StartedAt, DurationMs)
|
||||
VALUES
|
||||
(@CycleID, @sch, @tbl,
|
||||
@pending, @distinctRecs,
|
||||
@merged, @deleted, @applied,
|
||||
@errCnt, @deadCnt,
|
||||
@outcome, @errMsg, @t0,
|
||||
DATEDIFF(millisecond, @t0, SYSDATETIME()));
|
||||
END TRY
|
||||
BEGIN CATCH
|
||||
PRINT 'SyncApplyRunLog insert failed: ' + ERROR_MESSAGE();
|
||||
END CATCH
|
||||
END
|
||||
|
||||
FETCH NEXT FROM cur INTO @sch, @tbl;
|
||||
END
|
||||
|
||||
|
||||
Reference in New Issue
Block a user