Refine incremental sync audit observability

- Add per-cycle correlation ID ([cyc:xxxxxxxx]) threaded through Python
  logs and SQL audit tables for end-to-end traceability of any divergence.
- New ProductionDataBaseSync.SyncApplyRunLog table + @CycleID on usp_SyncApply
  for per-table apply auditing (pending/merged/deleted/applied, dead/error
  counts, duration). Audit write isolated in its own TRY/CATCH outside txn.
- Capture/cleanup phases now emit full detail: Insert->Delete downgrade
  warning with record_id/log_id, dedup_skipped as apply-stall signal,
  per-file summary, and access log-id ranges on cleanup.
- Queue health check surfaces error/dead rows with recent samples instead of
  silent accumulation (previously the top cause of data divergence).
- sql_writer uses INSERT...SELECT...WHERE NOT EXISTS for observable dedup;
  idle cycles lowered to DEBUG with periodic heartbeat.
- Backward compatible: old proc callers still work (CycleID nullable; legacy
  coarse-grained logging with one-time notice).

Excluded from this commit: CODE.md, build_code_doc.py (doc generation).

sql/02_sync_apply.sql

@@ -10,15 +10,33 @@
 -- Delete), which let a stale Delete outrank a newer Insert for the same RecordID
 -- and silently drop a row whose true last op was an Insert. Routing off the one
 -- winning row's OperateType guarantees only the genuine last op wins.
+--
+-- AUDIT NOTE: when ProductionDataBaseSync.SyncApplyRunLog exists (see
+-- sql/04_sync_apply_runlog.sql), one audit row is written per target table and
+-- per invocation: pending/distinct counts going in, the MERGE/DELETE rowcounts,
+-- how many queue rows were flipped to applied (or to error/dead on failure),
+-- the error message, the duration, and the caller-supplied @CycleID that ties
+-- the row to the Python service log's [cyc:...] lines. The audit insert is
+-- best-effort (wrapped in its own TRY/CATCH, outside the data transaction) and
+-- can never fail the apply itself. @CycleID defaults to NULL so the legacy
+-- single-parameter EXEC keeps working.
 
 CREATE OR ALTER PROCEDURE ProductionDataBaseSync.usp_SyncApply
-    @MaxRetries INT = 5
+    @MaxRetries INT = 5,
+    @CycleID    NVARCHAR(40) = NULL
 AS
 BEGIN
     SET NOCOUNT ON;
 
     DECLARE @sch NVARCHAR(128), @tbl NVARCHAR(255), @FullName NVARCHAR(514);
     DECLARE @sql NVARCHAR(MAX), @cols NVARCHAR(MAX), @upd NVARCHAR(MAX), @ins NVARCHAR(MAX);
+    DECLARE @hasRunLog bit =
+        CASE WHEN OBJECT_ID('ProductionDataBaseSync.SyncApplyRunLog', 'U') IS NOT NULL
+             THEN 1 ELSE 0 END;
+    DECLARE @t0 DATETIME2, @pending INT, @distinctRecs INT,
+            @merged INT, @deleted INT, @applied INT,
+            @errCnt INT, @deadCnt INT,
+            @errMsg NVARCHAR(MAX), @outcome VARCHAR(10);
 
     -- Re-queue rows still under the retry budget.
     UPDATE ProductionDataBaseSync.SyncQueue
@@ -37,6 +55,19 @@ BEGIN
     BEGIN
         SET @FullName = QUOTENAME(@sch) + N'.' + QUOTENAME(@tbl);
 
+        -- Per-table audit state. Explicit reset every iteration: local
+        -- variables keep their previous value across cursor loops.
+        SELECT @t0 = SYSDATETIME(),
+               @merged = NULL, @deleted = NULL, @applied = NULL,
+               @errCnt = NULL, @deadCnt = NULL,
+               @errMsg = NULL, @outcome = 'ok',
+               @cols = NULL, @upd = NULL, @ins = NULL;
+
+        SELECT @pending      = COUNT(*),
+               @distinctRecs = COUNT(DISTINCT RecordID)
+          FROM ProductionDataBaseSync.SyncQueue
+         WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
+
         BEGIN TRY
             BEGIN TRAN;
             -- @cols: comma-quoted column names (for INSERT target list)
@@ -66,6 +97,9 @@ BEGIN
                 -- pending ops so the rn=1 row is the true last op for that
                 -- RecordID; an earlier Delete can no longer outrank a newer
                 -- Insert for the same RecordID.
+                -- AUDIT: @@ROWCOUNT is captured into @MergedOut IMMEDIATELY
+                -- after the MERGE -- SET IDENTITY_INSERT (like any SET option)
+                -- resets @@ROWCOUNT, so the order below is load-bearing.
                 SET @sql = N'SET IDENTITY_INSERT ' + @FullName + N' ON;'
                          + N';WITH ranked AS ('
                          + N'  SELECT RecordID, OperateType, RowData,'
@@ -81,9 +115,11 @@ BEGIN
                          + N' WHEN MATCHED THEN UPDATE SET ' + @upd
                          + N' WHEN NOT MATCHED THEN INSERT (ID,' + @cols + N')'
                          + N'  VALUES (TRY_CAST(src.RecordID AS int),' + @ins + N');'
+                         + N'SET @MergedOut = @@ROWCOUNT;'
                          + N'SET IDENTITY_INSERT ' + @FullName + N' OFF;';
                 EXEC sp_executesql @sql,
-                     N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
+                     N'@sch NVARCHAR(128),@tbl NVARCHAR(255),@MergedOut INT OUTPUT',
+                     @sch, @tbl, @MergedOut = @merged OUTPUT;
 
                 -- Delete: winners (rn=1) whose winning OperateType is Delete.
                 -- Same ranked CTE — only the genuine last op can be a delete.
@@ -95,14 +131,17 @@ BEGIN
                          + N')'
                          + N'DELETE t FROM ' + @FullName + N' t'
                          + N' JOIN (SELECT RecordID FROM ranked WHERE rn=1 AND OperateType=''Delete'') d'
-                         + N'   ON t.ID = TRY_CAST(d.RecordID AS int);';
+                         + N'   ON t.ID = TRY_CAST(d.RecordID AS int);'
+                         + N'SET @DeletedOut = @@ROWCOUNT;';
                 EXEC sp_executesql @sql,
-                     N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
+                     N'@sch NVARCHAR(128),@tbl NVARCHAR(255),@DeletedOut INT OUTPUT',
+                     @sch, @tbl, @DeletedOut = @deleted OUTPUT;
             END
 
             UPDATE ProductionDataBaseSync.SyncQueue
                SET Status = 'applied', AppliedAt = SYSDATETIME()
              WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
+            SET @applied = @@ROWCOUNT;
 
             COMMIT;
         END TRY
@@ -113,13 +152,48 @@ BEGIN
             -- issues its own rollback. We roll back here so the partial per-table
             -- work is discarded before marking rows as 'error'/'dead'.
             IF @@TRANCOUNT > 0 ROLLBACK;
+            SELECT @errMsg = ERROR_MESSAGE(), @outcome = 'error';
+
+            -- Split of the previous single CASE update so the audit row can
+            -- report exactly how many rows died vs. how many will be retried.
+            -- Net effect on SyncQueue is identical.
             UPDATE ProductionDataBaseSync.SyncQueue
-               SET Status      = CASE WHEN RetryCount + 1 >= @MaxRetries THEN 'dead' ELSE 'error' END,
-                   RetryCount   = RetryCount + 1,
-                   ErrorMsg     = ERROR_MESSAGE()
+               SET Status = 'dead', RetryCount = RetryCount + 1, ErrorMsg = @errMsg
+             WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending'
+               AND RetryCount + 1 >= @MaxRetries;
+            SET @deadCnt = @@ROWCOUNT;
+
+            UPDATE ProductionDataBaseSync.SyncQueue
+               SET Status = 'error', RetryCount = RetryCount + 1, ErrorMsg = @errMsg
              WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
+            SET @errCnt = @@ROWCOUNT;
         END CATCH
 
+        -- Best-effort audit row: sits outside the data transaction (after
+        -- COMMIT/ROLLBACK) so it survives either outcome, and its own failure
+        -- can never break the apply loop.
+        IF @hasRunLog = 1
+        BEGIN
+            BEGIN TRY
+                INSERT ProductionDataBaseSync.SyncApplyRunLog
+                    (CycleID, TargetSchema, TargetTable,
+                     PendingCount, DistinctRecords,
+                     MergedCount, DeletedCount, AppliedCount,
+                     ErrorCount, DeadCount,
+                     Outcome, ErrorMsg, StartedAt, DurationMs)
+                VALUES
+                    (@CycleID, @sch, @tbl,
+                     @pending, @distinctRecs,
+                     @merged, @deleted, @applied,
+                     @errCnt, @deadCnt,
+                     @outcome, @errMsg, @t0,
+                     DATEDIFF(millisecond, @t0, SYSDATETIME()));
+            END TRY
+            BEGIN CATCH
+                PRINT 'SyncApplyRunLog insert failed: ' + ERROR_MESSAGE();
+            END CATCH
+        END
+
         FETCH NEXT FROM cur INTO @sch, @tbl;
     END
 

sql/04_sync_apply_runlog.sql

@@ -0,0 +1,75 @@
+-- SyncApplyRunLog: per-invocation, per-target-table audit of what usp_SyncApply
+-- actually did. This closes the biggest observability gap in the pipeline: the
+-- apply phase used to be a black box ("apply done") -- queue rows could flip
+-- to 'error' or 'dead' with no trace in the service log, and there was no
+-- record of how many rows a MERGE/DELETE touched at any point in time. With
+-- this table, "what did apply do to table X around time T, and did it fail?"
+-- is a single indexed query, and CycleID joins each row back to the exact
+-- [cyc:xxxxxxxx] lines in the Python service log.
+--
+-- Written by usp_SyncApply (sql/02_sync_apply.sql) as a best-effort insert per
+-- (invocation, target table). Idle cycles write nothing (the proc's cursor
+-- only visits tables that have pending rows), so growth tracks real change
+-- traffic, not poll frequency. Retention: unmanaged by default; if it ever
+-- grows large, purge by StartedAt, e.g.
+--   DELETE FROM ProductionDataBaseSync.SyncApplyRunLog
+--    WHERE StartedAt < DATEADD(day, -90, SYSDATETIME());
+--
+-- Lives under the ProductionDataBaseSync schema (run 00_schema.sql first).
+-- Idempotent: safe to re-run. Deploy alongside the updated 02_sync_apply.sql;
+-- ordering is forgiving either way (the proc checks for this table and skips
+-- the audit insert when it is absent).
+
+IF OBJECT_ID('ProductionDataBaseSync.SyncApplyRunLog', 'U') IS NULL
+BEGIN
+    CREATE TABLE ProductionDataBaseSync.SyncApplyRunLog (
+        RunLogID        bigint IDENTITY(1,1) NOT NULL,
+        CycleID         nvarchar(40)   NULL,      -- correlation id from the Python service ([cyc:...])
+        TargetSchema    nvarchar(128)  NOT NULL,
+        TargetTable     nvarchar(255)  NOT NULL,
+        PendingCount    int            NOT NULL,  -- pending queue rows seen for this table
+        DistinctRecords int            NULL,      -- distinct RecordIDs among them
+        MergedCount     int            NULL,      -- rows affected by the MERGE (insert + update)
+        DeletedCount    int            NULL,      -- rows affected by the DELETE
+        AppliedCount    int            NULL,      -- queue rows flipped to 'applied'
+        ErrorCount      int            NULL,      -- queue rows flipped to 'error' (will retry)
+        DeadCount       int            NULL,      -- queue rows flipped to 'dead' (retries exhausted)
+        Outcome         varchar(10)    NOT NULL,  -- 'ok' | 'error'
+        ErrorMsg        nvarchar(max)  NULL,
+        StartedAt       datetime2      NOT NULL
+                        CONSTRAINT DF_SyncApplyRunLog_Started DEFAULT sysdatetime(),
+        DurationMs      int            NULL,
+        CONSTRAINT PK_SyncApplyRunLog PRIMARY KEY CLUSTERED (RunLogID)
+    );
+END
+GO
+
+-- Join back to the Python service log of one cycle.
+IF NOT EXISTS (SELECT 1 FROM sys.indexes
+               WHERE name = 'IX_SyncApplyRunLog_Cycle'
+                 AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncApplyRunLog'))
+BEGIN
+    CREATE INDEX IX_SyncApplyRunLog_Cycle
+        ON ProductionDataBaseSync.SyncApplyRunLog(CycleID);
+END
+GO
+
+-- "What happened to this table around time T?"
+IF NOT EXISTS (SELECT 1 FROM sys.indexes
+               WHERE name = 'IX_SyncApplyRunLog_Table'
+                 AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncApplyRunLog'))
+BEGIN
+    CREATE INDEX IX_SyncApplyRunLog_Table
+        ON ProductionDataBaseSync.SyncApplyRunLog(TargetSchema, TargetTable, StartedAt);
+END
+GO
+
+-- Fast scan for failed applies.
+IF NOT EXISTS (SELECT 1 FROM sys.indexes
+               WHERE name = 'IX_SyncApplyRunLog_Outcome'
+                 AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncApplyRunLog'))
+BEGIN
+    CREATE INDEX IX_SyncApplyRunLog_Outcome
+        ON ProductionDataBaseSync.SyncApplyRunLog(Outcome, StartedAt);
+END
+GO