feat: SyncQueue table and set-based apply stored procedure

- sql/01_sync_queue.sql: idempotent DDL for dbo.SyncQueue (PK + unique
  dedup index + pending lookup index), safe to re-run.
- sql/02_sync_apply.sql: dbo.usp_SyncApply (@MaxRetries INT=5). Per
  distinct (TargetSchema,TargetTable) it builds column projections from
  sys.columns (excludes ID key/computed/identity/rowversion) and runs a
  dynamic-SQL MERGE (last-write-wins via ROW_NUMBER over SourceLogID DESC)
  for Insert/Update plus a DELETE for the last op = Delete.
  SET IDENTITY_INSERT ON preserves Access PKs.
- tests/conftest.py: sql_conn fixture reads conn_str from gitignored
  config.yaml via load_config; skipped without RUN_INTEGRATION=1.
- tests/test_apply_proc.py: integration test covering IDENTITY-preserving
  INSERT, last-write-wins UPDATE, BIT conversion, and DELETE; cleans up.

Deviation from the brief's procedure (root-cause fix, design preserved):
every JSON path key is quoted ('$."col"') so non-ASCII column names
(e.g. Chinese 名字/数量) parse correctly. Without quoting, JSON_VALUE
raises "JSON path format is not correct" on Chinese columns, which is the
real target schema for this Access->SQL Server sync.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-07-14 12:12:21 +08:00
parent 7c6cb10b91
commit 65939f5e85
4 changed files with 268 additions and 0 deletions

115
sql/02_sync_apply.sql Normal file
View File

@@ -0,0 +1,115 @@
-- usp_SyncApply: set-based apply of pending SyncQueue rows.
-- Per distinct (TargetSchema, TargetTable) it builds column lists from sys.columns
-- (excluding the ID key, computed, identity and rowversion columns) and runs a
-- dynamic-SQL MERGE (last-write-wins by SourceLogID DESC) for Insert/Update and a
-- DELETE for the last op = Delete. SET IDENTITY_INSERT ON preserves Access PKs.
CREATE OR ALTER PROCEDURE dbo.usp_SyncApply
@MaxRetries INT = 5
AS
BEGIN
SET NOCOUNT ON;
DECLARE @sch NVARCHAR(128), @tbl NVARCHAR(255), @FullName NVARCHAR(514);
DECLARE @sql NVARCHAR(MAX), @cols NVARCHAR(MAX), @upd NVARCHAR(MAX), @ins NVARCHAR(MAX);
-- Re-queue rows still under the retry budget.
UPDATE dbo.SyncQueue
SET Status = 'pending'
WHERE Status = 'error' AND RetryCount < @MaxRetries;
DECLARE cur CURSOR LOCAL FAST_FORWARD FOR
SELECT DISTINCT TargetSchema, TargetTable
FROM dbo.SyncQueue
WHERE Status = 'pending';
OPEN cur;
FETCH NEXT FROM cur INTO @sch, @tbl;
WHILE @@FETCH_STATUS = 0
BEGIN
SET @FullName = QUOTENAME(@sch) + N'.' + QUOTENAME(@tbl);
BEGIN TRY
BEGIN TRAN;
-- @cols: comma-quoted column names (for INSERT target list)
-- @upd: tgt.col = JSON_VALUE(src.RowData,'$.col')
-- @ins: JSON_VALUE(src.RowData,'$.col') (for INSERT VALUES)
-- NOTE: every JSON path key is wrapped in double quotes ('$."col"').
-- This is mandatory for non-ASCII column names (e.g. Chinese names
-- like 名字) and harmless for ASCII names, so we quote unconditionally.
SELECT
@cols = STRING_AGG(QUOTENAME(c.name), N',')
WITHIN GROUP (ORDER BY c.column_id),
@upd = STRING_AGG(QUOTENAME(c.name) + N'=JSON_VALUE(src.RowData,''$."' + c.name + N'"'')', N',')
WITHIN GROUP (ORDER BY c.column_id),
@ins = STRING_AGG(N'JSON_VALUE(src.RowData,''$."' + c.name + N'"'')', N',')
WITHIN GROUP (ORDER BY c.column_id)
FROM sys.columns c
WHERE c.object_id = OBJECT_ID(@FullName)
AND c.is_computed = 0
AND c.is_identity = 0
AND TYPE_NAME(c.system_type_id) <> 'timestamp'
AND c.name <> 'ID';
IF @cols IS NOT NULL
BEGIN
-- Upsert: last write wins (ROW_NUMBER over SourceLogID DESC).
SET @sql = N'SET IDENTITY_INSERT ' + @FullName + N' ON;'
+ N'MERGE ' + @FullName + N' WITH (HOLDLOCK) AS tgt'
+ N' USING ('
+ N' SELECT RecordID, RowData FROM ('
+ N' SELECT *, ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn'
+ N' FROM dbo.SyncQueue'
+ N' WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'''
+ N' AND OperateType IN (''Insert'',''Update'') AND RowData IS NOT NULL'
+ N' ) x WHERE rn=1'
+ N') AS src ON tgt.ID = TRY_CAST(src.RecordID AS int)'
+ N' WHEN MATCHED THEN UPDATE SET ' + @upd
+ N' WHEN NOT MATCHED THEN INSERT (ID,' + @cols + N')'
+ N' VALUES (TRY_CAST(src.RecordID AS int),' + @ins + N');'
+ N'SET IDENTITY_INSERT ' + @FullName + N' OFF;';
EXEC sp_executesql @sql,
N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
-- Delete: last op = Delete wins.
SET @sql = N'DELETE t FROM ' + @FullName + N' t'
+ N' JOIN ('
+ N' SELECT RecordID FROM ('
+ N' SELECT *, ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn'
+ N' FROM dbo.SyncQueue'
+ N' WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'''
+ N' AND OperateType=''Delete'''
+ N' ) x WHERE rn=1'
+ N') d ON t.ID = TRY_CAST(d.RecordID AS int);';
EXEC sp_executesql @sql,
N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
END
UPDATE dbo.SyncQueue
SET Status = 'applied', AppliedAt = SYSDATETIME()
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
COMMIT;
END TRY
BEGIN CATCH
-- ROLLBACK unwinds the per-table transaction (and, if the caller
-- started one, the caller's too). A proc that errors under an outer
-- transaction is expected to leave the txn uncommittable; the caller
-- issues its own rollback. We roll back here so the partial per-table
-- work is discarded before marking rows as 'error'/'dead'.
IF @@TRANCOUNT > 0 ROLLBACK;
UPDATE dbo.SyncQueue
SET Status = CASE WHEN RetryCount + 1 >= @MaxRetries THEN 'dead' ELSE 'error' END,
RetryCount = RetryCount + 1,
ErrorMsg = ERROR_MESSAGE()
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
END CATCH
FETCH NEXT FROM cur INTO @sch, @tbl;
END
CLOSE cur;
DEALLOCATE cur;
END
GO