feat: SyncQueue table and set-based apply stored procedure
- sql/01_sync_queue.sql: idempotent DDL for dbo.SyncQueue (PK + unique
dedup index + pending lookup index), safe to re-run.
- sql/02_sync_apply.sql: dbo.usp_SyncApply (@MaxRetries INT=5). Per
distinct (TargetSchema,TargetTable) it builds column projections from
sys.columns (excludes ID key/computed/identity/rowversion) and runs a
dynamic-SQL MERGE (last-write-wins via ROW_NUMBER over SourceLogID DESC)
for Insert/Update plus a DELETE for the last op = Delete.
SET IDENTITY_INSERT ON preserves Access PKs.
- tests/conftest.py: sql_conn fixture reads conn_str from gitignored
config.yaml via load_config; skipped without RUN_INTEGRATION=1.
- tests/test_apply_proc.py: integration test covering IDENTITY-preserving
INSERT, last-write-wins UPDATE, BIT conversion, and DELETE; cleans up.
Deviation from the brief's procedure (root-cause fix, design preserved):
every JSON path key is quoted ('$."col"') so non-ASCII column names
(e.g. Chinese 名字/数量) parse correctly. Without quoting, JSON_VALUE
raises "JSON path format is not correct" on Chinese columns, which is the
real target schema for this Access->SQL Server sync.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
42
sql/01_sync_queue.sql
Normal file
42
sql/01_sync_queue.sql
Normal file
@@ -0,0 +1,42 @@
|
||||
-- SyncQueue: staging table for the Access -> SQL Server one-way sync.
|
||||
-- Idempotent: safe to re-run (table created only if absent; indexes only if absent).
|
||||
|
||||
IF OBJECT_ID('dbo.SyncQueue', 'U') IS NULL
|
||||
BEGIN
|
||||
CREATE TABLE dbo.SyncQueue (
|
||||
QueueID bigint IDENTITY(1,1) NOT NULL,
|
||||
SourceFile nvarchar(255) NOT NULL,
|
||||
SourceTable nvarchar(255) NOT NULL,
|
||||
SourceLogID bigint NOT NULL,
|
||||
TargetSchema nvarchar(128) NOT NULL,
|
||||
TargetTable nvarchar(255) NOT NULL,
|
||||
RecordID nvarchar(50) NOT NULL,
|
||||
OperateType varchar(10) NOT NULL,
|
||||
RowData nvarchar(max) NULL,
|
||||
Status varchar(10) NOT NULL CONSTRAINT DF_SyncQueue_Status DEFAULT 'pending',
|
||||
RetryCount int NOT NULL CONSTRAINT DF_SyncQueue_Retry DEFAULT 0,
|
||||
ErrorMsg nvarchar(max) NULL,
|
||||
CapturedAt datetime2 NOT NULL CONSTRAINT DF_SyncQueue_Captured DEFAULT sysdatetime(),
|
||||
AppliedAt datetime2 NULL,
|
||||
CONSTRAINT PK_SyncQueue PRIMARY KEY CLUSTERED (QueueID)
|
||||
);
|
||||
END
|
||||
GO
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM sys.indexes
|
||||
WHERE name = 'UX_SyncQueue_Dedup'
|
||||
AND object_id = OBJECT_ID('dbo.SyncQueue'))
|
||||
BEGIN
|
||||
CREATE UNIQUE INDEX UX_SyncQueue_Dedup
|
||||
ON dbo.SyncQueue(SourceFile, SourceTable, SourceLogID);
|
||||
END
|
||||
GO
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM sys.indexes
|
||||
WHERE name = 'IX_SyncQueue_Pending'
|
||||
AND object_id = OBJECT_ID('dbo.SyncQueue'))
|
||||
BEGIN
|
||||
CREATE INDEX IX_SyncQueue_Pending
|
||||
ON dbo.SyncQueue(Status, TargetSchema, TargetTable);
|
||||
END
|
||||
GO
|
||||
115
sql/02_sync_apply.sql
Normal file
115
sql/02_sync_apply.sql
Normal file
@@ -0,0 +1,115 @@
|
||||
-- usp_SyncApply: set-based apply of pending SyncQueue rows.
|
||||
-- Per distinct (TargetSchema, TargetTable) it builds column lists from sys.columns
|
||||
-- (excluding the ID key, computed, identity and rowversion columns) and runs a
|
||||
-- dynamic-SQL MERGE (last-write-wins by SourceLogID DESC) for Insert/Update and a
|
||||
-- DELETE for the last op = Delete. SET IDENTITY_INSERT ON preserves Access PKs.
|
||||
|
||||
CREATE OR ALTER PROCEDURE dbo.usp_SyncApply
|
||||
@MaxRetries INT = 5
|
||||
AS
|
||||
BEGIN
|
||||
SET NOCOUNT ON;
|
||||
|
||||
DECLARE @sch NVARCHAR(128), @tbl NVARCHAR(255), @FullName NVARCHAR(514);
|
||||
DECLARE @sql NVARCHAR(MAX), @cols NVARCHAR(MAX), @upd NVARCHAR(MAX), @ins NVARCHAR(MAX);
|
||||
|
||||
-- Re-queue rows still under the retry budget.
|
||||
UPDATE dbo.SyncQueue
|
||||
SET Status = 'pending'
|
||||
WHERE Status = 'error' AND RetryCount < @MaxRetries;
|
||||
|
||||
DECLARE cur CURSOR LOCAL FAST_FORWARD FOR
|
||||
SELECT DISTINCT TargetSchema, TargetTable
|
||||
FROM dbo.SyncQueue
|
||||
WHERE Status = 'pending';
|
||||
|
||||
OPEN cur;
|
||||
FETCH NEXT FROM cur INTO @sch, @tbl;
|
||||
|
||||
WHILE @@FETCH_STATUS = 0
|
||||
BEGIN
|
||||
SET @FullName = QUOTENAME(@sch) + N'.' + QUOTENAME(@tbl);
|
||||
|
||||
BEGIN TRY
|
||||
BEGIN TRAN;
|
||||
-- @cols: comma-quoted column names (for INSERT target list)
|
||||
-- @upd: tgt.col = JSON_VALUE(src.RowData,'$.col')
|
||||
-- @ins: JSON_VALUE(src.RowData,'$.col') (for INSERT VALUES)
|
||||
-- NOTE: every JSON path key is wrapped in double quotes ('$."col"').
|
||||
-- This is mandatory for non-ASCII column names (e.g. Chinese names
|
||||
-- like 名字) and harmless for ASCII names, so we quote unconditionally.
|
||||
SELECT
|
||||
@cols = STRING_AGG(QUOTENAME(c.name), N',')
|
||||
WITHIN GROUP (ORDER BY c.column_id),
|
||||
@upd = STRING_AGG(QUOTENAME(c.name) + N'=JSON_VALUE(src.RowData,''$."' + c.name + N'"'')', N',')
|
||||
WITHIN GROUP (ORDER BY c.column_id),
|
||||
@ins = STRING_AGG(N'JSON_VALUE(src.RowData,''$."' + c.name + N'"'')', N',')
|
||||
WITHIN GROUP (ORDER BY c.column_id)
|
||||
FROM sys.columns c
|
||||
WHERE c.object_id = OBJECT_ID(@FullName)
|
||||
AND c.is_computed = 0
|
||||
AND c.is_identity = 0
|
||||
AND TYPE_NAME(c.system_type_id) <> 'timestamp'
|
||||
AND c.name <> 'ID';
|
||||
|
||||
IF @cols IS NOT NULL
|
||||
BEGIN
|
||||
-- Upsert: last write wins (ROW_NUMBER over SourceLogID DESC).
|
||||
SET @sql = N'SET IDENTITY_INSERT ' + @FullName + N' ON;'
|
||||
+ N'MERGE ' + @FullName + N' WITH (HOLDLOCK) AS tgt'
|
||||
+ N' USING ('
|
||||
+ N' SELECT RecordID, RowData FROM ('
|
||||
+ N' SELECT *, ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn'
|
||||
+ N' FROM dbo.SyncQueue'
|
||||
+ N' WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'''
|
||||
+ N' AND OperateType IN (''Insert'',''Update'') AND RowData IS NOT NULL'
|
||||
+ N' ) x WHERE rn=1'
|
||||
+ N') AS src ON tgt.ID = TRY_CAST(src.RecordID AS int)'
|
||||
+ N' WHEN MATCHED THEN UPDATE SET ' + @upd
|
||||
+ N' WHEN NOT MATCHED THEN INSERT (ID,' + @cols + N')'
|
||||
+ N' VALUES (TRY_CAST(src.RecordID AS int),' + @ins + N');'
|
||||
+ N'SET IDENTITY_INSERT ' + @FullName + N' OFF;';
|
||||
EXEC sp_executesql @sql,
|
||||
N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
|
||||
|
||||
-- Delete: last op = Delete wins.
|
||||
SET @sql = N'DELETE t FROM ' + @FullName + N' t'
|
||||
+ N' JOIN ('
|
||||
+ N' SELECT RecordID FROM ('
|
||||
+ N' SELECT *, ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn'
|
||||
+ N' FROM dbo.SyncQueue'
|
||||
+ N' WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'''
|
||||
+ N' AND OperateType=''Delete'''
|
||||
+ N' ) x WHERE rn=1'
|
||||
+ N') d ON t.ID = TRY_CAST(d.RecordID AS int);';
|
||||
EXEC sp_executesql @sql,
|
||||
N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
|
||||
END
|
||||
|
||||
UPDATE dbo.SyncQueue
|
||||
SET Status = 'applied', AppliedAt = SYSDATETIME()
|
||||
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
|
||||
|
||||
COMMIT;
|
||||
END TRY
|
||||
BEGIN CATCH
|
||||
-- ROLLBACK unwinds the per-table transaction (and, if the caller
|
||||
-- started one, the caller's too). A proc that errors under an outer
|
||||
-- transaction is expected to leave the txn uncommittable; the caller
|
||||
-- issues its own rollback. We roll back here so the partial per-table
|
||||
-- work is discarded before marking rows as 'error'/'dead'.
|
||||
IF @@TRANCOUNT > 0 ROLLBACK;
|
||||
UPDATE dbo.SyncQueue
|
||||
SET Status = CASE WHEN RetryCount + 1 >= @MaxRetries THEN 'dead' ELSE 'error' END,
|
||||
RetryCount = RetryCount + 1,
|
||||
ErrorMsg = ERROR_MESSAGE()
|
||||
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
|
||||
END CATCH
|
||||
|
||||
FETCH NEXT FROM cur INTO @sch, @tbl;
|
||||
END
|
||||
|
||||
CLOSE cur;
|
||||
DEALLOCATE cur;
|
||||
END
|
||||
GO
|
||||
Reference in New Issue
Block a user