AllForCompany/ProductionDataBaseSync_DataMacro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate sync objects into ProductionDataBaseSync schema + add permanent audit archive

Browse Source 
- Add sql/00_schema.sql: create dedicated ProductionDataBaseSync schema (idempotent)
- Move SyncQueue and usp_SyncApply from dbo into ProductionDataBaseSync
- Add sql/03_sync_log_archive.sql: permanent, append-only SyncLogArchive that
  records both OriginalOperateType and ProcessedOperateType plus the Access log
  OriginalTime, so pipeline divergences (e.g. Insert applied as Delete) stay
  reconstructible forever (SyncQueue is transient and only keeps processed type)
- config.py: inject sync_queue_table / archive_table / apply_proc (default to the
  new schema); SqlWriter takes these names instead of hardcoding dbo
- sql_writer.py: add ArchiveRow + insert_archive_row (dedup on source keys),
  parametrize queue/archive/proc names throughout
- capture.py: archive every consumed log row before enqueue (preserves evidence
  before cleanup deletes the Access log)
- service.py: pass the three names into SqlWriter
- tests: read queue/proc names from config instead of hardcoding dbo.SyncQueue
This commit is contained in:
Misaka_Company
2026-07-16 12:32:22 +08:00
parent 4179d3e232
commit d71b7eab62
11 changed files with 242 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
sql/00_schema.sql Normal file
View File

@@ -0,0 +1,12 @@
-- ProductionDataBaseSync: dedicated schema that groups every object owned by
-- the Access -> SQL Server sync (SyncQueue staging, SyncLogArchive audit store,
-- and the usp_SyncApply procedure). Keeping them out of dbo makes ownership and
-- housekeeping explicit.
-- Idempotent: CREATE SCHEMA must be the only statement in its batch, so it is
-- wrapped in EXEC() behind an existence check.
IF NOT EXISTS (SELECT 1 FROM sys.schemas WHERE name = 'ProductionDataBaseSync')
BEGIN
EXEC('CREATE SCHEMA ProductionDataBaseSync');
END
GO

29
sql/01_sync_queue.sql
View File

@@ -1,9 +1,10 @@
-- SyncQueue: staging table for the Access -> SQL Server one-way sync.
-- Lives under the ProductionDataBaseSync schema (run 00_schema.sql first).
-- Idempotent: safe to re-run (table created only if absent; indexes only if absent).
IF OBJECT_ID('dbo.SyncQueue', 'U') IS NULL
IF OBJECT_ID('ProductionDataBaseSync.SyncQueue', 'U') IS NULL
BEGIN
CREATE TABLE dbo.SyncQueue (
CREATE TABLE ProductionDataBaseSync.SyncQueue (
QueueID bigint IDENTITY(1,1) NOT NULL,
SourceFile nvarchar(255) NOT NULL,
SourceTable nvarchar(255) NOT NULL,
@@ -18,6 +19,7 @@ BEGIN
ErrorMsg nvarchar(max) NULL,
CapturedAt datetime2 NOT NULL CONSTRAINT DF_SyncQueue_Captured DEFAULT sysdatetime(),
AppliedAt datetime2 NULL,
CleanedAt datetime2 NULL,
CONSTRAINT PK_SyncQueue PRIMARY KEY CLUSTERED (QueueID)
);
END
@@ -25,38 +27,27 @@ GO
IF NOT EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'UX_SyncQueue_Dedup'
AND object_id = OBJECT_ID('dbo.SyncQueue'))
AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncQueue'))
BEGIN
CREATE UNIQUE INDEX UX_SyncQueue_Dedup
ON dbo.SyncQueue(SourceFile, SourceTable, SourceLogID);
ON ProductionDataBaseSync.SyncQueue(SourceFile, SourceTable, SourceLogID);
END
GO
IF NOT EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'IX_SyncQueue_Pending'
AND object_id = OBJECT_ID('dbo.SyncQueue'))
AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncQueue'))
BEGIN
CREATE INDEX IX_SyncQueue_Pending
ON dbo.SyncQueue(Status, TargetSchema, TargetTable);
END
GO
-- Cleanup bookkeeping: track when a queue row's Access log counterpart has
-- been physically deleted, so cleanup never re-deletes the same IDs and the
-- table can be purged to bound its growth.
IF NOT EXISTS (SELECT 1 FROM sys.columns
WHERE object_id = OBJECT_ID('dbo.SyncQueue')
AND name = 'CleanedAt')
BEGIN
ALTER TABLE dbo.SyncQueue ADD CleanedAt datetime2 NULL;
ON ProductionDataBaseSync.SyncQueue(Status, TargetSchema, TargetTable);
END
GO
IF NOT EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'IX_SyncQueue_Cleaned'
AND object_id = OBJECT_ID('dbo.SyncQueue'))
AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncQueue'))
BEGIN
CREATE INDEX IX_SyncQueue_Cleaned
ON dbo.SyncQueue(Status, CleanedAt);
ON ProductionDataBaseSync.SyncQueue(Status, CleanedAt);
END
GO

14
sql/02_sync_apply.sql
View File

@@ -11,7 +11,7 @@
-- and silently drop a row whose true last op was an Insert. Routing off the one
-- winning row's OperateType guarantees only the genuine last op wins.
CREATE OR ALTER PROCEDURE dbo.usp_SyncApply
CREATE OR ALTER PROCEDURE ProductionDataBaseSync.usp_SyncApply
@MaxRetries INT = 5
AS
BEGIN
@@ -21,13 +21,13 @@ BEGIN
DECLARE @sql NVARCHAR(MAX), @cols NVARCHAR(MAX), @upd NVARCHAR(MAX), @ins NVARCHAR(MAX);
-- Re-queue rows still under the retry budget.
UPDATE dbo.SyncQueue
UPDATE ProductionDataBaseSync.SyncQueue
SET Status = 'pending'
WHERE Status = 'error' AND RetryCount < @MaxRetries;
DECLARE cur CURSOR LOCAL FAST_FORWARD FOR
SELECT DISTINCT TargetSchema, TargetTable
FROM dbo.SyncQueue
FROM ProductionDataBaseSync.SyncQueue
WHERE Status = 'pending';
OPEN cur;
@@ -70,7 +70,7 @@ BEGIN
+ N';WITH ranked AS ('
+ N' SELECT RecordID, OperateType, RowData,'
+ N' ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn'
+ N' FROM dbo.SyncQueue'
+ N' FROM ProductionDataBaseSync.SyncQueue'
+ N' WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'''
+ N')'
+ N'MERGE ' + @FullName + N' WITH (HOLDLOCK) AS tgt'
@@ -90,7 +90,7 @@ BEGIN
SET @sql = N';WITH ranked AS ('
+ N' SELECT RecordID, OperateType,'
+ N' ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn'
+ N' FROM dbo.SyncQueue'
+ N' FROM ProductionDataBaseSync.SyncQueue'
+ N' WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'''
+ N')'
+ N'DELETE t FROM ' + @FullName + N' t'
@@ -100,7 +100,7 @@ BEGIN
N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl;
END
UPDATE dbo.SyncQueue
UPDATE ProductionDataBaseSync.SyncQueue
SET Status = 'applied', AppliedAt = SYSDATETIME()
WHERE TargetSchema = @sch AND TargetTable = @tbl AND Status = 'pending';
@@ -113,7 +113,7 @@ BEGIN
-- issues its own rollback. We roll back here so the partial per-table
-- work is discarded before marking rows as 'error'/'dead'.
IF @@TRANCOUNT > 0 ROLLBACK;
UPDATE dbo.SyncQueue
UPDATE ProductionDataBaseSync.SyncQueue
SET Status = CASE WHEN RetryCount + 1 >= @MaxRetries THEN 'dead' ELSE 'error' END,
RetryCount = RetryCount + 1,
ErrorMsg = ERROR_MESSAGE()

71
sql/03_sync_log_archive.sql Normal file
View File

@@ -0,0 +1,71 @@
-- SyncLogArchive: permanent, append-only audit store for every Access change-log
-- row consumed by the incremental sync. This is the durable evidence layer that
-- SyncQueue is NOT: SyncQueue is a transient work queue (purged 24h after a row
-- is cleaned) and it only stores the *processed* OperateType, so a downgrade
-- (e.g. capture turning an Insert into a Delete when the row is momentarily
-- unreadable) erases the original intent. This table preserves both the ORIGINAL
-- operate type recorded by the Access data macro and the PROCESSED type actually
-- sent to SQL Server, plus the original Access log timestamp and the row payload.
--
-- With this in place, cases like the 14287 incident (a real Insert applied to SQL
-- as a Delete) stay fully reconstructible: OriginalOperateType != ProcessedOperateType
-- flags exactly where the pipeline diverged from the source.
--
-- Retention: PERMANENT. No purge job touches this table. SyncQueue keeps its
-- short-lived queue role; this table keeps history. Lives under the
-- ProductionDataBaseSync schema (run 00_schema.sql first).
-- Idempotent: safe to re-run.
IF OBJECT_ID('ProductionDataBaseSync.SyncLogArchive', 'U') IS NULL
BEGIN
CREATE TABLE ProductionDataBaseSync.SyncLogArchive (
ArchiveID bigint IDENTITY(1,1) NOT NULL,
SourceFile nvarchar(255) NOT NULL,
SourceTable nvarchar(255) NOT NULL,
SourceLogID bigint NOT NULL,
RecordID nvarchar(50) NOT NULL,
TargetSchema nvarchar(128) NOT NULL,
TargetTable nvarchar(255) NOT NULL,
OriginalOperateType varchar(10) NOT NULL, -- as recorded by the Access data macro
ProcessedOperateType varchar(10) NOT NULL, -- as actually sent to SyncQueue / SQL
RowData nvarchar(max) NULL, -- captured row payload (NULL for Delete)
OriginalTime datetime2 NULL, -- Access TableChangeLog.Time (previously discarded)
CapturedAt datetime2 NOT NULL
CONSTRAINT DF_SyncLogArchive_Captured DEFAULT sysdatetime(),
CONSTRAINT PK_SyncLogArchive PRIMARY KEY CLUSTERED (ArchiveID)
);
END
GO
-- Dedup: one archive row per source log entry. Capture may re-run the same log
-- row if a prior cycle's apply failed (the Access log is only deleted after a
-- successful apply), so the write path uses IF NOT EXISTS on these keys.
IF NOT EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'UX_SyncLogArchive_Dedup'
AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncLogArchive'))
BEGIN
CREATE UNIQUE INDEX UX_SyncLogArchive_Dedup
ON ProductionDataBaseSync.SyncLogArchive(SourceFile, SourceTable, SourceLogID);
END
GO
-- Evidence lookup by table + record (e.g. "show every log ever seen for ID 14287").
IF NOT EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'IX_SyncLogArchive_Record'
AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncLogArchive'))
BEGIN
CREATE INDEX IX_SyncLogArchive_Record
ON ProductionDataBaseSync.SyncLogArchive(SourceTable, RecordID);
END
GO
-- Fast filter for the anomaly the archive exists to catch: original != processed.
IF NOT EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'IX_SyncLogArchive_Divergence'
AND object_id = OBJECT_ID('ProductionDataBaseSync.SyncLogArchive'))
BEGIN
CREATE INDEX IX_SyncLogArchive_Divergence
ON ProductionDataBaseSync.SyncLogArchive(OriginalOperateType, ProcessedOperateType)
INCLUDE (SourceFile, SourceTable, RecordID, CapturedAt);
END
GO