Migrate sync objects into ProductionDataBaseSync schema + add permanent audit archive
- Add sql/00_schema.sql: create dedicated ProductionDataBaseSync schema (idempotent) - Move SyncQueue and usp_SyncApply from dbo into ProductionDataBaseSync - Add sql/03_sync_log_archive.sql: permanent, append-only SyncLogArchive that records both OriginalOperateType and ProcessedOperateType plus the Access log OriginalTime, so pipeline divergences (e.g. Insert applied as Delete) stay reconstructible forever (SyncQueue is transient and only keeps processed type) - config.py: inject sync_queue_table / archive_table / apply_proc (default to the new schema); SqlWriter takes these names instead of hardcoding dbo - sql_writer.py: add ArchiveRow + insert_archive_row (dedup on source keys), parametrize queue/archive/proc names throughout - capture.py: archive every consumed log row before enqueue (preserves evidence before cleanup deletes the Access log) - service.py: pass the three names into SqlWriter - tests: read queue/proc names from config instead of hardcoding dbo.SyncQueue
This commit is contained in:
Misaka_Company
@@ -1,8 +1,12 @@
|
||||
"""SQL-side writer for the Access -> SQL Server sync.
|
||||
|
||||
SqlWriter owns the pyodbc connection used to (a) dedup-insert rows into
|
||||
``dbo.SyncQueue``, (b) invoke ``dbo.usp_SyncApply`` to drain the queue, and
|
||||
(c) report which SourceLogIDs have been applied.
|
||||
SqlWriter owns the pyodbc connection used to (a) dedup-insert rows into the
|
||||
staging queue (``ProductionDataBaseSync.SyncQueue`` by default), (b) invoke the
|
||||
apply proc (``ProductionDataBaseSync.usp_SyncApply``) to drain the queue,
|
||||
(c) report which SourceLogIDs have been applied, and (d) append every consumed
|
||||
Access change-log row to the permanent audit store
|
||||
(``ProductionDataBaseSync.SyncLogArchive``). The queue/archive/proc names are
|
||||
injected from config so the whole sync can live under a dedicated schema.
|
||||
|
||||
The connection is opened with ``autocommit=True`` on purpose. ``usp_SyncApply``
|
||||
manages its own transaction internally (BEGIN TRAN ... ROLLBACK on error); if
|
||||
@@ -20,7 +24,7 @@ import pyodbc
|
||||
|
||||
@dataclass
|
||||
class QueueRow:
|
||||
"""A single staged change to enqueue into ``dbo.SyncQueue``."""
|
||||
"""A single staged change to enqueue into the SyncQueue table."""
|
||||
|
||||
source_file: str
|
||||
source_table: str
|
||||
@@ -32,12 +36,43 @@ class QueueRow:
|
||||
row_data: str | None
|
||||
|
||||
|
||||
class SqlWriter:
|
||||
"""Writes to SyncQueue and drives the apply proc over a pyodbc connection."""
|
||||
@dataclass
|
||||
class ArchiveRow:
|
||||
"""A single Access change-log row to append to ``SyncLogArchive``.
|
||||
|
||||
def __init__(self, conn_str: str, queue_table: str = "dbo.SyncQueue"):
|
||||
Captures both the ORIGINAL operate type recorded by the Access data macro
|
||||
and the PROCESSED operate type actually sent to the queue, so a downgrade
|
||||
(e.g. Insert -> Delete when the row is momentarily unreadable) stays visible
|
||||
forever. ``original_time`` preserves the Access log's own timestamp, which
|
||||
the queue path discards.
|
||||
"""
|
||||
|
||||
source_file: str
|
||||
source_table: str
|
||||
source_log_id: int
|
||||
record_id: str
|
||||
target_schema: str
|
||||
target_table: str
|
||||
original_operate_type: str
|
||||
processed_operate_type: str
|
||||
row_data: str | None
|
||||
original_time: object
|
||||
|
||||
|
||||
class SqlWriter:
|
||||
"""Writes to SyncQueue/SyncLogArchive and drives the apply proc."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
conn_str: str,
|
||||
queue_table: str = "ProductionDataBaseSync.SyncQueue",
|
||||
archive_table: str = "ProductionDataBaseSync.SyncLogArchive",
|
||||
apply_proc: str = "ProductionDataBaseSync.usp_SyncApply",
|
||||
):
|
||||
self.conn_str = conn_str
|
||||
self.queue_table = queue_table
|
||||
self.archive_table = archive_table
|
||||
self.apply_proc = apply_proc
|
||||
# autocommit=True: usp_SyncApply manages its own transaction internally.
|
||||
# An outer pyodbc transaction would conflict on ROLLBACK (SQL error 266).
|
||||
self._conn = pyodbc.connect(conn_str, autocommit=True)
|
||||
@@ -53,9 +88,9 @@ class SqlWriter:
|
||||
# twice (once for the EXISTS check, once for the INSERT).
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
"IF NOT EXISTS (SELECT 1 FROM dbo.SyncQueue "
|
||||
f"IF NOT EXISTS (SELECT 1 FROM {self.queue_table} "
|
||||
"WHERE SourceFile=? AND SourceTable=? AND SourceLogID=?) "
|
||||
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,"
|
||||
f"INSERT {self.queue_table}(SourceFile,SourceTable,SourceLogID,"
|
||||
"TargetSchema,TargetTable,RecordID,OperateType,RowData,Status) "
|
||||
"VALUES (?,?,?,?,?,?,?,?, 'pending')",
|
||||
row.source_file,
|
||||
@@ -72,19 +107,52 @@ class SqlWriter:
|
||||
)
|
||||
# autocommit: statement already committed.
|
||||
|
||||
def insert_archive_row(self, row: ArchiveRow) -> None:
|
||||
"""Append ``row`` to the permanent audit store (dedup on source keys).
|
||||
|
||||
Written during capture, BEFORE cleanup deletes the Access log, so the
|
||||
original evidence survives even after the queue row is purged. Stores
|
||||
both the original and processed operate types plus the Access log time.
|
||||
``IF NOT EXISTS`` keeps the first archive record if capture re-runs the
|
||||
same log id (a prior cycle's apply failed and the Access log persisted).
|
||||
"""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
f"IF NOT EXISTS (SELECT 1 FROM {self.archive_table} "
|
||||
"WHERE SourceFile=? AND SourceTable=? AND SourceLogID=?) "
|
||||
f"INSERT {self.archive_table}(SourceFile,SourceTable,SourceLogID,"
|
||||
"RecordID,TargetSchema,TargetTable,OriginalOperateType,"
|
||||
"ProcessedOperateType,RowData,OriginalTime) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,?)",
|
||||
row.source_file,
|
||||
row.source_table,
|
||||
row.source_log_id,
|
||||
row.source_file,
|
||||
row.source_table,
|
||||
row.source_log_id,
|
||||
row.record_id,
|
||||
row.target_schema,
|
||||
row.target_table,
|
||||
row.original_operate_type,
|
||||
row.processed_operate_type,
|
||||
row.row_data,
|
||||
row.original_time,
|
||||
)
|
||||
# autocommit: statement already committed.
|
||||
|
||||
def call_apply(self, max_retries: int) -> None:
|
||||
"""Drain the pending queue via the stored procedure.
|
||||
|
||||
``usp_SyncApply`` flips rows to ``applied`` (or ``error`` after retries).
|
||||
"""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute("EXEC dbo.usp_SyncApply ?", max_retries)
|
||||
cur.execute(f"EXEC {self.apply_proc} ?", max_retries)
|
||||
|
||||
def applied_log_ids(self, source_file: str) -> list[int]:
|
||||
"""Return applied SourceLogIDs for ``source_file`` in ascending order."""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT SourceLogID FROM dbo.SyncQueue "
|
||||
f"SELECT SourceLogID FROM {self.queue_table} "
|
||||
"WHERE SourceFile=? AND Status='applied' ORDER BY SourceLogID",
|
||||
source_file,
|
||||
)
|
||||
@@ -107,7 +175,7 @@ class SqlWriter:
|
||||
chunk = source_log_ids[i:i + 1000]
|
||||
placeholders = ",".join("?" * len(chunk))
|
||||
cur.execute(
|
||||
f"UPDATE dbo.SyncQueue SET Status='cleaned', "
|
||||
f"UPDATE {self.queue_table} SET Status='cleaned', "
|
||||
f"CleanedAt=sysdatetime() "
|
||||
f"WHERE SourceFile=? AND Status='applied' "
|
||||
f"AND SourceLogID IN ({placeholders})",
|
||||
@@ -120,12 +188,12 @@ class SqlWriter:
|
||||
|
||||
Keeps the table bounded: cleanup marks rows ``cleaned`` every cycle,
|
||||
and this removes the old ones after a short audit/debug window so
|
||||
``dbo.SyncQueue`` stops growing without bound. Returns the number of
|
||||
the queue table stops growing without bound. Returns the number of
|
||||
rows removed.
|
||||
"""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
"DELETE FROM dbo.SyncQueue "
|
||||
f"DELETE FROM {self.queue_table} "
|
||||
"WHERE Status='cleaned' "
|
||||
"AND CleanedAt < DATEADD(hour, -?, GETDATE())",
|
||||
retention_hours,
|
||||
|
||||
Reference in New Issue
Block a user