d71b7eab62
- Add sql/00_schema.sql: create dedicated ProductionDataBaseSync schema (idempotent) - Move SyncQueue and usp_SyncApply from dbo into ProductionDataBaseSync - Add sql/03_sync_log_archive.sql: permanent, append-only SyncLogArchive that records both OriginalOperateType and ProcessedOperateType plus the Access log OriginalTime, so pipeline divergences (e.g. Insert applied as Delete) stay reconstructible forever (SyncQueue is transient and only keeps processed type) - config.py: inject sync_queue_table / archive_table / apply_proc (default to the new schema); SqlWriter takes these names instead of hardcoding dbo - sql_writer.py: add ArchiveRow + insert_archive_row (dedup on source keys), parametrize queue/archive/proc names throughout - capture.py: archive every consumed log row before enqueue (preserves evidence before cleanup deletes the Access log) - service.py: pass the three names into SqlWriter - tests: read queue/proc names from config instead of hardcoding dbo.SyncQueue
305 lines
12 KiB
Python
305 lines
12 KiB
Python
"""SQL-side writer for the Access -> SQL Server sync.
|
|
|
|
SqlWriter owns the pyodbc connection used to (a) dedup-insert rows into the
|
|
staging queue (``ProductionDataBaseSync.SyncQueue`` by default), (b) invoke the
|
|
apply proc (``ProductionDataBaseSync.usp_SyncApply``) to drain the queue,
|
|
(c) report which SourceLogIDs have been applied, and (d) append every consumed
|
|
Access change-log row to the permanent audit store
|
|
(``ProductionDataBaseSync.SyncLogArchive``). The queue/archive/proc names are
|
|
injected from config so the whole sync can live under a dedicated schema.
|
|
|
|
The connection is opened with ``autocommit=True`` on purpose. ``usp_SyncApply``
|
|
manages its own transaction internally (BEGIN TRAN ... ROLLBACK on error); if
|
|
the caller held an outer implicit transaction, the proc's ROLLBACK would
|
|
cascade and raise SQL error 266. The dedup ``IF NOT EXISTS ... INSERT`` is a
|
|
single statement that is atomic under autocommit, so no explicit transaction is
|
|
needed on the write path either.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
import pyodbc
|
|
|
|
|
|
@dataclass
|
|
class QueueRow:
|
|
"""A single staged change to enqueue into the SyncQueue table."""
|
|
|
|
source_file: str
|
|
source_table: str
|
|
record_id: str
|
|
target_schema: str
|
|
target_table: str
|
|
source_log_id: int
|
|
operate_type: str
|
|
row_data: str | None
|
|
|
|
|
|
@dataclass
|
|
class ArchiveRow:
|
|
"""A single Access change-log row to append to ``SyncLogArchive``.
|
|
|
|
Captures both the ORIGINAL operate type recorded by the Access data macro
|
|
and the PROCESSED operate type actually sent to the queue, so a downgrade
|
|
(e.g. Insert -> Delete when the row is momentarily unreadable) stays visible
|
|
forever. ``original_time`` preserves the Access log's own timestamp, which
|
|
the queue path discards.
|
|
"""
|
|
|
|
source_file: str
|
|
source_table: str
|
|
source_log_id: int
|
|
record_id: str
|
|
target_schema: str
|
|
target_table: str
|
|
original_operate_type: str
|
|
processed_operate_type: str
|
|
row_data: str | None
|
|
original_time: object
|
|
|
|
|
|
class SqlWriter:
|
|
"""Writes to SyncQueue/SyncLogArchive and drives the apply proc."""
|
|
|
|
def __init__(
|
|
self,
|
|
conn_str: str,
|
|
queue_table: str = "ProductionDataBaseSync.SyncQueue",
|
|
archive_table: str = "ProductionDataBaseSync.SyncLogArchive",
|
|
apply_proc: str = "ProductionDataBaseSync.usp_SyncApply",
|
|
):
|
|
self.conn_str = conn_str
|
|
self.queue_table = queue_table
|
|
self.archive_table = archive_table
|
|
self.apply_proc = apply_proc
|
|
# autocommit=True: usp_SyncApply manages its own transaction internally.
|
|
# An outer pyodbc transaction would conflict on ROLLBACK (SQL error 266).
|
|
self._conn = pyodbc.connect(conn_str, autocommit=True)
|
|
|
|
def insert_queue_row(self, row: QueueRow) -> None:
|
|
"""Idempotently enqueue ``row`` (dedup on SourceFile/Table/LogID).
|
|
|
|
``IF NOT EXISTS ... INSERT`` is a single statement, atomic under
|
|
autocommit. The unique index UX_SyncQueue_Dedup is the DB backstop.
|
|
"""
|
|
# IF NOT EXISTS and the VALUES list each carry their own ? markers;
|
|
# pyodbc binds them positionally, so the 3 dedup keys are supplied
|
|
# twice (once for the EXISTS check, once for the INSERT).
|
|
cur = self._conn.cursor()
|
|
cur.execute(
|
|
f"IF NOT EXISTS (SELECT 1 FROM {self.queue_table} "
|
|
"WHERE SourceFile=? AND SourceTable=? AND SourceLogID=?) "
|
|
f"INSERT {self.queue_table}(SourceFile,SourceTable,SourceLogID,"
|
|
"TargetSchema,TargetTable,RecordID,OperateType,RowData,Status) "
|
|
"VALUES (?,?,?,?,?,?,?,?, 'pending')",
|
|
row.source_file,
|
|
row.source_table,
|
|
row.source_log_id,
|
|
row.source_file,
|
|
row.source_table,
|
|
row.source_log_id,
|
|
row.target_schema,
|
|
row.target_table,
|
|
row.record_id,
|
|
row.operate_type,
|
|
row.row_data,
|
|
)
|
|
# autocommit: statement already committed.
|
|
|
|
def insert_archive_row(self, row: ArchiveRow) -> None:
|
|
"""Append ``row`` to the permanent audit store (dedup on source keys).
|
|
|
|
Written during capture, BEFORE cleanup deletes the Access log, so the
|
|
original evidence survives even after the queue row is purged. Stores
|
|
both the original and processed operate types plus the Access log time.
|
|
``IF NOT EXISTS`` keeps the first archive record if capture re-runs the
|
|
same log id (a prior cycle's apply failed and the Access log persisted).
|
|
"""
|
|
cur = self._conn.cursor()
|
|
cur.execute(
|
|
f"IF NOT EXISTS (SELECT 1 FROM {self.archive_table} "
|
|
"WHERE SourceFile=? AND SourceTable=? AND SourceLogID=?) "
|
|
f"INSERT {self.archive_table}(SourceFile,SourceTable,SourceLogID,"
|
|
"RecordID,TargetSchema,TargetTable,OriginalOperateType,"
|
|
"ProcessedOperateType,RowData,OriginalTime) "
|
|
"VALUES (?,?,?,?,?,?,?,?,?,?)",
|
|
row.source_file,
|
|
row.source_table,
|
|
row.source_log_id,
|
|
row.source_file,
|
|
row.source_table,
|
|
row.source_log_id,
|
|
row.record_id,
|
|
row.target_schema,
|
|
row.target_table,
|
|
row.original_operate_type,
|
|
row.processed_operate_type,
|
|
row.row_data,
|
|
row.original_time,
|
|
)
|
|
# autocommit: statement already committed.
|
|
|
|
def call_apply(self, max_retries: int) -> None:
|
|
"""Drain the pending queue via the stored procedure.
|
|
|
|
``usp_SyncApply`` flips rows to ``applied`` (or ``error`` after retries).
|
|
"""
|
|
cur = self._conn.cursor()
|
|
cur.execute(f"EXEC {self.apply_proc} ?", max_retries)
|
|
|
|
def applied_log_ids(self, source_file: str) -> list[int]:
|
|
"""Return applied SourceLogIDs for ``source_file`` in ascending order."""
|
|
cur = self._conn.cursor()
|
|
cur.execute(
|
|
f"SELECT SourceLogID FROM {self.queue_table} "
|
|
"WHERE SourceFile=? AND Status='applied' ORDER BY SourceLogID",
|
|
source_file,
|
|
)
|
|
return [r[0] for r in cur.fetchall()]
|
|
|
|
def mark_cleaned(self, source_file: str, source_log_ids: list[int]) -> None:
|
|
"""Mark applied queue rows for ``source_file`` as ``cleaned``.
|
|
|
|
Called after the corresponding Access log rows have been physically
|
|
deleted. Once a row is ``cleaned``, ``applied_log_ids`` no longer
|
|
returns it, so the same IDs are never deleted from Access twice. Rows
|
|
are matched on ``(SourceFile, SourceLogID)`` and constrained to
|
|
``Status='applied'``, so a row that errored out is never silently
|
|
marked clean. Chunked to stay under SQL Server's 2100-param limit.
|
|
"""
|
|
if not source_log_ids:
|
|
return
|
|
cur = self._conn.cursor()
|
|
for i in range(0, len(source_log_ids), 1000):
|
|
chunk = source_log_ids[i:i + 1000]
|
|
placeholders = ",".join("?" * len(chunk))
|
|
cur.execute(
|
|
f"UPDATE {self.queue_table} SET Status='cleaned', "
|
|
f"CleanedAt=sysdatetime() "
|
|
f"WHERE SourceFile=? AND Status='applied' "
|
|
f"AND SourceLogID IN ({placeholders})",
|
|
source_file,
|
|
*chunk,
|
|
)
|
|
|
|
def purge_cleaned(self, retention_hours: int) -> int:
|
|
"""Delete ``cleaned`` rows older than ``retention_hours``.
|
|
|
|
Keeps the table bounded: cleanup marks rows ``cleaned`` every cycle,
|
|
and this removes the old ones after a short audit/debug window so
|
|
the queue table stops growing without bound. Returns the number of
|
|
rows removed.
|
|
"""
|
|
cur = self._conn.cursor()
|
|
cur.execute(
|
|
f"DELETE FROM {self.queue_table} "
|
|
"WHERE Status='cleaned' "
|
|
"AND CleanedAt < DATEADD(hour, -?, GETDATE())",
|
|
retention_hours,
|
|
)
|
|
return cur.rowcount
|
|
|
|
def table_exists(self, schema: str, table: str) -> bool:
|
|
"""True if ``[schema].[table]`` exists in the target database."""
|
|
cur = self._conn.cursor()
|
|
cur.execute(
|
|
"SELECT 1 FROM sys.tables t JOIN sys.schemas s "
|
|
"ON s.schema_id = t.schema_id "
|
|
"WHERE s.name = ? AND t.name = ?",
|
|
schema, table,
|
|
)
|
|
return cur.fetchone() is not None
|
|
|
|
def count_target(self, schema: str, table: str) -> int:
|
|
"""Return ``COUNT(*)`` for ``[schema].[table]`` (compare count check)."""
|
|
cur = self._conn.cursor()
|
|
cur.execute(f"SELECT COUNT(*) FROM [{schema}].[{table}]")
|
|
return cur.fetchone()[0]
|
|
|
|
def read_target_ids(self, schema: str, table: str) -> list:
|
|
"""Return every ``ID`` from ``[schema].[table]``, ascending (compare IDs)."""
|
|
cur = self._conn.cursor()
|
|
cur.execute(f"SELECT ID FROM [{schema}].[{table}] ORDER BY ID")
|
|
return [r[0] for r in cur.fetchall()]
|
|
|
|
def _has_identity(self, schema: str, table: str) -> bool:
|
|
"""True if ``[schema].[table]`` has an IDENTITY column (the ``ID`` PK)."""
|
|
cur = self._conn.cursor()
|
|
cur.execute(
|
|
"SELECT 1 FROM sys.tables t JOIN sys.schemas s "
|
|
"ON s.schema_id = t.schema_id JOIN sys.columns c "
|
|
"ON c.object_id = t.object_id "
|
|
"WHERE s.name = ? AND t.name = ? AND c.is_identity = 1",
|
|
schema, table,
|
|
)
|
|
return cur.fetchone() is not None
|
|
|
|
def truncate_target(self, schema: str, table: str) -> None:
|
|
"""Empty ``[schema].[table]`` prior to a full reload.
|
|
|
|
Prefers ``TRUNCATE TABLE`` (fast, minimal logging). This project's
|
|
target schemas have no foreign keys, so TRUNCATE always succeeds; the
|
|
``DELETE FROM`` fallback only matters if a future FK is added or the
|
|
table sits under snapshot isolation.
|
|
"""
|
|
full = f"[{schema}].[{table}]"
|
|
cur = self._conn.cursor()
|
|
try:
|
|
cur.execute(f"TRUNCATE TABLE {full}")
|
|
except pyodbc.Error:
|
|
cur.execute(f"DELETE FROM {full}")
|
|
|
|
def bulk_insert(
|
|
self, schema: str, table: str, columns: list[str], rows: list[tuple]
|
|
) -> int:
|
|
"""Insert every ``row`` into ``[schema].[table]``.
|
|
|
|
``columns`` and the tuples in ``rows`` must be positionally aligned.
|
|
When the target has an IDENTITY column (the ``ID`` PK in every target
|
|
here), ``SET IDENTITY_INSERT`` is enabled so the original Access primary
|
|
keys are preserved — required for the incremental sync's RecordID
|
|
matching to keep working afterwards.
|
|
|
|
Rows are inserted in 1000-row chunks. ``fast_executemany`` is tried
|
|
first for speed; if a row's types can't be inferred (e.g. a leading
|
|
NULL), it transparently retries the chunk without it. Returns the total
|
|
number of rows inserted.
|
|
"""
|
|
if not columns or not rows:
|
|
return 0
|
|
full = f"[{schema}].[{table}]"
|
|
col_list = ", ".join(f"[{c}]" for c in columns)
|
|
placeholders = ", ".join("?" for _ in columns)
|
|
has_identity = self._has_identity(schema, table)
|
|
cur = self._conn.cursor()
|
|
if has_identity:
|
|
cur.execute(f"SET IDENTITY_INSERT {full} ON")
|
|
try:
|
|
inserted = 0
|
|
fast = True
|
|
for i in range(0, len(rows), 1000):
|
|
chunk = rows[i:i + 1000]
|
|
for attempt in range(2):
|
|
try:
|
|
cur.fast_executemany = fast
|
|
cur.executemany(
|
|
f"INSERT INTO {full} ({col_list}) VALUES ({placeholders})",
|
|
chunk,
|
|
)
|
|
break
|
|
except pyodbc.Error:
|
|
if attempt == 0 and fast:
|
|
fast = False # retry this chunk without fast_executemany
|
|
continue
|
|
raise
|
|
inserted += len(chunk)
|
|
return inserted
|
|
finally:
|
|
if has_identity:
|
|
cur.execute(f"SET IDENTITY_INSERT {full} OFF")
|
|
|
|
def close(self) -> None:
|
|
"""Close the underlying pyodbc connection."""
|
|
self._conn.close()
|