feat: sql writer with dedup insert and apply call
Adds SqlWriter: a pyodbc-backed writer that dedup-inserts into dbo.SyncQueue (IF NOT EXISTS guarded by UX_SyncQueue_Dedup), invokes dbo.usp_SyncApply, and reports applied SourceLogIDs. Connection is opened with autocommit=True per the controller revision: usp_SyncApply manages its own transaction internally (BEGIN/ROLLBACK), and an outer pyodbc transaction would conflict on ROLLBACK (SQL error 266). The dedup IF NOT EXISTS...INSERT is a single atomic statement. Integration test self-cleans via SourceFile='sqlw_test.accdb' marker; conn_str comes from the gitignored config.yaml (no hardcoded creds). Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
95
src/sync/sql_writer.py
Normal file
95
src/sync/sql_writer.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""SQL-side writer for the Access -> SQL Server sync.
|
||||
|
||||
SqlWriter owns the pyodbc connection used to (a) dedup-insert rows into
|
||||
``dbo.SyncQueue``, (b) invoke ``dbo.usp_SyncApply`` to drain the queue, and
|
||||
(c) report which SourceLogIDs have been applied.
|
||||
|
||||
The connection is opened with ``autocommit=True`` on purpose. ``usp_SyncApply``
|
||||
manages its own transaction internally (BEGIN TRAN ... ROLLBACK on error); if
|
||||
the caller held an outer implicit transaction, the proc's ROLLBACK would
|
||||
cascade and raise SQL error 266. The dedup ``IF NOT EXISTS ... INSERT`` is a
|
||||
single statement that is atomic under autocommit, so no explicit transaction is
|
||||
needed on the write path either.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pyodbc
|
||||
|
||||
|
||||
@dataclass
|
||||
class QueueRow:
|
||||
"""A single staged change to enqueue into ``dbo.SyncQueue``."""
|
||||
|
||||
source_file: str
|
||||
source_table: str
|
||||
record_id: str
|
||||
target_schema: str
|
||||
target_table: str
|
||||
source_log_id: int
|
||||
operate_type: str
|
||||
row_data: str | None
|
||||
|
||||
|
||||
class SqlWriter:
|
||||
"""Writes to SyncQueue and drives the apply proc over a pyodbc connection."""
|
||||
|
||||
def __init__(self, conn_str: str, queue_table: str = "dbo.SyncQueue"):
|
||||
self.conn_str = conn_str
|
||||
self.queue_table = queue_table
|
||||
# autocommit=True: usp_SyncApply manages its own transaction internally.
|
||||
# An outer pyodbc transaction would conflict on ROLLBACK (SQL error 266).
|
||||
self._conn = pyodbc.connect(conn_str, autocommit=True)
|
||||
|
||||
def insert_queue_row(self, row: QueueRow) -> None:
|
||||
"""Idempotently enqueue ``row`` (dedup on SourceFile/Table/LogID).
|
||||
|
||||
``IF NOT EXISTS ... INSERT`` is a single statement, atomic under
|
||||
autocommit. The unique index UX_SyncQueue_Dedup is the DB backstop.
|
||||
"""
|
||||
# IF NOT EXISTS and the VALUES list each carry their own ? markers;
|
||||
# pyodbc binds them positionally, so the 3 dedup keys are supplied
|
||||
# twice (once for the EXISTS check, once for the INSERT).
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
"IF NOT EXISTS (SELECT 1 FROM dbo.SyncQueue "
|
||||
"WHERE SourceFile=? AND SourceTable=? AND SourceLogID=?) "
|
||||
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,"
|
||||
"TargetSchema,TargetTable,RecordID,OperateType,RowData,Status) "
|
||||
"VALUES (?,?,?,?,?,?,?,?, 'pending')",
|
||||
row.source_file,
|
||||
row.source_table,
|
||||
row.source_log_id,
|
||||
row.source_file,
|
||||
row.source_table,
|
||||
row.source_log_id,
|
||||
row.target_schema,
|
||||
row.target_table,
|
||||
row.record_id,
|
||||
row.operate_type,
|
||||
row.row_data,
|
||||
)
|
||||
# autocommit: statement already committed.
|
||||
|
||||
def call_apply(self, max_retries: int) -> None:
|
||||
"""Drain the pending queue via the stored procedure.
|
||||
|
||||
``usp_SyncApply`` flips rows to ``applied`` (or ``error`` after retries).
|
||||
"""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute("EXEC dbo.usp_SyncApply ?", max_retries)
|
||||
|
||||
def applied_log_ids(self, source_file: str) -> list[int]:
|
||||
"""Return applied SourceLogIDs for ``source_file`` in ascending order."""
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT SourceLogID FROM dbo.SyncQueue "
|
||||
"WHERE SourceFile=? AND Status='applied' ORDER BY SourceLogID",
|
||||
source_file,
|
||||
)
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the underlying pyodbc connection."""
|
||||
self._conn.close()
|
||||
Reference in New Issue
Block a user