"""SQL-side writer for the Access -> SQL Server sync. SqlWriter owns the pyodbc connection used to (a) dedup-insert rows into the staging queue (``ProductionDataBaseSync.SyncQueue`` by default), (b) invoke the apply proc (``ProductionDataBaseSync.usp_SyncApply``) to drain the queue, (c) report which SourceLogIDs have been applied, and (d) append every consumed Access change-log row to the permanent audit store (``ProductionDataBaseSync.SyncLogArchive``). The queue/archive/proc names are injected from config so the whole sync can live under a dedicated schema. The connection is opened with ``autocommit=True`` on purpose. ``usp_SyncApply`` manages its own transaction internally (BEGIN TRAN ... ROLLBACK on error); if the caller held an outer implicit transaction, the proc's ROLLBACK would cascade and raise SQL error 266. The dedup ``IF NOT EXISTS ... INSERT`` is a single statement that is atomic under autocommit, so no explicit transaction is needed on the write path either. """ from __future__ import annotations from dataclasses import dataclass import pyodbc @dataclass class QueueRow: """A single staged change to enqueue into the SyncQueue table.""" source_file: str source_table: str record_id: str target_schema: str target_table: str source_log_id: int operate_type: str row_data: str | None @dataclass class ArchiveRow: """A single Access change-log row to append to ``SyncLogArchive``. Captures both the ORIGINAL operate type recorded by the Access data macro and the PROCESSED operate type actually sent to the queue, so a downgrade (e.g. Insert -> Delete when the row is momentarily unreadable) stays visible forever. ``original_time`` preserves the Access log's own timestamp, which the queue path discards. """ source_file: str source_table: str source_log_id: int record_id: str target_schema: str target_table: str original_operate_type: str processed_operate_type: str row_data: str | None original_time: object class SqlWriter: """Writes to SyncQueue/SyncLogArchive and drives the apply proc.""" def __init__( self, conn_str: str, queue_table: str = "ProductionDataBaseSync.SyncQueue", archive_table: str = "ProductionDataBaseSync.SyncLogArchive", apply_proc: str = "ProductionDataBaseSync.usp_SyncApply", ): self.conn_str = conn_str self.queue_table = queue_table self.archive_table = archive_table self.apply_proc = apply_proc # autocommit=True: usp_SyncApply manages its own transaction internally. # An outer pyodbc transaction would conflict on ROLLBACK (SQL error 266). self._conn = pyodbc.connect(conn_str, autocommit=True) def insert_queue_row(self, row: QueueRow) -> None: """Idempotently enqueue ``row`` (dedup on SourceFile/Table/LogID). ``IF NOT EXISTS ... INSERT`` is a single statement, atomic under autocommit. The unique index UX_SyncQueue_Dedup is the DB backstop. """ # IF NOT EXISTS and the VALUES list each carry their own ? markers; # pyodbc binds them positionally, so the 3 dedup keys are supplied # twice (once for the EXISTS check, once for the INSERT). cur = self._conn.cursor() cur.execute( f"IF NOT EXISTS (SELECT 1 FROM {self.queue_table} " "WHERE SourceFile=? AND SourceTable=? AND SourceLogID=?) " f"INSERT {self.queue_table}(SourceFile,SourceTable,SourceLogID," "TargetSchema,TargetTable,RecordID,OperateType,RowData,Status) " "VALUES (?,?,?,?,?,?,?,?, 'pending')", row.source_file, row.source_table, row.source_log_id, row.source_file, row.source_table, row.source_log_id, row.target_schema, row.target_table, row.record_id, row.operate_type, row.row_data, ) # autocommit: statement already committed. def insert_archive_row(self, row: ArchiveRow) -> None: """Append ``row`` to the permanent audit store (dedup on source keys). Written during capture, BEFORE cleanup deletes the Access log, so the original evidence survives even after the queue row is purged. Stores both the original and processed operate types plus the Access log time. ``IF NOT EXISTS`` keeps the first archive record if capture re-runs the same log id (a prior cycle's apply failed and the Access log persisted). """ cur = self._conn.cursor() cur.execute( f"IF NOT EXISTS (SELECT 1 FROM {self.archive_table} " "WHERE SourceFile=? AND SourceTable=? AND SourceLogID=?) " f"INSERT {self.archive_table}(SourceFile,SourceTable,SourceLogID," "RecordID,TargetSchema,TargetTable,OriginalOperateType," "ProcessedOperateType,RowData,OriginalTime) " "VALUES (?,?,?,?,?,?,?,?,?,?)", row.source_file, row.source_table, row.source_log_id, row.source_file, row.source_table, row.source_log_id, row.record_id, row.target_schema, row.target_table, row.original_operate_type, row.processed_operate_type, row.row_data, row.original_time, ) # autocommit: statement already committed. def call_apply(self, max_retries: int) -> None: """Drain the pending queue via the stored procedure. ``usp_SyncApply`` flips rows to ``applied`` (or ``error`` after retries). """ cur = self._conn.cursor() cur.execute(f"EXEC {self.apply_proc} ?", max_retries) def applied_log_ids(self, source_file: str) -> list[int]: """Return applied SourceLogIDs for ``source_file`` in ascending order.""" cur = self._conn.cursor() cur.execute( f"SELECT SourceLogID FROM {self.queue_table} " "WHERE SourceFile=? AND Status='applied' ORDER BY SourceLogID", source_file, ) return [r[0] for r in cur.fetchall()] def mark_cleaned(self, source_file: str, source_log_ids: list[int]) -> None: """Mark applied queue rows for ``source_file`` as ``cleaned``. Called after the corresponding Access log rows have been physically deleted. Once a row is ``cleaned``, ``applied_log_ids`` no longer returns it, so the same IDs are never deleted from Access twice. Rows are matched on ``(SourceFile, SourceLogID)`` and constrained to ``Status='applied'``, so a row that errored out is never silently marked clean. Chunked to stay under SQL Server's 2100-param limit. """ if not source_log_ids: return cur = self._conn.cursor() for i in range(0, len(source_log_ids), 1000): chunk = source_log_ids[i:i + 1000] placeholders = ",".join("?" * len(chunk)) cur.execute( f"UPDATE {self.queue_table} SET Status='cleaned', " f"CleanedAt=sysdatetime() " f"WHERE SourceFile=? AND Status='applied' " f"AND SourceLogID IN ({placeholders})", source_file, *chunk, ) def purge_cleaned(self, retention_hours: int) -> int: """Delete ``cleaned`` rows older than ``retention_hours``. Keeps the table bounded: cleanup marks rows ``cleaned`` every cycle, and this removes the old ones after a short audit/debug window so the queue table stops growing without bound. Returns the number of rows removed. """ cur = self._conn.cursor() cur.execute( f"DELETE FROM {self.queue_table} " "WHERE Status='cleaned' " "AND CleanedAt < DATEADD(hour, -?, GETDATE())", retention_hours, ) return cur.rowcount def table_exists(self, schema: str, table: str) -> bool: """True if ``[schema].[table]`` exists in the target database.""" cur = self._conn.cursor() cur.execute( "SELECT 1 FROM sys.tables t JOIN sys.schemas s " "ON s.schema_id = t.schema_id " "WHERE s.name = ? AND t.name = ?", schema, table, ) return cur.fetchone() is not None def count_target(self, schema: str, table: str) -> int: """Return ``COUNT(*)`` for ``[schema].[table]`` (compare count check).""" cur = self._conn.cursor() cur.execute(f"SELECT COUNT(*) FROM [{schema}].[{table}]") return cur.fetchone()[0] def read_target_ids(self, schema: str, table: str) -> list: """Return every ``ID`` from ``[schema].[table]``, ascending (compare IDs).""" cur = self._conn.cursor() cur.execute(f"SELECT ID FROM [{schema}].[{table}] ORDER BY ID") return [r[0] for r in cur.fetchall()] def _has_identity(self, schema: str, table: str) -> bool: """True if ``[schema].[table]`` has an IDENTITY column (the ``ID`` PK).""" cur = self._conn.cursor() cur.execute( "SELECT 1 FROM sys.tables t JOIN sys.schemas s " "ON s.schema_id = t.schema_id JOIN sys.columns c " "ON c.object_id = t.object_id " "WHERE s.name = ? AND t.name = ? AND c.is_identity = 1", schema, table, ) return cur.fetchone() is not None def truncate_target(self, schema: str, table: str) -> None: """Empty ``[schema].[table]`` prior to a full reload. Prefers ``TRUNCATE TABLE`` (fast, minimal logging). This project's target schemas have no foreign keys, so TRUNCATE always succeeds; the ``DELETE FROM`` fallback only matters if a future FK is added or the table sits under snapshot isolation. """ full = f"[{schema}].[{table}]" cur = self._conn.cursor() try: cur.execute(f"TRUNCATE TABLE {full}") except pyodbc.Error: cur.execute(f"DELETE FROM {full}") def bulk_insert( self, schema: str, table: str, columns: list[str], rows: list[tuple] ) -> int: """Insert every ``row`` into ``[schema].[table]``. ``columns`` and the tuples in ``rows`` must be positionally aligned. When the target has an IDENTITY column (the ``ID`` PK in every target here), ``SET IDENTITY_INSERT`` is enabled so the original Access primary keys are preserved — required for the incremental sync's RecordID matching to keep working afterwards. Rows are inserted in 1000-row chunks. ``fast_executemany`` is tried first for speed; if a row's types can't be inferred (e.g. a leading NULL), it transparently retries the chunk without it. Returns the total number of rows inserted. """ if not columns or not rows: return 0 full = f"[{schema}].[{table}]" col_list = ", ".join(f"[{c}]" for c in columns) placeholders = ", ".join("?" for _ in columns) has_identity = self._has_identity(schema, table) cur = self._conn.cursor() if has_identity: cur.execute(f"SET IDENTITY_INSERT {full} ON") try: inserted = 0 fast = True for i in range(0, len(rows), 1000): chunk = rows[i:i + 1000] for attempt in range(2): try: cur.fast_executemany = fast cur.executemany( f"INSERT INTO {full} ({col_list}) VALUES ({placeholders})", chunk, ) break except pyodbc.Error: if attempt == 0 and fast: fast = False # retry this chunk without fast_executemany continue raise inserted += len(chunk) return inserted finally: if has_identity: cur.execute(f"SET IDENTITY_INSERT {full} OFF") def close(self) -> None: """Close the underlying pyodbc connection.""" self._conn.close()