feat: access reader and value serialization

Add to_jsonable() (Access/pyodbc value -> JSON-safe) with strict TDD
(5 tests: datetime iso, bool preserved before int, decimal->str,
none/numbers, dict round-trip). bool branch precedes int because bool
is a subclass of int in Python.

Add AccessReader: reads TableChangeLog (read_log), reads a full source
row by RecordID via to_jsonable (read_row, warns on >4000-char values
that JSON_VALUE would truncate), and deletes processed log IDs in
chunked, retried batches (delete_log_ids, no-op on empty list).
Connects via the ACE ODBC driver from config.access.driver, shared
autocommit mode to coexist with the live .laccdb client.

Integration test against the real 氩弧焊.accdb (read-only; delete_log_ids
only exercised with []) passes.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-07-14 12:22:48 +08:00
parent 65939f5e85
commit 1f7191421a
4 changed files with 211 additions and 0 deletions

119
src/sync/access_reader.py Normal file
View File

@@ -0,0 +1,119 @@
"""Read Access change-log rows and full source rows via the ACE ODBC driver.
Opened in shared/autocommit mode so it coexists with the live client that holds
the ``.laccdb`` lock on the production ``.accdb`` files. The driver name comes
from ``config.access.driver`` (no secret in the connection string — Windows
auth / no password on the Access share).
"""
from __future__ import annotations
import logging
import time
from dataclasses import dataclass
import pyodbc
from .serialize import to_jsonable
log = logging.getLogger(__name__)
@dataclass
class LogRow:
"""A single row from the Access ``TableChangeLog`` table."""
id: int
table_name: str
record_id: str
operate_type: str # "Insert" | "Update" | "Delete"
time: object # raw Access datetime (serialized later by caller)
class AccessReader:
"""Thin pyodbc wrapper around one ``.accdb`` file.
The connection is lazily created and reused across calls. ``close()`` must
be called when the reader is no longer needed.
"""
def __init__(self, db_path: str, driver: str):
self.db_path = db_path
self.driver = driver
self._conn = None
def _connect(self):
if self._conn is None:
# ReadOnly=0: shared read/write open. The capture path is read-only,
# but the cleanup path (delete_log_ids) genuinely deletes processed
# log rows. autocommit=True avoids a held transaction that would
# block the live client.
conn_str = f"Driver={self.driver};DBQ={self.db_path};ReadOnly=0;"
self._conn = pyodbc.connect(conn_str, autocommit=True)
return self._conn
def read_log(self, batch_size: int) -> list[LogRow]:
"""Return up to ``batch_size`` oldest log rows ordered by ID."""
cur = self._connect().cursor()
cur.execute(
f"SELECT TOP {int(batch_size)} ID, TableName, RecordID, "
f"OperateType, Time FROM TableChangeLog ORDER BY ID"
)
return [
LogRow(r[0], r[1], str(r[2]), (r[3] or "").strip(), r[4])
for r in cur.fetchall()
]
def read_row(self, table: str, record_id: str) -> dict | None:
"""Return the full source row for ``record_id`` as a JSONable dict.
Returns ``None`` if the row no longer exists. Emits a warning when any
string value exceeds 4000 chars (the downstream ``JSON_VALUE`` on SQL
Server ``NVARCHAR(MAX)`` -> ``NVARCHAR(4000)`` would silently truncate).
"""
cur = self._connect().cursor()
cur.execute(f'SELECT * FROM "{table}" WHERE ID = ?', record_id)
cols = [c[0] for c in cur.description]
row = cur.fetchone()
if row is None:
return None
d = {cols[i]: to_jsonable(row[i]) for i in range(len(cols))}
for k, v in d.items():
if isinstance(v, str) and len(v) > 4000:
log.warning(
"value >4000 chars in %s.ID=%s col=%s "
"(JSON_VALUE will truncate)",
table, record_id, k,
)
return d
def delete_log_ids(
self, ids: list[int], batch_size: int, retries: int
) -> None:
"""Delete the given log-row IDs in chunks, retrying on lock contention.
A no-op when ``ids`` is empty (never raises). Retries with linear
backoff because the live client may briefly hold a page lock on
``TableChangeLog``.
"""
if not ids:
return
cur = self._connect().cursor()
for i in range(0, len(ids), batch_size):
chunk = ids[i:i + batch_size]
placeholders = ",".join("?" * len(chunk))
for attempt in range(retries):
try:
cur.execute(
f"DELETE FROM TableChangeLog WHERE ID IN ({placeholders})",
*chunk,
)
break
except pyodbc.OperationalError:
if attempt < retries - 1:
time.sleep(0.2 * (attempt + 1))
else:
raise
def close(self):
if self._conn:
self._conn.close()
self._conn = None

34
src/sync/serialize.py Normal file
View File

@@ -0,0 +1,34 @@
"""Access value serialization helpers.
Converts pyodbc/Access cell values into JSON-serializable Python types so a
captured row dict can be ``json.dumps``-ed before being enqueued on SQL Server.
Order matters: ``bool`` MUST be checked before ``int`` because ``bool`` is a
subclass of ``int`` in Python (``isinstance(True, int) is True``). If the order
were reversed, ``True`` would be returned as ``1`` and lose its type.
"""
import datetime
import decimal
def to_jsonable(v):
"""Coerce a single Access/pyodbc value to a JSON-serializable form.
- ``None`` -> ``None``
- ``bool`` -> ``bool`` (must precede the ``int`` branch)
- ``datetime``/``date`` -> ISO-8601 string (``"2026-07-14T08:41:18"``)
- ``Decimal`` -> string (preserves exact scale, e.g. ``"12.50"``)
- ``int``/``float``/``str`` -> unchanged
- anything else (e.g. ``bytes`` for OLE) -> ``str(v)`` fallback
"""
if v is None:
return None
if isinstance(v, bool): # MUST be before int
return v
if isinstance(v, (datetime.datetime, datetime.date)):
return v.isoformat()
if isinstance(v, decimal.Decimal):
return str(v)
if isinstance(v, (int, float, str)):
return v
return str(v)