feat: access reader and value serialization
Add to_jsonable() (Access/pyodbc value -> JSON-safe) with strict TDD (5 tests: datetime iso, bool preserved before int, decimal->str, none/numbers, dict round-trip). bool branch precedes int because bool is a subclass of int in Python. Add AccessReader: reads TableChangeLog (read_log), reads a full source row by RecordID via to_jsonable (read_row, warns on >4000-char values that JSON_VALUE would truncate), and deletes processed log IDs in chunked, retried batches (delete_log_ids, no-op on empty list). Connects via the ACE ODBC driver from config.access.driver, shared autocommit mode to coexist with the live .laccdb client. Integration test against the real 氩弧焊.accdb (read-only; delete_log_ids only exercised with []) passes. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
119
src/sync/access_reader.py
Normal file
119
src/sync/access_reader.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""Read Access change-log rows and full source rows via the ACE ODBC driver.
|
||||
|
||||
Opened in shared/autocommit mode so it coexists with the live client that holds
|
||||
the ``.laccdb`` lock on the production ``.accdb`` files. The driver name comes
|
||||
from ``config.access.driver`` (no secret in the connection string — Windows
|
||||
auth / no password on the Access share).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pyodbc
|
||||
|
||||
from .serialize import to_jsonable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LogRow:
|
||||
"""A single row from the Access ``TableChangeLog`` table."""
|
||||
id: int
|
||||
table_name: str
|
||||
record_id: str
|
||||
operate_type: str # "Insert" | "Update" | "Delete"
|
||||
time: object # raw Access datetime (serialized later by caller)
|
||||
|
||||
|
||||
class AccessReader:
|
||||
"""Thin pyodbc wrapper around one ``.accdb`` file.
|
||||
|
||||
The connection is lazily created and reused across calls. ``close()`` must
|
||||
be called when the reader is no longer needed.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str, driver: str):
|
||||
self.db_path = db_path
|
||||
self.driver = driver
|
||||
self._conn = None
|
||||
|
||||
def _connect(self):
|
||||
if self._conn is None:
|
||||
# ReadOnly=0: shared read/write open. The capture path is read-only,
|
||||
# but the cleanup path (delete_log_ids) genuinely deletes processed
|
||||
# log rows. autocommit=True avoids a held transaction that would
|
||||
# block the live client.
|
||||
conn_str = f"Driver={self.driver};DBQ={self.db_path};ReadOnly=0;"
|
||||
self._conn = pyodbc.connect(conn_str, autocommit=True)
|
||||
return self._conn
|
||||
|
||||
def read_log(self, batch_size: int) -> list[LogRow]:
|
||||
"""Return up to ``batch_size`` oldest log rows ordered by ID."""
|
||||
cur = self._connect().cursor()
|
||||
cur.execute(
|
||||
f"SELECT TOP {int(batch_size)} ID, TableName, RecordID, "
|
||||
f"OperateType, Time FROM TableChangeLog ORDER BY ID"
|
||||
)
|
||||
return [
|
||||
LogRow(r[0], r[1], str(r[2]), (r[3] or "").strip(), r[4])
|
||||
for r in cur.fetchall()
|
||||
]
|
||||
|
||||
def read_row(self, table: str, record_id: str) -> dict | None:
|
||||
"""Return the full source row for ``record_id`` as a JSONable dict.
|
||||
|
||||
Returns ``None`` if the row no longer exists. Emits a warning when any
|
||||
string value exceeds 4000 chars (the downstream ``JSON_VALUE`` on SQL
|
||||
Server ``NVARCHAR(MAX)`` -> ``NVARCHAR(4000)`` would silently truncate).
|
||||
"""
|
||||
cur = self._connect().cursor()
|
||||
cur.execute(f'SELECT * FROM "{table}" WHERE ID = ?', record_id)
|
||||
cols = [c[0] for c in cur.description]
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
d = {cols[i]: to_jsonable(row[i]) for i in range(len(cols))}
|
||||
for k, v in d.items():
|
||||
if isinstance(v, str) and len(v) > 4000:
|
||||
log.warning(
|
||||
"value >4000 chars in %s.ID=%s col=%s "
|
||||
"(JSON_VALUE will truncate)",
|
||||
table, record_id, k,
|
||||
)
|
||||
return d
|
||||
|
||||
def delete_log_ids(
|
||||
self, ids: list[int], batch_size: int, retries: int
|
||||
) -> None:
|
||||
"""Delete the given log-row IDs in chunks, retrying on lock contention.
|
||||
|
||||
A no-op when ``ids`` is empty (never raises). Retries with linear
|
||||
backoff because the live client may briefly hold a page lock on
|
||||
``TableChangeLog``.
|
||||
"""
|
||||
if not ids:
|
||||
return
|
||||
cur = self._connect().cursor()
|
||||
for i in range(0, len(ids), batch_size):
|
||||
chunk = ids[i:i + batch_size]
|
||||
placeholders = ",".join("?" * len(chunk))
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
cur.execute(
|
||||
f"DELETE FROM TableChangeLog WHERE ID IN ({placeholders})",
|
||||
*chunk,
|
||||
)
|
||||
break
|
||||
except pyodbc.OperationalError:
|
||||
if attempt < retries - 1:
|
||||
time.sleep(0.2 * (attempt + 1))
|
||||
else:
|
||||
raise
|
||||
|
||||
def close(self):
|
||||
if self._conn:
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
34
src/sync/serialize.py
Normal file
34
src/sync/serialize.py
Normal file
@@ -0,0 +1,34 @@
|
||||
"""Access value serialization helpers.
|
||||
|
||||
Converts pyodbc/Access cell values into JSON-serializable Python types so a
|
||||
captured row dict can be ``json.dumps``-ed before being enqueued on SQL Server.
|
||||
|
||||
Order matters: ``bool`` MUST be checked before ``int`` because ``bool`` is a
|
||||
subclass of ``int`` in Python (``isinstance(True, int) is True``). If the order
|
||||
were reversed, ``True`` would be returned as ``1`` and lose its type.
|
||||
"""
|
||||
import datetime
|
||||
import decimal
|
||||
|
||||
|
||||
def to_jsonable(v):
|
||||
"""Coerce a single Access/pyodbc value to a JSON-serializable form.
|
||||
|
||||
- ``None`` -> ``None``
|
||||
- ``bool`` -> ``bool`` (must precede the ``int`` branch)
|
||||
- ``datetime``/``date`` -> ISO-8601 string (``"2026-07-14T08:41:18"``)
|
||||
- ``Decimal`` -> string (preserves exact scale, e.g. ``"12.50"``)
|
||||
- ``int``/``float``/``str`` -> unchanged
|
||||
- anything else (e.g. ``bytes`` for OLE) -> ``str(v)`` fallback
|
||||
"""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, bool): # MUST be before int
|
||||
return v
|
||||
if isinstance(v, (datetime.datetime, datetime.date)):
|
||||
return v.isoformat()
|
||||
if isinstance(v, decimal.Decimal):
|
||||
return str(v)
|
||||
if isinstance(v, (int, float, str)):
|
||||
return v
|
||||
return str(v)
|
||||
37
tests/test_access_reader.py
Normal file
37
tests/test_access_reader.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""Integration test for AccessReader against the real 氩弧焊.accdb.
|
||||
|
||||
READ-ONLY: this test never mutates production data. ``delete_log_ids`` is only
|
||||
exercised with an empty list (a guaranteed no-op) to confirm it does not raise.
|
||||
No real log rows are deleted.
|
||||
"""
|
||||
import os
|
||||
import pytest
|
||||
|
||||
from sync.access_reader import AccessReader
|
||||
from sync.config import load_config
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_read_log_and_row_and_delete():
|
||||
if not os.environ.get("RUN_INTEGRATION"):
|
||||
pytest.skip("needs RUN_INTEGRATION=1")
|
||||
cfg = load_config("config.yaml")
|
||||
# Real live database (has a .laccdb lock from the client). Shared-mode
|
||||
# read via ACE is validated to coexist with the lock.
|
||||
db = os.environ.get(
|
||||
"TEST_ACCDB",
|
||||
r"\\192.168.110.114\生产进度表\2026年数据\氩弧焊.accdb",
|
||||
)
|
||||
r = AccessReader(db, cfg.access.driver)
|
||||
try:
|
||||
rows = r.read_log(500)
|
||||
assert isinstance(rows, list)
|
||||
if rows:
|
||||
lr = rows[0]
|
||||
assert lr.table_name and lr.record_id
|
||||
assert lr.operate_type in ("Insert", "Update", "Delete")
|
||||
d = r.read_row(lr.table_name, lr.record_id)
|
||||
assert d is None or "ID" in d
|
||||
r.delete_log_ids([], 100, 3) # empty list -> no-op, must not raise
|
||||
finally:
|
||||
r.close()
|
||||
21
tests/test_serialize.py
Normal file
21
tests/test_serialize.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import datetime, decimal
|
||||
from sync.serialize import to_jsonable
|
||||
import json
|
||||
|
||||
def test_datetime_iso():
|
||||
assert to_jsonable(datetime.datetime(2026,7,14,8,41,18)) == "2026-07-14T08:41:18"
|
||||
|
||||
def test_bool_preserved():
|
||||
assert to_jsonable(True) is True and to_jsonable(False) is False
|
||||
|
||||
def test_decimal_to_str():
|
||||
assert to_jsonable(decimal.Decimal("12.50")) == "12.50"
|
||||
|
||||
def test_none_and_numbers():
|
||||
assert to_jsonable(None) is None
|
||||
assert to_jsonable(5) == 5
|
||||
assert to_jsonable("x") == "x"
|
||||
|
||||
def test_dict_serializes():
|
||||
d = {"d": datetime.date(2026,1,1), "b": True, "n": None}
|
||||
assert json.loads(json.dumps({k: to_jsonable(v) for k,v in d.items()})) == {"d":"2026-01-01","b":True,"n":None}
|
||||
Reference in New Issue
Block a user