♻️ refactor(sync): extract shared target-table resolution
This commit is contained in:
@@ -3,18 +3,15 @@ import json, logging
|
|||||||
from .access_reader import AccessReader
|
from .access_reader import AccessReader
|
||||||
from .sql_writer import SqlWriter, QueueRow
|
from .sql_writer import SqlWriter, QueueRow
|
||||||
from .config import FileMapping, SyncConfig
|
from .config import FileMapping, SyncConfig
|
||||||
|
from .targets import is_synced_table
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
def capture_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter, cfg: SyncConfig) -> int:
|
def capture_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter, cfg: SyncConfig) -> int:
|
||||||
exclude = set(fm.exclude_tables or [])
|
|
||||||
include = set(fm.include_tables) if fm.include_tables else None
|
|
||||||
rows = reader.read_log(cfg.runtime.capture_batch_size)
|
rows = reader.read_log(cfg.runtime.capture_batch_size)
|
||||||
n = 0
|
n = 0
|
||||||
for lr in rows:
|
for lr in rows:
|
||||||
if lr.table_name in exclude:
|
if not is_synced_table(fm, lr.table_name):
|
||||||
continue
|
|
||||||
if include is not None and lr.table_name not in include:
|
|
||||||
continue
|
continue
|
||||||
op = lr.operate_type
|
op = lr.operate_type
|
||||||
row_data = None
|
row_data = None
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from .config import load_config, FileMapping, SyncConfig
|
|||||||
from .access_reader import AccessReader
|
from .access_reader import AccessReader
|
||||||
from .sql_writer import SqlWriter
|
from .sql_writer import SqlWriter
|
||||||
from .logging_setup import setup_logging
|
from .logging_setup import setup_logging
|
||||||
|
from .targets import resolve_synced_tables
|
||||||
|
|
||||||
log = logging.getLogger("sync.fullsync")
|
log = logging.getLogger("sync.fullsync")
|
||||||
|
|
||||||
@@ -29,21 +30,11 @@ log = logging.getLogger("sync.fullsync")
|
|||||||
def resolve_tables(reader: AccessReader, fm: FileMapping) -> list[str]:
|
def resolve_tables(reader: AccessReader, fm: FileMapping) -> list[str]:
|
||||||
"""Tables to fully sync for one file, after exclude/include rules.
|
"""Tables to fully sync for one file, after exclude/include rules.
|
||||||
|
|
||||||
Mirrors the precedence used by ``capture.capture_file``: a table in
|
Thin wrapper over ``targets.resolve_synced_tables`` so fullsync, capture
|
||||||
``exclude_tables`` is dropped even if it also appears in ``include_tables``.
|
and compare share one resolution path. System tables (``MSys*`` / ``~*``)
|
||||||
System tables (``MSys*`` / ``~*``) are already filtered by
|
are already filtered by ``AccessReader.list_user_tables``.
|
||||||
``AccessReader.list_user_tables``.
|
|
||||||
"""
|
"""
|
||||||
exclude = set(fm.exclude_tables or [])
|
return resolve_synced_tables(fm, reader)
|
||||||
include = set(fm.include_tables) if fm.include_tables else None
|
|
||||||
out = []
|
|
||||||
for t in reader.list_user_tables():
|
|
||||||
if t in exclude:
|
|
||||||
continue
|
|
||||||
if include is not None and t not in include:
|
|
||||||
continue
|
|
||||||
out.append(t)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def full_sync_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter) -> dict:
|
def full_sync_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter) -> dict:
|
||||||
|
|||||||
37
src/sync/targets.py
Normal file
37
src/sync/targets.py
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
"""Shared target-table resolution for fullsync, incremental capture, and compare.
|
||||||
|
|
||||||
|
All three pipelines must agree on which Access tables are in scope and how each
|
||||||
|
maps to its SQL Server target. Centralising the exclude/include rule here makes
|
||||||
|
that guarantee structural instead of duplicated across three files.
|
||||||
|
|
||||||
|
- ``is_synced_table`` applies the per-file exclude/include rule (exclude wins).
|
||||||
|
- ``resolve_synced_tables`` returns the in-scope user tables in
|
||||||
|
``list_user_tables`` order; system tables (``MSys*`` / ``~*``) are already
|
||||||
|
filtered by ``AccessReader.list_user_tables``.
|
||||||
|
|
||||||
|
Target *naming* is shared via ``FileMapping.target_table`` (name + year_suffix)
|
||||||
|
and ``FileMapping.schema``, so a given Access table resolves to the same
|
||||||
|
``(schema, table)`` everywhere.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from .config import FileMapping
|
||||||
|
|
||||||
|
|
||||||
|
def is_synced_table(fm: FileMapping, table_name: str) -> bool:
|
||||||
|
"""True if ``table_name`` is in sync scope for ``fm``.
|
||||||
|
|
||||||
|
``exclude_tables`` wins over ``include_tables``: a table listed in both is
|
||||||
|
excluded. When ``include_tables`` is None, every non-excluded table is in
|
||||||
|
scope.
|
||||||
|
"""
|
||||||
|
if table_name in (fm.exclude_tables or []):
|
||||||
|
return False
|
||||||
|
if fm.include_tables is not None:
|
||||||
|
return table_name in fm.include_tables
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_synced_tables(fm: FileMapping, reader) -> list[str]:
|
||||||
|
"""In-scope user tables for ``fm``, in ``list_user_tables`` order."""
|
||||||
|
return [t for t in reader.list_user_tables() if is_synced_table(fm, t)]
|
||||||
53
tests/test_targets.py
Normal file
53
tests/test_targets.py
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from sync.config import FileMapping
|
||||||
|
from sync.targets import is_synced_table, resolve_synced_tables
|
||||||
|
|
||||||
|
|
||||||
|
def _fm(**kw):
|
||||||
|
base = dict(file="x.accdb", root="2026", schema="s", year_suffix="_YEAR2026")
|
||||||
|
base.update(kw)
|
||||||
|
return FileMapping(**base)
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_synced_table_excludes_listed():
|
||||||
|
fm = _fm(exclude_tables=["TableChangeLog", "一车间每日催货落实记录_停"])
|
||||||
|
assert is_synced_table(fm, "TableChangeLog") is False
|
||||||
|
assert is_synced_table(fm, "一车间每日催货落实记录_停") is False
|
||||||
|
assert is_synced_table(fm, "表壳焊接记录") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_synced_table_include_restricts():
|
||||||
|
fm = _fm(exclude_tables=["TableChangeLog"], include_tables=["检验合格记录表"])
|
||||||
|
assert is_synced_table(fm, "检验合格记录表") is True
|
||||||
|
assert is_synced_table(fm, "其它表") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_synced_table_exclude_beats_include():
|
||||||
|
fm = _fm(exclude_tables=["TableChangeLog"],
|
||||||
|
include_tables=["TableChangeLog", "检验合格记录表"])
|
||||||
|
assert is_synced_table(fm, "TableChangeLog") is False
|
||||||
|
assert is_synced_table(fm, "检验合格记录表") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_synced_table_no_filters_includes_all():
|
||||||
|
fm = _fm()
|
||||||
|
assert is_synced_table(fm, "任意表") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_synced_tables_filters_user_tables():
|
||||||
|
fm = _fm(exclude_tables=["TableChangeLog"])
|
||||||
|
reader = MagicMock()
|
||||||
|
reader.list_user_tables.return_value = [
|
||||||
|
"TableChangeLog", "表壳焊接记录", "超压", "氩弧焊每日催货落实记录_停",
|
||||||
|
]
|
||||||
|
assert resolve_synced_tables(fm, reader) == [
|
||||||
|
"表壳焊接记录", "超压", "氩弧焊每日催货落实记录_停",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_synced_tables_with_include():
|
||||||
|
fm = _fm(exclude_tables=["TableChangeLog"], include_tables=["检验合格记录表"])
|
||||||
|
reader = MagicMock()
|
||||||
|
reader.list_user_tables.return_value = ["TableChangeLog", "检验合格记录表", "其它表"]
|
||||||
|
assert resolve_synced_tables(fm, reader) == ["检验合格记录表"]
|
||||||
Reference in New Issue
Block a user