feat(sync): add one-shot full sync (TRUNCATE + bulk INSERT)

- New `sync.fullsync` CLI: `python -m sync.fullsync config.yaml [--db F] [--table T] [--clear-change-log]`
- AccessReader: list_user_tables / read_all_rows / read_all_log_ids
- SqlWriter: table_exists / truncate_target (TRUNCATE w/ DELETE fallback) / bulk_insert (SET IDENTITY_INSERT + chunked fast_executemany)
- Reuses FileMapping exclude/include rules (exclude beats include, same as capture)
- Preserves Access IDs via IDENTITY_INSERT; target schemas have no FKs so TRUNCATE is safe
- TableChangeLog NOT cleared by default (opt-in --clear-change-log)
- tests/test_fullsync.py covers resolve_tables exclude/include precedence
This commit is contained in:
Misaka_Company
2026-07-14 18:06:39 +08:00
parent ea48de290f
commit a227aed0cb
4 changed files with 339 additions and 0 deletions

View File

@@ -118,6 +118,43 @@ class AccessReader:
raise
return total
def list_user_tables(self) -> list[str]:
"""Return user table names in this Access DB.
Excludes system tables (``MSys*``) and temporary ``~*`` tables. The
change-log table ``TableChangeLog`` is NOT auto-excluded here — callers
apply their own ``FileMapping.exclude_tables`` / ``include_tables`` rules
on top of this list (see ``sync.fullsync.resolve_tables``).
"""
cur = self._connect().cursor()
names = []
for r in cur.tables(tableType="TABLE"):
n = r.table_name
if n.startswith("MSys") or n.startswith("~"):
continue
names.append(n)
return names
def read_all_rows(self, table: str):
"""Return ``(columns, rows)`` for every row in ``table``.
``columns`` is the ordered list of column names; ``rows`` is a list of
tuples of **raw** pyodbc cell values (native Python types preserved) so
the downstream bulk insert keeps correct SQL Server types. An empty
table yields ``([...], [])`` with the real column list.
"""
cur = self._connect().cursor()
cur.execute(f'SELECT * FROM "{table}"')
cols = [c[0] for c in cur.description]
rows = [tuple(row) for row in cur.fetchall()]
return cols, rows
def read_all_log_ids(self) -> list[int]:
"""Return every ``ID`` from this file's ``TableChangeLog``, ascending."""
cur = self._connect().cursor()
cur.execute("SELECT ID FROM TableChangeLog ORDER BY ID")
return [r[0] for r in cur.fetchall()]
def close(self):
if self._conn:
self._conn.close()