151 lines
5.8 KiB
Python
151 lines
5.8 KiB
Python
"""One-shot full sync: TRUNCATE each target table, then bulk-insert every row
|
|
from the corresponding Access table.
|
|
|
|
This is a manual, out-of-band operation — ``python -m sync.fullsync
|
|
config.yaml [--db FILE] [--table NAME] [--clear-change-log]``. It deliberately
|
|
bypasses the incremental ``SyncQueue`` pipeline and writes straight to the SQL
|
|
Server mirror tables. Use it to (re)seed a schema from scratch or to repair
|
|
drift between Access and SQL Server.
|
|
|
|
Contract with the target tables (same as ``usp_SyncApply``): they already
|
|
exist and mirror the Access schema column-for-column, with ``ID`` as an
|
|
IDENTITY primary key. Because ``ID`` is an identity, ``SET IDENTITY_INSERT`` is
|
|
enabled during insert so the original Access primary keys survive — that is
|
|
what keeps the incremental sync's RecordID matching correct afterwards.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import logging
|
|
|
|
from .config import load_config, FileMapping, SyncConfig
|
|
from .access_reader import AccessReader
|
|
from .sql_writer import SqlWriter
|
|
from .logging_setup import setup_logging
|
|
from .targets import resolve_synced_tables
|
|
|
|
log = logging.getLogger("sync.fullsync")
|
|
|
|
|
|
def resolve_tables(reader: AccessReader, fm: FileMapping) -> list[str]:
|
|
"""Tables to fully sync for one file, after exclude/include rules.
|
|
|
|
Thin wrapper over ``targets.resolve_synced_tables`` so fullsync, capture
|
|
and compare share one resolution path. System tables (``MSys*`` / ``~*``)
|
|
are already filtered by ``AccessReader.list_user_tables``.
|
|
"""
|
|
return resolve_synced_tables(fm, reader)
|
|
|
|
|
|
def full_sync_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter) -> dict:
|
|
"""Full-sync every eligible table in one Access file. Returns a summary."""
|
|
summary = {"file": fm.file, "tables": 0, "rows": 0, "skipped": []}
|
|
for access_table in resolve_tables(reader, fm):
|
|
target = fm.target_table(access_table)
|
|
if not writer.table_exists(fm.schema, target):
|
|
log.warning(
|
|
"skip %s -> %s.%s (target table missing)",
|
|
access_table, fm.schema, target,
|
|
)
|
|
summary["skipped"].append(access_table)
|
|
continue
|
|
try:
|
|
cols, rows = reader.read_all_rows(access_table)
|
|
writer.truncate_target(fm.schema, target)
|
|
n = writer.bulk_insert(fm.schema, target, cols, rows)
|
|
summary["tables"] += 1
|
|
summary["rows"] += n
|
|
log.info(
|
|
"full-synced %s -> %s.%s : %d rows",
|
|
access_table, fm.schema, target, n,
|
|
)
|
|
except Exception:
|
|
log.exception(
|
|
"full sync failed for %s -> %s.%s",
|
|
access_table, fm.schema, target,
|
|
)
|
|
summary["skipped"].append(access_table)
|
|
return summary
|
|
|
|
|
|
def full_sync(
|
|
cfg: SyncConfig,
|
|
db_filter: str | None = None,
|
|
table_filter: str | None = None,
|
|
clear_change_log: bool = False,
|
|
) -> list[dict]:
|
|
"""Run a full sync across all (optionally filtered) files.
|
|
|
|
``db_filter`` limits to a single Access file (matched on ``FileMapping.file``).
|
|
``table_filter`` restricts every considered file to that one table by
|
|
overriding ``include_tables``. ``clear_change_log`` additionally empties the
|
|
``TableChangeLog`` on each synced file after loading (use with care — it
|
|
mutates the Access side so the incremental service won't replay old deltas).
|
|
"""
|
|
files = cfg.files
|
|
if db_filter:
|
|
files = [f for f in files if f.file == db_filter]
|
|
if not files:
|
|
log.warning("no file matches --db %r", db_filter)
|
|
return []
|
|
if table_filter:
|
|
files = [f.model_copy(update={"include_tables": [table_filter]}) for f in files]
|
|
|
|
writer = SqlWriter(cfg.sql_server.conn_str, cfg.sql_server.sync_queue_table)
|
|
summaries = []
|
|
try:
|
|
for fm in files:
|
|
reader = AccessReader(fm.source_path(cfg), cfg.access.driver)
|
|
try:
|
|
summary = full_sync_file(fm, reader, writer)
|
|
summaries.append(summary)
|
|
if clear_change_log:
|
|
try:
|
|
ids = reader.read_all_log_ids()
|
|
deleted = reader.delete_log_ids(
|
|
ids, 500, 3
|
|
) if ids else 0
|
|
log.info("cleared %d TableChangeLog rows from %s", deleted, fm.file)
|
|
except Exception:
|
|
log.exception("clear-change-log failed for %s", fm.file)
|
|
finally:
|
|
reader.close()
|
|
finally:
|
|
writer.close()
|
|
return summaries
|
|
|
|
|
|
def main():
|
|
"""CLI entry point: ``python -m sync.fullsync config.yaml [options]``."""
|
|
ap = argparse.ArgumentParser(
|
|
description="One-shot full sync: Access -> SQL Server (TRUNCATE + bulk INSERT)."
|
|
)
|
|
ap.add_argument("config", nargs="?", default="config.yaml",
|
|
help="path to config.yaml (default: config.yaml)")
|
|
ap.add_argument("--db", help="limit to one Access file (by FileMapping.file)")
|
|
ap.add_argument("--table", help="limit to one table (applies to all matched files)")
|
|
ap.add_argument("--clear-change-log", action="store_true",
|
|
help="after loading, clear TableChangeLog on the synced files")
|
|
args = ap.parse_args()
|
|
|
|
cfg = load_config(args.config)
|
|
setup_logging(cfg.logging)
|
|
|
|
summaries = full_sync(
|
|
cfg,
|
|
db_filter=args.db,
|
|
table_filter=args.table,
|
|
clear_change_log=args.clear_change_log,
|
|
)
|
|
|
|
total_tables = sum(s["tables"] for s in summaries)
|
|
total_rows = sum(s["rows"] for s in summaries)
|
|
log.info("FULL SYNC COMPLETE: %d tables, %d rows", total_tables, total_rows)
|
|
for s in summaries:
|
|
if s["skipped"]:
|
|
log.warning(" %s: skipped %s", s["file"], s["skipped"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|