Migrate sync objects into ProductionDataBaseSync schema + add permanent audit archive
- Add sql/00_schema.sql: create dedicated ProductionDataBaseSync schema (idempotent) - Move SyncQueue and usp_SyncApply from dbo into ProductionDataBaseSync - Add sql/03_sync_log_archive.sql: permanent, append-only SyncLogArchive that records both OriginalOperateType and ProcessedOperateType plus the Access log OriginalTime, so pipeline divergences (e.g. Insert applied as Delete) stay reconstructible forever (SyncQueue is transient and only keeps processed type) - config.py: inject sync_queue_table / archive_table / apply_proc (default to the new schema); SqlWriter takes these names instead of hardcoding dbo - sql_writer.py: add ArchiveRow + insert_archive_row (dedup on source keys), parametrize queue/archive/proc names throughout - capture.py: archive every consumed log row before enqueue (preserves evidence before cleanup deletes the Access log) - service.py: pass the three names into SqlWriter - tests: read queue/proc names from config instead of hardcoding dbo.SyncQueue
This commit is contained in:
@@ -21,10 +21,16 @@ def test_insert_dedup_and_applied_ids():
|
||||
if not os.environ.get("RUN_INTEGRATION"):
|
||||
pytest.skip("integration")
|
||||
cfg = load_config("config.yaml")
|
||||
w = SqlWriter(cfg.sql_server.conn_str, "dbo.SyncQueue")
|
||||
qt = cfg.sql_server.sync_queue_table
|
||||
w = SqlWriter(
|
||||
cfg.sql_server.conn_str,
|
||||
qt,
|
||||
cfg.sql_server.archive_table,
|
||||
cfg.sql_server.apply_proc,
|
||||
)
|
||||
try:
|
||||
cur = w._conn.cursor()
|
||||
cur.execute("DELETE dbo.SyncQueue WHERE SourceFile='sqlw_test.accdb'")
|
||||
cur.execute(f"DELETE {qt} WHERE SourceFile='sqlw_test.accdb'")
|
||||
qr = QueueRow(
|
||||
source_file="sqlw_test.accdb",
|
||||
source_table="T",
|
||||
@@ -38,7 +44,7 @@ def test_insert_dedup_and_applied_ids():
|
||||
w.insert_queue_row(qr)
|
||||
w.insert_queue_row(qr) # duplicate must be deduped (ignored)
|
||||
cur.execute(
|
||||
"SELECT COUNT(*) FROM dbo.SyncQueue "
|
||||
f"SELECT COUNT(*) FROM {qt} "
|
||||
"WHERE SourceFile='sqlw_test.accdb' AND SourceLogID=100"
|
||||
)
|
||||
assert cur.fetchone()[0] == 1
|
||||
@@ -47,12 +53,12 @@ def test_insert_dedup_and_applied_ids():
|
||||
w.call_apply(max_retries=5)
|
||||
|
||||
cur.execute(
|
||||
"UPDATE dbo.SyncQueue SET Status='applied' "
|
||||
f"UPDATE {qt} SET Status='applied' "
|
||||
"WHERE SourceFile='sqlw_test.accdb'"
|
||||
)
|
||||
assert w.applied_log_ids("sqlw_test.accdb") == [100]
|
||||
finally:
|
||||
cur.execute("DELETE dbo.SyncQueue WHERE SourceFile='sqlw_test.accdb'")
|
||||
cur.execute(f"DELETE {qt} WHERE SourceFile='sqlw_test.accdb'")
|
||||
w.close()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user