Migrate sync objects into ProductionDataBaseSync schema + add permanent audit archive

- Add sql/00_schema.sql: create dedicated ProductionDataBaseSync schema (idempotent)
- Move SyncQueue and usp_SyncApply from dbo into ProductionDataBaseSync
- Add sql/03_sync_log_archive.sql: permanent, append-only SyncLogArchive that
  records both OriginalOperateType and ProcessedOperateType plus the Access log
  OriginalTime, so pipeline divergences (e.g. Insert applied as Delete) stay
  reconstructible forever (SyncQueue is transient and only keeps processed type)
- config.py: inject sync_queue_table / archive_table / apply_proc (default to the
  new schema); SqlWriter takes these names instead of hardcoding dbo
- sql_writer.py: add ArchiveRow + insert_archive_row (dedup on source keys),
  parametrize queue/archive/proc names throughout
- capture.py: archive every consumed log row before enqueue (preserves evidence
  before cleanup deletes the Access log)
- service.py: pass the three names into SqlWriter
- tests: read queue/proc names from config instead of hardcoding dbo.SyncQueue
This commit is contained in:
Misaka_Company
2026-07-16 12:32:22 +08:00
parent 4179d3e232
commit d71b7eab62
11 changed files with 242 additions and 61 deletions

View File

@@ -1,15 +1,23 @@
"""Integration test for dbo.usp_SyncApply.
"""Integration test for usp_SyncApply.
Validates: IDENTITY-preserving INSERT, last-write-wins UPDATE, BIT conversion
from JSON ``true``/``false``, and DELETE of the last op. Creates a throwaway
schema ``sync_test`` and table ``ApplyDemo_YEAR2026`` and cleans them up at the
end so no residue is left on CompanyDB.
Queue table and apply proc names are read from ``config.yaml`` so the test
follows whichever schema the deployment targets (currently ProductionDataBaseSync).
"""
import pytest
from sync.config import load_config
@pytest.mark.integration
def test_upsert_insert_update_delete_last_write_wins(sql_conn):
cfg = load_config("config.yaml")
qt = cfg.sql_server.sync_queue_table
proc = cfg.sql_server.apply_proc
cur = sql_conn.cursor()
sch, tbl = "sync_test", "ApplyDemo_YEAR2026"
@@ -27,17 +35,17 @@ def test_upsert_insert_update_delete_last_write_wins(sql_conn):
"时间 DATETIME2 NULL, "
"标记 BIT NULL)"
)
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
cur.execute("DELETE " + qt + " WHERE TargetSchema='sync_test'")
# Insert then a later Update for the same RecordID=1 -> last write wins.
cur.execute(
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"INSERT " + qt + "(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"TargetTable,RecordID,OperateType,RowData,Status) "
"VALUES('t.accdb','ApplyDemo',1,'sync_test','ApplyDemo_YEAR2026','1',"
"'Insert','{\"ID\":1,\"名字\":\"A\",\"数量\":3,\"时间\":\"2026-01-01T00:00:00\",\"标记\":true}','pending')"
)
cur.execute(
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"INSERT " + qt + "(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"TargetTable,RecordID,OperateType,RowData,Status) "
"VALUES('t.accdb','ApplyDemo',2,'sync_test','ApplyDemo_YEAR2026','1',"
"'Update','{\"ID\":1,\"名字\":\"A2\",\"数量\":5,\"时间\":\"2026-01-02T00:00:00\",\"标记\":false}','pending')"
@@ -45,7 +53,7 @@ def test_upsert_insert_update_delete_last_write_wins(sql_conn):
sql_conn.commit()
# --- Act 1: apply upsert ----------------------------------------------
cur.execute("EXEC dbo.usp_SyncApply @MaxRetries=5")
cur.execute("EXEC " + proc + " @MaxRetries=5")
sql_conn.commit()
# --- Assert 1: the later Update wins; BIT false -> 0 ------------------
@@ -57,15 +65,15 @@ def test_upsert_insert_update_delete_last_write_wins(sql_conn):
assert row.标记 == 0 # BIT false
# --- Act 2: a later Delete wins ---------------------------------------
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
cur.execute("DELETE " + qt + " WHERE TargetSchema='sync_test'")
cur.execute(
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"INSERT " + qt + "(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"TargetTable,RecordID,OperateType,RowData,Status) "
"VALUES('t.accdb','ApplyDemo',3,'sync_test','ApplyDemo_YEAR2026','1',"
"'Delete',NULL,'pending')"
)
sql_conn.commit()
cur.execute("EXEC dbo.usp_SyncApply @MaxRetries=5")
cur.execute("EXEC " + proc + " @MaxRetries=5")
sql_conn.commit()
# --- Assert 2: row removed --------------------------------------------
@@ -77,5 +85,5 @@ def test_upsert_insert_update_delete_last_write_wins(sql_conn):
"IF OBJECT_ID('sync_test.ApplyDemo_YEAR2026') IS NOT NULL "
"DROP TABLE sync_test.ApplyDemo_YEAR2026"
)
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
cur.execute("DELETE " + qt + " WHERE TargetSchema='sync_test'")
sql_conn.commit()

View File

@@ -21,10 +21,16 @@ def test_insert_dedup_and_applied_ids():
if not os.environ.get("RUN_INTEGRATION"):
pytest.skip("integration")
cfg = load_config("config.yaml")
w = SqlWriter(cfg.sql_server.conn_str, "dbo.SyncQueue")
qt = cfg.sql_server.sync_queue_table
w = SqlWriter(
cfg.sql_server.conn_str,
qt,
cfg.sql_server.archive_table,
cfg.sql_server.apply_proc,
)
try:
cur = w._conn.cursor()
cur.execute("DELETE dbo.SyncQueue WHERE SourceFile='sqlw_test.accdb'")
cur.execute(f"DELETE {qt} WHERE SourceFile='sqlw_test.accdb'")
qr = QueueRow(
source_file="sqlw_test.accdb",
source_table="T",
@@ -38,7 +44,7 @@ def test_insert_dedup_and_applied_ids():
w.insert_queue_row(qr)
w.insert_queue_row(qr) # duplicate must be deduped (ignored)
cur.execute(
"SELECT COUNT(*) FROM dbo.SyncQueue "
f"SELECT COUNT(*) FROM {qt} "
"WHERE SourceFile='sqlw_test.accdb' AND SourceLogID=100"
)
assert cur.fetchone()[0] == 1
@@ -47,12 +53,12 @@ def test_insert_dedup_and_applied_ids():
w.call_apply(max_retries=5)
cur.execute(
"UPDATE dbo.SyncQueue SET Status='applied' "
f"UPDATE {qt} SET Status='applied' "
"WHERE SourceFile='sqlw_test.accdb'"
)
assert w.applied_log_ids("sqlw_test.accdb") == [100]
finally:
cur.execute("DELETE dbo.SyncQueue WHERE SourceFile='sqlw_test.accdb'")
cur.execute(f"DELETE {qt} WHERE SourceFile='sqlw_test.accdb'")
w.close()