feat: SyncQueue table and set-based apply stored procedure

- sql/01_sync_queue.sql: idempotent DDL for dbo.SyncQueue (PK + unique
  dedup index + pending lookup index), safe to re-run.
- sql/02_sync_apply.sql: dbo.usp_SyncApply (@MaxRetries INT=5). Per
  distinct (TargetSchema,TargetTable) it builds column projections from
  sys.columns (excludes ID key/computed/identity/rowversion) and runs a
  dynamic-SQL MERGE (last-write-wins via ROW_NUMBER over SourceLogID DESC)
  for Insert/Update plus a DELETE for the last op = Delete.
  SET IDENTITY_INSERT ON preserves Access PKs.
- tests/conftest.py: sql_conn fixture reads conn_str from gitignored
  config.yaml via load_config; skipped without RUN_INTEGRATION=1.
- tests/test_apply_proc.py: integration test covering IDENTITY-preserving
  INSERT, last-write-wins UPDATE, BIT conversion, and DELETE; cleans up.

Deviation from the brief's procedure (root-cause fix, design preserved):
every JSON path key is quoted ('$."col"') so non-ASCII column names
(e.g. Chinese 名字/数量) parse correctly. Without quoting, JSON_VALUE
raises "JSON path format is not correct" on Chinese columns, which is the
real target schema for this Access->SQL Server sync.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-07-14 12:12:21 +08:00
parent 7c6cb10b91
commit 65939f5e85
4 changed files with 268 additions and 0 deletions

30
tests/conftest.py Normal file
View File

@@ -0,0 +1,30 @@
"""Pytest fixtures for the Access -> SQL Server sync integration tests.
Credential safety: the SQL connection string is read from the gitignored
``config.yaml`` via ``sync.config.load_config``. No password is ever hardcoded
in a committed file. The ``.sql`` artefacts contain no credentials.
"""
import os
import pytest
import pyodbc
from sync.config import load_config
@pytest.fixture
def sql_conn():
"""A pyodbc connection to the CompanyDB SQL Server.
Skipped unless ``RUN_INTEGRATION=1`` is set. The connection is opened in a
transaction (autocommit=False); each test owns its commit/rollback and is
responsible for cleaning up any temporary objects it creates.
"""
if not os.environ.get("RUN_INTEGRATION"):
pytest.skip("set RUN_INTEGRATION=1 to run SQL integration tests")
cfg = load_config("config.yaml") # gitignored, real creds
conn = pyodbc.connect(cfg.sql_server.conn_str, autocommit=False)
try:
yield conn
finally:
conn.rollback()
conn.close()

81
tests/test_apply_proc.py Normal file
View File

@@ -0,0 +1,81 @@
"""Integration test for dbo.usp_SyncApply.
Validates: IDENTITY-preserving INSERT, last-write-wins UPDATE, BIT conversion
from JSON ``true``/``false``, and DELETE of the last op. Creates a throwaway
schema ``sync_test`` and table ``ApplyDemo_YEAR2026`` and cleans them up at the
end so no residue is left on CompanyDB.
"""
import pytest
@pytest.mark.integration
def test_upsert_insert_update_delete_last_write_wins(sql_conn):
cur = sql_conn.cursor()
sch, tbl = "sync_test", "ApplyDemo_YEAR2026"
# --- Arrange: fresh target table + empty queue -------------------------
cur.execute("IF SCHEMA_ID('sync_test') IS NULL EXEC('CREATE SCHEMA sync_test')")
cur.execute(
"IF OBJECT_ID('sync_test.ApplyDemo_YEAR2026') IS NOT NULL "
"DROP TABLE sync_test.ApplyDemo_YEAR2026"
)
cur.execute(
"CREATE TABLE sync_test.ApplyDemo_YEAR2026 ("
"ID INT IDENTITY(1,1) PRIMARY KEY, "
"名字 NVARCHAR(255) NULL, "
"数量 INT NULL, "
"时间 DATETIME2 NULL, "
"标记 BIT NULL)"
)
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
# Insert then a later Update for the same RecordID=1 -> last write wins.
cur.execute(
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"TargetTable,RecordID,OperateType,RowData,Status) "
"VALUES('t.accdb','ApplyDemo',1,'sync_test','ApplyDemo_YEAR2026','1',"
"'Insert','{\"ID\":1,\"名字\":\"A\",\"数量\":3,\"时间\":\"2026-01-01T00:00:00\",\"标记\":true}','pending')"
)
cur.execute(
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"TargetTable,RecordID,OperateType,RowData,Status) "
"VALUES('t.accdb','ApplyDemo',2,'sync_test','ApplyDemo_YEAR2026','1',"
"'Update','{\"ID\":1,\"名字\":\"A2\",\"数量\":5,\"时间\":\"2026-01-02T00:00:00\",\"标记\":false}','pending')"
)
sql_conn.commit()
# --- Act 1: apply upsert ----------------------------------------------
cur.execute("EXEC dbo.usp_SyncApply @MaxRetries=5")
sql_conn.commit()
# --- Assert 1: the later Update wins; BIT false -> 0 ------------------
cur.execute("SELECT 名字,数量,标记 FROM sync_test.ApplyDemo_YEAR2026 WHERE ID=1")
row = cur.fetchone()
assert row is not None, "expected ID=1 to exist after upsert"
assert row.名字 == "A2"
assert row.数量 == 5
assert row.标记 == 0 # BIT false
# --- Act 2: a later Delete wins ---------------------------------------
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
cur.execute(
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
"TargetTable,RecordID,OperateType,RowData,Status) "
"VALUES('t.accdb','ApplyDemo',3,'sync_test','ApplyDemo_YEAR2026','1',"
"'Delete',NULL,'pending')"
)
sql_conn.commit()
cur.execute("EXEC dbo.usp_SyncApply @MaxRetries=5")
sql_conn.commit()
# --- Assert 2: row removed --------------------------------------------
cur.execute("SELECT COUNT(*) FROM sync_test.ApplyDemo_YEAR2026 WHERE ID=1")
assert cur.fetchone()[0] == 0
# --- Cleanup ----------------------------------------------------------
cur.execute(
"IF OBJECT_ID('sync_test.ApplyDemo_YEAR2026') IS NOT NULL "
"DROP TABLE sync_test.ApplyDemo_YEAR2026"
)
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
sql_conn.commit()