feat: SyncQueue table and set-based apply stored procedure
- sql/01_sync_queue.sql: idempotent DDL for dbo.SyncQueue (PK + unique
dedup index + pending lookup index), safe to re-run.
- sql/02_sync_apply.sql: dbo.usp_SyncApply (@MaxRetries INT=5). Per
distinct (TargetSchema,TargetTable) it builds column projections from
sys.columns (excludes ID key/computed/identity/rowversion) and runs a
dynamic-SQL MERGE (last-write-wins via ROW_NUMBER over SourceLogID DESC)
for Insert/Update plus a DELETE for the last op = Delete.
SET IDENTITY_INSERT ON preserves Access PKs.
- tests/conftest.py: sql_conn fixture reads conn_str from gitignored
config.yaml via load_config; skipped without RUN_INTEGRATION=1.
- tests/test_apply_proc.py: integration test covering IDENTITY-preserving
INSERT, last-write-wins UPDATE, BIT conversion, and DELETE; cleans up.
Deviation from the brief's procedure (root-cause fix, design preserved):
every JSON path key is quoted ('$."col"') so non-ASCII column names
(e.g. Chinese 名字/数量) parse correctly. Without quoting, JSON_VALUE
raises "JSON path format is not correct" on Chinese columns, which is the
real target schema for this Access->SQL Server sync.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
30
tests/conftest.py
Normal file
30
tests/conftest.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""Pytest fixtures for the Access -> SQL Server sync integration tests.
|
||||
|
||||
Credential safety: the SQL connection string is read from the gitignored
|
||||
``config.yaml`` via ``sync.config.load_config``. No password is ever hardcoded
|
||||
in a committed file. The ``.sql`` artefacts contain no credentials.
|
||||
"""
|
||||
import os
|
||||
import pytest
|
||||
import pyodbc
|
||||
|
||||
from sync.config import load_config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sql_conn():
|
||||
"""A pyodbc connection to the CompanyDB SQL Server.
|
||||
|
||||
Skipped unless ``RUN_INTEGRATION=1`` is set. The connection is opened in a
|
||||
transaction (autocommit=False); each test owns its commit/rollback and is
|
||||
responsible for cleaning up any temporary objects it creates.
|
||||
"""
|
||||
if not os.environ.get("RUN_INTEGRATION"):
|
||||
pytest.skip("set RUN_INTEGRATION=1 to run SQL integration tests")
|
||||
cfg = load_config("config.yaml") # gitignored, real creds
|
||||
conn = pyodbc.connect(cfg.sql_server.conn_str, autocommit=False)
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
81
tests/test_apply_proc.py
Normal file
81
tests/test_apply_proc.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""Integration test for dbo.usp_SyncApply.
|
||||
|
||||
Validates: IDENTITY-preserving INSERT, last-write-wins UPDATE, BIT conversion
|
||||
from JSON ``true``/``false``, and DELETE of the last op. Creates a throwaway
|
||||
schema ``sync_test`` and table ``ApplyDemo_YEAR2026`` and cleans them up at the
|
||||
end so no residue is left on CompanyDB.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_upsert_insert_update_delete_last_write_wins(sql_conn):
|
||||
cur = sql_conn.cursor()
|
||||
sch, tbl = "sync_test", "ApplyDemo_YEAR2026"
|
||||
|
||||
# --- Arrange: fresh target table + empty queue -------------------------
|
||||
cur.execute("IF SCHEMA_ID('sync_test') IS NULL EXEC('CREATE SCHEMA sync_test')")
|
||||
cur.execute(
|
||||
"IF OBJECT_ID('sync_test.ApplyDemo_YEAR2026') IS NOT NULL "
|
||||
"DROP TABLE sync_test.ApplyDemo_YEAR2026"
|
||||
)
|
||||
cur.execute(
|
||||
"CREATE TABLE sync_test.ApplyDemo_YEAR2026 ("
|
||||
"ID INT IDENTITY(1,1) PRIMARY KEY, "
|
||||
"名字 NVARCHAR(255) NULL, "
|
||||
"数量 INT NULL, "
|
||||
"时间 DATETIME2 NULL, "
|
||||
"标记 BIT NULL)"
|
||||
)
|
||||
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
|
||||
|
||||
# Insert then a later Update for the same RecordID=1 -> last write wins.
|
||||
cur.execute(
|
||||
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
|
||||
"TargetTable,RecordID,OperateType,RowData,Status) "
|
||||
"VALUES('t.accdb','ApplyDemo',1,'sync_test','ApplyDemo_YEAR2026','1',"
|
||||
"'Insert','{\"ID\":1,\"名字\":\"A\",\"数量\":3,\"时间\":\"2026-01-01T00:00:00\",\"标记\":true}','pending')"
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
|
||||
"TargetTable,RecordID,OperateType,RowData,Status) "
|
||||
"VALUES('t.accdb','ApplyDemo',2,'sync_test','ApplyDemo_YEAR2026','1',"
|
||||
"'Update','{\"ID\":1,\"名字\":\"A2\",\"数量\":5,\"时间\":\"2026-01-02T00:00:00\",\"标记\":false}','pending')"
|
||||
)
|
||||
sql_conn.commit()
|
||||
|
||||
# --- Act 1: apply upsert ----------------------------------------------
|
||||
cur.execute("EXEC dbo.usp_SyncApply @MaxRetries=5")
|
||||
sql_conn.commit()
|
||||
|
||||
# --- Assert 1: the later Update wins; BIT false -> 0 ------------------
|
||||
cur.execute("SELECT 名字,数量,标记 FROM sync_test.ApplyDemo_YEAR2026 WHERE ID=1")
|
||||
row = cur.fetchone()
|
||||
assert row is not None, "expected ID=1 to exist after upsert"
|
||||
assert row.名字 == "A2"
|
||||
assert row.数量 == 5
|
||||
assert row.标记 == 0 # BIT false
|
||||
|
||||
# --- Act 2: a later Delete wins ---------------------------------------
|
||||
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
|
||||
cur.execute(
|
||||
"INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,"
|
||||
"TargetTable,RecordID,OperateType,RowData,Status) "
|
||||
"VALUES('t.accdb','ApplyDemo',3,'sync_test','ApplyDemo_YEAR2026','1',"
|
||||
"'Delete',NULL,'pending')"
|
||||
)
|
||||
sql_conn.commit()
|
||||
cur.execute("EXEC dbo.usp_SyncApply @MaxRetries=5")
|
||||
sql_conn.commit()
|
||||
|
||||
# --- Assert 2: row removed --------------------------------------------
|
||||
cur.execute("SELECT COUNT(*) FROM sync_test.ApplyDemo_YEAR2026 WHERE ID=1")
|
||||
assert cur.fetchone()[0] == 0
|
||||
|
||||
# --- Cleanup ----------------------------------------------------------
|
||||
cur.execute(
|
||||
"IF OBJECT_ID('sync_test.ApplyDemo_YEAR2026') IS NOT NULL "
|
||||
"DROP TABLE sync_test.ApplyDemo_YEAR2026"
|
||||
)
|
||||
cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'")
|
||||
sql_conn.commit()
|
||||
Reference in New Issue
Block a user