From fb025bb061b42f61b7a77f4579f7e5dc7cbb4692 Mon Sep 17 00:00:00 2001 From: Misaka_Company Date: Tue, 14 Jul 2026 11:12:34 +0800 Subject: [PATCH] chore: project scaffold for access-datamacro sync --- .claude/skills/nssm-114/SKILL.md | 134 ++ .claude/skills/nssm-114/evals/evals.json | 29 + .../with_skill/outputs/result.md | 28 + .../with_skill/timing.json | 1 + .../with_skill/outputs/result.md | 58 + .../with_skill/timing.json | 1 + .../nssm-114/references/nssm-commands.md | 257 ++++ .gitignore | 8 + README.md | 11 + .../plans/2026-07-14-access-datamacro-sync.md | 1235 +++++++++++++++++ ...2026-07-14-access-datamacro-sync-design.md | 326 +++++ requirements.txt | 4 + src/sync/__init__.py | 0 tests/__init__.py | 0 14 files changed, 2092 insertions(+) create mode 100644 .claude/skills/nssm-114/SKILL.md create mode 100644 .claude/skills/nssm-114/evals/evals.json create mode 100644 .claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-1-list-services/with_skill/outputs/result.md create mode 100644 .claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-1-list-services/with_skill/timing.json create mode 100644 .claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-2-inspect-webtable/with_skill/outputs/result.md create mode 100644 .claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-2-inspect-webtable/with_skill/timing.json create mode 100644 .claude/skills/nssm-114/references/nssm-commands.md create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/superpowers/plans/2026-07-14-access-datamacro-sync.md create mode 100644 docs/superpowers/specs/2026-07-14-access-datamacro-sync-design.md create mode 100644 requirements.txt create mode 100644 src/sync/__init__.py create mode 100644 tests/__init__.py diff --git a/.claude/skills/nssm-114/SKILL.md b/.claude/skills/nssm-114/SKILL.md new file mode 100644 index 0000000..6a2b251 --- /dev/null +++ b/.claude/skills/nssm-114/SKILL.md @@ -0,0 +1,134 @@ +--- +name: nssm-114 +description: NSSM (Non-Sucking Service Manager) operations and Windows service maintenance targeting the internal host 114. Use this skill whenever the user mentions NSSM, nssm.exe, or any nssm command (install, remove, set, get, reset, dump, start, stop, restart, pause, continue, rotate, status), or asks to manage/configure/restart a Windows service that you know or suspect is an NSSM-wrapped service. Also use it when the user says "运维/管理/重启/查一下服务" in connection with 114 or an internal service. In this environment, NSSM-related ops ALWAYS target host 114 (SSH alias `114`) — never run nssm locally. Route everything through `ssh 114 "..."`. +--- + +# NSSM Ops on Host 114 + +## The core routing rule + +**Every NSSM operation targets host 114 over SSH.** Never run `nssm` on the local machine. + +- SSH alias: `114` (Windows Server host, non-interactive) +- All commands go through: `ssh 114 "nssm ..."` or `ssh 114 "cmd /c ..."` / `ssh 114 "powershell -Command \"...\""` +- If the user says "114" + anything about services, or just "查一下/重启一下服务" with NSSM context, that's host 114. + +## How SSH to 114 actually works (read this before running anything) + +This is the part that bites. SSH to 114 is **non-interactive**: + +1. **No session.** Each `ssh 114 "..."` is a one-shot. `stdin` is not a TTY, so you cannot open an interactive shell and type follow-ups. Run one command per invocation, or chain with `&&` / `;`. +2. **`Pseudo-terminal will not be allocated`** warnings are normal — ignore them. +3. **Bash double-quote escaping eats PowerShell `$` variables.** When you run `ssh 114 "powershell -Command \"... $_ ...\""`, bash mangles `$_` and `$var` into `\extglob` garbage, producing hundreds of errors. This is the #1 source of broken commands. + - **Avoid `$_`, `$PSItem`, and any `$var` in PowerShell sent over SSH-through-bash.** + - Prefer plain `cmd.exe` tools: `findstr`, `where`, `dir`, `type`, `reg query`. + - If you need PowerShell, avoid pipeline `$_` (use `Where-Object`/`ForEach-Object` scriptblocks that don't reference `$_`, or filter in cmd). + - For complex logic, write a `.ps1` or `.py` file to the host first (via scp or a heredoc), then run it. Don't try to cram multi-line PowerShell into a quoted `-Command`. +4. **Verify command output.** Many `nssm` subcommands print `操作成功完成` / `Set parameter ... for service ...` on success. Read what comes back before declaring done. + +## Listing services (always start here when scope is unclear) + +``` +ssh 114 "nssm list" +``` + +This enumerates all NSSM-managed services on the host. If the user says "查一下服务" / "有哪些服务", run this first. + +## The NSSM command reference (essentials) + +Full official command reference is bundled at `references/nssm-commands.md` — read it when you need exact syntax, subparameters, or less-common parameters. The everyday set: + +### Lifecycle +``` +nssm install [] # install +nssm remove [confirm] # uninstall (confirm skips prompt) +nssm start +nssm stop +nssm restart +nssm status # 0=stopped, 1=running, etc. +``` + +### Inspect config +``` +nssm dump # full config as nssm set commands (great for snapshots) +nssm get [] +``` + +### Edit config +``` +nssm set [] +nssm reset [] # back to default / delete reg entry +``` + +Common parameters: `Application`/`AppProgram`, `AppParameters`, `AppDirectory`, `AppStdout`, `AppStderr`, `AppExit`, `AppEnvironmentExtra`, `AppEnvironment`, `AppRotateFiles`, `AppRotateBytes`, `AppRotateOnline`, `Description`, `DisplayName`, `ObjectName`, `Start`, `Type`, `DependOnService`. + +### Service controls +``` +nssm pause # SERVICE_PAUSE_PENDING capable +nssm continue +nssm rotate # online log rotation (needs AppRotateFiles + AppRotateOnline) +``` + +## NSSM gotchas (these will save you) + +### Gotcha 1 — `AppEnvironmentExtra` replaces, it does not append + +`nssm set AppEnvironmentExtra ` writes the **entire** environment block. Calling it multiple times in a row (e.g. chained with `&&`) wipes everything except the last value. The official docs confirm: + +> When setting an environment block with `nssm set`, each variable should be specified as a KEY=VALUE pair in a separate argument. + +**Correct — all variables in ONE call:** +``` +nssm set AppEnvironmentExtra VAR1=v1 VAR2=v2 VAR3=v3 +``` + +**Wrong — only VAR3 survives:** +``` +nssm set AppEnvironmentExtra VAR1=v1 && nssm set AppEnvironmentExtra VAR2=v2 && nssm set AppEnvironmentExtra VAR3=v3 +``` + +Before adding env vars, snapshot the current block so you can include existing entries: +``` +ssh 114 "nssm get AppEnvironmentExtra" # or: reg query ...\Parameters /v AppEnvironmentExtra +``` + +### Gotcha 2 — what `nssm dump` prefixes mean + +`nssm dump ` prints each env var with a prefix that encodes its semantics: + +- `:NAME=value` — **conditional**: set only if not already present in the process environment. Won't override a value inherited from the machine/user environment. +- `+NAME=value` — **override**: force-set, replacing any inherited value. + +The raw registry (`reg query`) stores values without these prefixes; the prefix is NSSM's display convention. If you need a service-level env var to **override** a machine-level one (very common for proxies), make sure it shows as `+` in `nssm dump`, i.e. write it without a leading `:` in the `nssm set` value. + +### Gotcha 3 — restart to apply env/log/path changes + +`nssm set` only edits the registry. The running process won't pick up `AppParameters`, `AppDirectory`, `AppEnvironmentExtra`, log paths, etc. until you **restart** the service. Snapshot logs first if the service is mid-task: + +``` +ssh 114 "nssm restart " +``` + +### Gotcha 4 — LocalSystem vs user-account services + +Most NSSM services run as `LocalSystem`, which can't access network shares or the current user's profile/credentials. If a service needs LAN file access, it runs as a real account (e.g. `.\peng`). Check with `nssm get ObjectName` before assuming network access works the same way as your shell. + +## Working safely on a production host + +114 runs real services. Before mutating state: + +1. **Snapshot first.** `nssm dump > before.txt` (or capture relevant `nssm get` output) so you have an exact rollback recipe. +2. **One change at a time**, verify output, then move on. +3. **Prefer `nssm restart`** over stop+start; it's atomic from NSSM's perspective. +4. **Tail logs** after changes: `nssm get AppStdout` / `AppStderr` to find the log paths, then read the tail for new errors. +5. **Rollback** = re-run the snapshot's `nssm set` lines, or `nssm reset` the parameter, then restart. + +## When to read the bundled reference + +Read `references/nssm-commands.md` when you need: +- Exact subparameter syntax for `AppExit`, `AppEnvironment(Extra)`, `DependOnService`, `ObjectName`, `Start`, `Type` +- The full list of native vs non-standard parameters +- Priority class constants for `AppPriority` +- Confirmation that a parameter exists before guessing + +Do NOT read it for the everyday lifecycle/config commands above — those are the hot path. diff --git a/.claude/skills/nssm-114/evals/evals.json b/.claude/skills/nssm-114/evals/evals.json new file mode 100644 index 0000000..8f72700 --- /dev/null +++ b/.claude/skills/nssm-114/evals/evals.json @@ -0,0 +1,29 @@ +{ + "skill_name": "nssm-114", + "evals": [ + { + "id": 1, + "name": "list-services-readonly", + "prompt": "114上有哪些nssm服务？列一下", + "expected_output": "通过 ssh 114 \"nssm list\" 列出服务清单；不执行任何写操作；遵循非交互SSH约定", + "files": [], + "safety": "READ-ONLY — safe to actually run against 114" + }, + { + "id": 2, + "name": "inspect-config-readonly", + "prompt": "帮我看一下 114 上 WebTable 这个服务的完整配置，日志轮转开了吗", + "expected_output": "用 ssh 114 \"nssm dump WebTable\" 拿到完整配置；解读 AppRotateFiles/AppRotateBytes/AppRotateOnline；指出日志路径；遵循SSH坑约束", + "files": [], + "safety": "READ-ONLY — safe to actually run against 114" + }, + { + "id": 3, + "name": "mutating-env-config", + "prompt": "给 114 上某个服务加一个环境变量 FOO=bar，和它已有的 PYTHONUTF8=1 共存", + "expected_output": "先 nssm get AppEnvironmentExtra 看现状；正确地用单条 nssm set 同时传 PYTHONUTF8=1 和 FOO=bar（避免覆盖丢失坑）；提醒改完要 restart；不实际执行破坏性写入，或先快照", + "files": [], + "safety": "MUTATING — eval subagent would write to production. Recommend NOT auto-running; review guidance only." + } + ] +} diff --git a/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-1-list-services/with_skill/outputs/result.md b/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-1-list-services/with_skill/outputs/result.md new file mode 100644 index 0000000..c26e7aa --- /dev/null +++ b/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-1-list-services/with_skill/outputs/result.md @@ -0,0 +1,28 @@ +# NSSM Services on Host 114 + +## Command Run + +``` +ssh 114 "nssm list" +``` + +This is the canonical read-only enumeration command per the skill. No stdin/TTY interaction, no PowerShell `$` variables involved, so no escaping hazards. + +## Output (the list of NSSM-managed services) + +``` +BLD_IncrementalSync +CargotraceFastAPIService +ExcelToSqlServer +WarehouseQuery +WebTable +``` + +5 NSSM-managed services total. + +## SSH Behavior Notes + +- Single one-shot `ssh 114 "nssm list"` invocation — no interactive shell, no follow-up commands needed. +- No `Pseudo-terminal will not be allocated` warning appeared in this run, but per the skill it would be safe to ignore if it had. +- No PowerShell involved, so the `$_` / `$var` mangling gotcha does not apply. +- `nssm list` printed one service name per line with no extra header/footer — clean output. diff --git a/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-1-list-services/with_skill/timing.json b/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-1-list-services/with_skill/timing.json new file mode 100644 index 0000000..513a9da --- /dev/null +++ b/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-1-list-services/with_skill/timing.json @@ -0,0 +1 @@ +{"total_tokens": 45, "duration_ms": 38890, "total_duration_seconds": 38.9} diff --git a/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-2-inspect-webtable/with_skill/outputs/result.md b/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-2-inspect-webtable/with_skill/outputs/result.md new file mode 100644 index 0000000..c588e11 --- /dev/null +++ b/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-2-inspect-webtable/with_skill/outputs/result.md @@ -0,0 +1,58 @@ +# WebTable Service Inspection on Host 114 + +## Commands Ran (all read-only) + +``` +ssh 114 "nssm dump WebTable" +ssh 114 "nssm status WebTable" +ssh 114 "nssm get WebTable AppRotateFiles & nssm get WebTable AppRotateBytes & nssm get WebTable AppRotateOnline & nssm get WebTable AppRotateSeconds" +``` + +## Full Config (`nssm dump WebTable`) + +``` +nssm install WebTable "C:\Program Files\nodejs\npx.cmd" +nssm set WebTable AppParameters "next start -H 0.0.0.0 -p 3081" +nssm set WebTable AppDirectory C:\Users\peng\projects\ProductionDatabase\web-table +nssm set WebTable AppExit Default Restart +nssm set WebTable AppStdout C:\Users\peng\projects\ProductionDatabase\web-table\logs\stdout.log +nssm set WebTable AppStderr C:\Users\peng\projects\ProductionDatabase\web-table\logs\stderr.log +nssm set WebTable DisplayName WebTable +nssm set WebTable ObjectName LocalSystem +nssm set WebTable Start SERVICE_AUTO_START +nssm set WebTable Type SERVICE_WIN32_OWN_PROCESS +``` + +## Service Status + +``` +SERVICE_RUNNING +``` + +## Log Rotation Parameters + +| Parameter | Value | Meaning | +|--------------------|-------|----------------------------------| +| `AppRotateFiles` | `0` | Rotation disabled (no file rotation) | +| `AppRotateBytes` | `0` | No byte-size threshold set | +| `AppRotateOnline` | `0` | Online rotation disabled | +| `AppRotateSeconds` | `0` | No time-based rotation threshold | + +## Answer: Is Log Rotation Enabled? + +**No. Log rotation is fully disabled for WebTable.** + +All four rotation-related parameters are `0`: +- `AppRotateFiles = 0` — NSSM will not rotate log files at all. +- `AppRotateBytes = 0` — no size threshold configured. +- `AppRotateOnline = 0` — online rotation not enabled. +- `AppRotateSeconds = 0` — no time-based rotation configured. + +The `nssm dump` output also omits any `AppRotate*` lines, confirming these are all at their default (unset/disabled) state. Logs are written directly to `C:\Users\peng\projects\ProductionDatabase\web-table\logs\stdout.log` and `stderr.log` and will grow without limit. + +## SSH Behavior Notes + +- All three SSH commands completed successfully in a single one-shot invocation each (non-interactive, as documented in the skill). +- The `Pseudo-terminal will not be allocated` warning did not appear in this run, but is expected and harmless. +- Chaining four `nssm get` calls with `&` in a single SSH invocation worked correctly on the Windows host's cmd.exe, returning four lines of output in order. +- No PowerShell `$_` issues were encountered because all commands used plain `nssm` CLI (cmd.exe-native), per the skill's recommendation to prefer cmd tools over PowerShell over SSH. diff --git a/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-2-inspect-webtable/with_skill/timing.json b/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-2-inspect-webtable/with_skill/timing.json new file mode 100644 index 0000000..3fd0179 --- /dev/null +++ b/.claude/skills/nssm-114/nssm-114-workspace/iteration-1/eval-2-inspect-webtable/with_skill/timing.json @@ -0,0 +1 @@ +{"total_tokens": 116, "duration_ms": 93003, "total_duration_seconds": 93.0} diff --git a/.claude/skills/nssm-114/references/nssm-commands.md b/.claude/skills/nssm-114/references/nssm-commands.md new file mode 100644 index 0000000..3934a35 --- /dev/null +++ b/.claude/skills/nssm-114/references/nssm-commands.md @@ -0,0 +1,257 @@ +# NSSM Commands + +> Source: + +## Managing services from the command line + +NSSM's core functionality has always been available from the command line. + +## Service installation + +``` +nssm install +nssm install +nssm install +nssm install [] +``` + +By default the service's startup directory will be set to the directory containing the `program`. The startup directory can be overridden after the service has been installed. + +``` +nssm set AppDirectory +``` + +## Service removal + +``` +nssm remove +nssm remove +nssm remove confirm +``` + +## Service management + +As of version 2.22, NSSM offers basic service management functionality. NSSM will also accept a service `displayname` anywhere that a `servicename` is expected, since Windows does not allow a service to have a name or display name which conflicts with either the name or display name of another service. Both the service name (also called the service key name) and its display name uniquely identify a service. + +### Starting and stopping a service + +``` +nssm start +nssm stop +nssm restart +``` + +### Querying a service's status + +``` +nssm status +``` + +### Sending controls to services + +``` +nssm pause +nssm continue +nssm rotate +``` + +`nssm rotate` triggers on-demand rotation for NSSM services with I/O redirection and online rotation enabled. NSSM accepts user-defined control 128 as a cue to begin output file rotation. Non-NSSM services might respond to control 128 in their own way (or ignore it, or crash). + +## Service editing + +As of version 2.22, **all** parameters understood by NSSM can be queried or configured on the command line. A subset of system parameters can also be queried and, in some cases, modified. + +### General syntax + +Parameters can usually be queried as follows. + +``` +nssm get +``` + +Some parameters are ambiguous and require a subparameter. See below. + +``` +nssm get +``` + +Parameters can usually be set in a similar way. + +``` +nssm set +nssm set +``` + +Most parameters can be reset to their defaults, which is equivalent to removing the associated registry entry. + +``` +nssm reset +nssm reset +``` + +As a convenience, NSSM will accept additional arguments beyond the `value` required, and concatenate them together, separated by single spaces. Thus the following two invocations are identical: + +``` +nssm set AppParameters "-classpath C:\Classes" +nssm set AppParameters -classpath C:\Classes +``` + +### Parameters + +A `parameter` is usually a string with the same name as the registry entry which controls the associated functionality. So, for example, the following command sets the startup directory for a service: + +``` +nssm set AppDirectory +``` + +### Values + +Most parameters are configured by setting the same value as is documented for the associated registry entry. To enable file rotation, for example, you would use the following command: + +``` +nssm set AppRotation 1 +``` + +See below for a list of parameters whose values are set differently. + +### Native parameters + +Certain parameters configure properties of the service itself rather than the behaviour of NSSM. They too are named after their associated registry values. + +- **DependOnGroup:** Load order groups whose members must start before the service can start. +- **DependOnService:** Services which must start before the service can start. +- **Description:** The service's description. +- **DisplayName:** The service's display name, eg *Application Layer Gateway Service*. This is the name shown under the *Name* column in *services.msc*. +- **ImagePath:** Path to the service executable, eg *C:\Windows\System32\alg.exe*. For NSSM services, this will be the path to *nssm.exe*. +- **ObjectName:** The name of the user account under which the service runs. The default is *LOCALSYSTEM*. +- **Name:** The service key name, eg *ALG*. The key name cannot be changed. You can use `nssm get Name` to find out the key name of a service. +- **Start:** The service's startup type, eg *Automatic*. +- **Type:** The service type. NSSM can only edit services of type `SERVICE_WIN32_OWN_PROCESS`. + +### Non-standard parameters + +- When used with `nssm get`, **AppEnvironment** and **AppEnvironmentExtra** accept an optional subparameter. If no subparameter is given, `nssm get` will print all configured environment variables, one per line in the form *KEY=VALUE*. If a subparameter is given, `nssm get` will print the value configured for the named environment variable, or the empty string if that variable is not present in the environment block. + + For example, suppose that **AppEnvironmentExtra** were configured with two variables, *CLASSPATH=C:\Classes* and *TEMP=C:\Temp*. The following invocation: + + ``` + nssm get AppEnvironmentExtra + ``` + + would print: + + ``` + CLASSPATH=C:\Classes + TEMP=C:\Temp + ``` + + Whereas the syntax below: + + ``` + nssm get AppEnvironmentExtra CLASSPATH + ``` + + would print: + + ``` + C:\Classes + ``` + + When setting an environment block with `nssm set`, each variable should be specified as a *KEY=VALUE* pair in a separate argument. For example: + + ``` + nssm set AppEnvironmentExtra CLASSPATH=C:\Classes TEMP=C:\Temp + ``` + +- The **AppExit** parameter requires a subparameter specifying the exit code to get or set. The default action can be specified with the string *Default*. + + For example, to get the default exit action for a service you should run: + + ``` + nssm get AppExit Default + ``` + + To get the exit action when the application exits with exit code 2, run: + + ``` + nssm get AppExit 2 + ``` + + Note that if no explicit action is configured for a specified exit code, NSSM will print the default exit action. + + To configure the service to stop when the application exits with an exit code of 2, run: + + ``` + nssm set AppExit 2 Exit + ``` + +- The **AppPriority** parameter takes a priority class constant as specified in the `SetPriorityClass()` documentation. Valid priorities are: + + - *REALTIME_PRIORITY_CLASS* + - *HIGH_PRIORITY_CLASS* + - *ABOVE_NORMAL_PRIORITY_CLASS* + - *NORMAL_PRIORITY_CLASS* + - *BELOW_NORMAL_PRIORITY_CLASS* + - *IDLE_PRIORITY_CLASS* + +### Non-standard native parameters + +- When used with `nssm set`, the **DependOnGroup** and **DependOnService** parameters treat each subsequent command line argument as a dependency group or service. + + Groups can be specified with or without the `SC_GROUP_IDENTIFIER` prefix (the *+* symbol). Services can be specified via their key name or display name. + + The following two invocations are equivalent: + + ``` + nssm set DependOnService RpcSS LanmanWorkstation + nssm set DependOnService "Remote Procedure Call (RPC)" LanmanWorkstation + ``` + + Groups will always be prefixed by the `SC_GROUP_IDENTIFIER` when queried with `nssm get`. + +- When used with `nssm set`, the **ObjectName** parameter requires an additional argument specifying the password of the user which will run the service. + + To retrieve the username, run: + + ``` + nssm get ObjectName + ``` + + To set the username and password, run: + + ``` + nssm set ObjectName + ``` + + Note that the rules of argument concatenation still apply. The following invocation will have the expected effect: + + ``` + nssm set ObjectName correct horse battery staple + ``` + + If you absolutely must configure an account with a blank password, run: + + ``` + nssm set ObjectName "" + ``` + +- Valid values for the **Start** parameter are: + + - *SERVICE_AUTO_START:* Automatic startup at boot. + - *SERVICE_DELAYED_AUTO_START:* Delayed startup at boot. + - *SERVICE_DEMAND_START:* Manual startup. + - *SERVICE_DISABLED:* Service is disabled. + + Note that *SERVICE_DELAYED_AUTO_START* is not supported on versions of Windows prior to Vista. NSSM will set the service to automatic startup if delayed start is unavailable. + +- The **Type** parameter is used to query or set the service type. NSSM recognises all currently documented service types but will only allow setting one of two types: + + - *SERVICE_WIN32_OWN_PROCESS:* A standalone service. This is the default. + - *SERVICE_INTERACTIVE_PROCESS:* A service which can interact with the desktop. + + A service may only be configured as interactive if it runs under the *LOCALSYSTEM* account. To guarantee success when attempting to configure an interactive service, run two commands in sequence: + + ``` + nssm reset ObjectName + nssm set Type SERVICE_INTERACTIVE_PROCESS + ``` diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e7d4451 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.venv/ +__pycache__/ +*.pyc +logs/ +*.log +.pytest_cache/ +config.yaml +config.local.yaml diff --git a/README.md b/README.md new file mode 100644 index 0000000..48c9505 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# ProductionDataBaseSync_DataMacro + +Access → SQL Server 增量同步（数据宏驱动）。详见 `docs/superpowers/specs/2026-07-14-access-datamacro-sync-design.md`。 + +## 运行 +```bash +.venv/Scripts/python.exe -m sync.service +``` + +## 配置 +编辑 `config.yaml`（从 `config.example.yaml` 复制并填入真实凭据）。 diff --git a/docs/superpowers/plans/2026-07-14-access-datamacro-sync.md b/docs/superpowers/plans/2026-07-14-access-datamacro-sync.md new file mode 100644 index 0000000..ccc98e6 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-access-datamacro-sync.md @@ -0,0 +1,1235 @@ +# Access → SQL Server 数据宏增量同步 实现计划 + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 构建一个 Python 同步服务（host 114, NSSM 常驻），读取各 Access 后端的本地 `TableChangeLog`（由数据宏写入），经 SQL Server `SyncQueue` 暂存，由存储过程集合化 MERGE/DELETE 同步到业务表，成功后清理 Access 日志。 + +**Architecture:** Capture（Python+pyodbc 读 Access 日志→回读整行→去重写入 `dbo.SyncQueue`）→ Apply（`dbo.usp_SyncApply` 存储过程按目标表保序 MERGE/DELETE，幂等以 ID 为键）→ Cleanup（Python 回删已 applied 的 Access 日志行）。配置全部在 `config.yaml`。无水位线表——Access 日志"应用成功即删"，`SyncQueue` 唯一索引去重。 + +**Tech Stack:** Python 3.10+、pyodbc（Access ACE 驱动 + SQL Server ODBC Driver 17）、PyYAML、Pydantic v2、pytest；SQL Server（STRING_AGG 需 2017+）；NSSM Windows 服务。 + +## Global Constraints + +- **当前阶段运行主机**：当前开发主机（经 UNC `\\192.168.110.114\生产进度表\` 读 Access、连 114 上 SQL Server）。**部署到 host 114（`C:\Users\peng\Projects\ProductionDataBaseSync_DataMacro`）+ NSSM 服务化 + VBA 切换均延后**，待流程在当前主机验证通过后再做。 +- **Python 执行**：全程在项目 `.venv` 虚拟环境内（CLAUDE.md 规定）。 +- **ODBC 驱动前置**（114 上需已安装，非 pip 可装）： + - `Microsoft Access Driver (*.accdb, *.mdb)`（ACE Redist 2016） + - `ODBC Driver 17 for SQL Server` +- **SQL Server**：`192.168.110.114,1433`，DB `CompanyDB`，账号 `peng`（需对目标表有 `ALTER` 权限以 `SET IDENTITY_INSERT`）。连接串来自 `config.yaml`，**勿硬编码凭据**。 +- **SQL Server 版本**：≥ 2017（用 `STRING_AGG ... WITHIN GROUP`）。 +- **排序规则**：新表 nvarchar 列继承 DB 默认 `Chinese_PRC_CI_AS`。 +- **类型映射**（Access→SQL）：AutoNumber→`INT IDENTITY(1,1) PK`；Text(255)→`NVARCHAR(255)`；Memo→`NVARCHAR(MAX)`；Long→`INT`；Date/Time→`DATETIME2`；Boolean→`BIT`；Currency→`MONEY`；Double→`FLOAT`。 +- **年份映射**：源文件 `2026年数据\`→目标表加 `_YEAR2026` 后缀；`2025年数据\`→无后缀；合同表（表名含年份）→同名无后缀。 +- **幂等**：upsert/delete 均以 `ID` 为键，可重放、可与旧 VBA 并行。 +- **日志保留**：Access `TableChangeLog` 应用成功即删；`dbo.SyncQueue` 长期保留（审计+重试），老 applied 行定期归档（运维任务，本期不实现）。 +- **4 个文件忽略**：缺料数据、精密表记录、技术部、成品物料号（无业务数据）。 +- **排除表**：每库的 `TableChangeLog`（日志表本身）、`*_停` 表、`dbo_TableChangeLog`、`USysApplicationLog` 不参与同步。 +- **Memo 长度**：实测最大 2144 字符 < 4000，故 apply 用 `JSON_VALUE`（nvarchar(4000)）即可；capture 侧对 >4000 的值打 WARNING。 +- **本期执行范围**：Task 1–7（构建）+ Task 9（pilot 验证于当前主机）。Task 8（NSSM）、Task 10（部署/切换）延后，不执行。 + +--- + +## File Structure + +``` +D:\projects\ProductionDataBaseSync_DataMacro\ + config.yaml # 所有配置（连接串、路径、映射、运行参数） + requirements.txt + .gitignore + README.md + sql\ + 01_sync_queue.sql # CREATE TABLE dbo.SyncQueue + 索引 + 02_sync_apply.sql # CREATE PROC dbo.usp_SyncApply + src\sync\ + __init__.py + config.py # Pydantic 模型 + load_config() + serialize.py # to_jsonable()：Access 值→JSON 可序列化 + access_reader.py # AccessReader：read_log()/read_row() + sql_writer.py # SqlWriter：insert_queue_row()/call_apply()/applied_log_ids() + capture.py # capture_file()：读日志→回读整行→写 SyncQueue + cleanup.py # cleanup_file()：回删 Access 已应用日志 + service.py # run()：主循环 capture→apply→cleanup + logging_setup.py # 日志配置 + tests\ + conftest.py # pytest fixtures（含集成测试跳过标记） + test_config.py + test_serialize.py + test_apply_proc.py # 集成：验证存储过程逻辑 + test_access_reader.py # 集成：验证 Access 读取 + test_sql_writer.py # 集成：验证 SyncQueue 写入/调用/清理 + test_capture.py # 单元：capture 编排（mock reader/writer） + scripts\ + install_service.bat # NSSM 安装脚本 +``` + +每个文件单一职责；`capture.py`/`cleanup.py` 编排，`access_reader.py`/`sql_writer.py` 封装外部 IO，`service.py` 主循环。 + +--- + +## Task 1: 项目脚手架 + +**Files:** +- Create: `requirements.txt`, `.gitignore`, `README.md`, `src/sync/__init__.py`, `tests/__init__.py` + +**Interfaces:** +- Produces: 可用的 `.venv` 与依赖；空包结构。 + +- [ ] **Step 1: 创建目录结构与 .gitignore** + +`D:\projects\ProductionDataBaseSync_DataMacro\.gitignore`: +``` +.venv/ +__pycache__/ +*.pyc +logs/ +*.log +.pytest_cache/ +config.local.yaml +``` + +`src/sync/__init__.py`（空文件）和 `tests/__init__.py`（空文件）。 + +- [ ] **Step 2: requirements.txt** + +`requirements.txt`: +``` +pyodbc>=5.0.1 +PyYAML>=6.0.1 +pydantic>=2.6.0 +pytest>=8.0.0 +``` + +- [ ] **Step 3: 创建 venv 并安装依赖** + +Run: +```bash +cd /d/projects/ProductionDataBaseSync_DataMacro +python -m venv .venv +.venv/Scripts/python.exe -m pip install --upgrade pip +.venv/Scripts/python.exe -m pip install -r requirements.txt +.venv/Scripts/python.exe -m pytest --version +``` +Expected: 输出 pytest 版本号，无错误。 + +- [ ] **Step 4: README.md** + +`README.md`: +````markdown +# ProductionDataBaseSync_DataMacro + +Access → SQL Server 增量同步（数据宏驱动）。详见 `docs/superpowers/specs/2026-07-14-access-datamacro-sync-design.md`。 + +## 运行 +```bash +.venv/Scripts/python.exe -m sync.service +``` + +## 配置 +编辑 `config.yaml`。 +```` + +- [ ] **Step 5: git 初始化与首次提交** + +```bash +cd /d/projects/ProductionDataBaseSync_DataMacro +git init +git add . +git commit -m "chore: project scaffold for access-datamacro sync" +``` +Expected: 首次提交成功。 + +--- + +## Task 2: 配置模型与加载器 + +**Files:** +- Create: `config.yaml`, `src/sync/config.py`, `tests/test_config.py` + +**Interfaces:** +- Produces: `load_config(path: str) -> SyncConfig`；`SyncConfig` 含 `.sql_server.conn_str`、`.access.driver`、`.access.roots`、`.runtime.*`、`.files: list[FileMapping]`；`FileMapping` 含 `.file`、`.root`、`.schema`、`.year_suffix`、`.exclude_tables`、`.include_tables`、`source_path()`。 + +- [ ] **Step 1: 写失败测试** + +`tests/test_config.py`: +```python +import pathlib, textwrap +from sync.config import load_config, FileMapping + +def test_load_config_parses_fields(tmp_path): + cfg_text = textwrap.dedent(""" + sql_server: + conn_str: "Driver={ODBC Driver 17 for SQL Server};Server=s;Database=d;UID=u;PWD=p;" + sync_queue_table: "dbo.SyncQueue" + access: + driver: "{Microsoft Access Driver (*.accdb, *.mdb)}" + roots: + 2026: "\\\\\\\\srv\\\\2026" + 2025: "\\\\\\\\srv\\\\2025" + runtime: + poll_interval_seconds: 10 + capture_batch_size: 500 + apply_batch_size: 200 + max_retries: 5 + retry_backoff_seconds: 30 + cleanup_batch_size: 200 + cleanup_lock_retries: 3 + files: + - file: "氩弧焊.accdb" + root: 2026 + schema: "TIGWelding" + year_suffix: "_YEAR2026" + exclude_tables: ["TableChangeLog", "氩弧焊每日催货落实记录_停"] + """) + p = tmp_path / "config.yaml" + p.write_text(cfg_text, encoding="utf-8") + cfg = load_config(str(p)) + assert cfg.sql_server.sync_queue_table == "dbo.SyncQueue" + assert cfg.runtime.poll_interval_seconds == 10 + assert cfg.files[0].schema == "TIGWelding" + assert cfg.files[0].year_suffix == "_YEAR2026" + assert "2026" in cfg.files[0].source_path(cfg) + +def test_file_mapping_target_table_applies_year_suffix(): + fm = FileMapping(file="x.accdb", root="2026", schema="s", year_suffix="_YEAR2026", + exclude_tables=[], include_tables=None) + assert fm.target_table("一车间记录") == "一车间记录_YEAR2026" + fm2 = FileMapping(file="y.accdb", root="2025", schema="s", year_suffix="", + exclude_tables=[], include_tables=None) + assert fm2.target_table("26年压力表合同数据") == "26年压力表合同数据" +``` + +- [ ] **Step 2: 运行测试，确认失败** + +Run: `.venv/Scripts/python.exe -m pytest tests/test_config.py -v` +Expected: FAIL（`ModuleNotFoundError: No module named 'sync'` 或导入错误）。 + +- [ ] **Step 3: 在项目根建 pyproject.toml 让 src 可导入** + +`pyproject.toml`: +```toml +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] +markers = ["integration: marks tests requiring real Access/SQL Server"] +``` + +- [ ] **Step 4: 实现 config.py** + +`src/sync/config.py`: +```python +from __future__ import annotations +from pathlib import Path +import yaml +from pydantic import BaseModel, Field + +class SqlServerConfig(BaseModel): + conn_str: str + sync_queue_table: str = "dbo.SyncQueue" + +class AccessConfig(BaseModel): + driver: str + roots: dict[str, str] + +class RuntimeConfig(BaseModel): + poll_interval_seconds: int = 10 + capture_batch_size: int = 500 + apply_batch_size: int = 200 + max_retries: int = 5 + retry_backoff_seconds: int = 30 + cleanup_batch_size: int = 200 + cleanup_lock_retries: int = 3 + +class FileMapping(BaseModel): + file: str + root: str + schema: str + year_suffix: str = "" + exclude_tables: list[str] = Field(default_factory=list) + include_tables: list[str] | None = None + + def source_path(self, cfg: "SyncConfig") -> str: + base = cfg.access.roots[self.root] + return f"{base}\\{self.file}" + + def target_table(self, access_table: str) -> str: + return f"{access_table}{self.year_suffix}" + +class SyncConfig(BaseModel): + sql_server: SqlServerConfig + access: AccessConfig + runtime: RuntimeConfig + files: list[FileMapping] + logging: dict | None = None + +def load_config(path: str) -> SyncConfig: + with open(path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + return SyncConfig(**data) +``` + +- [ ] **Step 5: 运行测试，确认通过** + +Run: `.venv/Scripts/python.exe -m pytest tests/test_config.py -v` +Expected: 2 passed。 + +- [ ] **Step 6: 写真实 config.yaml** + +`config.yaml`（完整在作用域文件清单；凭据用占位 `${...}` 由部署时替换，或直接填——本机即 peng 账号）: +```yaml +sql_server: + conn_str: "Driver={ODBC Driver 17 for SQL Server};Server=192.168.110.114,1433;Database=CompanyDB;UID=peng;PWD=Cqbld123456.;Encrypt=yes;TrustServerCertificate=yes;" + sync_queue_table: "dbo.SyncQueue" + +access: + driver: "{Microsoft Access Driver (*.accdb, *.mdb)}" + roots: + 2026: "\\\\192.168.110.114\\生产进度表\\2026年数据" + 2025: "\\\\192.168.110.114\\生产进度表\\2025年数据" + +runtime: + poll_interval_seconds: 10 + capture_batch_size: 500 + apply_batch_size: 200 + max_retries: 5 + retry_backoff_seconds: 30 + cleanup_batch_size: 200 + cleanup_lock_retries: 3 + +logging: + level: INFO + path: "D:\\projects\\ProductionDataBaseSync_DataMacro\\logs\\sync.log" + +files: + - {file: "一车间.accdb", root: 2026, schema: "workshopOne", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "一车间每日催货落实记录_停"]} + - {file: "二车间.accdb", root: 2026, schema: "workshopTwo", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "三车间.accdb", root: 2026, schema: "workshopThree", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "弯管车间.accdb", root: 2026, schema: "tubeBending", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "氩弧焊.accdb", root: 2026, schema: "TIGWelding", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "氩弧焊每日催货落实记录_停"]} + - {file: "机加工.accdb", root: 2026, schema: "machining", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "零件库.accdb", root: 2026, schema: "partsWarehouse", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "成品入库.accdb", root: 2026, schema: "productWarehousing", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "检验记录数据库.accdb", root: 2026, schema: "inspectionRecords", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"], include_tables: ["检验合格记录表"]} + - {file: "温度计记录.accdb", root: 2026, schema: "thermometerRecord", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "锡焊数据.accdb", root: 2026, schema: "solderingData", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "计划.accdb", root: 2026, schema: "contractPlanning", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "每日催货合同号_停", "每日催货缺件落实记录_停"]} + - {file: "隔膜数据.accdb", root: 2026, schema: "diaphragmData", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "执行卡下发记录.accdb", root: 2026, schema: "executionCardIssuanceRecord", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - {file: "OEM.accdb", root: 2026, schema: "OEM", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]} + - file: "生产合同数据.accdb" + root: 2025 + schema: "productionContractData" + year_suffix: "" + exclude_tables: ["TableChangeLog", "dbo_TableChangeLog", "USysApplicationLog", "温度计数据_修复"] +``` +> 注：`include_tables` 未列出则同步该库除 `exclude_tables` 外的全部业务表。 + +- [ ] **Step 7: 提交** + +```bash +git add config.yaml src/sync/config.py tests/test_config.py pyproject.toml +git commit -m "feat: config model and yaml loader" +``` + +--- + +## Task 3: SyncQueue 表与 usp_SyncApply 存储过程 + +**Files:** +- Create: `sql/01_sync_queue.sql`, `sql/02_sync_apply.sql`, `tests/test_apply_proc.py` + +**Interfaces:** +- Consumes: `SyncConfig.sql_server` +- Produces: SQL 端 `dbo.SyncQueue` 表 + `dbo.usp_SyncApply` 过程。 + +- [ ] **Step 1: 写 SyncQueue 建表脚本** + +`sql/01_sync_queue.sql`: +```sql +IF OBJECT_ID('dbo.SyncQueue','U') IS NULL +CREATE TABLE dbo.SyncQueue ( + QueueID bigint IDENTITY(1,1) NOT NULL, + SourceFile nvarchar(255) NOT NULL, + SourceTable nvarchar(255) NOT NULL, + SourceLogID bigint NOT NULL, + TargetSchema nvarchar(128) NOT NULL, + TargetTable nvarchar(255) NOT NULL, + RecordID nvarchar(50) NOT NULL, + OperateType varchar(10) NOT NULL, + RowData nvarchar(max) NULL, + Status varchar(10) NOT NULL CONSTRAINT DF_SyncQueue_Status DEFAULT 'pending', + RetryCount int NOT NULL CONSTRAINT DF_SyncQueue_Retry DEFAULT 0, + ErrorMsg nvarchar(max) NULL, + CapturedAt datetime2 NOT NULL CONSTRAINT DF_SyncQueue_Captured DEFAULT sysdatetime(), + AppliedAt datetime2 NULL, + CONSTRAINT PK_SyncQueue PRIMARY KEY CLUSTERED (QueueID) +); +CREATE UNIQUE INDEX UX_SyncQueue_Dedup ON dbo.SyncQueue(SourceFile, SourceTable, SourceLogID); +CREATE INDEX IX_SyncQueue_Pending ON dbo.SyncQueue(Status, TargetSchema, TargetTable); +``` + +- [ ] **Step 2: 写 usp_SyncApply 存储过程** + +`sql/02_sync_apply.sql`: +```sql +CREATE OR ALTER PROCEDURE dbo.usp_SyncApply + @MaxRetries INT = 5 +AS +BEGIN + SET NOCOUNT ON; + DECLARE @sch NVARCHAR(128), @tbl NVARCHAR(255), @FullName NVARCHAR(514); + DECLARE @sql NVARCHAR(MAX), @cols NVARCHAR(MAX), @upd NVARCHAR(MAX), @ins NVARCHAR(MAX); + + -- 重试：error 且未超限 → 重置 pending + UPDATE dbo.SyncQueue SET Status='pending' + WHERE Status='error' AND RetryCount < @MaxRetries; + + DECLARE cur CURSOR LOCAL FAST_FORWARD FOR + SELECT DISTINCT TargetSchema, TargetTable + FROM dbo.SyncQueue WHERE Status='pending'; + OPEN cur; + FETCH NEXT FROM cur INTO @sch, @tbl; + WHILE @@FETCH_STATUS = 0 + BEGIN + SET @FullName = QUOTENAME(@sch) + N'.' + QUOTENAME(@tbl); + BEGIN TRY + BEGIN TRAN; + + -- 非键列（排除 ID 键、computed、identity、rowversion） + SELECT + @cols = STRING_AGG(QUOTENAME(c.name), N',') WITHIN GROUP (ORDER BY c.column_id), + @upd = STRING_AGG(QUOTENAME(c.name) + N'=JSON_VALUE(src.RowData,''$.' + c.name + N''')', N',') WITHIN GROUP (ORDER BY c.column_id), + @ins = STRING_AGG(N'JSON_VALUE(src.RowData,''$.' + c.name + N''')', N',') WITHIN GROUP (ORDER BY c.column_id) + FROM sys.columns c + WHERE c.object_id = OBJECT_ID(@FullName) + AND c.is_computed = 0 + AND c.is_identity = 0 + AND TYPE_NAME(c.system_type_id) <> 'timestamp' + AND c.name <> 'ID'; + + IF @cols IS NOT NULL + BEGIN + -- Upsert（最后操作为 Insert/Update），保序"最后操作胜" + SET @sql = N'SET IDENTITY_INSERT ' + @FullName + N' ON; + MERGE ' + @FullName + N' WITH (HOLDLOCK) AS tgt + USING ( + SELECT RecordID, RowData FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn + FROM dbo.SyncQueue + WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'' + AND OperateType IN (''Insert'',''Update'') AND RowData IS NOT NULL + ) x WHERE rn=1 + ) AS src ON tgt.ID = TRY_CAST(src.RecordID AS int) + WHEN MATCHED THEN UPDATE SET ' + @upd + N' + WHEN NOT MATCHED THEN INSERT (ID,' + @cols + N') VALUES (TRY_CAST(src.RecordID AS int),' + @ins + N'); + SET IDENTITY_INSERT ' + @FullName + N' OFF;'; + EXEC sp_executesql @sql, N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl; + + -- Delete（最后操作为 Delete） + SET @sql = N'DELETE t FROM ' + @FullName + N' t + JOIN ( + SELECT RecordID FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn + FROM dbo.SyncQueue + WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'' + AND OperateType=''Delete'' + ) x WHERE rn=1 + ) d ON t.ID = TRY_CAST(d.RecordID AS int);'; + EXEC sp_executesql @sql, N'@sch NVARCHAR(128),@tbl NVARCHAR(255)', @sch, @tbl; + END + + UPDATE dbo.SyncQueue SET Status='applied', AppliedAt=SYSDATETIME() + WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status='pending'; + COMMIT; + END TRY + BEGIN CATCH + IF @@TRANCOUNT > 0 ROLLBACK; + UPDATE dbo.SyncQueue + SET Status = CASE WHEN RetryCount + 1 >= @MaxRetries THEN 'dead' ELSE 'error' END, + RetryCount = RetryCount + 1, + ErrorMsg = ERROR_MESSAGE() + WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status='pending'; + END CATCH + FETCH NEXT FROM cur INTO @sch, @tbl; + END + CLOSE cur; DEALLOCATE cur; +END +``` + +- [ ] **Step 3: 在 CompanyDB 执行两脚本** + +Run: +```bash +sqlcmd -S 192.168.110.114,1433 -U peng -P "Cqbld123456." -d CompanyDB -C -N o -i sql/01_sync_queue.sql +sqlcmd -S 192.168.110.114,1433 -U peng -P "Cqbld123456." -d CompanyDB -C -N o -i sql/02_sync_apply.sql +``` +Expected: 各命令无错误输出（成功无回显）。 + +- [ ] **Step 4: 验证对象存在** + +Run: +```bash +sqlcmd -S 192.168.110.114,1433 -U peng -P "Cqbld123456." -d CompanyDB -C -N o -Q "SELECT name FROM sys.tables WHERE name='SyncQueue'; SELECT name FROM sys.procedures WHERE name='usp_SyncApply'" +``` +Expected: 两行 `SyncQueue`、`usp_SyncApply`。 + +- [ ] **Step 5: 写存储过程集成测试（失败先）** + +`tests/conftest.py`: +```python +import os, pytest +import pyodbc + +CONN = ("Driver={ODBC Driver 17 for SQL Server};Server=192.168.110.114,1433;" + "Database=CompanyDB;UID=peng;PWD=Cqbld123456.;Encrypt=yes;TrustServerCertificate=yes;") + +@pytest.fixture +def sql_conn(): + if not os.environ.get("RUN_INTEGRATION"): + pytest.skip("set RUN_INTEGRATION=1 to run SQL integration tests") + c = pyodbc.connect(CONN, autocommit=False) + yield c + c.rollback() + c.close() +``` + +`tests/test_apply_proc.py`: +```python +import pytest + +@pytest.mark.integration +def test_upsert_insert_update_delete_last_write_wins(sql_conn): + cur = sql_conn.cursor() + sch, tbl = "sync_test", "ApplyDemo_YEAR2026" + cur.execute(f"IF SCHEMA_ID('sync_test') IS NULL EXEC('CREATE SCHEMA sync_test')") + cur.execute(f"IF OBJECT_ID('sync_test.ApplyDemo_YEAR2026') IS NOT NULL DROP TABLE sync_test.ApplyDemo_YEAR2026") + cur.execute(f"CREATE TABLE sync_test.ApplyDemo_YEAR2026 (ID INT IDENTITY(1,1) PRIMARY KEY, 名字 NVARCHAR(255) NULL, 数量 INT NULL, 时间 DATETIME2 NULL, 标记 BIT NULL)") + # 清空相关 SyncQueue 行 + cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'") + # Insert ID=1 + cur.execute("INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,TargetTable,RecordID,OperateType,RowData,Status) VALUES('t.accdb','ApplyDemo',1,'sync_test','ApplyDemo_YEAR2026','1','Insert','{\"ID\":1,\"名字\":\"A\",\"数量\":3,\"时间\":\"2026-01-01T00:00:00\",\"标记\":true}','pending')") + # Update ID=1 (last op wins) + cur.execute("INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,TargetTable,RecordID,OperateType,RowData,Status) VALUES('t.accdb','ApplyDemo',2,'sync_test','ApplyDemo_YEAR2026','1','Update','{\"ID\":1,\"名字\":\"A2\",\"数量\":5,\"时间\":\"2026-01-02T00:00:00\",\"标记\":false}','pending')") + sql_conn.commit() + + cur.execute("EXEC dbo.usp_SyncApply @MaxRetries=5") + sql_conn.commit() + + cur.execute("SELECT 名字,数量,标记 FROM sync_test.ApplyDemo_YEAR2026 WHERE ID=1") + row = cur.fetchone() + assert row.名字 == "A2" and row.数量 == 5 and row.标记 == 0 # last-write-wins, bit cast + + # Delete ID=1 + cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'") + cur.execute("INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,TargetTable,RecordID,OperateType,RowData,Status) VALUES('t.accdb','ApplyDemo',3,'sync_test','ApplyDemo_YEAR2026','1','Delete',NULL,'pending')") + sql_conn.commit() + cur.execute("EXEC dbo.usp_SyncApply @MaxRetries=5") + sql_conn.commit() + cur.execute("SELECT COUNT(*) FROM sync_test.ApplyDemo_YEAR2026 WHERE ID=1") + assert cur.fetchone()[0] == 0 + + cur.execute("IF OBJECT_ID('sync_test.ApplyDemo_YEAR2026') IS NOT NULL DROP TABLE sync_test.ApplyDemo_YEAR2026") + cur.execute("DELETE dbo.SyncQueue WHERE TargetSchema='sync_test'") + sql_conn.commit() +``` + +- [ ] **Step 6: 运行集成测试** + +Run: `RUN_INTEGRATION=1 .venv/Scripts/python.exe -m pytest tests/test_apply_proc.py -v` +Expected: PASS（验证 upsert/保序/delete/bit 转换/IDENTITY 保留）。 + +- [ ] **Step 7: 提交** + +```bash +git add sql/ tests/test_apply_proc.py tests/conftest.py +git commit -m "feat: SyncQueue table and set-based apply stored procedure" +``` + +--- + +## Task 4: Access 读取器与值序列化 + +**Files:** +- Create: `src/sync/serialize.py`, `src/sync/access_reader.py`, `tests/test_serialize.py`, `tests/test_access_reader.py` + +**Interfaces:** +- Consumes: `AccessConfig.driver` +- Produces: `to_jsonable(v)`；`AccessReader(db_path, driver)` 含 `.read_log(batch_size) -> list[LogRow]`、`.read_row(table, record_id) -> dict|None`、`.delete_log_ids(ids, batch_size, retries)`、`.close()`；`LogRow` 为 dataclass `(id, table_name, record_id, operate_type, time)`。 + +- [ ] **Step 1: 写 serialize 失败测试** + +`tests/test_serialize.py`: +```python +import datetime, decimal +from sync.serialize import to_jsonable +import json + +def test_datetime_iso(): + assert to_jsonable(datetime.datetime(2026,7,14,8,41,18)) == "2026-07-14T08:41:18" + +def test_bool_preserved(): + assert to_jsonable(True) is True and to_jsonable(False) is False + +def test_decimal_to_str(): + assert to_jsonable(decimal.Decimal("12.50")) == "12.50" + +def test_none_and_numbers(): + assert to_jsonable(None) is None + assert to_jsonable(5) == 5 + assert to_jsonable("x") == "x" + +def test_dict_serializes(): + d = {"d": datetime.date(2026,1,1), "b": True, "n": None} + assert json.loads(json.dumps({k: to_jsonable(v) for k,v in d.items()})) == {"d":"2026-01-01","b":True,"n":None} +``` + +- [ ] **Step 2: 运行，确认失败** + +Run: `.venv/Scripts/python.exe -m pytest tests/test_serialize.py -v` +Expected: FAIL（模块不存在）。 + +- [ ] **Step 3: 实现 serialize.py** + +`src/sync/serialize.py`: +```python +import datetime, decimal + +def to_jsonable(v): + if v is None: + return None + if isinstance(v, bool): # 必须在 int 之前 + return v + if isinstance(v, (datetime.datetime, datetime.date)): + return v.isoformat() + if isinstance(v, decimal.Decimal): + return str(v) + if isinstance(v, (int, float, str)): + return v + return str(v) +``` + +- [ ] **Step 4: 运行，确认通过** + +Run: `.venv/Scripts/python.exe -m pytest tests/test_serialize.py -v` +Expected: 5 passed。 + +- [ ] **Step 5: 实现 access_reader.py** + +`src/sync/access_reader.py`: +```python +from __future__ import annotations +import json, logging, time +from dataclasses import dataclass +import pyodbc +from .serialize import to_jsonable + +log = logging.getLogger(__name__) + +@dataclass +class LogRow: + id: int + table_name: str + record_id: str + operate_type: str + time: object + +class AccessReader: + def __init__(self, db_path: str, driver: str): + self.db_path = db_path + self.driver = driver + self._conn = None + + def _connect(self): + if self._conn is None: + conn_str = f"Driver={self.driver};DBQ={self.db_path};ReadOnly=0;" + self._conn = pyodbc.connect(conn_str, autocommit=True) + return self._conn + + def read_log(self, batch_size: int) -> list[LogRow]: + cur = self._connect().cursor() + cur.execute(f"SELECT TOP {int(batch_size)} ID, TableName, RecordID, OperateType, Time " + f"FROM TableChangeLog ORDER BY ID") + return [LogRow(r[0], r[1], str(r[2]), (r[3] or "").strip(), r[4]) for r in cur.fetchall()] + + def read_row(self, table: str, record_id: str) -> dict | None: + cur = self._connect().cursor() + cur.execute(f'SELECT * FROM "{table}" WHERE ID = ?', record_id) + cols = [c[0] for c in cur.description] + row = cur.fetchone() + if row is None: + return None + d = {cols[i]: to_jsonable(row[i]) for i in range(len(cols))} + # 防御：超长值告警（JSON_VALUE 上限 4000） + for k, v in d.items(): + if isinstance(v, str) and len(v) > 4000: + log.warning("value >4000 chars in %s.ID=%s col=%s (JSON_VALUE will truncate)", table, record_id, k) + return d + + def delete_log_ids(self, ids: list[int], batch_size: int, retries: int): + if not ids: + return + conn = self._connect() + cur = conn.cursor() + for i in range(0, len(ids), batch_size): + chunk = ids[i:i+batch_size] + placeholders = ",".join("?" * len(chunk)) + for attempt in range(retries): + try: + cur.execute(f"DELETE FROM TableChangeLog WHERE ID IN ({placeholders})", *chunk) + break + except pyodbc.OperationalError as e: + if attempt < retries - 1: + time.sleep(0.2 * (attempt + 1)) + else: + raise + + def close(self): + if self._conn: + self._conn.close() + self._conn = None +``` + +- [ ] **Step 6: 写 Access 集成测试** + +`tests/test_access_reader.py`: +```python +import os, pytest, pyodbc +from sync.access_reader import AccessReader + +DRIVER = "{Microsoft Access Driver (*.accdb, *.mdb)}" +TEST_DB = os.environ.get("TEST_ACCDB") # 指向一个测试用 .accdb + +@pytest.mark.integration +def test_read_log_and_row_and_delete(): + if not os.environ.get("RUN_INTEGRATION") or not TEST_DB: + pytest.skip("needs RUN_INTEGRATION=1 and TEST_ACCDB=path") + r = AccessReader(TEST_DB, DRIVER) + rows = r.read_log(500) + assert isinstance(rows, list) + if rows: + lr = rows[0] + d = r.read_row(lr.table_name, lr.record_id) + assert d is None or "ID" in d + r.delete_log_ids([], 100, 3) # 空列表不报错 + r.close() +``` + +> 测试用 .accdb 由执行者准备：复制一份含数据宏的小库（或用 pilot 前的 氩弧焊 副本），确保有 TableChangeLog 与业务表。 + +- [ ] **Step 7: 运行（集成，可选先跳过）+ 提交** + +Run: `RUN_INTEGRATION=1 TEST_ACCDB= .venv/Scripts/python.exe -m pytest tests/test_access_reader.py -v`（暂可 skip） +```bash +git add src/sync/serialize.py src/sync/access_reader.py tests/test_serialize.py tests/test_access_reader.py +git commit -m "feat: access reader and value serialization" +``` + +--- + +## Task 5: SqlWriter（SyncQueue 写入/调用/查询） + +**Files:** +- Create: `src/sync/sql_writer.py` + +**Interfaces:** +- Consumes: `SqlServerConfig.conn_str`, `sync_queue_table` +- Produces: `SqlWriter(conn_str, queue_table)` 含 `.insert_queue_row(row: QueueRow) -> None`（去重 INSERT）、`.call_apply(max_retries) -> None`、`.applied_log_ids(source_file) -> list[int]`、`.close()`；`QueueRow` dataclass。 + +- [ ] **Step 1: 写 SqlWriter 集成测试（失败先）** + +`tests/test_sql_writer.py`: +```python +import os, pytest +from sync.sql_writer import SqlWriter, QueueRow + +@pytest.mark.integration +def test_insert_dedup_and_applied_ids(sql_conn): + if not os.environ.get("RUN_INTEGRATION"): + pytest.skip("integration") + w = SqlWriter(("Driver={ODBC Driver 17 for SQL Server};Server=192.168.110.114,1433;" + "Database=CompanyDB;UID=peng;PWD=Cqbld123456.;Encrypt=yes;TrustServerCertificate=yes;"), + "dbo.SyncQueue") + cur = w._conn.cursor() + cur.execute("DELETE dbo.SyncQueue WHERE SourceFile='sqlw_test.accdb'") + w._conn.commit() + qr = QueueRow(source_file="sqlw_test.accdb", source_table="T", record_id="7", + target_schema="sync_test", target_table="T_YEAR2026", + source_log_id=100, operate_type="Insert", row_data='{"ID":7}') + w.insert_queue_row(qr) + w.insert_queue_row(qr) # 重复应被忽略 + cur.execute("SELECT COUNT(*) FROM dbo.SyncQueue WHERE SourceFile='sqlw_test.accdb' AND SourceLogID=100") + assert cur.fetchone()[0] == 1 + # 模拟 applied + cur.execute("UPDATE dbo.SyncQueue SET Status='applied' WHERE SourceFile='sqlw_test.accdb'") + w._conn.commit() + assert w.applied_log_ids("sqlw_test.accdb") == [100] + cur.execute("DELETE dbo.SyncQueue WHERE SourceFile='sqlw_test.accdb'") + w._conn.commit() + w.close() +``` + +- [ ] **Step 2: 实现 sql_writer.py** + +`src/sync/sql_writer.py`: +```python +from __future__ import annotations +from dataclasses import dataclass +import pyodbc + +@dataclass +class QueueRow: + source_file: str + source_table: str + record_id: str + target_schema: str + target_table: str + source_log_id: int + operate_type: str + row_data: str | None + +class SqlWriter: + def __init__(self, conn_str: str, queue_table: str = "dbo.SyncQueue"): + self.conn_str = conn_str + self.queue_table = queue_table + self._conn = pyodbc.connect(conn_str, autocommit=False) + + def insert_queue_row(self, row: QueueRow): + cur = self._conn.cursor() + cur.execute( + "IF NOT EXISTS (SELECT 1 FROM dbo.SyncQueue " + "WHERE SourceFile=? AND SourceTable=? AND SourceLogID=?) " + "INSERT dbo.SyncQueue(SourceFile,SourceTable,SourceLogID,TargetSchema,TargetTable," + "RecordID,OperateType,RowData,Status) VALUES (?,?,?,?,?,?,?,?, 'pending')", + row.source_file, row.source_table, row.source_log_id, + row.target_schema, row.target_table, row.record_id, + row.operate_type, row.row_data) + self._conn.commit() + + def call_apply(self, max_retries: int): + cur = self._conn.cursor() + cur.execute("EXEC dbo.usp_SyncApply ?", max_retries) + self._conn.commit() + + def applied_log_ids(self, source_file: str) -> list[int]: + cur = self._conn.cursor() + cur.execute("SELECT SourceLogID FROM dbo.SyncQueue " + "WHERE SourceFile=? AND Status='applied' ORDER BY SourceLogID", source_file) + return [r[0] for r in cur.fetchall()] + + def close(self): + self._conn.close() +``` + +- [ ] **Step 3: 运行集成测试** + +Run: `RUN_INTEGRATION=1 .venv/Scripts/python.exe -m pytest tests/test_sql_writer.py -v` +Expected: PASS（去重 + applied 查询）。 + +- [ ] **Step 4: 提交** + +```bash +git add src/sync/sql_writer.py tests/test_sql_writer.py +git commit -m "feat: sql writer with dedup insert and apply call" +``` + +--- + +## Task 6: Capture 编排 + +**Files:** +- Create: `src/sync/capture.py`, `tests/test_capture.py` + +**Interfaces:** +- Consumes: `FileMapping`, `AccessReader`, `SqlWriter` +- Produces: `capture_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter, cfg: SyncConfig) -> int`（返回捕获条数）。 + +- [ ] **Step 1: 写 capture 单元测试（mock reader/writer）** + +`tests/test_capture.py`: +```python +from unittest.mock import MagicMock +from sync.config import FileMapping, SyncConfig, AccessConfig, RuntimeConfig, SqlServerConfig +from sync.access_reader import LogRow +from sync.capture import capture_file + +def _cfg(): + return SyncConfig(sql_server=SqlServerConfig(conn_str="x"), + access=AccessConfig(driver="d", roots={"2026":"r"}), + runtime=RuntimeConfig(), + files=[]) + +def test_capture_insert_reads_row_and_queues(): + cfg = _cfg() + fm = FileMapping(file="氩弧焊.accdb", root="2026", schema="TIGWelding", year_suffix="_YEAR2026", exclude_tables=["TableChangeLog"]) + reader = MagicMock() + reader.read_log.return_value = [LogRow(10, "表壳焊接记录", "34041", "Insert", None)] + reader.read_row.return_value = {"ID": 34041, "订单号": "X1"} + writer = MagicMock() + n = capture_file(fm, reader, writer, cfg) + assert n == 1 + args = writer.insert_queue_row.call_args[0][0] + assert args.target_schema == "TIGWelding" + assert args.target_table == "表壳焊接记录_YEAR2026" + assert args.operate_type == "Insert" + assert '"订单号": "X1"' in args.row_data + assert args.source_log_id == 10 + +def test_capture_update_missing_row_downgrades_to_delete(): + cfg = _cfg() + fm = FileMapping(file="x.accdb", root="2026", schema="s", year_suffix="_YEAR2026", exclude_tables=["TableChangeLog"]) + reader = MagicMock() + reader.read_log.return_value = [LogRow(11, "T", "5", "Update", None)] + reader.read_row.return_value = None # 行已删 + writer = MagicMock() + n = capture_file(fm, reader, writer, cfg) + assert n == 1 + args = writer.insert_queue_row.call_args[0][0] + assert args.operate_type == "Delete" + assert args.row_data is None + +def test_capture_skips_excluded_tables(): + cfg = _cfg() + fm = FileMapping(file="x.accdb", root="2026", schema="s", year_suffix="_YEAR2026", exclude_tables=["TableChangeLog", "氩弧焊每日催货落实记录_停"]) + reader = MagicMock() + reader.read_log.return_value = [LogRow(1, "TableChangeLog", "1", "Insert", None), + LogRow(2, "氩弧焊每日催货落实记录_停", "1", "Insert", None)] + writer = MagicMock() + assert capture_file(fm, reader, writer, cfg) == 0 + writer.insert_queue_row.assert_not_called() + +def test_capture_include_tables_filter(): + cfg = _cfg() + fm = FileMapping(file="x.accdb", root="2026", schema="inspectionRecords", year_suffix="_YEAR2026", + exclude_tables=["TableChangeLog"], include_tables=["检验合格记录表"]) + reader = MagicMock() + reader.read_log.return_value = [LogRow(1, "检验合格记录表", "1", "Insert", None), + LogRow(2, "其它表", "1", "Insert", None)] + reader.read_row.return_value = {"ID": 1} + writer = MagicMock() + assert capture_file(fm, reader, writer, cfg) == 1 + assert writer.insert_queue_row.call_args[0][0].target_table == "检验合格记录表_YEAR2026" +``` + +- [ ] **Step 2: 运行，确认失败** + +Run: `.venv/Scripts/python.exe -m pytest tests/test_capture.py -v` +Expected: FAIL（模块不存在）。 + +- [ ] **Step 3: 实现 capture.py** + +`src/sync/capture.py`: +```python +from __future__ import annotations +import json, logging +from .access_reader import AccessReader +from .sql_writer import SqlWriter, QueueRow +from .config import FileMapping, SyncConfig + +log = logging.getLogger(__name__) + +def capture_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter, cfg: SyncConfig) -> int: + exclude = set(fm.exclude_tables or []) + include = set(fm.include_tables) if fm.include_tables else None + rows = reader.read_log(cfg.runtime.capture_batch_size) + n = 0 + for lr in rows: + if lr.table_name in exclude: + continue + if include is not None and lr.table_name not in include: + continue + op = lr.operate_type + row_data = None + if op in ("Insert", "Update"): + d = reader.read_row(lr.table_name, lr.record_id) + if d is None: + op = "Delete" # 行已删，降级 + else: + row_data = json.dumps(d, ensure_ascii=False) + elif op != "Delete": + log.warning("unknown OperateType %r in %s log %s", op, fm.file, lr.id) + continue + qr = QueueRow( + source_file=fm.file, + source_table=lr.table_name, + record_id=lr.record_id, + target_schema=fm.schema, + target_table=fm.target_table(lr.table_name), + source_log_id=lr.id, + operate_type=op, + row_data=row_data, + ) + writer.insert_queue_row(qr) + n += 1 + return n +``` + +- [ ] **Step 4: 运行，确认通过** + +Run: `.venv/Scripts/python.exe -m pytest tests/test_capture.py -v` +Expected: 4 passed。 + +- [ ] **Step 5: 提交** + +```bash +git add src/sync/capture.py tests/test_capture.py +git commit -m "feat: capture orchestration with include/exclude and delete-downgrade" +``` + +--- + +## Task 7: Cleanup 与主服务循环 + +**Files:** +- Create: `src/sync/cleanup.py`, `src/sync/logging_setup.py`, `src/sync/service.py` + +**Interfaces:** +- Consumes: Tasks 4-6 +- Produces: `cleanup_file(fm, reader, writer, cfg) -> int`；`run(cfg)` 主循环。 + +- [ ] **Step 1: 实现 cleanup.py** + +`src/sync/cleanup.py`: +```python +from __future__ import annotations +import logging +from .access_reader import AccessReader +from .sql_writer import SqlWriter +from .config import FileMapping, SyncConfig + +log = logging.getLogger(__name__) + +def cleanup_file(fm: FileMapping, reader: AccessReader, writer: SqlWriter, cfg: SyncConfig) -> int: + ids = writer.applied_log_ids(fm.file) + if not ids: + return 0 + reader.delete_log_ids(ids, cfg.runtime.cleanup_batch_size, cfg.runtime.cleanup_lock_retries) + return len(ids) +``` + +- [ ] **Step 2: 实现 logging_setup.py** + +`src/sync/logging_setup.py`: +```python +import logging, logging.handlers, os + +def setup_logging(cfg_dict: dict | None): + level = getattr(logging, (cfg_dict or {}).get("level", "INFO")) if cfg_dict else logging.INFO + path = (cfg_dict or {}).get("path", "sync.log") + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + h = logging.handlers.RotatingFileHandler(path, maxBytes=10*1024*1024, backupCount=5, encoding="utf-8") + h.setFormatter(logging.Formatter("%(asctime)s %(levelname)s [%(name)s] %(message)s")) + root = logging.getLogger() + root.setLevel(level) + root.addHandler(h) + sh = logging.StreamHandler() + sh.setFormatter(logging.Formatter("%(levelname)s %(message)s")) + root.addHandler(sh) +``` + +- [ ] **Step 3: 实现 service.py 主循环** + +`src/sync/service.py`: +```python +from __future__ import annotations +import sys, time, logging +from .config import load_config +from .access_reader import AccessReader +from .sql_writer import SqlWriter +from .capture import capture_file +from .cleanup import cleanup_file +from .logging_setup import setup_logging + +log = logging.getLogger("sync.service") + +def run(cfg): + setup_logging(cfg.logging) + while True: + cycle(cfg) + time.sleep(cfg.runtime.poll_interval_seconds) + +def cycle(cfg): + writer = SqlWriter(cfg.sql_server.conn_str, cfg.sql_server.sync_queue_table) + try: + total_captured = 0 + for fm in cfg.files: + reader = AccessReader(fm.source_path(cfg), cfg.access.driver) + try: + n = capture_file(fm, reader, writer, cfg) + total_captured += n + except Exception: + log.exception("capture failed for %s", fm.file) + finally: + reader.close() + log.info("captured %d rows", total_captured) + + try: + writer.call_apply(cfg.runtime.max_retries) + log.info("apply done") + except Exception: + log.exception("apply failed") + + for fm in cfg.files: + reader = AccessReader(fm.source_path(cfg), cfg.access.driver) + try: + c = cleanup_file(fm, reader, writer, cfg) + if c: + log.info("cleaned %d log rows from %s", c, fm.file) + except Exception: + log.exception("cleanup failed for %s", fm.file) + finally: + reader.close() + finally: + writer.close() + +def main(): + cfg_path = sys.argv[1] if len(sys.argv) > 1 else "config.yaml" + cfg = load_config(cfg_path) + try: + run(cfg) + except KeyboardInterrupt: + log.info("stopped") + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 4: 手动冒烟（单次 cycle，对 pilot 库）** + +先用仅含 氩弧焊 的临时 config 跑一次（见 Task 9 pilot 配置）。手动触发： +```bash +.venv/Scripts/python.exe -c "from sync.config import load_config; from sync.service import cycle; cycle(load_config('config.pilot.yaml'))" +``` +观察 `logs/sync.log` 与 `dbo.SyncQueue`、Access `TableChangeLog`、SQL `TIGWelding.表壳焊接记录_YEAR2026` 行数变化。Expected: 捕获→应用→清理一气呵成，无异常。 + +- [ ] **Step 5: 提交** + +```bash +git add src/sync/cleanup.py src/sync/logging_setup.py src/sync/service.py +git commit -m "feat: cleanup phase and main service loop" +``` + +--- + +## Task 8: NSSM 服务化（延后，本期不执行） + +**Files:** +- Create: `scripts/install_service.bat` + +- [ ] **Step 1: 安装脚本** + +`scripts/install_service.bat`: +```bat +@echo off +set SRV=AccessDataMacroSync +set ROOT=C:\Users\peng\Projects\ProductionDataBaseSync_DataMacro +nssm stop %SRV% 2>nul +nssm remove %SRV% confirm 2>nul +nssm install %SRV% "%ROOT%\.venv\Scripts\python.exe" "-m sync.service %ROOT%\config.yaml" +nssm set %SRV% AppDirectory %ROOT% +nssm set %SRV% AppStdout %ROOT%\logs\service.out.log +nssm set %SRV% AppStderr %ROOT%\logs\service.err.log +nssm set %SRV% AppRotateFiles 1 +nssm set %SRV% AppRotateBytes 10485760 +nssm set %SRV% Start SERVICE_AUTO_START +nssm start %SRV% +nssm status %SRV% +``` +> NSSM 操作须在 host 114 上执行；本机已配 `114` SSH 别名（见 nssm-114 skill），可 `ssh 114 "..."` 远程操作。 + +- [ ] **Step 2: 部署与验证（pilot 阶段做，见 Task 9）** + +pilot 验证通过后再以此脚本安装为服务。验证：`nssm status` 显示运行；`logs/sync.log` 持续滚动。 + +- [ ] **Step 3: 提交** + +```bash +git add scripts/install_service.bat +git commit -m "feat: nssm service install script" +``` + +--- + +## Task 9: Pilot（氩弧焊）验证 + +**Files:** +- Create: `config.pilot.yaml`（仅 氩弧焊 + 合同库，或仅 氩弧焊） + +- [ ] **Step 1: pilot 配置** + +复制 `config.yaml` 为 `config.pilot.yaml`，`files:` 仅保留 `氩弧焊.accdb` 一条。`runtime.poll_interval_seconds: 30`（pilot 期放慢便于观察）。 + +- [ ] **Step 2: 前置校验：目标表存在性** + +确认 `TIGWelding.表壳焊接记录_YEAR2026` 等已存在（前期已建）。记录 pilot 启动前各目标表行数： +```bash +sqlcmd -S 192.168.110.114,1433 -U peng -P "Cqbld123456." -d CompanyDB -C -N o -Q "SELECT '表壳焊接记录_YEAR2026' t, COUNT(*) c FROM TIGWelding.表壳焊接记录_YEAR2026 UNION ALL SELECT '超压_YEAR2026', COUNT(*) FROM TIGWelding.超压_YEAR2026" +``` + +- [ ] **Step 3: 启动 pilot（前台）** + +```bash +.venv/Scripts/python.exe -m sync.service config.pilot.yaml +``` +观察 1-2 个轮询周期。 + +- [ ] **Step 4: 触发变更并验证端到端** + +请用户在 氩弧焊 前端对 `表壳焊接记录` 插入/修改/删除各一条。等待 ≤30s。验证： +1. Access `氩弧焊.accdb` 的 `TableChangeLog` 该行已被清理（应用成功即删）。 +2. `dbo.SyncQueue` 对应行 `Status='applied'`。 +3. `TIGWelding.表壳焊接记录_YEAR2026` 数据与 Access 一致（插入的行出现、修改生效、删除的行消失）。 + +- [ ] **Step 5: 校验脚本（行数比对）** + +`scripts/verify_pilot.py`（手动跑）：比对 `氩弧焊.accdb.表壳焊接记录` 与 `TIGWelding.表壳焊接记录_YEAR2026` 的行数与抽样字段。Expected: 一致或差异可解释（同步窗口内）。 + +- [ ] **Step 6: 提交** + +```bash +git add config.pilot.yaml scripts/verify_pilot.py +git commit -m "test: pilot config and verification for 氩弧焊" +``` + +--- + +## Task 10: 全量上线与切换（延后，本期不执行；部署目标 host 114 `C:\Users\peng\Projects`） + +**Files:** +- Modify: `config.yaml`（确认全部 16 个文件）、`README.md`（切换 runbook） + +- [ ] **Step 1: 逐文件扩展 pilot** + +每加入一个文件，观察 1 个周期无 error/dead 后再加下一个。重点关注 `dbo.SyncQueue` 中 `Status='error'`/`'dead'` 行。 + +- [ ] **Step 2: 并行期校验** + +新管线与旧 VBA→`dbo.TableChangeLog`→apply 并行运行（幂等，不冲突）。运行 ≥1 天，比对各表行数与抽样，确认无差异、无持续 error。 + +- [ ] **Step 3: 安装为 NSSM 服务** + +在 114 上执行 `scripts/install_service.bat`（经 `ssh 114`）。验证服务自启、日志滚动。 + +- [ ] **Step 4: 下线旧 VBA** + +1. 编辑各桌面客户端前端 .accdb，移除/禁用写入 `dbo.TableChangeLog` 的 VBA（人工分发）。 +2. 停用旧 `dbo.TableChangeLog` 的 apply 进程。 +3. `dbo.TableChangeLog` 残留 5606 待同步：由旧 apply 跑完，或放弃（新管线覆盖此后增量）。 + +- [ ] **Step 5: 文档化切换 runbook** + +`README.md` 增补「切换步骤」「故障排查（dead 行处理、IDENTITY_INSERT 权限、Access 锁）」「SyncQueue 归档」章节。 + +- [ ] **Step 6: 提交** + +```bash +git add config.yaml README.md +git commit -m "docs: rollout and cutover runbook" +``` + +--- + +## 风险与回退 + +- **存储过程是最高风险组件**：动态 SQL + JSON_VALUE 隐式转换。pilot（Task 9）必须充分验证 upsert/delete/保序/类型/bit/datetime。若 proc 在 pilot 暴露系统性问题，回退方案：将 apply 改为 Python 逐行参数化 upsert/delete（`SqlWriter` 内实现，保留 `SyncQueue` 暂存与重试语义），牺牲"集合化"换简单可靠。 +- **IDENTITY_INSERT 权限**：`peng` 需对目标表 `ALTER`。若权限不足，proc 报错→行进 error→dead，需 DBA 授权。 +- **Access 锁冲突**：cleanup DELETE 与数据宏 INSERT 可能瞬时冲突，已用小批+重试缓解。 +- **JSON_VALUE 4000 截断**：实测 Memo 最大 2144，安全；capture 侧对 >4000 打 WARNING 以预警增长。 diff --git a/docs/superpowers/specs/2026-07-14-access-datamacro-sync-design.md b/docs/superpowers/specs/2026-07-14-access-datamacro-sync-design.md new file mode 100644 index 0000000..85f09a4 --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-access-datamacro-sync-design.md @@ -0,0 +1,326 @@ +# Access → SQL Server 增量同步设计（数据宏驱动） + +- **状态**：已通过设计评审，待写实现计划 +- **日期**：2026-07-14 +- **作者**：Claude（与系统负责人共同设计） + +## 1. 背景与问题 + +当前生产数据实际承载在 Access 数据库中（网络共享 `\\192.168.110.114\生产进度表\` 下，按业务/年份分库）。正在逐步迁移到 SQL Server。 + +**旧机制**：客户端前端 .accdb 内置 VBA，数据改动时由 VBA 写一条记录到 SQL Server `dbo.TableChangeLog`，再由 apply 逻辑把变更落到 SQL Server 业务表。 + +**问题**：VBA 只在特定代码路径/表单事件中触发，批量更新、直接改表、其它客户端写入等路径会绕过 VBA → **数据遗漏**。现 `dbo.TableChangeLog` 已累积 41 万+已同步、5606 待同步。 + +**新机制**：改用 Access **数据宏（Data Macro）**。数据宏是表级触发器，绑定在引擎上，任何数据修改路径都必触发，理论上 100% 捕获变更，且对客户端表单零侵入（免打扰）。各 Access 库的每张业务表已挂 `After Insert / After Update / After Delete` 数据宏，变更写入各库本地 `TableChangeLog`。本设计即"读取各 Access 本地日志 → 增量同步到 SQL Server"的同步程序。 + +## 2. 目标与非目标 + +**目标** +- 以 Access 数据宏日志为唯一变更源，单向增量同步到 SQL Server 业务表。 +- 100% 捕获（Insert/Update/Delete），最终一致，不漏不重。 +- 幂等、可重放、可补跑、可与旧 VBA 管线并行不冲突。 +- 同步成功后清理 Access 日志表，控制 .accdb 体积。 + +**非目标** +- 不做 SQL Server → Access 反向同步。 +- 不在本期重构 SQL Server 表结构（保留无后缀=2025、`_YEAR2026`=2026 现状）。 +- 不处理 SQL Server 原生模块（`executionCard.PG/TM_年份`、`ERPAuto`、`btprint`、`CargoTrace`、`procurementVisibilityHub`、`perf` 等，它们无 Access 对应源）。 + +## 3. 架构 + +选定方案 **B：轮询 + 暂存 + 集合化 apply**。全部运行于 host 114（Access 文件本地、读取快；已有 NSSM 服务基建），Python 实现，NSSM 常驻。 + +### 3.1 组件 + +| 组件 | 位置 | 职责 | +|---|---|---| +| **Capture** | Python（114, NSSM 服务） | 轮询各 Access 后端本地 `TableChangeLog`，回读整行，写入 `SyncQueue`；应用成功后回删 Access 日志 | +| **SyncQueue** | SQL Server 表 | 持久暂存缓冲：存待应用变更 + 状态 + 审计 | +| **Apply** | SQL Server 存储过程 | 按 目标表 集合化 MERGE / DELETE，保序"最后操作胜" | +| **config.yaml** | 114 本地文件 | 所有配置：连接串、路径、映射、轮询/重试参数 | + +> 不再使用 `SyncWatermark` 表：Access 日志"应用成功即删"，日志本身即待处理队列；`SyncQueue` 唯一索引去重保证重复捕获幂等。 + +### 3.2 数据流 + +``` +┌─────────── host 114 · Python NSSM 服务（轮询 ~10s）──────────┐ +│ ① CAPTURE 读各 Access TableChangeLog 全部剩余行 │ ──读──► Access .accdb +│ I/U 按 RecordID 回读整行 → SyncQueue(pending) │ (共享打开, 与客户端并发) +│ （SyncQueue 唯一键 (file,table,source_log_id) 去重）│ +│ ② APPLY 存储过程：按目标表分组，保序 MERGE/DELETE │ ──写──► SQL Server 业务表 +│ 成功→applied / 失败→error(retry) / 超限→dead │ +│ ③ CLEANUP DELETE Access TableChangeLog │ +│ WHERE ID IN (本文件 SyncQueue status='applied') │ +└─────────────────────────────────────────────────────────────┘ +``` + +## 4. 详细设计 + +### 4.1 配置（config.yaml） + +```yaml +sql_server: + conn_str: "Driver={ODBC Driver 17 for SQL Server};Server=;Database=;Trusted_Connection=yes;" + sync_queue_table: "dbo.SyncQueue" + +access: + driver: "{Microsoft Access Driver (*.accdb, *.mdb)}" + roots: + 2026: "\\\\192.168.110.114\\生产进度表\\2026年数据" + 2025: "\\\\192.168.110.114\\生产进度表\\2025年数据" + +runtime: + poll_interval_seconds: 10 + capture_batch_size: 500 + apply_batch_size: 200 + max_retries: 5 + retry_backoff_seconds: 30 + cleanup_batch_size: 200 + cleanup_lock_retries: 3 + +files: + - file: "氩弧焊.accdb" + root: 2026 + schema: "TIGWelding" + year_suffix: "_YEAR2026" # 2025 文件为 "" + exclude_tables: ["TableChangeLog", "氩弧焊每日催货落实记录_停"] + - file: "生产合同数据.accdb" + root: 2025 + schema: "productionContractData" + year_suffix: "" # 合同表年份在表名里，不加后缀 + include_tables: ["26年压力表合同数据", "26年温度计合同数据", "26年变送器合同数据", "26年OEM数据"] + # ... 其余文件见 §7 映射表 + +logging: + level: INFO + path: "D:\\projects\\ProductionDataBaseSync_DataMacro\\logs\\sync.log" +``` + +映射放 YAML 而非 SQL 表。Capture 写 `SyncQueue` 时即写入按 YAML 解析好的 `target_schema` / `target_table`。 + +### 4.2 SyncQueue 表结构（SQL Server） + +```sql +CREATE TABLE dbo.SyncQueue ( + QueueID bigint IDENTITY(1,1) PRIMARY KEY, + SourceFile nvarchar(255) NOT NULL, -- 源 .accdb 文件名 + SourceTable nvarchar(255) NOT NULL, -- Access 表名 + SourceLogID bigint NOT NULL, -- Access TableChangeLog.ID（去重键） + TargetSchema nvarchar(128) NOT NULL, + TargetTable nvarchar(255) NOT NULL, -- 已解析（含年份后缀） + RecordID nvarchar(50) NOT NULL, -- 业务行 ID（字符串） + OperateType varchar(10) NOT NULL, -- Insert/Update/Delete + RowData nvarchar(max) NULL, -- I/U 整行 JSON；Delete 为 NULL + Status varchar(10) NOT NULL DEFAULT 'pending', -- pending/applied/error/dead + RetryCount int NOT NULL DEFAULT 0, + ErrorMsg nvarchar(max) NULL, + CapturedAt datetime2 NOT NULL DEFAULT sysdatetime(), + AppliedAt datetime2 NULL +); +CREATE UNIQUE INDEX UX_SyncQueue_Dedup ON dbo.SyncQueue(SourceFile, SourceTable, SourceLogID); +CREATE INDEX IX_SyncQueue_Pending ON dbo.SyncQueue(Status, TargetSchema, TargetTable); +``` + +`SyncQueue` 同时是工作队列与审计/重试日志，长期保留。 + +### 4.3 Capture 逻辑（Python） + +1. 共享模式打开各 .accdb（`pyodbc` + ACE 驱动，不独占，与客户端并发共存）。 +2. `SELECT ID, TableName, RecordID, OperateType, Time FROM TableChangeLog ORDER BY ID`（取 `capture_batch_size` 条）。 +3. 对每行： + - `Insert/Update`：`SELECT * FROM "" WHERE ID = ` 回读整行当前状态。 + - 若读不到（记录已被删，Insert 后 Delete 未及同步）→ 降级为 `Delete`、`RowData=NULL`（最终态正确）。 + - 否则按列序列化为 JSON（类型规则见 §4.5）。 + - `Delete`：`RowData=NULL`。 +4. 按 YAML 解析目标（`schema` + `表名 + year_suffix`；合同表用 `include_tables` 同名）。 +5. `INSERT … SELECT … WHERE NOT EXISTS(同 SourceFile+SourceTable+SourceLogID)` → 重复捕获幂等忽略。 +6. 全部文件捕获完 → 调用 Apply 存储过程。 +7. Cleanup（见 §4.5）。 + +### 4.4 Apply 逻辑（SQL 存储过程，集合化 + 保序） + +核心：同一 `RecordID` 可能有多个操作（先 Insert 后 Delete），必须按日志顺序**最后操作胜**，否则 delete→insert 错序会插回已删行。用窗口函数取每个 ID 的最后一条操作： + +```sql +CREATE PROCEDURE dbo.usp_SyncApply + @MaxRetries int = 5 -- 来自 config.yaml runtime.max_retries +AS +BEGIN + -- 枚举待处理目标表 + DECLARE cur CURSOR FOR + SELECT DISTINCT TargetSchema, TargetTable + FROM dbo.SyncQueue WHERE Status='pending'; + + OPEN cur; FETCH NEXT FROM cur INTO @sch, @tbl; + WHILE @@FETCH_STATUS=0 BEGIN + BEGIN TRY + BEGIN TRAN; + + -- 动态从 sys.columns 取目标列（排除 ID[键]、SSMA_TimeStamp、computed、非ID identity） + -- 构造列列表 @cols_for_update / @cols_for_insert + + -- (a) 最后操作为 Insert/Update → upsert（保 ID） + SET @sql = N'SET IDENTITY_INSERT ['+@sch+'].['+@tbl+'] ON; + MERGE ['+@sch+'].['+@tbl+'] WITH (HOLDLOCK) AS tgt + USING (SELECT RecordID, RowData FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn + FROM dbo.SyncQueue + WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'' + AND OperateType IN (''Insert'',''Update'')) x WHERE rn=1) AS src + ON tgt.ID = TRY_CAST(src.RecordID AS int) + WHEN MATCHED THEN UPDATE SET ' + @update_clause + ' + WHEN NOT MATCHED THEN INSERT (ID,' + @insert_cols + ') VALUES (TRY_CAST(src.RecordID AS int),' + @insert_vals + '); + SET IDENTITY_INSERT ['+@sch+'].['+@tbl+'] OFF;'; + EXEC sp_executesql @sql, N'@sch nvarchar(128),@tbl nvarchar(255)', @sch, @tbl; + + -- (b) 最后操作为 Delete → 删 + SET @sql = N'DELETE t FROM ['+@sch+'].['+@tbl+'] t + JOIN (SELECT RecordID FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY RecordID ORDER BY SourceLogID DESC) rn + FROM dbo.SyncQueue + WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status=''pending'' + AND OperateType=''Delete'') x WHERE rn=1) d + ON t.ID = TRY_CAST(d.RecordID AS int);'; + EXEC sp_executesql @sql, N'@sch nvarchar(128),@tbl nvarchar(255)', @sch, @tbl; + + -- (c) 标记成功 + UPDATE dbo.SyncQueue SET Status='applied', AppliedAt=sysdatetime() + WHERE TargetSchema=@sch AND TargetTable=@tbl AND Status='pending'; + + COMMIT; + END TRY + BEGIN CATCH + ROLLBACK; + -- 标记失败：retry 未超限→error(待重试)，超限→dead + UPDATE q SET q.Status=CASE WHEN q.RetryCount>=@MaxRetries THEN 'dead' ELSE 'error' END, + q.RetryCount=q.RetryCount+1, q.ErrorMsg=ERROR_MESSAGE() + FROM dbo.SyncQueue q + WHERE q.TargetSchema=@sch AND q.TargetTable=@tbl AND q.Status='pending'; + END CATCH + FETCH NEXT FROM cur INTO @sch, @tbl; + END + -- 重试：error 且未超限 → 重置 pending 待下轮 + UPDATE dbo.SyncQueue SET Status='pending' + WHERE Status='error' AND RetryCount < @MaxRetries; +END +``` + +**关键实现点**： +- **列不硬编码**：运行时从 `sys.columns` 取目标表列名，排除 `ID`（MERGE 键，但 INSERT 时需保留）、`SSMA_TimeStamp`（rowversion 自增）、computed 列、非 ID 的 identity 列，动态拼 `@update_clause` / `@insert_cols` / `@insert_vals`。 +- **`SET IDENTITY_INSERT ON`**：SQL 端 ID 多为 IDENTITY，必须开此开关才能写入 Access 原 ID（否则 ID 错位，后续 update/delete 找不到行）。每个目标表 MERGE 前后开关。 +- **`TRY_CAST`**：JSON 值→目标类型容错；转换失败→NULL（配合 `error` 标记排查）。 +- **每目标表一个事务**：一张表失败只回滚该表，其它表照常；失败行 `error`，超 `max_retries` 置 `dead` 待人工。 +- **`HOLDLOCK`**：防 MERGE 并发条件竞争。 +- **保序"最后操作胜"**：`ROW_NUMBER() PARTITION BY RecordID ORDER BY SourceLogID DESC` 取 rn=1，保证最终态与 Access 一致。 + +### 4.5 Cleanup 逻辑（Python） + +```python +applied = sql.execute( + "SELECT SourceLogID FROM dbo.SyncQueue WHERE SourceFile=? AND Status='applied'", f +).fetchall() +# 小批删除，遇锁重试 +for batch in chunks(applied, cleanup_batch_size): + cnxn.execute(f"DELETE FROM TableChangeLog WHERE ID IN ({ids})", ...) # 锁冲突重试 cleanup_lock_retries 次 +cnxn.commit() +``` + +只删 `applied` 行；`error`/`pending`/`dead` 保留。`dead` 行需人工排查后手动处理。 + +### 4.6 类型序列化与转换 + +| Access 类型 | JSON 承载 | SQL 端转换 | +|---|---|---| +| Text / Memo | str | `JSON_VALUE` → nvarchar | +| Long / Integer / Byte | int | `TRY_CAST(... AS int)` | +| Double / Single | float | `TRY_CAST(... AS float)` | +| Currency | Decimal(str) | `TRY_CAST(... AS money)` | +| Date/Time | ISO 8601 str | `TRY_CAST(... AS datetime2)` | +| Yes/No | bool | `TRY_CAST(... AS bit)` | +| Null | null | NULL | + +### 4.7 并发与锁 + +- Access 共享打开，读 `TableChangeLog` + 源行 与数据宏并发 INSERT 不冲突（Jet 共享模式）。 +- Cleanup 的 DELETE 与宏 INSERT 可能瞬时锁冲突 → 小批（`cleanup_batch_size`）+ 重试 `cleanup_lock_retries` 次。 +- Apply 每表事务 + `HOLDLOCK`。 +- 最终一致性：同一 ID 多操作"最后操作胜"。 + +### 4.8 错误处理与可观测 + +- **per-row/per-table 隔离**：单行/单表失败不阻塞其它。 +- **重试**：`error` 行 `RetryCount < max_retries` → 下轮重置 `pending` 重试；超限→`dead`。 +- **dead-letter**：`dead` 行保留在 `SyncQueue`，`ErrorMsg` 记原因，待人工处理。 +- **日志**：Python 写 `logs/sync.log`（轮询、捕获数、apply 数、错误）；SQL 侧 `SyncQueue` 全留档。 +- **监控指标**（可选）：每轮各文件 pending/applied/error 计数，可接告警。 + +## 5. 切换计划（Cutover） + +1. **部署新管线**（Capture + SyncQueue + Apply），覆盖试点文件。 +2. **并行运行**：新管线与旧 VBA→`dbo.TableChangeLog`→apply 同时跑。两者均以 ID 为键幂等写业务表，不冲突。并行期用于校验。 +3. **校验**：抽样比对 Access 源行与 SQL Server 目标行；比对各表行数；检查 `SyncQueue` 无持续 `error`/`dead`。 +4. **下线旧管线**：校验通过后—— + - 停用客户端前端内的 VBA 捕获代码（编辑各客户端 .accdb 移除/禁用日志写入 VBA）。 + - 停用旧 `dbo.TableChangeLog` 的 apply 进程。 + - `dbo.TableChangeLog` 中残留 5606 待同步：由旧 apply 跑完，或直接放弃（新管线覆盖此后增量；历史已同步）。 +5. `dbo.TableChangeLog` 保留为历史审计（后续可归档/删除）。 + +## 6. 范围与试点 + +**同步范围（活跃源）**： +- `2026年数据\` 全部 21 个 .accdb（活跃车间/业务数据）。 +- `2025年数据\生产合同数据.accdb`（合同数据总库，含实时更新的 26年表）。 + +**排除**： +- `2026年数据\生产数据库_停.accdb`（旧中央汇总库，已停）。 +- `2026年数据\生产合同数据 .accdb`（空壳前端，0 行）。 +- `2025年数据\生产数据库.accdb`（旧中央汇总库，dormant）——**特别注意**：此类汇总库是衍生副本，绝不可纳入同步，否则会循环。 +- `2025年数据\` 其余车间库（已迁移到无后缀表，dormant，无数据宏活动）。 + +**试点**：先以 **`氩弧焊.accdb`**（2026，最活跃，数据宏已验证在写日志）跑通端到端，校验无误后再批量纳管其余文件。 + +## 7. 映射表（草稿，需确认/补全） + +| Access 文件 | 根 | SQL schema | year_suffix | 说明 / 待确认 | +|---|---|---|---|---| +| 一车间.accdb | 2026 | workshopOne | _YEAR2026 | 一车间记录；排除 一车间每日催货落实记录_停 | +| 二车间.accdb | 2026 | workshopTwo | _YEAR2026 | | +| 三车间.accdb | 2026 | workshopThree | _YEAR2026 | 含 温度计调校记录 | +| 弯管车间.accdb | 2026 | tubeBending | _YEAR2026 | 烘洗 | +| 氩弧焊.accdb | 2026 | TIGWelding | _YEAR2026 | 表壳焊接/超压/氦测/接收/退火/氩弧焊记录/氩弧焊领料；排除 *_停 | +| 机加工.accdb | 2026 | machining | _YEAR2026 | 车波纹/隔膜机加接收/膜片焊接/膜片接收/喷涂寄出/各每日催货 | +| 零件库.accdb | 2026 | partsWarehouse | _YEAR2026 | 表盘/法兰/部件/出库单/缺件/温度计法兰 入库&缺件记录 | +| 缺料数据.accdb | — | — | — | **忽略**：无实际业务数据，不同步 | +| 成品入库.accdb | 2026 | productWarehousing | _YEAR2026 | 成品交检/入库记录 | +| 检验记录数据库.accdb | 2026 | inspectionRecords | _YEAR2026 | 仅同步 检验合格记录表（多余表已由负责人移除） | +| 温度计记录.accdb | 2026 | thermometerRecord | _YEAR2026 | 采技机记录/温度计调校/检验/组装 | +| 锡焊数据.accdb | 2026 | solderingData | _YEAR2026 | 操作者1/2完成记录/零件到车间记录 | +| 计划.accdb | 2026 | contractPlanning | _YEAR2026 | 各接单记录/下单记录；排除 *_停 | +| 精密表记录.accdb | — | — | — | **忽略**：无实际业务数据，不同步 | +| 隔膜数据.accdb | 2026 | diaphragmData | _YEAR2026 | **新建**(2026-07-14)：schema diaphragmData + 隔膜BOM/隔膜类型/技术确认数据_YEAR2026 | +| 执行卡下发记录.accdb | 2026 | executionCardIssuanceRecord | _YEAR2026 | 执行卡下发记录 + **新建**(2026-07-14) 货期修改记录_YEAR2026 | +| 技术部.accdb | — | — | — | **忽略**：无实际业务数据，不同步 | +| OEM.accdb | 2026 | OEM | _YEAR2026 | OEM合同数据(既有,无后缀,2025/legacy) + **新建**(2026-07-14) OEM盘图/OEM请购/OEM外协/OEM自产转回_YEAR2026 | +| 成品物料号.accdb | — | — | — | **忽略**：无实际业务数据，不同步 | +| 生产合同数据.accdb | 2025 | productionContractData | "" | 18–26年合同表同名映射；include_tables 限定活跃年份表 | + +> GAP 项已由系统负责人确认（2026-07-14）：隔膜数据/货期修改记录/OEM 4 表已在 SQL Server 新建（新 schema `diaphragmData`，新表统一 `_YEAR2026` 后缀，Text(255)→nvarchar(255)）；缺料数据/精密表记录/技术部/成品物料号 无业务数据，忽略；检验记录数据库仅同步 检验合格记录表。 + +## 8. 测试策略 + +- **单元**：JSON 序列化/类型转换；目标列动态构建（排除规则）；"最后操作胜"保序逻辑（构造 Insert→Delete、Delete→Insert、多次 Update 用例）。 +- **集成（试点）**：以 `氩弧焊.accdb` 端到端：插入/修改/删除各触发一次，验证 SQL Server 目标行一致；验证 Access 日志被清理。 +- **幂等**：人为重复运行 Capture，确认无重复、无错写。 +- **并发**：客户端写入同时跑同步，验证无锁死、无丢失。 +- **故障注入**：Apply 中途杀进程，验证重启后续跑（pending 续处理、applied 已清理的不重做）。 +- **校验脚本**：全表行数比对 Access↔SQL Server，差异告警。 + +## 9. 待确认 / 风险 + +- **§7 映射 GAP**：已解决（2026-07-14）。新建 schema `diaphragmData` 及 8 张表（diaphragmData.隔膜BOM/隔膜类型/技术确认数据_YEAR2026、executionCardIssuanceRecord.货期修改记录_YEAR2026、OEM.OEM盘图/OEM请购/OEM外协/OEM自产转回_YEAR2026）；4 个无业务数据文件忽略。 +- **数据宏 Delete 取值**：确认 `After Delete` 宏确能记录被删行 ID（Access 宏上下文含旧值，应可；试点验证）。 +- **IDENTITY_INSERT 权限**：执行账号需对目标表有 `ALTER` 权限（`SET IDENTITY_INSERT` 要求）。 +- **Access 2GB / 日志体积**：清理后日志保持小体量；`SyncQueue` 长期增长需定期归档（可按 `AppliedAt` 归档老数据）。 +- **客户端 VBA 下线**：需在各桌面客户端 .accdb 中移除/禁用旧 VBA，属人工分发工作。 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2b00cfb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pyodbc>=5.0.1 +PyYAML>=6.0.1 +pydantic>=2.6.0 +pytest>=8.0.0 diff --git a/src/sync/__init__.py b/src/sync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29