feat: config model and yaml loader

Add Pydantic config models (SqlServerConfig, AccessConfig, RuntimeConfig,
FileMapping, SyncConfig) and a YAML loader (load_config). FileMapping provides
source_path() and target_table() helpers; number-typed YAML keys/values (e.g.
root: 2026) are coerced to str via coerce_numbers_to_str. Includes
config.example.yaml template (config.yaml with real credentials stays
gitignored) and pyproject.toml pytest config (pythonpath=src).

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-07-14 11:53:20 +08:00
parent fb025bb061
commit 7c6cb10b91
4 changed files with 142 additions and 0 deletions

44
config.example.yaml Normal file
View File

@@ -0,0 +1,44 @@
sql_server:
conn_str: "Driver={ODBC Driver 17 for SQL Server};Server=<SERVER>,1433;Database=<DB>;UID=<USER>;PWD=<PASSWORD>;Encrypt=yes;TrustServerCertificate=yes;"
sync_queue_table: "dbo.SyncQueue"
access:
driver: "{Microsoft Access Driver (*.accdb, *.mdb)}"
roots:
2026: "\\\\<server>\\生产进度表\\<年份>数据"
2025: "\\\\<server>\\生产进度表\\<年份>数据"
runtime:
poll_interval_seconds: 10
capture_batch_size: 500
apply_batch_size: 200
max_retries: 5
retry_backoff_seconds: 30
cleanup_batch_size: 200
cleanup_lock_retries: 3
logging:
level: INFO
path: "<LOG_PATH>"
files:
- {file: "一车间.accdb", root: 2026, schema: "workshopOne", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "一车间每日催货落实记录_停"]}
- {file: "二车间.accdb", root: 2026, schema: "workshopTwo", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "三车间.accdb", root: 2026, schema: "workshopThree", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "弯管车间.accdb", root: 2026, schema: "tubeBending", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "氩弧焊.accdb", root: 2026, schema: "TIGWelding", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "氩弧焊每日催货落实记录_停"]}
- {file: "机加工.accdb", root: 2026, schema: "machining", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "零件库.accdb", root: 2026, schema: "partsWarehouse", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "成品入库.accdb", root: 2026, schema: "productWarehousing", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "检验记录数据库.accdb", root: 2026, schema: "inspectionRecords", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"], include_tables: ["检验合格记录表"]}
- {file: "温度计记录.accdb", root: 2026, schema: "thermometerRecord", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "锡焊数据.accdb", root: 2026, schema: "solderingData", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "计划.accdb", root: 2026, schema: "contractPlanning", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "每日催货合同号_停", "每日催货缺件落实记录_停"]}
- {file: "隔膜数据.accdb", root: 2026, schema: "diaphragmData", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "执行卡下发记录.accdb", root: 2026, schema: "executionCardIssuanceRecord", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- {file: "OEM.accdb", root: 2026, schema: "OEM", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
- file: "生产合同数据.accdb"
root: 2025
schema: "productionContractData"
year_suffix: ""
exclude_tables: ["TableChangeLog", "dbo_TableChangeLog", "USysApplicationLog", "温度计数据_修复"]

4
pyproject.toml Normal file
View File

@@ -0,0 +1,4 @@
[tool.pytest.ini_options]
pythonpath = ["src"]
testpaths = ["tests"]
markers = ["integration: marks tests requiring real Access/SQL Server"]

50
src/sync/config.py Normal file
View File

@@ -0,0 +1,50 @@
from __future__ import annotations
from pathlib import Path
import yaml
from pydantic import BaseModel, ConfigDict, Field
class SqlServerConfig(BaseModel):
conn_str: str
sync_queue_table: str = "dbo.SyncQueue"
class AccessConfig(BaseModel):
model_config = ConfigDict(coerce_numbers_to_str=True)
driver: str
roots: dict[str, str]
class RuntimeConfig(BaseModel):
poll_interval_seconds: int = 10
capture_batch_size: int = 500
apply_batch_size: int = 200
max_retries: int = 5
retry_backoff_seconds: int = 30
cleanup_batch_size: int = 200
cleanup_lock_retries: int = 3
class FileMapping(BaseModel):
model_config = ConfigDict(coerce_numbers_to_str=True)
file: str
root: str
schema: str
year_suffix: str = ""
exclude_tables: list[str] = Field(default_factory=list)
include_tables: list[str] | None = None
def source_path(self, cfg: "SyncConfig") -> str:
base = cfg.access.roots[self.root]
return f"{base}\\{self.file}"
def target_table(self, access_table: str) -> str:
return f"{access_table}{self.year_suffix}"
class SyncConfig(BaseModel):
sql_server: SqlServerConfig
access: AccessConfig
runtime: RuntimeConfig
files: list[FileMapping]
logging: dict | None = None
def load_config(path: str) -> SyncConfig:
with open(path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
return SyncConfig(**data)

44
tests/test_config.py Normal file
View File

@@ -0,0 +1,44 @@
import pathlib, textwrap
from sync.config import load_config, FileMapping
def test_load_config_parses_fields(tmp_path):
cfg_text = textwrap.dedent("""
sql_server:
conn_str: "Driver={ODBC Driver 17 for SQL Server};Server=s;Database=d;UID=u;PWD=p;"
sync_queue_table: "dbo.SyncQueue"
access:
driver: "{Microsoft Access Driver (*.accdb, *.mdb)}"
roots:
2026: "\\\\\\\\srv\\\\2026"
2025: "\\\\\\\\srv\\\\2025"
runtime:
poll_interval_seconds: 10
capture_batch_size: 500
apply_batch_size: 200
max_retries: 5
retry_backoff_seconds: 30
cleanup_batch_size: 200
cleanup_lock_retries: 3
files:
- file: "氩弧焊.accdb"
root: 2026
schema: "TIGWelding"
year_suffix: "_YEAR2026"
exclude_tables: ["TableChangeLog", "氩弧焊每日催货落实记录_停"]
""")
p = tmp_path / "config.yaml"
p.write_text(cfg_text, encoding="utf-8")
cfg = load_config(str(p))
assert cfg.sql_server.sync_queue_table == "dbo.SyncQueue"
assert cfg.runtime.poll_interval_seconds == 10
assert cfg.files[0].schema == "TIGWelding"
assert cfg.files[0].year_suffix == "_YEAR2026"
assert "2026" in cfg.files[0].source_path(cfg)
def test_file_mapping_target_table_applies_year_suffix():
fm = FileMapping(file="x.accdb", root="2026", schema="s", year_suffix="_YEAR2026",
exclude_tables=[], include_tables=None)
assert fm.target_table("一车间记录") == "一车间记录_YEAR2026"
fm2 = FileMapping(file="y.accdb", root="2025", schema="s", year_suffix="",
exclude_tables=[], include_tables=None)
assert fm2.target_table("26年压力表合同数据") == "26年压力表合同数据"