feat: config model and yaml loader
Add Pydantic config models (SqlServerConfig, AccessConfig, RuntimeConfig, FileMapping, SyncConfig) and a YAML loader (load_config). FileMapping provides source_path() and target_table() helpers; number-typed YAML keys/values (e.g. root: 2026) are coerced to str via coerce_numbers_to_str. Includes config.example.yaml template (config.yaml with real credentials stays gitignored) and pyproject.toml pytest config (pythonpath=src). Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
44
config.example.yaml
Normal file
44
config.example.yaml
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
sql_server:
|
||||||
|
conn_str: "Driver={ODBC Driver 17 for SQL Server};Server=<SERVER>,1433;Database=<DB>;UID=<USER>;PWD=<PASSWORD>;Encrypt=yes;TrustServerCertificate=yes;"
|
||||||
|
sync_queue_table: "dbo.SyncQueue"
|
||||||
|
|
||||||
|
access:
|
||||||
|
driver: "{Microsoft Access Driver (*.accdb, *.mdb)}"
|
||||||
|
roots:
|
||||||
|
2026: "\\\\<server>\\生产进度表\\<年份>数据"
|
||||||
|
2025: "\\\\<server>\\生产进度表\\<年份>数据"
|
||||||
|
|
||||||
|
runtime:
|
||||||
|
poll_interval_seconds: 10
|
||||||
|
capture_batch_size: 500
|
||||||
|
apply_batch_size: 200
|
||||||
|
max_retries: 5
|
||||||
|
retry_backoff_seconds: 30
|
||||||
|
cleanup_batch_size: 200
|
||||||
|
cleanup_lock_retries: 3
|
||||||
|
|
||||||
|
logging:
|
||||||
|
level: INFO
|
||||||
|
path: "<LOG_PATH>"
|
||||||
|
|
||||||
|
files:
|
||||||
|
- {file: "一车间.accdb", root: 2026, schema: "workshopOne", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "一车间每日催货落实记录_停"]}
|
||||||
|
- {file: "二车间.accdb", root: 2026, schema: "workshopTwo", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "三车间.accdb", root: 2026, schema: "workshopThree", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "弯管车间.accdb", root: 2026, schema: "tubeBending", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "氩弧焊.accdb", root: 2026, schema: "TIGWelding", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "氩弧焊每日催货落实记录_停"]}
|
||||||
|
- {file: "机加工.accdb", root: 2026, schema: "machining", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "零件库.accdb", root: 2026, schema: "partsWarehouse", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "成品入库.accdb", root: 2026, schema: "productWarehousing", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "检验记录数据库.accdb", root: 2026, schema: "inspectionRecords", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"], include_tables: ["检验合格记录表"]}
|
||||||
|
- {file: "温度计记录.accdb", root: 2026, schema: "thermometerRecord", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "锡焊数据.accdb", root: 2026, schema: "solderingData", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "计划.accdb", root: 2026, schema: "contractPlanning", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog", "每日催货合同号_停", "每日催货缺件落实记录_停"]}
|
||||||
|
- {file: "隔膜数据.accdb", root: 2026, schema: "diaphragmData", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "执行卡下发记录.accdb", root: 2026, schema: "executionCardIssuanceRecord", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- {file: "OEM.accdb", root: 2026, schema: "OEM", year_suffix: "_YEAR2026", exclude_tables: ["TableChangeLog"]}
|
||||||
|
- file: "生产合同数据.accdb"
|
||||||
|
root: 2025
|
||||||
|
schema: "productionContractData"
|
||||||
|
year_suffix: ""
|
||||||
|
exclude_tables: ["TableChangeLog", "dbo_TableChangeLog", "USysApplicationLog", "温度计数据_修复"]
|
||||||
4
pyproject.toml
Normal file
4
pyproject.toml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
[tool.pytest.ini_options]
|
||||||
|
pythonpath = ["src"]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
markers = ["integration: marks tests requiring real Access/SQL Server"]
|
||||||
50
src/sync/config.py
Normal file
50
src/sync/config.py
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
from pathlib import Path
|
||||||
|
import yaml
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
class SqlServerConfig(BaseModel):
|
||||||
|
conn_str: str
|
||||||
|
sync_queue_table: str = "dbo.SyncQueue"
|
||||||
|
|
||||||
|
class AccessConfig(BaseModel):
|
||||||
|
model_config = ConfigDict(coerce_numbers_to_str=True)
|
||||||
|
driver: str
|
||||||
|
roots: dict[str, str]
|
||||||
|
|
||||||
|
class RuntimeConfig(BaseModel):
|
||||||
|
poll_interval_seconds: int = 10
|
||||||
|
capture_batch_size: int = 500
|
||||||
|
apply_batch_size: int = 200
|
||||||
|
max_retries: int = 5
|
||||||
|
retry_backoff_seconds: int = 30
|
||||||
|
cleanup_batch_size: int = 200
|
||||||
|
cleanup_lock_retries: int = 3
|
||||||
|
|
||||||
|
class FileMapping(BaseModel):
|
||||||
|
model_config = ConfigDict(coerce_numbers_to_str=True)
|
||||||
|
file: str
|
||||||
|
root: str
|
||||||
|
schema: str
|
||||||
|
year_suffix: str = ""
|
||||||
|
exclude_tables: list[str] = Field(default_factory=list)
|
||||||
|
include_tables: list[str] | None = None
|
||||||
|
|
||||||
|
def source_path(self, cfg: "SyncConfig") -> str:
|
||||||
|
base = cfg.access.roots[self.root]
|
||||||
|
return f"{base}\\{self.file}"
|
||||||
|
|
||||||
|
def target_table(self, access_table: str) -> str:
|
||||||
|
return f"{access_table}{self.year_suffix}"
|
||||||
|
|
||||||
|
class SyncConfig(BaseModel):
|
||||||
|
sql_server: SqlServerConfig
|
||||||
|
access: AccessConfig
|
||||||
|
runtime: RuntimeConfig
|
||||||
|
files: list[FileMapping]
|
||||||
|
logging: dict | None = None
|
||||||
|
|
||||||
|
def load_config(path: str) -> SyncConfig:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f)
|
||||||
|
return SyncConfig(**data)
|
||||||
44
tests/test_config.py
Normal file
44
tests/test_config.py
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
import pathlib, textwrap
|
||||||
|
from sync.config import load_config, FileMapping
|
||||||
|
|
||||||
|
def test_load_config_parses_fields(tmp_path):
|
||||||
|
cfg_text = textwrap.dedent("""
|
||||||
|
sql_server:
|
||||||
|
conn_str: "Driver={ODBC Driver 17 for SQL Server};Server=s;Database=d;UID=u;PWD=p;"
|
||||||
|
sync_queue_table: "dbo.SyncQueue"
|
||||||
|
access:
|
||||||
|
driver: "{Microsoft Access Driver (*.accdb, *.mdb)}"
|
||||||
|
roots:
|
||||||
|
2026: "\\\\\\\\srv\\\\2026"
|
||||||
|
2025: "\\\\\\\\srv\\\\2025"
|
||||||
|
runtime:
|
||||||
|
poll_interval_seconds: 10
|
||||||
|
capture_batch_size: 500
|
||||||
|
apply_batch_size: 200
|
||||||
|
max_retries: 5
|
||||||
|
retry_backoff_seconds: 30
|
||||||
|
cleanup_batch_size: 200
|
||||||
|
cleanup_lock_retries: 3
|
||||||
|
files:
|
||||||
|
- file: "氩弧焊.accdb"
|
||||||
|
root: 2026
|
||||||
|
schema: "TIGWelding"
|
||||||
|
year_suffix: "_YEAR2026"
|
||||||
|
exclude_tables: ["TableChangeLog", "氩弧焊每日催货落实记录_停"]
|
||||||
|
""")
|
||||||
|
p = tmp_path / "config.yaml"
|
||||||
|
p.write_text(cfg_text, encoding="utf-8")
|
||||||
|
cfg = load_config(str(p))
|
||||||
|
assert cfg.sql_server.sync_queue_table == "dbo.SyncQueue"
|
||||||
|
assert cfg.runtime.poll_interval_seconds == 10
|
||||||
|
assert cfg.files[0].schema == "TIGWelding"
|
||||||
|
assert cfg.files[0].year_suffix == "_YEAR2026"
|
||||||
|
assert "2026" in cfg.files[0].source_path(cfg)
|
||||||
|
|
||||||
|
def test_file_mapping_target_table_applies_year_suffix():
|
||||||
|
fm = FileMapping(file="x.accdb", root="2026", schema="s", year_suffix="_YEAR2026",
|
||||||
|
exclude_tables=[], include_tables=None)
|
||||||
|
assert fm.target_table("一车间记录") == "一车间记录_YEAR2026"
|
||||||
|
fm2 = FileMapping(file="y.accdb", root="2025", schema="s", year_suffix="",
|
||||||
|
exclude_tables=[], include_tables=None)
|
||||||
|
assert fm2.target_table("26年压力表合同数据") == "26年压力表合同数据"
|
||||||
Reference in New Issue
Block a user