feat(compare): 核对结果写入 logs/ 并为 114 增加开机计划任务
- compare.py: 新增 write_report(),每次运行把报告以日志形式写到 logs/compare_<granularity>_<日期>.log(含生成时间 + 汇总头部), --report 仍兼容作为额外输出路径 - main.py: compare 子命令改用 write_report,默认落 logs/,stdout 仍打印 - run_compare_ids.cmd: 114 主机开机计划任务包装脚本(ID 级核对) Co-Authored-By: WorkBuddy <workbuddy@tencent.com>
This commit is contained in:
17
main.py
17
main.py
@@ -17,6 +17,7 @@ was launched or whether the venv already has ``src`` on its path.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
@@ -27,9 +28,10 @@ from sync.config import load_config
|
|||||||
from sync.logging_setup import setup_logging
|
from sync.logging_setup import setup_logging
|
||||||
from sync.fullsync import full_sync
|
from sync.fullsync import full_sync
|
||||||
from sync import service
|
from sync import service
|
||||||
from sync.compare import compare, any_mismatch, format_report
|
from sync.compare import compare, any_mismatch, format_report, write_report
|
||||||
|
|
||||||
CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.yaml")
|
CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.yaml")
|
||||||
|
log = logging.getLogger("main")
|
||||||
|
|
||||||
|
|
||||||
def _parse_args(argv):
|
def _parse_args(argv):
|
||||||
@@ -99,11 +101,14 @@ def main(argv=None) -> int:
|
|||||||
if args.command == "compare":
|
if args.command == "compare":
|
||||||
results = compare(cfg, granularity=args.granularity,
|
results = compare(cfg, granularity=args.granularity,
|
||||||
db_filter=args.db, table_filter=args.table)
|
db_filter=args.db, table_filter=args.table)
|
||||||
report = format_report(results, args.granularity)
|
# Persist the report as a dated log file under the logging directory
|
||||||
print(report)
|
# (logs/ by default); --report still allows an extra custom path.
|
||||||
if args.report:
|
log_dir = os.path.dirname((cfg.logging or {}).get("path", "sync.log")) or "."
|
||||||
with open(args.report, "w", encoding="utf-8") as f:
|
written = write_report(results, args.granularity, log_dir=log_dir,
|
||||||
f.write(report + "\n")
|
extra_path=args.report)
|
||||||
|
print(format_report(results, args.granularity))
|
||||||
|
log.info("compare finished (granularity=%s) report=%s mismatch=%s",
|
||||||
|
args.granularity, written, any_mismatch(results))
|
||||||
return 1 if any_mismatch(results) else 0
|
return 1 if any_mismatch(results) else 0
|
||||||
|
|
||||||
return 2 # unreachable: argparse requires a subcommand
|
return 2 # unreachable: argparse requires a subcommand
|
||||||
|
|||||||
3
run_compare_ids.cmd
Normal file
3
run_compare_ids.cmd
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
@echo off
|
||||||
|
cd /d C:\Users\peng\Projects\ProductionDataBaseSync_DataMacro
|
||||||
|
.venv\Scripts\python.exe main.py compare --granularity ids >> logs\compare_task_stdout.log 2>&1
|
||||||
@@ -11,7 +11,9 @@ as full sync -- empirically confirming the two stay aligned.
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import datetime as _dt
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
from .config import FileMapping, SyncConfig
|
from .config import FileMapping, SyncConfig
|
||||||
@@ -139,3 +141,52 @@ def format_report(results: list[TableResult], granularity: str = "count") -> str
|
|||||||
else:
|
else:
|
||||||
lines.append(f"{base} access={r.access_count} sql={r.sql_count} [{r.status.upper()}]")
|
lines.append(f"{base} access={r.access_count} sql={r.sql_count} [{r.status.upper()}]")
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def summarize(results: list[TableResult]) -> dict:
|
||||||
|
"""Tally the outcome buckets: match / mismatch / skipped / error."""
|
||||||
|
s = {"match": 0, "mismatch": 0, "skipped": 0, "error": 0}
|
||||||
|
for r in results:
|
||||||
|
s[r.status] = s.get(r.status, 0) + 1
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def write_report(results: list[TableResult], granularity: str = "count",
|
||||||
|
log_dir: str = ".", run_dt: _dt.datetime | None = None,
|
||||||
|
extra_path: str | None = None) -> str:
|
||||||
|
"""Render the full report and persist it as a dated log file under *log_dir*.
|
||||||
|
|
||||||
|
Always writes ``<log_dir>/compare_<granularity>_<YYYY-MM-DD>.log``,
|
||||||
|
overwriting the day's previous run (one report per day; the logging system's
|
||||||
|
per-day archival later moves yesterday's file into ``logs/Archive/``). When
|
||||||
|
*extra_path* is given (the ``--report`` CLI option) the identical content is
|
||||||
|
also written there for backward compatibility. Returns the primary log path.
|
||||||
|
"""
|
||||||
|
run_dt = run_dt or _dt.datetime.now()
|
||||||
|
stats = summarize(results)
|
||||||
|
body = format_report(results, granularity)
|
||||||
|
header = (
|
||||||
|
"============================================================\n"
|
||||||
|
" 数据一致性核对报告 / Compare Report\n"
|
||||||
|
f" 生成时间 : {run_dt.strftime('%Y-%m-%d %H:%M:%S')}\n"
|
||||||
|
f" 粒度 : {granularity}\n"
|
||||||
|
f" 比对表合计 : {stats['match'] + stats['mismatch']}\n"
|
||||||
|
f" 一致 MATCH : {stats['match']}\n"
|
||||||
|
f" 不一致 MISMATCH : {stats['mismatch']}\n"
|
||||||
|
f" 跳过 SKIPPED : {stats['skipped']}\n"
|
||||||
|
f" 错误 ERROR : {stats['error']}\n"
|
||||||
|
"============================================================\n"
|
||||||
|
)
|
||||||
|
report = header + body + "\n"
|
||||||
|
|
||||||
|
log_dir = log_dir or "."
|
||||||
|
os.makedirs(log_dir, exist_ok=True)
|
||||||
|
primary = os.path.join(
|
||||||
|
log_dir, f"compare_{granularity}_{run_dt.strftime('%Y-%m-%d')}.log"
|
||||||
|
)
|
||||||
|
with open(primary, "w", encoding="utf-8") as f:
|
||||||
|
f.write(report)
|
||||||
|
if extra_path:
|
||||||
|
with open(extra_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(report)
|
||||||
|
return primary
|
||||||
|
|||||||
Reference in New Issue
Block a user