feat: --limit 全表扫描支持按真实 ID 排序/升降序/范围过滤
此前 --limit 仅按总排号排序取前 N 个。现扩展为:
- 排序按数据库真实 ID 列(config.id_field),新增 --order {asc,desc}(默认 asc)
- 新增 --range START,END 按真实 ID 闭区间过滤(与 --order/--limit 可组合)
- 上述三参数仅全表扫描生效,与 --id/--sn/--ids-file 互斥
db.py: fetch_all_ids 新增 order/id_min/id_max 参数,用派生表
(内层 DISTINCT 取 (总排号,ID) 配对,外层按 ID 排序)规避
SQL Server "SELECT DISTINCT 时 ORDER BY 列须在选择列表" 的限制;
id_field 未配置时降级按总排号排序并告警。
write_attachments.py: 新增 --order/--range CLI 参数并接线,含 --range
格式校验(须为 START,END 两个整数);模块 docstring 补示例。
README.md: 写入用法块与项目结构注释补充 --order/--range 说明。
自测(小数据量,只读+部分 dry-run):--limit 3 升/降序、--range 800,805
升序、--range 800,805 --order desc --limit 3 均验证返回总排号按真实 ID
正确排序且在范围内;dry-run 不落库;--range 格式错误正确报错退出。
Co-Authored-By: WorkBuddy <workbuddy@tencent.com>
This commit is contained in:
52
db.py
52
db.py
@@ -143,21 +143,57 @@ SENTINEL_MINOR = "无"
|
||||
NO_MINOR = "无"
|
||||
|
||||
|
||||
def fetch_all_ids(db_cfg: dict[str, Any], limit: int | None = None) -> list[str]:
|
||||
"""拉取源表全部总排号(去重、排除 NULL),用于全量分类写入。
|
||||
def fetch_all_ids(
|
||||
db_cfg: dict[str, Any],
|
||||
limit: int | None = None,
|
||||
order: str = "asc",
|
||||
id_min: int | None = None,
|
||||
id_max: int | None = None,
|
||||
) -> list[str]:
|
||||
"""拉取源表总排号列表,供全量/范围分类写入。
|
||||
|
||||
limit 用于测试时只取前 N 个(按总排号排序)。返回的总排号列表可直接
|
||||
喂给 classify_batch。
|
||||
排序按数据库真实 ID 列(config.id_field);未配置 id_field 时降级为按总排号
|
||||
列排序并告警。order 取 "asc" / "desc"。id_min / id_max 按真实 ID 列做闭区间
|
||||
过滤(含端点);任一为 None 表示不限制该侧。limit 取排序+过滤后的前 N 条。
|
||||
|
||||
返回的仍是总排号列表,可直接喂给 classify_batch(键类型统一为 sn)。
|
||||
用派生表(内层 DISTINCT 取 (总排号, ID) 配对,外层按 ID 排序)规避 SQL Server
|
||||
"SELECT DISTINCT 时 ORDER BY 列须出现在选择列表"的限制。
|
||||
"""
|
||||
schema = db_cfg["schema"]
|
||||
table = db_cfg["table"]
|
||||
id_col = db_cfg["id_column"]
|
||||
sn_col = db_cfg["id_column"] # 总排号列(返回列)
|
||||
id_field = db_cfg.get("id_field") # 真实 ID 列(排序/过滤用)
|
||||
qualified = f"[{schema}].[{table}]"
|
||||
sql = f"SELECT DISTINCT [{id_col}] FROM {qualified} WHERE [{id_col}] IS NOT NULL"
|
||||
params: list[Any] = []
|
||||
|
||||
order_dir = "DESC" if order == "desc" else "ASC"
|
||||
inner_where = [f"[{sn_col}] IS NOT NULL"]
|
||||
inner_params: list[Any] = []
|
||||
|
||||
if id_field:
|
||||
inner_where.append(f"[{id_field}] IS NOT NULL")
|
||||
if id_min is not None:
|
||||
inner_where.append(f"[{id_field}] >= ?")
|
||||
inner_params.append(int(id_min))
|
||||
if id_max is not None:
|
||||
inner_where.append(f"[{id_field}] <= ?")
|
||||
inner_params.append(int(id_max))
|
||||
order_expr = "[id]" # 派生表别名
|
||||
inner_select = f"SELECT DISTINCT [{sn_col}] AS sn, [{id_field}] AS id"
|
||||
else:
|
||||
logger.warning(
|
||||
"未配置 database.id_field,--limit/--range 将按总排号列排序(无法按真实 ID 排序/范围过滤)"
|
||||
)
|
||||
order_expr = "[sn]"
|
||||
inner_select = f"SELECT DISTINCT [{sn_col}] AS sn"
|
||||
|
||||
inner_sql = f"{inner_select} FROM {qualified} WHERE {' AND '.join(inner_where)}"
|
||||
sql = f"SELECT [sn] FROM ({inner_sql}) AS t ORDER BY {order_expr} {order_dir}"
|
||||
params = list(inner_params)
|
||||
if limit is not None:
|
||||
sql += f" ORDER BY [{id_col}] OFFSET 0 ROWS FETCH NEXT ? ROWS ONLY"
|
||||
sql += " OFFSET 0 ROWS FETCH NEXT ? ROWS ONLY"
|
||||
params.append(int(limit))
|
||||
|
||||
conn_str = _build_conn_str(db_cfg)
|
||||
try:
|
||||
with pyodbc.connect(conn_str, timeout=db_cfg["connect_timeout"]) as conn:
|
||||
|
||||
Reference in New Issue
Block a user