修复 Access CSV 导入编码:数据 UTF-8 + schema.ini cp936
- 真数据含 GBK 外字符(如 ø),原 cp936 写 CSV 触发 UnicodeEncodeError - 数据 CSV 改 UTF-8 + schema.ini CharacterSet=65001(覆盖全部 Unicode) - schema.ini 列名是中文,Access Text ISAM 按系统 ANSI(cp936) 读 ini,故 ini 仍用 cp936 写(仅数据文件用 UTF-8) - _schema_type 列宽为 None 时兜底 255(规避 Text Width 0 非法) - 本地集成测试(真实 sync_access, 35043 行, 含 ø)通过:行数+校验和一致
This commit is contained in:
@@ -131,7 +131,9 @@ def _schema_type(col, meta):
|
|||||||
tn, size = meta.get(col, ("", None))
|
tn, size = meta.get(col, ("", None))
|
||||||
cat = _TYPE_MAP.get(tn, "text")
|
cat = _TYPE_MAP.get(tn, "text")
|
||||||
if cat == "text":
|
if cat == "text":
|
||||||
return f"Text Width {size if size else 0}"
|
# ACE 对部分表(尤其 ADOX 建的)报告的列宽可能为 None,
|
||||||
|
# Text ISAM 不接受 Width 0,故无宽度时兜底 255
|
||||||
|
return f"Text Width {size if size else 255}"
|
||||||
return {"date": "DateTime", "int": "Integer",
|
return {"date": "DateTime", "int": "Integer",
|
||||||
"float": "Double", "bit": "Bit"}.get(cat, "Text Width 0")
|
"float": "Double", "bit": "Bit"}.get(cat, "Text Width 0")
|
||||||
|
|
||||||
@@ -151,23 +153,26 @@ def _csv_val(v):
|
|||||||
def _access_csv_import(cur, table_expr, insert_cols, san_rows, meta):
|
def _access_csv_import(cur, table_expr, insert_cols, san_rows, meta):
|
||||||
"""CSV + Text 驱动单语句批量导入(规避 executemany 逐行往返开销)。
|
"""CSV + Text 驱动单语句批量导入(规避 executemany 逐行往返开销)。
|
||||||
|
|
||||||
把 san_rows 写成临时 Tab 分隔 CSV(GBK/cp936,匹配中文 Windows ANSI),
|
把 san_rows 写成临时 Tab 分隔 CSV(UTF-8),动态生成 schema.ini
|
||||||
动态生成 schema.ini,再用一条 INSERT...SELECT FROM [Text;...] 整批写入。
|
(CharacterSet=65001 即 UTF-8,覆盖全部 Unicode,含 GBK 之外的字符如 ø),
|
||||||
|
再用一条 INSERT...SELECT FROM [Text;...] 整批写入。
|
||||||
临时目录用系统 TEMP(本地快盘),仅最终写入跨到目标库一次。
|
临时目录用系统 TEMP(本地快盘),仅最终写入跨到目标库一次。
|
||||||
"""
|
"""
|
||||||
tmp = tempfile.mkdtemp(prefix="pvhub_csv_")
|
tmp = tempfile.mkdtemp(prefix="pvhub_csv_")
|
||||||
csv_path = os.path.join(tmp, "import.csv")
|
csv_path = os.path.join(tmp, "import.csv")
|
||||||
ini_path = os.path.join(tmp, "schema.ini")
|
ini_path = os.path.join(tmp, "schema.ini")
|
||||||
try:
|
try:
|
||||||
with open(csv_path, "w", encoding="cp936", newline="") as f:
|
with open(csv_path, "w", encoding="utf-8", newline="") as f:
|
||||||
w = csv.writer(f, delimiter="\t")
|
w = csv.writer(f, delimiter="\t")
|
||||||
for row in san_rows:
|
for row in san_rows:
|
||||||
w.writerow([_csv_val(v) for v in row])
|
w.writerow([_csv_val(v) for v in row])
|
||||||
|
# 注意:schema.ini 的列名是中文,Access Text ISAM 按系统 ANSI(cp936) 读 ini,
|
||||||
|
# 故 ini 必须用 cp936 写;数据 CSV 才用 UTF-8 + CharacterSet=65001。
|
||||||
with open(ini_path, "w", encoding="cp936") as f:
|
with open(ini_path, "w", encoding="cp936") as f:
|
||||||
f.write("[import.csv]\r\n")
|
f.write("[import.csv]\r\n")
|
||||||
f.write("Format=TabDelimited\r\n")
|
f.write("Format=TabDelimited\r\n")
|
||||||
f.write("ColNameHeader=False\r\n")
|
f.write("ColNameHeader=False\r\n")
|
||||||
f.write("CharacterSet=936\r\n")
|
f.write("CharacterSet=65001\r\n")
|
||||||
for i, c in enumerate(insert_cols, 1):
|
for i, c in enumerate(insert_cols, 1):
|
||||||
f.write(f"Col{i}={c} {_schema_type(c, meta)}\r\n")
|
f.write(f"Col{i}={c} {_schema_type(c, meta)}\r\n")
|
||||||
sel = ",".join(f"[{c}]" for c in insert_cols)
|
sel = ",".join(f"[{c}]" for c in insert_cols)
|
||||||
|
|||||||
Reference in New Issue
Block a user