Character Encoding Reference

Practical Snippet

Key points: (1) strip the BOM on read, never add one to JSON or API responses, and do add it to CSV so Excel recognises UTF-8; (2) extensions carry no encoding information, so never branch on if (ext === ’.csv’); (3) detect in the order BOM → byte-level validation → user-specified → heuristics with a confidence score; (4) once double mojibake is persisted the information is gone and only the original data can restore it, so block it at the entry point; (5) allow UTF-8 end to end and convert only at explicit boundaries.

// ① 检测 BOM（读文件前必做）
function stripBom(buf) {
  const b = new Uint8Array(buf)
  if (b.length >= 3 && b[0] === 0xef && b[1] === 0xbb && b[2] === 0xbf) return b.slice(3)
  return b
}

// ② 按 BOM 判断 UTF-16 / UTF-8
function sniffEncoding(buf) {
  const b = new Uint8Array(buf)
  if (b.length >= 2 && b[0] === 0xff && b[1] === 0xfe) return 'utf-16le'
  if (b.length >= 2 && b[0] === 0xfe && b[1] === 0xff) return 'utf-16be'
  if (b.length >= 3 && b[0] === 0xef && b[1] === 0xbb && b[2] === 0xbf) return 'utf-8-bom'
  return 'unknown'   // 未命中就走内容探测或让用户选
}

// ③ 🔴 带 BOM 的 JSON 会让 JSON.parse 直接抛错
const badJson = new TextDecoder('utf-8').decode(
  new Uint8Array([0xef, 0xbb, 0xbf, ...new TextEncoder().encode('{"a":1}')]))
try { JSON.parse(badJson) } catch (e) { console.log('抛错:', e.message) }   // Unexpected token

// ④ 读取 CSV：先剥 BOM，再按声明/探测的编码解码
import { parse } from 'papaparse'
function readCsv(buf, encoding = 'utf-8') {
  const text = new TextDecoder(encoding).decode(stripBom(buf))
  return parse(text, { header: true, skipEmptyLines: true })
}

// ⑤ 写入侧：JSON 绝不加 BOM；CSV 加 BOM（让 Excel 正确识别 UTF-8）
const jsonStr = JSON.stringify({ a: 1 })
const csvStr = '' + 'name
age
'          // BOM 前置
console.log(jsonStr.charCodeAt(0) === 0xfeff)        // false ✅

// ⑥ 检测是否发生双重乱码（出现典型乱码字符）
const mojibake = /[ÃÂæ¼][€-￿]/
console.log(mojibake.test('æ¼ ä¸­æ–‡'))             // true ⇒ 需从原始数据恢复