Text Processing Cheat Sheet

Practical Snippet

Key points: (1) text often carries zero-width characters (U+200B–U+200D), non-breaking spaces (U+00A0), BOMs and quote variants that make identical-looking text fail to match — normalise with NFKC and strip zero-width characters before comparing, and print character codes instead of eyeballing; (2) CSV needs a real parser because RFC 4180 permits newlines inside quoted fields, so splitting by line cuts one record in two; (3) count matches before replacing and anchor with character classes — a bare dot in a regex matches any character (a.c matches abc) and a global dot replacement corrupts decimals and version numbers; (4) deduplicate by business key, not by whole lines.

// ① 规范化：去不可见字符 + NFKC + 引号统一
const ZERO_WIDTH = /[​-‍]/g
const QUOTES = { \u2018: "'", \u2019: "'", \u201C: '"', \u201D: '"' }

function normalizeText(s) {
  return s
    .replace(ZERO_WIDTH, '')                 // 零宽字符与 BOM
    .replace(/ /g, ' ')                 // 不换行空格 → 普通空格
    .replace(/[‘’]/g, "'")         // 中文引号 → ASCII
    .replace(/[“”]/g, '"')
    .normalize('NFKC')                        // 全角 → 半角、兼容字符归一
    .replace(/
/g, '
')                 // 换行统一
    .trim()
}
console.log(normalizeText('a​b c').length)   // 3 而不是 5

// ② 调试时打印字符码而不是肉眼比对
function showCodes(s) {
  return [...s].map(c => {
    const cp = c.codePointAt(0)
    const hex = cp.toString(16).toUpperCase().padStart(4, '0')
  // 用普通字符串而非模板串：这一行只是演示如何显示字符码，
  // 用模板串会与本页大量出现的美元符号加大括号字面量冲突（TS 会当插值解析）。
  return hex + (cp < 128 ? ' ' : ' ')
  }).join(' ')
}
// console.log(showCodes('a​b'))  → 0061 200B 0062

// ③ CSV 必须用解析器，不能按行 split（字段内可能有换行）
import { parse } from 'papaparse'
const csv = 'a,b
"line1
line2",c'         // 第二行含引号包裹的换行
const wrong = csv.split('
').filter(Boolean)      // ❌ 拆成 3 段
const right = parse(csv, { header: true }).data     // ✅ 2 条记录
console.log(wrong.length, right.length)              // 3 2

// ④ 替换前先统计命中数，且用字符类限定边界
function countMatches(text, re) {
  return (text.match(new RegExp(re.source, 'g')) || []).length
}
const text = '1.5 and 1.2.3 and v1.0 and 2.25'
// ❌ /./g 会命中所有点：4 处
// ✅ 只匹配小数：d+.d+
console.log('所有点:', countMatches(text, /./g))        // 4
console.log('仅小数:', countMatches(text, /\d+\.\d+/g))  // 2
const fixed = text.replace(/d+.d+/g, m => m.replace('.', ','))
console.log(fixed)   // 1,5 and 1.2.3 and v1.0 and 2,25 ← 版本号未被动

// ⑤ 去重排序要按业务键，不是按整行
const lines = ['b,2', 'a,1', 'b,2']
const seen = new Set()
const uniq = lines.filter(l => { if (seen.has(l)) return false; seen.add(l); return true })
console.log(uniq)   // ['b,2', 'a,1'] —— 重复的 b,2 被去掉