multi-byte.js 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962
  1. import { E_STRING } from './_utils.js'
  2. import { nativeEncoder } from './platform.js'
  3. import { asciiPrefix, decodeAscii, decodeLatin1, decodeUCS2 } from './latin1.js'
  4. import { getTable } from './multi-byte.table.js'
  5. export const E_STRICT = 'Input is not well-formed for this encoding'
  6. /* Decoders */
  7. // If the decoder is not cleared properly, state can be preserved between non-streaming calls!
  8. // See comment about fatal stream
  9. // All except iso-2022-jp are ASCII supersets
  10. // When adding something that is not an ASCII superset, ajust the ASCII fast path
  11. const mappers = {
  12. // https://encoding.spec.whatwg.org/#euc-kr-decoder
  13. 'euc-kr': (err) => {
  14. const euc = getTable('euc-kr')
  15. let lead = 0
  16. let oi = 0
  17. let o16
  18. const decodeLead = (b) => {
  19. if (b < 0x41 || b > 0xfe) {
  20. lead = 0
  21. o16[oi++] = err()
  22. if (b < 128) o16[oi++] = b
  23. } else {
  24. const p = euc[(lead - 0x81) * 190 + b - 0x41]
  25. lead = 0
  26. if (p) {
  27. o16[oi++] = p
  28. } else {
  29. o16[oi++] = err()
  30. if (b < 128) o16[oi++] = b
  31. }
  32. }
  33. }
  34. const decode = (arr, start, end, stream) => {
  35. let i = start
  36. o16 = new Uint16Array(end - start + (lead ? 1 : 0)) // there are pairs but they consume more than one byte
  37. oi = 0
  38. // Fast path
  39. if (!lead) {
  40. for (const last1 = end - 1; i < last1; ) {
  41. const l = arr[i]
  42. if (l < 128) {
  43. o16[oi++] = l
  44. i++
  45. } else {
  46. if (l === 0x80 || l === 0xff) break
  47. const b = arr[i + 1]
  48. if (b < 0x41 || b === 0xff) break
  49. const p = euc[(l - 0x81) * 190 + b - 0x41]
  50. if (!p) break
  51. o16[oi++] = p
  52. i += 2
  53. }
  54. }
  55. }
  56. if (lead && i < end) decodeLead(arr[i++])
  57. while (i < end) {
  58. const b = arr[i++]
  59. if (b < 128) {
  60. o16[oi++] = b
  61. } else if (b === 0x80 || b === 0xff) {
  62. o16[oi++] = err()
  63. } else {
  64. lead = b
  65. if (i < end) decodeLead(arr[i++])
  66. }
  67. }
  68. if (lead && !stream) {
  69. lead = 0
  70. o16[oi++] = err()
  71. }
  72. const res = decodeUCS2(o16, oi)
  73. o16 = null
  74. return res
  75. }
  76. return { decode, isAscii: () => lead === 0 }
  77. },
  78. // https://encoding.spec.whatwg.org/#euc-jp-decoder
  79. 'euc-jp': (err) => {
  80. const jis0208 = getTable('jis0208')
  81. const jis0212 = getTable('jis0212')
  82. let j12 = false
  83. let lead = 0
  84. let oi = 0
  85. let o16
  86. const decodeLead = (b) => {
  87. if (lead === 0x8e && b >= 0xa1 && b <= 0xdf) {
  88. lead = 0
  89. o16[oi++] = 0xfe_c0 + b
  90. } else if (lead === 0x8f && b >= 0xa1 && b <= 0xfe) {
  91. j12 = true
  92. lead = b
  93. } else {
  94. let cp
  95. if (lead >= 0xa1 && lead <= 0xfe && b >= 0xa1 && b <= 0xfe) {
  96. cp = (j12 ? jis0212 : jis0208)[(lead - 0xa1) * 94 + b - 0xa1]
  97. }
  98. lead = 0
  99. j12 = false
  100. if (cp) {
  101. o16[oi++] = cp
  102. } else {
  103. o16[oi++] = err()
  104. if (b < 128) o16[oi++] = b
  105. }
  106. }
  107. }
  108. const decode = (arr, start, end, stream) => {
  109. let i = start
  110. o16 = new Uint16Array(end - start + (lead ? 1 : 0))
  111. oi = 0
  112. // Fast path, non-j12
  113. // lead = 0 means j12 = 0
  114. if (!lead) {
  115. for (const last1 = end - 1; i < last1; ) {
  116. const l = arr[i]
  117. if (l < 128) {
  118. o16[oi++] = l
  119. i++
  120. } else {
  121. const b = arr[i + 1]
  122. if (l === 0x8e && b >= 0xa1 && b <= 0xdf) {
  123. o16[oi++] = 0xfe_c0 + b
  124. i += 2
  125. } else {
  126. if (l < 0xa1 || l === 0xff || b < 0xa1 || b === 0xff) break
  127. const cp = jis0208[(l - 0xa1) * 94 + b - 0xa1]
  128. if (!cp) break
  129. o16[oi++] = cp
  130. i += 2
  131. }
  132. }
  133. }
  134. }
  135. if (lead && i < end) decodeLead(arr[i++])
  136. if (lead && i < end) decodeLead(arr[i++]) // could be two leads, but no more
  137. while (i < end) {
  138. const b = arr[i++]
  139. if (b < 128) {
  140. o16[oi++] = b
  141. } else if ((b < 0xa1 && b !== 0x8e && b !== 0x8f) || b === 0xff) {
  142. o16[oi++] = err()
  143. } else {
  144. lead = b
  145. if (i < end) decodeLead(arr[i++])
  146. if (lead && i < end) decodeLead(arr[i++]) // could be two leads
  147. }
  148. }
  149. if (lead && !stream) {
  150. lead = 0
  151. j12 = false // can be true only when lead is non-zero
  152. o16[oi++] = err()
  153. }
  154. const res = decodeUCS2(o16, oi)
  155. o16 = null
  156. return res
  157. }
  158. return { decode, isAscii: () => lead === 0 } // j12 can be true only when lead is non-zero
  159. },
  160. // https://encoding.spec.whatwg.org/#iso-2022-jp-decoder
  161. 'iso-2022-jp': (err) => {
  162. const jis0208 = getTable('jis0208')
  163. let dState = 1
  164. let oState = 1
  165. let lead = 0 // 0 or 0x21-0x7e
  166. let out = false
  167. const bytes = (pushback, b) => {
  168. if (dState < 5 && b === 0x1b) {
  169. dState = 6 // escape start
  170. return
  171. }
  172. switch (dState) {
  173. case 1:
  174. case 2:
  175. // ASCII, Roman (common)
  176. out = false
  177. if (dState === 2) {
  178. if (b === 0x5c) return 0xa5
  179. if (b === 0x7e) return 0x20_3e
  180. }
  181. if (b <= 0x7f && b !== 0x0e && b !== 0x0f) return b
  182. return err()
  183. case 3:
  184. // Katakana
  185. out = false
  186. if (b >= 0x21 && b <= 0x5f) return 0xff_40 + b
  187. return err()
  188. case 4:
  189. // Leading byte
  190. out = false
  191. if (b < 0x21 || b > 0x7e) return err()
  192. lead = b
  193. dState = 5
  194. return
  195. case 5:
  196. // Trailing byte
  197. out = false
  198. if (b === 0x1b) {
  199. dState = 6 // escape start
  200. return err()
  201. }
  202. dState = 4
  203. if (b >= 0x21 && b <= 0x7e) {
  204. const cp = jis0208[(lead - 0x21) * 94 + b - 0x21]
  205. if (cp) return cp
  206. }
  207. return err()
  208. case 6:
  209. // Escape start
  210. if (b === 0x24 || b === 0x28) {
  211. lead = b
  212. dState = 7
  213. return
  214. }
  215. out = false
  216. dState = oState
  217. pushback.push(b)
  218. return err()
  219. case 7: {
  220. // Escape
  221. const l = lead
  222. lead = 0
  223. let s
  224. if (l === 0x28) {
  225. // eslint-disable-next-line unicorn/prefer-switch
  226. if (b === 0x42) {
  227. s = 1
  228. } else if (b === 0x4a) {
  229. s = 2
  230. } else if (b === 0x49) {
  231. s = 3
  232. }
  233. } else if (l === 0x24 && (b === 0x40 || b === 0x42)) {
  234. s = 4
  235. }
  236. if (s) {
  237. dState = oState = s
  238. const output = out
  239. out = true
  240. return output ? err() : undefined
  241. }
  242. out = false
  243. dState = oState
  244. pushback.push(b, l)
  245. return err()
  246. }
  247. }
  248. }
  249. const eof = (pushback) => {
  250. if (dState < 5) return null
  251. out = false
  252. switch (dState) {
  253. case 5:
  254. dState = 4
  255. return err()
  256. case 6:
  257. dState = oState
  258. return err()
  259. case 7: {
  260. dState = oState
  261. pushback.push(lead)
  262. lead = 0
  263. return err()
  264. }
  265. }
  266. }
  267. const decode = (arr, start, end, stream) => {
  268. const o16 = new Uint16Array(end - start + 2) // err in eof + lead from state
  269. let oi = 0
  270. let i = start
  271. const pushback = [] // local and auto-cleared
  272. // First, dump everything until EOF
  273. // Same as the full loop, but without EOF handling
  274. while (i < end || pushback.length > 0) {
  275. const c = bytes(pushback, pushback.length > 0 ? pushback.pop() : arr[i++])
  276. if (c !== undefined) o16[oi++] = c // 16-bit
  277. }
  278. // Then, dump EOF. This needs the same loop as the characters can be pushed back
  279. if (!stream) {
  280. while (i <= end || pushback.length > 0) {
  281. if (i < end || pushback.length > 0) {
  282. const c = bytes(pushback, pushback.length > 0 ? pushback.pop() : arr[i++])
  283. if (c !== undefined) o16[oi++] = c // 16-bit
  284. } else {
  285. const c = eof(pushback)
  286. if (c === null) break // clean exit
  287. o16[oi++] = c
  288. }
  289. }
  290. }
  291. // Chrome and WebKit fail on this, we don't: completely destroy the old decoder state when finished streaming
  292. // > If this’s do not flush is false, then set this’s decoder to a new instance of this’s encoding’s decoder,
  293. // > Set this’s do not flush to options["stream"]
  294. if (!stream) {
  295. dState = oState = 1
  296. lead = 0
  297. out = false
  298. }
  299. return decodeUCS2(o16, oi)
  300. }
  301. return { decode, isAscii: () => false }
  302. },
  303. // https://encoding.spec.whatwg.org/#shift_jis-decoder
  304. shift_jis: (err) => {
  305. const jis0208 = getTable('jis0208')
  306. let lead = 0
  307. let oi = 0
  308. let o16
  309. const decodeLead = (b) => {
  310. const l = lead
  311. lead = 0
  312. if (b >= 0x40 && b <= 0xfc && b !== 0x7f) {
  313. const p = (l - (l < 0xa0 ? 0x81 : 0xc1)) * 188 + b - (b < 0x7f ? 0x40 : 0x41)
  314. if (p >= 8836 && p <= 10_715) {
  315. o16[oi++] = 0xe0_00 - 8836 + p
  316. return
  317. }
  318. const cp = jis0208[p]
  319. if (cp) {
  320. o16[oi++] = cp
  321. return
  322. }
  323. }
  324. o16[oi++] = err()
  325. if (b < 128) o16[oi++] = b
  326. }
  327. const decode = (arr, start, end, stream) => {
  328. o16 = new Uint16Array(end - start + (lead ? 1 : 0))
  329. oi = 0
  330. let i = start
  331. // Fast path
  332. if (!lead) {
  333. for (const last1 = end - 1; i < last1; ) {
  334. const l = arr[i]
  335. if (l <= 0x80) {
  336. o16[oi++] = l
  337. i++
  338. } else if (l >= 0xa1 && l <= 0xdf) {
  339. o16[oi++] = 0xfe_c0 + l
  340. i++
  341. } else {
  342. if (l === 0xa0 || l > 0xfc) break
  343. const b = arr[i + 1]
  344. if (b < 0x40 || b > 0xfc || b === 0x7f) break
  345. const p = (l - (l < 0xa0 ? 0x81 : 0xc1)) * 188 + b - (b < 0x7f ? 0x40 : 0x41)
  346. if (p >= 8836 && p <= 10_715) {
  347. o16[oi++] = 0xe0_00 - 8836 + p
  348. i += 2
  349. } else {
  350. const cp = jis0208[p]
  351. if (!cp) break
  352. o16[oi++] = cp
  353. i += 2
  354. }
  355. }
  356. }
  357. }
  358. if (lead && i < end) decodeLead(arr[i++])
  359. while (i < end) {
  360. const b = arr[i++]
  361. if (b <= 0x80) {
  362. o16[oi++] = b // 0x80 is allowed
  363. } else if (b >= 0xa1 && b <= 0xdf) {
  364. o16[oi++] = 0xfe_c0 + b
  365. } else if (b === 0xa0 || b > 0xfc) {
  366. o16[oi++] = err()
  367. } else {
  368. lead = b
  369. if (i < end) decodeLead(arr[i++])
  370. }
  371. }
  372. if (lead && !stream) {
  373. lead = 0
  374. o16[oi++] = err()
  375. }
  376. const res = decodeUCS2(o16, oi)
  377. o16 = null
  378. return res
  379. }
  380. return { decode, isAscii: () => lead === 0 }
  381. },
  382. // https://encoding.spec.whatwg.org/#gbk-decoder
  383. gbk: (err) => mappers.gb18030(err), // 10.1.1. GBK’s decoder is gb18030’s decoder
  384. // https://encoding.spec.whatwg.org/#gb18030-decoder
  385. gb18030: (err) => {
  386. const gb18030 = getTable('gb18030')
  387. const gb18030r = getTable('gb18030-ranges')
  388. let g1 = 0, g2 = 0, g3 = 0 // prettier-ignore
  389. const index = (p) => {
  390. if ((p > 39_419 && p < 189_000) || p > 1_237_575) return
  391. if (p === 7457) return 0xe7_c7
  392. let a = 0, b = 0 // prettier-ignore
  393. for (const [c, d] of gb18030r) {
  394. if (c > p) break
  395. a = c
  396. b = d
  397. }
  398. return b + p - a
  399. }
  400. // g1 is 0 or 0x81-0xfe
  401. // g2 is 0 or 0x30-0x39
  402. // g3 is 0 or 0x81-0xfe
  403. const decode = (arr, start, end, stream) => {
  404. const o16 = new Uint16Array(end - start + (g1 ? 3 : 0)) // even with pushback it's at most 1 char per byte
  405. let oi = 0
  406. let i = start
  407. const pushback = [] // local and auto-cleared
  408. // Fast path for 2-byte only
  409. // pushback is always empty ad start, and g1 = 0 means g2 = g3 = 0
  410. if (g1 === 0) {
  411. for (const last1 = end - 1; i < last1; ) {
  412. const b = arr[i]
  413. if (b < 128) {
  414. o16[oi++] = b
  415. i++
  416. } else if (b === 0x80) {
  417. o16[oi++] = 0x20_ac
  418. i++
  419. } else {
  420. if (b === 0xff) break
  421. const n = arr[i + 1]
  422. let cp
  423. if (n < 0x7f) {
  424. if (n < 0x40) break
  425. cp = gb18030[(b - 0x81) * 190 + n - 0x40]
  426. } else {
  427. if (n === 0xff || n === 0x7f) break
  428. cp = gb18030[(b - 0x81) * 190 + n - 0x41]
  429. }
  430. if (!cp) break
  431. o16[oi++] = cp // 16-bit
  432. i += 2
  433. }
  434. }
  435. }
  436. // First, dump everything until EOF
  437. // Same as the full loop, but without EOF handling
  438. while (i < end || pushback.length > 0) {
  439. const b = pushback.length > 0 ? pushback.pop() : arr[i++]
  440. if (g1) {
  441. // g2 can be set only when g1 is set, g3 can be set only when g2 is set
  442. // hence, 3 checks for g3 is faster than 3 checks for g1
  443. if (g2) {
  444. if (g3) {
  445. if (b <= 0x39 && b >= 0x30) {
  446. const p = index(
  447. (g1 - 0x81) * 12_600 + (g2 - 0x30) * 1260 + (g3 - 0x81) * 10 + b - 0x30
  448. )
  449. g1 = g2 = g3 = 0
  450. if (p === undefined) {
  451. o16[oi++] = err()
  452. } else if (p <= 0xff_ff) {
  453. o16[oi++] = p // Can validly return replacement
  454. } else {
  455. const d = p - 0x1_00_00
  456. o16[oi++] = 0xd8_00 | (d >> 10)
  457. o16[oi++] = 0xdc_00 | (d & 0x3_ff)
  458. }
  459. } else {
  460. pushback.push(b, g3, g2)
  461. g1 = g2 = g3 = 0
  462. o16[oi++] = err()
  463. }
  464. } else if (b >= 0x81 && b <= 0xfe) {
  465. g3 = b
  466. } else {
  467. pushback.push(b, g2)
  468. g1 = g2 = 0
  469. o16[oi++] = err()
  470. }
  471. } else if (b <= 0x39 && b >= 0x30) {
  472. g2 = b
  473. } else {
  474. let cp
  475. if (b >= 0x40 && b <= 0xfe && b !== 0x7f) {
  476. cp = gb18030[(g1 - 0x81) * 190 + b - (b < 0x7f ? 0x40 : 0x41)]
  477. }
  478. g1 = 0
  479. if (cp) {
  480. o16[oi++] = cp // 16-bit
  481. } else {
  482. o16[oi++] = err()
  483. if (b < 128) o16[oi++] = b // can be processed immediately
  484. }
  485. }
  486. } else if (b < 128) {
  487. o16[oi++] = b
  488. } else if (b === 0x80) {
  489. o16[oi++] = 0x20_ac
  490. } else if (b === 0xff) {
  491. o16[oi++] = err()
  492. } else {
  493. g1 = b
  494. }
  495. }
  496. // if g1 = 0 then g2 = g3 = 0
  497. if (g1 && !stream) {
  498. g1 = g2 = g3 = 0
  499. o16[oi++] = err()
  500. }
  501. return decodeUCS2(o16, oi)
  502. }
  503. return { decode, isAscii: () => g1 === 0 } // if g1 = 0 then g2 = g3 = 0
  504. },
  505. // https://encoding.spec.whatwg.org/#big5
  506. big5: (err) => {
  507. // The only decoder which returns multiple codepoints per byte, also has non-charcode codepoints
  508. // We store that as strings
  509. const big5 = getTable('big5')
  510. let lead = 0
  511. let oi = 0
  512. let o16
  513. const decodeLead = (b) => {
  514. if (b < 0x40 || (b > 0x7e && b < 0xa1) || b === 0xff) {
  515. lead = 0
  516. o16[oi++] = err()
  517. if (b < 128) o16[oi++] = b
  518. } else {
  519. const p = big5[(lead - 0x81) * 157 + b - (b < 0x7f ? 0x40 : 0x62)]
  520. lead = 0
  521. if (p > 0x1_00_00) {
  522. o16[oi++] = p >> 16
  523. o16[oi++] = p & 0xff_ff
  524. } else if (p) {
  525. o16[oi++] = p
  526. } else {
  527. o16[oi++] = err()
  528. if (b < 128) o16[oi++] = b
  529. }
  530. }
  531. }
  532. // eslint-disable-next-line sonarjs/no-identical-functions
  533. const decode = (arr, start, end, stream) => {
  534. let i = start
  535. o16 = new Uint16Array(end - start + (lead ? 1 : 0)) // there are pairs but they consume more than one byte
  536. oi = 0
  537. // Fast path
  538. if (!lead) {
  539. for (const last1 = end - 1; i < last1; ) {
  540. const l = arr[i]
  541. if (l < 128) {
  542. o16[oi++] = l
  543. i++
  544. } else {
  545. if (l === 0x80 || l === 0xff) break
  546. const b = arr[i + 1]
  547. if (b < 0x40 || (b > 0x7e && b < 0xa1) || b === 0xff) break
  548. const p = big5[(l - 0x81) * 157 + b - (b < 0x7f ? 0x40 : 0x62)]
  549. if (p > 0x1_00_00) {
  550. o16[oi++] = p >> 16
  551. o16[oi++] = p & 0xff_ff
  552. } else {
  553. if (!p) break
  554. o16[oi++] = p
  555. }
  556. i += 2
  557. }
  558. }
  559. }
  560. if (lead && i < end) decodeLead(arr[i++])
  561. while (i < end) {
  562. const b = arr[i++]
  563. if (b < 128) {
  564. o16[oi++] = b
  565. } else if (b === 0x80 || b === 0xff) {
  566. o16[oi++] = err()
  567. } else {
  568. lead = b
  569. if (i < end) decodeLead(arr[i++])
  570. }
  571. }
  572. if (lead && !stream) {
  573. lead = 0
  574. o16[oi++] = err()
  575. }
  576. const res = decodeUCS2(o16, oi)
  577. o16 = null
  578. return res
  579. }
  580. return { decode, isAscii: () => lead === 0 }
  581. },
  582. }
  583. export const isAsciiSuperset = (enc) => enc !== 'iso-2022-jp' // all others are ASCII supersets and can use fast path
  584. export function multibyteDecoder(enc, loose = false) {
  585. if (typeof loose !== 'boolean') throw new TypeError('loose option should be boolean')
  586. if (!Object.hasOwn(mappers, enc)) throw new RangeError('Unsupported encoding')
  587. // Input is assumed to be typechecked already
  588. let mapper
  589. const asciiSuperset = isAsciiSuperset(enc)
  590. let streaming // because onErr is cached in mapper
  591. const onErr = loose
  592. ? () => 0xff_fd
  593. : () => {
  594. // The correct way per spec seems to be not destoying the decoder state in stream mode, even when fatal
  595. // Decoders big5, euc-jp, euc-kr, shift_jis, gb18030 / gbk - all clear state before throwing unless EOF, so not affected
  596. // iso-2022-jp is the only tricky one one where this !stream check matters in non-stream mode
  597. if (!streaming) mapper = null // destroy state, effectively the same as 'do not flush' = false, but early
  598. throw new TypeError(E_STRICT)
  599. }
  600. return (arr, stream = false) => {
  601. let res = ''
  602. if (asciiSuperset && (!mapper || mapper.isAscii?.())) {
  603. const prefixLen = asciiPrefix(arr)
  604. if (prefixLen === arr.length) return decodeAscii(arr) // ascii
  605. res = decodeLatin1(arr, 0, prefixLen) // TODO: check if decodeAscii with subarray is faster for small prefixes too
  606. }
  607. streaming = stream // affects onErr
  608. if (!mapper) mapper = mappers[enc](onErr)
  609. return res + mapper.decode(arr, res.length, arr.length, stream)
  610. }
  611. }
  612. /* Encoders */
  613. const maps = new Map()
  614. const e7 = [[148, 236], [149, 237], [150, 243]] // prettier-ignore
  615. const e8 = [[30, 89], [38, 97], [43, 102], [44, 103], [50, 109], [67, 126], [84, 144], [100, 160]] // prettier-ignore
  616. const preencoders = {
  617. __proto__: null,
  618. big5: (p) => ((((p / 157) | 0) + 0x81) << 8) | ((p % 157 < 0x3f ? 0x40 : 0x62) + (p % 157)),
  619. shift_jis: (p) => {
  620. const l = (p / 188) | 0
  621. const t = p % 188
  622. return ((l + (l < 0x1f ? 0x81 : 0xc1)) << 8) | ((t < 0x3f ? 0x40 : 0x41) + t)
  623. },
  624. 'iso-2022-jp': (p) => ((((p / 94) | 0) + 0x21) << 8) | ((p % 94) + 0x21),
  625. 'euc-jp': (p) => ((((p / 94) | 0) + 0xa1) << 8) | ((p % 94) + 0xa1),
  626. 'euc-kr': (p) => ((((p / 190) | 0) + 0x81) << 8) | ((p % 190) + 0x41),
  627. gb18030: (p) => ((((p / 190) | 0) + 0x81) << 8) | ((p % 190 < 0x3f ? 0x40 : 0x41) + (p % 190)),
  628. }
  629. preencoders.gbk = preencoders.gb18030
  630. // We accept that encoders use non-trivial amount of mem, for perf
  631. // most are are 128 KiB mem, big5 is 380 KiB, lazy-loaded at first use
  632. function getMap(id, size, ascii) {
  633. const cached = maps.get(id)
  634. if (cached) return cached
  635. let tname = id
  636. const sjis = id === 'shift_jis'
  637. const iso2022jp = id === 'iso-2022-jp'
  638. if (iso2022jp) tname = 'jis0208'
  639. if (id === 'gbk') tname = 'gb18030'
  640. if (id === 'euc-jp' || sjis) tname = 'jis0208'
  641. const table = getTable(tname)
  642. const map = new Uint16Array(size)
  643. const enc = preencoders[id] || ((p) => p + 1)
  644. for (let i = 0; i < table.length; i++) {
  645. const c = table[i]
  646. if (!c) continue
  647. if (id === 'big5') {
  648. if (i < 5024) continue // this also skips multi-codepoint strings
  649. // In big5, all return first entries except for these
  650. if (
  651. map[c] &&
  652. c !== 0x25_50 &&
  653. c !== 0x25_5e &&
  654. c !== 0x25_61 &&
  655. c !== 0x25_6a &&
  656. c !== 0x53_41 &&
  657. c !== 0x53_45
  658. ) {
  659. continue
  660. }
  661. } else {
  662. if (sjis && i >= 8272 && i <= 8835) continue
  663. if (map[c]) continue
  664. }
  665. if (c > 0xff_ff) {
  666. // always a single codepoint here
  667. const s = String.fromCharCode(c >> 16, c & 0xff_ff)
  668. map[s.codePointAt(0)] = enc(i)
  669. } else {
  670. map[c] = enc(i)
  671. }
  672. }
  673. if (ascii) for (let i = 0; i < 0x80; i++) map[i] = i
  674. if (sjis || id === 'euc-jp') {
  675. if (sjis) map[0x80] = 0x80
  676. const d = sjis ? 0xfe_c0 : 0x70_c0
  677. for (let i = 0xff_61; i <= 0xff_9f; i++) map[i] = i - d
  678. map[0x22_12] = map[0xff_0d]
  679. map[0xa5] = 0x5c
  680. map[0x20_3e] = 0x7e
  681. } else if (tname === 'gb18030') {
  682. if (id === 'gbk') map[0x20_ac] = 0x80
  683. for (let i = 0xe7_8d; i <= 0xe7_93; i++) map[i] = i - 0x40_b4
  684. for (const [a, b] of e7) map[0xe7_00 | a] = 0xa6_00 | b
  685. for (const [a, b] of e8) map[0xe8_00 | a] = 0xfe_00 | b
  686. }
  687. maps.set(id, map)
  688. return map
  689. }
  690. const NON_LATIN = /[^\x00-\xFF]/ // eslint-disable-line no-control-regex
  691. let gb18030r, katakana
  692. export function multibyteEncoder(enc, onError) {
  693. if (!Object.hasOwn(mappers, enc)) throw new RangeError('Unsupported encoding')
  694. const size = enc === 'big5' ? 0x2_f8_a7 : 0x1_00_00 // for big5, max codepoint in table + 1
  695. const iso2022jp = enc === 'iso-2022-jp'
  696. const gb18030 = enc === 'gb18030'
  697. const ascii = isAsciiSuperset(enc)
  698. const width = iso2022jp ? 5 : gb18030 ? 4 : 2
  699. const tailsize = iso2022jp ? 3 : 0
  700. const map = getMap(enc, size, ascii)
  701. if (gb18030 && !gb18030r) gb18030r = getTable('gb18030-ranges')
  702. if (iso2022jp && !katakana) katakana = getTable('iso-2022-jp-katakana')
  703. return (str) => {
  704. if (typeof str !== 'string') throw new TypeError(E_STRING)
  705. if (ascii && nativeEncoder && !NON_LATIN.test(str)) {
  706. const u8 = nativeEncoder.encode(str)
  707. if (u8.length === str.length) return u8
  708. }
  709. const length = str.length
  710. const u8 = new Uint8Array(length * width + tailsize)
  711. let i = 0
  712. if (ascii) {
  713. while (i < length) {
  714. const x = str.charCodeAt(i)
  715. if (x >= 128) break
  716. u8[i++] = x
  717. }
  718. }
  719. // eslint-disable-next-line unicorn/consistent-function-scoping
  720. const err = (code) => {
  721. if (onError) return onError(code, u8, i)
  722. throw new TypeError(E_STRICT)
  723. }
  724. if (!map || map.length < size) /* c8 ignore next */ throw new Error('Unreachable') // Important for perf
  725. if (iso2022jp) {
  726. let state = 0 // 0 = ASCII, 1 = Roman, 2 = jis0208
  727. const restore = () => {
  728. state = 0
  729. u8[i++] = 0x1b
  730. u8[i++] = 0x28
  731. u8[i++] = 0x42
  732. }
  733. for (let j = 0; j < length; j++) {
  734. let x = str.charCodeAt(j)
  735. if (x >= 0xd8_00 && x < 0xe0_00) {
  736. if (state === 2) restore()
  737. if (x >= 0xdc_00 || j + 1 === length) {
  738. i += err(x) // lone
  739. } else {
  740. const x1 = str.charCodeAt(j + 1)
  741. if (x1 < 0xdc_00 || x1 >= 0xe0_00) {
  742. i += err(x) // lone
  743. } else {
  744. j++ // consume x1
  745. i += err(0x1_00_00 + ((x1 & 0x3_ff) | ((x & 0x3_ff) << 10)))
  746. }
  747. }
  748. } else if (x < 0x80) {
  749. if (state === 2 || (state === 1 && (x === 0x5c || x === 0x7e))) restore()
  750. if (x === 0xe || x === 0xf || x === 0x1b) {
  751. i += err(0xff_fd) // 12.2.2. step 3: This returns U+FFFD rather than codePoint to prevent attacks
  752. } else {
  753. u8[i++] = x
  754. }
  755. } else if (x === 0xa5 || x === 0x20_3e) {
  756. if (state !== 1) {
  757. state = 1
  758. u8[i++] = 0x1b
  759. u8[i++] = 0x28
  760. u8[i++] = 0x4a
  761. }
  762. u8[i++] = x === 0xa5 ? 0x5c : 0x7e
  763. } else {
  764. if (x === 0x22_12) x = 0xff_0d
  765. if (x >= 0xff_61 && x <= 0xff_9f) x = katakana[x - 0xff_61]
  766. const e = map[x]
  767. if (e) {
  768. if (state !== 2) {
  769. state = 2
  770. u8[i++] = 0x1b
  771. u8[i++] = 0x24
  772. u8[i++] = 0x42
  773. }
  774. u8[i++] = e >> 8
  775. u8[i++] = e & 0xff
  776. } else {
  777. if (state === 2) restore()
  778. i += err(x)
  779. }
  780. }
  781. }
  782. if (state) restore()
  783. } else if (gb18030) {
  784. // Deduping this branch hurts other encoders perf
  785. const encode = (cp) => {
  786. let a = 0, b = 0 // prettier-ignore
  787. for (const [c, d] of gb18030r) {
  788. if (d > cp) break
  789. a = c
  790. b = d
  791. }
  792. let rp = cp === 0xe7_c7 ? 7457 : a + cp - b
  793. u8[i++] = 0x81 + ((rp / 12_600) | 0)
  794. rp %= 12_600
  795. u8[i++] = 0x30 + ((rp / 1260) | 0)
  796. rp %= 1260
  797. u8[i++] = 0x81 + ((rp / 10) | 0)
  798. u8[i++] = 0x30 + (rp % 10)
  799. }
  800. for (let j = i; j < length; j++) {
  801. const x = str.charCodeAt(j)
  802. if (x >= 0xd8_00 && x < 0xe0_00) {
  803. if (x >= 0xdc_00 || j + 1 === length) {
  804. i += err(x) // lone
  805. } else {
  806. const x1 = str.charCodeAt(j + 1)
  807. if (x1 < 0xdc_00 || x1 >= 0xe0_00) {
  808. i += err(x) // lone
  809. } else {
  810. j++ // consume x1
  811. encode(0x1_00_00 + ((x1 & 0x3_ff) | ((x & 0x3_ff) << 10)))
  812. }
  813. }
  814. } else {
  815. const e = map[x]
  816. if (e & 0xff_00) {
  817. u8[i++] = e >> 8
  818. u8[i++] = e & 0xff
  819. } else if (e || x === 0) {
  820. u8[i++] = e
  821. } else if (x === 0xe5_e5) {
  822. i += err(x)
  823. } else {
  824. encode(x)
  825. }
  826. }
  827. }
  828. } else {
  829. const long =
  830. enc === 'big5'
  831. ? (x) => {
  832. const e = map[x]
  833. if (e & 0xff_00) {
  834. u8[i++] = e >> 8
  835. u8[i++] = e & 0xff
  836. } else if (e || x === 0) {
  837. u8[i++] = e
  838. } else {
  839. i += err(x)
  840. }
  841. }
  842. : (x) => {
  843. i += err(x)
  844. }
  845. for (let j = i; j < length; j++) {
  846. const x = str.charCodeAt(j)
  847. if (x >= 0xd8_00 && x < 0xe0_00) {
  848. if (x >= 0xdc_00 || j + 1 === length) {
  849. i += err(x) // lone
  850. } else {
  851. const x1 = str.charCodeAt(j + 1)
  852. if (x1 < 0xdc_00 || x1 >= 0xe0_00) {
  853. i += err(x) // lone
  854. } else {
  855. j++ // consume x1
  856. long(0x1_00_00 + ((x1 & 0x3_ff) | ((x & 0x3_ff) << 10)))
  857. }
  858. }
  859. } else {
  860. const e = map[x]
  861. if (e & 0xff_00) {
  862. u8[i++] = e >> 8
  863. u8[i++] = e & 0xff
  864. } else if (e || x === 0) {
  865. u8[i++] = e
  866. } else {
  867. i += err(x)
  868. }
  869. }
  870. }
  871. }
  872. return i === u8.length ? u8 : u8.slice(0, i)
  873. }
  874. }