utf8.node.js 2.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667
  1. import { assertU8, fromBuffer, E_STRING, E_STRICT_UNICODE } from './fallback/_utils.js'
  2. import { E_STRICT } from './fallback/utf8.js'
  3. import { isAscii } from 'node:buffer'
  4. if (Buffer.TYPED_ARRAY_SUPPORT) throw new Error('Unexpected Buffer polyfill')
  5. let decoderFatal
  6. const decoderLoose = new TextDecoder('utf-8', { ignoreBOM: true })
  7. const { isWellFormed } = String.prototype
  8. const isDeno = !!globalThis.Deno
  9. try {
  10. decoderFatal = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })
  11. } catch {
  12. // Without ICU, Node.js doesn't support fatal option for utf-8
  13. }
  14. function encode(str, loose, format) {
  15. if (typeof str !== 'string') throw new TypeError(E_STRING)
  16. const strLength = str.length
  17. if (strLength === 0) return new Uint8Array() // faster than Uint8Array.of
  18. let res
  19. if (strLength > 0x4_00 && !isDeno) {
  20. // Faster for large strings
  21. const byteLength = Buffer.byteLength(str)
  22. res = format === 'buffer' ? Buffer.allocUnsafe(byteLength) : Buffer.allocUnsafeSlow(byteLength)
  23. const written = byteLength === strLength ? res.latin1Write(str) : res.utf8Write(str)
  24. if (written !== byteLength) throw new Error('Failed to write all bytes') // safeguard just in case
  25. } else {
  26. res = Buffer.from(str)
  27. }
  28. // Loose and ascii do not need the check
  29. if (!loose && res.length !== strLength && !isWellFormed.call(str)) {
  30. throw new TypeError(E_STRICT_UNICODE)
  31. }
  32. return fromBuffer(res, format)
  33. }
  34. function decode(arr, loose = false) {
  35. assertU8(arr)
  36. const byteLength = arr.byteLength
  37. if (byteLength === 0) return ''
  38. if (byteLength > 0x6_00 && !(isDeno && loose) && isAscii(arr)) {
  39. // On non-ascii strings, this loses ~10% * [relative position of the first non-ascii byte] (up to 10% total)
  40. // On ascii strings, this wins 1.5x on loose = false and 1.3x on loose = true
  41. // Only makes sense for large enough strings
  42. const buf = Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength)
  43. if (isDeno) return buf.toString() // Deno suffers from .latin1Slice
  44. return buf.latin1Slice(0, arr.byteLength) // .latin1Slice is faster than .asciiSlice
  45. }
  46. if (loose) return decoderLoose.decode(arr)
  47. if (decoderFatal) return decoderFatal.decode(arr)
  48. // We are in an env without native fatal decoder support (non-fixed Node.js without ICU)
  49. // Well, just recheck against encode if it contains replacement then, this is still faster than js impl
  50. const str = decoderLoose.decode(arr)
  51. if (str.includes('\uFFFD') && !Buffer.from(str).equals(arr)) throw new TypeError(E_STRICT)
  52. return str
  53. }
  54. export const utf8fromString = (str, format = 'uint8') => encode(str, false, format)
  55. export const utf8fromStringLoose = (str, format = 'uint8') => encode(str, true, format)
  56. export const utf8toString = (arr) => decode(arr, false)
  57. export const utf8toStringLoose = (arr) => decode(arr, true)