fix: preserve utf-8 charset for text uploads

Add conservative upload content-type handling that appends charset=utf-8 only when text uploads validate as UTF-8, while preserving existing charsets and non-UTF-8 text metadata.

Fixes rustfs/rustfs#3095

lib/upload-content-type.js

@@ -0,0 +1,77 @@
+const EXTENSION_MIME_TYPES = {
+  txt: "text/plain",
+  md: "text/markdown",
+  markdown: "text/markdown",
+  csv: "text/csv",
+  json: "application/json",
+  jsonl: "application/x-ndjson",
+  ndjson: "application/x-ndjson",
+  xml: "application/xml",
+  html: "text/html",
+  htm: "text/html",
+  css: "text/css",
+  js: "application/javascript",
+  mjs: "application/javascript",
+  svg: "image/svg+xml",
+  yml: "application/yaml",
+  yaml: "application/yaml",
+}
+
+function inferMimeTypeFromObjectKey(objectKey) {
+  const ext = objectKey.split(".").pop()?.toLowerCase() ?? ""
+  return EXTENSION_MIME_TYPES[ext] ?? "application/octet-stream"
+}
+
+function hasCharset(contentType) {
+  return /(?:^|;)\s*charset\s*=/i.test(contentType)
+}
+
+function isTextualContentType(contentType) {
+  const mime = contentType.split(";")[0]?.trim().toLowerCase() ?? ""
+  return (
+    mime.startsWith("text/") ||
+    mime === "application/json" ||
+    mime === "application/ld+json" ||
+    mime === "application/xml" ||
+    mime.endsWith("+xml") ||
+    mime === "application/javascript" ||
+    mime === "application/x-javascript" ||
+    mime === "application/ecmascript" ||
+    mime === "application/x-ndjson" ||
+    mime === "application/ndjson" ||
+    mime === "application/yaml" ||
+    mime === "application/x-yaml" ||
+    mime === "image/svg+xml"
+  )
+}
+
+async function isValidUtf8(file) {
+  const decoder = new TextDecoder("utf-8", { fatal: true })
+  const reader = file.stream().getReader()
+  try {
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      decoder.decode(value, { stream: true })
+    }
+    decoder.decode()
+    return true
+  } catch {
+    return false
+  } finally {
+    reader.releaseLock()
+  }
+}
+
+export async function getUploadContentType(file, objectKey) {
+  const baseContentType = file.type || inferMimeTypeFromObjectKey(objectKey)
+  if (!isTextualContentType(baseContentType) || hasCharset(baseContentType)) {
+    return baseContentType
+  }
+
+  if (!(await isValidUtf8(file))) {
+    return baseContentType
+  }
+
+  return `${baseContentType}; charset=utf-8`
+}

lib/upload-content-type.ts

@@ -0,0 +1,77 @@
+const EXTENSION_MIME_TYPES: Record<string, string> = {
+  txt: "text/plain",
+  md: "text/markdown",
+  markdown: "text/markdown",
+  csv: "text/csv",
+  json: "application/json",
+  jsonl: "application/x-ndjson",
+  ndjson: "application/x-ndjson",
+  xml: "application/xml",
+  html: "text/html",
+  htm: "text/html",
+  css: "text/css",
+  js: "application/javascript",
+  mjs: "application/javascript",
+  svg: "image/svg+xml",
+  yml: "application/yaml",
+  yaml: "application/yaml",
+}
+
+function inferMimeTypeFromObjectKey(objectKey: string): string {
+  const ext = objectKey.split(".").pop()?.toLowerCase() ?? ""
+  return EXTENSION_MIME_TYPES[ext] ?? "application/octet-stream"
+}
+
+function hasCharset(contentType: string): boolean {
+  return /(?:^|;)\s*charset\s*=/i.test(contentType)
+}
+
+function isTextualContentType(contentType: string): boolean {
+  const mime = contentType.split(";")[0]?.trim().toLowerCase() ?? ""
+  return (
+    mime.startsWith("text/") ||
+    mime === "application/json" ||
+    mime === "application/ld+json" ||
+    mime === "application/xml" ||
+    mime.endsWith("+xml") ||
+    mime === "application/javascript" ||
+    mime === "application/x-javascript" ||
+    mime === "application/ecmascript" ||
+    mime === "application/x-ndjson" ||
+    mime === "application/ndjson" ||
+    mime === "application/yaml" ||
+    mime === "application/x-yaml" ||
+    mime === "image/svg+xml"
+  )
+}
+
+async function isValidUtf8(file: Blob): Promise<boolean> {
+  const decoder = new TextDecoder("utf-8", { fatal: true })
+  const reader = file.stream().getReader()
+  try {
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      decoder.decode(value, { stream: true })
+    }
+    decoder.decode()
+    return true
+  } catch {
+    return false
+  } finally {
+    reader.releaseLock()
+  }
+}
+
+export async function getUploadContentType(file: File, objectKey: string): Promise<string> {
+  const baseContentType = file.type || inferMimeTypeFromObjectKey(objectKey)
+  if (!isTextualContentType(baseContentType) || hasCharset(baseContentType)) {
+    return baseContentType
+  }
+
+  if (!(await isValidUtf8(file))) {
+    return baseContentType
+  }
+
+  return `${baseContentType}; charset=utf-8`
+}

lib/upload-task.ts

@@ -8,6 +8,7 @@ import {
 import type { ManagedTask, TaskHandler, TaskLifecycleStatus } from "./task-manager"
 import { formatBytes } from "./functions"
 import { createTaskId } from "./task-id"
+import { getUploadContentType } from "./upload-content-type"
 
 export type UploadStatus = "pending" | "running" | "completed" | "failed" | "canceled" | "paused"
 
@@ -129,13 +130,14 @@ async function putObject(task: UploadTask, s3Client: S3Client) {
   const { file, bucketName, key } = task
   const abortController = new AbortController()
   task.abortController = abortController
+  const contentType = await getUploadContentType(file, key)
 
   await s3Client.send(
     new PutObjectCommand({
       Bucket: bucketName,
       Key: key,
       Body: file,
-      ContentType: file.type || "application/octet-stream",
+      ContentType: contentType,
     }),
     { abortSignal: abortController.signal },
   )
@@ -149,6 +151,7 @@ async function multipartUpload(task: UploadTask, s3Client: S3Client, chunkSize:
 
   let uploadId: string | undefined = task.uploadId
   const completedParts: { ETag: string; PartNumber: number }[] = task.completedParts || []
+  const contentType = await getUploadContentType(file, key)
 
   try {
     if (!uploadId) {
@@ -156,7 +159,7 @@ async function multipartUpload(task: UploadTask, s3Client: S3Client, chunkSize:
         new CreateMultipartUploadCommand({
           Bucket: bucketName,
           Key: key,
-          ContentType: file.type || "application/octet-stream",
+          ContentType: contentType,
         }),
         { abortSignal: abortController.signal },
       )

tests/lib/upload-content-type.test.js

@@ -0,0 +1,54 @@
+import test from "node:test"
+import assert from "node:assert/strict"
+import { getUploadContentType } from "../../lib/upload-content-type.js"
+
+function uploadBlob(parts, name, type = "") {
+  const blob = new Blob(parts, { type })
+  Object.defineProperty(blob, "name", { value: name })
+  return blob
+}
+
+test("getUploadContentType adds utf-8 charset for valid UTF-8 text uploads", async () => {
+  const file = uploadBlob(["中文内容"], "notes.txt", "text/plain")
+
+  assert.equal(await getUploadContentType(file, "notes.txt"), "text/plain; charset=utf-8")
+})
+
+test("getUploadContentType does not mark non-UTF-8 text as UTF-8", async () => {
+  const gbkChineseBytes = new Uint8Array([0xd6, 0xd0, 0xce, 0xc4])
+  const file = uploadBlob([gbkChineseBytes], "gbk.txt", "text/plain")
+
+  assert.equal(await getUploadContentType(file, "gbk.txt"), "text/plain")
+})
+
+test("getUploadContentType validates the full file before adding UTF-8 charset", async () => {
+  const asciiPrefix = "a".repeat(70 * 1024)
+  const invalidUtf8Suffix = new Uint8Array([0xd6, 0xd0])
+  const file = uploadBlob([asciiPrefix, invalidUtf8Suffix], "mixed.txt", "text/plain")
+
+  assert.equal(await getUploadContentType(file, "mixed.txt"), "text/plain")
+})
+
+test("getUploadContentType preserves an existing charset", async () => {
+  const file = uploadBlob(["中文内容"], "gbk.txt", "text/plain; charset=gbk")
+
+  assert.equal(await getUploadContentType(file, "gbk.txt"), "text/plain; charset=gbk")
+})
+
+test("getUploadContentType keeps binary content types unchanged", async () => {
+  const file = uploadBlob([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], "image.png", "image/png")
+
+  assert.equal(await getUploadContentType(file, "image.png"), "image/png")
+})
+
+test("getUploadContentType infers markdown MIME type and charset when browser type is empty", async () => {
+  const file = uploadBlob(["# 标题"], "readme.md")
+
+  assert.equal(await getUploadContentType(file, "readme.md"), "text/markdown; charset=utf-8")
+})
+
+test("getUploadContentType falls back to octet-stream for unknown empty MIME types", async () => {
+  const file = uploadBlob(["content"], "archive.unknownext")
+
+  assert.equal(await getUploadContentType(file, "archive.unknownext"), "application/octet-stream")
+})