From 929105662e31ca213446a15551c42be4ee904f03 Mon Sep 17 00:00:00 2001 From: jenken827 Date: Sun, 24 Aug 2025 18:18:11 +0800 Subject: [PATCH 01/18] feat(fs):slice upload --- src/lang/en/home.json | 3 +- src/pages/home/uploads/Upload.tsx | 11 ++ src/pages/home/uploads/form.ts | 6 + src/pages/home/uploads/slice_upload.ts | 253 +++++++++++++++++++++++++ src/pages/home/uploads/stream.ts | 19 +- src/pages/home/uploads/types.ts | 8 + src/pages/home/uploads/util.ts | 170 +++++++++++++++-- src/types/resp.ts | 20 ++ 8 files changed, 467 insertions(+), 23 deletions(-) create mode 100644 src/pages/home/uploads/slice_upload.ts diff --git a/src/lang/en/home.json b/src/lang/en/home.json index caa04855..1f497509 100644 --- a/src/lang/en/home.json +++ b/src/lang/en/home.json @@ -139,7 +139,8 @@ "success": "Success", "error": "Error", "back": "Back to Upload", - "clear_done": "Clear Done" + "clear_done": "Clear Done", + "slice_upload": "Slice upload" }, "local_settings": { "aria2_rpc_url": "Aria2 RPC URL", diff --git a/src/pages/home/uploads/Upload.tsx b/src/pages/home/uploads/Upload.tsx index 26381aeb..3097c925 100644 --- a/src/pages/home/uploads/Upload.tsx +++ b/src/pages/home/uploads/Upload.tsx @@ -78,6 +78,7 @@ const Upload = () => { const [uploading, setUploading] = createSignal(false) const [asTask, setAsTask] = createSignal(false) const [overwrite, setOverwrite] = createSignal(false) + const [sliceup, setSliceup] = createSignal(false) const [rapid, setRapid] = createSignal(true) const [uploadFiles, setUploadFiles] = createStore<{ uploads: UploadFileProps[] @@ -122,6 +123,7 @@ const Upload = () => { asTask(), overwrite(), rapid(), + sliceup(), ) if (!err) { setUpload(path, "status", "success") @@ -304,6 +306,15 @@ const Upload = () => { > {t("home.upload.add_as_task")} + { + setSliceup(!sliceup()) + }} + > + {t("home.upload.slice_upload")} + + { diff --git a/src/pages/home/uploads/form.ts b/src/pages/home/uploads/form.ts index 85d90fe2..ca2d38d8 100644 --- a/src/pages/home/uploads/form.ts +++ b/src/pages/home/uploads/form.ts @@ -3,6 +3,7 @@ import { EmptyResp } from "~/types" import { r } from "~/utils" import { SetUpload, Upload } from "./types" import { calculateHash } from "./util" +import { sliceupload } from "./slice_upload" export const FormUpload: Upload = async ( uploadPath: string, file: File, @@ -10,7 +11,12 @@ export const FormUpload: Upload = async ( asTask = false, overwrite = false, rapid = false, + sliceup = false, ): Promise => { + if (sliceup) { + return sliceupload(uploadPath, file, setUpload, overwrite, asTask) + } + let oldTimestamp = new Date().valueOf() let oldLoaded = 0 const form = new FormData() diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts new file mode 100644 index 00000000..47b17ae2 --- /dev/null +++ b/src/pages/home/uploads/slice_upload.ts @@ -0,0 +1,253 @@ +import { password } from "~/store" +import { EmptyResp } from "~/types" +import { r, pathDir, log } from "~/utils" +import { SetUpload, Upload } from "./types" +import pLimit from "p-limit" +import { + calculateHash, + calculateSliceHash, + fsUploadInfo, + fsPreup, + FsSliceupComplete, + HashType, +} from "./util" +import createMutex from "~/utils/mutex" + +const progressMutex = createMutex() + +export const sliceupload = async ( + uploadPath: string, + file: File, + setUpload: SetUpload, + overwrite = false, + asTask = false, +): Promise => { + let hashtype: string = HashType.Md5 + let slicehash: string[] = [] + let sliceupstatus: Uint8Array + let ht: string[] = [] + + const dir = pathDir(uploadPath) + + //获取上传需要的信息 + const resp = await fsUploadInfo(dir) + if (resp.code != 200) { + return new Error(resp.message) + } + + // hash计算 + if (resp.data.hash_md5_need) { + ht.push(HashType.Md5) + hashtype = HashType.Md5 + } + if (resp.data.hash_sha1_need) { + ht.push(HashType.Sha1) + hashtype = HashType.Sha1 + } + if (resp.data.hash_md5_256kb_need) { + ht.push(HashType.Md5256kb) + } + const hash = await calculateHash(file, ht) + // 预上传 + const resp1 = await fsPreup( + dir, + file.name, + file.size, + hash, + overwrite, + asTask, + ) + if (resp1.code != 200) { + return new Error(resp1.message) + } + if (resp1.data.reuse) { + setUpload("progress", "100") + setUpload("status", "success") + setUpload("speed", "0") + return + } + //计算分片hash + if (resp.data.slice_hash_need) { + slicehash = await calculateSliceHash(file, resp1.data.slice_size, hashtype) + } + // 分片上传状态 + sliceupstatus = base64ToUint8Array(resp1.data.slice_upload_status) + + // 进度和速度统计 + let uploadedBytes = 0 + let lastTimestamp = Date.now() + let lastUploadedBytes = 0 + const totalSize = file.size + let completeFlag = false + + // 上传分片的核心函数，带进度和速度统计 + const uploadChunk = async ( + chunk: Blob, + idx: number, + slice_hash: string, + upload_id: number, + ) => { + const formData = new FormData() + formData.append("upload_id", upload_id.toString()) + formData.append("slice_hash", slice_hash) + formData.append("slice_num", idx.toString()) + formData.append("slice", chunk) + + let oldTimestamp = Date.now() + let oldLoaded = 0 + + const resp: EmptyResp = await r.post("/fs/slice_upload", formData, { + headers: { + "File-Path": encodeURIComponent(dir), + "Content-Type": "multipart/form-data", + Password: password(), + }, + onUploadProgress: async (progressEvent) => { + log() + if (!progressEvent.lengthComputable) { + return + } + //获取锁 + const release = await progressMutex.acquire() + try { + const sliceuploaded = progressEvent.loaded - oldLoaded + log("progress event trigger", idx, sliceuploaded, Date.now()) + uploadedBytes += sliceuploaded + oldLoaded = progressEvent.loaded + } finally { + progressMutex.release() + } + }, + }) + + if (resp.code != 200) { + throw new Error(resp.message) + } + } + + // 进度速度计算 + let speedInterval = setInterval(() => { + if (completeFlag) { + clearInterval(speedInterval) + return + } + + const intervalLoaded = uploadedBytes - lastUploadedBytes + if (intervalLoaded < 1000) { + //进度太小，不更新 + return + } + const speed = intervalLoaded / ((Date.now() - lastTimestamp) / 1000) + const complete = Math.min(100, ((uploadedBytes / file.size) * 100) | 0) + setUpload("speed", speed) + setUpload("progress", complete) + lastTimestamp = Date.now() + lastUploadedBytes = uploadedBytes + }, 1000) // 更高频更新 + + // 开始计时 + lastTimestamp = Date.now() + + // 先上传第一个分片，slicehash全部用逗号拼接传递 + if (!isSliceUploaded(sliceupstatus, 0)) { + const chunk = file.slice(0, resp1.data.slice_size) + try { + await uploadChunk( + chunk, + 0, + slicehash.length == 0 ? "" : slicehash.join(","), + resp1.data.upload_id, + ) + } catch (err) { + completeFlag = true + setUpload("status", "error") + setUpload("speed", 0) + return err as Error + } + } else { + uploadedBytes += Math.min(resp1.data.slice_size, totalSize) + } + + // 后续分片并发上传，限制并发数为3，后续也可以通过fsUploadInfo接口获取配置 + const limit = pLimit(3) + + const tasks: Promise[] = [] + const errors: Error[] = [] + for (let i = 1; i < resp1.data.slice_cnt; i++) { + if (!isSliceUploaded(sliceupstatus, i)) { + const chunk = file.slice( + i * resp1.data.slice_size, + (i + 1) * resp1.data.slice_size, + ) + tasks.push( + limit(async () => { + try { + await uploadChunk( + chunk, + i, + slicehash.length == 0 ? "" : slicehash[i], + resp1.data.upload_id, + ) + } catch (err) { + errors.push(err as Error) + } + }), + ) + } else { + uploadedBytes += Math.min( + resp1.data.slice_size, + totalSize - i * resp1.data.slice_size, + ) + } + } + await Promise.all(tasks) + completeFlag = true + + // 最终处理上传结果 + if (errors.length > 0) { + setUpload("status", "error") + setUpload("speed", 0) + setUpload( + "progress", + Math.min(100, ((uploadedBytes / totalSize) * 100) | 0), + ) + return errors[0] + } else { + const resp = await FsSliceupComplete(dir, resp1.data.upload_id) + if (resp.code != 200) { + setUpload("status", "error") + return new Error(resp.message) + } else if (resp.data.complete == 0) { + setUpload("status", "error") + return new Error("slice missing, please reupload") + } else if (resp.data.complete == 2) { + //后台或任务上传中 + setUpload("status", "backending") + return + } + setUpload("progress", 100) + setUpload("status", "success") + setUpload("speed", 0) + return + } +} + +// 解码 base64 字符串为 Uint8Array +const base64ToUint8Array = (base64: string): Uint8Array => { + const binary = atob(base64) + const len = binary.length + const bytes = new Uint8Array(len) + for (let i = 0; i < len; i++) { + bytes[i] = binary.charCodeAt(i) + } + return bytes +} + +// 判断第 idx 个分片是否已上传 +const isSliceUploaded = (status: Uint8Array, idx: number): boolean => { + // const bytes = base64ToUint8Array(statusBase64) + const byteIdx = Math.floor(idx / 8) + const bitIdx = idx % 8 + if (byteIdx >= status.length) return false + return (status[byteIdx] & (1 << bitIdx)) !== 0 +} diff --git a/src/pages/home/uploads/stream.ts b/src/pages/home/uploads/stream.ts index 72980cc1..d32df985 100644 --- a/src/pages/home/uploads/stream.ts +++ b/src/pages/home/uploads/stream.ts @@ -2,7 +2,8 @@ import { password } from "~/store" import { EmptyResp } from "~/types" import { r } from "~/utils" import { SetUpload, Upload } from "./types" -import { calculateHash } from "./util" +import { calculateHash, HashType } from "./util" +import { sliceupload } from "./slice_upload" export const StreamUpload: Upload = async ( uploadPath: string, file: File, @@ -10,7 +11,11 @@ export const StreamUpload: Upload = async ( asTask = false, overwrite = false, rapid = false, + sliceup = false, ): Promise => { + if (sliceup) { + return sliceupload(uploadPath, file, setUpload, overwrite, asTask) + } let oldTimestamp = new Date().valueOf() let oldLoaded = 0 let headers: { [k: string]: any } = { @@ -22,10 +27,14 @@ export const StreamUpload: Upload = async ( Overwrite: overwrite.toString(), } if (rapid) { - const { md5, sha1, sha256 } = await calculateHash(file) - headers["X-File-Md5"] = md5 - headers["X-File-Sha1"] = sha1 - headers["X-File-Sha256"] = sha256 + const hash = await calculateHash(file, [ + HashType.Md5, + HashType.Sha1, + HashType.Sha256, + ]) + headers["X-File-Md5"] = hash.md5 + headers["X-File-Sha1"] = hash.sha1 + headers["X-File-Sha256"] = hash.sha256 } const resp: EmptyResp = await r.put("/fs/put", file, { headers: headers, diff --git a/src/pages/home/uploads/types.ts b/src/pages/home/uploads/types.ts index ade472f7..2845d2f1 100644 --- a/src/pages/home/uploads/types.ts +++ b/src/pages/home/uploads/types.ts @@ -23,4 +23,12 @@ export type Upload = ( asTask: boolean, overwrite: boolean, rapid: boolean, + sliceup: boolean, ) => Promise + +export type HashInfo = { + md5: string + md5_256kb: string + sha1: string + sha256: string +} diff --git a/src/pages/home/uploads/util.ts b/src/pages/home/uploads/util.ts index ee7fa88c..5036f3e9 100644 --- a/src/pages/home/uploads/util.ts +++ b/src/pages/home/uploads/util.ts @@ -1,5 +1,7 @@ -import { UploadFileProps } from "./types" +import { HashInfo, UploadFileProps } from "./types" +import { FsUpinfoResp, FsPreupResp, FsSliceupCompleteResp } from "~/types" import { createMD5, createSHA1, createSHA256 } from "hash-wasm" +import { r } from "~/utils" export const traverseFileTree = async (entry: FileSystemEntry) => { let res: File[] = [] @@ -50,6 +52,57 @@ export const traverseFileTree = async (entry: FileSystemEntry) => { return res } +export const fsUploadInfo = (path: string = "/"): Promise => { + return r.get("/fs/upload/info", { + headers: { + "File-Path": encodeURIComponent(path), + }, + }) +} + +export const fsPreup = async ( + path: string, + name: string, + size: number, + hash: HashInfo, + overwrite: boolean, + as_task: boolean, +): Promise => { + return r.post( + "/fs/preup", + { + path, + name, + size, + hash, + overwrite, + as_task, + }, + { + headers: { + "File-Path": encodeURIComponent(path), + }, + }, + ) +} + +export const FsSliceupComplete = async ( + path: string, + upload_id: number, +): Promise => { + return r.post( + "/fs/slice_upload_complete", + { + upload_id, + }, + { + headers: { + "File-Path": encodeURIComponent(path), + }, + }, + ) +} + export const File2Upload = (file: File): UploadFileProps => { return { name: file.name, @@ -61,24 +114,107 @@ export const File2Upload = (file: File): UploadFileProps => { } } -export const calculateHash = async (file: File) => { - const md5Digest = await createMD5() - const sha1Digest = await createSHA1() - const sha256Digest = await createSHA256() +export enum HashType { + Md5 = "md5", + Md5256kb = "md5_256kb", + Sha1 = "sha1", + Sha256 = "sha256", +} + +export const calculateHash = async ( + file: File, + hashType: string[] = [HashType.Md5], +) => { + let md5Digest: any, md5256kbDigest: any, sha1Digest: any, sha256Digest: any + let hash: HashInfo = { + md5: "", + md5_256kb: "", + sha1: "", + sha256: "", + } + // 初始化需要的 hash 实例 + for (const ht of hashType) { + if (ht === HashType.Md5) { + md5Digest = await createMD5() + } else if (ht === HashType.Md5256kb) { + md5256kbDigest = await createMD5() + } else if (ht === HashType.Sha1) { + sha1Digest = await createSHA1() + } else if (ht === HashType.Sha256) { + sha256Digest = await createSHA256() + } + } + const reader = file.stream().getReader() - const read = async () => { + let readBytes = 0 + const KB256 = 256 * 1024 + let md5256kbDone = false + + while (true) { const { done, value } = await reader.read() - if (done) { - return + if (done) break + + if (md5Digest) md5Digest.update(value) + if (sha1Digest) sha1Digest.update(value) + if (sha256Digest) sha256Digest.update(value) + + // 计算前256KB的md5 + if (md5256kbDigest && !md5256kbDone) { + let chunk = value + if (readBytes + chunk.length > KB256) { + // 只取剩余需要的部分 + chunk = chunk.slice(0, KB256 - readBytes) + md5256kbDone = true + } + md5256kbDigest.update(chunk) + readBytes += chunk.length + if (readBytes >= KB256) { + md5256kbDone = true + } } - md5Digest.update(value) - sha1Digest.update(value) - sha256Digest.update(value) - await read() } - await read() - const md5 = md5Digest.digest("hex") - const sha1 = sha1Digest.digest("hex") - const sha256 = sha256Digest.digest("hex") - return { md5, sha1, sha256 } + + if (md5Digest) hash.md5 = await md5Digest.digest("hex") + if (md5256kbDigest) hash.md5_256kb = await md5256kbDigest.digest("hex") + if (sha1Digest) hash.sha1 = await sha1Digest.digest("hex") + if (sha256Digest) hash.sha256 = await sha256Digest.digest("hex") + + return hash +} + +export const calculateSliceHash = async ( + file: File, + sliceSize: number, + hashType: string, +) => { + const sliceCount = Math.ceil(file.size / sliceSize) + const results: string[] = [] + + for (let i = 0; i < sliceCount; i++) { + const start = i * sliceSize + const end = Math.min(file.size, start + sliceSize) + const blob = file.slice(start, end) + const arrayBuffer = await blob.arrayBuffer() + let hash: string = "" + + if (hashType === HashType.Md5) { + const md5 = await createMD5() + md5.update(new Uint8Array(arrayBuffer)) + hash = await md5.digest("hex") + } else if (hashType === HashType.Sha1) { + const sha1 = await createSHA1() + sha1.update(new Uint8Array(arrayBuffer)) + hash = await sha1.digest("hex") + } else if (hashType === HashType.Sha256) { + const sha256 = await createSHA256() + sha256.update(new Uint8Array(arrayBuffer)) + hash = await sha256.digest("hex") + } else { + throw new Error("Unsupported hash type: " + hashType) + } + + results.push(hash) + } + + return results // 每个分片的hash组成的数组 } diff --git a/src/types/resp.ts b/src/types/resp.ts index 8934099c..9a829114 100644 --- a/src/types/resp.ts +++ b/src/types/resp.ts @@ -41,6 +41,26 @@ export type FsGetResp = Resp< } > +export type FsPreupResp = Resp<{ + upload_id: number + slice_size: number + slice_cnt: number + slice_upload_status: string + reuse: boolean +}> +export type FsUpinfoResp = Resp<{ + slice_hash_need: boolean //是否需要分片哈希 + hash_md5_need: boolean //是否需要md5 + hash_md5_256kb_need: boolean //是否需要前256KB的md5 + hash_sha1_need: boolean //是否需要sha1 +}> + +export type FsSliceupCompleteResp = Resp<{ + upload_id: number + slice_upload_status: string + complete: number +}> + export type EmptyResp = Resp<{}> export type PResp = Promise> From 88ad0148837271caac5feacfc9f740e0b65767fb Mon Sep 17 00:00:00 2001 From: jenken827 Date: Sun, 24 Aug 2025 19:53:51 +0800 Subject: [PATCH 02/18] feat:slice upload --- vite.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/vite.config.ts b/vite.config.ts index 08fe5c88..971f0294 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -72,6 +72,7 @@ export default defineConfig({ // target: "es2015", //next // polyfillDynamicImport: false, rollupOptions: { + external: ["p-limit"], output: { assetFileNames: (assetInfo) => assetInfo.names?.some((name) => name.endsWith("pdf.worker.min.mjs")) From a4ece2dd183772e7dc1c63bba5fecb865f841fce Mon Sep 17 00:00:00 2001 From: jenken827 Date: Mon, 25 Aug 2025 11:02:39 +0800 Subject: [PATCH 03/18] feat(fs): slice upload --- src/pages/home/uploads/slice_upload.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index 47b17ae2..cd95b18e 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -1,6 +1,6 @@ import { password } from "~/store" import { EmptyResp } from "~/types" -import { r, pathDir, log } from "~/utils" +import { r, pathDir } from "~/utils" import { SetUpload, Upload } from "./types" import pLimit from "p-limit" import { @@ -103,7 +103,6 @@ export const sliceupload = async ( Password: password(), }, onUploadProgress: async (progressEvent) => { - log() if (!progressEvent.lengthComputable) { return } @@ -111,7 +110,6 @@ export const sliceupload = async ( const release = await progressMutex.acquire() try { const sliceuploaded = progressEvent.loaded - oldLoaded - log("progress event trigger", idx, sliceuploaded, Date.now()) uploadedBytes += sliceuploaded oldLoaded = progressEvent.loaded } finally { From 757a74fcd5f75310bf6e1bdaa208857768734be2 Mon Sep 17 00:00:00 2001 From: jenken827 Date: Tue, 26 Aug 2025 12:54:21 +0800 Subject: [PATCH 04/18] feat(fs): implement slice upload --- src/lang/en/home.json | 1 + src/pages/home/uploads/Upload.tsx | 2 +- src/pages/home/uploads/slice_upload.ts | 19 ++++++------------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/lang/en/home.json b/src/lang/en/home.json index 1f497509..f8574bac 100644 --- a/src/lang/en/home.json +++ b/src/lang/en/home.json @@ -136,6 +136,7 @@ "pending": "Pending", "uploading": "Uploading", "backending": "Uploading in the backend", + "tasked": "Successfully added to the task", "success": "Success", "error": "Error", "back": "Back to Upload", diff --git a/src/pages/home/uploads/Upload.tsx b/src/pages/home/uploads/Upload.tsx index 3097c925..0a9c0371 100644 --- a/src/pages/home/uploads/Upload.tsx +++ b/src/pages/home/uploads/Upload.tsx @@ -126,7 +126,7 @@ const Upload = () => { sliceup(), ) if (!err) { - setUpload(path, "status", "success") + setUpload(path, "status", asTask() ? "tasked" : "success") setUpload(path, "progress", 100) } else { setUpload(path, "status", "error") diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index cd95b18e..426aba58 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -141,7 +141,7 @@ export const sliceupload = async ( setUpload("progress", complete) lastTimestamp = Date.now() lastUploadedBytes = uploadedBytes - }, 1000) // 更高频更新 + }, 1000) // 开始计时 lastTimestamp = Date.now() @@ -199,33 +199,26 @@ export const sliceupload = async ( } } await Promise.all(tasks) - completeFlag = true // 最终处理上传结果 if (errors.length > 0) { - setUpload("status", "error") - setUpload("speed", 0) setUpload( "progress", Math.min(100, ((uploadedBytes / totalSize) * 100) | 0), ) return errors[0] } else { + if (!asTask) { + setUpload("status", "backending") + } const resp = await FsSliceupComplete(dir, resp1.data.upload_id) + completeFlag = true if (resp.code != 200) { - setUpload("status", "error") return new Error(resp.message) } else if (resp.data.complete == 0) { - setUpload("status", "error") return new Error("slice missing, please reupload") - } else if (resp.data.complete == 2) { - //后台或任务上传中 - setUpload("status", "backending") - return } - setUpload("progress", 100) - setUpload("status", "success") - setUpload("speed", 0) + //状态处理交给上层 return } } From 0e484dddde8bfb4f29e45772b69314e17e019489 Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Fri, 5 Sep 2025 22:26:41 +0800 Subject: [PATCH 05/18] fix(fs): update error messages and change upload_id to task_id in slice upload functions --- src/pages/home/uploads/slice_upload.ts | 18 +++++++++--------- src/pages/home/uploads/util.ts | 4 ++-- src/types/resp.ts | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index 426aba58..45bb4465 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -32,7 +32,7 @@ export const sliceupload = async ( //获取上传需要的信息 const resp = await fsUploadInfo(dir) if (resp.code != 200) { - return new Error(resp.message) + return new Error(`Upload info failed: ${resp.code} - ${resp.message}`) } // hash计算 @@ -58,7 +58,7 @@ export const sliceupload = async ( asTask, ) if (resp1.code != 200) { - return new Error(resp1.message) + return new Error(`Preup failed: ${resp1.code} - ${resp1.message}`) } if (resp1.data.reuse) { setUpload("progress", "100") @@ -85,10 +85,10 @@ export const sliceupload = async ( chunk: Blob, idx: number, slice_hash: string, - upload_id: number, + task_id: string, ) => { const formData = new FormData() - formData.append("upload_id", upload_id.toString()) + formData.append("task_id", task_id) formData.append("slice_hash", slice_hash) formData.append("slice_num", idx.toString()) formData.append("slice", chunk) @@ -102,7 +102,7 @@ export const sliceupload = async ( "Content-Type": "multipart/form-data", Password: password(), }, - onUploadProgress: async (progressEvent) => { + onUploadProgress: async (progressEvent: any) => { if (!progressEvent.lengthComputable) { return } @@ -154,7 +154,7 @@ export const sliceupload = async ( chunk, 0, slicehash.length == 0 ? "" : slicehash.join(","), - resp1.data.upload_id, + resp1.data.task_id, ) } catch (err) { completeFlag = true @@ -184,7 +184,7 @@ export const sliceupload = async ( chunk, i, slicehash.length == 0 ? "" : slicehash[i], - resp1.data.upload_id, + resp1.data.task_id, ) } catch (err) { errors.push(err as Error) @@ -211,10 +211,10 @@ export const sliceupload = async ( if (!asTask) { setUpload("status", "backending") } - const resp = await FsSliceupComplete(dir, resp1.data.upload_id) + const resp = await FsSliceupComplete(dir, resp1.data.task_id) completeFlag = true if (resp.code != 200) { - return new Error(resp.message) + return new Error(`Upload complete failed: ${resp.code} - ${resp.message}`) } else if (resp.data.complete == 0) { return new Error("slice missing, please reupload") } diff --git a/src/pages/home/uploads/util.ts b/src/pages/home/uploads/util.ts index 5036f3e9..2d9dd56e 100644 --- a/src/pages/home/uploads/util.ts +++ b/src/pages/home/uploads/util.ts @@ -88,12 +88,12 @@ export const fsPreup = async ( export const FsSliceupComplete = async ( path: string, - upload_id: number, + task_id: string, ): Promise => { return r.post( "/fs/slice_upload_complete", { - upload_id, + task_id, }, { headers: { diff --git a/src/types/resp.ts b/src/types/resp.ts index 9a829114..b96739a4 100644 --- a/src/types/resp.ts +++ b/src/types/resp.ts @@ -42,7 +42,7 @@ export type FsGetResp = Resp< > export type FsPreupResp = Resp<{ - upload_id: number + task_id: string slice_size: number slice_cnt: number slice_upload_status: string @@ -56,7 +56,7 @@ export type FsUpinfoResp = Resp<{ }> export type FsSliceupCompleteResp = Resp<{ - upload_id: number + task_id: string slice_upload_status: string complete: number }> From 39752d1fc8f20246c01222039ccda1268afd48a5 Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Fri, 5 Sep 2025 22:37:06 +0800 Subject: [PATCH 06/18] feat(fs): enhance slice upload with retry logic, upload state management, and memory optimization --- src/pages/home/uploads/slice_upload.ts | 231 ++++++++++++++++++++----- 1 file changed, 191 insertions(+), 40 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index 45bb4465..ddc9e88f 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -13,25 +13,92 @@ import { } from "./util" import createMutex from "~/utils/mutex" +// 重试配置 +const RETRY_CONFIG = { + maxRetries: 3, + retryDelay: 1000, // 1秒 + backoffMultiplier: 2, // 指数退避 +} + +// 大文件优化配置 +const MEMORY_OPTIMIZATION = { + largeFileThreshold: 100 * 1024 * 1024, // 100MB + maxConcurrentSlices: 2, // 大文件时减少并发 + chunkReadSize: 64 * 1024, // 64KB 分块读取 +} + const progressMutex = createMutex() +// 重试函数 +const retryWithBackoff = async ( + fn: () => Promise, + maxRetries: number = RETRY_CONFIG.maxRetries, + delay: number = RETRY_CONFIG.retryDelay +): Promise => { + let lastError: Error + for (let i = 0; i <= maxRetries; i++) { + try { + return await fn() + } catch (error) { + lastError = error as Error + if (i === maxRetries) { + throw lastError + } + // 指数退避延迟 + const waitTime = delay * Math.pow(RETRY_CONFIG.backoffMultiplier, i) + await new Promise(resolve => setTimeout(resolve, waitTime)) + } + } + throw lastError! +} + +// 上传状态管理 +interface UploadState { + isPaused: boolean + isCancelled: boolean + totalBytes: number + uploadedBytes: number +} + export const sliceupload = async ( uploadPath: string, file: File, setUpload: SetUpload, overwrite = false, asTask = false, + uploadState?: UploadState, ): Promise => { let hashtype: string = HashType.Md5 let slicehash: string[] = [] let sliceupstatus: Uint8Array let ht: string[] = [] + // 初始化上传状态 + const state: UploadState = uploadState || { + isPaused: false, + isCancelled: false, + totalBytes: file.size, + uploadedBytes: 0, + } + + // 注册到上传队列 + uploadQueue.addUpload(uploadPath, state) + + // 清理函数 + let speedInterval: any + const cleanup = () => { + if (speedInterval) { + clearInterval(speedInterval) + } + uploadQueue.removeUpload(uploadPath) + } + const dir = pathDir(uploadPath) //获取上传需要的信息 const resp = await fsUploadInfo(dir) if (resp.code != 200) { + cleanup() return new Error(`Upload info failed: ${resp.code} - ${resp.message}`) } @@ -58,12 +125,14 @@ export const sliceupload = async ( asTask, ) if (resp1.code != 200) { + cleanup() return new Error(`Preup failed: ${resp1.code} - ${resp1.message}`) } if (resp1.data.reuse) { setUpload("progress", "100") setUpload("status", "success") setUpload("speed", "0") + cleanup() return } //计算分片hash @@ -74,19 +143,37 @@ export const sliceupload = async ( sliceupstatus = base64ToUint8Array(resp1.data.slice_upload_status) // 进度和速度统计 - let uploadedBytes = 0 let lastTimestamp = Date.now() let lastUploadedBytes = 0 - const totalSize = file.size let completeFlag = false - // 上传分片的核心函数，带进度和速度统计 + // 计算已上传的字节数（用于断点续传） + for (let i = 0; i < resp1.data.slice_cnt; i++) { + if (isSliceUploaded(sliceupstatus, i)) { + state.uploadedBytes += Math.min( + resp1.data.slice_size, + state.totalBytes - i * resp1.data.slice_size, + ) + } + } + + // 上传分片的核心函数，带进度、速度统计、重试和暂停支持 const uploadChunk = async ( chunk: Blob, idx: number, slice_hash: string, task_id: string, ) => { + // 检查是否被取消 + if (state.isCancelled) { + throw new Error("Upload cancelled by user") + } + + // 检查是否暂停，等待恢复 + while (state.isPaused && !state.isCancelled) { + await new Promise(resolve => setTimeout(resolve, 100)) + } + const formData = new FormData() formData.append("task_id", task_id) formData.append("slice_hash", slice_hash) @@ -96,51 +183,54 @@ export const sliceupload = async ( let oldTimestamp = Date.now() let oldLoaded = 0 - const resp: EmptyResp = await r.post("/fs/slice_upload", formData, { - headers: { - "File-Path": encodeURIComponent(dir), - "Content-Type": "multipart/form-data", - Password: password(), - }, - onUploadProgress: async (progressEvent: any) => { - if (!progressEvent.lengthComputable) { - return - } - //获取锁 - const release = await progressMutex.acquire() - try { - const sliceuploaded = progressEvent.loaded - oldLoaded - uploadedBytes += sliceuploaded - oldLoaded = progressEvent.loaded - } finally { - progressMutex.release() - } - }, - }) + return retryWithBackoff(async () => { + const resp: EmptyResp = await r.post("/fs/slice_upload", formData, { + headers: { + "File-Path": encodeURIComponent(dir), + "Content-Type": "multipart/form-data", + Password: password(), + }, + onUploadProgress: async (progressEvent: any) => { + if (!progressEvent.lengthComputable || state.isCancelled) { + return + } + //获取锁 + const release = await progressMutex.acquire() + try { + const sliceuploaded = progressEvent.loaded - oldLoaded + state.uploadedBytes += sliceuploaded + oldLoaded = progressEvent.loaded + } finally { + progressMutex.release() + } + }, + }) - if (resp.code != 200) { - throw new Error(resp.message) - } + if (resp.code != 200) { + throw new Error(`Slice upload failed: ${resp.code} - ${resp.message}`) + } + return resp + }) } // 进度速度计算 - let speedInterval = setInterval(() => { - if (completeFlag) { + speedInterval = setInterval(() => { + if (completeFlag || state.isCancelled) { clearInterval(speedInterval) return } - const intervalLoaded = uploadedBytes - lastUploadedBytes + const intervalLoaded = state.uploadedBytes - lastUploadedBytes if (intervalLoaded < 1000) { //进度太小，不更新 return } const speed = intervalLoaded / ((Date.now() - lastTimestamp) / 1000) - const complete = Math.min(100, ((uploadedBytes / file.size) * 100) | 0) + const complete = Math.min(100, ((state.uploadedBytes / state.totalBytes) * 100) | 0) setUpload("speed", speed) setUpload("progress", complete) lastTimestamp = Date.now() - lastUploadedBytes = uploadedBytes + lastUploadedBytes = state.uploadedBytes }, 1000) // 开始计时 @@ -162,12 +252,14 @@ export const sliceupload = async ( setUpload("speed", 0) return err as Error } - } else { - uploadedBytes += Math.min(resp1.data.slice_size, totalSize) - } + } else { + state.uploadedBytes += Math.min(resp1.data.slice_size, state.totalBytes) + } // 后续分片并发上传，根据文件大小动态调整并发数 + const isLargeFile = file.size > MEMORY_OPTIMIZATION.largeFileThreshold + const concurrentLimit = isLargeFile ? MEMORY_OPTIMIZATION.maxConcurrentSlices : 3 + const limit = pLimit(concurrentLimit) - // 后续分片并发上传，限制并发数为3，后续也可以通过fsUploadInfo接口获取配置 - const limit = pLimit(3) + console.log(`File size: ${(file.size / 1024 / 1024).toFixed(2)}MB, using ${concurrentLimit} concurrent uploads`) const tasks: Promise[] = [] const errors: Error[] = [] @@ -192,9 +284,9 @@ export const sliceupload = async ( }), ) } else { - uploadedBytes += Math.min( + state.uploadedBytes += Math.min( resp1.data.slice_size, - totalSize - i * resp1.data.slice_size, + state.totalBytes - i * resp1.data.slice_size, ) } } @@ -204,8 +296,9 @@ export const sliceupload = async ( if (errors.length > 0) { setUpload( "progress", - Math.min(100, ((uploadedBytes / totalSize) * 100) | 0), + Math.min(100, ((state.uploadedBytes / state.totalBytes) * 100) | 0), ) + cleanup() return errors[0] } else { if (!asTask) { @@ -213,6 +306,7 @@ export const sliceupload = async ( } const resp = await FsSliceupComplete(dir, resp1.data.task_id) completeFlag = true + cleanup() if (resp.code != 200) { return new Error(`Upload complete failed: ${resp.code} - ${resp.message}`) } else if (resp.data.complete == 0) { @@ -242,3 +336,60 @@ const isSliceUploaded = (status: Uint8Array, idx: number): boolean => { if (byteIdx >= status.length) return false return (status[byteIdx] & (1 << bitIdx)) !== 0 } + +// 上传队列管理 +class UploadQueue { + private static instance: UploadQueue + private uploads: Map = new Map() + + static getInstance(): UploadQueue { + if (!UploadQueue.instance) { + UploadQueue.instance = new UploadQueue() + } + return UploadQueue.instance + } + + addUpload(uploadPath: string, state: UploadState): void { + this.uploads.set(uploadPath, state) + } + + pauseUpload(uploadPath: string): void { + const state = this.uploads.get(uploadPath) + if (state) { + state.isPaused = true + } + } + + resumeUpload(uploadPath: string): void { + const state = this.uploads.get(uploadPath) + if (state) { + state.isPaused = false + } + } + + cancelUpload(uploadPath: string): void { + const state = this.uploads.get(uploadPath) + if (state) { + state.isCancelled = true + } + } + + removeUpload(uploadPath: string): void { + this.uploads.delete(uploadPath) + } + + getUploadState(uploadPath: string): UploadState | undefined { + return this.uploads.get(uploadPath) + } + + getAllUploads(): Array<{path: string, state: UploadState}> { + return Array.from(this.uploads.entries()).map(([path, state]) => ({path, state})) + } +} + +// 导出队列管理函数 +export const uploadQueue = UploadQueue.getInstance() + +export const pauseUpload = (uploadPath: string) => uploadQueue.pauseUpload(uploadPath) +export const resumeUpload = (uploadPath: string) => uploadQueue.resumeUpload(uploadPath) +export const cancelUpload = (uploadPath: string) => uploadQueue.cancelUpload(uploadPath) From 17566278e365087737c6a5c2e3a882174482dbf4 Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Fri, 5 Sep 2025 23:51:32 +0800 Subject: [PATCH 07/18] feat(fs): Enhanced multi-part upload functionality, added server health check, error handling, and retry mechanisms --- src/pages/home/uploads/slice_upload.ts | 474 ++++++++++++++++++++++--- 1 file changed, 423 insertions(+), 51 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index ddc9e88f..4621ea54 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -15,37 +15,286 @@ import createMutex from "~/utils/mutex" // 重试配置 const RETRY_CONFIG = { - maxRetries: 3, - retryDelay: 1000, // 1秒 + maxRetries: 5, // 增加重试次数 + retryDelay: 1000, // 基础延迟1秒 + maxDelay: 30000, // 最大延迟30秒 backoffMultiplier: 2, // 指数退避 + // 服务器重启检测 + serverHealthCheckDelay: 5000, // 服务器健康检查延迟 + serverRestartRetries: 3, // 服务器重启后的特殊重试次数 } -// 大文件优化配置 -const MEMORY_OPTIMIZATION = { - largeFileThreshold: 100 * 1024 * 1024, // 100MB - maxConcurrentSlices: 2, // 大文件时减少并发 - chunkReadSize: 64 * 1024, // 64KB 分块读取 +// 服务器状态检测 +class ServerHealthChecker { + private static instance: ServerHealthChecker + private lastHealthCheck = 0 + private serverOnline = true + private checkPromise: Promise | null = null + + static getInstance(): ServerHealthChecker { + if (!ServerHealthChecker.instance) { + ServerHealthChecker.instance = new ServerHealthChecker() + } + return ServerHealthChecker.instance + } + + async isServerHealthy(): Promise { + const now = Date.now() + + // 如果最近检查过且结果为在线，直接返回 + if (this.serverOnline && now - this.lastHealthCheck < 10000) { + return true + } + + // 防止并发检查 + if (this.checkPromise) { + return this.checkPromise + } + + this.checkPromise = this.performHealthCheck() + try { + const result = await this.checkPromise + this.lastHealthCheck = now + this.serverOnline = result + return result + } finally { + this.checkPromise = null + } + } + + private async performHealthCheck(): Promise { + try { + const response = await r.get('/ping', { + timeout: 5000, + headers: { Password: password() } + }) + return response.status === 200 + } catch (error: any) { + console.warn('Server health check failed:', error.message) + return false + } + } + + markServerOffline(): void { + this.serverOnline = false + } + + async waitForServerRecovery(maxWaitTime = 60000): Promise { + const startTime = Date.now() + let attempt = 1 + + console.log('等待服务器恢复...') + + while (Date.now() - startTime < maxWaitTime) { + const isHealthy = await this.isServerHealthy() + if (isHealthy) { + console.log(`服务器已恢复 (第${attempt}次检查)`) + return true + } + + const waitTime = Math.min(5000 * attempt, 15000) // 渐进式等待 + console.log(`服务器检查失败，${waitTime/1000}秒后重试...`) + await new Promise(resolve => setTimeout(resolve, waitTime)) + attempt++ + } + + console.error('服务器恢复超时') + return false + } +} + +// 错误类型定义 +enum UploadErrorType { + NETWORK_ERROR = 'network_error', + SERVER_ERROR = 'server_error', + FILE_ERROR = 'file_error', + CANCEL_ERROR = 'cancel_error', + TIMEOUT_ERROR = 'timeout_error', + HASH_ERROR = 'hash_error', + MEMORY_ERROR = 'memory_error' +} + +class UploadError extends Error { + public type: UploadErrorType + public statusCode?: number + public retryable: boolean + public userMessage: string + + constructor( + type: UploadErrorType, + message: string, + userMessage: string, + statusCode?: number, + retryable: boolean = true + ) { + super(message) + this.type = type + this.statusCode = statusCode + this.retryable = retryable + this.userMessage = userMessage + this.name = 'UploadError' + } + + static fromAxiosError(error: any, chunkIndex?: number): UploadError { + const chunkMsg = chunkIndex !== undefined ? `分片 ${chunkIndex + 1}` : '文件' + + if (error.code === 'ECONNABORTED' || error.message?.includes('timeout')) { + return new UploadError( + UploadErrorType.TIMEOUT_ERROR, + `Upload timeout: ${error.message}`, + `${chunkMsg}上传超时，请检查网络连接`, + error.response?.status, + true + ) + } + + if (!error.response) { + return new UploadError( + UploadErrorType.NETWORK_ERROR, + `Network error: ${error.message}`, + `网络连接失败，请检查网络状态`, + undefined, + true + ) + } + + const status = error.response.status + const data = error.response.data + + if (status >= 500) { + return new UploadError( + UploadErrorType.SERVER_ERROR, + `Server error ${status}: ${data?.message || error.message}`, + `服务器暂时不可用 (${status})，正在重试...`, + status, + true + ) + } else if (status === 413) { + return new UploadError( + UploadErrorType.FILE_ERROR, + `File too large: ${data?.message || error.message}`, + `${chunkMsg}过大，请选择较小的文件`, + status, + false + ) + } else if (status === 401 || status === 403) { + return new UploadError( + UploadErrorType.SERVER_ERROR, + `Authorization failed: ${data?.message || error.message}`, + `认证失败，请重新登录`, + status, + false + ) + } else { + return new UploadError( + UploadErrorType.SERVER_ERROR, + `HTTP ${status}: ${data?.message || error.message}`, + `上传失败 (${status})，${data?.message || '未知错误'}`, + status, + status >= 400 && status < 500 ? false : true + ) + } + } + + static fromGenericError(error: any, context: string = ''): UploadError { + if (error instanceof UploadError) { + return error + } + + const message = error.message || String(error) + if (message.includes('memory') || message.includes('Memory')) { + return new UploadError( + UploadErrorType.MEMORY_ERROR, + `Memory error in ${context}: ${message}`, + `内存不足，请关闭其他程序或选择较小的文件`, + undefined, + false + ) + } + + return new UploadError( + UploadErrorType.FILE_ERROR, + `${context} error: ${message}`, + `文件处理出错: ${message}`, + undefined, + false + ) + } +} + +// 进度详情接口 +interface UploadProgress { + uploadedBytes: number + totalBytes: number + percentage: number + speed: number // bytes per second + remainingTime: number // seconds + activeChunks: number + completedChunks: number + totalChunks: number + lastError?: UploadError + stage: 'preparing' | 'hashing' | 'uploading' | 'completing' | 'completed' | 'error' } const progressMutex = createMutex() -// 重试函数 +// 智能重试函数，支持服务器重启检测 const retryWithBackoff = async ( fn: () => Promise, maxRetries: number = RETRY_CONFIG.maxRetries, - delay: number = RETRY_CONFIG.retryDelay + delay: number = RETRY_CONFIG.retryDelay, + context: string = 'operation' ): Promise => { + const healthChecker = ServerHealthChecker.getInstance() let lastError: Error + for (let i = 0; i <= maxRetries; i++) { try { return await fn() } catch (error) { lastError = error as Error + + // 如果是最后一次重试，直接抛出错误 if (i === maxRetries) { throw lastError } - // 指数退避延迟 - const waitTime = delay * Math.pow(RETRY_CONFIG.backoffMultiplier, i) + + // 检查是否是服务器相关错误 + const isServerError = error instanceof UploadError && + (error.type === UploadErrorType.SERVER_ERROR || error.type === UploadErrorType.NETWORK_ERROR) + + if (isServerError && error instanceof UploadError) { + // 标记服务器可能离线 + healthChecker.markServerOffline() + + // 检查服务器状态 + const isServerHealthy = await healthChecker.isServerHealthy() + + if (!isServerHealthy) { + console.log(`服务器似乎离线，等待恢复... (${context}, 重试 ${i + 1}/${maxRetries})`) + + // 等待服务器恢复，使用更长的等待时间 + const recovered = await healthChecker.waitForServerRecovery(30000) + + if (!recovered) { + // 服务器恢复失败，但还有重试机会，继续重试 + console.warn(`服务器恢复失败，继续重试 (${context})`) + } else { + console.log(`服务器已恢复，继续上传 (${context})`) + } + } + } + + // 计算延迟时间，对服务器错误使用更长的延迟 + let waitTime = delay * Math.pow(RETRY_CONFIG.backoffMultiplier, i) + if (isServerError) { + waitTime = Math.max(waitTime, RETRY_CONFIG.serverHealthCheckDelay) + } + waitTime = Math.min(waitTime, RETRY_CONFIG.maxDelay) + + console.log(`${context} 失败，${waitTime/1000}秒后重试 (${i + 1}/${maxRetries}):`, + (error as any) instanceof UploadError ? (error as UploadError).userMessage : (error as Error).message) + await new Promise(resolve => setTimeout(resolve, waitTime)) } } @@ -58,6 +307,12 @@ interface UploadState { isCancelled: boolean totalBytes: number uploadedBytes: number + completedChunks: number + totalChunks: number + activeChunks: number + speed: number + lastError?: UploadError + onProgress?: (progress: UploadProgress) => void } export const sliceupload = async ( @@ -79,6 +334,10 @@ export const sliceupload = async ( isCancelled: false, totalBytes: file.size, uploadedBytes: 0, + completedChunks: 0, + totalChunks: 0, + activeChunks: 0, + speed: 0, } // 注册到上传队列 @@ -128,6 +387,10 @@ export const sliceupload = async ( cleanup() return new Error(`Preup failed: ${resp1.code} - ${resp1.message}`) } + + // 设置总分片数 + state.totalChunks = resp1.data.slice_cnt + if (resp1.data.reuse) { setUpload("progress", "100") setUpload("status", "success") @@ -166,7 +429,13 @@ export const sliceupload = async ( ) => { // 检查是否被取消 if (state.isCancelled) { - throw new Error("Upload cancelled by user") + throw new UploadError( + UploadErrorType.CANCEL_ERROR, + 'Upload cancelled by user', + '上传已取消', + undefined, + false + ) } // 检查是否暂停，等待恢复 @@ -184,33 +453,60 @@ export const sliceupload = async ( let oldLoaded = 0 return retryWithBackoff(async () => { - const resp: EmptyResp = await r.post("/fs/slice_upload", formData, { - headers: { - "File-Path": encodeURIComponent(dir), - "Content-Type": "multipart/form-data", - Password: password(), - }, - onUploadProgress: async (progressEvent: any) => { - if (!progressEvent.lengthComputable || state.isCancelled) { - return - } - //获取锁 - const release = await progressMutex.acquire() - try { - const sliceuploaded = progressEvent.loaded - oldLoaded - state.uploadedBytes += sliceuploaded - oldLoaded = progressEvent.loaded - } finally { - progressMutex.release() - } - }, - }) - - if (resp.code != 200) { - throw new Error(`Slice upload failed: ${resp.code} - ${resp.message}`) + try { + const resp: EmptyResp = await r.post("/fs/slice_upload", formData, { + headers: { + "File-Path": encodeURIComponent(dir), + "Content-Type": "multipart/form-data", + Password: password(), + }, + onUploadProgress: async (progressEvent: any) => { + if (!progressEvent.lengthComputable || state.isCancelled) { + return + } + //获取锁 + const release = await progressMutex.acquire() + try { + const sliceuploaded = progressEvent.loaded - oldLoaded + state.uploadedBytes += sliceuploaded + oldLoaded = progressEvent.loaded + + // 更新完成的分片数（估算） + state.completedChunks = Math.floor(state.uploadedBytes / (state.totalBytes / state.totalChunks)) + + // 实时进度更新 + const progress = Math.min(100, ((state.uploadedBytes / state.totalBytes) * 100) | 0) + setUpload("progress", progress) + + } finally { + progressMutex.release() + } + }, + }) + + if (resp.code != 200) { + throw new UploadError( + UploadErrorType.SERVER_ERROR, + `Slice upload failed: ${resp.code} - ${resp.message}`, + `分片 ${idx + 1} 上传失败: ${resp.message || '服务器错误'}`, + resp.code, + resp.code >= 500 + ) + } + return resp + } catch (err: any) { + // 转换为结构化错误 + const uploadError = err instanceof UploadError + ? err + : UploadError.fromAxiosError(err, idx) + + // 记录最后的错误 + state.lastError = uploadError + + console.error(`Chunk ${idx + 1} upload failed:`, uploadError.userMessage) + throw uploadError } - return resp - }) + }, RETRY_CONFIG.maxRetries, RETRY_CONFIG.retryDelay, `slice_${idx + 1}_upload`) } // 进度速度计算 @@ -254,9 +550,8 @@ export const sliceupload = async ( } } else { state.uploadedBytes += Math.min(resp1.data.slice_size, state.totalBytes) - } // 后续分片并发上传，根据文件大小动态调整并发数 - const isLargeFile = file.size > MEMORY_OPTIMIZATION.largeFileThreshold - const concurrentLimit = isLargeFile ? MEMORY_OPTIMIZATION.maxConcurrentSlices : 3 + } // 后续分片并发上传 + const concurrentLimit = 3 // 固定3个并发 const limit = pLimit(concurrentLimit) console.log(`File size: ${(file.size / 1024 / 1024).toFixed(2)}MB, using ${concurrentLimit} concurrent uploads`) @@ -298,22 +593,62 @@ export const sliceupload = async ( "progress", Math.min(100, ((state.uploadedBytes / state.totalBytes) * 100) | 0), ) + setUpload("status", "error") cleanup() - return errors[0] + + // 返回最具代表性的错误 + const serverErrors = errors.filter(e => e instanceof UploadError && e.type === UploadErrorType.SERVER_ERROR) + const networkErrors = errors.filter(e => e instanceof UploadError && e.type === UploadErrorType.NETWORK_ERROR) + + if (serverErrors.length > 0) { + return serverErrors[0] + } else if (networkErrors.length > 0) { + return networkErrors[0] + } else { + return errors[0] + } } else { if (!asTask) { setUpload("status", "backending") } - const resp = await FsSliceupComplete(dir, resp1.data.task_id) - completeFlag = true - cleanup() - if (resp.code != 200) { - return new Error(`Upload complete failed: ${resp.code} - ${resp.message}`) - } else if (resp.data.complete == 0) { - return new Error("slice missing, please reupload") + + try { + const resp = await retryWithBackoff( + () => FsSliceupComplete(dir, resp1.data.task_id), + RETRY_CONFIG.maxRetries, + RETRY_CONFIG.retryDelay, + 'upload_complete' + ) + + completeFlag = true + cleanup() + + if (resp.code != 200) { + return new UploadError( + UploadErrorType.SERVER_ERROR, + `Upload complete failed: ${resp.code} - ${resp.message}`, + `上传完成确认失败: ${resp.message}`, + resp.code, + resp.code >= 500 + ) + } else if (resp.data.complete == 0) { + return new UploadError( + UploadErrorType.SERVER_ERROR, + "slice missing, please reupload", + "文件分片缺失，请重新上传", + undefined, + true + ) + } + + //状态处理交给上层 + return + } catch (error) { + cleanup() + return error instanceof UploadError + ? error + : UploadError.fromGenericError(error, 'upload_complete') } - //状态处理交给上层 - return } } @@ -393,3 +728,40 @@ export const uploadQueue = UploadQueue.getInstance() export const pauseUpload = (uploadPath: string) => uploadQueue.pauseUpload(uploadPath) export const resumeUpload = (uploadPath: string) => uploadQueue.resumeUpload(uploadPath) export const cancelUpload = (uploadPath: string) => uploadQueue.cancelUpload(uploadPath) + +// 导出错误类型和辅助函数 +export { UploadError, UploadErrorType } +export type { UploadProgress } + +// 导出服务器健康检查器 +export const serverHealthChecker = ServerHealthChecker.getInstance() + +// 获取上传详细信息的辅助函数 +export const getUploadDetails = (uploadPath: string): { + state?: UploadState, + progress?: UploadProgress, + errorMessage?: string +} => { + const state = uploadQueue.getUploadState(uploadPath) + if (!state) return {} + + const progress: UploadProgress = { + uploadedBytes: state.uploadedBytes, + totalBytes: state.totalBytes, + percentage: Math.min(100, ((state.uploadedBytes / state.totalBytes) * 100) | 0), + speed: state.speed, + remainingTime: state.speed > 0 ? (state.totalBytes - state.uploadedBytes) / state.speed : 0, + activeChunks: state.activeChunks, + completedChunks: state.completedChunks, + totalChunks: state.totalChunks, + lastError: state.lastError, + stage: state.isCancelled ? 'error' : + state.uploadedBytes >= state.totalBytes ? 'completed' : 'uploading' + } + + return { + state, + progress, + errorMessage: state.lastError?.userMessage + } +} From 91d83a068c48d8580ee73caccde96aee70570f93 Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Sat, 6 Sep 2025 00:37:38 +0800 Subject: [PATCH 08/18] feat(fs): Added task status synchronization function, optimized the processing logic after server restart, and adjusted the retry configuration --- src/pages/home/uploads/slice_upload.ts | 140 ++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 1 deletion(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index 4621ea54..e02c4b93 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -15,13 +15,16 @@ import createMutex from "~/utils/mutex" // 重试配置 const RETRY_CONFIG = { - maxRetries: 5, // 增加重试次数 + maxRetries: 10, // 增加重试次数以应对服务器重启 retryDelay: 1000, // 基础延迟1秒 maxDelay: 30000, // 最大延迟30秒 backoffMultiplier: 2, // 指数退避 // 服务器重启检测 serverHealthCheckDelay: 5000, // 服务器健康检查延迟 serverRestartRetries: 3, // 服务器重启后的特殊重试次数 + serverRecoveryMaxWait: 120000, // 最大等待服务器恢复时间（2分钟） + // 任务状态同步 + taskSyncRetries: 3, // 任务状态同步重试次数 } // 服务器状态检测 @@ -103,6 +106,92 @@ class ServerHealthChecker { } } +// 任务状态同步器 +class TaskSyncManager { + private static async syncTaskStatus( + dir: string, + fileName: string, + fileSize: number, + hash: any, + overwrite: boolean, + asTask: boolean, + expectedTaskId?: string + ) { + try { + const resp = await fsPreup(dir, fileName, fileSize, hash, overwrite, asTask) + if (resp.code === 200) { + return { + success: true, + taskId: resp.data.task_id, + sliceSize: resp.data.slice_size, + sliceCnt: resp.data.slice_cnt, + sliceUploadStatus: resp.data.slice_upload_status, + isExpectedTask: !expectedTaskId || resp.data.task_id === expectedTaskId + } + } + return { success: false, error: `Sync failed: ${resp.code} - ${resp.message}` } + } catch (error) { + return { success: false, error: `Sync error: ${error}` } + } + } + + static async handleServerRecovery( + dir: string, + fileName: string, + fileSize: number, + hash: any, + overwrite: boolean, + asTask: boolean, + currentTaskId: string, + currentSliceStatus: Uint8Array + ) { + console.log(`检测到服务器重启，正在同步任务状态: ${currentTaskId}`) + + for (let attempt = 0; attempt < RETRY_CONFIG.taskSyncRetries; attempt++) { + const syncResult = await this.syncTaskStatus( + dir, fileName, fileSize, hash, overwrite, asTask, currentTaskId + ) + + if (syncResult.success) { + const serverSliceStatus = base64ToUint8Array(syncResult.sliceUploadStatus!) + + if (syncResult.isExpectedTask) { + // 服务器任务ID匹配，比较状态 + console.log(`任务状态同步成功，任务ID匹配: ${currentTaskId}`) + return { + success: true, + needResync: !this.compareSliceStatus(currentSliceStatus, serverSliceStatus), + serverStatus: syncResult + } + } else { + // 服务器返回了不同的任务ID，可能是新任务 + console.log(`服务器返回新任务ID: ${syncResult.taskId}，原任务: ${currentTaskId}`) + return { + success: true, + needRestart: true, + serverStatus: syncResult + } + } + } + + if (attempt < RETRY_CONFIG.taskSyncRetries - 1) { + console.log(`任务同步失败，${2 ** attempt}秒后重试...`) + await new Promise(resolve => setTimeout(resolve, 2 ** attempt * 1000)) + } + } + + return { success: false, error: 'Task sync failed after all retries' } + } + + private static compareSliceStatus(local: Uint8Array, server: Uint8Array): boolean { + if (local.length !== server.length) return false + for (let i = 0; i < local.length; i++) { + if (local[i] !== server[i]) return false + } + return true + } +} + // 错误类型定义 enum UploadErrorType { NETWORK_ERROR = 'network_error', @@ -327,6 +416,14 @@ export const sliceupload = async ( let slicehash: string[] = [] let sliceupstatus: Uint8Array let ht: string[] = [] + + // 任务信息，用于状态同步 + let taskInfo: { + taskId: string + hash: any + sliceSize: number + sliceCnt: number + } | null = null // 初始化上传状态 const state: UploadState = uploadState || { @@ -391,6 +488,14 @@ export const sliceupload = async ( // 设置总分片数 state.totalChunks = resp1.data.slice_cnt + // 保存任务信息用于状态同步 + taskInfo = { + taskId: resp1.data.task_id, + hash, + sliceSize: resp1.data.slice_size, + sliceCnt: resp1.data.slice_cnt + } + if (resp1.data.reuse) { setUpload("progress", "100") setUpload("status", "success") @@ -495,6 +600,39 @@ export const sliceupload = async ( } return resp } catch (err: any) { + // 检查是否是服务器重启导致的任务不一致 + if (err?.response?.status === 400 && taskInfo) { + const errorMsg = err?.response?.data?.message || err.message || '' + if (errorMsg.includes('task') || errorMsg.includes('TaskID') || errorMsg.includes('failed get slice upload')) { + console.log(`检测到任务ID不一致，尝试同步任务状态: ${task_id}`) + + try { + const syncResult = await TaskSyncManager.handleServerRecovery( + dir, file.name, file.size, taskInfo.hash, overwrite, asTask, task_id, sliceupstatus + ) + + if (syncResult.success && syncResult.serverStatus) { + if (syncResult.needRestart) { + throw new UploadError( + UploadErrorType.SERVER_ERROR, + 'Task ID changed, need restart', + '服务器任务状态已变更，需要重新开始上传', + undefined, + false // 不可重试，需要重新开始 + ) + } else if (syncResult.needResync) { + // 更新本地状态 + sliceupstatus = base64ToUint8Array(syncResult.serverStatus.sliceUploadStatus!) + console.log('任务状态已同步，继续上传') + // 重新抛出原始错误，让重试机制处理 + } + } + } catch (syncError) { + console.warn('任务状态同步失败:', syncError) + } + } + } + // 转换为结构化错误 const uploadError = err instanceof UploadError ? err From 5b2f8fbcaf1aadde7ce3870291bb1dc52521814c Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Sat, 6 Sep 2025 12:38:25 +0800 Subject: [PATCH 09/18] =?UTF-8?q?feat(fs):=20=E5=A2=9E=E5=8A=A0=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E9=85=8D=E7=BD=AE=EF=BC=8C=E4=BC=98=E5=8C=96=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E5=99=A8=E9=87=8D=E5=90=AF=E5=90=8E=E7=9A=84=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=8A=B6=E6=80=81=E5=90=8C=E6=AD=A5=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=8C=E6=94=B9=E8=BF=9B=E9=94=99=E8=AF=AF=E5=A4=84=E7=90=86?= =?UTF-8?q?=E5=92=8C=E6=97=A5=E5=BF=97=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home/uploads/slice_upload.ts | 164 ++++++++++++++++--------- 1 file changed, 107 insertions(+), 57 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index e02c4b93..a9478471 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -15,16 +15,19 @@ import createMutex from "~/utils/mutex" // 重试配置 const RETRY_CONFIG = { - maxRetries: 10, // 增加重试次数以应对服务器重启 + maxRetries: 15, // 增加重试次数以更好应对服务器重启 retryDelay: 1000, // 基础延迟1秒 maxDelay: 30000, // 最大延迟30秒 backoffMultiplier: 2, // 指数退避 - // 服务器重启检测 - serverHealthCheckDelay: 5000, // 服务器健康检查延迟 - serverRestartRetries: 3, // 服务器重启后的特殊重试次数 - serverRecoveryMaxWait: 120000, // 最大等待服务器恢复时间（2分钟） + // 服务器重启检测和恢复 + serverHealthCheckDelay: 3000, // 服务器健康检查延迟 + serverRestartRetries: 5, // 服务器重启后的特殊重试次数 + serverRecoveryMaxWait: 180000, // 最大等待服务器恢复时间（3分钟） // 任务状态同步 - taskSyncRetries: 3, // 任务状态同步重试次数 + taskSyncRetries: 5, // 任务状态同步重试次数 + taskSyncDelay: 2000, // 任务同步延迟 + // 原生分片上传优化 + nativeSliceRetries: 8, // 原生分片上传额外重试次数 } // 服务器状态检测 @@ -101,7 +104,7 @@ class ServerHealthChecker { attempt++ } - console.error('服务器恢复超时') + console.error('Server recovery timeout') return false } } @@ -145,42 +148,71 @@ class TaskSyncManager { currentTaskId: string, currentSliceStatus: Uint8Array ) { - console.log(`检测到服务器重启，正在同步任务状态: ${currentTaskId}`) + console.log(`🔄 Server restart detected, syncing task status: ${currentTaskId}`) for (let attempt = 0; attempt < RETRY_CONFIG.taskSyncRetries; attempt++) { - const syncResult = await this.syncTaskStatus( - dir, fileName, fileSize, hash, overwrite, asTask, currentTaskId - ) - - if (syncResult.success) { - const serverSliceStatus = base64ToUint8Array(syncResult.sliceUploadStatus!) + try { + const syncResult = await this.syncTaskStatus( + dir, fileName, fileSize, hash, overwrite, asTask, currentTaskId + ) - if (syncResult.isExpectedTask) { - // 服务器任务ID匹配，比较状态 - console.log(`任务状态同步成功，任务ID匹配: ${currentTaskId}`) - return { - success: true, - needResync: !this.compareSliceStatus(currentSliceStatus, serverSliceStatus), - serverStatus: syncResult - } - } else { - // 服务器返回了不同的任务ID，可能是新任务 - console.log(`服务器返回新任务ID: ${syncResult.taskId}，原任务: ${currentTaskId}`) - return { - success: true, - needRestart: true, - serverStatus: syncResult + if (syncResult.success) { + const serverSliceStatus = base64ToUint8Array(syncResult.sliceUploadStatus!) + + if (syncResult.isExpectedTask) { + // Server task ID matches, compare status + const statusMatches = this.compareSliceStatus(currentSliceStatus, serverSliceStatus) + const serverCompletedSlices = this.countCompletedSlices(serverSliceStatus) + const localCompletedSlices = this.countCompletedSlices(currentSliceStatus) + + console.log(`✅ Task status sync successful - TaskID: ${currentTaskId}`) + console.log(`📊 Server completed slices: ${serverCompletedSlices}, local records: ${localCompletedSlices}`) + + return { + success: true, + needResync: !statusMatches, + serverStatus: syncResult, + message: `Task recovery successful, server has completed ${serverCompletedSlices} slices` + } + } else { + // Server returned different task ID, need to restart + console.log(`⚠️ Server returned new task ID: ${syncResult.taskId}, original task invalid: ${currentTaskId}`) + return { + success: true, + needRestart: true, + serverStatus: syncResult, + message: 'Server task has changed, need to restart upload' + } } } + } catch (error) { + console.warn(`🔄 Task sync attempt ${attempt + 1} failed:`, error) } if (attempt < RETRY_CONFIG.taskSyncRetries - 1) { - console.log(`任务同步失败，${2 ** attempt}秒后重试...`) - await new Promise(resolve => setTimeout(resolve, 2 ** attempt * 1000)) + const waitTime = RETRY_CONFIG.taskSyncDelay * (attempt + 1) + console.log(`⏳ Retrying task sync in ${waitTime/1000} seconds...`) + await new Promise(resolve => setTimeout(resolve, waitTime)) } } - return { success: false, error: 'Task sync failed after all retries' } + return { + success: false, + error: 'Task sync failed after all retries', + message: 'Task status sync failed, please restart upload' + } + } + + private static countCompletedSlices(sliceStatus: Uint8Array): number { + let count = 0 + for (let i = 0; i < sliceStatus.length * 8; i++) { + const byteIndex = Math.floor(i / 8) + const bitIndex = i % 8 + if (byteIndex < sliceStatus.length && (sliceStatus[byteIndex] & (1 << bitIndex)) !== 0) { + count++ + } + } + return count } private static compareSliceStatus(local: Uint8Array, server: Uint8Array): boolean { @@ -348,40 +380,40 @@ const retryWithBackoff = async ( throw lastError } - // 检查是否是服务器相关错误 + // Check if server-related error const isServerError = error instanceof UploadError && (error.type === UploadErrorType.SERVER_ERROR || error.type === UploadErrorType.NETWORK_ERROR) if (isServerError && error instanceof UploadError) { - // 标记服务器可能离线 + // Mark server as possibly offline healthChecker.markServerOffline() - // 检查服务器状态 + // Check server status const isServerHealthy = await healthChecker.isServerHealthy() if (!isServerHealthy) { - console.log(`服务器似乎离线，等待恢复... (${context}, 重试 ${i + 1}/${maxRetries})`) + console.log(`Server appears offline, waiting for recovery... (${context}, retry ${i + 1}/${maxRetries})`) - // 等待服务器恢复，使用更长的等待时间 + // Wait for server recovery with longer wait time const recovered = await healthChecker.waitForServerRecovery(30000) if (!recovered) { - // 服务器恢复失败，但还有重试机会，继续重试 - console.warn(`服务器恢复失败，继续重试 (${context})`) + // Server recovery failed, but still have retry chances, continue retrying + console.warn(`Server recovery failed, continue retrying (${context})`) } else { - console.log(`服务器已恢复，继续上传 (${context})`) + console.log(`Server recovered, continue upload (${context})`) } } } - // 计算延迟时间，对服务器错误使用更长的延迟 + // Calculate delay time, use longer delay for server errors let waitTime = delay * Math.pow(RETRY_CONFIG.backoffMultiplier, i) if (isServerError) { waitTime = Math.max(waitTime, RETRY_CONFIG.serverHealthCheckDelay) } waitTime = Math.min(waitTime, RETRY_CONFIG.maxDelay) - console.log(`${context} 失败，${waitTime/1000}秒后重试 (${i + 1}/${maxRetries}):`, + console.log(`${context} failed, retrying in ${waitTime/1000} seconds (${i + 1}/${maxRetries}):`, (error as any) instanceof UploadError ? (error as UploadError).userMessage : (error as Error).message) await new Promise(resolve => setTimeout(resolve, waitTime)) @@ -390,7 +422,7 @@ const retryWithBackoff = async ( throw lastError! } -// 上传状态管理 +// Upload state management interface UploadState { isPaused: boolean isCancelled: boolean @@ -600,48 +632,66 @@ export const sliceupload = async ( } return resp } catch (err: any) { - // 检查是否是服务器重启导致的任务不一致 + // 🔍 Smart error detection: server restart / task lost if (err?.response?.status === 400 && taskInfo) { const errorMsg = err?.response?.data?.message || err.message || '' - if (errorMsg.includes('task') || errorMsg.includes('TaskID') || errorMsg.includes('failed get slice upload')) { - console.log(`检测到任务ID不一致，尝试同步任务状态: ${task_id}`) + const isTaskNotFound = errorMsg.includes('task') || + errorMsg.includes('TaskID') || + errorMsg.includes('failed get slice upload') + + if (isTaskNotFound) { + console.log(`🚨 Task lost detected, starting smart recovery: ${task_id} (slice ${idx + 1})`) try { const syncResult = await TaskSyncManager.handleServerRecovery( dir, file.name, file.size, taskInfo.hash, overwrite, asTask, task_id, sliceupstatus ) - if (syncResult.success && syncResult.serverStatus) { + if (syncResult.success) { if (syncResult.needRestart) { + // Task needs to restart + console.log(`❌ ${syncResult.message}`) throw new UploadError( UploadErrorType.SERVER_ERROR, 'Task ID changed, need restart', - '服务器任务状态已变更，需要重新开始上传', + syncResult.message || 'Server task status changed, need to restart upload', undefined, - false // 不可重试，需要重新开始 + false // Not retryable, need restart ) } else if (syncResult.needResync) { - // 更新本地状态 - sliceupstatus = base64ToUint8Array(syncResult.serverStatus.sliceUploadStatus!) - console.log('任务状态已同步，继续上传') - // 重新抛出原始错误，让重试机制处理 + // Status synced, update local status and continue retry + sliceupstatus = base64ToUint8Array(syncResult.serverStatus!.sliceUploadStatus!) + console.log(`✅ ${syncResult.message}, continuing upload slice ${idx + 1}`) + + // Check if current slice is already completed on server + if (isSliceUploaded(sliceupstatus, idx)) { + console.log(`✅ Slice ${idx + 1} already completed on server, skipping upload`) + return { code: 200, message: 'Slice already uploaded on server' } as EmptyResp + } + + // Re-throw error to let retry mechanism continue + console.log(`🔄 Slice ${idx + 1} needs to be re-uploaded`) + } else { + console.log(`✅ ${syncResult.message}`) } + } else { + console.warn(`❌ Task status sync failed: ${syncResult.error}`) } } catch (syncError) { - console.warn('任务状态同步失败:', syncError) + console.warn('🔧 Error during task status sync:', syncError) } } } - // 转换为结构化错误 + // Convert to structured error const uploadError = err instanceof UploadError ? err : UploadError.fromAxiosError(err, idx) - // 记录最后的错误 + // Record last error state.lastError = uploadError - console.error(`Chunk ${idx + 1} upload failed:`, uploadError.userMessage) + console.error(`💥 Slice ${idx + 1} upload failed:`, uploadError.userMessage) throw uploadError } }, RETRY_CONFIG.maxRetries, RETRY_CONFIG.retryDelay, `slice_${idx + 1}_upload`) From ae51cfaae49953474809870b590deb0f50fedace Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Sat, 6 Sep 2025 13:04:48 +0800 Subject: [PATCH 10/18] =?UTF-8?q?feat(deps):=20=E6=B7=BB=E5=8A=A0=20p-limi?= =?UTF-8?q?t=20=E4=BE=9D=E8=B5=96=EF=BC=8C=E6=9B=B4=E6=96=B0=20pnpm-lock.y?= =?UTF-8?q?aml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + pnpm-lock.yaml | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/package.json b/package.json index 5aa393c9..88db0ab5 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "mitt": "^3.0.1", "monaco-editor": "^0.52.2", "mpegts.js": "^1.8.0", + "p-limit": "^7.1.1", "pdfjs-dist": "^5.3.31", "qrcode": "^1.5.4", "rehype-katex": "6.0.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e76abf1..bbd3ad16 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -121,6 +121,9 @@ importers: mpegts.js: specifier: ^1.8.0 version: 1.8.0 + p-limit: + specifier: ^7.1.1 + version: 7.1.1 pdfjs-dist: specifier: ^5.3.31 version: 5.4.54 @@ -1040,30 +1043,35 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-arm64-musl@0.1.74': resolution: {integrity: sha512-tfFqLHGtSEabBigOnPUfZviSTGmW2xHv5tYZYPBWmgGiTkoNJ7lEWFUxHjwvV5HXGqLs8ok/O7g1enSpxO6lmQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@napi-rs/canvas-linux-riscv64-gnu@0.1.74': resolution: {integrity: sha512-j6H9dHTMtr1y3tu/zGm1ythYIL9vTl4EEv9f6CMx0n3Zn2M+OruUUwh9ylCj4afzSNEK9T8cr6zMnmTPzkpBvQ==} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-x64-gnu@0.1.74': resolution: {integrity: sha512-73DIV4E7Y9CpIJuUXVl9H6+MEQXyRy4VJQoUGA1tOlcKQiStxqhq6UErL4decI28NxjyQXBhtYZKj5q8AJEuOg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-x64-musl@0.1.74': resolution: {integrity: sha512-FgDMEFdGIJT3I2xejflRJ82/ZgDphyirS43RgtoLaIXI6zihLiZcQ7rczpqeWgAwlJNjR0He2EustsKe1SkUOg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@napi-rs/canvas-win32-x64-msvc@0.1.74': resolution: {integrity: sha512-x6bhwlhn0wU7dfiP46mt5Bi6PowSUH4CJ4PTzGj58LRQ1HVasEIJgoMx7MLC48F738eJpzbfg3WR/D8+e9CeTA==} @@ -1117,56 +1125,67 @@ packages: resolution: {integrity: sha512-RkdOTu2jK7brlu+ZwjMIZfdV2sSYHK2qR08FUWcIoqJC2eywHbXr0L8T/pONFwkGukQqERDheaGTeedG+rra6Q==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.45.1': resolution: {integrity: sha512-3kJ8pgfBt6CIIr1o+HQA7OZ9mp/zDk3ctekGl9qn/pRBgrRgfwiffaUmqioUGN9hv0OHv2gxmvdKOkARCtRb8Q==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.45.1': resolution: {integrity: sha512-k3dOKCfIVixWjG7OXTCOmDfJj3vbdhN0QYEqB+OuGArOChek22hn7Uy5A/gTDNAcCy5v2YcXRJ/Qcnm4/ma1xw==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.45.1': resolution: {integrity: sha512-PmI1vxQetnM58ZmDFl9/Uk2lpBBby6B6rF4muJc65uZbxCs0EA7hhKCk2PKlmZKuyVSHAyIw3+/SiuMLxKxWog==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loongarch64-gnu@4.45.1': resolution: {integrity: sha512-9UmI0VzGmNJ28ibHW2GpE2nF0PBQqsyiS4kcJ5vK+wuwGnV5RlqdczVocDSUfGX/Na7/XINRVoUgJyFIgipoRg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-powerpc64le-gnu@4.45.1': resolution: {integrity: sha512-7nR2KY8oEOUTD3pBAxIBBbZr0U7U+R9HDTPNy+5nVVHDXI4ikYniH1oxQz9VoB5PbBU1CZuDGHkLJkd3zLMWsg==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.45.1': resolution: {integrity: sha512-nlcl3jgUultKROfZijKjRQLUu9Ma0PeNv/VFHkZiKbXTBQXhpytS8CIj5/NfBeECZtY2FJQubm6ltIxm/ftxpw==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.45.1': resolution: {integrity: sha512-HJV65KLS51rW0VY6rvZkiieiBnurSzpzore1bMKAhunQiECPuxsROvyeaot/tcK3A3aGnI+qTHqisrpSgQrpgA==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.45.1': resolution: {integrity: sha512-NITBOCv3Qqc6hhwFt7jLV78VEO/il4YcBzoMGGNxznLgRQf43VQDae0aAzKiBeEPIxnDrACiMgbqjuihx08OOw==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.45.1': resolution: {integrity: sha512-+E/lYl6qu1zqgPEnTrs4WysQtvc/Sh4fC2nByfFExqgYrqkKWp1tWIbe+ELhixnenSpBbLXNi6vbEEJ8M7fiHw==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.45.1': resolution: {integrity: sha512-a6WIAp89p3kpNoYStITT9RbTbTnqarU7D8N8F2CV+4Cl9fwCOZraLVuVFvlpsW0SbIiYtEnhCZBPLoNdRkjQFw==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-win32-arm64-msvc@4.45.1': resolution: {integrity: sha512-T5Bi/NS3fQiJeYdGvRpTAP5P02kqSOpqiopwhj0uaXB6nzs5JVi2XMJb18JUSKhCOX8+UE1UKQufyD6Or48dJg==} @@ -1287,24 +1306,28 @@ packages: engines: {node: '>=10'} cpu: [arm64] os: [linux] + libc: [glibc] '@swc/core-linux-arm64-musl@1.12.14': resolution: {integrity: sha512-ZkOOIpSMXuPAjfOXEIAEQcrPOgLi6CaXvA5W+GYnpIpFG21Nd0qb0WbwFRv4K8BRtl993Q21v0gPpOaFHU+wdA==} engines: {node: '>=10'} cpu: [arm64] os: [linux] + libc: [musl] '@swc/core-linux-x64-gnu@1.12.14': resolution: {integrity: sha512-71EPPccwJiJUxd2aMwNlTfom2mqWEWYGdbeTju01tzSHsEuD7E6ePlgC3P3ngBqB3urj41qKs87z7zPOswT5Iw==} engines: {node: '>=10'} cpu: [x64] os: [linux] + libc: [glibc] '@swc/core-linux-x64-musl@1.12.14': resolution: {integrity: sha512-nImF1hZJqKTcl0WWjHqlelOhvuB9rU9kHIw/CmISBUZXogjLIvGyop1TtJNz0ULcz2Oxr3Q2YpwfrzsgvgbGkA==} engines: {node: '>=10'} cpu: [x64] os: [linux] + libc: [musl] '@swc/core-win32-arm64-msvc@1.12.14': resolution: {integrity: sha512-sABFQFxSuStFoxvEWZUHWYldtB1B4A9eDNFd4Ty50q7cemxp7uoscFoaCqfXSGNBwwBwpS5EiPB6YN4y6hqmLQ==} @@ -2729,6 +2752,10 @@ packages: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} + p-limit@7.1.1: + resolution: {integrity: sha512-i8PyM2JnsNChVSYWLr2BAjNoLi0BAYC+wecOnZnVV+YSNJkzP7cWmvI34dk0WArWfH9KwBHNoZI3P3MppImlIA==} + engines: {node: '>=20'} + p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -3379,6 +3406,10 @@ packages: resolution: {integrity: sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==} engines: {node: '>=12'} + yocto-queue@1.2.1: + resolution: {integrity: sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==} + engines: {node: '>=12.20'} + zustand@5.0.5: resolution: {integrity: sha512-mILtRfKW9xM47hqxGIxCv12gXusoY/xTSHBYApXozR0HmQv299whhBeeAcRy+KrPPybzosvJBCOmVjq6x12fCg==} engines: {node: '>=12.20.0'} @@ -6228,6 +6259,10 @@ snapshots: dependencies: p-try: 2.2.0 + p-limit@7.1.1: + dependencies: + yocto-queue: 1.2.1 + p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -6919,6 +6954,8 @@ snapshots: buffer-crc32: 0.2.13 pend: 1.2.0 + yocto-queue@1.2.1: {} + zustand@5.0.5: {} zwitch@2.0.4: {} From 8ab5c5d2e441f220843f78693f6e97f8506dbc30 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 6 Sep 2025 13:31:58 +0800 Subject: [PATCH 11/18] =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home/uploads/slice_upload.ts | 608 +++++++++++++++---------- 1 file changed, 363 insertions(+), 245 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index a9478471..bbcc3c06 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -46,7 +46,7 @@ class ServerHealthChecker { async isServerHealthy(): Promise { const now = Date.now() - + // 如果最近检查过且结果为在线，直接返回 if (this.serverOnline && now - this.lastHealthCheck < 10000) { return true @@ -70,13 +70,13 @@ class ServerHealthChecker { private async performHealthCheck(): Promise { try { - const response = await r.get('/ping', { + const response = await r.get("/ping", { timeout: 5000, - headers: { Password: password() } + headers: { Password: password() }, }) return response.status === 200 } catch (error: any) { - console.warn('Server health check failed:', error.message) + console.warn("Server health check failed:", error.message) return false } } @@ -88,23 +88,23 @@ class ServerHealthChecker { async waitForServerRecovery(maxWaitTime = 60000): Promise { const startTime = Date.now() let attempt = 1 - - console.log('等待服务器恢复...') - + + console.log("等待服务器恢复...") + while (Date.now() - startTime < maxWaitTime) { const isHealthy = await this.isServerHealthy() if (isHealthy) { console.log(`服务器已恢复 (第${attempt}次检查)`) return true } - + const waitTime = Math.min(5000 * attempt, 15000) // 渐进式等待 - console.log(`服务器检查失败，${waitTime/1000}秒后重试...`) - await new Promise(resolve => setTimeout(resolve, waitTime)) + console.log(`服务器检查失败，${waitTime / 1000}秒后重试...`) + await new Promise((resolve) => setTimeout(resolve, waitTime)) attempt++ } - - console.error('Server recovery timeout') + + console.error("Server recovery timeout") return false } } @@ -112,16 +112,23 @@ class ServerHealthChecker { // 任务状态同步器 class TaskSyncManager { private static async syncTaskStatus( - dir: string, - fileName: string, - fileSize: number, - hash: any, - overwrite: boolean, + dir: string, + fileName: string, + fileSize: number, + hash: any, + overwrite: boolean, asTask: boolean, - expectedTaskId?: string + expectedTaskId?: string, ) { try { - const resp = await fsPreup(dir, fileName, fileSize, hash, overwrite, asTask) + const resp = await fsPreup( + dir, + fileName, + fileSize, + hash, + overwrite, + asTask, + ) if (resp.code === 200) { return { success: true, @@ -129,93 +136,124 @@ class TaskSyncManager { sliceSize: resp.data.slice_size, sliceCnt: resp.data.slice_cnt, sliceUploadStatus: resp.data.slice_upload_status, - isExpectedTask: !expectedTaskId || resp.data.task_id === expectedTaskId + isExpectedTask: + !expectedTaskId || resp.data.task_id === expectedTaskId, } } - return { success: false, error: `Sync failed: ${resp.code} - ${resp.message}` } + return { + success: false, + error: `Sync failed: ${resp.code} - ${resp.message}`, + } } catch (error) { return { success: false, error: `Sync error: ${error}` } } } static async handleServerRecovery( - dir: string, - fileName: string, - fileSize: number, - hash: any, - overwrite: boolean, + dir: string, + fileName: string, + fileSize: number, + hash: any, + overwrite: boolean, asTask: boolean, currentTaskId: string, - currentSliceStatus: Uint8Array + currentSliceStatus: Uint8Array, ) { - console.log(`🔄 Server restart detected, syncing task status: ${currentTaskId}`) - + console.log( + `🔄 Server restart detected, syncing task status: ${currentTaskId}`, + ) + for (let attempt = 0; attempt < RETRY_CONFIG.taskSyncRetries; attempt++) { try { const syncResult = await this.syncTaskStatus( - dir, fileName, fileSize, hash, overwrite, asTask, currentTaskId + dir, + fileName, + fileSize, + hash, + overwrite, + asTask, + currentTaskId, ) - + if (syncResult.success) { - const serverSliceStatus = base64ToUint8Array(syncResult.sliceUploadStatus!) - + const serverSliceStatus = base64ToUint8Array( + syncResult.sliceUploadStatus!, + ) + if (syncResult.isExpectedTask) { // Server task ID matches, compare status - const statusMatches = this.compareSliceStatus(currentSliceStatus, serverSliceStatus) - const serverCompletedSlices = this.countCompletedSlices(serverSliceStatus) - const localCompletedSlices = this.countCompletedSlices(currentSliceStatus) - - console.log(`✅ Task status sync successful - TaskID: ${currentTaskId}`) - console.log(`📊 Server completed slices: ${serverCompletedSlices}, local records: ${localCompletedSlices}`) - + const statusMatches = this.compareSliceStatus( + currentSliceStatus, + serverSliceStatus, + ) + const serverCompletedSlices = + this.countCompletedSlices(serverSliceStatus) + const localCompletedSlices = + this.countCompletedSlices(currentSliceStatus) + + console.log( + `✅ Task status sync successful - TaskID: ${currentTaskId}`, + ) + console.log( + `📊 Server completed slices: ${serverCompletedSlices}, local records: ${localCompletedSlices}`, + ) + return { success: true, needResync: !statusMatches, serverStatus: syncResult, - message: `Task recovery successful, server has completed ${serverCompletedSlices} slices` + message: `Task recovery successful, server has completed ${serverCompletedSlices} slices`, } } else { // Server returned different task ID, need to restart - console.log(`⚠️ Server returned new task ID: ${syncResult.taskId}, original task invalid: ${currentTaskId}`) + console.log( + `⚠️ Server returned new task ID: ${syncResult.taskId}, original task invalid: ${currentTaskId}`, + ) return { success: true, needRestart: true, serverStatus: syncResult, - message: 'Server task has changed, need to restart upload' + message: "Server task has changed, need to restart upload", } } } } catch (error) { console.warn(`🔄 Task sync attempt ${attempt + 1} failed:`, error) } - + if (attempt < RETRY_CONFIG.taskSyncRetries - 1) { const waitTime = RETRY_CONFIG.taskSyncDelay * (attempt + 1) - console.log(`⏳ Retrying task sync in ${waitTime/1000} seconds...`) - await new Promise(resolve => setTimeout(resolve, waitTime)) + console.log(`⏳ Retrying task sync in ${waitTime / 1000} seconds...`) + await new Promise((resolve) => setTimeout(resolve, waitTime)) } } - - return { - success: false, - error: 'Task sync failed after all retries', - message: 'Task status sync failed, please restart upload' + + return { + success: false, + error: "Task sync failed after all retries", + message: "Task status sync failed, please restart upload", } } - + private static countCompletedSlices(sliceStatus: Uint8Array): number { let count = 0 for (let i = 0; i < sliceStatus.length * 8; i++) { const byteIndex = Math.floor(i / 8) const bitIndex = i % 8 - if (byteIndex < sliceStatus.length && (sliceStatus[byteIndex] & (1 << bitIndex)) !== 0) { + if ( + byteIndex < sliceStatus.length && + (sliceStatus[byteIndex] & (1 << bitIndex)) !== 0 + ) { count++ } } return count } - - private static compareSliceStatus(local: Uint8Array, server: Uint8Array): boolean { + + private static compareSliceStatus( + local: Uint8Array, + server: Uint8Array, + ): boolean { if (local.length !== server.length) return false for (let i = 0; i < local.length; i++) { if (local[i] !== server[i]) return false @@ -226,13 +264,13 @@ class TaskSyncManager { // 错误类型定义 enum UploadErrorType { - NETWORK_ERROR = 'network_error', - SERVER_ERROR = 'server_error', - FILE_ERROR = 'file_error', - CANCEL_ERROR = 'cancel_error', - TIMEOUT_ERROR = 'timeout_error', - HASH_ERROR = 'hash_error', - MEMORY_ERROR = 'memory_error' + NETWORK_ERROR = "network_error", + SERVER_ERROR = "server_error", + FILE_ERROR = "file_error", + CANCEL_ERROR = "cancel_error", + TIMEOUT_ERROR = "timeout_error", + HASH_ERROR = "hash_error", + MEMORY_ERROR = "memory_error", } class UploadError extends Error { @@ -242,40 +280,41 @@ class UploadError extends Error { public userMessage: string constructor( - type: UploadErrorType, - message: string, + type: UploadErrorType, + message: string, userMessage: string, statusCode?: number, - retryable: boolean = true + retryable: boolean = true, ) { super(message) this.type = type this.statusCode = statusCode this.retryable = retryable this.userMessage = userMessage - this.name = 'UploadError' + this.name = "UploadError" } static fromAxiosError(error: any, chunkIndex?: number): UploadError { - const chunkMsg = chunkIndex !== undefined ? `分片 ${chunkIndex + 1}` : '文件' - - if (error.code === 'ECONNABORTED' || error.message?.includes('timeout')) { + const chunkMsg = + chunkIndex !== undefined ? `分片 ${chunkIndex + 1}` : "文件" + + if (error.code === "ECONNABORTED" || error.message?.includes("timeout")) { return new UploadError( UploadErrorType.TIMEOUT_ERROR, `Upload timeout: ${error.message}`, `${chunkMsg}上传超时，请检查网络连接`, error.response?.status, - true + true, ) } - + if (!error.response) { return new UploadError( UploadErrorType.NETWORK_ERROR, `Network error: ${error.message}`, `网络连接失败，请检查网络状态`, undefined, - true + true, ) } @@ -288,7 +327,7 @@ class UploadError extends Error { `Server error ${status}: ${data?.message || error.message}`, `服务器暂时不可用 (${status})，正在重试...`, status, - true + true, ) } else if (status === 413) { return new UploadError( @@ -296,7 +335,7 @@ class UploadError extends Error { `File too large: ${data?.message || error.message}`, `${chunkMsg}过大，请选择较小的文件`, status, - false + false, ) } else if (status === 401 || status === 403) { return new UploadError( @@ -304,41 +343,41 @@ class UploadError extends Error { `Authorization failed: ${data?.message || error.message}`, `认证失败，请重新登录`, status, - false + false, ) } else { return new UploadError( UploadErrorType.SERVER_ERROR, `HTTP ${status}: ${data?.message || error.message}`, - `上传失败 (${status})，${data?.message || '未知错误'}`, + `上传失败 (${status})，${data?.message || "未知错误"}`, status, - status >= 400 && status < 500 ? false : true + status >= 400 && status < 500 ? false : true, ) } } - static fromGenericError(error: any, context: string = ''): UploadError { + static fromGenericError(error: any, context: string = ""): UploadError { if (error instanceof UploadError) { return error } - + const message = error.message || String(error) - if (message.includes('memory') || message.includes('Memory')) { + if (message.includes("memory") || message.includes("Memory")) { return new UploadError( UploadErrorType.MEMORY_ERROR, `Memory error in ${context}: ${message}`, `内存不足，请关闭其他程序或选择较小的文件`, undefined, - false + false, ) } - + return new UploadError( UploadErrorType.FILE_ERROR, `${context} error: ${message}`, `文件处理出错: ${message}`, undefined, - false + false, ) } } @@ -348,13 +387,19 @@ interface UploadProgress { uploadedBytes: number totalBytes: number percentage: number - speed: number // bytes per second - remainingTime: number // seconds + speed: number // bytes per second + remainingTime: number // seconds activeChunks: number completedChunks: number totalChunks: number lastError?: UploadError - stage: 'preparing' | 'hashing' | 'uploading' | 'completing' | 'completed' | 'error' + stage: + | "preparing" + | "hashing" + | "uploading" + | "completing" + | "completed" + | "error" } const progressMutex = createMutex() @@ -364,59 +409,69 @@ const retryWithBackoff = async ( fn: () => Promise, maxRetries: number = RETRY_CONFIG.maxRetries, delay: number = RETRY_CONFIG.retryDelay, - context: string = 'operation' + context: string = "operation", ): Promise => { const healthChecker = ServerHealthChecker.getInstance() let lastError: Error - + for (let i = 0; i <= maxRetries; i++) { try { return await fn() } catch (error) { lastError = error as Error - + // 如果是最后一次重试，直接抛出错误 if (i === maxRetries) { throw lastError } // Check if server-related error - const isServerError = error instanceof UploadError && - (error.type === UploadErrorType.SERVER_ERROR || error.type === UploadErrorType.NETWORK_ERROR) - + const isServerError = + error instanceof UploadError && + (error.type === UploadErrorType.SERVER_ERROR || + error.type === UploadErrorType.NETWORK_ERROR) + if (isServerError && error instanceof UploadError) { // Mark server as possibly offline healthChecker.markServerOffline() - + // Check server status const isServerHealthy = await healthChecker.isServerHealthy() - + if (!isServerHealthy) { - console.log(`Server appears offline, waiting for recovery... (${context}, retry ${i + 1}/${maxRetries})`) - + console.log( + `Server appears offline, waiting for recovery... (${context}, retry ${i + 1}/${maxRetries})`, + ) + // Wait for server recovery with longer wait time const recovered = await healthChecker.waitForServerRecovery(30000) - + if (!recovered) { // Server recovery failed, but still have retry chances, continue retrying - console.warn(`Server recovery failed, continue retrying (${context})`) + console.warn( + `Server recovery failed, continue retrying (${context})`, + ) } else { console.log(`Server recovered, continue upload (${context})`) } } } - + // Calculate delay time, use longer delay for server errors let waitTime = delay * Math.pow(RETRY_CONFIG.backoffMultiplier, i) if (isServerError) { waitTime = Math.max(waitTime, RETRY_CONFIG.serverHealthCheckDelay) } waitTime = Math.min(waitTime, RETRY_CONFIG.maxDelay) - - console.log(`${context} failed, retrying in ${waitTime/1000} seconds (${i + 1}/${maxRetries}):`, - (error as any) instanceof UploadError ? (error as UploadError).userMessage : (error as Error).message) - - await new Promise(resolve => setTimeout(resolve, waitTime)) + + console.log( + `${context} failed, retrying in ${waitTime / 1000} seconds (${i + 1}/${maxRetries}):`, + (error as any) instanceof UploadError + ? (error as UploadError).userMessage + : (error as Error).message, + ) + + await new Promise((resolve) => setTimeout(resolve, waitTime)) } } throw lastError! @@ -448,7 +503,7 @@ export const sliceupload = async ( let slicehash: string[] = [] let sliceupstatus: Uint8Array let ht: string[] = [] - + // 任务信息，用于状态同步 let taskInfo: { taskId: string @@ -516,18 +571,18 @@ export const sliceupload = async ( cleanup() return new Error(`Preup failed: ${resp1.code} - ${resp1.message}`) } - + // 设置总分片数 state.totalChunks = resp1.data.slice_cnt - + // 保存任务信息用于状态同步 taskInfo = { taskId: resp1.data.task_id, hash, sliceSize: resp1.data.slice_size, - sliceCnt: resp1.data.slice_cnt + sliceCnt: resp1.data.slice_cnt, } - + if (resp1.data.reuse) { setUpload("progress", "100") setUpload("status", "success") @@ -568,16 +623,16 @@ export const sliceupload = async ( if (state.isCancelled) { throw new UploadError( UploadErrorType.CANCEL_ERROR, - 'Upload cancelled by user', - '上传已取消', + "Upload cancelled by user", + "上传已取消", undefined, - false + false, ) } // 检查是否暂停，等待恢复 while (state.isPaused && !state.isCancelled) { - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) } const formData = new FormData() @@ -589,112 +644,147 @@ export const sliceupload = async ( let oldTimestamp = Date.now() let oldLoaded = 0 - return retryWithBackoff(async () => { - try { - const resp: EmptyResp = await r.post("/fs/slice_upload", formData, { - headers: { - "File-Path": encodeURIComponent(dir), - "Content-Type": "multipart/form-data", - Password: password(), - }, - onUploadProgress: async (progressEvent: any) => { - if (!progressEvent.lengthComputable || state.isCancelled) { - return - } - //获取锁 - const release = await progressMutex.acquire() - try { - const sliceuploaded = progressEvent.loaded - oldLoaded - state.uploadedBytes += sliceuploaded - oldLoaded = progressEvent.loaded - - // 更新完成的分片数（估算） - state.completedChunks = Math.floor(state.uploadedBytes / (state.totalBytes / state.totalChunks)) - - // 实时进度更新 - const progress = Math.min(100, ((state.uploadedBytes / state.totalBytes) * 100) | 0) - setUpload("progress", progress) - - } finally { - progressMutex.release() - } - }, - }) - - if (resp.code != 200) { - throw new UploadError( - UploadErrorType.SERVER_ERROR, - `Slice upload failed: ${resp.code} - ${resp.message}`, - `分片 ${idx + 1} 上传失败: ${resp.message || '服务器错误'}`, - resp.code, - resp.code >= 500 - ) - } - return resp - } catch (err: any) { - // 🔍 Smart error detection: server restart / task lost - if (err?.response?.status === 400 && taskInfo) { - const errorMsg = err?.response?.data?.message || err.message || '' - const isTaskNotFound = errorMsg.includes('task') || - errorMsg.includes('TaskID') || - errorMsg.includes('failed get slice upload') - - if (isTaskNotFound) { - console.log(`🚨 Task lost detected, starting smart recovery: ${task_id} (slice ${idx + 1})`) - - try { - const syncResult = await TaskSyncManager.handleServerRecovery( - dir, file.name, file.size, taskInfo.hash, overwrite, asTask, task_id, sliceupstatus + return retryWithBackoff( + async () => { + try { + const resp: EmptyResp = await r.post("/fs/slice_upload", formData, { + headers: { + "File-Path": encodeURIComponent(dir), + "Content-Type": "multipart/form-data", + Password: password(), + }, + onUploadProgress: async (progressEvent: any) => { + if (!progressEvent.lengthComputable || state.isCancelled) { + return + } + //获取锁 + const release = await progressMutex.acquire() + try { + const sliceuploaded = progressEvent.loaded - oldLoaded + state.uploadedBytes += sliceuploaded + oldLoaded = progressEvent.loaded + + // 更新完成的分片数（估算） + state.completedChunks = Math.floor( + state.uploadedBytes / (state.totalBytes / state.totalChunks), + ) + + // 实时进度更新 + const progress = Math.min( + 100, + ((state.uploadedBytes / state.totalBytes) * 100) | 0, + ) + setUpload("progress", progress) + } finally { + progressMutex.release() + } + }, + }) + + if (resp.code != 200) { + throw new UploadError( + UploadErrorType.SERVER_ERROR, + `Slice upload failed: ${resp.code} - ${resp.message}`, + `分片 ${idx + 1} 上传失败: ${resp.message || "服务器错误"}`, + resp.code, + resp.code >= 500, + ) + } + return resp + } catch (err: any) { + // 🔍 Smart error detection: server restart / task lost + if (err?.response?.status === 400 && taskInfo) { + const errorMsg = err?.response?.data?.message || err.message || "" + const isTaskNotFound = + errorMsg.includes("task") || + errorMsg.includes("TaskID") || + errorMsg.includes("failed get slice upload") + + if (isTaskNotFound) { + console.log( + `🚨 Task lost detected, starting smart recovery: ${task_id} (slice ${idx + 1})`, ) - - if (syncResult.success) { - if (syncResult.needRestart) { - // Task needs to restart - console.log(`❌ ${syncResult.message}`) - throw new UploadError( - UploadErrorType.SERVER_ERROR, - 'Task ID changed, need restart', - syncResult.message || 'Server task status changed, need to restart upload', - undefined, - false // Not retryable, need restart - ) - } else if (syncResult.needResync) { - // Status synced, update local status and continue retry - sliceupstatus = base64ToUint8Array(syncResult.serverStatus!.sliceUploadStatus!) - console.log(`✅ ${syncResult.message}, continuing upload slice ${idx + 1}`) - - // Check if current slice is already completed on server - if (isSliceUploaded(sliceupstatus, idx)) { - console.log(`✅ Slice ${idx + 1} already completed on server, skipping upload`) - return { code: 200, message: 'Slice already uploaded on server' } as EmptyResp + + try { + const syncResult = await TaskSyncManager.handleServerRecovery( + dir, + file.name, + file.size, + taskInfo.hash, + overwrite, + asTask, + task_id, + sliceupstatus, + ) + + if (syncResult.success) { + if (syncResult.needRestart) { + // Task needs to restart + console.log(`❌ ${syncResult.message}`) + throw new UploadError( + UploadErrorType.SERVER_ERROR, + "Task ID changed, need restart", + syncResult.message || + "Server task status changed, need to restart upload", + undefined, + false, // Not retryable, need restart + ) + } else if (syncResult.needResync) { + // Status synced, update local status and continue retry + sliceupstatus = base64ToUint8Array( + syncResult.serverStatus!.sliceUploadStatus!, + ) + console.log( + `✅ ${syncResult.message}, continuing upload slice ${idx + 1}`, + ) + + // Check if current slice is already completed on server + if (isSliceUploaded(sliceupstatus, idx)) { + console.log( + `✅ Slice ${idx + 1} already completed on server, skipping upload`, + ) + return { + code: 200, + message: "Slice already uploaded on server", + } as EmptyResp + } + + // Re-throw error to let retry mechanism continue + console.log(`🔄 Slice ${idx + 1} needs to be re-uploaded`) + } else { + console.log(`✅ ${syncResult.message}`) } - - // Re-throw error to let retry mechanism continue - console.log(`🔄 Slice ${idx + 1} needs to be re-uploaded`) } else { - console.log(`✅ ${syncResult.message}`) + console.warn( + `❌ Task status sync failed: ${syncResult.error}`, + ) } - } else { - console.warn(`❌ Task status sync failed: ${syncResult.error}`) + } catch (syncError) { + console.warn("🔧 Error during task status sync:", syncError) } - } catch (syncError) { - console.warn('🔧 Error during task status sync:', syncError) } } + + // Convert to structured error + const uploadError = + err instanceof UploadError + ? err + : UploadError.fromAxiosError(err, idx) + + // Record last error + state.lastError = uploadError + + console.error( + `💥 Slice ${idx + 1} upload failed:`, + uploadError.userMessage, + ) + throw uploadError } - - // Convert to structured error - const uploadError = err instanceof UploadError - ? err - : UploadError.fromAxiosError(err, idx) - - // Record last error - state.lastError = uploadError - - console.error(`💥 Slice ${idx + 1} upload failed:`, uploadError.userMessage) - throw uploadError - } - }, RETRY_CONFIG.maxRetries, RETRY_CONFIG.retryDelay, `slice_${idx + 1}_upload`) + }, + RETRY_CONFIG.maxRetries, + RETRY_CONFIG.retryDelay, + `slice_${idx + 1}_upload`, + ) } // 进度速度计算 @@ -710,7 +800,10 @@ export const sliceupload = async ( return } const speed = intervalLoaded / ((Date.now() - lastTimestamp) / 1000) - const complete = Math.min(100, ((state.uploadedBytes / state.totalBytes) * 100) | 0) + const complete = Math.min( + 100, + ((state.uploadedBytes / state.totalBytes) * 100) | 0, + ) setUpload("speed", speed) setUpload("progress", complete) lastTimestamp = Date.now() @@ -736,13 +829,15 @@ export const sliceupload = async ( setUpload("speed", 0) return err as Error } - } else { - state.uploadedBytes += Math.min(resp1.data.slice_size, state.totalBytes) - } // 后续分片并发上传 + } else { + state.uploadedBytes += Math.min(resp1.data.slice_size, state.totalBytes) + } // 后续分片并发上传 const concurrentLimit = 3 // 固定3个并发 const limit = pLimit(concurrentLimit) - console.log(`File size: ${(file.size / 1024 / 1024).toFixed(2)}MB, using ${concurrentLimit} concurrent uploads`) + console.log( + `File size: ${(file.size / 1024 / 1024).toFixed(2)}MB, using ${concurrentLimit} concurrent uploads`, + ) const tasks: Promise[] = [] const errors: Error[] = [] @@ -783,11 +878,17 @@ export const sliceupload = async ( ) setUpload("status", "error") cleanup() - + // 返回最具代表性的错误 - const serverErrors = errors.filter(e => e instanceof UploadError && e.type === UploadErrorType.SERVER_ERROR) - const networkErrors = errors.filter(e => e instanceof UploadError && e.type === UploadErrorType.NETWORK_ERROR) - + const serverErrors = errors.filter( + (e) => + e instanceof UploadError && e.type === UploadErrorType.SERVER_ERROR, + ) + const networkErrors = errors.filter( + (e) => + e instanceof UploadError && e.type === UploadErrorType.NETWORK_ERROR, + ) + if (serverErrors.length > 0) { return serverErrors[0] } else if (networkErrors.length > 0) { @@ -799,25 +900,25 @@ export const sliceupload = async ( if (!asTask) { setUpload("status", "backending") } - + try { const resp = await retryWithBackoff( () => FsSliceupComplete(dir, resp1.data.task_id), RETRY_CONFIG.maxRetries, RETRY_CONFIG.retryDelay, - 'upload_complete' + "upload_complete", ) - + completeFlag = true cleanup() - + if (resp.code != 200) { return new UploadError( UploadErrorType.SERVER_ERROR, `Upload complete failed: ${resp.code} - ${resp.message}`, `上传完成确认失败: ${resp.message}`, resp.code, - resp.code >= 500 + resp.code >= 500, ) } else if (resp.data.complete == 0) { return new UploadError( @@ -825,17 +926,17 @@ export const sliceupload = async ( "slice missing, please reupload", "文件分片缺失，请重新上传", undefined, - true + true, ) } - + //状态处理交给上层 return } catch (error) { cleanup() - return error instanceof UploadError - ? error - : UploadError.fromGenericError(error, 'upload_complete') + return error instanceof UploadError + ? error + : UploadError.fromGenericError(error, "upload_complete") } } } @@ -905,17 +1006,23 @@ class UploadQueue { return this.uploads.get(uploadPath) } - getAllUploads(): Array<{path: string, state: UploadState}> { - return Array.from(this.uploads.entries()).map(([path, state]) => ({path, state})) + getAllUploads(): Array<{ path: string; state: UploadState }> { + return Array.from(this.uploads.entries()).map(([path, state]) => ({ + path, + state, + })) } } // 导出队列管理函数 export const uploadQueue = UploadQueue.getInstance() -export const pauseUpload = (uploadPath: string) => uploadQueue.pauseUpload(uploadPath) -export const resumeUpload = (uploadPath: string) => uploadQueue.resumeUpload(uploadPath) -export const cancelUpload = (uploadPath: string) => uploadQueue.cancelUpload(uploadPath) +export const pauseUpload = (uploadPath: string) => + uploadQueue.pauseUpload(uploadPath) +export const resumeUpload = (uploadPath: string) => + uploadQueue.resumeUpload(uploadPath) +export const cancelUpload = (uploadPath: string) => + uploadQueue.cancelUpload(uploadPath) // 导出错误类型和辅助函数 export { UploadError, UploadErrorType } @@ -925,31 +1032,42 @@ export type { UploadProgress } export const serverHealthChecker = ServerHealthChecker.getInstance() // 获取上传详细信息的辅助函数 -export const getUploadDetails = (uploadPath: string): { - state?: UploadState, - progress?: UploadProgress, +export const getUploadDetails = ( + uploadPath: string, +): { + state?: UploadState + progress?: UploadProgress errorMessage?: string } => { const state = uploadQueue.getUploadState(uploadPath) if (!state) return {} - + const progress: UploadProgress = { uploadedBytes: state.uploadedBytes, totalBytes: state.totalBytes, - percentage: Math.min(100, ((state.uploadedBytes / state.totalBytes) * 100) | 0), + percentage: Math.min( + 100, + ((state.uploadedBytes / state.totalBytes) * 100) | 0, + ), speed: state.speed, - remainingTime: state.speed > 0 ? (state.totalBytes - state.uploadedBytes) / state.speed : 0, + remainingTime: + state.speed > 0 + ? (state.totalBytes - state.uploadedBytes) / state.speed + : 0, activeChunks: state.activeChunks, completedChunks: state.completedChunks, totalChunks: state.totalChunks, lastError: state.lastError, - stage: state.isCancelled ? 'error' : - state.uploadedBytes >= state.totalBytes ? 'completed' : 'uploading' + stage: state.isCancelled + ? "error" + : state.uploadedBytes >= state.totalBytes + ? "completed" + : "uploading", } - + return { state, progress, - errorMessage: state.lastError?.userMessage + errorMessage: state.lastError?.userMessage, } } From 55793217034e4715bfadbe32c66cd185b08a0946 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 6 Sep 2025 13:33:05 +0800 Subject: [PATCH 12/18] =?UTF-8?q?=E6=B5=81=E5=BC=8F=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E5=88=86=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home/uploads/slice_upload.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index bbcc3c06..8802d7e5 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -634,23 +634,18 @@ export const sliceupload = async ( while (state.isPaused && !state.isCancelled) { await new Promise((resolve) => setTimeout(resolve, 100)) } - - const formData = new FormData() - formData.append("task_id", task_id) - formData.append("slice_hash", slice_hash) - formData.append("slice_num", idx.toString()) - formData.append("slice", chunk) - - let oldTimestamp = Date.now() let oldLoaded = 0 return retryWithBackoff( async () => { try { - const resp: EmptyResp = await r.post("/fs/slice_upload", formData, { + const slice = chunk.slice(0, chunk.size) + const resp: EmptyResp = await r.put("/fs/slice_upload", slice, { headers: { "File-Path": encodeURIComponent(dir), - "Content-Type": "multipart/form-data", + "X-Task-ID": task_id, + "X-Slice-Num": idx.toString(), + "X-Slice-Hash": slice_hash, Password: password(), }, onUploadProgress: async (progressEvent: any) => { From 3766f0ba688263f7ec26e389975830c914509f8d Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Sat, 6 Sep 2025 13:36:44 +0800 Subject: [PATCH 13/18] feat(fs): fix p-limit --- src/pages/home/uploads/slice_upload.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index a9478471..73a96ee9 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -2,7 +2,6 @@ import { password } from "~/store" import { EmptyResp } from "~/types" import { r, pathDir } from "~/utils" import { SetUpload, Upload } from "./types" -import pLimit from "p-limit" import { calculateHash, calculateSliceHash, @@ -740,6 +739,7 @@ export const sliceupload = async ( state.uploadedBytes += Math.min(resp1.data.slice_size, state.totalBytes) } // 后续分片并发上传 const concurrentLimit = 3 // 固定3个并发 + const { default: pLimit } = await import("p-limit") const limit = pLimit(concurrentLimit) console.log(`File size: ${(file.size / 1024 / 1024).toFixed(2)}MB, using ${concurrentLimit} concurrent uploads`) From 54dab7a6f99c35f3a1a8f7f47e6cecb67750f8f4 Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 6 Sep 2025 14:02:12 +0800 Subject: [PATCH 14/18] =?UTF-8?q?feat(upload):=20=E6=B7=BB=E5=8A=A0=20"tas?= =?UTF-8?q?ked"=20=E7=8A=B6=E6=80=81=E6=94=AF=E6=8C=81=EF=BC=8C=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E7=8A=B6=E6=80=81=E6=A3=80=E6=9F=A5=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home/uploads/Upload.tsx | 5 +++-- src/pages/home/uploads/types.ts | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pages/home/uploads/Upload.tsx b/src/pages/home/uploads/Upload.tsx index 0a9c0371..b690f570 100644 --- a/src/pages/home/uploads/Upload.tsx +++ b/src/pages/home/uploads/Upload.tsx @@ -87,7 +87,7 @@ const Upload = () => { }) const allDone = () => { return uploadFiles.uploads.every(({ status }) => - ["success", "error"].includes(status), + ["success", "error", "tasked"].includes(status), ) } let fileInput: HTMLInputElement @@ -150,7 +150,8 @@ const Upload = () => { onClick={() => { setUploadFiles("uploads", (_uploads) => _uploads.filter( - ({ status }) => !["success", "error"].includes(status), + ({ status }) => + !["success", "error", "tasked"].includes(status), ), ) console.log(uploadFiles.uploads) diff --git a/src/pages/home/uploads/types.ts b/src/pages/home/uploads/types.ts index 2845d2f1..74d1f254 100644 --- a/src/pages/home/uploads/types.ts +++ b/src/pages/home/uploads/types.ts @@ -12,6 +12,7 @@ export const StatusBadge = { pending: "neutral", uploading: "info", backending: "info", + tasked: "info", success: "success", error: "danger", } as const From 238896f9c81abd88a07d636504859a13de8ad930 Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Sat, 6 Sep 2025 14:29:11 +0800 Subject: [PATCH 15/18] =?UTF-8?q?refactor(upload):=20=E7=A7=BB=E9=99=A4=20?= =?UTF-8?q?p-limit=20=E4=BE=9D=E8=B5=96=EF=BC=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0=E5=B9=B6=E5=8F=91=E5=A4=84?= =?UTF-8?q?=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 - pnpm-lock.yaml | 17 ---- src/pages/home/uploads/slice_upload.ts | 123 ++++++++++++------------- 3 files changed, 58 insertions(+), 83 deletions(-) diff --git a/package.json b/package.json index 88db0ab5..5aa393c9 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,6 @@ "mitt": "^3.0.1", "monaco-editor": "^0.52.2", "mpegts.js": "^1.8.0", - "p-limit": "^7.1.1", "pdfjs-dist": "^5.3.31", "qrcode": "^1.5.4", "rehype-katex": "6.0.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bbd3ad16..002b2ba5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -121,9 +121,6 @@ importers: mpegts.js: specifier: ^1.8.0 version: 1.8.0 - p-limit: - specifier: ^7.1.1 - version: 7.1.1 pdfjs-dist: specifier: ^5.3.31 version: 5.4.54 @@ -2752,10 +2749,6 @@ packages: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} - p-limit@7.1.1: - resolution: {integrity: sha512-i8PyM2JnsNChVSYWLr2BAjNoLi0BAYC+wecOnZnVV+YSNJkzP7cWmvI34dk0WArWfH9KwBHNoZI3P3MppImlIA==} - engines: {node: '>=20'} - p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -3406,10 +3399,6 @@ packages: resolution: {integrity: sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==} engines: {node: '>=12'} - yocto-queue@1.2.1: - resolution: {integrity: sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==} - engines: {node: '>=12.20'} - zustand@5.0.5: resolution: {integrity: sha512-mILtRfKW9xM47hqxGIxCv12gXusoY/xTSHBYApXozR0HmQv299whhBeeAcRy+KrPPybzosvJBCOmVjq6x12fCg==} engines: {node: '>=12.20.0'} @@ -6259,10 +6248,6 @@ snapshots: dependencies: p-try: 2.2.0 - p-limit@7.1.1: - dependencies: - yocto-queue: 1.2.1 - p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -6954,8 +6939,6 @@ snapshots: buffer-crc32: 0.2.13 pend: 1.2.0 - yocto-queue@1.2.1: {} - zustand@5.0.5: {} zwitch@2.0.4: {} diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index 2f7f525d..57ce5ceb 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -14,19 +14,16 @@ import createMutex from "~/utils/mutex" // 重试配置 const RETRY_CONFIG = { - maxRetries: 15, // 增加重试次数以更好应对服务器重启 - retryDelay: 1000, // 基础延迟1秒 - maxDelay: 30000, // 最大延迟30秒 - backoffMultiplier: 2, // 指数退避 - // 服务器重启检测和恢复 - serverHealthCheckDelay: 3000, // 服务器健康检查延迟 - serverRestartRetries: 5, // 服务器重启后的特殊重试次数 - serverRecoveryMaxWait: 180000, // 最大等待服务器恢复时间（3分钟） - // 任务状态同步 - taskSyncRetries: 5, // 任务状态同步重试次数 - taskSyncDelay: 2000, // 任务同步延迟 - // 原生分片上传优化 - nativeSliceRetries: 8, // 原生分片上传额外重试次数 + maxRetries: 15, + retryDelay: 1000, + maxDelay: 30000, + backoffMultiplier: 2, + serverHealthCheckDelay: 3000, + serverRestartRetries: 5, + serverRecoveryMaxWait: 180000, + taskSyncRetries: 5, + taskSyncDelay: 2000, + nativeSliceRetries: 8, } // 服务器状态检测 @@ -159,7 +156,7 @@ class TaskSyncManager { currentSliceStatus: Uint8Array, ) { console.log( - `🔄 Server restart detected, syncing task status: ${currentTaskId}`, + `Server restart detected, syncing task status: ${currentTaskId}`, ) for (let attempt = 0; attempt < RETRY_CONFIG.taskSyncRetries; attempt++) { @@ -191,10 +188,10 @@ class TaskSyncManager { this.countCompletedSlices(currentSliceStatus) console.log( - `✅ Task status sync successful - TaskID: ${currentTaskId}`, + `Task status sync successful - TaskID: ${currentTaskId}`, ) console.log( - `📊 Server completed slices: ${serverCompletedSlices}, local records: ${localCompletedSlices}`, + `Server completed slices: ${serverCompletedSlices}, local records: ${localCompletedSlices}`, ) return { @@ -419,37 +416,24 @@ const retryWithBackoff = async ( } catch (error) { lastError = error as Error - // 如果是最后一次重试，直接抛出错误 if (i === maxRetries) { throw lastError } - // Check if server-related error const isServerError = error instanceof UploadError && (error.type === UploadErrorType.SERVER_ERROR || error.type === UploadErrorType.NETWORK_ERROR) if (isServerError && error instanceof UploadError) { - // Mark server as possibly offline healthChecker.markServerOffline() - - // Check server status const isServerHealthy = await healthChecker.isServerHealthy() if (!isServerHealthy) { - console.log( - `Server appears offline, waiting for recovery... (${context}, retry ${i + 1}/${maxRetries})`, - ) - - // Wait for server recovery with longer wait time + console.log(`Server offline, waiting for recovery... (${context}, retry ${i + 1}/${maxRetries})`) const recovered = await healthChecker.waitForServerRecovery(30000) - if (!recovered) { - // Server recovery failed, but still have retry chances, continue retrying - console.warn( - `Server recovery failed, continue retrying (${context})`, - ) + console.warn(`Server recovery failed, continue retrying (${context})`) } else { console.log(`Server recovered, continue upload (${context})`) } @@ -696,7 +680,7 @@ export const sliceupload = async ( if (isTaskNotFound) { console.log( - `🚨 Task lost detected, starting smart recovery: ${task_id} (slice ${idx + 1})`, + `Task lost detected, starting smart recovery: ${task_id} (slice ${idx + 1})`, ) try { @@ -729,13 +713,13 @@ export const sliceupload = async ( syncResult.serverStatus!.sliceUploadStatus!, ) console.log( - `✅ ${syncResult.message}, continuing upload slice ${idx + 1}`, + `${syncResult.message}, continuing upload slice ${idx + 1}`, ) // Check if current slice is already completed on server if (isSliceUploaded(sliceupstatus, idx)) { console.log( - `✅ Slice ${idx + 1} already completed on server, skipping upload`, + `Slice ${idx + 1} already completed on server, skipping upload`, ) return { code: 200, @@ -744,9 +728,9 @@ export const sliceupload = async ( } // Re-throw error to let retry mechanism continue - console.log(`🔄 Slice ${idx + 1} needs to be re-uploaded`) + console.log(`Slice ${idx + 1} needs to be re-uploaded`) } else { - console.log(`✅ ${syncResult.message}`) + console.log(syncResult.message) } } else { console.warn( @@ -825,44 +809,53 @@ export const sliceupload = async ( } } else { state.uploadedBytes += Math.min(resp1.data.slice_size, state.totalBytes) - } // 后续分片并发上传 - const concurrentLimit = 3 // 固定3个并发 - const { default: pLimit } = await import("p-limit") - const limit = pLimit(concurrentLimit) + } + // 后续分片并发上传 + const concurrentLimit = 3 // 固定3个并发 console.log( `File size: ${(file.size / 1024 / 1024).toFixed(2)}MB, using ${concurrentLimit} concurrent uploads`, ) - const tasks: Promise[] = [] - const errors: Error[] = [] + // 原生并发控制实现 + const pendingSlices: number[] = [] for (let i = 1; i < resp1.data.slice_cnt; i++) { if (!isSliceUploaded(sliceupstatus, i)) { - const chunk = file.slice( - i * resp1.data.slice_size, - (i + 1) * resp1.data.slice_size, - ) - tasks.push( - limit(async () => { - try { - await uploadChunk( - chunk, - i, - slicehash.length == 0 ? "" : slicehash[i], - resp1.data.task_id, - ) - } catch (err) { - errors.push(err as Error) - } - }), - ) - } else { - state.uploadedBytes += Math.min( - resp1.data.slice_size, - state.totalBytes - i * resp1.data.slice_size, - ) + pendingSlices.push(i) + } + } + + const errors: Error[] = [] + let currentIndex = 0 + + // 并发处理函数 + const processNextSlice = async (): Promise => { + while (currentIndex < pendingSlices.length) { + const sliceIndex = pendingSlices[currentIndex++] + + try { + const chunk = file.slice( + sliceIndex * resp1.data.slice_size, + (sliceIndex + 1) * resp1.data.slice_size, + ) + await uploadChunk( + chunk, + sliceIndex, + slicehash.length == 0 ? "" : slicehash[sliceIndex], + resp1.data.task_id, + ) + } catch (err) { + errors.push(err as Error) + } } } + + // 启动并发任务 + const tasks: Promise[] = [] + for (let i = 0; i < Math.min(concurrentLimit, pendingSlices.length); i++) { + tasks.push(processNextSlice()) + } + await Promise.all(tasks) // 最终处理上传结果 From 9e2272a0def7cee97009cf2e1730f71568b5d95c Mon Sep 17 00:00:00 2001 From: j2rong4cn Date: Sat, 6 Sep 2025 14:35:40 +0800 Subject: [PATCH 16/18] =?UTF-8?q?=E6=8A=8A=E5=88=86=E7=89=87=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E7=A7=BB=E5=8A=A8=E5=88=B0=E4=B8=8A=E4=BC=A0=E6=96=B9?= =?UTF-8?q?=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home/uploads/Upload.tsx | 28 +++++++++----------------- src/pages/home/uploads/form.ts | 6 ------ src/pages/home/uploads/slice_upload.ts | 7 +++---- src/pages/home/uploads/stream.ts | 5 ----- src/pages/home/uploads/types.ts | 1 - src/pages/home/uploads/uploads.ts | 6 ++++++ 6 files changed, 19 insertions(+), 34 deletions(-) diff --git a/src/pages/home/uploads/Upload.tsx b/src/pages/home/uploads/Upload.tsx index b690f570..39f0cb05 100644 --- a/src/pages/home/uploads/Upload.tsx +++ b/src/pages/home/uploads/Upload.tsx @@ -78,7 +78,6 @@ const Upload = () => { const [uploading, setUploading] = createSignal(false) const [asTask, setAsTask] = createSignal(false) const [overwrite, setOverwrite] = createSignal(false) - const [sliceup, setSliceup] = createSignal(false) const [rapid, setRapid] = createSignal(true) const [uploadFiles, setUploadFiles] = createStore<{ uploads: UploadFileProps[] @@ -123,7 +122,6 @@ const Upload = () => { asTask(), overwrite(), rapid(), - sliceup(), ) if (!err) { setUpload(path, "status", asTask() ? "tasked" : "success") @@ -307,14 +305,6 @@ const Upload = () => { > {t("home.upload.add_as_task")} - { - setSliceup(!sliceup()) - }} - > - {t("home.upload.slice_upload")} - { > {t("home.conflict_policy.overwrite_existing")} - { - setRapid(!rapid()) - }} - > - {t("home.upload.try_rapid")} - + + { + setRapid(!rapid()) + }} + > + {t("home.upload.try_rapid")} + + diff --git a/src/pages/home/uploads/form.ts b/src/pages/home/uploads/form.ts index ca2d38d8..85d90fe2 100644 --- a/src/pages/home/uploads/form.ts +++ b/src/pages/home/uploads/form.ts @@ -3,7 +3,6 @@ import { EmptyResp } from "~/types" import { r } from "~/utils" import { SetUpload, Upload } from "./types" import { calculateHash } from "./util" -import { sliceupload } from "./slice_upload" export const FormUpload: Upload = async ( uploadPath: string, file: File, @@ -11,12 +10,7 @@ export const FormUpload: Upload = async ( asTask = false, overwrite = false, rapid = false, - sliceup = false, ): Promise => { - if (sliceup) { - return sliceupload(uploadPath, file, setUpload, overwrite, asTask) - } - let oldTimestamp = new Date().valueOf() let oldLoaded = 0 const form = new FormData() diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index 57ce5ceb..99c7fb8c 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -474,13 +474,12 @@ interface UploadState { onProgress?: (progress: UploadProgress) => void } -export const sliceupload = async ( +export const SliceUpload: Upload = async ( uploadPath: string, file: File, setUpload: SetUpload, - overwrite = false, asTask = false, - uploadState?: UploadState, + overwrite = false, ): Promise => { let hashtype: string = HashType.Md5 let slicehash: string[] = [] @@ -496,7 +495,7 @@ export const sliceupload = async ( } | null = null // 初始化上传状态 - const state: UploadState = uploadState || { + const state: UploadState = { isPaused: false, isCancelled: false, totalBytes: file.size, diff --git a/src/pages/home/uploads/stream.ts b/src/pages/home/uploads/stream.ts index d32df985..9369b0ad 100644 --- a/src/pages/home/uploads/stream.ts +++ b/src/pages/home/uploads/stream.ts @@ -3,7 +3,6 @@ import { EmptyResp } from "~/types" import { r } from "~/utils" import { SetUpload, Upload } from "./types" import { calculateHash, HashType } from "./util" -import { sliceupload } from "./slice_upload" export const StreamUpload: Upload = async ( uploadPath: string, file: File, @@ -11,11 +10,7 @@ export const StreamUpload: Upload = async ( asTask = false, overwrite = false, rapid = false, - sliceup = false, ): Promise => { - if (sliceup) { - return sliceupload(uploadPath, file, setUpload, overwrite, asTask) - } let oldTimestamp = new Date().valueOf() let oldLoaded = 0 let headers: { [k: string]: any } = { diff --git a/src/pages/home/uploads/types.ts b/src/pages/home/uploads/types.ts index 74d1f254..fa8534fc 100644 --- a/src/pages/home/uploads/types.ts +++ b/src/pages/home/uploads/types.ts @@ -24,7 +24,6 @@ export type Upload = ( asTask: boolean, overwrite: boolean, rapid: boolean, - sliceup: boolean, ) => Promise export type HashInfo = { diff --git a/src/pages/home/uploads/uploads.ts b/src/pages/home/uploads/uploads.ts index 86b34053..47ad9634 100644 --- a/src/pages/home/uploads/uploads.ts +++ b/src/pages/home/uploads/uploads.ts @@ -2,6 +2,7 @@ import { objStore } from "~/store" import { FormUpload } from "./form" import { StreamUpload } from "./stream" import { Upload } from "./types" +import { SliceUpload } from "./slice_upload" type Uploader = { upload: Upload @@ -20,6 +21,11 @@ const AllUploads: Uploader[] = [ upload: FormUpload, provider: /.*/, }, + { + name: "Slice", + upload: SliceUpload, + provider: /.*/, + }, ] export const getUploads = (): Pick[] => { From 6b77cb0d89edb19e200ce66e6b1642f138d68b71 Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Sat, 6 Sep 2025 15:54:36 +0800 Subject: [PATCH 17/18] =?UTF-8?q?refactor(upload):=20=E7=A7=BB=E9=99=A4?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=99=A8=E5=81=A5=E5=BA=B7=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E5=92=8C=E4=BB=BB=E5=8A=A1=E7=8A=B6=E6=80=81=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E7=AE=80=E5=8C=96=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home/uploads/slice_upload.ts | 347 +------------------------ 1 file changed, 6 insertions(+), 341 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index 99c7fb8c..6cc2f46e 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -18,246 +18,9 @@ const RETRY_CONFIG = { retryDelay: 1000, maxDelay: 30000, backoffMultiplier: 2, - serverHealthCheckDelay: 3000, - serverRestartRetries: 5, - serverRecoveryMaxWait: 180000, - taskSyncRetries: 5, - taskSyncDelay: 2000, nativeSliceRetries: 8, } -// 服务器状态检测 -class ServerHealthChecker { - private static instance: ServerHealthChecker - private lastHealthCheck = 0 - private serverOnline = true - private checkPromise: Promise | null = null - - static getInstance(): ServerHealthChecker { - if (!ServerHealthChecker.instance) { - ServerHealthChecker.instance = new ServerHealthChecker() - } - return ServerHealthChecker.instance - } - - async isServerHealthy(): Promise { - const now = Date.now() - - // 如果最近检查过且结果为在线，直接返回 - if (this.serverOnline && now - this.lastHealthCheck < 10000) { - return true - } - - // 防止并发检查 - if (this.checkPromise) { - return this.checkPromise - } - - this.checkPromise = this.performHealthCheck() - try { - const result = await this.checkPromise - this.lastHealthCheck = now - this.serverOnline = result - return result - } finally { - this.checkPromise = null - } - } - - private async performHealthCheck(): Promise { - try { - const response = await r.get("/ping", { - timeout: 5000, - headers: { Password: password() }, - }) - return response.status === 200 - } catch (error: any) { - console.warn("Server health check failed:", error.message) - return false - } - } - - markServerOffline(): void { - this.serverOnline = false - } - - async waitForServerRecovery(maxWaitTime = 60000): Promise { - const startTime = Date.now() - let attempt = 1 - - console.log("等待服务器恢复...") - - while (Date.now() - startTime < maxWaitTime) { - const isHealthy = await this.isServerHealthy() - if (isHealthy) { - console.log(`服务器已恢复 (第${attempt}次检查)`) - return true - } - - const waitTime = Math.min(5000 * attempt, 15000) // 渐进式等待 - console.log(`服务器检查失败，${waitTime / 1000}秒后重试...`) - await new Promise((resolve) => setTimeout(resolve, waitTime)) - attempt++ - } - - console.error("Server recovery timeout") - return false - } -} - -// 任务状态同步器 -class TaskSyncManager { - private static async syncTaskStatus( - dir: string, - fileName: string, - fileSize: number, - hash: any, - overwrite: boolean, - asTask: boolean, - expectedTaskId?: string, - ) { - try { - const resp = await fsPreup( - dir, - fileName, - fileSize, - hash, - overwrite, - asTask, - ) - if (resp.code === 200) { - return { - success: true, - taskId: resp.data.task_id, - sliceSize: resp.data.slice_size, - sliceCnt: resp.data.slice_cnt, - sliceUploadStatus: resp.data.slice_upload_status, - isExpectedTask: - !expectedTaskId || resp.data.task_id === expectedTaskId, - } - } - return { - success: false, - error: `Sync failed: ${resp.code} - ${resp.message}`, - } - } catch (error) { - return { success: false, error: `Sync error: ${error}` } - } - } - - static async handleServerRecovery( - dir: string, - fileName: string, - fileSize: number, - hash: any, - overwrite: boolean, - asTask: boolean, - currentTaskId: string, - currentSliceStatus: Uint8Array, - ) { - console.log( - `Server restart detected, syncing task status: ${currentTaskId}`, - ) - - for (let attempt = 0; attempt < RETRY_CONFIG.taskSyncRetries; attempt++) { - try { - const syncResult = await this.syncTaskStatus( - dir, - fileName, - fileSize, - hash, - overwrite, - asTask, - currentTaskId, - ) - - if (syncResult.success) { - const serverSliceStatus = base64ToUint8Array( - syncResult.sliceUploadStatus!, - ) - - if (syncResult.isExpectedTask) { - // Server task ID matches, compare status - const statusMatches = this.compareSliceStatus( - currentSliceStatus, - serverSliceStatus, - ) - const serverCompletedSlices = - this.countCompletedSlices(serverSliceStatus) - const localCompletedSlices = - this.countCompletedSlices(currentSliceStatus) - - console.log( - `Task status sync successful - TaskID: ${currentTaskId}`, - ) - console.log( - `Server completed slices: ${serverCompletedSlices}, local records: ${localCompletedSlices}`, - ) - - return { - success: true, - needResync: !statusMatches, - serverStatus: syncResult, - message: `Task recovery successful, server has completed ${serverCompletedSlices} slices`, - } - } else { - // Server returned different task ID, need to restart - console.log( - `⚠️ Server returned new task ID: ${syncResult.taskId}, original task invalid: ${currentTaskId}`, - ) - return { - success: true, - needRestart: true, - serverStatus: syncResult, - message: "Server task has changed, need to restart upload", - } - } - } - } catch (error) { - console.warn(`🔄 Task sync attempt ${attempt + 1} failed:`, error) - } - - if (attempt < RETRY_CONFIG.taskSyncRetries - 1) { - const waitTime = RETRY_CONFIG.taskSyncDelay * (attempt + 1) - console.log(`⏳ Retrying task sync in ${waitTime / 1000} seconds...`) - await new Promise((resolve) => setTimeout(resolve, waitTime)) - } - } - - return { - success: false, - error: "Task sync failed after all retries", - message: "Task status sync failed, please restart upload", - } - } - - private static countCompletedSlices(sliceStatus: Uint8Array): number { - let count = 0 - for (let i = 0; i < sliceStatus.length * 8; i++) { - const byteIndex = Math.floor(i / 8) - const bitIndex = i % 8 - if ( - byteIndex < sliceStatus.length && - (sliceStatus[byteIndex] & (1 << bitIndex)) !== 0 - ) { - count++ - } - } - return count - } - - private static compareSliceStatus( - local: Uint8Array, - server: Uint8Array, - ): boolean { - if (local.length !== server.length) return false - for (let i = 0; i < local.length; i++) { - if (local[i] !== server[i]) return false - } - return true - } -} - // 错误类型定义 enum UploadErrorType { NETWORK_ERROR = "network_error", @@ -400,14 +163,13 @@ interface UploadProgress { const progressMutex = createMutex() -// 智能重试函数，支持服务器重启检测 +// 标准重试函数 const retryWithBackoff = async ( fn: () => Promise, maxRetries: number = RETRY_CONFIG.maxRetries, delay: number = RETRY_CONFIG.retryDelay, context: string = "operation", ): Promise => { - const healthChecker = ServerHealthChecker.getInstance() let lastError: Error for (let i = 0; i <= maxRetries; i++) { @@ -420,32 +182,11 @@ const retryWithBackoff = async ( throw lastError } - const isServerError = - error instanceof UploadError && - (error.type === UploadErrorType.SERVER_ERROR || - error.type === UploadErrorType.NETWORK_ERROR) - - if (isServerError && error instanceof UploadError) { - healthChecker.markServerOffline() - const isServerHealthy = await healthChecker.isServerHealthy() - - if (!isServerHealthy) { - console.log(`Server offline, waiting for recovery... (${context}, retry ${i + 1}/${maxRetries})`) - const recovered = await healthChecker.waitForServerRecovery(30000) - if (!recovered) { - console.warn(`Server recovery failed, continue retrying (${context})`) - } else { - console.log(`Server recovered, continue upload (${context})`) - } - } - } - - // Calculate delay time, use longer delay for server errors - let waitTime = delay * Math.pow(RETRY_CONFIG.backoffMultiplier, i) - if (isServerError) { - waitTime = Math.max(waitTime, RETRY_CONFIG.serverHealthCheckDelay) - } - waitTime = Math.min(waitTime, RETRY_CONFIG.maxDelay) + // Calculate delay time with exponential backoff + const waitTime = Math.min( + delay * Math.pow(RETRY_CONFIG.backoffMultiplier, i), + RETRY_CONFIG.maxDelay, + ) console.log( `${context} failed, retrying in ${waitTime / 1000} seconds (${i + 1}/${maxRetries}):`, @@ -669,79 +410,6 @@ export const SliceUpload: Upload = async ( } return resp } catch (err: any) { - // 🔍 Smart error detection: server restart / task lost - if (err?.response?.status === 400 && taskInfo) { - const errorMsg = err?.response?.data?.message || err.message || "" - const isTaskNotFound = - errorMsg.includes("task") || - errorMsg.includes("TaskID") || - errorMsg.includes("failed get slice upload") - - if (isTaskNotFound) { - console.log( - `Task lost detected, starting smart recovery: ${task_id} (slice ${idx + 1})`, - ) - - try { - const syncResult = await TaskSyncManager.handleServerRecovery( - dir, - file.name, - file.size, - taskInfo.hash, - overwrite, - asTask, - task_id, - sliceupstatus, - ) - - if (syncResult.success) { - if (syncResult.needRestart) { - // Task needs to restart - console.log(`❌ ${syncResult.message}`) - throw new UploadError( - UploadErrorType.SERVER_ERROR, - "Task ID changed, need restart", - syncResult.message || - "Server task status changed, need to restart upload", - undefined, - false, // Not retryable, need restart - ) - } else if (syncResult.needResync) { - // Status synced, update local status and continue retry - sliceupstatus = base64ToUint8Array( - syncResult.serverStatus!.sliceUploadStatus!, - ) - console.log( - `${syncResult.message}, continuing upload slice ${idx + 1}`, - ) - - // Check if current slice is already completed on server - if (isSliceUploaded(sliceupstatus, idx)) { - console.log( - `Slice ${idx + 1} already completed on server, skipping upload`, - ) - return { - code: 200, - message: "Slice already uploaded on server", - } as EmptyResp - } - - // Re-throw error to let retry mechanism continue - console.log(`Slice ${idx + 1} needs to be re-uploaded`) - } else { - console.log(syncResult.message) - } - } else { - console.warn( - `❌ Task status sync failed: ${syncResult.error}`, - ) - } - } catch (syncError) { - console.warn("🔧 Error during task status sync:", syncError) - } - } - } - // Convert to structured error const uploadError = err instanceof UploadError @@ -1015,9 +683,6 @@ export const cancelUpload = (uploadPath: string) => export { UploadError, UploadErrorType } export type { UploadProgress } -// 导出服务器健康检查器 -export const serverHealthChecker = ServerHealthChecker.getInstance() - // 获取上传详细信息的辅助函数 export const getUploadDetails = ( uploadPath: string, From 7b7e6b9ab8a6c91b9c0bec99e080ce6a8a23eeb9 Mon Sep 17 00:00:00 2001 From: Suyunmeng Date: Sat, 6 Sep 2025 17:17:02 +0800 Subject: [PATCH 18/18] =?UTF-8?q?refactor(upload):=20=E7=A7=BB=E9=99=A4?= =?UTF-8?q?=E5=86=97=E4=BD=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home/uploads/slice_upload.ts | 39 +------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/src/pages/home/uploads/slice_upload.ts b/src/pages/home/uploads/slice_upload.ts index 6cc2f46e..92113d2e 100644 --- a/src/pages/home/uploads/slice_upload.ts +++ b/src/pages/home/uploads/slice_upload.ts @@ -12,7 +12,6 @@ import { } from "./util" import createMutex from "~/utils/mutex" -// 重试配置 const RETRY_CONFIG = { maxRetries: 15, retryDelay: 1000, @@ -21,7 +20,6 @@ const RETRY_CONFIG = { nativeSliceRetries: 8, } -// 错误类型定义 enum UploadErrorType { NETWORK_ERROR = "network_error", SERVER_ERROR = "server_error", @@ -141,7 +139,6 @@ class UploadError extends Error { } } -// 进度详情接口 interface UploadProgress { uploadedBytes: number totalBytes: number @@ -163,7 +160,6 @@ interface UploadProgress { const progressMutex = createMutex() -// 标准重试函数 const retryWithBackoff = async ( fn: () => Promise, maxRetries: number = RETRY_CONFIG.maxRetries, @@ -227,7 +223,6 @@ export const SliceUpload: Upload = async ( let sliceupstatus: Uint8Array let ht: string[] = [] - // 任务信息，用于状态同步 let taskInfo: { taskId: string hash: any @@ -247,10 +242,8 @@ export const SliceUpload: Upload = async ( speed: 0, } - // 注册到上传队列 uploadQueue.addUpload(uploadPath, state) - // 清理函数 let speedInterval: any const cleanup = () => { if (speedInterval) { @@ -261,7 +254,6 @@ export const SliceUpload: Upload = async ( const dir = pathDir(uploadPath) - //获取上传需要的信息 const resp = await fsUploadInfo(dir) if (resp.code != 200) { cleanup() @@ -281,7 +273,6 @@ export const SliceUpload: Upload = async ( ht.push(HashType.Md5256kb) } const hash = await calculateHash(file, ht) - // 预上传 const resp1 = await fsPreup( dir, file.name, @@ -295,10 +286,8 @@ export const SliceUpload: Upload = async ( return new Error(`Preup failed: ${resp1.code} - ${resp1.message}`) } - // 设置总分片数 state.totalChunks = resp1.data.slice_cnt - // 保存任务信息用于状态同步 taskInfo = { taskId: resp1.data.task_id, hash, @@ -313,19 +302,15 @@ export const SliceUpload: Upload = async ( cleanup() return } - //计算分片hash if (resp.data.slice_hash_need) { slicehash = await calculateSliceHash(file, resp1.data.slice_size, hashtype) } - // 分片上传状态 sliceupstatus = base64ToUint8Array(resp1.data.slice_upload_status) - // 进度和速度统计 let lastTimestamp = Date.now() let lastUploadedBytes = 0 let completeFlag = false - // 计算已上传的字节数（用于断点续传） for (let i = 0; i < resp1.data.slice_cnt; i++) { if (isSliceUploaded(sliceupstatus, i)) { state.uploadedBytes += Math.min( @@ -335,14 +320,12 @@ export const SliceUpload: Upload = async ( } } - // 上传分片的核心函数，带进度、速度统计、重试和暂停支持 const uploadChunk = async ( chunk: Blob, idx: number, slice_hash: string, task_id: string, ) => { - // 检查是否被取消 if (state.isCancelled) { throw new UploadError( UploadErrorType.CANCEL_ERROR, @@ -353,7 +336,6 @@ export const SliceUpload: Upload = async ( ) } - // 检查是否暂停，等待恢复 while (state.isPaused && !state.isCancelled) { await new Promise((resolve) => setTimeout(resolve, 100)) } @@ -375,19 +357,16 @@ export const SliceUpload: Upload = async ( if (!progressEvent.lengthComputable || state.isCancelled) { return } - //获取锁 const release = await progressMutex.acquire() try { const sliceuploaded = progressEvent.loaded - oldLoaded state.uploadedBytes += sliceuploaded oldLoaded = progressEvent.loaded - // 更新完成的分片数（估算） state.completedChunks = Math.floor( state.uploadedBytes / (state.totalBytes / state.totalChunks), ) - // 实时进度更新 const progress = Math.min( 100, ((state.uploadedBytes / state.totalBytes) * 100) | 0, @@ -420,7 +399,7 @@ export const SliceUpload: Upload = async ( state.lastError = uploadError console.error( - `💥 Slice ${idx + 1} upload failed:`, + `Slice ${idx + 1} upload failed:`, uploadError.userMessage, ) throw uploadError @@ -432,7 +411,6 @@ export const SliceUpload: Upload = async ( ) } - // 进度速度计算 speedInterval = setInterval(() => { if (completeFlag || state.isCancelled) { clearInterval(speedInterval) @@ -441,7 +419,6 @@ export const SliceUpload: Upload = async ( const intervalLoaded = state.uploadedBytes - lastUploadedBytes if (intervalLoaded < 1000) { - //进度太小，不更新 return } const speed = intervalLoaded / ((Date.now() - lastTimestamp) / 1000) @@ -455,10 +432,8 @@ export const SliceUpload: Upload = async ( lastUploadedBytes = state.uploadedBytes }, 1000) - // 开始计时 lastTimestamp = Date.now() - // 先上传第一个分片，slicehash全部用逗号拼接传递 if (!isSliceUploaded(sliceupstatus, 0)) { const chunk = file.slice(0, resp1.data.slice_size) try { @@ -478,13 +453,11 @@ export const SliceUpload: Upload = async ( state.uploadedBytes += Math.min(resp1.data.slice_size, state.totalBytes) } - // 后续分片并发上传 const concurrentLimit = 3 // 固定3个并发 console.log( `File size: ${(file.size / 1024 / 1024).toFixed(2)}MB, using ${concurrentLimit} concurrent uploads`, ) - // 原生并发控制实现 const pendingSlices: number[] = [] for (let i = 1; i < resp1.data.slice_cnt; i++) { if (!isSliceUploaded(sliceupstatus, i)) { @@ -495,7 +468,6 @@ export const SliceUpload: Upload = async ( const errors: Error[] = [] let currentIndex = 0 - // 并发处理函数 const processNextSlice = async (): Promise => { while (currentIndex < pendingSlices.length) { const sliceIndex = pendingSlices[currentIndex++] @@ -517,7 +489,6 @@ export const SliceUpload: Upload = async ( } } - // 启动并发任务 const tasks: Promise[] = [] for (let i = 0; i < Math.min(concurrentLimit, pendingSlices.length); i++) { tasks.push(processNextSlice()) @@ -525,7 +496,6 @@ export const SliceUpload: Upload = async ( await Promise.all(tasks) - // 最终处理上传结果 if (errors.length > 0) { setUpload( "progress", @@ -534,7 +504,6 @@ export const SliceUpload: Upload = async ( setUpload("status", "error") cleanup() - // 返回最具代表性的错误 const serverErrors = errors.filter( (e) => e instanceof UploadError && e.type === UploadErrorType.SERVER_ERROR, @@ -585,7 +554,6 @@ export const SliceUpload: Upload = async ( ) } - //状态处理交给上层 return } catch (error) { cleanup() @@ -596,7 +564,6 @@ export const SliceUpload: Upload = async ( } } -// 解码 base64 字符串为 Uint8Array const base64ToUint8Array = (base64: string): Uint8Array => { const binary = atob(base64) const len = binary.length @@ -607,7 +574,6 @@ const base64ToUint8Array = (base64: string): Uint8Array => { return bytes } -// 判断第 idx 个分片是否已上传 const isSliceUploaded = (status: Uint8Array, idx: number): boolean => { // const bytes = base64ToUint8Array(statusBase64) const byteIdx = Math.floor(idx / 8) @@ -669,7 +635,6 @@ class UploadQueue { } } -// 导出队列管理函数 export const uploadQueue = UploadQueue.getInstance() export const pauseUpload = (uploadPath: string) => @@ -679,11 +644,9 @@ export const resumeUpload = (uploadPath: string) => export const cancelUpload = (uploadPath: string) => uploadQueue.cancelUpload(uploadPath) -// 导出错误类型和辅助函数 export { UploadError, UploadErrorType } export type { UploadProgress } -// 获取上传详细信息的辅助函数 export const getUploadDetails = ( uploadPath: string, ): {