diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7b76e62394..0bf864ce11d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,23 @@ jobs: name: Release runs-on: ${{ matrix.platform }} steps: + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Prerelease uses: irongut/EditRelease@v1.2.0 with: diff --git a/drivers/115/driver.go b/drivers/115/driver.go index f6fb6b05618..4f584cd7b51 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -79,28 +79,60 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return link, nil } -func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { +func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err } - if _, err := d.client.Mkdir(parentDir.GetID(), dirName); err != nil { - return err + + result := driver115.MkdirResp{} + form := map[string]string{ + "pid": parentDir.GetID(), + "cname": dirName, } - return nil + req := d.client.NewRequest(). + SetFormData(form). + SetResult(&result). + ForceContentType("application/json;charset=UTF-8") + + resp, err := req.Post(driver115.ApiDirAdd) + + err = driver115.CheckErr(err, &result, resp) + if err != nil { + return nil, err + } + f, err := d.getNewFile(result.FileID) + if err != nil { + return nil, nil + } + return f, nil } -func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err } - return d.client.Move(dstDir.GetID(), srcObj.GetID()) + if err := d.client.Move(dstDir.GetID(), srcObj.GetID()); err != nil { + return nil, err + } + f, err := d.getNewFile(srcObj.GetID()) + if err != nil { + return nil, nil + } + return f, nil } -func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) error { +func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err + } + if err := d.client.Rename(srcObj.GetID(), newName); err != nil { + return nil, err } - return d.client.Rename(srcObj.GetID(), newName) + f, err := d.getNewFile((srcObj.GetID())) + if err != nil { + return nil, nil + } + return f, nil } func (d *Pan115) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -117,9 +149,9 @@ func (d *Pan115) Remove(ctx context.Context, obj model.Obj) error { return d.client.Delete(obj.GetID()) } -func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err } var ( @@ -128,10 +160,10 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr ) if ok, err := d.client.UploadAvailable(); err != nil || !ok { - return err + return nil, err } if stream.GetSize() > d.client.UploadMetaInfo.SizeLimit { - return driver115.ErrUploadTooLarge + return nil, driver115.ErrUploadTooLarge } //if digest, err = d.client.GetDigestResult(stream); err != nil { // return err @@ -144,22 +176,22 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: hashSize}) if err != nil { - return err + return nil, err } preHash, err := utils.HashReader(utils.SHA1, reader) if err != nil { - return err + return nil, err } preHash = strings.ToUpper(preHash) fullHash := stream.GetHash().GetHash(utils.SHA1) if len(fullHash) <= 0 { tmpF, err := stream.CacheFullInTempFile() if err != nil { - return err + return nil, err } fullHash, err = utils.HashFile(utils.SHA1, tmpF) if err != nil { - return err + return nil, err } } fullHash = strings.ToUpper(fullHash) @@ -168,20 +200,36 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr // note that 115 add timeout for rapid-upload, // and "sig invalid" err is thrown even when the hash is correct after timeout. if fastInfo, err = d.rapidUpload(stream.GetSize(), stream.GetName(), dirID, preHash, fullHash, stream); err != nil { - return err + return nil, err } if matched, err := fastInfo.Ok(); err != nil { - return err + return nil, err } else if matched { - return nil + f, err := d.getNewFileByPickCode(fastInfo.PickCode) + if err != nil { + return nil, nil + } + return f, nil } + var uploadResult *UploadResult // 闪传失败,上传 - if stream.GetSize() <= utils.KB { // 文件大小小于1KB,改用普通模式上传 - return d.client.UploadByOSS(&fastInfo.UploadOSSParams, stream, dirID) + if stream.GetSize() <= 10*utils.MB { // 文件大小小于10MB,改用普通模式上传 + if uploadResult, err = d.UploadByOSS(&fastInfo.UploadOSSParams, stream, dirID); err != nil { + return nil, err + } + } else { + // 分片上传 + if uploadResult, err = d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID); err != nil { + return nil, err + } + } + + file, err := d.getNewFile(uploadResult.Data.FileID) + if err != nil { + return nil, nil } - // 分片上传 - return d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID) + return file, nil } func (d *Pan115) OfflineList(ctx context.Context) ([]*driver115.OfflineTask, error) { diff --git a/drivers/115/meta.go b/drivers/115/meta.go index d9526775229..3b192291a43 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -10,7 +10,7 @@ type Addition struct { QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` - LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` + LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate ([limit]r/1s)"` driver.RootID } diff --git a/drivers/115/util.go b/drivers/115/util.go index 381ef0bd185..33e345706d2 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -74,6 +74,34 @@ func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { return res, nil } +func (d *Pan115) getNewFile(fileId string) (*FileObj, error) { + file, err := d.client.GetFile(fileId) + if err != nil { + return nil, err + } + return &FileObj{*file}, nil +} + +func (d *Pan115) getNewFileByPickCode(pickCode string) (*FileObj, error) { + result := driver115.GetFileInfoResponse{} + req := d.client.NewRequest(). + SetQueryParam("pick_code", pickCode). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + resp, err := req.Get(driver115.ApiFileInfo) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + if len(result.Files) == 0 { + return nil, errors.New("not get file info") + } + fileInfo := result.Files[0] + + f := &FileObj{} + f.From(fileInfo) + return f, nil +} + func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } @@ -244,8 +272,38 @@ func UploadDigestRange(stream model.FileStreamer, rangeSpec string) (result stri return } +// UploadByOSS use aliyun sdk to upload +func (c *Pan115) UploadByOSS(params *driver115.UploadOSSParams, r io.Reader, dirID string) (*UploadResult, error) { + ossToken, err := c.client.GetOSSToken() + if err != nil { + return nil, err + } + ossClient, err := oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret) + if err != nil { + return nil, err + } + bucket, err := ossClient.Bucket(params.Bucket) + if err != nil { + return nil, err + } + + var bodyBytes []byte + if err = bucket.PutObject(params.Object, r, append( + driver115.OssOption(params, ossToken), + oss.CallbackResult(&bodyBytes), + )...); err != nil { + return nil, err + } + + var uploadResult UploadResult + if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { + return nil, err + } + return &uploadResult, uploadResult.Err(string(bodyBytes)) +} + // UploadByMultipart upload by mutipart blocks -func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize int64, stream model.FileStreamer, dirID string, opts ...driver115.UploadMultipartOption) error { +func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize int64, stream model.FileStreamer, dirID string, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart @@ -259,7 +317,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i tmpF, err := stream.CacheFullInTempFile() if err != nil { - return err + return nil, err } options := driver115.DefalutUploadMultipartOptions() @@ -272,15 +330,15 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i options.ThreadsNum = 1 if ossToken, err = d.client.GetOSSToken(); err != nil { - return err + return nil, err } if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { - return err + return nil, err } if bucket, err = ossClient.Bucket(params.Bucket); err != nil { - return err + return nil, err } // ossToken一小时后就会失效,所以每50分钟重新获取一次 @@ -290,7 +348,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i timeout := time.NewTimer(options.Timeout) if chunks, err = SplitFile(fileSize); err != nil { - return err + return nil, err } if imur, err = bucket.InitiateMultipartUpload(params.Object, @@ -298,7 +356,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i oss.UserAgentHeader(driver115.OSSUserAgent), oss.EnableSha1(), oss.Sequential(), ); err != nil { - return err + return nil, err } wg := sync.WaitGroup{} @@ -364,14 +422,14 @@ LOOP: case <-ticker.C: // 到时重新获取ossToken if ossToken, err = d.client.GetOSSToken(); err != nil { - return err + return nil, err } case <-quit: break LOOP case <-errCh: - return err + return nil, err case <-timeout.C: - return fmt.Errorf("time out") + return nil, fmt.Errorf("time out") } } @@ -381,14 +439,14 @@ LOOP: driver115.OssOption(params, ossToken), oss.CallbackResult(&bodyBytes), )...); err != nil { - return err + return nil, err } var uploadResult UploadResult if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { - return err + return nil, err } - return uploadResult.Err(string(bodyBytes)) + return &uploadResult, uploadResult.Err(string(bodyBytes)) } func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 232fc00917e..11495af27dc 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,10 +6,12 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "github.com/google/uuid" "golang.org/x/time/rate" "io" "net/http" "net/url" + "strings" "sync" "time" @@ -56,10 +58,14 @@ func (d *Pan123) Init(ctx context.Context) error { d.params.AppVersion = TVAndroidAppVer } + if d.Addition.LoginUuid == "" { + d.Addition.LoginUuid = strings.ReplaceAll(uuid.New().String(), "-", "") + } + d.params.OsVersion = d.OsVersion - d.params.LoginUuid = d.LoginUuid d.params.DeviceName = d.DeviceName d.params.DeviceType = d.DeiveType + d.params.LoginUuid = d.Addition.LoginUuid _, err := d.request(UserInfo, http.MethodGet, nil, nil) return err @@ -102,7 +108,7 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) "type": f.Type, } resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) { - + req.SetBody(data).SetHeaders(headers) }, nil) if err != nil { diff --git a/drivers/123/meta.go b/drivers/123/meta.go index 662d6ee0067..6767d70f462 100644 --- a/drivers/123/meta.go +++ b/drivers/123/meta.go @@ -18,7 +18,7 @@ type Addition struct { DeviceName string `json:"devicename" default:"XiaoMi"` DeiveType string `json:"devicetype" default:"houji"` OsVersion string `json:"osversion" default:"14"` - LoginUuid string `json:"loginuuid" default:"1fce20b2428d30899fd537f4cf231dfb"` + LoginUuid string `json:"loginuuid" default:""` } var config = driver.Config{ diff --git a/drivers/123/util.go b/drivers/123/util.go index 619f0ea0df4..c1eda67f153 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -53,11 +53,11 @@ const ( ) const ( - AndroidUserAgentPrefix = "123pan/v2.4.7" // 123pan/v2.4.7(Android_14;XiaoMi) + AndroidUserAgentPrefix = "123pan/v2.4.8" // 123pan/v2.4.8(Android_14;XiaoMi) AndroidPlatformParam = "android" - AndroidAppVer = "69" - AndroidXAppVer = "2.4.7" - AndroidXChannel = "1002" + AndroidAppVer = "70" + AndroidXAppVer = "2.4.8" + AndroidXChannel = "1001" TVUserAgentPrefix = "123pan_android_tv/1.0.0" // 123pan_android_tv/1.0.0(14;samsung SM-X800) TVPlatformParam = "android_tv" TVAndroidAppVer = "100" diff --git a/drivers/123_share/driver.go b/drivers/123_share/driver.go index 66624a8f223..cecf024a47f 100644 --- a/drivers/123_share/driver.go +++ b/drivers/123_share/driver.go @@ -4,9 +4,11 @@ import ( "context" "encoding/base64" "fmt" + "github.com/google/uuid" "golang.org/x/time/rate" "net/http" "net/url" + "strings" "sync" "time" @@ -50,10 +52,14 @@ func (d *Pan123Share) Init(ctx context.Context) error { d.params.AppVersion = TVAndroidAppVer } + if d.Addition.LoginUuid == "" { + d.Addition.LoginUuid = strings.ReplaceAll(uuid.New().String(), "-", "") + } + d.params.OsVersion = d.OsVersion - d.params.LoginUuid = d.LoginUuid d.params.DeviceName = d.DeviceName d.params.DeviceType = d.DeiveType + d.params.LoginUuid = d.LoginUuid _, err := d.request(UserInfo, http.MethodGet, nil, nil) return err diff --git a/drivers/123_share/meta.go b/drivers/123_share/meta.go index c1cee306cb8..a929ee43ca0 100644 --- a/drivers/123_share/meta.go +++ b/drivers/123_share/meta.go @@ -20,7 +20,7 @@ type Addition struct { DeviceName string `json:"devicename" default:"XiaoMi"` DeiveType string `json:"devicetype" default:"houji"` OsVersion string `json:"osversion" default:"14"` - LoginUuid string `json:"loginuuid" default:"1fce20b2428d30899fd537f4cf231dfb"` + LoginUuid string `json:"loginuuid" default:""` } var config = driver.Config{ diff --git a/drivers/123_share/util.go b/drivers/123_share/util.go index 87a6a89d846..aa835f3d159 100644 --- a/drivers/123_share/util.go +++ b/drivers/123_share/util.go @@ -39,11 +39,11 @@ const ( ) const ( - AndroidUserAgentPrefix = "123pan/v2.4.7" // 123pan/v2.4.7(Android_14;XiaoMi) + AndroidUserAgentPrefix = "123pan/v2.4.8" // 123pan/v2.4.8(Android_14;XiaoMi) AndroidPlatformParam = "android" - AndroidAppVer = "69" - AndroidXAppVer = "2.4.7" - AndroidXChannel = "1002" + AndroidAppVer = "70" + AndroidXAppVer = "2.4.8" + AndroidXChannel = "1001" TVUserAgentPrefix = "123pan_android_tv/1.0.0" // 123pan_android_tv/1.0.0(14;samsung SM-X800) TVPlatformParam = "android_tv" TVAndroidAppVer = "100" diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index 31115417a69..d6b1507b61f 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -6,7 +6,7 @@ import ( ) type Addition struct { - DriveType string `json:"drive_type" type:"select" options:"default,resource,backup" default:"default"` + DriveType string `json:"drive_type" type:"select" options:"default,resource,backup" default:"resource"` driver.RootID RefreshToken string `json:"refresh_token"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index cbec0bcfcd6..6f3bf13b3e4 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -6,6 +6,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" ) type TokenErrResp struct { @@ -72,7 +73,7 @@ func fileToObj(f File) *model.ObjThumb { IsFolder: f.Isdir == 1, // 直接获取的MD5是错误的 - // HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), + HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index ac1f06e807e..ca1a6805a04 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -1,11 +1,14 @@ package baidu_netdisk import ( + "encoding/hex" "errors" "fmt" "net/http" "strconv" + "strings" "time" + "unicode" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/errs" @@ -153,8 +156,6 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model u = res.Header().Get("location") //} - updateObjMd5(file, "pan.baidu.com", u) - return &model.Link{ URL: u, Header: http.Header{ @@ -178,8 +179,6 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li return nil, err } - updateObjMd5(file, d.CustomCrackUA, resp.Info[0].Dlink) - return &model.Link{ URL: resp.Info[0].Dlink, Header: http.Header{ @@ -229,19 +228,6 @@ func joinTime(form map[string]string, ctime, mtime int64) { form["local_ctime"] = strconv.FormatInt(ctime, 10) } -func updateObjMd5(obj model.Obj, userAgent, u string) { - object := model.GetRawObject(obj) - if object != nil { - req, _ := http.NewRequest(http.MethodHead, u, nil) - req.Header.Add("User-Agent", userAgent) - resp, _ := base.HttpClient.Do(req) - if resp != nil { - contentMd5 := resp.Header.Get("Content-Md5") - object.HashInfo = utils.NewHashInfo(utils.MD5, contentMd5) - } - } -} - const ( DefaultSliceSize int64 = 4 * utils.MB VipSliceSize = 16 * utils.MB @@ -267,3 +253,40 @@ func (d *BaiduNetdisk) getSliceSize() int64 { // r = strings.ReplaceAll(r, "+", "%20") // return r // } + +func DecryptMd5(encryptMd5 string) string { + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} + +func EncryptMd5(originalMd5 string) string { + reversed := originalMd5[8:16] + originalMd5[:8] + originalMd5[24:32] + originalMd5[16:24] + + var out strings.Builder + out.Grow(len(reversed)) + for i, n := 0, int64(0); i < len(reversed); i++ { + n, _ = strconv.ParseInt(reversed[i:i+1], 16, 64) + n ^= int64(15 & i) + if i == 9 { + out.WriteRune(rune(n) + 'g') + } else { + out.WriteString(strconv.FormatInt(n, 16)) + } + } + return out.String() +} diff --git a/drivers/baidu_photo/types.go b/drivers/baidu_photo/types.go index 2bbacd303f7..0e5cbb2cdd5 100644 --- a/drivers/baidu_photo/types.go +++ b/drivers/baidu_photo/types.go @@ -72,7 +72,7 @@ func (c *File) Thumb() string { } func (c *File) GetHash() utils.HashInfo { - return utils.NewHashInfo(utils.MD5, c.Md5) + return utils.NewHashInfo(utils.MD5, DecryptMd5(c.Md5)) } /*相册部分*/ diff --git a/drivers/baidu_photo/utils.go b/drivers/baidu_photo/utils.go index be0ed1336e6..c8c5b7ee88b 100644 --- a/drivers/baidu_photo/utils.go +++ b/drivers/baidu_photo/utils.go @@ -2,8 +2,12 @@ package baiduphoto import ( "context" + "encoding/hex" "fmt" "net/http" + "strconv" + "strings" + "unicode" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/errs" @@ -476,3 +480,40 @@ func (d *BaiduPhoto) uInfo() (*UInfo, error) { } return &info, nil } + +func DecryptMd5(encryptMd5 string) string { + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} + +func EncryptMd5(originalMd5 string) string { + reversed := originalMd5[8:16] + originalMd5[:8] + originalMd5[24:32] + originalMd5[16:24] + + var out strings.Builder + out.Grow(len(reversed)) + for i, n := 0, int64(0); i < len(reversed); i++ { + n, _ = strconv.ParseInt(reversed[i:i+1], 16, 64) + n ^= int64(15 & i) + if i == 9 { + out.WriteRune(rune(n) + 'g') + } else { + out.WriteString(strconv.FormatInt(n, 16)) + } + } + return out.String() +} diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index de122c36c4d..360c6e3d01d 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -67,7 +67,9 @@ func (d *ChaoXing) Init(ctx context.Context) error { } func (d *ChaoXing) Drop(ctx context.Context) error { - d.cron.Stop() + if d.cron != nil { + d.cron.Stop() + } return nil } diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 4208bb8765a..24de24d4f59 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -91,8 +91,8 @@ func (d *PikPak) Init(ctx context.Context) (err error) { ClientID: d.ClientID, ClientSecret: d.ClientSecret, Endpoint: oauth2.Endpoint{ - AuthURL: "https://user.mypikpak.com/v1/auth/signin", - TokenURL: "https://user.mypikpak.com/v1/auth/token", + AuthURL: "https://user.mypikpak.net/v1/auth/signin", + TokenURL: "https://user.mypikpak.net/v1/auth/token", AuthStyle: oauth2.AuthStyleInParams, }, } @@ -124,7 +124,7 @@ func (d *PikPak) Init(ctx context.Context) (err error) { } // 获取CaptchaToken - err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Common.GetUserID()) + err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.net/drive/v1/files"), d.Common.GetUserID()) if err != nil { return err } @@ -174,7 +174,7 @@ func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if !d.DisableMediaLink { queryParams["usage"] = "CACHE" } - _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.com/drive/v1/files/%s", file.GetID()), + _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.net/drive/v1/files/%s", file.GetID()), http.MethodGet, func(req *resty.Request) { req.SetQueryParams(queryParams) }, &resp) @@ -200,7 +200,7 @@ func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *PikPak) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "kind": "drive#folder", "parent_id": parentDir.GetID(), @@ -211,7 +211,7 @@ func (d *PikPak) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *PikPak) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchMove", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchMove", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{srcObj.GetID()}, "to": base.Json{ @@ -223,7 +223,7 @@ func (d *PikPak) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *PikPak) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files/"+srcObj.GetID(), http.MethodPatch, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files/"+srcObj.GetID(), http.MethodPatch, func(req *resty.Request) { req.SetBody(base.Json{ "name": newName, }) @@ -232,7 +232,7 @@ func (d *PikPak) Rename(ctx context.Context, srcObj model.Obj, newName string) e } func (d *PikPak) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchCopy", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchCopy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{srcObj.GetID()}, "to": base.Json{ @@ -244,7 +244,7 @@ func (d *PikPak) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *PikPak) Remove(ctx context.Context, obj model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchTrash", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchTrash", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{obj.GetID()}, }) @@ -268,7 +268,7 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } var resp UploadTaskData - res, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + res, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "kind": "drive#file", "name": stream.GetName(), @@ -292,9 +292,9 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr params := resp.Resumable.Params //endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") - // web 端上传 返回的endpoint 为 `mypikpak.com` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.com`· + // web 端上传 返回的endpoint 为 `mypikpak.net` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.net`· if d.Addition.Platform == "android" { - params.Endpoint = "mypikpak.com" + params.Endpoint = "mypikpak.net" } if stream.GetSize() <= 10*utils.MB { // 文件大小 小于10MB,改用普通模式上传 @@ -318,7 +318,7 @@ func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir } var resp OfflineDownloadResp - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(requestBody) }, &resp) @@ -336,7 +336,7 @@ PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING */ func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase []string) ([]OfflineTask, error) { res := make([]OfflineTask, 0) - url := "https://api-drive.mypikpak.com/drive/v1/tasks" + url := "https://api-drive.mypikpak.net/drive/v1/tasks" if len(phase) == 0 { phase = []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_COMPLETE", "PHASE_TYPE_PENDING"} @@ -377,7 +377,7 @@ func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase [] } func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { - url := "https://api-drive.mypikpak.com/drive/v1/tasks" + url := "https://api-drive.mypikpak.net/drive/v1/tasks" params := map[string]string{ "task_ids": strings.Join(taskIDs, ","), "delete_files": strconv.FormatBool(deleteFiles), diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 1fd26020a60..6c5c88ad4b2 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -86,51 +86,51 @@ const ( WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" - WebPackageName = "mypikpak.com" + WebPackageName = "mypikpak.net" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" PCClientVersion = "undefined" // 2.5.6.4831 - PCPackageName = "mypikpak.com" + PCPackageName = "mypikpak.net" PCSdkVersion = "8.0.3" ) var DlAddr = []string{ - "dl-a10b-0621.mypikpak.com", - "dl-a10b-0622.mypikpak.com", - "dl-a10b-0623.mypikpak.com", - "dl-a10b-0624.mypikpak.com", - "dl-a10b-0625.mypikpak.com", - "dl-a10b-0858.mypikpak.com", - "dl-a10b-0859.mypikpak.com", - "dl-a10b-0860.mypikpak.com", - "dl-a10b-0861.mypikpak.com", - "dl-a10b-0862.mypikpak.com", - "dl-a10b-0863.mypikpak.com", - "dl-a10b-0864.mypikpak.com", - "dl-a10b-0865.mypikpak.com", - "dl-a10b-0866.mypikpak.com", - "dl-a10b-0867.mypikpak.com", - "dl-a10b-0868.mypikpak.com", - "dl-a10b-0869.mypikpak.com", - "dl-a10b-0870.mypikpak.com", - "dl-a10b-0871.mypikpak.com", - "dl-a10b-0872.mypikpak.com", - "dl-a10b-0873.mypikpak.com", - "dl-a10b-0874.mypikpak.com", - "dl-a10b-0875.mypikpak.com", - "dl-a10b-0876.mypikpak.com", - "dl-a10b-0877.mypikpak.com", - "dl-a10b-0878.mypikpak.com", - "dl-a10b-0879.mypikpak.com", - "dl-a10b-0880.mypikpak.com", - "dl-a10b-0881.mypikpak.com", - "dl-a10b-0882.mypikpak.com", - "dl-a10b-0883.mypikpak.com", - "dl-a10b-0884.mypikpak.com", - "dl-a10b-0885.mypikpak.com", - "dl-a10b-0886.mypikpak.com", - "dl-a10b-0887.mypikpak.com", + "dl-a10b-0621.mypikpak.net", + "dl-a10b-0622.mypikpak.net", + "dl-a10b-0623.mypikpak.net", + "dl-a10b-0624.mypikpak.net", + "dl-a10b-0625.mypikpak.net", + "dl-a10b-0858.mypikpak.net", + "dl-a10b-0859.mypikpak.net", + "dl-a10b-0860.mypikpak.net", + "dl-a10b-0861.mypikpak.net", + "dl-a10b-0862.mypikpak.net", + "dl-a10b-0863.mypikpak.net", + "dl-a10b-0864.mypikpak.net", + "dl-a10b-0865.mypikpak.net", + "dl-a10b-0866.mypikpak.net", + "dl-a10b-0867.mypikpak.net", + "dl-a10b-0868.mypikpak.net", + "dl-a10b-0869.mypikpak.net", + "dl-a10b-0870.mypikpak.net", + "dl-a10b-0871.mypikpak.net", + "dl-a10b-0872.mypikpak.net", + "dl-a10b-0873.mypikpak.net", + "dl-a10b-0874.mypikpak.net", + "dl-a10b-0875.mypikpak.net", + "dl-a10b-0876.mypikpak.net", + "dl-a10b-0877.mypikpak.net", + "dl-a10b-0878.mypikpak.net", + "dl-a10b-0879.mypikpak.net", + "dl-a10b-0880.mypikpak.net", + "dl-a10b-0881.mypikpak.net", + "dl-a10b-0882.mypikpak.net", + "dl-a10b-0883.mypikpak.net", + "dl-a10b-0884.mypikpak.net", + "dl-a10b-0885.mypikpak.net", + "dl-a10b-0886.mypikpak.net", + "dl-a10b-0887.mypikpak.net", } func (d *PikPak) login() error { @@ -139,7 +139,7 @@ func (d *PikPak) login() error { return errors.New("username or password is empty") } - url := "https://user.mypikpak.com/v1/auth/signin" + url := "https://user.mypikpak.net/v1/auth/signin" // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) if d.GetCaptchaToken() == "" { if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { @@ -169,7 +169,7 @@ func (d *PikPak) login() error { } func (d *PikPak) refreshToken(refreshToken string) error { - url := "https://user.mypikpak.com/v1/auth/token" + url := "https://user.mypikpak.net/v1/auth/token" var e ErrResp res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e). SetHeader("user-agent", "").SetBody(base.Json{ @@ -307,7 +307,7 @@ func (d *PikPak) getFiles(id string) ([]File, error) { "page_token": pageToken, } var resp Files - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -473,7 +473,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err } var e ErrResp var resp CaptchaTokenResponse - _, err := d.request("https://user.mypikpak.com/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://user.mypikpak.net/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param).SetQueryParam("client_id", d.ClientID) }, &resp) diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index 91cb45ca1cf..f107ac17ba3 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -80,7 +80,7 @@ func (d *PikPakShare) Init(ctx context.Context) error { } // 获取CaptchaToken - err := d.RefreshCaptchaToken(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/share:batch_file_info"), "") + err := d.RefreshCaptchaToken(GetAction(http.MethodGet, "https://api-drive.mypikpak.net/drive/v1/share:batch_file_info"), "") if err != nil { return err } @@ -113,7 +113,7 @@ func (d *PikPakShare) Link(ctx context.Context, file model.Obj, args model.LinkA "file_id": file.GetID(), "pass_code_token": d.PassCodeToken, } - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share/file_info", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share/file_info", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index f333ca5f706..1b14a65aad6 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -68,51 +68,51 @@ const ( WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" - WebPackageName = "mypikpak.com" + WebPackageName = "mypikpak.net" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" PCClientVersion = "undefined" // 2.5.6.4831 - PCPackageName = "mypikpak.com" + PCPackageName = "mypikpak.net" PCSdkVersion = "8.0.3" ) var DlAddr = []string{ - "dl-a10b-0621.mypikpak.com", - "dl-a10b-0622.mypikpak.com", - "dl-a10b-0623.mypikpak.com", - "dl-a10b-0624.mypikpak.com", - "dl-a10b-0625.mypikpak.com", - "dl-a10b-0858.mypikpak.com", - "dl-a10b-0859.mypikpak.com", - "dl-a10b-0860.mypikpak.com", - "dl-a10b-0861.mypikpak.com", - "dl-a10b-0862.mypikpak.com", - "dl-a10b-0863.mypikpak.com", - "dl-a10b-0864.mypikpak.com", - "dl-a10b-0865.mypikpak.com", - "dl-a10b-0866.mypikpak.com", - "dl-a10b-0867.mypikpak.com", - "dl-a10b-0868.mypikpak.com", - "dl-a10b-0869.mypikpak.com", - "dl-a10b-0870.mypikpak.com", - "dl-a10b-0871.mypikpak.com", - "dl-a10b-0872.mypikpak.com", - "dl-a10b-0873.mypikpak.com", - "dl-a10b-0874.mypikpak.com", - "dl-a10b-0875.mypikpak.com", - "dl-a10b-0876.mypikpak.com", - "dl-a10b-0877.mypikpak.com", - "dl-a10b-0878.mypikpak.com", - "dl-a10b-0879.mypikpak.com", - "dl-a10b-0880.mypikpak.com", - "dl-a10b-0881.mypikpak.com", - "dl-a10b-0882.mypikpak.com", - "dl-a10b-0883.mypikpak.com", - "dl-a10b-0884.mypikpak.com", - "dl-a10b-0885.mypikpak.com", - "dl-a10b-0886.mypikpak.com", - "dl-a10b-0887.mypikpak.com", + "dl-a10b-0621.mypikpak.net", + "dl-a10b-0622.mypikpak.net", + "dl-a10b-0623.mypikpak.net", + "dl-a10b-0624.mypikpak.net", + "dl-a10b-0625.mypikpak.net", + "dl-a10b-0858.mypikpak.net", + "dl-a10b-0859.mypikpak.net", + "dl-a10b-0860.mypikpak.net", + "dl-a10b-0861.mypikpak.net", + "dl-a10b-0862.mypikpak.net", + "dl-a10b-0863.mypikpak.net", + "dl-a10b-0864.mypikpak.net", + "dl-a10b-0865.mypikpak.net", + "dl-a10b-0866.mypikpak.net", + "dl-a10b-0867.mypikpak.net", + "dl-a10b-0868.mypikpak.net", + "dl-a10b-0869.mypikpak.net", + "dl-a10b-0870.mypikpak.net", + "dl-a10b-0871.mypikpak.net", + "dl-a10b-0872.mypikpak.net", + "dl-a10b-0873.mypikpak.net", + "dl-a10b-0874.mypikpak.net", + "dl-a10b-0875.mypikpak.net", + "dl-a10b-0876.mypikpak.net", + "dl-a10b-0877.mypikpak.net", + "dl-a10b-0878.mypikpak.net", + "dl-a10b-0879.mypikpak.net", + "dl-a10b-0880.mypikpak.net", + "dl-a10b-0881.mypikpak.net", + "dl-a10b-0882.mypikpak.net", + "dl-a10b-0883.mypikpak.net", + "dl-a10b-0884.mypikpak.net", + "dl-a10b-0885.mypikpak.net", + "dl-a10b-0886.mypikpak.net", + "dl-a10b-0887.mypikpak.net", } func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { @@ -159,7 +159,7 @@ func (d *PikPakShare) getSharePassToken() error { "limit": "100", } var resp ShareResp - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -187,7 +187,7 @@ func (d *PikPakShare) getFiles(id string) ([]File, error) { "pass_code_token": d.PassCodeToken, } var resp ShareResp - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share/detail", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share/detail", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -345,7 +345,7 @@ func (d *PikPakShare) refreshCaptchaToken(action string, metas map[string]string } var e ErrResp var resp CaptchaTokenResponse - _, err := d.request("https://user.mypikpak.com/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://user.mypikpak.net/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param) }, &resp) diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index 11db351b75c..362de69e0a0 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -10,8 +10,6 @@ import ( "math" stdpath "path" "strconv" - "strings" - "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" @@ -24,9 +22,9 @@ import ( type Terabox struct { model.Storage Addition - JsToken string + JsToken string url_domain_prefix string - base_url string + base_url string } func (d *Terabox) Config() driver.Config { @@ -145,52 +143,24 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt } log.Debugln(locateupload_resp) - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - var Default int64 = 4 * 1024 * 1024 - defaultByteData := make([]byte, Default) - count := int(math.Ceil(float64(stream.GetSize()) / float64(Default))) - // cal md5 - h1 := md5.New() - h2 := md5.New() - block_list := make([]string, 0) - left := stream.GetSize() - for i := 0; i < count; i++ { - byteSize := Default - var byteData []byte - if left < Default { - byteSize = left - byteData = make([]byte, byteSize) - } else { - byteData = defaultByteData - } - left -= byteSize - _, err = io.ReadFull(tempFile, byteData) - if err != nil { - return err - } - h1.Write(byteData) - h2.Write(byteData) - block_list = append(block_list, fmt.Sprintf("\"%s\"", hex.EncodeToString(h2.Sum(nil)))) - h2.Reset() - } + // precreate file + rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) + path := encodeURIComponent(rawPath) - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err + var precreateBlockListStr string + if stream.GetSize() > initialChunkSize { + precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761","a5fc157d78e6ad1c7e114b056c92821e"]` + } else { + precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761"]` } - rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) - path := encodeURIComponent(rawPath) - block_list_str := fmt.Sprintf("[%s]", strings.Join(block_list, ",")) data := map[string]string{ - "path": rawPath, - "autoinit": "1", - "target_path": dstDir.GetPath(), - "block_list": block_list_str, - "local_mtime": strconv.FormatInt(time.Now().Unix(), 10), + "path": rawPath, + "autoinit": "1", + "target_path": dstDir.GetPath(), + "block_list": precreateBlockListStr, + "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), + "file_limit_switch_v34": "true", } var precreateResp PrecreateResp log.Debugln(data) @@ -206,6 +176,13 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt if precreateResp.ReturnType == 2 { return nil } + + // upload chunks + tempFile, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + params := map[string]string{ "method": "upload", "path": path, @@ -215,24 +192,37 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt "channel": "dubox", "clienttype": "0", } - left = stream.GetSize() - for i, partseq := range precreateResp.BlockList { + + streamSize := stream.GetSize() + chunkSize := calculateChunkSize(streamSize) + chunkByteData := make([]byte, chunkSize) + count := int(math.Ceil(float64(streamSize) / float64(chunkSize))) + left := streamSize + uploadBlockList := make([]string, 0, count) + h := md5.New() + for partseq := 0; partseq < count; partseq++ { if utils.IsCanceled(ctx) { return ctx.Err() } - byteSize := Default + byteSize := chunkSize var byteData []byte - if left < Default { + if left >= chunkSize { + byteData = chunkByteData + } else { byteSize = left byteData = make([]byte, byteSize) - } else { - byteData = defaultByteData } left -= byteSize _, err = io.ReadFull(tempFile, byteData) if err != nil { return err } + + // calculate md5 + h.Write(byteData) + uploadBlockList = append(uploadBlockList, hex.EncodeToString(h.Sum(nil))) + h.Reset() + u := "https://" + locateupload_resp.Host + "/rest/2.0/pcs/superfile2" params["partseq"] = strconv.Itoa(partseq) res, err := base.RestyClient.R(). @@ -245,25 +235,39 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt return err } log.Debugln(res.String()) - if len(precreateResp.BlockList) > 0 { - up(float64(i) * 100 / float64(len(precreateResp.BlockList))) + if count > 0 { + up(float64(partseq) * 100 / float64(count)) } } + + // create file params = map[string]string{ "isdir": "0", "rtype": "1", } + + uploadBlockListStr, err := utils.Json.MarshalToString(uploadBlockList) + if err != nil { + return err + } data = map[string]string{ "path": rawPath, "size": strconv.FormatInt(stream.GetSize(), 10), "uploadid": precreateResp.Uploadid, "target_path": dstDir.GetPath(), - "block_list": block_list_str, - "local_mtime": strconv.FormatInt(time.Now().Unix(), 10), + "block_list": uploadBlockListStr, + "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), } - res, err = d.post_form("/api/create", params, data, nil) + var createResp CreateResp + res, err = d.post_form("/api/create", params, data, &createResp) log.Debugln(string(res)) - return err + if err != nil { + return err + } + if createResp.Errno != 0 { + return fmt.Errorf("[terabox] failed to create file, errno: %d", createResp.Errno) + } + return nil } var _ driver.Driver = (*Terabox)(nil) diff --git a/drivers/terabox/types.go b/drivers/terabox/types.go index 8bdbc6fce1b..f4d50ddef37 100644 --- a/drivers/terabox/types.go +++ b/drivers/terabox/types.go @@ -99,3 +99,7 @@ type CheckLoginResp struct { type LocateUploadResp struct { Host string `json:"host"` } + +type CreateResp struct { + Errno int `json:"errno"` +} diff --git a/drivers/terabox/util.go b/drivers/terabox/util.go index e0f3d74e8f5..058eecd6085 100644 --- a/drivers/terabox/util.go +++ b/drivers/terabox/util.go @@ -17,6 +17,11 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + initialChunkSize int64 = 4 << 20 // 4MB + initialSizeThreshold int64 = 4 << 30 // 4GB +) + func getStrBetween(raw, start, end string) string { regexPattern := fmt.Sprintf(`%s(.*?)%s`, regexp.QuoteMeta(start), regexp.QuoteMeta(end)) regex := regexp.MustCompile(regexPattern) @@ -86,11 +91,15 @@ func (d *Terabox) request(rurl string, method string, callback base.ReqCallback, return d.request(rurl, method, callback, resp, true) } } else if errno == -6 { - log.Debugln(res.Header()) - d.url_domain_prefix = res.Header()["Url-Domain-Prefix"][0] - d.base_url = "https://" + d.url_domain_prefix + ".terabox.com" - log.Debugln("Redirect base_url to", d.base_url) - return d.request(rurl, method, callback, resp, noRetry...) + header := res.Header() + log.Debugln(header) + urlDomainPrefix := header.Get("Url-Domain-Prefix") + if len(urlDomainPrefix) > 0 { + d.url_domain_prefix = urlDomainPrefix + d.base_url = "https://" + d.url_domain_prefix + ".terabox.com" + log.Debugln("Redirect base_url to", d.base_url) + return d.request(rurl, method, callback, resp, noRetry...) + } } return res.Body(), nil } @@ -258,3 +267,19 @@ func encodeURIComponent(str string) string { r = strings.ReplaceAll(r, "+", "%20") return r } + +func calculateChunkSize(streamSize int64) int64 { + chunkSize := initialChunkSize + sizeThreshold := initialSizeThreshold + + if streamSize < chunkSize { + return streamSize + } + + for streamSize > sizeThreshold { + chunkSize <<= 1 + sizeThreshold <<= 1 + } + + return chunkSize +} diff --git a/drivers/vtencent/drive.go b/drivers/vtencent/drive.go index 676431439a9..36a9167234e 100644 --- a/drivers/vtencent/drive.go +++ b/drivers/vtencent/drive.go @@ -55,7 +55,9 @@ func (d *Vtencent) Init(ctx context.Context) error { } func (d *Vtencent) Drop(ctx context.Context) error { - d.cron.Stop() + if d.cron != nil { + d.cron.Stop() + } return nil } diff --git a/internal/conf/config.go b/internal/conf/config.go index c5dc9c521bf..aa29e1f506d 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -131,22 +131,22 @@ func DefaultConfig() *Config { TlsInsecureSkipVerify: true, Tasks: TasksConfig{ Download: TaskConfig{ - Workers: 5, - MaxRetry: 1, - TaskPersistant: true, + Workers: 5, + MaxRetry: 1, + // TaskPersistant: true, }, Transfer: TaskConfig{ - Workers: 5, - MaxRetry: 2, - TaskPersistant: true, + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, }, Upload: TaskConfig{ Workers: 5, }, Copy: TaskConfig{ - Workers: 5, - MaxRetry: 2, - TaskPersistant: true, + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, }, }, Cors: Cors{ diff --git a/internal/op/storage.go b/internal/op/storage.go index 6790a8dffa6..7d8831f548e 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -101,7 +101,7 @@ func initStorage(ctx context.Context, storage model.Storage, storageDriver drive log.Errorf("panic init storage: %s", errInfo) driverStorage.SetStatus(errInfo) MustSaveDriverStorage(storageDriver) - storagesMap.Delete(driverStorage.MountPath) + storagesMap.Store(driverStorage.MountPath, storageDriver) } }() // Unmarshal Addition