From 44ac75d98fba37d23acf0519196d731c197a446e Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Sat, 5 Oct 2024 07:54:45 +0530 Subject: [PATCH 1/2] Fix the retry mechanism and improve error handling. --- pikpakapi/PikpakException.py | 4 + pikpakapi/__init__.py | 141 +++++++++++++++++++---------------- test.py | 9 +++ 3 files changed, 90 insertions(+), 64 deletions(-) diff --git a/pikpakapi/PikpakException.py b/pikpakapi/PikpakException.py index 3172f31..f2d8cb3 100644 --- a/pikpakapi/PikpakException.py +++ b/pikpakapi/PikpakException.py @@ -1,3 +1,7 @@ class PikpakException(Exception): def __init__(self, message): super().__init__(message) + + +class PikpakRetryException(PikpakException): + pass diff --git a/pikpakapi/__init__.py b/pikpakapi/__init__.py index f7f8387..73c791f 100644 --- a/pikpakapi/__init__.py +++ b/pikpakapi/__init__.py @@ -4,7 +4,6 @@ import logging import asyncio from base64 import b64decode, b64encode -import sys import re from typing import Any, Dict, List, Optional from .utils import ( @@ -19,7 +18,7 @@ import httpx -from .PikpakException import PikpakException +from .PikpakException import PikpakException, PikpakRetryException from .enums import DownloadStatus @@ -28,8 +27,6 @@ class PikPakApi: PikPakApi class Attributes: - CLIENT_ID: str - PikPak API client id - CLIENT_SECRET: str - PikPak API client secret PIKPAK_API_HOST: str - PikPak API host PIKPAK_USER_HOST: str - PikPak user API host @@ -39,7 +36,6 @@ class PikPakApi: access_token: str - access token of the user , expire in 7200 refresh_token: str - refresh token of the user user_id: str - user id of the user - httpx_client_args: dict - extra arguments for httpx.AsyncClient (https://www.python-httpx.org/api/#asyncclient) """ @@ -51,19 +47,26 @@ def __init__( username: Optional[str] = None, password: Optional[str] = None, encoded_token: Optional[str] = None, - httpx_client_args: Optional[Dict[str, Any]] = {}, + httpx_client_args: Optional[Dict[str, Any]] = None, device_id: Optional[str] = None, + request_max_retries: int = 3, + request_initial_backoff: float = 3.0, ): """ username: str - username of the user password: str - password of the user encoded_token: str - encoded token of the user with access and refresh token httpx_client_args: dict - extra arguments for httpx.AsyncClient (https://www.python-httpx.org/api/#asyncclient) + device_id: str - device id to identify the device + request_max_retries: int - maximum number of retries for requests + request_initial_backoff: float - initial backoff time for retries """ self.username = username self.password = password self.encoded_token = encoded_token + self.max_retries = request_max_retries + self.initial_backoff = request_initial_backoff self.access_token = None self.refresh_token = None @@ -77,9 +80,8 @@ def __init__( ) self.captcha_token = None - self.httpx_client = httpx.AsyncClient( - **httpx_client_args if httpx_client_args else {} - ) + httpx_client_args = httpx_client_args or {"timeout": 10} + self.httpx_client = httpx.AsyncClient(**httpx_client_args) self._path_id_cache: Dict[str, Any] = {} @@ -124,62 +126,74 @@ def get_headers(self, access_token: Optional[str] = None) -> Dict[str, str]: return headers async def _make_request( - self, method: str, url: str, data=None, params=None, headers=None + self, + method: str, + url: str, + data: Optional[Dict[str, Any]] = None, + params: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: - backoff_seconds = 3 - error_decription = "" - for i in range(3): # retries - # headers can be different for each request with captcha - if headers is None: - req_headers = self.get_headers() - else: - req_headers = headers + last_error = None + + for attempt in range(self.max_retries): try: - response = await self.httpx_client.request( - method, - url, - json=data, - params=params, - headers=req_headers, + response = await self._send_request(method, url, data, params, headers) + return await self._handle_response(response) + except PikpakRetryException as error: + logging.info(f"Retry attempt {attempt + 1}/{self.max_retries}") + last_error = error + except PikpakException: + raise + except httpx.HTTPError as error: + logging.error( + f"HTTP Error on attempt {attempt + 1}/{self.max_retries}: {str(error)}" + ) + last_error = error + except Exception as error: + logging.error( + f"Unexpected error on attempt {attempt + 1}/{self.max_retries}: {str(error)}" ) - except httpx.HTTPError as e: - logging.error(e) - await asyncio.sleep(backoff_seconds) - backoff_seconds *= 2 # exponential backoff - continue - except KeyboardInterrupt as e: - sys.exit(0) - except Exception as e: - logging.error(e) - await asyncio.sleep(backoff_seconds) - backoff_seconds *= 2 # exponential backoff - continue + last_error = error + + await asyncio.sleep(self.initial_backoff * (2**attempt)) + + # If we've exhausted all retries, raise an exception with the last error + raise PikpakException(f"Max retries reached. Last error: {str(last_error)}") + async def _send_request(self, method, url, data, params, headers): + req_headers = headers or self.get_headers() + return await self.httpx_client.request( + method, + url, + json=data, + params=params, + headers=req_headers, + ) + + async def _handle_response(self, response) -> Dict[str, Any]: + try: json_data = response.json() - if json_data and "error" not in json_data: - # ok - return json_data - - if not json_data: - error_decription = "empty json data" - await asyncio.sleep(backoff_seconds) - backoff_seconds *= 2 # exponential backoff - continue - elif json_data["error_code"] == 16: - await self.refresh_access_token() - continue - # goes to next iteration in retry loop - else: - await asyncio.sleep(backoff_seconds) - backoff_seconds *= 2 # exponential backoff - continue + except ValueError: + if response.status_code == 200: + return {} + raise PikpakRetryException("Empty JSON data") - if error_decription == "" and "error_description" in json_data.keys(): - error_decription = json_data["error_description"] - else: - error_decription = "Unknown Error" + if not json_data: + if response.status_code == 200: + return {} + raise PikpakRetryException("Empty JSON data") + + if "error" not in json_data: + return json_data + + if json_data["error"] == "invalid_account_or_password": + raise PikpakException("Invalid username or password") + + if json_data.get("error_code") == 16: + await self.refresh_access_token() + raise PikpakRetryException("Token refreshed, please retry") - raise PikpakException(error_decription) + raise PikpakException(json_data.get("error_description", "Unknown Error")) async def _request_get( self, @@ -574,7 +588,6 @@ async def path_to_id(self, path: str, create: bool = False) -> List[Dict[str, st next_page_token = None while count < len(paths): - current_parent_path = "/" + "/".join(paths[:count]) data = await self.file_list( parent_id=parent_id, next_page_token=next_page_token ) @@ -603,9 +616,9 @@ async def path_to_id(self, path: str, create: bool = False) -> List[Dict[str, st next_page_token = data.get("next_page_token") elif create: data = await self.create_folder(name=paths[count], parent_id=parent_id) - id = data.get("file").get("id") + file_id = data.get("file").get("id") record = { - "id": id, + "id": file_id, "name": paths[count], "file_type": "folder", } @@ -613,7 +626,7 @@ async def path_to_id(self, path: str, create: bool = False) -> List[Dict[str, st current_path = "/" + "/".join(paths[: count + 1]) self._path_id_cache[current_path] = record count += 1 - parent_id = id + parent_id = file_id else: break return path_ids @@ -690,8 +703,8 @@ async def file_move_or_copy_by_path( from_ids: List[str] = [] for path in from_path: if path_ids := await self.path_to_id(path): - if id := path_ids[-1].get("id"): - from_ids.append(id) + if file_id := path_ids[-1].get("id"): + from_ids.append(file_id) if not from_ids: raise PikpakException("要移动的文件不存在") to_path_ids = await self.path_to_id(to_path, create=create) diff --git a/test.py b/test.py index 849ea95..c2f6a52 100644 --- a/test.py +++ b/test.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import httpx @@ -17,6 +18,11 @@ async def test(): ) await client.login() await client.refresh_access_token() + tasks = await client.offline_list() + print(json.dumps(tasks, indent=4)) + print("=" * 30, end="\n\n") + if tasks.get("tasks"): + await client.delete_tasks(task_ids=[tasks["tasks"][0]["id"]]) print(json.dumps(client.get_user_info(), indent=4)) print("=" * 30, end="\n\n") @@ -82,4 +88,7 @@ async def test(): if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) asyncio.run(test()) From 667b034455a8bd5eaff54290d0149223f790ab68 Mon Sep 17 00:00:00 2001 From: mhdzumair Date: Sat, 5 Oct 2024 08:19:35 +0530 Subject: [PATCH 2/2] Add support for FilePax alongside PikPak --- pikpakapi/__init__.py | 122 +++++++++++++++++++++++++++--------------- pikpakapi/utils.py | 7 +-- test.py | 7 +-- 3 files changed, 84 insertions(+), 52 deletions(-) diff --git a/pikpakapi/__init__.py b/pikpakapi/__init__.py index 8024c41..1485264 100644 --- a/pikpakapi/__init__.py +++ b/pikpakapi/__init__.py @@ -5,20 +5,21 @@ import re from base64 import b64decode, b64encode from hashlib import md5 -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Literal import httpx from .PikpakException import PikpakException, PikpakRetryException from .enums import DownloadStatus from .utils import ( - CLIENT_ID, + PIKPAK_CLIENT_ID, CLIENT_SECRET, CLIENT_VERSION, PACKAG_ENAME, build_custom_user_agent, captcha_sign, get_timestamp, + FILEPAX_CLIENT_ID, ) @@ -29,6 +30,8 @@ class PikPakApi: Attributes: PIKPAK_API_HOST: str - PikPak API host PIKPAK_USER_HOST: str - PikPak user API host + FILEPAX_API_HOST: str - FilePax API host + FILEPAX_USER_HOST: str - FilePax user API host username: str - username of the user password: str - password of the user @@ -41,6 +44,8 @@ class PikPakApi: PIKPAK_API_HOST = "api-drive.mypikpak.com" PIKPAK_USER_HOST = "user.mypikpak.com" + FILEPAX_API_HOST = "api-drive.filepax.com" + FILEPAX_USER_HOST = "user.filepax.com" def __init__( self, @@ -51,6 +56,7 @@ def __init__( device_id: Optional[str] = None, request_max_retries: int = 3, request_initial_backoff: float = 3.0, + host: Literal["pikpak", "filepax"] = "pikpak", ): """ username: str - username of the user @@ -60,6 +66,7 @@ def __init__( device_id: str - device id to identify the device request_max_retries: int - maximum number of retries for requests request_initial_backoff: float - initial backoff time for retries + host: str - host to use, pikpak or filepax """ self.username = username @@ -67,6 +74,13 @@ def __init__( self.encoded_token = encoded_token self.max_retries = request_max_retries self.initial_backoff = request_initial_backoff + self.host = host + self.api_host = ( + self.PIKPAK_API_HOST if host == "pikpak" else self.FILEPAX_API_HOST + ) + self.user_host = ( + self.PIKPAK_USER_HOST if host == "pikpak" else self.FILEPAX_USER_HOST + ) self.access_token = None self.refresh_token = None @@ -247,7 +261,7 @@ def encode_token(self): self.encoded_token = b64encode(json.dumps(token_data).encode()).decode() async def captcha_init(self, action: str, meta: dict = None) -> Dict[str, Any]: - url = f"https://{PikPakApi.PIKPAK_USER_HOST}/v1/shield/captcha/init" + url = f"https://{self.user_host}/v1/shield/captcha/init" if not meta: t = f"{get_timestamp()}" meta = { @@ -258,7 +272,9 @@ async def captcha_init(self, action: str, meta: dict = None) -> Dict[str, Any]: "timestamp": t, } params = { - "client_id": CLIENT_ID, + "client_id": ( + PIKPAK_CLIENT_ID if self.host == "pikpak" else FILEPAX_CLIENT_ID + ), "action": action, "device_id": self.device_id, "meta": meta, @@ -269,7 +285,8 @@ async def login(self) -> None: """ Login to PikPak """ - login_url = f"https://{PikPakApi.PIKPAK_USER_HOST}/v1/auth/signin" + login_path = "/v1/auth/signin" + login_url = f"https://{self.user_host}{login_path}" metas = {} if not self.username or not self.password: raise PikpakException("username and password are required") @@ -280,25 +297,36 @@ async def login(self) -> None: else: metas["username"] = self.username result = await self.captcha_init( - action=f"POST:{login_url}", + action=f"POST:{login_url if self.host == 'pikpak' else login_path}", meta=metas, ) captcha_token = result.get("captcha_token", "") if not captcha_token: raise PikpakException("captcha_token get failed") - login_data = { - "client_id": CLIENT_ID, - "client_secret": CLIENT_SECRET, - "password": self.password, - "username": self.username, - "captcha_token": captcha_token, - } + if self.host == "pikpak": + login_data = { + "client_id": PIKPAK_CLIENT_ID, + "client_secret": CLIENT_SECRET, + "password": self.password, + "username": self.username, + "captcha_token": captcha_token, + } + headers = { + "Content-Type": "application/x-www-form-urlencoded", + } + else: + login_data = { + "client_id": FILEPAX_CLIENT_ID, + "password": self.password, + "username": self.username, + } + headers = { + "Content-Type": "application/json", + } user_info = await self._request_post( login_url, login_data, - { - "Content-Type": "application/x-www-form-urlencoded", - }, + headers, ) self.access_token = user_info["access_token"] self.refresh_token = user_info["refresh_token"] @@ -309,9 +337,11 @@ async def refresh_access_token(self) -> None: """ Refresh access token """ - refresh_url = f"https://{self.PIKPAK_USER_HOST}/v1/auth/token" + refresh_url = f"https://{self.user_host}/v1/auth/token" refresh_data = { - "client_id": CLIENT_ID, + "client_id": ( + PIKPAK_CLIENT_ID if self.host == "pikpak" else FILEPAX_CLIENT_ID + ), "refresh_token": self.refresh_token, "grant_type": "refresh_token", } @@ -342,7 +372,7 @@ async def create_folder( 创建文件夹 """ - url = f"https://{self.PIKPAK_API_HOST}/drive/v1/files" + url = f"https://{self.api_host}/drive/v1/files" data = { "kind": "drive#folder", "name": name, @@ -357,7 +387,7 @@ async def delete_to_trash(self, ids: List[str]) -> Dict[str, Any]: 将文件夹、文件移动到回收站 """ - url = f"https://{self.PIKPAK_API_HOST}/drive/v1/files:batchTrash" + url = f"https://{self.api_host}/drive/v1/files:batchTrash" data = { "ids": ids, } @@ -370,7 +400,7 @@ async def untrash(self, ids: List[str]) -> Dict[str, Any]: 将文件夹、文件移出回收站 """ - url = f"https://{self.PIKPAK_API_HOST}/drive/v1/files:batchUntrash" + url = f"https://{self.api_host}/drive/v1/files:batchUntrash" data = { "ids": ids, } @@ -383,7 +413,7 @@ async def delete_forever(self, ids: List[str]) -> Dict[str, Any]: 永远删除文件夹、文件, 慎用 """ - url = f"https://{self.PIKPAK_API_HOST}/drive/v1/files:batchDelete" + url = f"https://{self.api_host}/drive/v1/files:batchDelete" data = { "ids": ids, } @@ -400,7 +430,7 @@ async def offline_download( 离线下载磁力链 """ - download_url = f"https://{self.PIKPAK_API_HOST}/drive/v1/files" + download_url = f"https://{self.api_host}/drive/v1/files" download_data = { "kind": "drive#file", "name": name, @@ -428,7 +458,7 @@ async def offline_list( """ if phase is None: phase = ["PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR"] - list_url = f"https://{self.PIKPAK_API_HOST}/drive/v1/tasks" + list_url = f"https://{self.api_host}/drive/v1/tasks" list_data = { "type": "offline", "thumbnail_size": "SIZE_SMALL", @@ -446,7 +476,7 @@ async def offline_file_info(self, file_id: str) -> Dict[str, Any]: 离线下载文件信息 """ - url = f"https://{self.PIKPAK_API_HOST}/drive/v1/files/{file_id}" + url = f"https://{self.api_host}/drive/v1/files/{file_id}" result = await self._request_get(url, {"thumbnail_size": "SIZE_LARGE"}) return result @@ -471,7 +501,7 @@ async def file_list( } if additional_filters: default_filters.update(additional_filters) - list_url = f"https://{self.PIKPAK_API_HOST}/drive/v1/files" + list_url = f"https://{self.api_host}/drive/v1/files" list_data = { "parent_id": parent_id, "thumbnail_size": "SIZE_MEDIUM", @@ -480,7 +510,13 @@ async def file_list( "page_token": next_page_token, "filters": json.dumps(default_filters), } + # FixME + response = await self.captcha_init( + action="GET:/drive/v1/files", + ) + self.captcha_token = response.get("captcha_token") result = await self._request_get(list_url, list_data) + self.captcha_token = None return result async def events( @@ -492,7 +528,7 @@ async def events( 获取最近添加事件列表 """ - list_url = f"https://{self.PIKPAK_API_HOST}/drive/v1/events" + list_url = f"https://{self.api_host}/drive/v1/events" list_data = { "thumbnail_size": "SIZE_MEDIUM", "limit": size, @@ -507,7 +543,7 @@ async def offline_task_retry(self, task_id: str) -> Dict[str, Any]: 重试离线下载任务 """ - list_url = f"https://{self.PIKPAK_API_HOST}/drive/v1/task" + list_url = f"https://{self.api_host}/drive/v1/task" list_data = { "type": "offline", "create_type": "RETRY", @@ -526,7 +562,7 @@ async def delete_tasks( delete tasks by task ids task_ids: List[str] - task ids to delete """ - delete_url = f"https://{self.PIKPAK_API_HOST}/drive/v1/tasks" + delete_url = f"https://{self.api_host}/drive/v1/tasks" params = { "task_ids": task_ids, "delete_files": delete_files, @@ -650,7 +686,7 @@ async def file_batch_move( else {} ) result = await self._request_post( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/files:batchMove", + url=f"https://{self.api_host}/drive/v1/files:batchMove", data={ "ids": ids, "to": to, @@ -677,7 +713,7 @@ async def file_batch_copy( else {} ) result = await self._request_post( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/files:batchCopy", + url=f"https://{self.api_host}/drive/v1/files:batchCopy", data={ "ids": ids, "to": to, @@ -731,7 +767,7 @@ async def get_download_url(self, file_id: str) -> Dict[str, Any]: ) self.captcha_token = result.get("captcha_token") result = await self._request_get( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/files/{file_id}?", + url=f"https://{self.api_host}/drive/v1/files/{file_id}?", ) self.captcha_token = None return result @@ -748,7 +784,7 @@ async def file_rename(self, id: str, new_file_name: str) -> Dict[str, Any]: "name": new_file_name, } result = await self._request_patch( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/files/{id}", + url=f"https://{self.api_host}/drive/v1/files/{id}", data=data, ) return result @@ -766,7 +802,7 @@ async def file_batch_star( "ids": ids, } result = await self._request_post( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/files:star", + url=f"https://{self.api_host}/drive/v1/files:star", data=data, ) return result @@ -784,7 +820,7 @@ async def file_batch_unstar( "ids": ids, } result = await self._request_post( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/files:unstar", + url=f"https://{self.api_host}/drive/v1/files:unstar", data=data, ) return result @@ -838,7 +874,7 @@ async def file_batch_share( "pass_code_option": "REQUIRED" if need_password else "NOT_REQUIRED", } result = await self._request_post( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/share", + url=f"https://{self.api_host}/drive/v1/share", data=data, ) return result @@ -862,19 +898,19 @@ async def get_quota_info(self) -> Dict[str, Any]: } """ result = await self._request_get( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/about", + url=f"https://{self.api_host}/drive/v1/about", ) return result async def get_invite_code(self): result = await self._request_get( - url=f"https://{self.PIKPAK_API_HOST}/vip/v1/activity/inviteCode", + url=f"https://{self.api_host}/vip/v1/activity/inviteCode", ) return result["code"] async def vip_info(self): result = await self._request_get( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/privilege/vip", + url=f"https://{self.api_host}/drive/v1/privilege/vip", ) return result @@ -882,7 +918,7 @@ async def get_transfer_quota(self) -> Dict[str, Any]: """ Get transfer quota """ - url = f"https://{self.PIKPAK_API_HOST}/vip/v1/quantity/list?type=transfer" + url = f"https://{self.api_host}/vip/v1/quantity/list?type=transfer" result = await self._request_get(url) return result @@ -905,7 +941,7 @@ async def get_share_folder( "parent_id": parent_id, "pass_code_token": pass_code_token, } - url = f"https://{self.PIKPAK_API_HOST}/drive/v1/share/detail" + url = f"https://{self.api_host}/drive/v1/share/detail" return await self._request_get(url, params=data) async def get_share_info( @@ -933,7 +969,7 @@ async def get_share_info( "parent_id": parent_id, "pass_code": pass_code, } - url = f"https://{self.PIKPAK_API_HOST}/drive/v1/share" + url = f"https://{self.api_host}/drive/v1/share" return await self._request_get(url, params=data) async def restore( @@ -952,6 +988,6 @@ async def restore( "file_ids": file_ids, } result = await self._request_post( - url=f"https://{self.PIKPAK_API_HOST}/drive/v1/share/restore", data=data + url=f"https://{self.api_host}/drive/v1/share/restore", data=data ) return result diff --git a/pikpakapi/utils.py b/pikpakapi/utils.py index e4fcf54..4ff68df 100644 --- a/pikpakapi/utils.py +++ b/pikpakapi/utils.py @@ -2,7 +2,8 @@ from uuid import uuid4 import time -CLIENT_ID = "YNxT9w7GMdWvEOKa" +PIKPAK_CLIENT_ID = "YNxT9w7GMdWvEOKa" +FILEPAX_CLIENT_ID = "ZbHPv-5xPR83hucb" CLIENT_SECRET = "dbw2OtmVEeuUvIptb1Coyg" CLIENT_VERSION = "1.47.1" PACKAG_ENAME = "com.pikcloud.pikpak" @@ -50,7 +51,7 @@ def captcha_sign(device_id: str, timestamp: str) -> str: 在网页端的js中, 搜索 captcha_sign, 可以找到对应的js代码 """ - sign = CLIENT_ID + CLIENT_VERSION + PACKAG_ENAME + device_id + timestamp + sign = PIKPAK_CLIENT_ID + CLIENT_VERSION + PACKAG_ENAME + device_id + timestamp for salt in SALTS: sign = hashlib.md5((sign + salt).encode()).hexdigest() return f"1.{sign}" @@ -81,7 +82,7 @@ def build_custom_user_agent(device_id, user_id): f"ANDROID-{APP_NAME}/{CLIENT_VERSION}", "protocolVersion/200", "accesstype/", - f"clientid/{CLIENT_ID}", + f"clientid/{PIKPAK_CLIENT_ID}", f"clientversion/{CLIENT_VERSION}", "action_type/", "networktype/WIFI", diff --git a/test.py b/test.py index 5b13a0b..f455239 100644 --- a/test.py +++ b/test.py @@ -9,12 +9,7 @@ async def test(): client = PikPakApi( - username="your_username", - password="your_password", - httpx_client_args={ - "proxy": "http://127.0.0.1:1081", - "transport": httpx.AsyncHTTPTransport(retries=3), - }, + username="mhdzumair@gmail.com", password="Zumair@786", host="pikpak" ) await client.login() await client.refresh_access_token()