From af795f6e2b7b3efaa1d83e068c6812e48d16a004 Mon Sep 17 00:00:00 2001 From: "j.Stonty" <29155206+sht2017@users.noreply.github.com> Date: Thu, 11 Jul 2024 21:46:25 +0800 Subject: [PATCH] update --- .github/workflows/CI.yaml | 4 +- .gitignore | 2 + config.example.yaml | 62 ++++++++++ requirements.txt | 5 +- setup.py | 66 +++++++++++ src/browser/__init__.py | 1 + src/browser/browser.py | 31 +++++ src/browser/remote_injector/__init__.py | 2 + src/browser/remote_injector/inject.py | 109 ++++++++++++++++++ src/browser/remote_injector/injector.py | 52 +++++++++ src/browser/remote_injector/server.py | 64 ++++++++++ .../template/invokeRemoteFunction.js | 27 +++++ src/custom/channel.py | 28 +++++ src/custom/config.py | 7 ++ src/custom/config_parser.py | 10 ++ src/custom/injection.py | 85 ++++++++++++++ src/custom/jsondb/__init__.py | 1 + src/custom/jsondb/database.py | 54 +++++++++ src/custom/main.py | 28 +++++ src/epg/__init__.py | 2 + 20 files changed, 636 insertions(+), 4 deletions(-) create mode 100644 config.example.yaml create mode 100644 setup.py create mode 100644 src/browser/__init__.py create mode 100644 src/browser/browser.py create mode 100644 src/browser/remote_injector/__init__.py create mode 100644 src/browser/remote_injector/inject.py create mode 100644 src/browser/remote_injector/injector.py create mode 100644 src/browser/remote_injector/server.py create mode 100644 src/browser/remote_injector/template/invokeRemoteFunction.js create mode 100644 src/custom/channel.py create mode 100644 src/custom/config.py create mode 100644 src/custom/config_parser.py create mode 100644 src/custom/injection.py create mode 100644 src/custom/jsondb/__init__.py create mode 100644 src/custom/jsondb/database.py create mode 100644 src/custom/main.py create mode 100644 src/epg/__init__.py diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 1c56a5e..8c1736a 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -11,7 +11,7 @@ jobs: - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: '3.12' + python-version: "3.12" - name: Install dependencies run: pip install -r requirements.txt - name: Run tests and collect coverage @@ -19,4 +19,4 @@ jobs: - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 with: - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 27bc199..7d37eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ __pycache__/ /.venv/ /.pytest_cache/ .coverage +/config.yaml +/db.json \ No newline at end of file diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..3ca017c --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,62 @@ +# The document template is enabled for this yaml config file. Which can be accessed as examples below: +# {{browser.headers.Accept}} -> text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 +# {{epg.authenticator.auth_method}} -> SALTED_MD5 +browser: + args: + - --disable-extensions + - --disable-gpu + - --disable-dev-shm-usage + - --disable-add-to-shelf + - --disable-background-networking + - --disable-background-timer-throttling + - --disable-backgrounding-occluded-windows + - --disable-breakpad + - --disable-checker-imaging + - --disable-datasaver-prompt + - --disable-default-apps + - --disable-desktop-notifications + - --disable-domain-reliability + - --disable-hang-monitor + - --disable-infobars + - --disable-logging + - --disable-notifications + - --disable-popup-blocking + - --disable-prompt-on-repost + - --disable-renderer-backgrounding + - --disable-sync + - --force-color-profile=srgb + - --force-device-scale-factor=1 + - --metrics-recording-only + - --mute-audio + - --no-default-browser-check + - --no-first-run + headers: + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + Accept-Charset: utf-8, iso-8859-1, utf-16, *;q=0.7 + Accept-Encoding: gzip + Accept-Language: en-us + User-Agent: + Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0) + # start_url: Endpoints where the authentication start + start_url: "http://127.0.0.1/login?UserID={{epg.credential.user_id}}" + # end_url: Endpoints for which configuration files have been distributed, wildcard is allowed to use + end_url: "**/everything_finish.html" +epg: + authenticator: + # auth_method(PLAIN, MD5, SALTED_MD5): The hashing algorithm your carrier use + auth_method: SALTED_MD5 + # salt[optional], default None: Only need if you are using SALTED_MD5 as hashing algorithm, THIS FIELD MUST BE STRING! + salt: "12345678" + credential: + # user_id: Account + user_id: acc_example + # password: Password + password: pass1234 + # ip: IP address of your STB, might not affect authentication since the IP is unstable + ip: 127.0.0.1 + # mac: Mac address of your STB, affect authentication + mac: 1A:2B:3C:4D:5E:6F + # product_id: Unique id of your STB, affect authentication + product_id: PRODUCTID1234567890 + # ctc[optional], default to CTC: Unknown, might affect authentication, I don't know what it means for god sake + ctc: SOMECTC diff --git a/requirements.txt b/requirements.txt index 4801267..19244f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ -aiohttp>=0.0.0 fastapi>=0.0.0 -playwright>=0.0.0 +jinja2>=0.0.0 +playwright==1.44.0 pycryptodome>=0.0.0 +pydantic>=0.0.0 uvicorn[standard]>=0.0.0 pytest>=0.0.0 pytest-cov>=0.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c82c589 --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +import os +import subprocess +import venv +from pathlib import Path + +VENV_PATH = ".venv" + + +def get_binary_path(path: str | Path) -> Path: + if not isinstance(path, Path): + path = Path(path) + for bin_dir in ["bin", "Scripts"]: + binary_path = path / bin_dir + if binary_path.exists(): + return binary_path + raise FileNotFoundError("binary path of venv not found") + + +def rm(path: str | Path) -> None: + if not isinstance(path, Path): + path = Path(path) + if path.exists(): + if path.is_dir() and not path.is_symlink(): + for sub_path in path.iterdir(): + rm(sub_path) + path.rmdir() + else: + path.unlink() + else: + raise FileNotFoundError() + + +def create_venv(*args, **kwargs) -> None: + path = Path(kwargs["env_dir"] if "env_dir" in kwargs else args[0]) + print(path) + try: + rm(path) + except FileNotFoundError: + pass + venv.create(*args, **kwargs) + + +if __name__ != "__main__": + raise NotImplementedError() + +print("setup venv") +create_venv( + VENV_PATH, + with_pip=True, + upgrade_deps=True, +) +binary_path = get_binary_path(VENV_PATH) +print("install packages") +subprocess.run( + [binary_path / "pip", "install", "-r", "requirements.txt"], check=True +) +print("install browser") +subprocess.run([binary_path / "playwright", "install", "chromium"], check=True) +match os.name: + case "nt": + os.system(f"mklink {binary_path}/") + case "posix": + os.system(f"ln -s {binary_path}/") + case _: + raise NotImplementedError() diff --git a/src/browser/__init__.py b/src/browser/__init__.py new file mode 100644 index 0000000..5868012 --- /dev/null +++ b/src/browser/__init__.py @@ -0,0 +1 @@ +from .browser import process \ No newline at end of file diff --git a/src/browser/browser.py b/src/browser/browser.py new file mode 100644 index 0000000..5936d85 --- /dev/null +++ b/src/browser/browser.py @@ -0,0 +1,31 @@ +from playwright.async_api import ( + Browser, + Page, + async_playwright, +) + +from . import remote_injector + + +async def process( + injector: remote_injector.Injector, + start_url: str, + end_url: str, + args: list | None = None, + headers: dict | None = None, + headless: bool = True, +): + async with async_playwright() as playwright: + browser: Browser = await playwright.chromium.launch( + args=args, headless=headless + ) + page: Page = await browser.new_page() + + await remote_injector.inject(page, injector) + if headers: + await page.set_extra_http_headers(headers) + + await page.goto(start_url) + + await page.wait_for_url(url=end_url, wait_until="domcontentloaded") + await browser.close() diff --git a/src/browser/remote_injector/__init__.py b/src/browser/remote_injector/__init__.py new file mode 100644 index 0000000..ef1f4d3 --- /dev/null +++ b/src/browser/remote_injector/__init__.py @@ -0,0 +1,2 @@ +from .inject import inject +from .injector import Injector diff --git a/src/browser/remote_injector/inject.py b/src/browser/remote_injector/inject.py new file mode 100644 index 0000000..22b7db1 --- /dev/null +++ b/src/browser/remote_injector/inject.py @@ -0,0 +1,109 @@ +import asyncio +import inspect +import socket +from pathlib import Path + +import uvicorn +from jinja2 import Environment +from playwright.async_api import ( + BrowserContext, + Page, +) + +from .injector import Injector +from .server import RemoteInvokeServer + + +def _get_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + port = s.getsockname()[1] + return port + + +def _generate_script(injector: Injector) -> str: + result = "" + for class_name, class_ in injector: + if class_name != "_": + result += f"class {class_name} {{}}\n" + for method_name, method in class_.items(): + if class_name == "_": + result += ( + f"{method_name} = " + f"function({','.join(inspect.signature(method).parameters)})" + "{return " + f'invokeRemoteFunction("{method_name}",' + "null," + f'{{{",".join( + f'"{para}":{para}' for para + in inspect.signature(method).parameters + )}}}' + ").result};" + "\n" + ) + else: + result += ( + f"{class_name}.{method_name} = " + f"function({','.join(inspect.signature(method).parameters)})" + "{return " + f'invokeRemoteFunction("{class_name}.{method_name}",' + "null," + f'{{{",".join( + f'"{para}":{para}' for para + in inspect.signature(method).parameters + )}}}' + ").result};" + "\n" + ) + return result + + +async def _inject_javascript( + target: Page | BrowserContext, injector: Injector, port: int +) -> None: + with open( + Path(__file__).resolve().parent + / "template" + / "invokeRemoteFunction.js", + encoding="utf-8", + ) as file: + await target.add_init_script( + Environment().from_string(file.read()).render(port=port) + ) + await target.add_init_script(_generate_script(injector)) + + +async def _override_cors_restrictions( + target: Page | BrowserContext, port: int +) -> None: + async def override_cors(route, request): + url = request.url + if f"localhost:{port}" in url or f"127.0.0.1:{port}" in url: + await route.continue_() + else: + response = await target.request.fetch(request) + headers = response.headers + headers["Access-Control-Allow-Origin"] = "*" + await route.fulfill(response=response, headers=headers) + + await target.route("**/*", override_cors) + + +async def inject(target: Page | BrowserContext, injector: Injector) -> None: + port = _get_free_port() + server = uvicorn.Server( + uvicorn.Config( + RemoteInvokeServer(injector=injector).app, + port=port, + log_level="warning", + ) + ) + + async def on_close(): + await server.shutdown() + + asyncio.create_task(server.serve()) + await _override_cors_restrictions(target, port) + await _inject_javascript(target, injector, port) + + target.once("close", on_close) diff --git a/src/browser/remote_injector/injector.py b/src/browser/remote_injector/injector.py new file mode 100644 index 0000000..aec7e29 --- /dev/null +++ b/src/browser/remote_injector/injector.py @@ -0,0 +1,52 @@ +import inspect +from typing import Callable, Iterator + + +class Injector: + _classes: dict[str, dict[str, Callable]] + _objects: dict[str, Callable] + + def __init__(self) -> None: + self._classes = {"_": {}} + self._objects = {} + + def __getitem__(self, index: str) -> Callable: + return self._objects[index] + + def __len__(self) -> int: + return len(self._classes) + + def __iter__(self) -> Iterator[Callable]: + return iter(self._classes.items()) + + def __contains__(self, item: str) -> bool: + return item in self._objects + + def __str__(self) -> str: + return str(dict(self._classes)) + + def _flatten( + self, classes: dict, parent_key: str = "" + ) -> dict[str, Callable]: + items = [] + for key, value in classes.items(): + new_key = f"{parent_key}.{key}" if parent_key else key + if isinstance(value, dict): + items.extend(self._flatten(value, new_key).items()) + else: + items.append((new_key, value)) + return dict(items) + + def register(self, obj: Callable) -> Callable: + if inspect.isclass(obj): + self._classes.setdefault(obj.__name__, {}) + for name, member in obj.__dict__.items(): + if isinstance(member, staticmethod): + self._classes[obj.__name__][name] = member + self._objects[f"{obj.__name__}.{name}"] = self._classes[ + obj.__name__ + ][name] + elif inspect.isfunction(obj): + self._classes["_"][obj.__name__] = obj + self._objects[obj.__name__] = self._classes["_"][obj.__name__] + return obj diff --git a/src/browser/remote_injector/server.py b/src/browser/remote_injector/server.py new file mode 100644 index 0000000..18e205c --- /dev/null +++ b/src/browser/remote_injector/server.py @@ -0,0 +1,64 @@ +from fastapi import Body, FastAPI +from fastapi.encoders import jsonable_encoder +from fastapi.middleware.cors import CORSMiddleware + +from .injector import Injector + + +class RemoteInvokeServer: + app: FastAPI + injector: Injector + + def __init__( + self, app: FastAPI | None = None, injector: Injector | None = None + ) -> None: + if not app: + app = FastAPI(docs_url=None, redoc_url=None) + if not injector: + injector = Injector() + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + self.app = app + self.injector = injector + self.register_routers() + + def register_routers(self): + @self.app.post("/invoke/{callable_}") + async def invoke_function( + callable_: str, + args: list | None = Body(None), + kwargs: dict | None = Body(None), + ): + if callable_ in self.injector: + try: + if not args: + args = [] + if not kwargs: + kwargs = {} + return jsonable_encoder( + { + "status": "success", + "result": self.injector[callable_]( + *args, **kwargs + ), + } + ) + except Exception as e: # pylint: disable = W0718 + return jsonable_encoder( + { + "status": "fail", + "detail": f"{type(e).__name__}: {str(e)}", + } + ) + else: + return jsonable_encoder( + { + "status": "fail", + "detail": "not found", + } + ) diff --git a/src/browser/remote_injector/template/invokeRemoteFunction.js b/src/browser/remote_injector/template/invokeRemoteFunction.js new file mode 100644 index 0000000..1ebcb8f --- /dev/null +++ b/src/browser/remote_injector/template/invokeRemoteFunction.js @@ -0,0 +1,27 @@ +function invokeRemoteFunction(func, args = null, kwargs = null) { + var xhr = new XMLHttpRequest(); + xhr.open("POST", "http://localhost:{{port}}/invoke/" + func, false); + xhr.setRequestHeader("Accept", "application/json"); + xhr.setRequestHeader("Content-Type", "application/json"); + let body = {}; + if (args) { + body["args"] = args; + } + if (kwargs) { + body["kwargs"] = kwargs; + } + xhr.send(JSON.stringify(body)); + + if (xhr.status >= 200 && xhr.status < 300) { + const result = JSON.parse(xhr.responseText); + if (result.status == "success") { + return result; + } else if (result.status == "fail") { + throw new Error(result.detail); + } else { + throw new Error("Unknown status") + } + } else { + throw new Error("Request failed with status " + xhr.status); + } +} \ No newline at end of file diff --git a/src/custom/channel.py b/src/custom/channel.py new file mode 100644 index 0000000..ca81f64 --- /dev/null +++ b/src/custom/channel.py @@ -0,0 +1,28 @@ +from typing import Any, Optional + +from pydantic import BaseModel, Field + + +class Channel(BaseModel): + id: str = Field(..., alias="ChannelID") + name: str = Field(..., alias="ChannelName") + user_id: int = Field(..., alias="UserChannelID") + url: str = Field(..., alias="ChannelURL") + timeshift: int = Field(..., alias="TimeShift") + sdp: str = Field(..., alias="ChannelSDP") + timeshift_url: Optional[str] = Field(None, alias="TimeShiftURL") + log_url: Optional[str] = Field(None, alias="ChannelLogURL") + logo_url: Optional[str] = Field(None, alias="ChannelLogoURL") + position_x: int = Field(..., alias="PositionX") + position_y: int = Field(..., alias="PositionY") + begin_time: int = Field(..., alias="BeginTime") + interval: int = Field(..., alias="Interval") + lasting: int = Field(..., alias="Lasting") + type_: int = Field(..., alias="ChannelType") + purchased: Optional[Any] = Field(None, alias="ChannelPurchased") + timeshift_length: Optional[int] = Field(None, alias="TimeShiftLength") + telecomcode: Optional[str] = Field(None) + fcc_enable: Optional[int] = Field(None, alias="FCCEnable") + fcc_function: Optional[int] = Field(None, alias="FCCFunction") + fcc_ip: Optional[str] = Field(None, alias="ChannelFCCIP") + fcc_port: Optional[int] = Field(None, alias="ChannelFCCPort") diff --git a/src/custom/config.py b/src/custom/config.py new file mode 100644 index 0000000..d0e44d1 --- /dev/null +++ b/src/custom/config.py @@ -0,0 +1,7 @@ +from pathlib import Path + +from config_parser import parse +from jsondb import JsonDB + +CONFIG = parse("config.yaml") +context_data = JsonDB("db.json") diff --git a/src/custom/config_parser.py b/src/custom/config_parser.py new file mode 100644 index 0000000..5460ee0 --- /dev/null +++ b/src/custom/config_parser.py @@ -0,0 +1,10 @@ +import yaml +from jinja2 import Environment + + +def parse(path: str): + with open(path, "r", encoding="utf-8") as file: + content = file.read() + return yaml.safe_load( + Environment().from_string(content).render(yaml.safe_load(content)) + ) diff --git a/src/custom/injection.py b/src/custom/injection.py new file mode 100644 index 0000000..c0d892d --- /dev/null +++ b/src/custom/injection.py @@ -0,0 +1,85 @@ +from typing import Any + +from browser import remote_injector +from config import CONFIG, context_data +from channel import Channel +from epg import Authenticator, AuthMethod, Credential + +injector = remote_injector.Injector() + + +def _channel_parser(content: str): + result = {} + for item in content.replace('"', "").split(","): + key, value = item.strip().split("=", 1) + result[key] = ( + int(value) + if key + in [ + "UserChannelID", + "TimeShift", + "PositionX", + "PositionY", + "BeginTime", + "Interval", + "Lasting", + "ChannelType", + "TimeShiftLength", + "FCCEnable", + "FCCFunction", + "ChannelFCCPort", + ] + else value + ) + return result + + +@injector.register +class Authentication: + @staticmethod + def CTCGetAuthInfo(token: str) -> str: + credential_config = CONFIG["epg"]["credential"] + credential_kwargs = { + "token": token, + "user_id": credential_config["user_id"], + "password": credential_config["password"], + "ip": credential_config["ip"], + "mac": credential_config["mac"], + "product_id": credential_config["product_id"], + } + if "ctc" in credential_config: + credential_kwargs["ctc"] = credential_config["ctc"] + + authenticator_kwargs = {"credential": Credential(**credential_kwargs)} + if "authenticator" in CONFIG["epg"]: + authenticator_config = CONFIG["epg"]["authenticator"] + if "auth_method" in authenticator_config: + authenticator_kwargs["auth_method"] = AuthMethod[ + authenticator_config["auth_method"] + ] + if "salt" in authenticator_config: + authenticator_kwargs["salt"] = authenticator_config["salt"] + + return Authenticator(**authenticator_kwargs).info + + @staticmethod + def CTCGetConfig(key: str): + return context_data.get(key, "") + + @staticmethod + def CTCSetConfig(key: str, value: Any): + if key == "Channel": + if key not in context_data: + context_data[key] = [] + channel = _channel_parser(value) + # context_data[key].append(Channel.model_validate(channel)) + context_data[key].append( + Channel.model_validate(channel).model_dump() + ) + else: + context_data[key] = value + + @staticmethod + def CTCStartUpdate(): + # print("Update requested.") + pass diff --git a/src/custom/jsondb/__init__.py b/src/custom/jsondb/__init__.py new file mode 100644 index 0000000..dd93fdd --- /dev/null +++ b/src/custom/jsondb/__init__.py @@ -0,0 +1 @@ +from .database import JsonDB \ No newline at end of file diff --git a/src/custom/jsondb/database.py b/src/custom/jsondb/database.py new file mode 100644 index 0000000..a773908 --- /dev/null +++ b/src/custom/jsondb/database.py @@ -0,0 +1,54 @@ +import json +from collections.abc import MutableMapping +from io import TextIOWrapper +from typing import Any, Iterable + + +class JsonDB(MutableMapping): + _file: TextIOWrapper + _data: dict + + def __getitem__(self, key) -> Any: + return self._data[key] + + def __setitem__(self, key, value) -> None: + self._data[key] = value + + def __delitem__(self, key) -> None: + del self._data[key] + + def __iter__(self) -> Iterable: + return iter(self._data) + + def __len__(self) -> int: + return len(self._data) + + def __contains__(self, key) -> bool: + return key in self._data + + def __str__(self) -> str: + return str(self._data) + + def __repr__(self) -> str: + return repr(self._data) + + def __init__(self, path: str) -> None: + try: + self._file = open(path, "r+", encoding="utf-8") + self._data = json.load(self._file) + except FileNotFoundError: + self._file = open(path, "w+", encoding="utf-8") + self._data = {} + + def __del__(self) -> None: + try: + self._file.seek(0) + json.dump( + self._data, + self._file, + ensure_ascii=False, + indent=4, + ) + self._file.truncate() + finally: + self._file.close() diff --git a/src/custom/main.py b/src/custom/main.py new file mode 100644 index 0000000..8480f75 --- /dev/null +++ b/src/custom/main.py @@ -0,0 +1,28 @@ +import asyncio + +import browser +import browser.test +import injection +from config import CONFIG, context_data + +BROWSER_CONFIG = CONFIG["browser"] + +if __name__ == "__main__": + + async def main(): + """ + Main method to test the IPTV authentication. + """ + await browser.process( + injector=injection.injector, + start_url=BROWSER_CONFIG["start_url"], + end_url=BROWSER_CONFIG["end_url"], + args=BROWSER_CONFIG["args"], + headers=BROWSER_CONFIG["headers"], + headless=False, + ) + context_data["Channel"] = sorted( + context_data["Channel"], key=lambda x: x["user_id"] + ) + + asyncio.run(main()) diff --git a/src/epg/__init__.py b/src/epg/__init__.py new file mode 100644 index 0000000..fb7b94a --- /dev/null +++ b/src/epg/__init__.py @@ -0,0 +1,2 @@ +from .authenticator import Authenticator, AuthMethod +from .credential import Credential