61 lines
1.9 KiB
Python
61 lines
1.9 KiB
Python
import logging
|
|
from typing import Literal
|
|
|
|
import aiohttp
|
|
from playwright.async_api import async_playwright
|
|
|
|
logger = logging.getLogger("source")
|
|
|
|
RequestMode = Literal["base", "browser"]
|
|
|
|
DEFAULT_REQUEST_MODE: RequestMode = "base"
|
|
|
|
|
|
class ApiSource:
|
|
DEFAULT_USER_AGENT = (
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
|
)
|
|
DEFAULT_TIMEOUT = 30.0
|
|
|
|
def __init__(
|
|
self,
|
|
base_url,
|
|
user_agent: str = DEFAULT_USER_AGENT,
|
|
timeout: float = DEFAULT_TIMEOUT,
|
|
cookies: dict[str, str] | None = None,
|
|
headers: dict[str, str] | None = None,
|
|
):
|
|
self._base_url = base_url
|
|
self._user_agent = user_agent
|
|
self._timeout = timeout
|
|
self._cookies = cookies
|
|
self._headers = headers
|
|
|
|
async def _base_request(self, url: str) -> str:
|
|
headers = {"User-Agent": self._user_agent, **(self._headers or {})}
|
|
async with aiohttp.ClientSession(
|
|
headers=headers,
|
|
cookies=self._cookies,
|
|
raise_for_status=True,
|
|
) as session:
|
|
async with session.request("GET", url, timeout=self._timeout) as response:
|
|
return await response.text()
|
|
|
|
async def _browser_request(self, url: str) -> str:
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch()
|
|
page = await browser.new_page()
|
|
await page.goto(url, wait_until="networkidle")
|
|
content = await page.content()
|
|
await browser.close()
|
|
return content
|
|
|
|
async def request(self, endpoint: str, /, mode: RequestMode = DEFAULT_REQUEST_MODE) -> str:
|
|
if endpoint.startswith("https:"):
|
|
url = endpoint
|
|
else:
|
|
url = f"{self._base_url}/{endpoint}"
|
|
logger.info(url)
|
|
method = getattr(self, f"_{mode}_request")
|
|
return await method(url)
|