diff --git a/gallery/main.py b/gallery/main.py index 5d62dfb..8c3e969 100644 --- a/gallery/main.py +++ b/gallery/main.py @@ -7,6 +7,7 @@ from gallery.easel import build_app from gallery.painting.gismeteo.api import GismeteoApi from gallery.painting.matchtv.api import MatchTvApi from gallery.painting.openweather.api import OpenWeatherApi +from gallery.painting.rp5.api import RP5WeatherApi from gallery.painting.yandextv.api import YandexTvApi from gallery.painting.yandexweather.api import YandexWeatherApi from gallery.sketch.bundle import ApiBundle @@ -20,6 +21,7 @@ api = ApiBundle( CachedWeatherApi(GismeteoApi()), CachedWeatherApi(OpenWeatherApi()), CachedWeatherApi(YandexWeatherApi()), + CachedWeatherApi(RP5WeatherApi()), ] ) app = build_app(api) diff --git a/gallery/painting/rp5/__init__.py b/gallery/painting/rp5/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gallery/painting/rp5/api.py b/gallery/painting/rp5/api.py new file mode 100644 index 0000000..5a5e7d4 --- /dev/null +++ b/gallery/painting/rp5/api.py @@ -0,0 +1,103 @@ +import datetime +import logging + +import aiohttp +from aiocache import cached +from bs4 import BeautifulSoup + +from gallery.sketch.source import ApiSource +from gallery.sketch.weather.api import WeatherApi +from gallery.sketch.weather.model import Location, WeatherResponse +from gallery.sketch.weather.util import merge_weather_values +from gallery.util import TimeUnit + +logger = logging.getLogger("rp5") + +HEADERS: dict[str, str] = { + "Accept": "application/json, text/javascript, */*; q=0.01", + "Accept-Encoding": "gzip, deflate, br, zstd", + "Accept-Language": "en-US,en;q=0.9,ru-RU;q=0.8,ru;q=0.7,uk;q=0.6", + "Connection": "keep-alive", + "Host": "rp5.ru", + "Referer": "https://rp5.ru/", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + "User-Agent": ApiSource.DEFAULT_USER_AGENT, + "X-Requested-With": "XMLHttpRequest", + "sec-ch-ua": '"Not;A=Brand";v="8", "Chromium";v="150", "Google Chrome";v="150"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Linux"', +} + + +class RP5WeatherApi(WeatherApi): + PROVIDER = "rp5" + + def _build_source(self) -> ApiSource: + return ApiSource( + "https://rp5.ru", + headers=HEADERS, + ) + + async def find_locations(self, query: str) -> list[Location]: + endpoint = "docs/search/ru" + data = aiohttp.FormData({"searchStr": query}) + response = await self.source.request(endpoint, method="POST", data=data) + soup = BeautifulSoup(response, features="html.parser") + table = soup.select_one(".searchResults") + result = [] + for row in table.select("tr")[1:]: + cells = row.select("td") + name = cells[0].text.strip() + location_id = cells[0].select_one("a").attrs["href"].lstrip("/") + country = cells[1].text.strip() + district = cells[2].text.strip() + subdistrict = cells[3].text.strip() + result.append( + Location( + id=location_id, + name=name, + provider=self.provider, + lat=0, + lon=0, + country=country, + country_code="ru", # TODO: contry_code from country + district=district, + subdistrict=subdistrict, + ) + ) + return result + + @cached( + key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast", + alias="redis", + ttl=TimeUnit.HOUR, + ) + async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]: + endpoint = f"{location_id}" + content = await self.source.request(endpoint) + soup = BeautifulSoup(content, features="html.parser") + table = soup.select_one("#forecastTable") + print("TABLE", table) + result = [] + rows = table.select("tr") + # TODO: implement + return result + + async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse: + forecast = await self._get_location_forecast(location_id) + for response in forecast: + if response.date == date: + return response + raise ValueError(f"Unsupported date: {date}") + + async def get_days(self, location_id: str, days: int) -> WeatherResponse: + forecast = await self._get_location_forecast(location_id) + values = [merge_weather_values(item.date, item.values) for item in forecast] + return WeatherResponse( + location=forecast[0].location, + date=datetime.date.today(), + period="days", + values=values, + ) diff --git a/gallery/sketch/source.py b/gallery/sketch/source.py index cf6b87b..90fa6a0 100644 --- a/gallery/sketch/source.py +++ b/gallery/sketch/source.py @@ -1,5 +1,5 @@ import logging -from typing import Literal +from typing import Any, Literal import aiohttp from playwright.async_api import async_playwright @@ -9,6 +9,7 @@ logger = logging.getLogger("source") RequestMode = Literal["base", "browser"] DEFAULT_REQUEST_MODE: RequestMode = "base" +DEFAULT_METHOD = "GET" class ApiSource: @@ -31,30 +32,47 @@ class ApiSource: self._cookies = cookies self._headers = headers - async def _base_request(self, url: str) -> str: + async def _base_request( + self, + url: str, + method: str = DEFAULT_METHOD, + data: Any | None = None, + ) -> str: headers = {"User-Agent": self._user_agent, **(self._headers or {})} async with aiohttp.ClientSession( headers=headers, cookies=self._cookies, raise_for_status=True, ) as session: - async with session.request("GET", url, timeout=self._timeout) as response: + async with session.request(method, url, timeout=self._timeout, data=data) as response: return await response.text() - async def _browser_request(self, url: str) -> str: + async def _browser_request( + self, + url: str, + method: str = DEFAULT_METHOD, + data: Any | None = None, + ) -> str: async with async_playwright() as p: browser = await p.chromium.launch() page = await browser.new_page() - await page.goto(url, wait_until="networkidle") + await page.goto(url, timeout=self._timeout, wait_until="networkidle") content = await page.content() await browser.close() return content - async def request(self, endpoint: str, /, mode: RequestMode = DEFAULT_REQUEST_MODE) -> str: + async def request( + self, + endpoint: str, + /, + mode: RequestMode = DEFAULT_REQUEST_MODE, + method: str = DEFAULT_METHOD, + data: Any | None = None, + ) -> str: if endpoint.startswith("https:"): url = endpoint else: url = f"{self._base_url}/{endpoint}" logger.info(url) - method = getattr(self, f"_{mode}_request") - return await method(url) + fun = getattr(self, f"_{mode}_request") + return await fun(url, method, data) diff --git a/tests/common/mock.py b/tests/common/mock.py index a3b0c14..ad78b7c 100644 --- a/tests/common/mock.py +++ b/tests/common/mock.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import TypeVar from gallery.sketch.api import Api, S -from gallery.sketch.source import DEFAULT_REQUEST_MODE, ApiSource, RequestMode +from gallery.sketch.source import ApiSource class MockSource(ApiSource): @@ -12,7 +12,7 @@ class MockSource(ApiSource): self._path = path self._mapping = mapping - async def request(self, endpoint: str, /, mode: RequestMode = DEFAULT_REQUEST_MODE) -> str: + async def request(self, endpoint: str, /, **kwargs) -> str: for pattern, filename in self._mapping.items(): if pattern in endpoint: return (self._path / filename).read_text() diff --git a/tests/data/rp5/__init__.py b/tests/data/rp5/__init__.py new file mode 100644 index 0000000..d3d151d --- /dev/null +++ b/tests/data/rp5/__init__.py @@ -0,0 +1,11 @@ +from pathlib import Path + +from tests.common.mock import MockSource + +RP5_MOCK_SOURCE = MockSource( + Path(__file__).parent, + { + "search": "search.html", + "test": "weather.html", + }, +) diff --git a/tests/data/rp5/search.html b/tests/data/rp5/search.html new file mode 100644 index 0000000..fd6cd19 --- /dev/null +++ b/tests/data/rp5/search.html @@ -0,0 +1,379 @@ + +Результат поиска + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
+
+ +
+
+ +
+
+
+ +

Результат поиска

Найдено населенных пунктов: 3

Населенный пункт
Страна
Регион
Район
Змиевка Россия Луганская Народная Республика Сватовский район
Змиевка Россия Орловская область Свердловский район
Змиевка Россия Херсонская область Бериславский район
+
+
+
+ +
+
+ + + + + + + \ No newline at end of file diff --git a/tests/data/rp5/weather.html b/tests/data/rp5/weather.html new file mode 100644 index 0000000..0866690 --- /dev/null +++ b/tests/data/rp5/weather.html @@ -0,0 +1,592 @@ + + + + + +Погода в Орле, Орловская область - РП5 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
+
+ +
+
+ +
+
+
+ +
+
+ + + +
+ + + + + + + + +
+ + + ru +
+ расписание
погоды +
+ 		- это сайт, с которого распечатан данный прогноз. Чтобы узнать больше, посетите сайт.
+
+
+
+
+
Завтра ожидается +23..+16 °C, без осадков, слабый ветер. Послезавтра: +14..+22 °C, cлабый дождь, легкий ветер.
7 суток
Завтра
Чт
Пт
Сб
Вс
Пн
Вт
Завтра, ср, 2 сентября
Чт, 3 сентября
Пт, 4 сентября
Сб, 5 сентября
Вс, 6 сентября
Пн, 7 сентября
Вт, 8 сентября
Местное время 030915210309152103091521030915210309152103091521030915
Облачность,
%
Осадки, мм
Температура,
°C
+15
+17
+23
+14
+11
+16
+22
+14
+10
+16
+23
+14
+14
+15
+19
+13
+12
+13
+19
+11
+10
+13
+18
+10
+7
+11
+18
Ощущается как,
°C
+8
Давление,
мм рт. ст.
748
748
747
748
748
749
748
748
747
748
746
744
741
735
736
738
739
742
742
743
744
746
747
748
749
751
750
Ветер: скорость,
м/с
1
3
4
1
1
2
3
1
1
2
3
2
3
6
6
4
4
4
3
3
3
5
5
1
1
2
3
порывы,
м/с
7
10
5
9
8
6
13
14
9
10
8
10
6
7
10
12
5
6
направление Ю-ЗЮ-ЗЗЮ-ЗЮ-ЗЮ-ЗЗЮ-ЗЮ-ЗЮ-ЗЮ-ЗЮЮЮЗЮ-ЗЗЗЗЗЗС-ЗС-ЗС-ЗС-ЗС-ЗС-З
Развернуть таблицу (Влажность. Солнце, Луна: восход, заход, фаза)
больше информации о погоде
на сайте rp5.ru
+
+
Новости погоды
+
+ За последние 12 часов максимальная температура воздуха наблюдалась в Ливенском районе в Ливнах: +28 oC + . +
+
+
+
+
Развернуть текст
+
+
+
+
+ +
+
+
+
+ +
+
+
+ +
+
+
+
0
+
+
+
+
+ + + + + + diff --git a/tests/test_rp5.py b/tests/test_rp5.py new file mode 100644 index 0000000..310da31 --- /dev/null +++ b/tests/test_rp5.py @@ -0,0 +1,27 @@ +import datetime + +import pytest + +from gallery.painting.rp5.api import RP5WeatherApi +from tests.common.mock import create_mock_api +from tests.data.rp5 import RP5_MOCK_SOURCE + + +@pytest.fixture(name="rp5_api", scope="module") +def rp5_api_fixture() -> RP5WeatherApi: + return create_mock_api(RP5WeatherApi, RP5_MOCK_SOURCE) + + +async def test_search(rp5_api: RP5WeatherApi): + result = await rp5_api.find_locations("test") + assert len(result) == 3 + + +async def test_day(rp5_api: RP5WeatherApi): + result = await rp5_api.get_day("test", datetime.date.today()) + assert len(result.values) == 4 + + +async def test_days(rp5_api: RP5WeatherApi): + result = await rp5_api.get_days("test", 10) + assert len(result.values) == 10