diff --git a/gallery/__init__.py b/gallery/__init__.py index 31bb90e..90e07cd 100644 --- a/gallery/__init__.py +++ b/gallery/__init__.py @@ -10,7 +10,7 @@ caches.set_config( { "default": { "cache": "aiocache.SimpleMemoryCache", - "serializer": {"class": "aiocache.serializers.StringSerializer"}, + "serializer": {"class": "aiocache.serializers.PickleSerializer"}, }, "redis": { "cache": "aiocache.RedisCache", diff --git a/gallery/main.py b/gallery/main.py index 5d62dfb..8c3e969 100644 --- a/gallery/main.py +++ b/gallery/main.py @@ -7,6 +7,7 @@ from gallery.easel import build_app from gallery.painting.gismeteo.api import GismeteoApi from gallery.painting.matchtv.api import MatchTvApi from gallery.painting.openweather.api import OpenWeatherApi +from gallery.painting.rp5.api import RP5WeatherApi from gallery.painting.yandextv.api import YandexTvApi from gallery.painting.yandexweather.api import YandexWeatherApi from gallery.sketch.bundle import ApiBundle @@ -20,6 +21,7 @@ api = ApiBundle( CachedWeatherApi(GismeteoApi()), CachedWeatherApi(OpenWeatherApi()), CachedWeatherApi(YandexWeatherApi()), + CachedWeatherApi(RP5WeatherApi()), ] ) app = build_app(api) diff --git a/gallery/painting/rp5/__init__.py b/gallery/painting/rp5/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gallery/painting/rp5/api.py b/gallery/painting/rp5/api.py new file mode 100644 index 0000000..759193f --- /dev/null +++ b/gallery/painting/rp5/api.py @@ -0,0 +1,103 @@ +import datetime +import logging + +import aiohttp +from aiocache import cached +from bs4 import BeautifulSoup + +from gallery.sketch.source import ApiSource +from gallery.sketch.weather.api import WeatherApi +from gallery.sketch.weather.model import ( + Location, + WeatherResponse, +) +from gallery.sketch.weather.util import merge_weather_values +from gallery.util import TimeUnit + +from .parser import parse_page + +logger = logging.getLogger("rp5") + +HEADERS: dict[str, str] = { + "Accept": "application/json, text/javascript, */*; q=0.01", + "Accept-Encoding": "gzip, deflate, br, zstd", + "Accept-Language": "en-US,en;q=0.9,ru-RU;q=0.8,ru;q=0.7,uk;q=0.6", + "Connection": "keep-alive", + "Host": "rp5.ru", + "Referer": "https://rp5.ru/", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + "User-Agent": ApiSource.DEFAULT_USER_AGENT, + "X-Requested-With": "XMLHttpRequest", + "sec-ch-ua": '"Not;A=Brand";v="8", "Chromium";v="150", "Google Chrome";v="150"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Linux"', +} + + +class RP5WeatherApi(WeatherApi): + PROVIDER = "rp5" + + def _build_source(self) -> ApiSource: + return ApiSource( + "https://rp5.ru", + headers=HEADERS, + ) + + async def find_locations(self, query: str) -> list[Location]: + endpoint = "docs/search/ru" + data = aiohttp.FormData({"searchStr": query}) + response = await self.source.request(endpoint, method="POST", data=data) + soup = BeautifulSoup(response, features="html.parser") + table = soup.select_one(".searchResults") + result = [] + for row in table.select("tr")[1:]: + cells = row.select("td") + name = cells[0].text.strip() + location_id = cells[0].select_one("a").attrs["href"].lstrip("/") + country = cells[1].text.strip() + district = cells[2].text.strip() + subdistrict = cells[3].text.strip() + result.append( + Location( + id=location_id, + name=name, + provider=self.provider, + lat=0, + lon=0, + country=country, + country_code="ru", # TODO: contry_code from country + district=district, + subdistrict=subdistrict, + ) + ) + return result + + @cached( + key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast", + alias="redis", + ttl=TimeUnit.HOUR, + ) + async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]: + endpoint = f"{location_id}" + content = await self.source.request(endpoint) + soup = BeautifulSoup(content, features="html.parser") + return parse_page(soup) + + async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse: + forecast = await self._get_location_forecast(location_id) + for response in forecast: + if response.date == date: + return response + raise ValueError(f"Unsupported date: {date}") + + async def get_days(self, location_id: str, days: int) -> WeatherResponse: + forecast = await self._get_location_forecast(location_id) + values = [merge_weather_values(item.date, item.values) for item in forecast] + return WeatherResponse( + location=forecast[0].location, + date=datetime.date.today(), + period="days", + values=values, + ) diff --git a/gallery/painting/rp5/parser.py b/gallery/painting/rp5/parser.py new file mode 100644 index 0000000..40ed803 --- /dev/null +++ b/gallery/painting/rp5/parser.py @@ -0,0 +1,108 @@ +import re + +import dateparser +from bs4 import BeautifulSoup, Tag + +from gallery.sketch.weather.model import ( + Cloudness, + Precipitation, + WeatherResponse, + WeatherValue, +) +from gallery.sketch.weather.parser import ParseError, parse_wind_direction +from gallery.sketch.weather.util import build_weather_value + +CLOUDNESS_MAP = { + "Ясно": Cloudness.CLEAR, + "Малооблачно": Cloudness.PARTLY_CLOUDY, + "Облачно с прояснениями": Cloudness.PARTLY_CLOUDY, + "Переменная облачность": Cloudness.PARTLY_CLOUDY, + "Небольшая облачность": Cloudness.CLOUDY, + "Облачно": Cloudness.CLOUDY, + "Значительная облачность": Cloudness.MAINLY_CLOUDY, + "Пасмурная погода": Cloudness.MAINLY_CLOUDY, +} + +PRECIPITATION_MAP = { + "Явления погоды отсутствуют": Precipitation.NO, + "Cлабый дождь": Precipitation.SMALL_RAIN, +} + + +def _parse_wind_speed(cell: Tag) -> int: + element = cell.select_one(".wv_0") + if element: + return int(element.text.strip() or "0") + return 0 + + +# pylint:disable=too-many-branches +def parse_page(page: BeautifulSoup) -> list[WeatherResponse]: + location = page.select(".intoLeftNavi > .verticalBottom > a")[-1].text.strip() + table = page.select_one("#forecastTable") + if table is None: + raise ParseError("#forecastTable not found") + ( + date_row, + time_row, + cloudness_row, + precipitation_row, + temperature_row, + _, + pressure_row, + wind_speed_row, + wind_gust_row, + wind_direction_row, + humidity_row, + *_, + ) = table.select("tr") + result: list[WeatherResponse] = [] + values: list[WeatherValue] = [] + for index, cell in enumerate(date_row.select("td")): + colspan = int(cell.attrs["colspan"]) + if index == 0: + colspan -= 1 + date_str = ",".join(cell.text.strip().split(",")[-2:]) + date = dateparser.parse(date_str, languages=["ru"]) + if date is None: + raise ParseError.invalid_date(date_str) + item_values = [build_weather_value(date) for _ in range(colspan)] + result.append( + WeatherResponse( + location=location, + date=date.date(), + period="day", + values=item_values, + ) + ) + values += item_values + for index, cell in enumerate(time_row.select("td")[1:]): + hour = int(cell.text.strip()) + values[index].date = values[index].date.replace(hour=hour) + for index, cell in enumerate(cloudness_row.select("td")[1:]): + onmouseover = cell.select_one(".cc_0").next_element.attrs["onmouseover"] + cloudness_str = re.compile("(.*?)").search(onmouseover).group(1).strip() + values[index].sky.cloudness = CLOUDNESS_MAP[cloudness_str] + for index, cell in enumerate(precipitation_row.select("td")[1:]): + onmouseover = cell.select_one(".pr_0").attrs["onmouseover"] + precipitation_str = onmouseover.split(",")[1].strip(" '") + precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str) + if precipitation_value_matched: + # one hour value + values[index].precipitation = float(precipitation_value_matched.group(1)) * 6 + if "(" in precipitation_str: + precipitation_str = precipitation_str.split(" (")[0] + values[index].sky.precipitation = PRECIPITATION_MAP[precipitation_str] + for index, cell in enumerate(temperature_row.select("td")[1:]): + values[index].temperature = [int(cell.select_one(".t_0").text.lstrip("+").strip())] + for index, cell in enumerate(pressure_row.select("td")[1:]): + values[index].pressure = [int(cell.select_one(".p_0").text.strip())] + for index, cell in enumerate(wind_speed_row.select("td")[1:]): + values[index].wind_speed = _parse_wind_speed(cell) + for index, cell in enumerate(wind_gust_row.select("td")[1:]): + values[index].wind_speed = _parse_wind_speed(cell) + for index, cell in enumerate(wind_direction_row.select("td")[1:]): + values[index].wind_direction = parse_wind_direction(cell.text.strip()) + for index, cell in enumerate(humidity_row.select("td")[1:]): + values[index].humidity = int(cell.text.strip()) + return result diff --git a/gallery/sketch/source.py b/gallery/sketch/source.py index cf6b87b..54345f2 100644 --- a/gallery/sketch/source.py +++ b/gallery/sketch/source.py @@ -1,5 +1,6 @@ +import datetime import logging -from typing import Literal +from typing import Any, Literal import aiohttp from playwright.async_api import async_playwright @@ -9,6 +10,7 @@ logger = logging.getLogger("source") RequestMode = Literal["base", "browser"] DEFAULT_REQUEST_MODE: RequestMode = "base" +DEFAULT_METHOD = "GET" class ApiSource: @@ -31,30 +33,47 @@ class ApiSource: self._cookies = cookies self._headers = headers - async def _base_request(self, url: str) -> str: + async def _base_request( + self, + url: str, + method: str = DEFAULT_METHOD, + data: Any | None = None, + ) -> str: headers = {"User-Agent": self._user_agent, **(self._headers or {})} async with aiohttp.ClientSession( headers=headers, cookies=self._cookies, raise_for_status=True, ) as session: - async with session.request("GET", url, timeout=self._timeout) as response: + async with session.request(method, url, timeout=self._timeout, data=data) as response: return await response.text() - async def _browser_request(self, url: str) -> str: + async def _browser_request( + self, + url: str, + method: str = DEFAULT_METHOD, + data: Any | None = None, + ) -> str: async with async_playwright() as p: browser = await p.chromium.launch() page = await browser.new_page() - await page.goto(url, wait_until="networkidle") + await page.goto(url, timeout=datetime.timedelta(seconds=self._timeout), wait_until="networkidle") content = await page.content() await browser.close() return content - async def request(self, endpoint: str, /, mode: RequestMode = DEFAULT_REQUEST_MODE) -> str: + async def request( + self, + endpoint: str, + /, + mode: RequestMode = DEFAULT_REQUEST_MODE, + method: str = DEFAULT_METHOD, + data: Any | None = None, + ) -> str: if endpoint.startswith("https:"): url = endpoint else: url = f"{self._base_url}/{endpoint}" logger.info(url) - method = getattr(self, f"_{mode}_request") - return await method(url) + fun = getattr(self, f"_{mode}_request") + return await fun(url, method, data) diff --git a/gallery/sketch/weather/parser.py b/gallery/sketch/weather/parser.py index 708915e..e67c87c 100644 --- a/gallery/sketch/weather/parser.py +++ b/gallery/sketch/weather/parser.py @@ -13,6 +13,7 @@ WIND_DIRECTION_MAP: dict[str, WindDirection] = { "—": WindDirection.CALM, "": WindDirection.CALM, "штиль": WindDirection.CALM, + "штл": WindDirection.CALM, "с": WindDirection.N, "св": WindDirection.NE, "в": WindDirection.E, @@ -25,4 +26,4 @@ WIND_DIRECTION_MAP: dict[str, WindDirection] = { def parse_wind_direction(value: str) -> WindDirectionDeg: - return WindDirectionDeg.from_direction(WIND_DIRECTION_MAP[value.lower()]) + return WindDirectionDeg.from_direction(WIND_DIRECTION_MAP[value.replace("-", "").lower()]) diff --git a/gallery/sketch/weather/util.py b/gallery/sketch/weather/util.py index 71a7749..f78e93f 100644 --- a/gallery/sketch/weather/util.py +++ b/gallery/sketch/weather/util.py @@ -27,7 +27,13 @@ def build_weather_value(date: datetime.datetime) -> WeatherValue: def _merge_values(values: list[int], mode: MergeMode) -> list[int]: - return [max(values), min(values)] if mode == "interval" else [round(sum(values) / len(values))] + if mode == "interval": + if (max_value := max(values)) == (min_value := min(values)): + return [max_value] + else: + return [max_value, min_value] + else: + return [round(sum(values) / len(values))] def merge_weather_values( diff --git a/tests/common/mock.py b/tests/common/mock.py index a3b0c14..ad78b7c 100644 --- a/tests/common/mock.py +++ b/tests/common/mock.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import TypeVar from gallery.sketch.api import Api, S -from gallery.sketch.source import DEFAULT_REQUEST_MODE, ApiSource, RequestMode +from gallery.sketch.source import ApiSource class MockSource(ApiSource): @@ -12,7 +12,7 @@ class MockSource(ApiSource): self._path = path self._mapping = mapping - async def request(self, endpoint: str, /, mode: RequestMode = DEFAULT_REQUEST_MODE) -> str: + async def request(self, endpoint: str, /, **kwargs) -> str: for pattern, filename in self._mapping.items(): if pattern in endpoint: return (self._path / filename).read_text() diff --git a/tests/conftest.py b/tests/conftest.py index e85daff..79d211a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,20 @@ from os import environ +from aiocache import caches + +import gallery # pylint:disable=unused-import + environ.setdefault("OPENWEATHER_KEY", "test") + +caches.set_config( + { + "default": { + "cache": "aiocache.SimpleMemoryCache", + "serializer": {"class": "aiocache.serializers.PickleSerializer"}, + }, + "redis": { + "cache": "aiocache.SimpleMemoryCache", + "serializer": {"class": "aiocache.serializers.PickleSerializer"}, + }, + } +) diff --git a/tests/data/rp5/__init__.py b/tests/data/rp5/__init__.py new file mode 100644 index 0000000..d3d151d --- /dev/null +++ b/tests/data/rp5/__init__.py @@ -0,0 +1,11 @@ +from pathlib import Path + +from tests.common.mock import MockSource + +RP5_MOCK_SOURCE = MockSource( + Path(__file__).parent, + { + "search": "search.html", + "test": "weather.html", + }, +) diff --git a/tests/data/rp5/search.html b/tests/data/rp5/search.html new file mode 100644 index 0000000..fd6cd19 --- /dev/null +++ b/tests/data/rp5/search.html @@ -0,0 +1,379 @@ + +Результат поиска + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
+
+ +
+
+ +
+
+
+ +

Результат поиска

Найдено населенных пунктов: 3

Населенный пункт
Страна
Регион
Район
Змиевка Россия Луганская Народная Республика Сватовский район
Змиевка Россия Орловская область Свердловский район
Змиевка Россия Херсонская область Бериславский район
+
+
+
+ +
+
+ + + + + + + \ No newline at end of file diff --git a/tests/data/rp5/weather.html b/tests/data/rp5/weather.html new file mode 100644 index 0000000..0866690 --- /dev/null +++ b/tests/data/rp5/weather.html @@ -0,0 +1,592 @@ + + + + + +Погода в Орле, Орловская область - РП5 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
+
+ +
+
+ +
+
+
+ +
+
+ + + +
+ + + + + + + + +
+ + + ru +
+ расписание
погоды +
+ 		- это сайт, с которого распечатан данный прогноз. Чтобы узнать больше, посетите сайт.
+
+
+
+
+
Завтра ожидается +23..+16 °C, без осадков, слабый ветер. Послезавтра: +14..+22 °C, cлабый дождь, легкий ветер.
7 суток
Завтра
Чт
Пт
Сб
Вс
Пн
Вт
Завтра, ср, 2 сентября
Чт, 3 сентября
Пт, 4 сентября
Сб, 5 сентября
Вс, 6 сентября
Пн, 7 сентября
Вт, 8 сентября
Местное время 030915210309152103091521030915210309152103091521030915
Облачность,
%
Осадки, мм
Температура,
°C
+15
+17
+23
+14
+11
+16
+22
+14
+10
+16
+23
+14
+14
+15
+19
+13
+12
+13
+19
+11
+10
+13
+18
+10
+7
+11
+18
Ощущается как,
°C
+8
Давление,
мм рт. ст.
748
748
747
748
748
749
748
748
747
748
746
744
741
735
736
738
739
742
742
743
744
746
747
748
749
751
750
Ветер: скорость,
м/с
1
3
4
1
1
2
3
1
1
2
3
2
3
6
6
4
4
4
3
3
3
5
5
1
1
2
3
порывы,
м/с
7
10
5
9
8
6
13
14
9
10
8
10
6
7
10
12
5
6
направление Ю-ЗЮ-ЗЗЮ-ЗЮ-ЗЮ-ЗЗЮ-ЗЮ-ЗЮ-ЗЮ-ЗЮЮЮЗЮ-ЗЗЗЗЗЗС-ЗС-ЗС-ЗС-ЗС-ЗС-З
Развернуть таблицу (Влажность. Солнце, Луна: восход, заход, фаза)
больше информации о погоде
на сайте rp5.ru
+
+
Новости погоды
+
+ За последние 12 часов максимальная температура воздуха наблюдалась в Ливенском районе в Ливнах: +28 oC + . +
+
+
+
+
Развернуть текст
+
+
+
+
+ +
+
+
+
+ +
+
+
+ +
+
+
+
0
+
+
+
+
+ + + + + + diff --git a/tests/test_rp5.py b/tests/test_rp5.py new file mode 100644 index 0000000..1f33389 --- /dev/null +++ b/tests/test_rp5.py @@ -0,0 +1,27 @@ +import datetime + +import pytest + +from gallery.painting.rp5.api import RP5WeatherApi +from tests.common.mock import create_mock_api +from tests.data.rp5 import RP5_MOCK_SOURCE + + +@pytest.fixture(name="rp5_api", scope="module") +def rp5_api_fixture() -> RP5WeatherApi: + return create_mock_api(RP5WeatherApi, RP5_MOCK_SOURCE) + + +async def test_search(rp5_api: RP5WeatherApi): + result = await rp5_api.find_locations("test") + assert len(result) == 3 + + +async def test_day(rp5_api: RP5WeatherApi): + result = await rp5_api.get_day("test", datetime.date(2026, 9, 2)) + assert len(result.values) == 4 + + +async def test_days(rp5_api: RP5WeatherApi): + result = await rp5_api.get_days("test", 10) + assert len(result.values) == 7