feat(rp5): add rp5 api

gallery/__init__.py

@@ -10,7 +10,7 @@ caches.set_config(
     {
         "default": {
             "cache": "aiocache.SimpleMemoryCache",
-            "serializer": {"class": "aiocache.serializers.StringSerializer"},
+            "serializer": {"class": "aiocache.serializers.PickleSerializer"},
         },
         "redis": {
             "cache": "aiocache.RedisCache",

gallery/main.py

@@ -7,6 +7,7 @@ from gallery.easel import build_app
 from gallery.painting.gismeteo.api import GismeteoApi
 from gallery.painting.matchtv.api import MatchTvApi
 from gallery.painting.openweather.api import OpenWeatherApi
+from gallery.painting.rp5.api import RP5WeatherApi
 from gallery.painting.yandextv.api import YandexTvApi
 from gallery.painting.yandexweather.api import YandexWeatherApi
 from gallery.sketch.bundle import ApiBundle
@@ -20,6 +21,7 @@ api = ApiBundle(
         CachedWeatherApi(GismeteoApi()),
         CachedWeatherApi(OpenWeatherApi()),
         CachedWeatherApi(YandexWeatherApi()),
+        CachedWeatherApi(RP5WeatherApi()),
     ]
 )
 app = build_app(api)

gallery/painting/rp5/api.py

@@ -0,0 +1,103 @@
+import datetime
+import logging
+
+import aiohttp
+from aiocache import cached
+from bs4 import BeautifulSoup
+
+from gallery.sketch.source import ApiSource
+from gallery.sketch.weather.api import WeatherApi
+from gallery.sketch.weather.model import (
+    Location,
+    WeatherResponse,
+)
+from gallery.sketch.weather.util import merge_weather_values
+from gallery.util import TimeUnit
+
+from .parser import parse_page
+
+logger = logging.getLogger("rp5")
+
+HEADERS: dict[str, str] = {
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept-Encoding": "gzip, deflate, br, zstd",
+    "Accept-Language": "en-US,en;q=0.9,ru-RU;q=0.8,ru;q=0.7,uk;q=0.6",
+    "Connection": "keep-alive",
+    "Host": "rp5.ru",
+    "Referer": "https://rp5.ru/",
+    "Sec-Fetch-Dest": "empty",
+    "Sec-Fetch-Mode": "cors",
+    "Sec-Fetch-Site": "same-origin",
+    "User-Agent": ApiSource.DEFAULT_USER_AGENT,
+    "X-Requested-With": "XMLHttpRequest",
+    "sec-ch-ua": '"Not;A=Brand";v="8", "Chromium";v="150", "Google Chrome";v="150"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"Linux"',
+}
+
+
+class RP5WeatherApi(WeatherApi):
+    PROVIDER = "rp5"
+
+    def _build_source(self) -> ApiSource:
+        return ApiSource(
+            "https://rp5.ru",
+            headers=HEADERS,
+        )
+
+    async def find_locations(self, query: str) -> list[Location]:
+        endpoint = "docs/search/ru"
+        data = aiohttp.FormData({"searchStr": query})
+        response = await self.source.request(endpoint, method="POST", data=data)
+        soup = BeautifulSoup(response, features="html.parser")
+        table = soup.select_one(".searchResults")
+        result = []
+        for row in table.select("tr")[1:]:
+            cells = row.select("td")
+            name = cells[0].text.strip()
+            location_id = cells[0].select_one("a").attrs["href"].lstrip("/")
+            country = cells[1].text.strip()
+            district = cells[2].text.strip()
+            subdistrict = cells[3].text.strip()
+            result.append(
+                Location(
+                    id=location_id,
+                    name=name,
+                    provider=self.provider,
+                    lat=0,
+                    lon=0,
+                    country=country,
+                    country_code="ru",  # TODO: contry_code from country
+                    district=district,
+                    subdistrict=subdistrict,
+                )
+            )
+        return result
+
+    @cached(
+        key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast",
+        alias="redis",
+        ttl=TimeUnit.HOUR,
+    )
+    async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]:
+        endpoint = f"{location_id}"
+        content = await self.source.request(endpoint)
+        soup = BeautifulSoup(content, features="html.parser")
+        return parse_page(soup)
+
+    async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
+        forecast = await self._get_location_forecast(location_id)
+        for response in forecast:
+            if response.date == date:
+                return response
+        raise ValueError(f"Unsupported date: {date}")
+
+    async def get_days(self, location_id: str, days: int) -> WeatherResponse:
+        forecast = await self._get_location_forecast(location_id)
+        values = [merge_weather_values(item.date, item.values) for item in forecast]
+        return WeatherResponse(
+            location=forecast[0].location,
+            date=datetime.date.today(),
+            period="days",
+            values=values,
+        )

gallery/painting/rp5/parser.py

@@ -0,0 +1,108 @@
+import re
+
+import dateparser
+from bs4 import BeautifulSoup, Tag
+
+from gallery.sketch.weather.model import (
+    Cloudness,
+    Precipitation,
+    WeatherResponse,
+    WeatherValue,
+)
+from gallery.sketch.weather.parser import ParseError, parse_wind_direction
+from gallery.sketch.weather.util import build_weather_value
+
+CLOUDNESS_MAP = {
+    "Ясно": Cloudness.CLEAR,
+    "Малооблачно": Cloudness.PARTLY_CLOUDY,
+    "Облачно с прояснениями": Cloudness.PARTLY_CLOUDY,
+    "Переменная облачность": Cloudness.PARTLY_CLOUDY,
+    "Небольшая облачность": Cloudness.CLOUDY,
+    "Облачно": Cloudness.CLOUDY,
+    "Значительная облачность": Cloudness.MAINLY_CLOUDY,
+    "Пасмурная погода": Cloudness.MAINLY_CLOUDY,
+}
+
+PRECIPITATION_MAP = {
+    "Явления погоды отсутствуют": Precipitation.NO,
+    "Cлабый дождь": Precipitation.SMALL_RAIN,
+}
+
+
+def _parse_wind_speed(cell: Tag) -> int:
+    element = cell.select_one(".wv_0")
+    if element:
+        return int(element.text.strip() or "0")
+    return 0
+
+
+# pylint:disable=too-many-branches
+def parse_page(page: BeautifulSoup) -> list[WeatherResponse]:
+    location = page.select(".intoLeftNavi > .verticalBottom > a")[-1].text.strip()
+    table = page.select_one("#forecastTable")
+    if table is None:
+        raise ParseError("#forecastTable not found")
+    (
+        date_row,
+        time_row,
+        cloudness_row,
+        precipitation_row,
+        temperature_row,
+        _,
+        pressure_row,
+        wind_speed_row,
+        wind_gust_row,
+        wind_direction_row,
+        humidity_row,
+        *_,
+    ) = table.select("tr")
+    result: list[WeatherResponse] = []
+    values: list[WeatherValue] = []
+    for index, cell in enumerate(date_row.select("td")):
+        colspan = int(cell.attrs["colspan"])
+        if index == 0:
+            colspan -= 1
+        date_str = ",".join(cell.text.strip().split(",")[-2:])
+        date = dateparser.parse(date_str, languages=["ru"])
+        if date is None:
+            raise ParseError.invalid_date(date_str)
+        item_values = [build_weather_value(date) for _ in range(colspan)]
+        result.append(
+            WeatherResponse(
+                location=location,
+                date=date.date(),
+                period="day",
+                values=item_values,
+            )
+        )
+        values += item_values
+    for index, cell in enumerate(time_row.select("td")[1:]):
+        hour = int(cell.text.strip())
+        values[index].date = values[index].date.replace(hour=hour)
+    for index, cell in enumerate(cloudness_row.select("td")[1:]):
+        onmouseover = cell.select_one(".cc_0").next_element.attrs["onmouseover"]
+        cloudness_str = re.compile("<b>(.*?)</b>").search(onmouseover).group(1).strip()
+        values[index].sky.cloudness = CLOUDNESS_MAP[cloudness_str]
+    for index, cell in enumerate(precipitation_row.select("td")[1:]):
+        onmouseover = cell.select_one(".pr_0").attrs["onmouseover"]
+        precipitation_str = onmouseover.split(",")[1].strip(" '")
+        precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str)
+        if precipitation_value_matched:
+            # one hour value
+            values[index].precipitation = float(precipitation_value_matched.group(1)) * 6
+        if "(" in precipitation_str:
+            precipitation_str = precipitation_str.split(" (")[0]
+        values[index].sky.precipitation = PRECIPITATION_MAP[precipitation_str]
+    for index, cell in enumerate(temperature_row.select("td")[1:]):
+        values[index].temperature = [int(cell.select_one(".t_0").text.lstrip("+").strip())]
+    for index, cell in enumerate(pressure_row.select("td")[1:]):
+        values[index].pressure = [int(cell.select_one(".p_0").text.strip())]
+    for index, cell in enumerate(wind_speed_row.select("td")[1:]):
+        values[index].wind_speed = _parse_wind_speed(cell)
+    for index, cell in enumerate(wind_gust_row.select("td")[1:]):
+        values[index].wind_speed = _parse_wind_speed(cell)
+    for index, cell in enumerate(wind_direction_row.select("td")[1:]):
+        values[index].wind_direction = parse_wind_direction(cell.text.strip())
+    for index, cell in enumerate(humidity_row.select("td")[1:]):
+        values[index].humidity = int(cell.text.strip())
+    return result

gallery/sketch/source.py

@@ -1,5 +1,6 @@
+import datetime
 import logging
-from typing import Literal
+from typing import Any, Literal
 
 import aiohttp
 from playwright.async_api import async_playwright
@@ -9,6 +10,7 @@ logger = logging.getLogger("source")
 RequestMode = Literal["base", "browser"]
 
 DEFAULT_REQUEST_MODE: RequestMode = "base"
+DEFAULT_METHOD = "GET"
 
 
 class ApiSource:
@@ -31,30 +33,47 @@ class ApiSource:
         self._cookies = cookies
         self._headers = headers
 
-    async def _base_request(self, url: str) -> str:
+    async def _base_request(
+        self,
+        url: str,
+        method: str = DEFAULT_METHOD,
+        data: Any | None = None,
+    ) -> str:
         headers = {"User-Agent": self._user_agent, **(self._headers or {})}
         async with aiohttp.ClientSession(
             headers=headers,
             cookies=self._cookies,
             raise_for_status=True,
         ) as session:
-            async with session.request("GET", url, timeout=self._timeout) as response:
+            async with session.request(method, url, timeout=self._timeout, data=data) as response:
                 return await response.text()
 
-    async def _browser_request(self, url: str) -> str:
+    async def _browser_request(
+        self,
+        url: str,
+        method: str = DEFAULT_METHOD,
+        data: Any | None = None,
+    ) -> str:
         async with async_playwright() as p:
             browser = await p.chromium.launch()
             page = await browser.new_page()
-            await page.goto(url, wait_until="networkidle")
+            await page.goto(url, timeout=datetime.timedelta(seconds=self._timeout), wait_until="networkidle")
             content = await page.content()
             await browser.close()
         return content
 
-    async def request(self, endpoint: str, /, mode: RequestMode = DEFAULT_REQUEST_MODE) -> str:
+    async def request(
+        self,
+        endpoint: str,
+        /,
+        mode: RequestMode = DEFAULT_REQUEST_MODE,
+        method: str = DEFAULT_METHOD,
+        data: Any | None = None,
+    ) -> str:
         if endpoint.startswith("https:"):
             url = endpoint
         else:
             url = f"{self._base_url}/{endpoint}"
         logger.info(url)
-        method = getattr(self, f"_{mode}_request")
-        return await method(url)
+        fun = getattr(self, f"_{mode}_request")
+        return await fun(url, method, data)

gallery/sketch/weather/parser.py

@@ -13,6 +13,7 @@ WIND_DIRECTION_MAP: dict[str, WindDirection] = {
     "—": WindDirection.CALM,
     "": WindDirection.CALM,
     "штиль": WindDirection.CALM,
+    "штл": WindDirection.CALM,
     "с": WindDirection.N,
     "св": WindDirection.NE,
     "в": WindDirection.E,
@@ -25,4 +26,4 @@ WIND_DIRECTION_MAP: dict[str, WindDirection] = {
 
 
 def parse_wind_direction(value: str) -> WindDirectionDeg:
-    return WindDirectionDeg.from_direction(WIND_DIRECTION_MAP[value.lower()])
+    return WindDirectionDeg.from_direction(WIND_DIRECTION_MAP[value.replace("-", "").lower()])

gallery/sketch/weather/util.py

@@ -27,7 +27,13 @@ def build_weather_value(date: datetime.datetime) -> WeatherValue:
 
 
 def _merge_values(values: list[int], mode: MergeMode) -> list[int]:
-    return [max(values), min(values)] if mode == "interval" else [round(sum(values) / len(values))]
+    if mode == "interval":
+        if (max_value := max(values)) == (min_value := min(values)):
+            return [max_value]
+        else:
+            return [max_value, min_value]
+    else:
+        return [round(sum(values) / len(values))]
 
 
 def merge_weather_values(