refactor(rp5): rework rp5 parser

gallery/painting/gismeteo/parser.py

@@ -17,12 +17,12 @@ from gallery.sketch.weather.model import (
     WindDirection,
 )
 from gallery.sketch.weather.parser import parse_wind_direction
-from gallery.sketch.weather.util import build_weather_value, merge_weather_values
+from gallery.sketch.weather.util import merge_weather_values
 
 logger = logging.getLogger("gismeteo")
 
 
-class LocationParser(Parser[str, str]):
+class LocationNameParser(Parser[str, str]):
     PATTERN = re.compile('{"ru":{"city":{"name":"(.*?)"')
 
     def parse(self, data: str) -> str:
@@ -36,7 +36,7 @@ class TimeParser(TableValueParser[datetime.datetime]):
     selector = ".widget-row.widget-row-datetime-time > .row-item > time-value"
     model_key = "date"
 
-    def parse(self, data: Tag) -> datetime.datetime:
+    def extract(self, data: Tag) -> datetime.datetime:
         timestamp = int(data.attrs["timestamp"])
         return datetime.datetime.fromtimestamp(timestamp)
 
@@ -45,7 +45,7 @@ class DateParser(TableValueParser[datetime.datetime]):
     selector = ".widget-row.widget-row-date > .row-item"
     model_key = "date"
 
-    def parse(self, data: Tag) -> datetime.datetime:
+    def extract(self, data: Tag) -> datetime.datetime:
         return parse_date(data.text)
 
 
@@ -89,7 +89,7 @@ class SkyParser(TableValueParser[Sky]):
         result_values = values - flag_values
         return result_values, len(result_values) < len(values)
 
-    def parse(self, data: Tag) -> Sky:
+    def extract(self, data: Tag) -> Sky:
         sky_str = data.attrs["data-tooltip"]
         values = {item.strip().lower() for item in sky_str.split(",")}
         cloudness = Cloudness.CLEAR
@@ -120,7 +120,7 @@ class TemperatureParser(TableValueParser[list[int]]):
     selector = ".widget-row-chart[data-row=temperature-air] > .chart > .values > .value"
     model_key = "temperature"
 
-    def parse(self, data: Tag) -> list[int]:
+    def extract(self, data: Tag) -> list[int]:
         return [int(value.attrs["value"]) for value in data.select("temperature-value")]
 
 
@@ -128,7 +128,7 @@ class WindSpeedParser(TableValueParser[int]):
     selector = ".widget-row-wind > .row-item > .wind-speed"
     model_key = "wind.speed"
 
-    def parse(self, data: Tag) -> int:
+    def extract(self, data: Tag) -> int:
         value = data.select_one("speed-value")
         return int(value.attrs["value"]) if value else 0
 
@@ -137,7 +137,7 @@ class WindGustParser(TableValueParser[int]):
     selector = ".widget-row-wind > .row-item > .wind-gust"
     model_key = "wind.gust"
 
-    def parse(self, data: Tag) -> int:
+    def extract(self, data: Tag) -> int:
         value = data.select_one("speed-value")
         return int(value.attrs["value"]) if value else 0
 
@@ -146,7 +146,7 @@ class WindDirectionParser(TableValueParser[WindDirection | None]):
     selector = ".widget-row-wind > .row-item > .wind-speed > .wind-direction"
     model_key = "wind.direction"
 
-    def parse(self, data: Tag) -> WindDirection | None:
+    def extract(self, data: Tag) -> WindDirection | None:
         return parse_wind_direction(data.text.strip())
 
 
@@ -154,7 +154,7 @@ class PrecipitationParser(TableValueParser[float]):
     selector = ".widget-row[data-row=precipitation-bars] > .row-item"
     model_key = "precipitation"
 
-    def parse(self, data: Tag) -> float:
+    def extract(self, data: Tag) -> float:
         value = data.select_one("precipitation-value")
         return float(value.attrs["value"]) if value else 0
 
@@ -163,7 +163,7 @@ class PressureParser(TableValueParser[list[int]]):
     selector = ".widget-row-chart[data-row=pressure] > .chart > .values > .value"
     model_key = "pressure"
 
-    def parse(self, data: Tag) -> list[int]:
+    def extract(self, data: Tag) -> list[int]:
         return [int(value.attrs["value"]) for value in data.select("pressure-value")]
 
 
@@ -171,13 +171,12 @@ class HumidityParser(TableValueParser[int]):
     selector = ".widget-row[data-row=humidity] > .row-item, .widget-row[data-row=humidity-avg] > .row-item"
     model_key = "humidity"
 
-    def parse(self, data: Tag) -> int:
+    def extract(self, data: Tag) -> int:
         return int(data.text)
 
 
 class GismeteoTableParser(TableParser[WeatherValue]):
     selector = ".widget .widget-items"
-    model_factory = lambda cls: build_weather_value()
     value_parsers = [
         TimeParser(),
         DateParser(),
@@ -191,6 +190,9 @@ class GismeteoTableParser(TableParser[WeatherValue]):
         HumidityParser(),
     ]
 
+    def create_model(self, data: dict) -> WeatherValue:
+        return WeatherValue.model_validate(data)
+
 
 class Context(NamedTuple):
     date: datetime.date
@@ -200,7 +202,7 @@ class Context(NamedTuple):
 
 
 class GismeteoParser(Parser[Context, WeatherResponse]):
-    location_parser = LocationParser()
+    location_parser = LocationNameParser()
     table_parser = GismeteoTableParser()
 
     def parse(self, data: Context) -> WeatherResponse:

gallery/painting/rp5/api.py

@@ -3,8 +3,8 @@ import logging
 
 import aiohttp
 from aiocache import cached
-from bs4 import BeautifulSoup
 
+from gallery.sketch.headers import build_headers
 from gallery.sketch.source import ApiSource
 from gallery.sketch.weather.api import WeatherApi
 from gallery.sketch.weather.model import (
@@ -14,8 +14,7 @@ from gallery.sketch.weather.model import (
 from gallery.sketch.weather.util import merge_weather_values
 from gallery.util import TimeUnit
 
-from ...sketch.headers import build_headers
+from .parser import RP5ForecastParser, RP5LocationParser
-from .parser import parse_page
 
 logger = logging.getLogger("rp5")
 
@@ -23,37 +22,17 @@ logger = logging.getLogger("rp5")
 class RP5WeatherApi(WeatherApi):
     PROVIDER = "rp5"
 
+    forecast_parser = RP5ForecastParser()
+    location_parser = RP5LocationParser()
+
     def _build_source(self) -> ApiSource:
         return ApiSource("https://rp5.ru", headers=build_headers("rp5.ru"))
 
     async def find_locations(self, query: str) -> list[Location]:
         endpoint = "docs/search/ru"
         data = aiohttp.FormData({"searchStr": query})
-        response = await self.source.request(endpoint, method="POST", data=data)
+        content = await self.source.request(endpoint, method="POST", data=data)
-        soup = BeautifulSoup(response, features="html.parser")
+        return self.location_parser.parse(content)
-        table = soup.select_one(".searchResults")
-        result = []
-        for row in table.select("tr")[1:]:
-            cells = row.select("td")
-            name = cells[0].text.strip()
-            location_id = cells[0].select_one("a").attrs["href"].lstrip("/")
-            country = cells[1].text.strip()
-            district = cells[2].text.strip()
-            subdistrict = cells[3].text.strip()
-            result.append(
-                Location(
-                    id=location_id,
-                    name=name,
-                    provider=self.provider,
-                    lat=0,
-                    lon=0,
-                    country=country,
-                    country_code="ru",  # TODO: contry_code from country
-                    district=district,
-                    subdistrict=subdistrict,
-                )
-            )
-        return result
 
     @cached(
         key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast",
@@ -63,8 +42,7 @@ class RP5WeatherApi(WeatherApi):
     async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]:
         endpoint = f"{location_id}"
         content = await self.source.request(endpoint)
-        soup = BeautifulSoup(content, features="html.parser")
+        return self.forecast_parser.parse(content)
-        return parse_page(soup)
 
     async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
         forecast = await self._get_location_forecast(location_id)

gallery/painting/rp5/parser.py

@@ -1,18 +1,59 @@
+import datetime
 import re
-from enum import Enum, auto
-from typing import Iterable
 
-import dateparser
 from bs4 import BeautifulSoup, Tag
 
+from gallery.sketch.parse.core import Parser
+from gallery.sketch.parse.date import parse_date
+from gallery.sketch.parse.html import (
+    TableParser,
+    TableValueParser,
+    TagIterParser,
+    TagParser,
+)
 from gallery.sketch.weather.model import (
     Cloudness,
+    Location,
     Precipitation,
     WeatherResponse,
     WeatherValue,
+    WindDirection,
 )
-from gallery.sketch.weather.parser import ParseError, parse_wind_direction
+from gallery.sketch.weather.parser import parse_wind_direction
-from gallery.sketch.weather.util import build_weather_value
+
+
+class LocationNameParser(TagParser[str]):
+    selector = ".intoLeftNavi > .verticalBottom > a"
+
+    def extract(self, data: Tag) -> str:
+        return data.text.strip()
+
+
+class DatesParser(TagIterParser[tuple[datetime.datetime, int]]):
+    selector = "#forecastTable > tr.forecastDate > td"
+
+    def extract(self, data: Tag) -> tuple[datetime.datetime, int]:
+        colspan = int(data.attrs["colspan"])
+        index = data.parent.index(data)
+        if index == 1:
+            colspan -= 1
+        date_str = ",".join(data.text.strip().split(",")[-2:])
+        date = parse_date(date_str)
+        return (date, colspan)
+
+
+class TimeParser(TableValueParser[datetime.datetime]):
+    selector = "tr.forecastTime > td:not(:first-of-type)"
+    model_key = "date"
+
+    def extract(self, data: Tag) -> datetime.datetime:
+        hour = int(data.text.strip())
+        return datetime.datetime.min.replace(hour=hour)
+
+
+class CloudnessParser(TableValueParser[Cloudness]):
+    selector = "tr:nth-of-type(3) > td:not(:first-of-type)"
+    model_key = "sky.cloudness"
 
     CLOUDNESS_MAP = {
         "Ясно": Cloudness.CLEAR,
@@ -25,122 +66,150 @@ CLOUDNESS_MAP = {
         "Пасмурная погода": Cloudness.MAINLY_CLOUDY,
     }
 
+    def extract(self, data: Tag) -> Cloudness:
+        onmouseover = data.select_one(".cc_0").next_element.attrs["onmouseover"]
+        cloudness_str = re.compile("<b>(.*?)</b>").search(onmouseover).group(1).strip()
+        return self.CLOUDNESS_MAP[cloudness_str]
+
+
+class PrecipitationParser(TableValueParser[tuple[Precipitation, float]]):
+    selector = "tr:nth-of-type(4) > td:not(:first-of-type)"
+    model_key = ["sky.precipitation", "precipitation"]
+
     PRECIPITATION_MAP = {
         "Явления погоды отсутствуют": Precipitation.NO,
         "Cлабый дождь": Precipitation.SMALL_RAIN,
     }
 
+    def extract(self, data: Tag) -> tuple[Precipitation, float]:
+        onmouseover = data.select_one(".pr_0").attrs["onmouseover"]
+        precipitation_str = onmouseover.split(",")[1].strip(" '")
+        precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str)
+        precipitation_value = 0
+        if precipitation_value_matched:
+            # one hour value
+            precipitation_value = float(precipitation_value_matched.group(1)) * 6
+        if "(" in precipitation_str:
+            precipitation_str = precipitation_str.split(" (")[0]
+        return self.PRECIPITATION_MAP[precipitation_str], precipitation_value
 
-def _parse_wind_speed(cell: Tag) -> int:
+
-    element = cell.select_one(".wv_0")
+class TemperatureParser(TableValueParser[list[int]]):
+    selector = "tr:nth-of-type(5) > td:not(:first-of-type)"
+    model_key = "temperature"
+
+    def extract(self, data: Tag) -> list[int]:
+        return [int(data.select_one(".t_0").text.lstrip("+").strip())]
+
+
+class PressureParser(TableValueParser[list[int]]):
+    selector = "tr:nth-last-of-type(8) > td:not(:first-of-type)"
+    model_key = "pressure"
+
+    def extract(self, data: Tag) -> list[int]:
+        return [int(data.select_one(".p_0").text.strip())]
+
+
+class WindSpeedParser(TableValueParser[int]):
+    selector = "tr:nth-last-of-type(7) > td:not(:first-of-type)"
+    model_key = "wind.speed"
+
+    def extract(self, data: Tag) -> int:
+        element = data.select_one(".wv_0")
         if element:
             return int(element.text.strip() or "0")
         return 0
 
 
-class RowType(Enum):
+class WindGustParser(WindSpeedParser):
-    DATE = auto()
+    selector = "tr:nth-last-of-type(6) > td:not(:first-of-type)"
-    TIME = auto()
+    model_key = "wind.gust"
-    CLOUDNESS = auto()
-    PRECIPITATION = auto()
-    TEMPERATURE = auto()
-    PRESSURE = auto()
-    WIND_SPEED = auto()
-    WIND_GUST = auto()
-    WIND_DIRECTION = auto()
-    HUMIDITY = auto()
 
 
-def _iter_table_rows(table: Tag) -> Iterable[tuple[RowType, Tag]]:
+class WindDirectionParser(TableValueParser[WindDirection | None]):
-    for row in table.select("tr"):
+    selector = "tr:nth-last-of-type(5) > td:not(:first-of-type)"
-        title = row.select_one("td").text.strip().lower()
+    model_key = "wind.direction"
-        if "сегодня" in title or "завтра" in title:
+
-            yield RowType.DATE, row
+    def extract(self, data: Tag) -> WindDirection | None:
-        elif "местное время" in title:
+        return parse_wind_direction(data.text.strip())
-            yield RowType.TIME, row
-        elif "облачность" in title:
-            yield RowType.CLOUDNESS, row
-        elif "осадки" in title:
-            yield RowType.PRECIPITATION, row
-        elif "температура" in title:
-            yield RowType.TEMPERATURE, row
-        elif "давление" in title:
-            yield RowType.PRESSURE, row
-        elif "скорость" in title:
-            yield RowType.WIND_SPEED, row
-        elif "порывы" in title:
-            yield RowType.WIND_GUST, row
-        elif "направление" in title:
-            yield RowType.WIND_DIRECTION, row
-        elif "влажность" in title:
-            yield RowType.HUMIDITY, row
 
 
-# pylint:disable=too-many-branches
+class HumidityParser(TableValueParser[int]):
-def parse_page(page: BeautifulSoup) -> list[WeatherResponse]:
+    selector = "tr:nth-last-of-type(4) > td:not(:first-of-type)"
-    location = page.select(".intoLeftNavi > .verticalBottom > a")[-1].text.strip()
+    model_key = "humidity"
-    table = page.select_one("#forecastTable")
+
-    if table is None:
+    def extract(self, data: Tag) -> int:
-        raise ParseError("#forecastTable not found")
+        return int(data.text.strip())
-    result: list[WeatherResponse] = []
+
-    values: list[WeatherValue] = []
+
-    for row_type, row in _iter_table_rows(table):
+class RP5TableParser(TableParser[WeatherValue]):
-        match row_type:
+    selector = "#forecastTable"
-            case RowType.DATE:
+    value_parsers = [
-                for index, cell in enumerate(row.select("td")):
+        TimeParser(),
-                    colspan = int(cell.attrs["colspan"])
+        CloudnessParser(),
-                    if index == 0:
+        PrecipitationParser(),
-                        colspan -= 1
+        TemperatureParser(),
-                    date_str = ",".join(cell.text.strip().split(",")[-2:])
+        PressureParser(),
-                    date = dateparser.parse(date_str, languages=["ru"])
+        WindSpeedParser(),
-                    if date is None:
+        WindGustParser(),
-                        raise ParseError.invalid_date(date_str)
+        WindDirectionParser(),
-                    item_values = [build_weather_value(date) for _ in range(colspan)]
+        HumidityParser(),
-                    result.append(
+    ]
+
+    def create_model(self, data: dict) -> WeatherValue:
+        return WeatherValue.model_validate(data)
+
+
+class RP5ForecastParser(Parser[str, list[WeatherResponse]]):
+    location_parser = LocationNameParser()
+    dates_parser = DatesParser()
+    table_parser = RP5TableParser()
+
+    def parse(self, data: str) -> list[WeatherResponse]:
+        soup = BeautifulSoup(data, features="html.parser")
+        location = self.location_parser.parse(soup)
+        dates = self.dates_parser.parse(soup)
+        values = self.table_parser.parse(soup)
+        responses = []
+        for date, count in dates:
+            item_values, values = values[:count], values[count:]
+            for value in item_values:
+                value.date = date.replace(hour=value.date.hour)
+            responses.append(
                 WeatherResponse(
                     location=location,
-                            date=date.date(),
+                    date=date,
                     period="day",
                     values=item_values,
                 )
             )
-                    values += item_values
+        return responses
-            case RowType.TIME:
+
-                for index, cell in enumerate(row.select("td")[1:]):
+
-                    hour = int(cell.text.strip())
+class RP5LocationParser(Parser[str, list[Location]]):
-                    values[index].date = values[index].date.replace(hour=hour)
+
-            case RowType.CLOUDNESS:
+    def parse(self, data: str) -> list[Location]:
-                for index, cell in enumerate(row.select("td")[1:]):
+        soup = BeautifulSoup(data, features="html.parser")
-                    onmouseover = cell.select_one(".cc_0").next_element.attrs["onmouseover"]
+        table = soup.select_one(".searchResults")
-                    cloudness_str = re.compile("<b>(.*?)</b>").search(onmouseover).group(1).strip()
+        result = []
-                    values[index].sky.cloudness = CLOUDNESS_MAP[cloudness_str]
+        for row in table.select("tr")[1:]:
-            case RowType.PRECIPITATION:
+            cells = row.select("td")
-                for index, cell in enumerate(row.select("td")[1:]):
+            name = cells[0].text.strip()
-                    onmouseover = cell.select_one(".pr_0").attrs["onmouseover"]
+            location_id = cells[0].select_one("a").attrs["href"].lstrip("/")
-                    precipitation_str = onmouseover.split(",")[1].strip(" '")
+            country = cells[1].text.strip()
-                    precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str)
+            district = cells[2].text.strip()
-                    if precipitation_value_matched:
+            subdistrict = cells[3].text.strip()
-                        # one hour value
+            result.append(
-                        values[index].precipitation = float(precipitation_value_matched.group(1)) * 6
+                Location(
-                    if "(" in precipitation_str:
+                    id=location_id,
-                        precipitation_str = precipitation_str.split(" (")[0]
+                    name=name,
-                    values[index].sky.precipitation = PRECIPITATION_MAP[precipitation_str]
+                    provider="rp5",  # TODO: provider module const
-            case RowType.TEMPERATURE:
+                    lat=0,
-                for index, cell in enumerate(row.select("td")[1:]):
+                    lon=0,
-                    values[index].temperature = [int(cell.select_one(".t_0").text.lstrip("+").strip())]
+                    country=country,
-            case RowType.PRESSURE:
+                    country_code="ru",  # TODO: contry_code from country
-                for index, cell in enumerate(row.select("td")[1:]):
+                    district=district,
-                    values[index].pressure = [int(cell.select_one(".p_0").text.strip())]
+                    subdistrict=subdistrict,
-            case RowType.WIND_SPEED:
+                )
-                for index, cell in enumerate(row.select("td")[1:]):
+            )
-                    values[index].wind.speed = _parse_wind_speed(cell)
-            case RowType.WIND_GUST:
-                for index, cell in enumerate(row.select("td")[1:]):
-                    values[index].wind.gust = _parse_wind_speed(cell)
-            case RowType.WIND_DIRECTION:
-                for index, cell in enumerate(row.select("td")[1:]):
-                    values[index].wind.direction = parse_wind_direction(cell.text.strip())
-            case RowType.HUMIDITY:
-                for index, cell in enumerate(row.select("td")[1:]):
-                    values[index].humidity = int(cell.text.strip())
         return result

gallery/sketch/parse/html.py

@@ -1,36 +1,58 @@
-from typing import Callable, Generic
+from typing import Generic, Iterable
 
 from bs4 import Tag
 
 from .core import ParseError, Parser, R
-from .util import set_nested_attr
+from .util import set_dict_nested_value
 
 
-class TableValueParser(Parser[Tag, R], Generic[R]):
+class TagParser(Parser[Tag, R], Generic[R]):
     selector: str
-    model_key: str
+
+    def extract(self, data: Tag) -> R:
+        raise NotImplementedError
 
     def parse(self, data: Tag) -> R:
+        tag = data.select_one(self.selector)
+        if tag is None:
+            raise ParseError("Tag not found", self.selector)
+        return self.extract(tag)
+
+
+class TagIterParser(Parser[Tag, Iterable[R]], Generic[R]):
+    selector: str
+
+    def extract(self, data: Tag) -> R:
+        raise NotImplementedError
+
+    def parse(self, data: Tag) -> Iterable[R]:
+        tags = data.select(self.selector)
+        for tag in tags:
+            yield self.extract(tag)
+
+
+class TableValueParser(TagIterParser[R], Generic[R]):
+    model_key: str | list[str]
+
+    def extract(self, data: Tag) -> R:
         raise NotImplementedError
 
 
-class TableParser(Parser[Tag, R], Generic[R]):
+class TableParser(TagParser[list[R]], Generic[R]):
-    selector: str
-    model_factory: Callable[[], R]
     value_parsers: list[TableValueParser]
 
-    def create_model(self) -> R:
+    def create_model(self, data: dict) -> R:
-        return self.model_factory()
+        raise NotImplementedError
 
-    def parse(self, data: Tag) -> list[R]:
+    def extract(self, data: Tag) -> list[R]:
-        table = data.select_one(self.selector)
+        results = []
-        if table is None:
-            raise ParseError("Table not found", self.selector)
-        result = []
         for parser in self.value_parsers:
-            for index, tag in enumerate(table.select(parser.selector)):
+            for index, value in enumerate(parser.parse(data)):
-                value = parser.parse(tag)
+                while index > len(results) - 1:
-                while index > len(result) - 1:
+                    results.append({})
-                    result.append(self.create_model())
+                if isinstance(parser.model_key, list):
-                set_nested_attr(result[index], parser.model_key, value)
+                    for model_key_index, model_key in enumerate(parser.model_key):
-        return result
+                        set_dict_nested_value(results[index], model_key, value[model_key_index])
+                else:
+                    set_dict_nested_value(results[index], parser.model_key, value)
+        return [self.create_model(item) for item in results]

gallery/sketch/weather/model.py

@@ -51,10 +51,10 @@ Precipitation.values = list(Precipitation)
 
 
 class Sky(Model):
-    cloudness: Cloudness
+    cloudness: Cloudness = Cloudness.CLEAR
-    precipitation: Precipitation
+    precipitation: Precipitation = Precipitation.NO
-    thunder: bool
+    thunder: bool = False
-    fog: bool
+    fog: bool = False
 
 
 class WindDirection(StrEnum):
@@ -69,8 +69,8 @@ class WindDirection(StrEnum):
 
 
 class Wind(Model):
-    speed: int
+    speed: int = 0
-    gust: int
+    gust: int = 0
     direction: WindDirection | None = None
 
 
@@ -79,9 +79,9 @@ class WeatherValue(Model):
     sky: Sky
     wind: Wind
     temperature: list[int]
-    precipitation: float
+    precipitation: float = 0
     pressure: list[int] = Field(default=[])
-    humidity: int = Field(default=-1)
+    humidity: int = -1
 
 
 class WeatherResponse(Model):