diff --git a/gallery/painting/gismeteo/parser.py b/gallery/painting/gismeteo/parser.py index 9d03386..18dbed1 100644 --- a/gallery/painting/gismeteo/parser.py +++ b/gallery/painting/gismeteo/parser.py @@ -17,12 +17,12 @@ from gallery.sketch.weather.model import ( WindDirection, ) from gallery.sketch.weather.parser import parse_wind_direction -from gallery.sketch.weather.util import build_weather_value, merge_weather_values +from gallery.sketch.weather.util import merge_weather_values logger = logging.getLogger("gismeteo") -class LocationParser(Parser[str, str]): +class LocationNameParser(Parser[str, str]): PATTERN = re.compile('{"ru":{"city":{"name":"(.*?)"') def parse(self, data: str) -> str: @@ -36,7 +36,7 @@ class TimeParser(TableValueParser[datetime.datetime]): selector = ".widget-row.widget-row-datetime-time > .row-item > time-value" model_key = "date" - def parse(self, data: Tag) -> datetime.datetime: + def extract(self, data: Tag) -> datetime.datetime: timestamp = int(data.attrs["timestamp"]) return datetime.datetime.fromtimestamp(timestamp) @@ -45,7 +45,7 @@ class DateParser(TableValueParser[datetime.datetime]): selector = ".widget-row.widget-row-date > .row-item" model_key = "date" - def parse(self, data: Tag) -> datetime.datetime: + def extract(self, data: Tag) -> datetime.datetime: return parse_date(data.text) @@ -89,7 +89,7 @@ class SkyParser(TableValueParser[Sky]): result_values = values - flag_values return result_values, len(result_values) < len(values) - def parse(self, data: Tag) -> Sky: + def extract(self, data: Tag) -> Sky: sky_str = data.attrs["data-tooltip"] values = {item.strip().lower() for item in sky_str.split(",")} cloudness = Cloudness.CLEAR @@ -120,7 +120,7 @@ class TemperatureParser(TableValueParser[list[int]]): selector = ".widget-row-chart[data-row=temperature-air] > .chart > .values > .value" model_key = "temperature" - def parse(self, data: Tag) -> list[int]: + def extract(self, data: Tag) -> list[int]: return [int(value.attrs["value"]) for value in data.select("temperature-value")] @@ -128,7 +128,7 @@ class WindSpeedParser(TableValueParser[int]): selector = ".widget-row-wind > .row-item > .wind-speed" model_key = "wind.speed" - def parse(self, data: Tag) -> int: + def extract(self, data: Tag) -> int: value = data.select_one("speed-value") return int(value.attrs["value"]) if value else 0 @@ -137,7 +137,7 @@ class WindGustParser(TableValueParser[int]): selector = ".widget-row-wind > .row-item > .wind-gust" model_key = "wind.gust" - def parse(self, data: Tag) -> int: + def extract(self, data: Tag) -> int: value = data.select_one("speed-value") return int(value.attrs["value"]) if value else 0 @@ -146,7 +146,7 @@ class WindDirectionParser(TableValueParser[WindDirection | None]): selector = ".widget-row-wind > .row-item > .wind-speed > .wind-direction" model_key = "wind.direction" - def parse(self, data: Tag) -> WindDirection | None: + def extract(self, data: Tag) -> WindDirection | None: return parse_wind_direction(data.text.strip()) @@ -154,7 +154,7 @@ class PrecipitationParser(TableValueParser[float]): selector = ".widget-row[data-row=precipitation-bars] > .row-item" model_key = "precipitation" - def parse(self, data: Tag) -> float: + def extract(self, data: Tag) -> float: value = data.select_one("precipitation-value") return float(value.attrs["value"]) if value else 0 @@ -163,7 +163,7 @@ class PressureParser(TableValueParser[list[int]]): selector = ".widget-row-chart[data-row=pressure] > .chart > .values > .value" model_key = "pressure" - def parse(self, data: Tag) -> list[int]: + def extract(self, data: Tag) -> list[int]: return [int(value.attrs["value"]) for value in data.select("pressure-value")] @@ -171,13 +171,12 @@ class HumidityParser(TableValueParser[int]): selector = ".widget-row[data-row=humidity] > .row-item, .widget-row[data-row=humidity-avg] > .row-item" model_key = "humidity" - def parse(self, data: Tag) -> int: + def extract(self, data: Tag) -> int: return int(data.text) class GismeteoTableParser(TableParser[WeatherValue]): selector = ".widget .widget-items" - model_factory = lambda cls: build_weather_value() value_parsers = [ TimeParser(), DateParser(), @@ -191,6 +190,9 @@ class GismeteoTableParser(TableParser[WeatherValue]): HumidityParser(), ] + def create_model(self, data: dict) -> WeatherValue: + return WeatherValue.model_validate(data) + class Context(NamedTuple): date: datetime.date @@ -200,7 +202,7 @@ class Context(NamedTuple): class GismeteoParser(Parser[Context, WeatherResponse]): - location_parser = LocationParser() + location_parser = LocationNameParser() table_parser = GismeteoTableParser() def parse(self, data: Context) -> WeatherResponse: diff --git a/gallery/painting/rp5/api.py b/gallery/painting/rp5/api.py index 8bcf809..c090643 100644 --- a/gallery/painting/rp5/api.py +++ b/gallery/painting/rp5/api.py @@ -3,8 +3,8 @@ import logging import aiohttp from aiocache import cached -from bs4 import BeautifulSoup +from gallery.sketch.headers import build_headers from gallery.sketch.source import ApiSource from gallery.sketch.weather.api import WeatherApi from gallery.sketch.weather.model import ( @@ -14,8 +14,7 @@ from gallery.sketch.weather.model import ( from gallery.sketch.weather.util import merge_weather_values from gallery.util import TimeUnit -from ...sketch.headers import build_headers -from .parser import parse_page +from .parser import RP5ForecastParser, RP5LocationParser logger = logging.getLogger("rp5") @@ -23,37 +22,17 @@ logger = logging.getLogger("rp5") class RP5WeatherApi(WeatherApi): PROVIDER = "rp5" + forecast_parser = RP5ForecastParser() + location_parser = RP5LocationParser() + def _build_source(self) -> ApiSource: return ApiSource("https://rp5.ru", headers=build_headers("rp5.ru")) async def find_locations(self, query: str) -> list[Location]: endpoint = "docs/search/ru" data = aiohttp.FormData({"searchStr": query}) - response = await self.source.request(endpoint, method="POST", data=data) - soup = BeautifulSoup(response, features="html.parser") - table = soup.select_one(".searchResults") - result = [] - for row in table.select("tr")[1:]: - cells = row.select("td") - name = cells[0].text.strip() - location_id = cells[0].select_one("a").attrs["href"].lstrip("/") - country = cells[1].text.strip() - district = cells[2].text.strip() - subdistrict = cells[3].text.strip() - result.append( - Location( - id=location_id, - name=name, - provider=self.provider, - lat=0, - lon=0, - country=country, - country_code="ru", # TODO: contry_code from country - district=district, - subdistrict=subdistrict, - ) - ) - return result + content = await self.source.request(endpoint, method="POST", data=data) + return self.location_parser.parse(content) @cached( key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast", @@ -63,8 +42,7 @@ class RP5WeatherApi(WeatherApi): async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]: endpoint = f"{location_id}" content = await self.source.request(endpoint) - soup = BeautifulSoup(content, features="html.parser") - return parse_page(soup) + return self.forecast_parser.parse(content) async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse: forecast = await self._get_location_forecast(location_id) diff --git a/gallery/painting/rp5/parser.py b/gallery/painting/rp5/parser.py index 718c617..1f82c54 100644 --- a/gallery/painting/rp5/parser.py +++ b/gallery/painting/rp5/parser.py @@ -1,146 +1,215 @@ +import datetime import re -from enum import Enum, auto -from typing import Iterable -import dateparser from bs4 import BeautifulSoup, Tag +from gallery.sketch.parse.core import Parser +from gallery.sketch.parse.date import parse_date +from gallery.sketch.parse.html import ( + TableParser, + TableValueParser, + TagIterParser, + TagParser, +) from gallery.sketch.weather.model import ( Cloudness, + Location, Precipitation, WeatherResponse, WeatherValue, + WindDirection, ) -from gallery.sketch.weather.parser import ParseError, parse_wind_direction -from gallery.sketch.weather.util import build_weather_value - -CLOUDNESS_MAP = { - "Ясно": Cloudness.CLEAR, - "Малооблачно": Cloudness.PARTLY_CLOUDY, - "Облачно с прояснениями": Cloudness.PARTLY_CLOUDY, - "Переменная облачность": Cloudness.PARTLY_CLOUDY, - "Небольшая облачность": Cloudness.CLOUDY, - "Облачно": Cloudness.CLOUDY, - "Значительная облачность": Cloudness.MAINLY_CLOUDY, - "Пасмурная погода": Cloudness.MAINLY_CLOUDY, -} - -PRECIPITATION_MAP = { - "Явления погоды отсутствуют": Precipitation.NO, - "Cлабый дождь": Precipitation.SMALL_RAIN, -} +from gallery.sketch.weather.parser import parse_wind_direction -def _parse_wind_speed(cell: Tag) -> int: - element = cell.select_one(".wv_0") - if element: - return int(element.text.strip() or "0") - return 0 +class LocationNameParser(TagParser[str]): + selector = ".intoLeftNavi > .verticalBottom > a" + + def extract(self, data: Tag) -> str: + return data.text.strip() -class RowType(Enum): - DATE = auto() - TIME = auto() - CLOUDNESS = auto() - PRECIPITATION = auto() - TEMPERATURE = auto() - PRESSURE = auto() - WIND_SPEED = auto() - WIND_GUST = auto() - WIND_DIRECTION = auto() - HUMIDITY = auto() +class DatesParser(TagIterParser[tuple[datetime.datetime, int]]): + selector = "#forecastTable > tr.forecastDate > td" + + def extract(self, data: Tag) -> tuple[datetime.datetime, int]: + colspan = int(data.attrs["colspan"]) + index = data.parent.index(data) + if index == 1: + colspan -= 1 + date_str = ",".join(data.text.strip().split(",")[-2:]) + date = parse_date(date_str) + return (date, colspan) -def _iter_table_rows(table: Tag) -> Iterable[tuple[RowType, Tag]]: - for row in table.select("tr"): - title = row.select_one("td").text.strip().lower() - if "сегодня" in title or "завтра" in title: - yield RowType.DATE, row - elif "местное время" in title: - yield RowType.TIME, row - elif "облачность" in title: - yield RowType.CLOUDNESS, row - elif "осадки" in title: - yield RowType.PRECIPITATION, row - elif "температура" in title: - yield RowType.TEMPERATURE, row - elif "давление" in title: - yield RowType.PRESSURE, row - elif "скорость" in title: - yield RowType.WIND_SPEED, row - elif "порывы" in title: - yield RowType.WIND_GUST, row - elif "направление" in title: - yield RowType.WIND_DIRECTION, row - elif "влажность" in title: - yield RowType.HUMIDITY, row +class TimeParser(TableValueParser[datetime.datetime]): + selector = "tr.forecastTime > td:not(:first-of-type)" + model_key = "date" + + def extract(self, data: Tag) -> datetime.datetime: + hour = int(data.text.strip()) + return datetime.datetime.min.replace(hour=hour) -# pylint:disable=too-many-branches -def parse_page(page: BeautifulSoup) -> list[WeatherResponse]: - location = page.select(".intoLeftNavi > .verticalBottom > a")[-1].text.strip() - table = page.select_one("#forecastTable") - if table is None: - raise ParseError("#forecastTable not found") - result: list[WeatherResponse] = [] - values: list[WeatherValue] = [] - for row_type, row in _iter_table_rows(table): - match row_type: - case RowType.DATE: - for index, cell in enumerate(row.select("td")): - colspan = int(cell.attrs["colspan"]) - if index == 0: - colspan -= 1 - date_str = ",".join(cell.text.strip().split(",")[-2:]) - date = dateparser.parse(date_str, languages=["ru"]) - if date is None: - raise ParseError.invalid_date(date_str) - item_values = [build_weather_value(date) for _ in range(colspan)] - result.append( - WeatherResponse( - location=location, - date=date.date(), - period="day", - values=item_values, - ) - ) - values += item_values - case RowType.TIME: - for index, cell in enumerate(row.select("td")[1:]): - hour = int(cell.text.strip()) - values[index].date = values[index].date.replace(hour=hour) - case RowType.CLOUDNESS: - for index, cell in enumerate(row.select("td")[1:]): - onmouseover = cell.select_one(".cc_0").next_element.attrs["onmouseover"] - cloudness_str = re.compile("(.*?)").search(onmouseover).group(1).strip() - values[index].sky.cloudness = CLOUDNESS_MAP[cloudness_str] - case RowType.PRECIPITATION: - for index, cell in enumerate(row.select("td")[1:]): - onmouseover = cell.select_one(".pr_0").attrs["onmouseover"] - precipitation_str = onmouseover.split(",")[1].strip(" '") - precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str) - if precipitation_value_matched: - # one hour value - values[index].precipitation = float(precipitation_value_matched.group(1)) * 6 - if "(" in precipitation_str: - precipitation_str = precipitation_str.split(" (")[0] - values[index].sky.precipitation = PRECIPITATION_MAP[precipitation_str] - case RowType.TEMPERATURE: - for index, cell in enumerate(row.select("td")[1:]): - values[index].temperature = [int(cell.select_one(".t_0").text.lstrip("+").strip())] - case RowType.PRESSURE: - for index, cell in enumerate(row.select("td")[1:]): - values[index].pressure = [int(cell.select_one(".p_0").text.strip())] - case RowType.WIND_SPEED: - for index, cell in enumerate(row.select("td")[1:]): - values[index].wind.speed = _parse_wind_speed(cell) - case RowType.WIND_GUST: - for index, cell in enumerate(row.select("td")[1:]): - values[index].wind.gust = _parse_wind_speed(cell) - case RowType.WIND_DIRECTION: - for index, cell in enumerate(row.select("td")[1:]): - values[index].wind.direction = parse_wind_direction(cell.text.strip()) - case RowType.HUMIDITY: - for index, cell in enumerate(row.select("td")[1:]): - values[index].humidity = int(cell.text.strip()) - return result +class CloudnessParser(TableValueParser[Cloudness]): + selector = "tr:nth-of-type(3) > td:not(:first-of-type)" + model_key = "sky.cloudness" + + CLOUDNESS_MAP = { + "Ясно": Cloudness.CLEAR, + "Малооблачно": Cloudness.PARTLY_CLOUDY, + "Облачно с прояснениями": Cloudness.PARTLY_CLOUDY, + "Переменная облачность": Cloudness.PARTLY_CLOUDY, + "Небольшая облачность": Cloudness.CLOUDY, + "Облачно": Cloudness.CLOUDY, + "Значительная облачность": Cloudness.MAINLY_CLOUDY, + "Пасмурная погода": Cloudness.MAINLY_CLOUDY, + } + + def extract(self, data: Tag) -> Cloudness: + onmouseover = data.select_one(".cc_0").next_element.attrs["onmouseover"] + cloudness_str = re.compile("(.*?)").search(onmouseover).group(1).strip() + return self.CLOUDNESS_MAP[cloudness_str] + + +class PrecipitationParser(TableValueParser[tuple[Precipitation, float]]): + selector = "tr:nth-of-type(4) > td:not(:first-of-type)" + model_key = ["sky.precipitation", "precipitation"] + + PRECIPITATION_MAP = { + "Явления погоды отсутствуют": Precipitation.NO, + "Cлабый дождь": Precipitation.SMALL_RAIN, + } + + def extract(self, data: Tag) -> tuple[Precipitation, float]: + onmouseover = data.select_one(".pr_0").attrs["onmouseover"] + precipitation_str = onmouseover.split(",")[1].strip(" '") + precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str) + precipitation_value = 0 + if precipitation_value_matched: + # one hour value + precipitation_value = float(precipitation_value_matched.group(1)) * 6 + if "(" in precipitation_str: + precipitation_str = precipitation_str.split(" (")[0] + return self.PRECIPITATION_MAP[precipitation_str], precipitation_value + + +class TemperatureParser(TableValueParser[list[int]]): + selector = "tr:nth-of-type(5) > td:not(:first-of-type)" + model_key = "temperature" + + def extract(self, data: Tag) -> list[int]: + return [int(data.select_one(".t_0").text.lstrip("+").strip())] + + +class PressureParser(TableValueParser[list[int]]): + selector = "tr:nth-last-of-type(8) > td:not(:first-of-type)" + model_key = "pressure" + + def extract(self, data: Tag) -> list[int]: + return [int(data.select_one(".p_0").text.strip())] + + +class WindSpeedParser(TableValueParser[int]): + selector = "tr:nth-last-of-type(7) > td:not(:first-of-type)" + model_key = "wind.speed" + + def extract(self, data: Tag) -> int: + element = data.select_one(".wv_0") + if element: + return int(element.text.strip() or "0") + return 0 + + +class WindGustParser(WindSpeedParser): + selector = "tr:nth-last-of-type(6) > td:not(:first-of-type)" + model_key = "wind.gust" + + +class WindDirectionParser(TableValueParser[WindDirection | None]): + selector = "tr:nth-last-of-type(5) > td:not(:first-of-type)" + model_key = "wind.direction" + + def extract(self, data: Tag) -> WindDirection | None: + return parse_wind_direction(data.text.strip()) + + +class HumidityParser(TableValueParser[int]): + selector = "tr:nth-last-of-type(4) > td:not(:first-of-type)" + model_key = "humidity" + + def extract(self, data: Tag) -> int: + return int(data.text.strip()) + + +class RP5TableParser(TableParser[WeatherValue]): + selector = "#forecastTable" + value_parsers = [ + TimeParser(), + CloudnessParser(), + PrecipitationParser(), + TemperatureParser(), + PressureParser(), + WindSpeedParser(), + WindGustParser(), + WindDirectionParser(), + HumidityParser(), + ] + + def create_model(self, data: dict) -> WeatherValue: + return WeatherValue.model_validate(data) + + +class RP5ForecastParser(Parser[str, list[WeatherResponse]]): + location_parser = LocationNameParser() + dates_parser = DatesParser() + table_parser = RP5TableParser() + + def parse(self, data: str) -> list[WeatherResponse]: + soup = BeautifulSoup(data, features="html.parser") + location = self.location_parser.parse(soup) + dates = self.dates_parser.parse(soup) + values = self.table_parser.parse(soup) + responses = [] + for date, count in dates: + item_values, values = values[:count], values[count:] + for value in item_values: + value.date = date.replace(hour=value.date.hour) + responses.append( + WeatherResponse( + location=location, + date=date, + period="day", + values=item_values, + ) + ) + return responses + + +class RP5LocationParser(Parser[str, list[Location]]): + + def parse(self, data: str) -> list[Location]: + soup = BeautifulSoup(data, features="html.parser") + table = soup.select_one(".searchResults") + result = [] + for row in table.select("tr")[1:]: + cells = row.select("td") + name = cells[0].text.strip() + location_id = cells[0].select_one("a").attrs["href"].lstrip("/") + country = cells[1].text.strip() + district = cells[2].text.strip() + subdistrict = cells[3].text.strip() + result.append( + Location( + id=location_id, + name=name, + provider="rp5", # TODO: provider module const + lat=0, + lon=0, + country=country, + country_code="ru", # TODO: contry_code from country + district=district, + subdistrict=subdistrict, + ) + ) + return result diff --git a/gallery/sketch/parse/html.py b/gallery/sketch/parse/html.py index fb00f59..9821a49 100644 --- a/gallery/sketch/parse/html.py +++ b/gallery/sketch/parse/html.py @@ -1,36 +1,58 @@ -from typing import Callable, Generic +from typing import Generic, Iterable from bs4 import Tag from .core import ParseError, Parser, R -from .util import set_nested_attr +from .util import set_dict_nested_value -class TableValueParser(Parser[Tag, R], Generic[R]): +class TagParser(Parser[Tag, R], Generic[R]): selector: str - model_key: str + + def extract(self, data: Tag) -> R: + raise NotImplementedError def parse(self, data: Tag) -> R: + tag = data.select_one(self.selector) + if tag is None: + raise ParseError("Tag not found", self.selector) + return self.extract(tag) + + +class TagIterParser(Parser[Tag, Iterable[R]], Generic[R]): + selector: str + + def extract(self, data: Tag) -> R: + raise NotImplementedError + + def parse(self, data: Tag) -> Iterable[R]: + tags = data.select(self.selector) + for tag in tags: + yield self.extract(tag) + + +class TableValueParser(TagIterParser[R], Generic[R]): + model_key: str | list[str] + + def extract(self, data: Tag) -> R: raise NotImplementedError -class TableParser(Parser[Tag, R], Generic[R]): - selector: str - model_factory: Callable[[], R] +class TableParser(TagParser[list[R]], Generic[R]): value_parsers: list[TableValueParser] - def create_model(self) -> R: - return self.model_factory() + def create_model(self, data: dict) -> R: + raise NotImplementedError - def parse(self, data: Tag) -> list[R]: - table = data.select_one(self.selector) - if table is None: - raise ParseError("Table not found", self.selector) - result = [] + def extract(self, data: Tag) -> list[R]: + results = [] for parser in self.value_parsers: - for index, tag in enumerate(table.select(parser.selector)): - value = parser.parse(tag) - while index > len(result) - 1: - result.append(self.create_model()) - set_nested_attr(result[index], parser.model_key, value) - return result + for index, value in enumerate(parser.parse(data)): + while index > len(results) - 1: + results.append({}) + if isinstance(parser.model_key, list): + for model_key_index, model_key in enumerate(parser.model_key): + set_dict_nested_value(results[index], model_key, value[model_key_index]) + else: + set_dict_nested_value(results[index], parser.model_key, value) + return [self.create_model(item) for item in results] diff --git a/gallery/sketch/weather/model.py b/gallery/sketch/weather/model.py index 31f08c9..7cc4477 100644 --- a/gallery/sketch/weather/model.py +++ b/gallery/sketch/weather/model.py @@ -51,10 +51,10 @@ Precipitation.values = list(Precipitation) class Sky(Model): - cloudness: Cloudness - precipitation: Precipitation - thunder: bool - fog: bool + cloudness: Cloudness = Cloudness.CLEAR + precipitation: Precipitation = Precipitation.NO + thunder: bool = False + fog: bool = False class WindDirection(StrEnum): @@ -69,8 +69,8 @@ class WindDirection(StrEnum): class Wind(Model): - speed: int - gust: int + speed: int = 0 + gust: int = 0 direction: WindDirection | None = None @@ -79,9 +79,9 @@ class WeatherValue(Model): sky: Sky wind: Wind temperature: list[int] - precipitation: float + precipitation: float = 0 pressure: list[int] = Field(default=[]) - humidity: int = Field(default=-1) + humidity: int = -1 class WeatherResponse(Model):