refactor(rp5): rework rp5 parser
This commit is contained in:
@@ -17,12 +17,12 @@ from gallery.sketch.weather.model import (
|
||||
WindDirection,
|
||||
)
|
||||
from gallery.sketch.weather.parser import parse_wind_direction
|
||||
from gallery.sketch.weather.util import build_weather_value, merge_weather_values
|
||||
from gallery.sketch.weather.util import merge_weather_values
|
||||
|
||||
logger = logging.getLogger("gismeteo")
|
||||
|
||||
|
||||
class LocationParser(Parser[str, str]):
|
||||
class LocationNameParser(Parser[str, str]):
|
||||
PATTERN = re.compile('{"ru":{"city":{"name":"(.*?)"')
|
||||
|
||||
def parse(self, data: str) -> str:
|
||||
@@ -36,7 +36,7 @@ class TimeParser(TableValueParser[datetime.datetime]):
|
||||
selector = ".widget-row.widget-row-datetime-time > .row-item > time-value"
|
||||
model_key = "date"
|
||||
|
||||
def parse(self, data: Tag) -> datetime.datetime:
|
||||
def extract(self, data: Tag) -> datetime.datetime:
|
||||
timestamp = int(data.attrs["timestamp"])
|
||||
return datetime.datetime.fromtimestamp(timestamp)
|
||||
|
||||
@@ -45,7 +45,7 @@ class DateParser(TableValueParser[datetime.datetime]):
|
||||
selector = ".widget-row.widget-row-date > .row-item"
|
||||
model_key = "date"
|
||||
|
||||
def parse(self, data: Tag) -> datetime.datetime:
|
||||
def extract(self, data: Tag) -> datetime.datetime:
|
||||
return parse_date(data.text)
|
||||
|
||||
|
||||
@@ -89,7 +89,7 @@ class SkyParser(TableValueParser[Sky]):
|
||||
result_values = values - flag_values
|
||||
return result_values, len(result_values) < len(values)
|
||||
|
||||
def parse(self, data: Tag) -> Sky:
|
||||
def extract(self, data: Tag) -> Sky:
|
||||
sky_str = data.attrs["data-tooltip"]
|
||||
values = {item.strip().lower() for item in sky_str.split(",")}
|
||||
cloudness = Cloudness.CLEAR
|
||||
@@ -120,7 +120,7 @@ class TemperatureParser(TableValueParser[list[int]]):
|
||||
selector = ".widget-row-chart[data-row=temperature-air] > .chart > .values > .value"
|
||||
model_key = "temperature"
|
||||
|
||||
def parse(self, data: Tag) -> list[int]:
|
||||
def extract(self, data: Tag) -> list[int]:
|
||||
return [int(value.attrs["value"]) for value in data.select("temperature-value")]
|
||||
|
||||
|
||||
@@ -128,7 +128,7 @@ class WindSpeedParser(TableValueParser[int]):
|
||||
selector = ".widget-row-wind > .row-item > .wind-speed"
|
||||
model_key = "wind.speed"
|
||||
|
||||
def parse(self, data: Tag) -> int:
|
||||
def extract(self, data: Tag) -> int:
|
||||
value = data.select_one("speed-value")
|
||||
return int(value.attrs["value"]) if value else 0
|
||||
|
||||
@@ -137,7 +137,7 @@ class WindGustParser(TableValueParser[int]):
|
||||
selector = ".widget-row-wind > .row-item > .wind-gust"
|
||||
model_key = "wind.gust"
|
||||
|
||||
def parse(self, data: Tag) -> int:
|
||||
def extract(self, data: Tag) -> int:
|
||||
value = data.select_one("speed-value")
|
||||
return int(value.attrs["value"]) if value else 0
|
||||
|
||||
@@ -146,7 +146,7 @@ class WindDirectionParser(TableValueParser[WindDirection | None]):
|
||||
selector = ".widget-row-wind > .row-item > .wind-speed > .wind-direction"
|
||||
model_key = "wind.direction"
|
||||
|
||||
def parse(self, data: Tag) -> WindDirection | None:
|
||||
def extract(self, data: Tag) -> WindDirection | None:
|
||||
return parse_wind_direction(data.text.strip())
|
||||
|
||||
|
||||
@@ -154,7 +154,7 @@ class PrecipitationParser(TableValueParser[float]):
|
||||
selector = ".widget-row[data-row=precipitation-bars] > .row-item"
|
||||
model_key = "precipitation"
|
||||
|
||||
def parse(self, data: Tag) -> float:
|
||||
def extract(self, data: Tag) -> float:
|
||||
value = data.select_one("precipitation-value")
|
||||
return float(value.attrs["value"]) if value else 0
|
||||
|
||||
@@ -163,7 +163,7 @@ class PressureParser(TableValueParser[list[int]]):
|
||||
selector = ".widget-row-chart[data-row=pressure] > .chart > .values > .value"
|
||||
model_key = "pressure"
|
||||
|
||||
def parse(self, data: Tag) -> list[int]:
|
||||
def extract(self, data: Tag) -> list[int]:
|
||||
return [int(value.attrs["value"]) for value in data.select("pressure-value")]
|
||||
|
||||
|
||||
@@ -171,13 +171,12 @@ class HumidityParser(TableValueParser[int]):
|
||||
selector = ".widget-row[data-row=humidity] > .row-item, .widget-row[data-row=humidity-avg] > .row-item"
|
||||
model_key = "humidity"
|
||||
|
||||
def parse(self, data: Tag) -> int:
|
||||
def extract(self, data: Tag) -> int:
|
||||
return int(data.text)
|
||||
|
||||
|
||||
class GismeteoTableParser(TableParser[WeatherValue]):
|
||||
selector = ".widget .widget-items"
|
||||
model_factory = lambda cls: build_weather_value()
|
||||
value_parsers = [
|
||||
TimeParser(),
|
||||
DateParser(),
|
||||
@@ -191,6 +190,9 @@ class GismeteoTableParser(TableParser[WeatherValue]):
|
||||
HumidityParser(),
|
||||
]
|
||||
|
||||
def create_model(self, data: dict) -> WeatherValue:
|
||||
return WeatherValue.model_validate(data)
|
||||
|
||||
|
||||
class Context(NamedTuple):
|
||||
date: datetime.date
|
||||
@@ -200,7 +202,7 @@ class Context(NamedTuple):
|
||||
|
||||
|
||||
class GismeteoParser(Parser[Context, WeatherResponse]):
|
||||
location_parser = LocationParser()
|
||||
location_parser = LocationNameParser()
|
||||
table_parser = GismeteoTableParser()
|
||||
|
||||
def parse(self, data: Context) -> WeatherResponse:
|
||||
|
||||
@@ -3,8 +3,8 @@ import logging
|
||||
|
||||
import aiohttp
|
||||
from aiocache import cached
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from gallery.sketch.headers import build_headers
|
||||
from gallery.sketch.source import ApiSource
|
||||
from gallery.sketch.weather.api import WeatherApi
|
||||
from gallery.sketch.weather.model import (
|
||||
@@ -14,8 +14,7 @@ from gallery.sketch.weather.model import (
|
||||
from gallery.sketch.weather.util import merge_weather_values
|
||||
from gallery.util import TimeUnit
|
||||
|
||||
from ...sketch.headers import build_headers
|
||||
from .parser import parse_page
|
||||
from .parser import RP5ForecastParser, RP5LocationParser
|
||||
|
||||
logger = logging.getLogger("rp5")
|
||||
|
||||
@@ -23,37 +22,17 @@ logger = logging.getLogger("rp5")
|
||||
class RP5WeatherApi(WeatherApi):
|
||||
PROVIDER = "rp5"
|
||||
|
||||
forecast_parser = RP5ForecastParser()
|
||||
location_parser = RP5LocationParser()
|
||||
|
||||
def _build_source(self) -> ApiSource:
|
||||
return ApiSource("https://rp5.ru", headers=build_headers("rp5.ru"))
|
||||
|
||||
async def find_locations(self, query: str) -> list[Location]:
|
||||
endpoint = "docs/search/ru"
|
||||
data = aiohttp.FormData({"searchStr": query})
|
||||
response = await self.source.request(endpoint, method="POST", data=data)
|
||||
soup = BeautifulSoup(response, features="html.parser")
|
||||
table = soup.select_one(".searchResults")
|
||||
result = []
|
||||
for row in table.select("tr")[1:]:
|
||||
cells = row.select("td")
|
||||
name = cells[0].text.strip()
|
||||
location_id = cells[0].select_one("a").attrs["href"].lstrip("/")
|
||||
country = cells[1].text.strip()
|
||||
district = cells[2].text.strip()
|
||||
subdistrict = cells[3].text.strip()
|
||||
result.append(
|
||||
Location(
|
||||
id=location_id,
|
||||
name=name,
|
||||
provider=self.provider,
|
||||
lat=0,
|
||||
lon=0,
|
||||
country=country,
|
||||
country_code="ru", # TODO: contry_code from country
|
||||
district=district,
|
||||
subdistrict=subdistrict,
|
||||
)
|
||||
)
|
||||
return result
|
||||
content = await self.source.request(endpoint, method="POST", data=data)
|
||||
return self.location_parser.parse(content)
|
||||
|
||||
@cached(
|
||||
key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast",
|
||||
@@ -63,8 +42,7 @@ class RP5WeatherApi(WeatherApi):
|
||||
async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]:
|
||||
endpoint = f"{location_id}"
|
||||
content = await self.source.request(endpoint)
|
||||
soup = BeautifulSoup(content, features="html.parser")
|
||||
return parse_page(soup)
|
||||
return self.forecast_parser.parse(content)
|
||||
|
||||
async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
|
||||
forecast = await self._get_location_forecast(location_id)
|
||||
|
||||
+171
-102
@@ -1,18 +1,59 @@
|
||||
import datetime
|
||||
import re
|
||||
from enum import Enum, auto
|
||||
from typing import Iterable
|
||||
|
||||
import dateparser
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from gallery.sketch.parse.core import Parser
|
||||
from gallery.sketch.parse.date import parse_date
|
||||
from gallery.sketch.parse.html import (
|
||||
TableParser,
|
||||
TableValueParser,
|
||||
TagIterParser,
|
||||
TagParser,
|
||||
)
|
||||
from gallery.sketch.weather.model import (
|
||||
Cloudness,
|
||||
Location,
|
||||
Precipitation,
|
||||
WeatherResponse,
|
||||
WeatherValue,
|
||||
WindDirection,
|
||||
)
|
||||
from gallery.sketch.weather.parser import ParseError, parse_wind_direction
|
||||
from gallery.sketch.weather.util import build_weather_value
|
||||
from gallery.sketch.weather.parser import parse_wind_direction
|
||||
|
||||
|
||||
class LocationNameParser(TagParser[str]):
|
||||
selector = ".intoLeftNavi > .verticalBottom > a"
|
||||
|
||||
def extract(self, data: Tag) -> str:
|
||||
return data.text.strip()
|
||||
|
||||
|
||||
class DatesParser(TagIterParser[tuple[datetime.datetime, int]]):
|
||||
selector = "#forecastTable > tr.forecastDate > td"
|
||||
|
||||
def extract(self, data: Tag) -> tuple[datetime.datetime, int]:
|
||||
colspan = int(data.attrs["colspan"])
|
||||
index = data.parent.index(data)
|
||||
if index == 1:
|
||||
colspan -= 1
|
||||
date_str = ",".join(data.text.strip().split(",")[-2:])
|
||||
date = parse_date(date_str)
|
||||
return (date, colspan)
|
||||
|
||||
|
||||
class TimeParser(TableValueParser[datetime.datetime]):
|
||||
selector = "tr.forecastTime > td:not(:first-of-type)"
|
||||
model_key = "date"
|
||||
|
||||
def extract(self, data: Tag) -> datetime.datetime:
|
||||
hour = int(data.text.strip())
|
||||
return datetime.datetime.min.replace(hour=hour)
|
||||
|
||||
|
||||
class CloudnessParser(TableValueParser[Cloudness]):
|
||||
selector = "tr:nth-of-type(3) > td:not(:first-of-type)"
|
||||
model_key = "sky.cloudness"
|
||||
|
||||
CLOUDNESS_MAP = {
|
||||
"Ясно": Cloudness.CLEAR,
|
||||
@@ -25,122 +66,150 @@ CLOUDNESS_MAP = {
|
||||
"Пасмурная погода": Cloudness.MAINLY_CLOUDY,
|
||||
}
|
||||
|
||||
def extract(self, data: Tag) -> Cloudness:
|
||||
onmouseover = data.select_one(".cc_0").next_element.attrs["onmouseover"]
|
||||
cloudness_str = re.compile("<b>(.*?)</b>").search(onmouseover).group(1).strip()
|
||||
return self.CLOUDNESS_MAP[cloudness_str]
|
||||
|
||||
|
||||
class PrecipitationParser(TableValueParser[tuple[Precipitation, float]]):
|
||||
selector = "tr:nth-of-type(4) > td:not(:first-of-type)"
|
||||
model_key = ["sky.precipitation", "precipitation"]
|
||||
|
||||
PRECIPITATION_MAP = {
|
||||
"Явления погоды отсутствуют": Precipitation.NO,
|
||||
"Cлабый дождь": Precipitation.SMALL_RAIN,
|
||||
}
|
||||
|
||||
def extract(self, data: Tag) -> tuple[Precipitation, float]:
|
||||
onmouseover = data.select_one(".pr_0").attrs["onmouseover"]
|
||||
precipitation_str = onmouseover.split(",")[1].strip(" '")
|
||||
precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str)
|
||||
precipitation_value = 0
|
||||
if precipitation_value_matched:
|
||||
# one hour value
|
||||
precipitation_value = float(precipitation_value_matched.group(1)) * 6
|
||||
if "(" in precipitation_str:
|
||||
precipitation_str = precipitation_str.split(" (")[0]
|
||||
return self.PRECIPITATION_MAP[precipitation_str], precipitation_value
|
||||
|
||||
def _parse_wind_speed(cell: Tag) -> int:
|
||||
element = cell.select_one(".wv_0")
|
||||
|
||||
class TemperatureParser(TableValueParser[list[int]]):
|
||||
selector = "tr:nth-of-type(5) > td:not(:first-of-type)"
|
||||
model_key = "temperature"
|
||||
|
||||
def extract(self, data: Tag) -> list[int]:
|
||||
return [int(data.select_one(".t_0").text.lstrip("+").strip())]
|
||||
|
||||
|
||||
class PressureParser(TableValueParser[list[int]]):
|
||||
selector = "tr:nth-last-of-type(8) > td:not(:first-of-type)"
|
||||
model_key = "pressure"
|
||||
|
||||
def extract(self, data: Tag) -> list[int]:
|
||||
return [int(data.select_one(".p_0").text.strip())]
|
||||
|
||||
|
||||
class WindSpeedParser(TableValueParser[int]):
|
||||
selector = "tr:nth-last-of-type(7) > td:not(:first-of-type)"
|
||||
model_key = "wind.speed"
|
||||
|
||||
def extract(self, data: Tag) -> int:
|
||||
element = data.select_one(".wv_0")
|
||||
if element:
|
||||
return int(element.text.strip() or "0")
|
||||
return 0
|
||||
|
||||
|
||||
class RowType(Enum):
|
||||
DATE = auto()
|
||||
TIME = auto()
|
||||
CLOUDNESS = auto()
|
||||
PRECIPITATION = auto()
|
||||
TEMPERATURE = auto()
|
||||
PRESSURE = auto()
|
||||
WIND_SPEED = auto()
|
||||
WIND_GUST = auto()
|
||||
WIND_DIRECTION = auto()
|
||||
HUMIDITY = auto()
|
||||
class WindGustParser(WindSpeedParser):
|
||||
selector = "tr:nth-last-of-type(6) > td:not(:first-of-type)"
|
||||
model_key = "wind.gust"
|
||||
|
||||
|
||||
def _iter_table_rows(table: Tag) -> Iterable[tuple[RowType, Tag]]:
|
||||
for row in table.select("tr"):
|
||||
title = row.select_one("td").text.strip().lower()
|
||||
if "сегодня" in title or "завтра" in title:
|
||||
yield RowType.DATE, row
|
||||
elif "местное время" in title:
|
||||
yield RowType.TIME, row
|
||||
elif "облачность" in title:
|
||||
yield RowType.CLOUDNESS, row
|
||||
elif "осадки" in title:
|
||||
yield RowType.PRECIPITATION, row
|
||||
elif "температура" in title:
|
||||
yield RowType.TEMPERATURE, row
|
||||
elif "давление" in title:
|
||||
yield RowType.PRESSURE, row
|
||||
elif "скорость" in title:
|
||||
yield RowType.WIND_SPEED, row
|
||||
elif "порывы" in title:
|
||||
yield RowType.WIND_GUST, row
|
||||
elif "направление" in title:
|
||||
yield RowType.WIND_DIRECTION, row
|
||||
elif "влажность" in title:
|
||||
yield RowType.HUMIDITY, row
|
||||
class WindDirectionParser(TableValueParser[WindDirection | None]):
|
||||
selector = "tr:nth-last-of-type(5) > td:not(:first-of-type)"
|
||||
model_key = "wind.direction"
|
||||
|
||||
def extract(self, data: Tag) -> WindDirection | None:
|
||||
return parse_wind_direction(data.text.strip())
|
||||
|
||||
|
||||
# pylint:disable=too-many-branches
|
||||
def parse_page(page: BeautifulSoup) -> list[WeatherResponse]:
|
||||
location = page.select(".intoLeftNavi > .verticalBottom > a")[-1].text.strip()
|
||||
table = page.select_one("#forecastTable")
|
||||
if table is None:
|
||||
raise ParseError("#forecastTable not found")
|
||||
result: list[WeatherResponse] = []
|
||||
values: list[WeatherValue] = []
|
||||
for row_type, row in _iter_table_rows(table):
|
||||
match row_type:
|
||||
case RowType.DATE:
|
||||
for index, cell in enumerate(row.select("td")):
|
||||
colspan = int(cell.attrs["colspan"])
|
||||
if index == 0:
|
||||
colspan -= 1
|
||||
date_str = ",".join(cell.text.strip().split(",")[-2:])
|
||||
date = dateparser.parse(date_str, languages=["ru"])
|
||||
if date is None:
|
||||
raise ParseError.invalid_date(date_str)
|
||||
item_values = [build_weather_value(date) for _ in range(colspan)]
|
||||
result.append(
|
||||
class HumidityParser(TableValueParser[int]):
|
||||
selector = "tr:nth-last-of-type(4) > td:not(:first-of-type)"
|
||||
model_key = "humidity"
|
||||
|
||||
def extract(self, data: Tag) -> int:
|
||||
return int(data.text.strip())
|
||||
|
||||
|
||||
class RP5TableParser(TableParser[WeatherValue]):
|
||||
selector = "#forecastTable"
|
||||
value_parsers = [
|
||||
TimeParser(),
|
||||
CloudnessParser(),
|
||||
PrecipitationParser(),
|
||||
TemperatureParser(),
|
||||
PressureParser(),
|
||||
WindSpeedParser(),
|
||||
WindGustParser(),
|
||||
WindDirectionParser(),
|
||||
HumidityParser(),
|
||||
]
|
||||
|
||||
def create_model(self, data: dict) -> WeatherValue:
|
||||
return WeatherValue.model_validate(data)
|
||||
|
||||
|
||||
class RP5ForecastParser(Parser[str, list[WeatherResponse]]):
|
||||
location_parser = LocationNameParser()
|
||||
dates_parser = DatesParser()
|
||||
table_parser = RP5TableParser()
|
||||
|
||||
def parse(self, data: str) -> list[WeatherResponse]:
|
||||
soup = BeautifulSoup(data, features="html.parser")
|
||||
location = self.location_parser.parse(soup)
|
||||
dates = self.dates_parser.parse(soup)
|
||||
values = self.table_parser.parse(soup)
|
||||
responses = []
|
||||
for date, count in dates:
|
||||
item_values, values = values[:count], values[count:]
|
||||
for value in item_values:
|
||||
value.date = date.replace(hour=value.date.hour)
|
||||
responses.append(
|
||||
WeatherResponse(
|
||||
location=location,
|
||||
date=date.date(),
|
||||
date=date,
|
||||
period="day",
|
||||
values=item_values,
|
||||
)
|
||||
)
|
||||
values += item_values
|
||||
case RowType.TIME:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
hour = int(cell.text.strip())
|
||||
values[index].date = values[index].date.replace(hour=hour)
|
||||
case RowType.CLOUDNESS:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
onmouseover = cell.select_one(".cc_0").next_element.attrs["onmouseover"]
|
||||
cloudness_str = re.compile("<b>(.*?)</b>").search(onmouseover).group(1).strip()
|
||||
values[index].sky.cloudness = CLOUDNESS_MAP[cloudness_str]
|
||||
case RowType.PRECIPITATION:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
onmouseover = cell.select_one(".pr_0").attrs["onmouseover"]
|
||||
precipitation_str = onmouseover.split(",")[1].strip(" '")
|
||||
precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str)
|
||||
if precipitation_value_matched:
|
||||
# one hour value
|
||||
values[index].precipitation = float(precipitation_value_matched.group(1)) * 6
|
||||
if "(" in precipitation_str:
|
||||
precipitation_str = precipitation_str.split(" (")[0]
|
||||
values[index].sky.precipitation = PRECIPITATION_MAP[precipitation_str]
|
||||
case RowType.TEMPERATURE:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
values[index].temperature = [int(cell.select_one(".t_0").text.lstrip("+").strip())]
|
||||
case RowType.PRESSURE:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
values[index].pressure = [int(cell.select_one(".p_0").text.strip())]
|
||||
case RowType.WIND_SPEED:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
values[index].wind.speed = _parse_wind_speed(cell)
|
||||
case RowType.WIND_GUST:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
values[index].wind.gust = _parse_wind_speed(cell)
|
||||
case RowType.WIND_DIRECTION:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
values[index].wind.direction = parse_wind_direction(cell.text.strip())
|
||||
case RowType.HUMIDITY:
|
||||
for index, cell in enumerate(row.select("td")[1:]):
|
||||
values[index].humidity = int(cell.text.strip())
|
||||
return responses
|
||||
|
||||
|
||||
class RP5LocationParser(Parser[str, list[Location]]):
|
||||
|
||||
def parse(self, data: str) -> list[Location]:
|
||||
soup = BeautifulSoup(data, features="html.parser")
|
||||
table = soup.select_one(".searchResults")
|
||||
result = []
|
||||
for row in table.select("tr")[1:]:
|
||||
cells = row.select("td")
|
||||
name = cells[0].text.strip()
|
||||
location_id = cells[0].select_one("a").attrs["href"].lstrip("/")
|
||||
country = cells[1].text.strip()
|
||||
district = cells[2].text.strip()
|
||||
subdistrict = cells[3].text.strip()
|
||||
result.append(
|
||||
Location(
|
||||
id=location_id,
|
||||
name=name,
|
||||
provider="rp5", # TODO: provider module const
|
||||
lat=0,
|
||||
lon=0,
|
||||
country=country,
|
||||
country_code="ru", # TODO: contry_code from country
|
||||
district=district,
|
||||
subdistrict=subdistrict,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -1,36 +1,58 @@
|
||||
from typing import Callable, Generic
|
||||
from typing import Generic, Iterable
|
||||
|
||||
from bs4 import Tag
|
||||
|
||||
from .core import ParseError, Parser, R
|
||||
from .util import set_nested_attr
|
||||
from .util import set_dict_nested_value
|
||||
|
||||
|
||||
class TableValueParser(Parser[Tag, R], Generic[R]):
|
||||
class TagParser(Parser[Tag, R], Generic[R]):
|
||||
selector: str
|
||||
model_key: str
|
||||
|
||||
def extract(self, data: Tag) -> R:
|
||||
raise NotImplementedError
|
||||
|
||||
def parse(self, data: Tag) -> R:
|
||||
tag = data.select_one(self.selector)
|
||||
if tag is None:
|
||||
raise ParseError("Tag not found", self.selector)
|
||||
return self.extract(tag)
|
||||
|
||||
|
||||
class TagIterParser(Parser[Tag, Iterable[R]], Generic[R]):
|
||||
selector: str
|
||||
|
||||
def extract(self, data: Tag) -> R:
|
||||
raise NotImplementedError
|
||||
|
||||
def parse(self, data: Tag) -> Iterable[R]:
|
||||
tags = data.select(self.selector)
|
||||
for tag in tags:
|
||||
yield self.extract(tag)
|
||||
|
||||
|
||||
class TableValueParser(TagIterParser[R], Generic[R]):
|
||||
model_key: str | list[str]
|
||||
|
||||
def extract(self, data: Tag) -> R:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class TableParser(Parser[Tag, R], Generic[R]):
|
||||
selector: str
|
||||
model_factory: Callable[[], R]
|
||||
class TableParser(TagParser[list[R]], Generic[R]):
|
||||
value_parsers: list[TableValueParser]
|
||||
|
||||
def create_model(self) -> R:
|
||||
return self.model_factory()
|
||||
def create_model(self, data: dict) -> R:
|
||||
raise NotImplementedError
|
||||
|
||||
def parse(self, data: Tag) -> list[R]:
|
||||
table = data.select_one(self.selector)
|
||||
if table is None:
|
||||
raise ParseError("Table not found", self.selector)
|
||||
result = []
|
||||
def extract(self, data: Tag) -> list[R]:
|
||||
results = []
|
||||
for parser in self.value_parsers:
|
||||
for index, tag in enumerate(table.select(parser.selector)):
|
||||
value = parser.parse(tag)
|
||||
while index > len(result) - 1:
|
||||
result.append(self.create_model())
|
||||
set_nested_attr(result[index], parser.model_key, value)
|
||||
return result
|
||||
for index, value in enumerate(parser.parse(data)):
|
||||
while index > len(results) - 1:
|
||||
results.append({})
|
||||
if isinstance(parser.model_key, list):
|
||||
for model_key_index, model_key in enumerate(parser.model_key):
|
||||
set_dict_nested_value(results[index], model_key, value[model_key_index])
|
||||
else:
|
||||
set_dict_nested_value(results[index], parser.model_key, value)
|
||||
return [self.create_model(item) for item in results]
|
||||
|
||||
@@ -51,10 +51,10 @@ Precipitation.values = list(Precipitation)
|
||||
|
||||
|
||||
class Sky(Model):
|
||||
cloudness: Cloudness
|
||||
precipitation: Precipitation
|
||||
thunder: bool
|
||||
fog: bool
|
||||
cloudness: Cloudness = Cloudness.CLEAR
|
||||
precipitation: Precipitation = Precipitation.NO
|
||||
thunder: bool = False
|
||||
fog: bool = False
|
||||
|
||||
|
||||
class WindDirection(StrEnum):
|
||||
@@ -69,8 +69,8 @@ class WindDirection(StrEnum):
|
||||
|
||||
|
||||
class Wind(Model):
|
||||
speed: int
|
||||
gust: int
|
||||
speed: int = 0
|
||||
gust: int = 0
|
||||
direction: WindDirection | None = None
|
||||
|
||||
|
||||
@@ -79,9 +79,9 @@ class WeatherValue(Model):
|
||||
sky: Sky
|
||||
wind: Wind
|
||||
temperature: list[int]
|
||||
precipitation: float
|
||||
precipitation: float = 0
|
||||
pressure: list[int] = Field(default=[])
|
||||
humidity: int = Field(default=-1)
|
||||
humidity: int = -1
|
||||
|
||||
|
||||
class WeatherResponse(Model):
|
||||
|
||||
Reference in New Issue
Block a user