Files
gallery/gallery/painting/gismeteo/parser.py
T

256 lines
8.8 KiB
Python

import datetime
import re
from dataclasses import dataclass
from typing import Any, Literal
from bs4 import Tag
from gallery.sketch.parse.core import ParsedContext, ParseError, Parser
from gallery.sketch.parse.date import DATE_PARSER
from gallery.sketch.parse.html import HTMLContextParser, TableParser, TableValueParser
from gallery.sketch.parse.json import JSONParser
from gallery.sketch.weather.model import (
Cloudness,
Location,
Precipitation,
Sky,
WeatherResponse,
WeatherValue,
WindDirection,
)
from gallery.sketch.weather.parse.wind import WIND_DIRECTION_PARSER
from gallery.sketch.weather.util import fix_dates_sequence, merge_weather_values
from . import PROVIDER
class LocationNameParser(Parser[str, str]):
PATTERN = re.compile('{"ru":{"city":{"name":"(.*?)"')
def parse(self, data: str) -> str:
match = self.PATTERN.search(data)
if match:
return match.group(1)
raise ParseError("Location not found")
class TimeParser(TableValueParser[datetime.datetime]):
selector = ".widget-row.widget-row-datetime-time > .row-item > time-value"
model_key = "date"
def extract(self, data: Tag) -> datetime.datetime:
timestamp = int(data.attrs["timestamp"])
return datetime.datetime.fromtimestamp(timestamp)
class DateParser(TableValueParser[datetime.datetime]):
selector = ".widget-row.widget-row-date > .row-item"
model_key = "date"
def extract(self, data: Tag) -> datetime.datetime:
return DATE_PARSER.parse(data.text)
class SkyParser(TableValueParser[Sky]):
selector = ".widget-row[data-row=icon-tooltip] > .row-item"
model_key = "sky"
CLOUDNESS_MAP: dict[str, Cloudness] = {
"ясно": Cloudness.CLEAR,
"безоблачно": Cloudness.CLEAR,
"малооблачно": Cloudness.PARTLY_CLOUDY,
"облачно": Cloudness.CLOUDY,
"пасмурно": Cloudness.MAINLY_CLOUDY,
}
PRECIPITATION_MAP: dict[str, Precipitation] = {
"без осадков": Precipitation.NO,
"без существенных осадков": Precipitation.NO,
"небольшой дождь": Precipitation.SMALL_RAIN,
"сильный дождь": Precipitation.HEAVY_RAIN,
"очень сильный дождь": Precipitation.HEAVY_RAIN,
"ливневый дождь": Precipitation.SHOWER,
"дождь": Precipitation.RAIN,
"ливень": Precipitation.SHOWER,
"снег": Precipitation.SNOW,
"небольшой снег": Precipitation.SNOW,
"сильный снег": Precipitation.HEAVY_SNOW,
"мокрый снег": Precipitation.SNOW,
"снег с дождём": Precipitation.SNOW,
"сильный снег с дождём": Precipitation.HEAVY_SNOW,
"небольшой снег с дождём": Precipitation.SNOW,
"небольшой мокрый снег": Precipitation.SNOW,
"град": Precipitation.HAIL,
"небольшая морось": Precipitation.SMALL_RAIN,
"морось": Precipitation.SMALL_RAIN,
}
THUNDER = {"гроза"}
FOG = {"дымка", "туман"}
@classmethod
def _detect_flag(cls, values: set[str], flag_values: set[str]) -> tuple[set[str], bool]:
result_values = values - flag_values
return result_values, len(result_values) < len(values)
def extract(self, data: Tag) -> Sky:
sky_str = data.attrs["data-tooltip"]
values = {item.strip().lower() for item in sky_str.split(",")}
cloudness = Cloudness.CLEAR
precipitation = Precipitation.NO
values, thunder = self._detect_flag(values, self.THUNDER)
values, fog = self._detect_flag(values, self.FOG)
for k, v in self.CLOUDNESS_MAP.items():
if k in values:
cloudness = v
values.remove(k)
break
for k, v in self.PRECIPITATION_MAP.items():
if k in values:
precipitation = v
values.remove(k)
break
if values:
raise ParseError("Unknown sky values", values)
return Sky(
cloudness=cloudness,
precipitation=precipitation,
thunder=thunder,
fog=fog,
)
class TemperatureParser(TableValueParser[list[int]]):
selector = ".widget-row-chart[data-row=temperature-air] > .chart > .values > .value"
model_key = "temperature"
def extract(self, data: Tag) -> list[int]:
return [int(value.attrs["value"]) for value in data.select("temperature-value")]
class WindSpeedParser(TableValueParser[int]):
selector = ".widget-row-wind > .row-item > .wind-speed"
model_key = "wind.speed"
def extract(self, data: Tag) -> int:
value = data.select_one("speed-value")
return int(value.attrs["value"]) if value else 0
class WindGustParser(TableValueParser[int]):
selector = ".widget-row-wind > .row-item > .wind-gust"
model_key = "wind.gust"
def extract(self, data: Tag) -> int:
value = data.select_one("speed-value")
return int(value.attrs["value"]) if value else 0
class WindDirectionParser(TableValueParser[WindDirection | None]):
selector = ".widget-row-wind > .row-item > .wind-speed > .wind-direction"
model_key = "wind.direction"
def extract(self, data: Tag) -> WindDirection | None:
return WIND_DIRECTION_PARSER.parse(data.text.strip())
class PrecipitationParser(TableValueParser[float]):
selector = ".widget-row[data-row=precipitation-bars] > .row-item"
model_key = "precipitation"
def extract(self, data: Tag) -> float:
value = data.select_one("precipitation-value")
return float(value.attrs["value"]) if value else 0
class PressureParser(TableValueParser[list[int]]):
selector = ".widget-row-chart[data-row=pressure] > .chart > .values > .value"
model_key = "pressure"
def extract(self, data: Tag) -> list[int]:
return [int(value.attrs["value"]) for value in data.select("pressure-value")]
class HumidityParser(TableValueParser[int]):
selector = ".widget-row[data-row=humidity] > .row-item, .widget-row[data-row=humidity-avg] > .row-item"
model_key = "humidity"
def extract(self, data: Tag) -> int:
return int(data.text)
class GismeteoTableParser(TableParser[WeatherValue]):
selector = ".widget .widget-items"
value_parsers = [
TimeParser(),
DateParser(),
SkyParser(),
TemperatureParser(),
WindSpeedParser(),
WindGustParser(),
WindDirectionParser(),
PrecipitationParser(),
PressureParser(),
HumidityParser(),
]
def create_model(self, data: dict) -> WeatherValue:
return WeatherValue.model_validate(data)
@dataclass
class ForecastData:
date: datetime.date
period: Literal["day", "days"]
data: str
values: int | None = None
class GismeteoForecastParser(HTMLContextParser[ForecastData, WeatherResponse]):
location_parser = LocationNameParser()
table_parser = GismeteoTableParser()
def extract(self, data: ParsedContext[ForecastData, Tag]) -> WeatherResponse:
location = self.location_parser.parse(data.context.data)
values = self.table_parser.parse(data.data)
if data.context.values and len(values) > data.context.values:
n = int(len((values)) / data.context.values)
values = [
merge_weather_values(values[i].date, values[i : i + n], "average") for i in range(0, len(values), n)
]
if data.context.period == "days":
fix_dates_sequence(values)
return WeatherResponse(
location=location,
date=data.context.date,
period=data.context.period,
values=values,
)
class GismeteoLocationParser(JSONParser[list[Location]]):
def extract(self, data: Any) -> list[Location]:
result = []
for item in data["data"]:
result.append(
Location(
id=f"{item['slug']}-{item['id']}",
name=item["translations"]["kk"]["city"]["name"],
provider=PROVIDER,
lat=item["coordinates"]["latitude"],
lon=item["coordinates"]["longitude"],
country=item["translations"]["kk"]["country"]["name"],
country_code=item["country"]["code"].lower(),
district=(
item["translations"]["kk"]["district"]["name"] if item["translations"]["kk"]["district"] else ""
),
subdistrict=(
item["translations"]["kk"]["subdistrict"]["name"]
if "subdistrict" in item["translations"]["kk"]
else ""
),
)
)
return result