feat(yandexweather): add yandexweather api

gallery/easel/route/view/weather/templates/weather.html

@@ -25,6 +25,7 @@
   </a>
   {% endif %}
   {% if response.period == 'days' %}
+  <span class="badge rounded-pill text-bg-secondary fs-6">{{provider}}</span>
   <span>{{response.location}} | {{format_date(response.date, DATE_FORMAT, locale=request.state.language)}}</span>
   <span>- {{format_date(response.date + datetime.timedelta(days=(response.values | length - 1)), DATE_FORMAT,
     locale=request.state.language)}}</span>
@@ -121,7 +122,7 @@
         <td class="wind"
             style="background-color: rgba(128, 128, 128, {{value.wind_speed * 0.05}});">
           <span class="speed">{{value.wind_speed}}</span>
-          {% if value.wind_gust != value.wind_speed %}
+          {% if value.wind_gust != value.wind_speed and value.wind_gust > 0 %}
           <span class="gust">
             ({{value.wind_gust}})
           </span>

gallery/main.py

@@ -8,6 +8,7 @@ from gallery.painting.gismeteo.api import GismeteoApi
 from gallery.painting.matchtv.api import MatchTvApi
 from gallery.painting.openweather.api import OpenWeatherApi
 from gallery.painting.yandextv.api import YandexTvApi
+from gallery.painting.yandexweather.api import YandexWeatherApi
 from gallery.sketch.bundle import ApiBundle
 from gallery.sketch.schedule.cached import CachedScheduleApi
 from gallery.sketch.weather.cached import CachedWeatherApi
@@ -18,6 +19,7 @@ api = ApiBundle(
         CachedScheduleApi(MatchTvApi()),
         CachedWeatherApi(GismeteoApi()),
         CachedWeatherApi(OpenWeatherApi()),
+        CachedWeatherApi(YandexWeatherApi()),
     ]
 )
 app = build_app(api)

gallery/painting/gismeteo/api.py

@@ -20,10 +20,12 @@ logger = logging.getLogger("gismeteo")
 
 class GismeteoApi(WeatherApi):
     PROVIDER = "gismeteo"
-    SOURCE = ApiSource(
-        "https://www.gismeteo.ru",
-        cookies={"xauth": auth} if (auth := environ.get("GISMETEO_AUTH")) else None,
-    )
+
+    def _build_source(self) -> ApiSource:
+        return ApiSource(
+            "https://www.gismeteo.ru",
+            cookies={"xauth": auth} if (auth := environ.get("GISMETEO_AUTH")) else None,
+        )
 
     def _parse_oneday(self, date: datetime.date, data: str) -> WeatherResponse:
         result: list[dict[str, Any]] = []
@@ -70,7 +72,7 @@ class GismeteoApi(WeatherApi):
         latitude = 52.968498
         longitude = 36.0695
         data = json.loads(
-            await self.SOURCE.request(
+            await self.source.request(
                 f"mq/city/q/?q={query}&geo={geo}&latitude={latitude}&longitude={longitude}&limit=10"
             )
         )
@@ -103,9 +105,9 @@ class GismeteoApi(WeatherApi):
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST, detail={"max_date": max_date.strftime("%Y-%m-%d")}
             )
-        data = await self.SOURCE.request(f"weather-{location_id}/{datehelp.dump(date)}")
+        data = await self.source.request(f"weather-{location_id}/{datehelp.dump(date)}")
         return self._parse_oneday(date, data)
 
     async def get_days(self, location_id: str, days: int) -> WeatherResponse:
-        data = await self.SOURCE.request(f"weather-{location_id}/{days}-days")
+        data = await self.source.request(f"weather-{location_id}/{days}-days")
         return self._parse_manydays(data)

gallery/painting/gismeteo/parser.py

@@ -11,8 +11,8 @@ from gallery.sketch.weather.model import (
     Precipitation,
     Sky,
     WindDirection,
-    WindDirectionDeg,
 )
+from gallery.sketch.weather.parser import parse_wind_direction
 
 from .core import BaseWidgetParser, RowParser
 
@@ -147,24 +147,10 @@ class WindGustParser(RowParser[int]):
 class WindDirectionParser(RowParser[WindDirection]):
     KEY = "wind_direction"
 
-    WIND_DIRECTION_MAP: dict[str, WindDirection] = {
-        "—": WindDirection.CALM,
-        "": WindDirection.CALM,
-        "штиль": WindDirection.CALM,
-        "с": WindDirection.N,
-        "св": WindDirection.NE,
-        "в": WindDirection.E,
-        "юв": WindDirection.SE,
-        "ю": WindDirection.S,
-        "юз": WindDirection.SW,
-        "з": WindDirection.W,
-        "сз": WindDirection.NW,
-    }
-
     def parse_row(self, tag: Tag) -> Iterable[float]:
         for item in tag.select(".widget-row-wind > .row-item > .wind-speed > .wind-direction"):
             wind_direction_str = item.text.lower().strip()
-            yield WindDirectionDeg.from_direction(self.WIND_DIRECTION_MAP[wind_direction_str]).value
+            yield parse_wind_direction(wind_direction_str)
 
 
 class PrecipitationParser(RowParser[float]):

gallery/painting/matchtv/api.py

@@ -11,11 +11,13 @@ logger = logging.getLogger("matchtv")
 
 class MatchTvApi(ScheduleApi):
     PROVIDER = "matchtv"
-    SOURCE = ApiSource("https://matchtv.ru")
+
+    def _build_source(self) -> ApiSource:
+        return ApiSource("https://matchtv.ru")
 
     async def find_channels(self, query: str) -> list[Channel]:
         endpoint = "api/v1/channels"
-        data = json.loads(await self.SOURCE.request(endpoint))
+        data = json.loads(await self.source.request(endpoint))
         result = []
         query = query.lower()
         for item in data["result"]:
@@ -27,7 +29,7 @@ class MatchTvApi(ScheduleApi):
 
     async def get_schedule(self, channel_id: str, date: datetime.date) -> Schedule:
         endpoint = f"api/v1/channels/{channel_id}/tv-schedule?date={date:%Y%m%d}"
-        data = json.loads(await self.SOURCE.request(endpoint))
+        data = json.loads(await self.source.request(endpoint))
         channel_data = data["result"]["channels"][0]
         channel = Channel(id=channel_data["alias"], name=channel_data["name"], provider=self.provider)
         values = []

gallery/painting/openweather/api.py

@@ -18,9 +18,11 @@ from .parser import FORECAST_ITEM_PARSER
 logger = logging.getLogger("openweather")
 
 
-class OpenWeatherApi(WeatherApi):
+class OpenWeatherApi(WeatherApi[OpenWeather]):
     PROVIDER = "openweather"
-    SOURCE = OpenWeather(environ["OPENWEATHER_KEY"])
+
+    def _build_source(self) -> OpenWeather:
+        return OpenWeather(environ["OPENWEATHER_KEY"])
 
     @classmethod
     def _parse_location(cls, location_id: str) -> tuple[float, float]:
@@ -32,7 +34,7 @@ class OpenWeatherApi(WeatherApi):
         ttl=TimeUnit.DAY,
     )
     async def _get_location(self, location_id: str) -> OpenWeatherLocation:
-        return await self.SOURCE.get_location(*self._parse_location(location_id))
+        return await self.source.get_location(*self._parse_location(location_id))
 
     @cached(
         key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast",
@@ -40,10 +42,10 @@ class OpenWeatherApi(WeatherApi):
         ttl=TimeUnit.HOUR,
     )
     async def _get_location_forecast(self, location_id: str) -> Forecast:
-        return await self.SOURCE.get_forecast(*self._parse_location(location_id))
+        return await self.source.get_forecast(*self._parse_location(location_id))
 
     async def find_locations(self, query: str) -> list[Location]:
-        result = await self.SOURCE.find_locations(query)
+        result = await self.source.find_locations(query)
         return [
             Location(
                 id=f"{item.lat}:{item.lon}",

gallery/painting/openweather/openweather.py

@@ -76,28 +76,28 @@ class Location(Model):
     local_names: dict[str, str] | None = None
 
 
-class OpenWeather:
+class OpenWeather(ApiSource):
     BASE_URL = "https://api.openweathermap.org"
 
     def __init__(self, api_key: str):
         self._api_key = api_key
-        self._source = ApiSource(self.BASE_URL)
+        super().__init__(self.BASE_URL)
 
     async def get_forecast(self, lat: float, lon: float) -> Forecast:
         endpoint = f"data/2.5/forecast?lat={lat}&lon={lon}&appid={self._api_key}&units=metric"
-        response = await self._source.request(endpoint)
+        response = await self.request(endpoint)
         response_data = json.loads(response)
         return Forecast.model_validate(response_data)
 
     async def find_locations(self, query: str, limit: int = 5) -> list[Location]:
         endpoint = f"geo/1.0/direct?q={query}&limit={limit}&appid={self._api_key}"
-        response = await self._source.request(endpoint)
+        response = await self.request(endpoint)
         response_data = json.loads(response)
         return [Location.model_validate(item) for item in response_data]
 
     async def get_location(self, lat: float, lon: float) -> Location:
         limit = 1
         endpoint = f"geo/1.0/reverse?lat={lat}&lon={lon}&limit={limit}&appid={self._api_key}"
-        response = await self._source.request(endpoint)
+        response = await self.request(endpoint)
         response_data = json.loads(response)
         return Location.model_validate(response_data[0])

gallery/painting/yandextv/api.py

@@ -43,14 +43,16 @@ HEADERS: dict[str, str] = {
 
 class YandexTvApi(ScheduleApi):
     PROVIDER = "yandextv"
-    SOURCE = ApiSource("https://tv.yandex.ru", headers=HEADERS)
+
+    def _build_source(self) -> ApiSource:
+        return ApiSource("https://tv.yandex.ru", headers=HEADERS)
 
     async def find_channels(self, query: str) -> list[Channel]:
         url = (
             "https://suggest-multi.yandex.ru/suggest-tv2?"
             f"v=4&uil=ru&lr=10&count_channels=4&count_programs=4&sn=50&part={query}"
         )
-        _, values = json.loads(await self.SOURCE.request(url))
+        _, values = json.loads(await self.source.request(url))
         result = []
         for _, name, content in values:
             if content["label"] == "Каналы":
@@ -60,7 +62,7 @@ class YandexTvApi(ScheduleApi):
 
     async def get_schedule(self, channel_id: str, date: datetime.date) -> Schedule:
         endpoint = f"channels/{channel_id}?date={date:%Y-%m-%d}"
-        data = await self.SOURCE.request(endpoint)
+        data = await self.source.request(endpoint)
         soup = BeautifulSoup(data, features="html.parser")
         if soup.select_one(".CheckboxCaptcha") is not None:
             raise RuntimeError("Captcha")

gallery/painting/yandexweather/api.py

@@ -0,0 +1,75 @@
+import datetime
+import json
+import logging
+
+from aiocache import cached
+from bs4 import BeautifulSoup
+
+from gallery.sketch.source import ApiSource
+from gallery.sketch.weather.api import WeatherApi
+from gallery.sketch.weather.model import (
+    Location,
+    WeatherResponse,
+)
+from gallery.sketch.weather.util import merge_weather_values
+from gallery.util import TimeUnit
+
+from .parser import parse_article
+
+logger = logging.getLogger("yandexweather")
+
+
+class YandexWeatherApi(WeatherApi):
+    PROVIDER = "yandex"
+
+    def _build_source(self) -> ApiSource:
+        return ApiSource("https://yandex.ru")
+
+    @cached(
+        key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast",
+        alias="redis",
+        ttl=TimeUnit.HOUR,
+    )
+    async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]:
+        endpoint = f"pogoda/ru/{location_id}/details"
+        content = await self.source.request(endpoint, mode="browser")
+        soup = BeautifulSoup(content, features="html.parser")
+        location = soup.select_one("[class^=Suggest_placeholder]").text.strip()
+        articles = soup.select("article[class^=AppForecastDay]")
+        return [parse_article(article, location) for article in articles]
+
+    async def find_locations(self, query: str) -> list[Location]:
+        endpoint = f"weather/api/suggest?part={query}&lang=ru&type=weather"
+        response = await self.source.request(endpoint)
+        data = json.loads(response)
+        return [
+            Location(
+                id=str(item["geoId"]),
+                name=item["title"],
+                provider=self.provider,
+                lat=item["coords"]["lat"],
+                lon=item["coords"]["lon"],
+                country="ru",
+                country_code="ru",
+                district=item["description"],
+                subdistrict="",
+            )
+            for item in data
+        ]
+
+    async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
+        forecast = await self._get_location_forecast(location_id)
+        for response in forecast:
+            if response.date == date:
+                return response
+        raise ValueError(f"Unsupported date: {date}")
+
+    async def get_days(self, location_id: str, days: int) -> WeatherResponse:
+        forecast = await self._get_location_forecast(location_id)
+        values = [merge_weather_values(item.date, item.values) for item in forecast]
+        return WeatherResponse(
+            location=forecast[0].location,
+            date=datetime.date.today(),
+            period="days",
+            values=values,
+        )

gallery/painting/yandexweather/parser.py

@@ -0,0 +1,89 @@
+import logging
+from enum import Enum
+
+import dateparser
+from bs4 import Tag
+
+from gallery.sketch.weather.model import Cloudness, Precipitation, Sky, WeatherResponse
+from gallery.sketch.weather.parser import ParseError, parse_wind_direction
+from gallery.sketch.weather.util import build_weather_value
+
+logger = logging.getLogger("yandexweather")
+
+
+class DayPart(Enum):
+    MORNING = ("Утром", 6)
+    AFTERNOON = ("Днём", 12)
+    EVENING = ("Вечером", 18)
+    NIGHT = ("Ночью", 0)
+
+    def __init__(self, title: str, hour: int):
+        super().__init__()
+        self.title = title
+        self.hour = hour
+
+
+def parse_sky(sky: str) -> Sky:
+    cloudness = Cloudness.CLEAR
+    precipitation = Precipitation.NO
+    thunder = False
+    fog = False
+    if sky == "ясно":
+        pass
+    elif sky == "облачно с прояснениями":
+        cloudness = Cloudness.PARTLY_CLOUDY
+    elif sky == "малооблачно":
+        cloudness = Cloudness.CLOUDY
+    elif sky == "пасмурно":
+        cloudness = Cloudness.MAINLY_CLOUDY
+    elif sky == "небольшой дождь":
+        precipitation = Precipitation.RAIN
+    elif sky == "дождь":
+        precipitation = Precipitation.SMALL_RAIN
+    else:
+        logger.warning("unknown sky value: %s:", sky)
+    return Sky(
+        cloudness=cloudness,
+        precipitation=precipitation,
+        thunder=thunder,
+        fog=fog,
+    )
+
+
+def parse_article(article: Tag, location: str) -> WeatherResponse:
+    date_str = article.select_one("[class^=AppForecastDayHeader_dayTitle]").text.split(",")[-1].strip()
+    date = dateparser.parse(date_str, languages=["ru"])
+    if date is None:
+        raise ParseError("Invalid date string", date_str)
+    values = []
+    for day_part in DayPart:
+        value = build_weather_value(date.replace(hour=day_part.hour))
+        day_part_tag = article.find(string=day_part.title)
+        temperature_tag = day_part_tag.next_element
+        temperature = int(temperature_tag.text.strip().rstrip("°"))
+        value.temperature.append(temperature)
+        sky_tag = temperature_tag.next_element.next_element.next_element
+        sky_str = sky_tag.text.strip()
+        value.sky = parse_sky(sky_str)
+        temperature_tag = sky_tag.next_element.next_element
+        temperature = int(temperature_tag.text.strip().rstrip("°"))
+        value.temperature.append(temperature)
+        wind_speed_tag = temperature_tag.next_element.next_element
+        wind_speed = int(wind_speed_tag.text.strip())
+        value.wind_speed = wind_speed
+        wind_direction_tag = wind_speed_tag.next_element.next_element.next_element.next_element
+        wind_direction_str = wind_direction_tag.text.strip()
+        value.wind_direction = parse_wind_direction(wind_direction_str)
+        humidity_tag = wind_direction_tag.next_element.next_element.next_element.next_element
+        humidity = int(humidity_tag.text.strip().rstrip("%"))
+        value.humidity = humidity
+        pressure_tag = humidity_tag.next_element.next_element
+        pressure = int(pressure_tag.text.strip())
+        value.pressure.append(pressure)
+        values.append(value)
+    return WeatherResponse(
+        location=location,
+        date=date,
+        period="day",
+        values=values,
+    )

gallery/sketch/api.py

@@ -1,10 +1,19 @@
 from typing import TypeVar
 
+from .source import ApiSource
 
-class Api:
+S = TypeVar("S", bound=ApiSource)
+
+
+class Api[S = ApiSource]:
     TYPE: str
     PROVIDER: str
 
+    _source: S | None = None
+
+    def _build_source(self) -> S:
+        raise NotImplementedError
+
     @property
     def type(self) -> str:
         return self.TYPE
@@ -13,5 +22,11 @@ class Api:
     def provider(self) -> str:
         return self.PROVIDER
 
+    @property
+    def source(self) -> S:
+        if self._source is None:
+            self._source = self._build_source()
+        return self._source
+
 
 API = TypeVar("API", bound=Api)

gallery/sketch/cached.py

@@ -1,5 +1,6 @@
 from typing import Generic, NamedTuple
 
+from gallery.sketch.source import ApiSource
 from gallery.util import TimeUnit
 
 from .api import API, Api
@@ -19,6 +20,9 @@ class CachedApi(Api, Generic[API]):
     def __init__(self, api: API):
         self._api = api
 
+    def _build_source(self) -> ApiSource:
+        raise RuntimeError("Source not used in CachedApi")
+
     @property
     def provider(self) -> str:
         return self._api.provider

gallery/sketch/schedule/api.py

@@ -1,10 +1,11 @@
 import datetime
 
 from ..api import Api
+from ..source import ApiSource
 from .model import Channel, Schedule
 
 
-class ScheduleApi(Api):
+class ScheduleApi[S = ApiSource](Api[S]):
     TYPE = "schedule"
 
     async def find_channels(self, query: str) -> list[Channel]:

gallery/sketch/source.py

@@ -1,9 +1,15 @@
 import logging
+from typing import Literal
 
 import aiohttp
+from playwright.async_api import async_playwright
 
 logger = logging.getLogger("source")
 
+RequestMode = Literal["base", "browser"]
+
+DEFAULT_REQUEST_MODE: RequestMode = "base"
+
 
 class ApiSource:
     DEFAULT_USER_AGENT = (
@@ -25,12 +31,7 @@ class ApiSource:
         self._cookies = cookies
         self._headers = headers
 
-    async def request(self, endpoint: str) -> str:
-        if endpoint.startswith("https:"):
-            url = endpoint
-        else:
-            url = f"{self._base_url}/{endpoint}"
-        logger.info(url)
+    async def _base_request(self, url: str) -> str:
         headers = {"User-Agent": self._user_agent, **(self._headers or {})}
         async with aiohttp.ClientSession(
             headers=headers,
@@ -39,3 +40,21 @@ class ApiSource:
         ) as session:
             async with session.request("GET", url, timeout=self._timeout) as response:
                 return await response.text()
+
+    async def _browser_request(self, url: str) -> str:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch()
+            page = await browser.new_page()
+            await page.goto(url, wait_until="networkidle")
+            content = await page.content()
+            await browser.close()
+        return content
+
+    async def request(self, endpoint: str, /, mode: RequestMode = DEFAULT_REQUEST_MODE) -> str:
+        if endpoint.startswith("https:"):
+            url = endpoint
+        else:
+            url = f"{self._base_url}/{endpoint}"
+        logger.info(url)
+        method = getattr(self, f"_{mode}_request")
+        return await method(url)

gallery/sketch/weather/api.py

@@ -1,10 +1,11 @@
 import datetime
 
 from ..api import Api
+from ..source import ApiSource
 from .model import Location, WeatherResponse
 
 
-class WeatherApi(Api):
+class WeatherApi[S = ApiSource](Api[S]):
     TYPE = "weather"
 
     async def find_locations(self, query: str) -> list[Location]:

gallery/sketch/weather/parser.py

@@ -0,0 +1,24 @@
+from gallery.sketch.weather.model import WindDirection, WindDirectionDeg
+
+
+class ParseError(Exception):
+    pass
+
+
+WIND_DIRECTION_MAP: dict[str, WindDirection] = {
+    "—": WindDirection.CALM,
+    "": WindDirection.CALM,
+    "штиль": WindDirection.CALM,
+    "с": WindDirection.N,
+    "св": WindDirection.NE,
+    "в": WindDirection.E,
+    "юв": WindDirection.SE,
+    "ю": WindDirection.S,
+    "юз": WindDirection.SW,
+    "з": WindDirection.W,
+    "сз": WindDirection.NW,
+}
+
+
+def parse_wind_direction(value: str) -> WindDirectionDeg:
+    return WindDirectionDeg.from_direction(WIND_DIRECTION_MAP[value.lower()])

gallery/sketch/weather/util.py

@@ -19,7 +19,7 @@ def build_weather_value(date: datetime.datetime) -> WeatherValue:
         wind_direction=WindDirectionDeg(-1),
         precipitation=0,
         pressure=[],
-        humidity=0,
+        humidity=-1,
     )