refactor(tv): rework tv parsers

README.md

@@ -66,7 +66,7 @@ https://api.shmyga.ru/api/schedule/{provider}/{channel_id}/{date}
 | Provider | Site | API sample |
 | :- | :- | :- |
 | matchtv | https://matchtv.ru/tvguide | https://api.shmyga.ru/api/schedule/matchtv/igra/today |
-| yandextv | https://tv.yandex.ru | https://api.shmyga.ru/api/schedule/yandextv/146/today |
+| yandex | https://tv.yandex.ru | https://api.shmyga.ru/api/schedule/yandex/146/today |
 
 ## Demo
 

gallery/easel/depends/schedule.py

@@ -16,10 +16,10 @@ class ChannelDef(NamedTuple):
 
 GROUP_CANNELS_MAP: dict[str, list[ChannelDef]] = {
     "central": [
-        ChannelDef("yandextv", "pervyy-16"),
+        ChannelDef("yandex", "pervyy-16"),
-        ChannelDef("yandextv", "rossiya-1-31"),
+        ChannelDef("yandex", "rossiya-1-31"),
-        ChannelDef("yandextv", "ntv-11"),
+        ChannelDef("yandex", "ntv-11"),
-        ChannelDef("yandextv", "sts-8"),
+        ChannelDef("yandex", "sts-8"),
     ],
     "sport": [
         ChannelDef("matchtv", "matchtv"),
@@ -29,7 +29,7 @@ GROUP_CANNELS_MAP: dict[str, list[ChannelDef]] = {
         ChannelDef("matchtv", "futbol-2"),
         ChannelDef("matchtv", "futbol-3"),
         ChannelDef("matchtv", "strana"),
-        ChannelDef("yandextv", "start-103"),
+        ChannelDef("yandex", "start-103"),
     ],
 }
 

gallery/main.py

@@ -5,10 +5,10 @@ import uvicorn
 
 from gallery.easel import build_app
 from gallery.painting.gismeteo.api import GismeteoApi
-from gallery.painting.matchtv.api import MatchTvApi
+from gallery.painting.matchtv.api import MatchTVApi
 from gallery.painting.openweather.api import OpenWeatherApi
 from gallery.painting.rp5.api import RP5WeatherApi
-from gallery.painting.yandextv.api import YandexTvApi
+from gallery.painting.yandextv.api import YandexTVApi
 from gallery.painting.yandexweather.api import YandexWeatherApi
 from gallery.sketch.bundle import ApiBundle
 from gallery.sketch.schedule.cached import CachedScheduleApi
@@ -16,8 +16,8 @@ from gallery.sketch.weather.cached import CachedWeatherApi
 
 api = ApiBundle(
     [
-        CachedScheduleApi(YandexTvApi()),
+        CachedScheduleApi(YandexTVApi()),
-        CachedScheduleApi(MatchTvApi()),
+        CachedScheduleApi(MatchTVApi()),
         CachedWeatherApi(GismeteoApi()),
         CachedWeatherApi(OpenWeatherApi()),
         CachedWeatherApi(YandexWeatherApi()),

gallery/painting/gismeteo/__init__.py

@@ -0,0 +1 @@
+PROVIDER = "gismeteo"

gallery/painting/gismeteo/api.py

@@ -1,5 +1,4 @@
 import datetime
-import logging
 from os import environ
 
 from fastapi import HTTPException, status
@@ -8,9 +7,8 @@ from gallery.sketch.source import ApiSource
 from gallery.sketch.weather.api import WeatherApi
 from gallery.sketch.weather.model import Location, WeatherResponse
 
-from .parser import Context, GismeteoForecastParser, GismeteoLocationParser
+from . import PROVIDER
-
+from .parser import ForecastData, GismeteoForecastParser, GismeteoLocationParser
-logger = logging.getLogger("gismeteo")
 
 
 def format_date(date: datetime.date) -> str:
@@ -25,7 +23,7 @@ def format_date(date: datetime.date) -> str:
 
 
 class GismeteoApi(WeatherApi):
-    PROVIDER = "gismeteo"
+    PROVIDER = PROVIDER
 
     forecast_parser = GismeteoForecastParser()
     location_parser = GismeteoLocationParser()
@@ -52,10 +50,10 @@ class GismeteoApi(WeatherApi):
                 status_code=status.HTTP_400_BAD_REQUEST, detail={"max_date": max_date.strftime("%Y-%m-%d")}
             )
         data = await self.source.request(f"weather-{location_id}/{format_date(date)}")
-        context = Context(date, "day", data, 8)
+        context = ForecastData(date, "day", data, 8)
         return self.forecast_parser.parse(context)
 
     async def get_days(self, location_id: str, days: int) -> WeatherResponse:
         data = await self.source.request(f"weather-{location_id}/{days}-days")
-        context = Context(datetime.date.today(), "days", data)
+        context = ForecastData(datetime.date.today(), "days", data)
         return self.forecast_parser.parse(context)

gallery/painting/gismeteo/parser.py

@@ -21,7 +21,9 @@ from gallery.sketch.weather.model import (
 from gallery.sketch.weather.parser import parse_wind_direction
 from gallery.sketch.weather.util import fix_dates_sequence, merge_weather_values
 
-logger = logging.getLogger("gismeteo")
+from . import PROVIDER
+
+logger = logging.getLogger(PROVIDER)
 
 
 class LocationNameParser(Parser[str, str]):
@@ -196,18 +198,18 @@ class GismeteoTableParser(TableParser[WeatherValue]):
         return WeatherValue.model_validate(data)
 
 
-class Context(NamedTuple):
+class ForecastData(NamedTuple):
     date: datetime.date
     period: Literal["day", "days"]
     data: str
     values: int | None = None
 
 
-class GismeteoForecastParser(Parser[Context, WeatherResponse]):
+class GismeteoForecastParser(Parser[ForecastData, WeatherResponse]):
     location_parser = LocationNameParser()
     table_parser = GismeteoTableParser()
 
-    def parse(self, data: Context) -> WeatherResponse:
+    def parse(self, data: ForecastData) -> WeatherResponse:
         location = self.location_parser.parse(data.data)
         soup = BeautifulSoup(data.data, features="html.parser")
         values = self.table_parser.parse(soup)
@@ -236,7 +238,7 @@ class GismeteoLocationParser(Parser[str, list[Location]]):
                 Location(
                     id=f"{item['slug']}-{item['id']}",
                     name=item["translations"]["kk"]["city"]["name"],
-                    provider="gismeteo",  # TODO: provider module const
+                    provider=PROVIDER,
                     lat=item["coordinates"]["latitude"],
                     lon=item["coordinates"]["longitude"],
                     country=item["translations"]["kk"]["country"]["name"],

gallery/painting/matchtv/__init__.py

@@ -0,0 +1 @@
+PROVIDER = "matchtv"

gallery/painting/matchtv/api.py

@@ -1,57 +1,33 @@
 import datetime
-import json
-import logging
 
 from gallery.sketch.schedule.api import ScheduleApi
-from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
+from gallery.sketch.schedule.model import Channel, Schedule
 from gallery.sketch.source import ApiSource
 
-logger = logging.getLogger("matchtv")
+from . import PROVIDER
+from .parser import (
+    ChannelData,
+    MatchTVChannelParser,
+    MatchTVScheduleParser,
+    ScheduleData,
+)
 
 
-class MatchTvApi(ScheduleApi):
+class MatchTVApi(ScheduleApi):
-    PROVIDER = "matchtv"
+    PROVIDER = PROVIDER
+
+    schedule_parser = MatchTVScheduleParser()
+    channel_parser = MatchTVChannelParser()
 
     def _build_source(self) -> ApiSource:
         return ApiSource("https://matchtv.ru")
 
     async def find_channels(self, query: str) -> list[Channel]:
         endpoint = "api/v1/channels"
-        data = json.loads(await self.source.request(endpoint))
+        data = await self.source.request(endpoint)
-        result = []
+        return self.channel_parser.parse(ChannelData(query, data))
-        query = query.lower()
-        for item in data["result"]:
-            if query in item["name"].lower() or query in item["alias"]:
-                channel_id = item["alias"]
-                name = item["name"].split("|")[0].strip()
-                result.append(Channel(id=channel_id, name=name, provider=self.provider))
-        return result
 
     async def get_schedule(self, channel_id: str, date: datetime.date) -> Schedule:
         endpoint = f"api/v1/channels/{channel_id}/tv-schedule?date={date:%Y%m%d}"
-        data = json.loads(await self.source.request(endpoint))
+        data = await self.source.request(endpoint)
-        channel_data = data["result"]["channels"][0]
+        return self.schedule_parser.parse(ScheduleData(date, data))
-        channel = Channel(id=channel_data["alias"], name=channel_data["name"], provider=self.provider)
-        values = []
-        current_day = datetime.datetime.combine(date.today(), datetime.datetime.min.time())
-        end = current_day + datetime.timedelta(days=1, hours=6)
-        prev_value: ScheduleValue | None = None
-        for item in channel_data["schedule"]:
-            title = item["title"]
-            time_str = item["time"]
-            hours, minutes = map(int, time_str.split(":"))
-            item_date = current_day.replace(hour=hours, minute=minutes)
-            if prev_value is not None and item_date.hour < prev_value.start.hour:
-                current_day += datetime.timedelta(days=1)
-                item_date += datetime.timedelta(days=1)
-            live = "Прямая трансляция" in title
-            value = ScheduleValue(start=item_date, end=end, label=title, live=live)
-            values.append(value)
-            if prev_value is not None:
-                prev_value.end = item_date
-            prev_value = value
-        return Schedule(
-            channel=channel,
-            date=date,
-            values=values,
-        )

gallery/painting/matchtv/parser.py

@@ -0,0 +1,70 @@
+import datetime
+import json
+import re
+from typing import NamedTuple
+
+from gallery.sketch.parse.core import Parser
+from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
+
+from . import PROVIDER
+
+
+class DateParser(Parser[str, datetime.date]):
+    def parse(self, data: str) -> datetime.date:
+        matched = re.compile('{\\"date\\":(\\d+)}').search(data)
+        return datetime.date.strptime(matched.group(1), data)
+
+
+class ScheduleData(NamedTuple):
+    date: datetime.date
+    data: str
+
+
+class MatchTVScheduleParser(Parser[ScheduleData, Schedule]):
+    date_parser = DateParser()
+
+    def parse(self, data: ScheduleData) -> Schedule:
+        content = json.loads(data.data)
+        channel_data = content["result"]["channels"][0]
+        channel = Channel(id=channel_data["alias"], name=channel_data["name"], provider=PROVIDER)
+        values = []
+        start = datetime.datetime.combine(data.date, datetime.time.min)
+        end = start + datetime.timedelta(days=1, hours=6)
+        prev_value: ScheduleValue | None = None
+        for item in channel_data["schedule"]:
+            title = item["title"]
+            time_str = item["time"]
+            hours, minutes = map(int, time_str.split(":"))
+            item_date = start.replace(hour=hours, minute=minutes)
+            if prev_value is not None and item_date.hour < prev_value.start.hour:
+                start += datetime.timedelta(days=1)
+                item_date += datetime.timedelta(days=1)
+            live = "Прямая трансляция" in title
+            value = ScheduleValue(start=item_date, end=end, label=title, live=live)
+            values.append(value)
+            if prev_value is not None:
+                prev_value.end = item_date
+            prev_value = value
+        return Schedule(
+            channel=channel,
+            date=data.date,
+            values=values,
+        )
+
+
+class ChannelData(NamedTuple):
+    query: str
+    data: str
+
+
+class MatchTVChannelParser(Parser[ChannelData, list[Channel]]):
+    def parse(self, data: ChannelData) -> list[Channel]:
+        context = json.loads(data.data)
+        result = []
+        query = data.query.lower()
+        for item in context["result"]:
+            if query in item["name"].lower() or query in item["alias"]:
+                channel_id = item["alias"]
+                name = item["name"].split("|")[0].strip()
+                result.append(Channel(id=channel_id, name=name, provider=PROVIDER))
+        return result

gallery/painting/openweather/__init__.py

@@ -0,0 +1 @@
+PROVIDER = "openweather"

gallery/painting/openweather/api.py

@@ -1,5 +1,4 @@
 import datetime
-import logging
 from itertools import groupby
 from os import environ
 
@@ -10,15 +9,14 @@ from gallery.sketch.weather.model import Location, WeatherResponse, WeatherValue
 from gallery.sketch.weather.util import merge_weather_values
 from gallery.util import TimeUnit
 
+from . import PROVIDER
 from .openweather import Location as OpenWeatherLocation
 from .openweather import OpenWeather
 from .parser import OpenWeatherForecastParser, OpenWeatherLocationParser
 
-logger = logging.getLogger("openweather")
-
 
 class OpenWeatherApi(WeatherApi[OpenWeather]):
-    PROVIDER = "openweather"
+    PROVIDER = PROVIDER
 
     forecast_parser = OpenWeatherForecastParser()
     location_parser = OpenWeatherLocationParser()

gallery/painting/rp5/__init__.py

@@ -0,0 +1 @@
+PROVIDER = "rp5"

gallery/painting/rp5/api.py

@@ -1,5 +1,4 @@
 import datetime
-import logging
 
 import aiohttp
 from aiocache import cached
@@ -14,13 +13,12 @@ from gallery.sketch.weather.model import (
 from gallery.sketch.weather.util import merge_weather_values
 from gallery.util import TimeUnit
 
+from . import PROVIDER
 from .parser import RP5ForecastParser, RP5LocationParser
 
-logger = logging.getLogger("rp5")
-
 
 class RP5WeatherApi(WeatherApi):
-    PROVIDER = "rp5"
+    PROVIDER = PROVIDER
 
     forecast_parser = RP5ForecastParser()
     location_parser = RP5LocationParser()

gallery/painting/rp5/parser.py

@@ -22,6 +22,8 @@ from gallery.sketch.weather.model import (
 from gallery.sketch.weather.parser import parse_wind_direction
 from gallery.sketch.weather.util import fix_dates_sequence
 
+from . import PROVIDER
+
 
 class LocationNameParser(TagParser[str]):
     selector = ".intoLeftNavi > .verticalBottom > a"
@@ -205,7 +207,7 @@ class RP5LocationParser(Parser[str, list[Location]]):
                 Location(
                     id=location_id,
                     name=name,
-                    provider="rp5",  # TODO: provider module const
+                    provider=PROVIDER,
                     lat=0,
                     lon=0,
                     country=country,

gallery/painting/yandextv/__init__.py

@@ -0,0 +1 @@
+PROVIDER = "yandex"

gallery/painting/yandextv/api.py

@@ -1,20 +1,19 @@
 import datetime
-import json
-import logging
-
-from bs4 import BeautifulSoup
 
+from gallery.sketch.headers import build_headers
 from gallery.sketch.schedule.api import ScheduleApi
-from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
+from gallery.sketch.schedule.model import Channel, Schedule
 from gallery.sketch.source import ApiSource
 
-from ...sketch.headers import build_headers
+from . import PROVIDER
-
+from .parser import ScheduleData, YandexTVChannelParser, YandexTVScheduleParser
-logger = logging.getLogger("matchtv")
 
 
-class YandexTvApi(ScheduleApi):
+class YandexTVApi(ScheduleApi):
-    PROVIDER = "yandextv"
+    PROVIDER = PROVIDER
+
+    schedule_parser = YandexTVScheduleParser()
+    channel_parser = YandexTVChannelParser()
 
     lr = 213  # magic id
 
@@ -26,41 +25,10 @@ class YandexTvApi(ScheduleApi):
             "https://suggest-multi.yandex.ru/suggest-tv2?"
             f"v=4&uil=ru&lr={self.lr}&count_channels=4&count_programs=4&sn=50&part={query}"
         )
-        _, values = json.loads(await self.source.request(url))
+        data = await self.source.request(url)
-        result = []
+        return self.channel_parser.parse(data)
-        for _, name, content in values:
-            if content["label"] == "Каналы":
-                channel_id = content["url"].split("/")[-1]
-                result.append(Channel(id=channel_id, name=name, provider=self.provider))
-        return result
 
     async def get_schedule(self, channel_id: str, date: datetime.date) -> Schedule:
         endpoint = f"{self.lr}/channel/{channel_id}?date={date:%Y-%m-%d}"
         data = await self.source.request(endpoint)
-        soup = BeautifulSoup(data, features="html.parser")
+        return self.schedule_parser.parse(ScheduleData(date, channel_id, data))
-        if soup.select_one(".CheckboxCaptcha") is not None:
-            raise RuntimeError("Captcha")
-        values = []
-        channel_name = soup.select_one(".channel-header__text").text.strip()
-        current_day = datetime.datetime.combine(date.today(), datetime.datetime.min.time())
-        end = current_day + datetime.timedelta(days=1, hours=6)
-        prev_value: ScheduleValue | None = None
-        for item in soup.select(".channel-schedule .channel-schedule__event"):
-            title = item.select_one(".channel-schedule__title").text.strip()
-            time_str = item.select_one(".channel-schedule__time").text.strip()
-            hours, minutes = map(int, time_str.split(":"))
-            item_date = current_day.replace(hour=hours, minute=minutes)
-            if prev_value is not None and item_date.hour < prev_value.start.hour:
-                current_day += datetime.timedelta(days=1)
-                item_date += datetime.timedelta(days=1)
-            live = item.select_one(".channel-schedule__info .icon_live") is not None
-            value = ScheduleValue(start=item_date, end=end, label=title, live=live)
-            values.append(value)
-            if prev_value is not None:
-                prev_value.end = item_date
-            prev_value = value
-        return Schedule(
-            channel=Channel(id=channel_id, name=channel_name, provider=self.provider),
-            date=date,
-            values=values,
-        )

gallery/painting/yandextv/parser.py

@@ -0,0 +1,58 @@
+import datetime
+import json
+from typing import NamedTuple
+
+from bs4 import BeautifulSoup
+
+from gallery.sketch.parse.core import Parser
+from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
+
+from . import PROVIDER
+
+
+class ScheduleData(NamedTuple):
+    date: datetime.date
+    channel_id: str
+    data: str
+
+
+class YandexTVScheduleParser(Parser[ScheduleData, Schedule]):
+    def parse(self, data: ScheduleData) -> Schedule:
+        soup = BeautifulSoup(data.data, features="html.parser")
+        if soup.select_one(".CheckboxCaptcha") is not None:
+            raise RuntimeError("Captcha")
+        values = []
+        channel_name = soup.select_one(".channel-header__text").text.strip()
+        current_day = datetime.datetime.combine(data.date, datetime.datetime.min.time())
+        end = current_day + datetime.timedelta(days=1, hours=6)
+        prev_value: ScheduleValue | None = None
+        for item in soup.select(".channel-schedule .channel-schedule__event"):
+            title = item.select_one(".channel-schedule__title").text.strip()
+            time_str = item.select_one(".channel-schedule__time").text.strip()
+            hours, minutes = map(int, time_str.split(":"))
+            item_date = current_day.replace(hour=hours, minute=minutes)
+            if prev_value is not None and item_date.hour < prev_value.start.hour:
+                current_day += datetime.timedelta(days=1)
+                item_date += datetime.timedelta(days=1)
+            live = item.select_one(".channel-schedule__info .icon_live") is not None
+            value = ScheduleValue(start=item_date, end=end, label=title, live=live)
+            values.append(value)
+            if prev_value is not None:
+                prev_value.end = item_date
+            prev_value = value
+        return Schedule(
+            channel=Channel(id=data.channel_id, name=channel_name, provider=PROVIDER),
+            date=data.date,
+            values=values,
+        )
+
+
+class YandexTVChannelParser(Parser[str, list[Channel]]):
+    def parse(self, data: str) -> list[Channel]:
+        _, values = json.loads(data)
+        result = []
+        for _, name, content in values:
+            if content["label"] == "Каналы":
+                channel_id = content["url"].split("/")[-1]
+                result.append(Channel(id=channel_id, name=name, provider=PROVIDER))
+        return result

gallery/painting/yandexweather/__init__.py

@@ -0,0 +1 @@
+PROVIDER = "yandex"

gallery/painting/yandexweather/api.py

@@ -1,5 +1,4 @@
 import datetime
-import logging
 
 from aiocache import cached
 
@@ -12,13 +11,12 @@ from gallery.sketch.weather.model import (
 from gallery.sketch.weather.util import merge_weather_values
 from gallery.util import TimeUnit
 
+from . import PROVIDER
 from .parser import YandexForecastParser, YandexLocationParser
 
-logger = logging.getLogger("yandexweather")
-
 
 class YandexWeatherApi(WeatherApi):
-    PROVIDER = "yandex"
+    PROVIDER = PROVIDER
 
     forecast_parser = YandexForecastParser()
     location_parser = YandexLocationParser()

gallery/painting/yandexweather/parser.py

@@ -18,7 +18,9 @@ from gallery.sketch.weather.model import (
 )
 from gallery.sketch.weather.parser import parse_wind_direction
 
-logger = logging.getLogger("yandexweather")
+from . import PROVIDER
+
+logger = logging.getLogger(PROVIDER)
 
 
 class LocationNameParser(TagParser[str]):
@@ -169,7 +171,7 @@ class YandexLocationParser(Parser[str, list[Location]]):
             Location(
                 id=str(item["geoId"]),
                 name=item["title"],
-                provider="yandex",  # TODO: provider module const
+                provider=PROVIDER,
                 lat=item["coords"]["lat"],
                 lon=item["coords"]["lon"],
                 country="ru",

tests/test_matchtv_api.py

@@ -2,22 +2,22 @@ import datetime
 
 import pytest
 
-from gallery.painting.matchtv.api import MatchTvApi
+from gallery.painting.matchtv.api import MatchTVApi
 from tests.common.mock import create_mock_api
 from tests.data.matchtv import MATCHTV_MOCK_SOURCE
 
 
 @pytest.fixture(name="matchtv_api", scope="module")
-def matchtv_api_fixture() -> MatchTvApi:
+def matchtv_api_fixture() -> MatchTVApi:
-    return create_mock_api(MatchTvApi, MATCHTV_MOCK_SOURCE)
+    return create_mock_api(MatchTVApi, MATCHTV_MOCK_SOURCE)
 
 
-async def test_search(matchtv_api: MatchTvApi):
+async def test_search(matchtv_api: MatchTVApi):
     result = await matchtv_api.find_channels("матч")
     assert len(result) == 6
 
 
-async def test_channel(matchtv_api: MatchTvApi):
+async def test_channel(matchtv_api: MatchTVApi):
     result = await matchtv_api.get_schedule("test", datetime.date.today())
     assert result is not None
     assert len(result.values) > 0

tests/test_yandextv_api.py

@@ -2,22 +2,22 @@ import datetime
 
 import pytest
 
-from gallery.painting.yandextv.api import YandexTvApi
+from gallery.painting.yandextv.api import YandexTVApi
 from tests.common.mock import create_mock_api
 from tests.data.yandextv import YANDEXTV_MOCK_SOURCE
 
 
 @pytest.fixture(name="yandextv_api", scope="module")
-def yandextv_api_fixture() -> YandexTvApi:
+def yandextv_api_fixture() -> YandexTVApi:
-    return create_mock_api(YandexTvApi, YANDEXTV_MOCK_SOURCE)
+    return create_mock_api(YandexTVApi, YANDEXTV_MOCK_SOURCE)
 
 
-async def test_search(yandextv_api: YandexTvApi):
+async def test_search(yandextv_api: YandexTVApi):
     result = await yandextv_api.find_channels("матч")
     assert len(result) == 4
 
 
-async def test_channel(yandextv_api: YandexTvApi):
+async def test_channel(yandextv_api: YandexTVApi):
     result = await yandextv_api.get_schedule("test", datetime.date.today())
     assert result is not None
     assert len(result.values) > 0