InfernalGames/gallery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(tv): rework tv parsers

Browse Source
This commit is contained in:
shmyga
2026-09-18 22:13:57 +03:00
parent a14f49742b
commit 5d025998b9
22 changed files with 208 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -66,7 +66,7 @@ https://api.shmyga.ru/api/schedule/{provider}/{channel_id}/{date}
| Provider | Site | API sample |
| :- | :- | :- |
| matchtv | https://matchtv.ru/tvguide | https://api.shmyga.ru/api/schedule/matchtv/igra/today |
| yandextv | https://tv.yandex.ru | https://api.shmyga.ru/api/schedule/yandextv/146/today |
| yandex | https://tv.yandex.ru | https://api.shmyga.ru/api/schedule/yandex/146/today |
## Demo
gallery/easel/depends/schedule.py
+5 -5
View File
@@ -16,10 +16,10 @@ class ChannelDef(NamedTuple):
GROUP_CANNELS_MAP: dict[str, list[ChannelDef]] = {
"central": [
ChannelDef("yandextv", "pervyy-16"),
ChannelDef("yandextv", "rossiya-1-31"),
ChannelDef("yandextv", "ntv-11"),
ChannelDef("yandextv", "sts-8"),
ChannelDef("yandex", "pervyy-16"),
ChannelDef("yandex", "rossiya-1-31"),
ChannelDef("yandex", "ntv-11"),
ChannelDef("yandex", "sts-8"),
],
"sport": [
ChannelDef("matchtv", "matchtv"),
@@ -29,7 +29,7 @@ GROUP_CANNELS_MAP: dict[str, list[ChannelDef]] = {
ChannelDef("matchtv", "futbol-2"),
ChannelDef("matchtv", "futbol-3"),
ChannelDef("matchtv", "strana"),
ChannelDef("yandextv", "start-103"),
ChannelDef("yandex", "start-103"),
],
}
gallery/main.py
+4 -4
View File
@@ -5,10 +5,10 @@ import uvicorn
from gallery.easel import build_app
from gallery.painting.gismeteo.api import GismeteoApi
from gallery.painting.matchtv.api import MatchTvApi
from gallery.painting.matchtv.api import MatchTVApi
from gallery.painting.openweather.api import OpenWeatherApi
from gallery.painting.rp5.api import RP5WeatherApi
from gallery.painting.yandextv.api import YandexTvApi
from gallery.painting.yandextv.api import YandexTVApi
from gallery.painting.yandexweather.api import YandexWeatherApi
from gallery.sketch.bundle import ApiBundle
from gallery.sketch.schedule.cached import CachedScheduleApi
@@ -16,8 +16,8 @@ from gallery.sketch.weather.cached import CachedWeatherApi
api = ApiBundle(
[
CachedScheduleApi(YandexTvApi()),
CachedScheduleApi(MatchTvApi()),
CachedScheduleApi(YandexTVApi()),
CachedScheduleApi(MatchTVApi()),
CachedWeatherApi(GismeteoApi()),
CachedWeatherApi(OpenWeatherApi()),
CachedWeatherApi(YandexWeatherApi()),
gallery/painting/gismeteo/__init__.py
+1
View File
@@ -0,0 +1 @@
PROVIDER = "gismeteo"
gallery/painting/gismeteo/api.py
+5 -7
View File
@@ -1,5 +1,4 @@
import datetime
import logging
from os import environ
from fastapi import HTTPException, status
@@ -8,9 +7,8 @@ from gallery.sketch.source import ApiSource
from gallery.sketch.weather.api import WeatherApi
from gallery.sketch.weather.model import Location, WeatherResponse
from .parser import Context, GismeteoForecastParser, GismeteoLocationParser
logger = logging.getLogger("gismeteo")
from . import PROVIDER
from .parser import ForecastData, GismeteoForecastParser, GismeteoLocationParser
def format_date(date: datetime.date) -> str:
@@ -25,7 +23,7 @@ def format_date(date: datetime.date) -> str:
class GismeteoApi(WeatherApi):
PROVIDER = "gismeteo"
PROVIDER = PROVIDER
forecast_parser = GismeteoForecastParser()
location_parser = GismeteoLocationParser()
@@ -52,10 +50,10 @@ class GismeteoApi(WeatherApi):
status_code=status.HTTP_400_BAD_REQUEST, detail={"max_date": max_date.strftime("%Y-%m-%d")}
)
data = await self.source.request(f"weather-{location_id}/{format_date(date)}")
context = Context(date, "day", data, 8)
context = ForecastData(date, "day", data, 8)
return self.forecast_parser.parse(context)
async def get_days(self, location_id: str, days: int) -> WeatherResponse:
data = await self.source.request(f"weather-{location_id}/{days}-days")
context = Context(datetime.date.today(), "days", data)
context = ForecastData(datetime.date.today(), "days", data)
return self.forecast_parser.parse(context)
gallery/painting/gismeteo/parser.py
+7 -5
View File
@@ -21,7 +21,9 @@ from gallery.sketch.weather.model import (
from gallery.sketch.weather.parser import parse_wind_direction
from gallery.sketch.weather.util import fix_dates_sequence, merge_weather_values
logger = logging.getLogger("gismeteo")
from . import PROVIDER
logger = logging.getLogger(PROVIDER)
class LocationNameParser(Parser[str, str]):
@@ -196,18 +198,18 @@ class GismeteoTableParser(TableParser[WeatherValue]):
return WeatherValue.model_validate(data)
class Context(NamedTuple):
class ForecastData(NamedTuple):
date: datetime.date
period: Literal["day", "days"]
data: str
values: int | None = None
class GismeteoForecastParser(Parser[Context, WeatherResponse]):
class GismeteoForecastParser(Parser[ForecastData, WeatherResponse]):
location_parser = LocationNameParser()
table_parser = GismeteoTableParser()
def parse(self, data: Context) -> WeatherResponse:
def parse(self, data: ForecastData) -> WeatherResponse:
location = self.location_parser.parse(data.data)
soup = BeautifulSoup(data.data, features="html.parser")
values = self.table_parser.parse(soup)
@@ -236,7 +238,7 @@ class GismeteoLocationParser(Parser[str, list[Location]]):
Location(
id=f"{item['slug']}-{item['id']}",
name=item["translations"]["kk"]["city"]["name"],
provider="gismeteo", # TODO: provider module const
provider=PROVIDER,
lat=item["coordinates"]["latitude"],
lon=item["coordinates"]["longitude"],
country=item["translations"]["kk"]["country"]["name"],
gallery/painting/matchtv/__init__.py
+1
View File
@@ -0,0 +1 @@
PROVIDER = "matchtv"
gallery/painting/matchtv/api.py
+17 -41
View File
@@ -1,57 +1,33 @@
import datetime
import json
import logging
from gallery.sketch.schedule.api import ScheduleApi
from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
from gallery.sketch.schedule.model import Channel, Schedule
from gallery.sketch.source import ApiSource
logger = logging.getLogger("matchtv")
from . import PROVIDER
from .parser import (
ChannelData,
MatchTVChannelParser,
MatchTVScheduleParser,
ScheduleData,
)
class MatchTvApi(ScheduleApi):
PROVIDER = "matchtv"
class MatchTVApi(ScheduleApi):
PROVIDER = PROVIDER
schedule_parser = MatchTVScheduleParser()
channel_parser = MatchTVChannelParser()
def _build_source(self) -> ApiSource:
return ApiSource("https://matchtv.ru")
async def find_channels(self, query: str) -> list[Channel]:
endpoint = "api/v1/channels"
data = json.loads(await self.source.request(endpoint))
result = []
query = query.lower()
for item in data["result"]:
if query in item["name"].lower() or query in item["alias"]:
channel_id = item["alias"]
name = item["name"].split("|")[0].strip()
result.append(Channel(id=channel_id, name=name, provider=self.provider))
return result
data = await self.source.request(endpoint)
return self.channel_parser.parse(ChannelData(query, data))
async def get_schedule(self, channel_id: str, date: datetime.date) -> Schedule:
endpoint = f"api/v1/channels/{channel_id}/tv-schedule?date={date:%Y%m%d}"
data = json.loads(await self.source.request(endpoint))
channel_data = data["result"]["channels"][0]
channel = Channel(id=channel_data["alias"], name=channel_data["name"], provider=self.provider)
values = []
current_day = datetime.datetime.combine(date.today(), datetime.datetime.min.time())
end = current_day + datetime.timedelta(days=1, hours=6)
prev_value: ScheduleValue | None = None
for item in channel_data["schedule"]:
title = item["title"]
time_str = item["time"]
hours, minutes = map(int, time_str.split(":"))
item_date = current_day.replace(hour=hours, minute=minutes)
if prev_value is not None and item_date.hour < prev_value.start.hour:
current_day += datetime.timedelta(days=1)
item_date += datetime.timedelta(days=1)
live = "Прямая трансляция" in title
value = ScheduleValue(start=item_date, end=end, label=title, live=live)
values.append(value)
if prev_value is not None:
prev_value.end = item_date
prev_value = value
return Schedule(
channel=channel,
date=date,
values=values,
)
data = await self.source.request(endpoint)
return self.schedule_parser.parse(ScheduleData(date, data))
gallery/painting/matchtv/parser.py
+70
View File
@@ -0,0 +1,70 @@
import datetime
import json
import re
from typing import NamedTuple
from gallery.sketch.parse.core import Parser
from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
from . import PROVIDER
class DateParser(Parser[str, datetime.date]):
def parse(self, data: str) -> datetime.date:
matched = re.compile('{\\"date\\":(\\d+)}').search(data)
return datetime.date.strptime(matched.group(1), data)
class ScheduleData(NamedTuple):
date: datetime.date
data: str
class MatchTVScheduleParser(Parser[ScheduleData, Schedule]):
date_parser = DateParser()
def parse(self, data: ScheduleData) -> Schedule:
content = json.loads(data.data)
channel_data = content["result"]["channels"][0]
channel = Channel(id=channel_data["alias"], name=channel_data["name"], provider=PROVIDER)
values = []
start = datetime.datetime.combine(data.date, datetime.time.min)
end = start + datetime.timedelta(days=1, hours=6)
prev_value: ScheduleValue | None = None
for item in channel_data["schedule"]:
title = item["title"]
time_str = item["time"]
hours, minutes = map(int, time_str.split(":"))
item_date = start.replace(hour=hours, minute=minutes)
if prev_value is not None and item_date.hour < prev_value.start.hour:
start += datetime.timedelta(days=1)
item_date += datetime.timedelta(days=1)
live = "Прямая трансляция" in title
value = ScheduleValue(start=item_date, end=end, label=title, live=live)
values.append(value)
if prev_value is not None:
prev_value.end = item_date
prev_value = value
return Schedule(
channel=channel,
date=data.date,
values=values,
)
class ChannelData(NamedTuple):
query: str
data: str
class MatchTVChannelParser(Parser[ChannelData, list[Channel]]):
def parse(self, data: ChannelData) -> list[Channel]:
context = json.loads(data.data)
result = []
query = data.query.lower()
for item in context["result"]:
if query in item["name"].lower() or query in item["alias"]:
channel_id = item["alias"]
name = item["name"].split("|")[0].strip()
result.append(Channel(id=channel_id, name=name, provider=PROVIDER))
return result
gallery/painting/openweather/__init__.py
+1
View File
@@ -0,0 +1 @@
PROVIDER = "openweather"
gallery/painting/openweather/api.py
+2 -4
View File
@@ -1,5 +1,4 @@
import datetime
import logging
from itertools import groupby
from os import environ
@@ -10,15 +9,14 @@ from gallery.sketch.weather.model import Location, WeatherResponse, WeatherValue
from gallery.sketch.weather.util import merge_weather_values
from gallery.util import TimeUnit
from . import PROVIDER
from .openweather import Location as OpenWeatherLocation
from .openweather import OpenWeather
from .parser import OpenWeatherForecastParser, OpenWeatherLocationParser
logger = logging.getLogger("openweather")
class OpenWeatherApi(WeatherApi[OpenWeather]):
PROVIDER = "openweather"
PROVIDER = PROVIDER
forecast_parser = OpenWeatherForecastParser()
location_parser = OpenWeatherLocationParser()
gallery/painting/rp5/__init__.py
+1
View File
@@ -0,0 +1 @@
PROVIDER = "rp5"
gallery/painting/rp5/api.py
+2 -4
View File
@@ -1,5 +1,4 @@
import datetime
import logging
import aiohttp
from aiocache import cached
@@ -14,13 +13,12 @@ from gallery.sketch.weather.model import (
from gallery.sketch.weather.util import merge_weather_values
from gallery.util import TimeUnit
from . import PROVIDER
from .parser import RP5ForecastParser, RP5LocationParser
logger = logging.getLogger("rp5")
class RP5WeatherApi(WeatherApi):
PROVIDER = "rp5"
PROVIDER = PROVIDER
forecast_parser = RP5ForecastParser()
location_parser = RP5LocationParser()
gallery/painting/rp5/parser.py
+3 -1
View File
@@ -22,6 +22,8 @@ from gallery.sketch.weather.model import (
from gallery.sketch.weather.parser import parse_wind_direction
from gallery.sketch.weather.util import fix_dates_sequence
from . import PROVIDER
class LocationNameParser(TagParser[str]):
selector = ".intoLeftNavi > .verticalBottom > a"
@@ -205,7 +207,7 @@ class RP5LocationParser(Parser[str, list[Location]]):
Location(
id=location_id,
name=name,
provider="rp5", # TODO: provider module const
provider=PROVIDER,
lat=0,
lon=0,
country=country,
gallery/painting/yandextv/__init__.py
+1
View File
@@ -0,0 +1 @@
PROVIDER = "yandex"
gallery/painting/yandextv/api.py
+12 -44
View File
@@ -1,20 +1,19 @@
import datetime
import json
import logging
from bs4 import BeautifulSoup
from gallery.sketch.headers import build_headers
from gallery.sketch.schedule.api import ScheduleApi
from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
from gallery.sketch.schedule.model import Channel, Schedule
from gallery.sketch.source import ApiSource
from ...sketch.headers import build_headers
logger = logging.getLogger("matchtv")
from . import PROVIDER
from .parser import ScheduleData, YandexTVChannelParser, YandexTVScheduleParser
class YandexTvApi(ScheduleApi):
PROVIDER = "yandextv"
class YandexTVApi(ScheduleApi):
PROVIDER = PROVIDER
schedule_parser = YandexTVScheduleParser()
channel_parser = YandexTVChannelParser()
lr = 213 # magic id
@@ -26,41 +25,10 @@ class YandexTvApi(ScheduleApi):
"https://suggest-multi.yandex.ru/suggest-tv2?"
f"v=4&uil=ru&lr={self.lr}&count_channels=4&count_programs=4&sn=50&part={query}"
)
_, values = json.loads(await self.source.request(url))
result = []
for _, name, content in values:
if content["label"] == "Каналы":
channel_id = content["url"].split("/")[-1]
result.append(Channel(id=channel_id, name=name, provider=self.provider))
return result
data = await self.source.request(url)
return self.channel_parser.parse(data)
async def get_schedule(self, channel_id: str, date: datetime.date) -> Schedule:
endpoint = f"{self.lr}/channel/{channel_id}?date={date:%Y-%m-%d}"
data = await self.source.request(endpoint)
soup = BeautifulSoup(data, features="html.parser")
if soup.select_one(".CheckboxCaptcha") is not None:
raise RuntimeError("Captcha")
values = []
channel_name = soup.select_one(".channel-header__text").text.strip()
current_day = datetime.datetime.combine(date.today(), datetime.datetime.min.time())
end = current_day + datetime.timedelta(days=1, hours=6)
prev_value: ScheduleValue | None = None
for item in soup.select(".channel-schedule .channel-schedule__event"):
title = item.select_one(".channel-schedule__title").text.strip()
time_str = item.select_one(".channel-schedule__time").text.strip()
hours, minutes = map(int, time_str.split(":"))
item_date = current_day.replace(hour=hours, minute=minutes)
if prev_value is not None and item_date.hour < prev_value.start.hour:
current_day += datetime.timedelta(days=1)
item_date += datetime.timedelta(days=1)
live = item.select_one(".channel-schedule__info .icon_live") is not None
value = ScheduleValue(start=item_date, end=end, label=title, live=live)
values.append(value)
if prev_value is not None:
prev_value.end = item_date
prev_value = value
return Schedule(
channel=Channel(id=channel_id, name=channel_name, provider=self.provider),
date=date,
values=values,
)
return self.schedule_parser.parse(ScheduleData(date, channel_id, data))
gallery/painting/yandextv/parser.py
+58
View File
@@ -0,0 +1,58 @@
import datetime
import json
from typing import NamedTuple
from bs4 import BeautifulSoup
from gallery.sketch.parse.core import Parser
from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
from . import PROVIDER
class ScheduleData(NamedTuple):
date: datetime.date
channel_id: str
data: str
class YandexTVScheduleParser(Parser[ScheduleData, Schedule]):
def parse(self, data: ScheduleData) -> Schedule:
soup = BeautifulSoup(data.data, features="html.parser")
if soup.select_one(".CheckboxCaptcha") is not None:
raise RuntimeError("Captcha")
values = []
channel_name = soup.select_one(".channel-header__text").text.strip()
current_day = datetime.datetime.combine(data.date, datetime.datetime.min.time())
end = current_day + datetime.timedelta(days=1, hours=6)
prev_value: ScheduleValue | None = None
for item in soup.select(".channel-schedule .channel-schedule__event"):
title = item.select_one(".channel-schedule__title").text.strip()
time_str = item.select_one(".channel-schedule__time").text.strip()
hours, minutes = map(int, time_str.split(":"))
item_date = current_day.replace(hour=hours, minute=minutes)
if prev_value is not None and item_date.hour < prev_value.start.hour:
current_day += datetime.timedelta(days=1)
item_date += datetime.timedelta(days=1)
live = item.select_one(".channel-schedule__info .icon_live") is not None
value = ScheduleValue(start=item_date, end=end, label=title, live=live)
values.append(value)
if prev_value is not None:
prev_value.end = item_date
prev_value = value
return Schedule(
channel=Channel(id=data.channel_id, name=channel_name, provider=PROVIDER),
date=data.date,
values=values,
)
class YandexTVChannelParser(Parser[str, list[Channel]]):
def parse(self, data: str) -> list[Channel]:
_, values = json.loads(data)
result = []
for _, name, content in values:
if content["label"] == "Каналы":
channel_id = content["url"].split("/")[-1]
result.append(Channel(id=channel_id, name=name, provider=PROVIDER))
return result
gallery/painting/yandexweather/__init__.py
+1
View File
@@ -0,0 +1 @@
PROVIDER = "yandex"
gallery/painting/yandexweather/api.py
+2 -4
View File
@@ -1,5 +1,4 @@
import datetime
import logging
from aiocache import cached
@@ -12,13 +11,12 @@ from gallery.sketch.weather.model import (
from gallery.sketch.weather.util import merge_weather_values
from gallery.util import TimeUnit
from . import PROVIDER
from .parser import YandexForecastParser, YandexLocationParser
logger = logging.getLogger("yandexweather")
class YandexWeatherApi(WeatherApi):
PROVIDER = "yandex"
PROVIDER = PROVIDER
forecast_parser = YandexForecastParser()
location_parser = YandexLocationParser()
gallery/painting/yandexweather/parser.py
+4 -2
View File
@@ -18,7 +18,9 @@ from gallery.sketch.weather.model import (
)
from gallery.sketch.weather.parser import parse_wind_direction
logger = logging.getLogger("yandexweather")
from . import PROVIDER
logger = logging.getLogger(PROVIDER)
class LocationNameParser(TagParser[str]):
@@ -169,7 +171,7 @@ class YandexLocationParser(Parser[str, list[Location]]):
Location(
id=str(item["geoId"]),
name=item["title"],
provider="yandex", # TODO: provider module const
provider=PROVIDER,
lat=item["coords"]["lat"],
lon=item["coords"]["lon"],
country="ru",
tests/test_matchtv_api.py
+5 -5
View File
@@ -2,22 +2,22 @@ import datetime
import pytest
from gallery.painting.matchtv.api import MatchTvApi
from gallery.painting.matchtv.api import MatchTVApi
from tests.common.mock import create_mock_api
from tests.data.matchtv import MATCHTV_MOCK_SOURCE
@pytest.fixture(name="matchtv_api", scope="module")
def matchtv_api_fixture() -> MatchTvApi:
return create_mock_api(MatchTvApi, MATCHTV_MOCK_SOURCE)
def matchtv_api_fixture() -> MatchTVApi:
return create_mock_api(MatchTVApi, MATCHTV_MOCK_SOURCE)
async def test_search(matchtv_api: MatchTvApi):
async def test_search(matchtv_api: MatchTVApi):
result = await matchtv_api.find_channels("матч")
assert len(result) == 6
async def test_channel(matchtv_api: MatchTvApi):
async def test_channel(matchtv_api: MatchTVApi):
result = await matchtv_api.get_schedule("test", datetime.date.today())
assert result is not None
assert len(result.values) > 0
tests/test_yandextv_api.py
+5 -5
View File
@@ -2,22 +2,22 @@ import datetime
import pytest
from gallery.painting.yandextv.api import YandexTvApi
from gallery.painting.yandextv.api import YandexTVApi
from tests.common.mock import create_mock_api
from tests.data.yandextv import YANDEXTV_MOCK_SOURCE
@pytest.fixture(name="yandextv_api", scope="module")
def yandextv_api_fixture() -> YandexTvApi:
return create_mock_api(YandexTvApi, YANDEXTV_MOCK_SOURCE)
def yandextv_api_fixture() -> YandexTVApi:
return create_mock_api(YandexTVApi, YANDEXTV_MOCK_SOURCE)
async def test_search(yandextv_api: YandexTvApi):
async def test_search(yandextv_api: YandexTVApi):
result = await yandextv_api.find_channels("матч")
assert len(result) == 4
async def test_channel(yandextv_api: YandexTvApi):
async def test_channel(yandextv_api: YandexTVApi):
result = await yandextv_api.get_schedule("test", datetime.date.today())
assert result is not None
assert len(result.values) > 0