Compare commits

...
30 Commits
Author SHA1 Message Date
shmyga 160e1f9156 ci(version): 0.10.5
release / release (push) Successful in 3m53s
test / test (push) Successful in 52s
2026-09-30 20:57:43 +03:00
shmyga ad8f29c5ef feat(easel): add api link button to views
test / test (push) Successful in 59s
2026-09-30 20:56:17 +03:00
shmyga 4fc1c8c8e5 docs(readme): add test status badge
test / test (push) Successful in 56s
2026-09-30 19:43:27 +03:00
shmyga e414a93242 ci(action): add deploy action
release / release (push) Successful in 46s
test / test (push) Successful in 44s
2026-09-30 19:23:07 +03:00
shmyga 714fd1567c ci(action): use project-manager action
release / release (push) Successful in 39s
test / test (push) Successful in 49s
2026-09-30 16:03:23 +03:00
shmyga 1d1ba69cc8 ci(version): 0.10.4
release / release (push) Successful in 8m42s
test / test (push) Successful in 56s
2026-09-29 22:54:33 +03:00
shmyga f35d3f8cb8 build(docker): fix playwright version 2026-09-29 22:42:41 +03:00
shmyga b0187506fd ci(action): add setup composite action
test / test (push) Successful in 35s
2026-09-28 19:10:52 +03:00
shmyga beafed8faf ci(version): 0.10.3
build / build (push) Successful in 3m17s
test / test (push) Successful in 42s
2026-09-28 15:28:43 +03:00
shmyga 52315cc8bd fix(weather): add query params to days tag route redirect 2026-09-28 15:28:18 +03:00
shmyga 80e9a80edd ci(version): 0.10.2
build / build (push) Successful in 3m26s
test / test (push) Successful in 42s
2026-09-27 21:00:51 +03:00
shmyga b3412d9e53 fix(weather): add deprecated days tag route 2026-09-27 20:59:33 +03:00
shmyga ba91d6d161 ci(action): fix build action name
test / test (push) Successful in 42s
2026-09-24 19:17:15 +03:00
shmyga 62efd54298 ci(version): 0.10.1
test / build (push) Successful in 2m59s
test / test (push) Successful in 51s
2026-09-24 19:04:01 +03:00
shmyga f3b831649e ci(action): add test and build actions
test / test (push) Successful in 45s
2026-09-24 12:11:08 +03:00
shmyga 829a12612e feat(parser): use ParserError in parsers 2026-09-23 16:12:13 +03:00
shmyga 73df7a2b5d ci(version): 0.10.0 2026-09-22 21:25:30 +03:00
shmyga 3612658323 refactor(weather): rework wind parser 2026-09-22 20:58:18 +03:00
shmyga c0503458e4 refactor(parse): rework date parser 2026-09-21 22:30:17 +03:00
shmyga 0baa1277ef refactor(parse): add base html and json parsers 2026-09-21 22:17:50 +03:00
shmyga 5d025998b9 refactor(tv): rework tv parsers 2026-09-18 22:13:57 +03:00
shmyga a14f49742b refactor(weather): move fix_dates_sequence to util 2026-09-18 16:12:18 +03:00
shmyga 6178c694f9 refactor(openweather): rework openweather parser 2026-09-17 22:03:24 +03:00
shmyga 2e9d04c10e refactor(yandexweather): rework yandexweather parser 2026-09-17 20:59:29 +03:00
shmyga ece1a58a27 refactor(gismeteo): add gismeteo location parser 2026-09-17 16:39:07 +03:00
shmyga a12a81e26d fix(weather): fix parsed dates sequence 2026-09-17 16:28:57 +03:00
shmyga d214849e2e refactor(rp5): rework rp5 parser 2026-09-17 16:03:42 +03:00
shmyga 9ae9d1be14 refactor: add base parse module 2026-09-17 00:47:01 +03:00
shmyga 65b6adbf16 ci(version): 0.9.0 2026-09-16 15:52:56 +03:00
shmyga 2500ba19bd feat(api): add schedule groups api 2026-09-16 15:52:38 +03:00
54 changed files with 1492 additions and 846 deletions
+22
View File
@@ -0,0 +1,22 @@
name: deploy
on:
workflow_run:
workflows: [release]
types: [completed]
jobs:
deploy:
if: ${{ github.event.workflow_run.conclusion == 'success' }}
runs-on: ubuntu-latest
steps:
- uses: appleboy/ssh-action@v1
with:
host: ${{ secrets.DEPLOY_HOST }}
username: ${{ secrets.DEPLOY_USERNAME }}
key: ${{ secrets.DEPLOY_KEY }}
passphrase: ${{ secrets.DEPLOY_PASSPHRASE }}
script: |
cd /opt/gallery
docker compose pull
docker compose up -d
+20
View File
@@ -0,0 +1,20 @@
name: release
on:
push:
tags:
- "[0-9]+.[0-9]+.[0-9]+"
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: https://git.shmyga.ru/InfernalGames/project-manager@master
- run: pm build:docker -vv
- uses: docker/login-action@v4
with:
registry: git.shmyga.ru
username: ${{ vars.PUBLISH_USERNAME }}
password: ${{ secrets.PUBLISH_TOKEN }}
- run: pm publish:docker -vv
+14
View File
@@ -0,0 +1,14 @@
name: test
on: push
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: https://git.shmyga.ru/InfernalGames/project-manager@master
with:
setup-python: true
- run: pm lint -vv
- run: pm test -vv
+3 -2
View File
@@ -27,7 +27,7 @@ FROM python:3.14-slim
ENV PLAYWRIGHT_BROWSERS_PATH="/opt/playwright-browsers"
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
pip install playwright && \
pip install playwright==1.62 && \
playwright install --with-deps chromium
ENV PATH="/app/.venv/bin:$PATH"
@@ -39,6 +39,7 @@ COPY --from=builder /app ./
COPY --from=node-builder /app/dist ./static/dist
COPY gallery gallery/
COPY --from=builder --parents locales/**/*.mo ./
# COPY --from=builder --parents locales/**/*.mo ./
COPY --from=builder /app/locales ./locales
CMD ["uvicorn", "gallery.main:app", "--host", "0.0.0.0", "--port", "80", "--log-config", "gallery/logging.yaml"]
+3 -1
View File
@@ -1,5 +1,7 @@
# API Gallery
[![test](https://git.shmyga.ru/InfernalGames/gallery/actions/workflows/test.yaml/badge.svg?branch=master)](https://git.shmyga.ru/InfernalGames/gallery/actions?workflow=test.yaml)
Currently implemented APIs:
- Weather forecast
@@ -66,7 +68,7 @@ https://api.shmyga.ru/api/schedule/{provider}/{channel_id}/{date}
| Provider | Site | API sample |
| :- | :- | :- |
| matchtv | https://matchtv.ru/tvguide | https://api.shmyga.ru/api/schedule/matchtv/igra/today |
| yandextv | https://tv.yandex.ru | https://api.shmyga.ru/api/schedule/yandextv/146/today |
| yandex | https://tv.yandex.ru | https://api.shmyga.ru/api/schedule/yandex/start-103/today |
## Demo
+7 -2
View File
@@ -1,10 +1,11 @@
from fastapi import FastAPI
from fastapi.responses import RedirectResponse
from fastapi.staticfiles import StaticFiles
from gallery.sketch.bundle import ApiBundle
from gallery.util import root_path
from .route import api, doc, view
from .route import api, view
def build_app(api_bundle: ApiBundle) -> FastAPI:
@@ -15,8 +16,12 @@ def build_app(api_bundle: ApiBundle) -> FastAPI:
)
app.state.api = api_bundle
app.mount("/static", StaticFiles(directory=root_path / "static/dist"))
doc.mount(app)
api.app.state.api = api_bundle
app.mount("/api", api.app)
app.include_router(view.router)
@app.get("/docs")
def get_docs():
return RedirectResponse("/api/docs")
return app
+33 -1
View File
@@ -1,4 +1,4 @@
from typing import Annotated
from typing import Annotated, NamedTuple
from fastapi import Depends
@@ -7,3 +7,35 @@ from gallery.sketch.schedule.api import ScheduleApi
from .api import api_resolver
ScheduleApiDepends = Annotated[ScheduleApi, Depends(api_resolver(ScheduleApi))]
class ChannelDef(NamedTuple):
provider: str
id: str
GROUP_CANNELS_MAP: dict[str, list[ChannelDef]] = {
"central": [
ChannelDef("yandex", "pervyy-16"),
ChannelDef("yandex", "rossiya-1-31"),
ChannelDef("yandex", "ntv-11"),
ChannelDef("yandex", "sts-8"),
],
"sport": [
ChannelDef("matchtv", "matchtv"),
ChannelDef("matchtv", "igra"),
ChannelDef("matchtv", "arena"),
ChannelDef("matchtv", "futbol-1"),
ChannelDef("matchtv", "futbol-2"),
ChannelDef("matchtv", "futbol-3"),
ChannelDef("matchtv", "strana"),
ChannelDef("yandex", "start-103"),
],
}
def resolve_group_channels(group_id: str) -> list[ChannelDef]:
return GROUP_CANNELS_MAP.get(group_id) or [ChannelDef(*item.split(":")) for item in group_id.split(",")]
ScheduleGroupChannelsDepends = Annotated[list[ChannelDef], Depends(resolve_group_channels)]
+7 -1
View File
@@ -1,9 +1,14 @@
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from gallery.easel.route import doc
from . import schedule, weather
app = FastAPI(openapi_prefix="/api")
app = FastAPI(
docs_url=None,
redoc_url=None,
)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
@@ -13,3 +18,4 @@ app.add_middleware(
)
app.include_router(weather.router)
app.include_router(schedule.router)
doc.mount(app, "/api")
+18 -1
View File
@@ -1,7 +1,11 @@
from fastapi import APIRouter
from gallery.easel.core import AppRequest
from gallery.easel.depends.schedule import ScheduleApiDepends
from gallery.easel.depends.schedule import (
GROUP_CANNELS_MAP,
ScheduleApiDepends,
ScheduleGroupChannelsDepends,
)
from gallery.sketch.schedule.api import ScheduleApi
from gallery.sketch.schedule.model import Channel, Schedule
@@ -10,6 +14,19 @@ from ...depends.date import DateDepends
router = APIRouter(prefix="/schedule", tags=["Schedule"])
@router.get("/groups")
async def get_api_schedule_groups():
return list(GROUP_CANNELS_MAP.keys())
@router.get("/group/{group_id}/{date}")
async def get_api_schedule_group_schedule(
request: AppRequest, channels: ScheduleGroupChannelsDepends, date: DateDepends
) -> list[Schedule]:
api = request.app.state.api
return [await api.get_api(ScheduleApi, channel.provider).get_schedule(channel.id, date) for channel in channels]
@router.get("/providers")
async def get_api_weather_providers(request: AppRequest) -> list[str]:
return request.app.state.api.get_api_providers(ScheduleApi)
+4 -3
View File
@@ -8,7 +8,8 @@ from fastapi.openapi.docs import (
from fastapi.staticfiles import StaticFiles
def mount(app: FastAPI):
def mount(app: FastAPI, prefix: str | None = None):
openapi_url = prefix + app.openapi_url if prefix else app.openapi_url
app.mount(
"/docs/static",
StaticFiles(directory=Path(__file__).parent / "static"),
@@ -17,7 +18,7 @@ def mount(app: FastAPI):
@app.get("/docs", include_in_schema=False)
async def custom_swagger_ui_html():
return get_swagger_ui_html(
openapi_url=app.openapi_url,
openapi_url=openapi_url,
title=app.title + " - Swagger UI",
oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
swagger_js_url="docs/static/swagger-ui-bundle.js",
@@ -31,7 +32,7 @@ def mount(app: FastAPI):
@app.get("/redoc", include_in_schema=False)
async def redoc_html():
return get_redoc_html(
openapi_url=app.openapi_url,
openapi_url=openapi_url,
title=app.title + " - ReDoc",
redoc_js_url="docs/static/redoc.standalone.js",
)
@@ -17,13 +17,13 @@
</div>
<h1>{{_("Docs")}}</h1>
<div class="list-group mb-3">
<app-link href="/docs"
<app-link href="/api/docs"
target="_blank"
icon="file-earmark-text"
class="list-group-item list-group-item-action">
Swagger UI
</app-link>
<app-link href="/redoc"
<app-link href="/api/redoc"
target="_blank"
icon="file-earmark-text"
class="list-group-item list-group-item-action">
@@ -1,18 +0,0 @@
GROUP_MAP = {
"central": [
("yandextv", "146"), # pervyy
("yandextv", "587"), # rossya-1
("yandextv", "162"), # ntv
("yandextv", "79"), # sts
],
"sport": [
("matchtv", "matchtv"),
("matchtv", "igra"),
("matchtv", "arena"),
("matchtv", "futbol-1"),
("matchtv", "futbol-2"),
("matchtv", "futbol-3"),
("matchtv", "strana"),
("yandextv", "1799"), # start
],
}
+34 -17
View File
@@ -1,4 +1,3 @@
import datetime
from pathlib import Path
from urllib import parse
@@ -8,13 +7,14 @@ from fastapi.responses import HTMLResponse, RedirectResponse
from gallery.easel.core import AppRequest
from gallery.easel.depends.api import api_resolver
from gallery.easel.depends.date import DateDepends
from gallery.easel.depends.schedule import ScheduleApiDepends
from gallery.sketch.bundle import ApiBundle
from gallery.easel.depends.schedule import (
GROUP_CANNELS_MAP,
ScheduleApiDepends,
ScheduleGroupChannelsDepends,
)
from gallery.sketch.schedule.api import ScheduleApi
from gallery.sketch.schedule.model import Schedule
from ..common.utils.template import build_templates
from .bundle import GROUP_MAP
from .filters import timedelta_format
@@ -45,29 +45,34 @@ async def get_schedule_index(request: AppRequest, provider: str | None = None, q
return templates.TemplateResponse(
request=request,
name="index.html",
context={"channels": [item.model_dump() for item in channels], "groups": GROUP_MAP.keys()},
context={"channels": [item.model_dump() for item in channels], "groups": GROUP_CANNELS_MAP.keys()},
)
async def _get_group_schedule(api: ApiBundle, channels: list[tuple[str, str]], date: datetime.date) -> list[Schedule]:
return [
await api.get_api(ScheduleApi, provider).get_schedule(channel_id, date) for provider, channel_id in channels
]
@router.get("/group/{group_id}", response_class=RedirectResponse)
async def get_group_default(group_id: str):
return RedirectResponse(f"{parse.quote(group_id)}/today")
@router.get("/group/{group_id}/{date}", response_class=HTMLResponse)
async def get_group(request: AppRequest, group_id: str, date: DateDepends):
channels = GROUP_MAP.get(group_id, [tuple(item.split(":")) for item in group_id.split(",")])
responses = await _get_group_schedule(request.app.state.api, channels, date)
async def get_group(request: AppRequest, channels: ScheduleGroupChannelsDepends, date: DateDepends):
responses = [
await request.app.state.api.get_api(ScheduleApi, channel.provider).get_schedule(channel.id, date)
for channel in channels
]
return templates.TemplateResponse(
request=request,
name="group.html",
context={"responses": responses},
context={
"responses": responses,
"api_route": "/".join(
[
"/api/schedule/group",
request.path_params["group_id"],
request.path_params["date"],
]
),
},
)
@@ -82,5 +87,17 @@ async def get_channel(request: AppRequest, schedule_api: ScheduleApiDepends, cha
return templates.TemplateResponse(
request=request,
name="channel.html",
context={"response": response},
context={
"response": response,
"api_route": (
"/".join(
[
"/api/schedule",
request.path_params["provider"],
request.path_params["channel"],
request.path_params["date"],
]
)
),
},
)
@@ -10,6 +10,9 @@ locale=request.state.language)}}
{% block header %}
<app-link href="/schedule"
icon="tv">{{_("TV program")}}</app-link>
<app-link target="_blank"
href="{{ api_route }}"
icon="filetype-json"></app-link>
{% endblock %}
{% block content %}
@@ -44,7 +47,8 @@ locale=request.state.language)}}
{% for value in response.values %}
<tr class="{{'d-none' if (live and not value.live) else 'table-success' if (not live and value.live) else ''}}">
<td>
<span>{{value.start.strftime('%H:%M')}}</span>&nbsp;<span class="small ms-1">({{(value.end - value.start) | timedelta_format}})</span>
<span>{{value.start.strftime('%H:%M')}}</span>&nbsp;<span class="small ms-1">({{(value.end - value.start) |
timedelta_format}})</span>
</td>
<td>{{value.label}}</td>
</tr>
@@ -9,6 +9,9 @@
{% block header %}
<app-link href="/schedule"
icon="tv">{{_("TV program")}}</app-link>
<app-link target="_blank"
href="{{ api_route }}"
icon="filetype-json"></app-link>
{% endblock %}
{% block content %}
+20 -1
View File
@@ -3,6 +3,7 @@ from typing import Literal
from fastapi import APIRouter
from fastapi.responses import HTMLResponse, RedirectResponse
from starlette.datastructures import URL
from gallery.easel.core import AppRequest
from gallery.easel.depends.api import api_resolver
@@ -36,7 +37,18 @@ def build_weather_response(request: AppRequest, response: WeatherResponse):
return templates.TemplateResponse(
request=request,
name="weather.html",
context={"response": response},
context={
"response": response,
"api_route": "/".join(
[
"/api/weather",
request.path_params["provider"],
request.path_params["location"],
response.period,
request.path_params.get("date") or request.path_params.get("days") or "",
]
),
},
)
@@ -68,6 +80,13 @@ async def get_weather_days(request: AppRequest, weather_api: WeatherApiDepends,
return build_weather_response(request, response)
# TODO: tag mode route deprecated, used in tizen app
@router.get("/{provider}/{location}/tag/days-{days}", response_class=RedirectResponse)
async def get_tag_days(request: AppRequest, days: int):
url = URL(path=f"../days/{days}", query=request.url.query)
return RedirectResponse(url)
# TODO: tag mode route deprecated, used in tizen app
@router.get("/{provider}/{location}/{mode:str}/{date}", response_class=HTMLResponse)
async def get_weather_day(
@@ -7,6 +7,9 @@ locale=request.state.language)}}{% endblock %}
{% block header %}
<app-link href="/weather"
icon="brightness-high">{{_("Weather")}}</app-link>
<app-link target="_blank"
href="{{ api_route }}"
icon="filetype-json"></app-link>
{% endblock %}
{% block content %}
+4 -4
View File
@@ -5,10 +5,10 @@ import uvicorn
from gallery.easel import build_app
from gallery.painting.gismeteo.api import GismeteoApi
from gallery.painting.matchtv.api import MatchTvApi
from gallery.painting.matchtv.api import MatchTVApi
from gallery.painting.openweather.api import OpenWeatherApi
from gallery.painting.rp5.api import RP5WeatherApi
from gallery.painting.yandextv.api import YandexTvApi
from gallery.painting.yandextv.api import YandexTVApi
from gallery.painting.yandexweather.api import YandexWeatherApi
from gallery.sketch.bundle import ApiBundle
from gallery.sketch.schedule.cached import CachedScheduleApi
@@ -16,8 +16,8 @@ from gallery.sketch.weather.cached import CachedWeatherApi
api = ApiBundle(
[
CachedScheduleApi(YandexTvApi()),
CachedScheduleApi(MatchTvApi()),
CachedScheduleApi(YandexTVApi()),
CachedScheduleApi(MatchTVApi()),
CachedWeatherApi(GismeteoApi()),
CachedWeatherApi(OpenWeatherApi()),
CachedWeatherApi(YandexWeatherApi()),
+1
View File
@@ -0,0 +1 @@
PROVIDER = "gismeteo"
+14 -93
View File
@@ -1,20 +1,12 @@
import datetime
import json
import logging
from os import environ
from typing import Any
from bs4 import BeautifulSoup
from fastapi import HTTPException, status
from gallery.sketch.source import ApiSource
from gallery.sketch.weather.api import WeatherApi
from gallery.sketch.weather.model import Location, WeatherResponse, WeatherValue
from gallery.sketch.weather.util import merge_weather_values
from gallery.sketch.weather.model import Location, WeatherResponse
from .parser import DAYS_PARSER, LOCATION_PARSER, ONE_DAY_PARSER, ROW_PARSERS
logger = logging.getLogger("gismeteo")
from . import PROVIDER
from .parser import ForecastData, GismeteoForecastParser, GismeteoLocationParser
def format_date(date: datetime.date) -> str:
@@ -28,17 +20,11 @@ def format_date(date: datetime.date) -> str:
return f"{days + 1}-day"
def set_dict_nested_value(target: dict, key: str, value: Any):
parts = key.split(".")
for part in parts[:-1]:
if part not in target:
target[part] = {}
target = target[part]
target[parts[-1]] = value
class GismeteoApi(WeatherApi):
PROVIDER = "gismeteo"
PROVIDER = PROVIDER
forecast_parser = GismeteoForecastParser()
location_parser = GismeteoLocationParser()
def _build_source(self) -> ApiSource:
return ApiSource(
@@ -46,89 +32,24 @@ class GismeteoApi(WeatherApi):
cookies={"xauth": auth} if (auth := environ.get("GISMETEO_AUTH")) else None,
)
def _parse_oneday(self, date: datetime.date, data: str) -> WeatherResponse:
result: list[dict[str, Any]] = []
soup = BeautifulSoup(data, features="html.parser")
location = LOCATION_PARSER.parse_location(data)
widget = ONE_DAY_PARSER.parse_widget(soup)
for parser in ROW_PARSERS:
for index, value in enumerate(parser.parse_row(widget)):
while len(result) < index + 1:
result.append({})
set_dict_nested_value(result[index], parser.KEY, value)
values = [WeatherValue(**item) for item in result]
# group values by 3 hours
if len(values) > 8:
n = 3
values = [
merge_weather_values(values[i].date, values[i : i + n], "average") for i in range(0, len(values), n)
]
return WeatherResponse(
location=location or "n/a",
date=date,
period="day",
values=values,
)
def _parse_manydays(self, data: str) -> WeatherResponse:
result: list[dict[str, Any]] = []
soup = BeautifulSoup(data, features="html.parser")
location = LOCATION_PARSER.parse_location(data)
widget = DAYS_PARSER.parse_widget(soup)
for parser in ROW_PARSERS:
for index, value in enumerate(parser.parse_row(widget)):
while len(result) < index + 1:
result.append({})
set_dict_nested_value(result[index], parser.KEY, value)
values = [WeatherValue(**item) for item in result]
return WeatherResponse(
location=location or "n/a",
date=datetime.date.today(),
period="days",
values=values,
)
async def find_locations(self, query: str) -> list[Location]:
geo = "ru"
latitude = 52.968498
longitude = 36.0695
data = json.loads(
await self.source.request(
data = await self.source.request(
f"mq/city/q/?q={query}&geo={geo}&latitude={latitude}&longitude={longitude}&limit=10"
)
)
result = []
for item in data["data"]:
result.append(
Location(
id=f"{item['slug']}-{item['id']}",
name=item["translations"]["kk"]["city"]["name"],
provider=self.provider,
lat=item["coordinates"]["latitude"],
lon=item["coordinates"]["longitude"],
country=item["translations"]["kk"]["country"]["name"],
country_code=item["country"]["code"].lower(),
district=(
item["translations"]["kk"]["district"]["name"] if item["translations"]["kk"]["district"] else ""
),
subdistrict=(
item["translations"]["kk"]["subdistrict"]["name"]
if "subdistrict" in item["translations"]["kk"]
else ""
),
)
)
return result
return self.location_parser.parse(data)
async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
max_date = datetime.date.today() + datetime.timedelta(days=9)
if date > max_date:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail={"max_date": max_date.strftime("%Y-%m-%d")}
)
raise ValueError(f"Invalid date: {date}, max: {max_date}")
data = await self.source.request(f"weather-{location_id}/{format_date(date)}")
return self._parse_oneday(date, data)
context = ForecastData(date, "day", data, 8)
return self.forecast_parser.parse(context)
async def get_days(self, location_id: str, days: int) -> WeatherResponse:
data = await self.source.request(f"weather-{location_id}/{days}-days")
return self._parse_manydays(data)
context = ForecastData(datetime.date.today(), "days", data)
return self.forecast_parser.parse(context)
+135 -113
View File
@@ -1,92 +1,59 @@
import datetime
import logging
import re
from typing import Generic, Iterable, Optional, TypeVar
from dataclasses import dataclass
from typing import Any, Literal
import dateparser
from bs4 import Tag
from gallery.sketch.parse.core import ParsedContext, ParseError, Parser
from gallery.sketch.parse.date import DATE_PARSER
from gallery.sketch.parse.html import HTMLContextParser, TableParser, TableValueParser
from gallery.sketch.parse.json import JSONParser
from gallery.sketch.weather.model import (
Cloudness,
Location,
Precipitation,
Sky,
WeatherResponse,
WeatherValue,
WindDirection,
)
from gallery.sketch.weather.parser import ParseError, parse_wind_direction
from gallery.sketch.weather.parse.wind import WIND_DIRECTION_PARSER
from gallery.sketch.weather.util import fix_dates_sequence, merge_weather_values
logger = logging.getLogger("gismeteo")
T = TypeVar("T")
from . import PROVIDER
class WidgetParser:
def parse_widget(self, tag: Tag) -> Tag:
raise NotImplementedError
class BaseWidgetParser(WidgetParser):
SELECT: str
def __init__(self, select: Optional[str] = None):
super().__init__()
self._select = select or self.SELECT
def parse_widget(self, tag: Tag) -> Tag:
widget = tag.select_one(self._select)
if widget is None:
raise ValueError(self._select)
return widget
class RowParser(Generic[T]):
KEY: str
def parse_row(self, tag: Tag) -> Iterable[T]:
raise NotImplementedError
ONE_DAY_PARSER = BaseWidgetParser(".widget.widget-oneday .widget-items")
DAYS_PARSER = BaseWidgetParser(".widget.widget-days .widget-items")
class LocationParser:
class LocationNameParser(Parser[str, str]):
PATTERN = re.compile('{"ru":{"city":{"name":"(.*?)"')
def parse_location(self, data: str) -> str | None:
def parse(self, data: str) -> str:
match = self.PATTERN.search(data)
if match:
return match.group(1)
return None
raise ParseError("Location not found")
LOCATION_PARSER = LocationParser()
class TimeParser(TableValueParser[datetime.datetime]):
selector = ".widget-row.widget-row-datetime-time > .row-item > time-value"
model_key = "date"
def extract(self, data: Tag) -> datetime.datetime:
timestamp = int(data.attrs["timestamp"])
return datetime.datetime.fromtimestamp(timestamp)
class DateParser(RowParser[datetime.datetime]):
KEY = "date"
class DateParser(TableValueParser[datetime.datetime]):
selector = ".widget-row.widget-row-date > .row-item"
model_key = "date"
def parse_row(self, tag: Tag) -> Iterable[datetime.datetime]:
datetime_time_row = tag.select_one(".widget-row.widget-row-datetime-time")
if datetime_time_row:
for item in datetime_time_row.select(".row-item > time-value"):
timestamp = int(item.attrs["timestamp"])
time = datetime.datetime.fromtimestamp(timestamp)
yield time
else:
month = None
for item in tag.select(".widget-row.widget-row-date > .row-item"):
date_str = item.text
date = dateparser.parse(date_str, languages=["ru"])
if date is None:
raise ParseError.invalid_date(date_str)
if month and month > date.month:
date = date.replace(month=month)
month = date.month
yield date
def extract(self, data: Tag) -> datetime.datetime:
return DATE_PARSER.parse(data.text)
class SkyParser(RowParser[Sky]):
KEY = "sky"
class SkyParser(TableValueParser[Sky]):
selector = ".widget-row[data-row=icon-tooltip] > .row-item"
model_key = "sky"
CLOUDNESS_MAP: dict[str, Cloudness] = {
"ясно": Cloudness.CLEAR,
@@ -124,9 +91,8 @@ class SkyParser(RowParser[Sky]):
result_values = values - flag_values
return result_values, len(result_values) < len(values)
def parse_row(self, tag: Tag) -> Iterable[Sky]:
for item in tag.select(".widget-row[data-row=icon-tooltip] > .row-item"):
sky_str = item.attrs["data-tooltip"]
def extract(self, data: Tag) -> Sky:
sky_str = data.attrs["data-tooltip"]
values = {item.strip().lower() for item in sky_str.split(",")}
cloudness = Cloudness.CLEAR
precipitation = Precipitation.NO
@@ -143,8 +109,8 @@ class SkyParser(RowParser[Sky]):
values.remove(k)
break
if values:
logger.warning("unknown sky values: %s:", values)
yield Sky(
raise ParseError("Unknown sky values", values)
return Sky(
cloudness=cloudness,
precipitation=precipitation,
thunder=thunder,
@@ -152,71 +118,69 @@ class SkyParser(RowParser[Sky]):
)
class TemperatureParser(RowParser[list[int]]):
KEY = "temperature"
class TemperatureParser(TableValueParser[list[int]]):
selector = ".widget-row-chart[data-row=temperature-air] > .chart > .values > .value"
model_key = "temperature"
def parse_row(self, tag: Tag) -> Iterable[list[int]]:
for item in tag.select(".widget-row-chart[data-row=temperature-air] > .chart > .values > .value"):
yield [int(value.attrs["value"]) for value in item.select("temperature-value")]
def extract(self, data: Tag) -> list[int]:
return [int(value.attrs["value"]) for value in data.select("temperature-value")]
class WindSpeedParser(RowParser[int]):
KEY = "wind.speed"
class WindSpeedParser(TableValueParser[int]):
selector = ".widget-row-wind > .row-item > .wind-speed"
model_key = "wind.speed"
def parse_row(self, tag: Tag) -> Iterable[int]:
for item in tag.select(".widget-row-wind > .row-item > .wind-speed > speed-value"):
yield int(item.attrs["value"])
def extract(self, data: Tag) -> int:
value = data.select_one("speed-value")
return int(value.attrs["value"]) if value else 0
class WindGustParser(RowParser[int]):
KEY = "wind.gust"
class WindGustParser(TableValueParser[int]):
selector = ".widget-row-wind > .row-item > .wind-gust"
model_key = "wind.gust"
def parse_row(self, tag: Tag) -> Iterable[int]:
for item in tag.select(".widget-row-wind > .row-item > .wind-gust"):
value = item.select_one("speed-value")
yield int(value.attrs["value"]) if value else 0
def extract(self, data: Tag) -> int:
value = data.select_one("speed-value")
return int(value.attrs["value"]) if value else 0
class WindDirectionParser(RowParser[WindDirection]):
KEY = "wind.direction"
class WindDirectionParser(TableValueParser[WindDirection | None]):
selector = ".widget-row-wind > .row-item > .wind-speed > .wind-direction"
model_key = "wind.direction"
def parse_row(self, tag: Tag) -> Iterable[float]:
for item in tag.select(".widget-row-wind > .row-item > .wind-speed > .wind-direction"):
wind_direction_str = item.text.lower().strip()
yield parse_wind_direction(wind_direction_str)
def extract(self, data: Tag) -> WindDirection | None:
return WIND_DIRECTION_PARSER.parse(data.text.strip())
class PrecipitationParser(RowParser[float]):
KEY = "precipitation"
class PrecipitationParser(TableValueParser[float]):
selector = ".widget-row[data-row=precipitation-bars] > .row-item"
model_key = "precipitation"
def parse_row(self, tag: Tag) -> Iterable[float]:
for item in tag.select(".widget-row[data-row=precipitation-bars] > .row-item"):
value = item.select_one("precipitation-value")
if value:
yield float(value.attrs["value"])
else:
yield 0
def extract(self, data: Tag) -> float:
value = data.select_one("precipitation-value")
return float(value.attrs["value"]) if value else 0
class PressureParser(RowParser[list[int]]):
KEY = "pressure"
class PressureParser(TableValueParser[list[int]]):
selector = ".widget-row-chart[data-row=pressure] > .chart > .values > .value"
model_key = "pressure"
def parse_row(self, tag: Tag) -> Iterable[list[int]]:
for item in tag.select(".widget-row-chart[data-row=pressure] > .chart > .values > .value"):
yield [int(value.attrs["value"]) for value in item.select("pressure-value")]
def extract(self, data: Tag) -> list[int]:
return [int(value.attrs["value"]) for value in data.select("pressure-value")]
class HumidityParser(RowParser[int]):
KEY = "humidity"
class HumidityParser(TableValueParser[int]):
selector = ".widget-row[data-row=humidity] > .row-item, .widget-row[data-row=humidity-avg] > .row-item"
model_key = "humidity"
def parse_row(self, tag: Tag) -> Iterable[int]:
for item in tag.select(
".widget-row[data-row=humidity] > .row-item, .widget-row[data-row=humidity-avg] > .row-item"
):
yield int(item.text)
def extract(self, data: Tag) -> int:
return int(data.text)
ROW_PARSERS: list[RowParser] = [
class GismeteoTableParser(TableParser[WeatherValue]):
selector = ".widget .widget-items"
value_parsers = [
TimeParser(),
DateParser(),
SkyParser(),
TemperatureParser(),
@@ -228,4 +192,62 @@ ROW_PARSERS: list[RowParser] = [
HumidityParser(),
]
ROW_PARSERS_MAP: dict[str, RowParser] = {parser.KEY: parser for parser in ROW_PARSERS}
def create_model(self, data: dict) -> WeatherValue:
return WeatherValue.model_validate(data)
@dataclass
class ForecastData:
date: datetime.date
period: Literal["day", "days"]
data: str
values: int | None = None
class GismeteoForecastParser(HTMLContextParser[ForecastData, WeatherResponse]):
location_parser = LocationNameParser()
table_parser = GismeteoTableParser()
def extract(self, data: ParsedContext[ForecastData, Tag]) -> WeatherResponse:
location = self.location_parser.parse(data.context.data)
values = self.table_parser.parse(data.data)
if data.context.values and len(values) > data.context.values:
n = int(len((values)) / data.context.values)
values = [
merge_weather_values(values[i].date, values[i : i + n], "average") for i in range(0, len(values), n)
]
if data.context.period == "days":
fix_dates_sequence(values)
return WeatherResponse(
location=location,
date=data.context.date,
period=data.context.period,
values=values,
)
class GismeteoLocationParser(JSONParser[list[Location]]):
def extract(self, data: Any) -> list[Location]:
result = []
for item in data["data"]:
result.append(
Location(
id=f"{item['slug']}-{item['id']}",
name=item["translations"]["kk"]["city"]["name"],
provider=PROVIDER,
lat=item["coordinates"]["latitude"],
lon=item["coordinates"]["longitude"],
country=item["translations"]["kk"]["country"]["name"],
country_code=item["country"]["code"].lower(),
district=(
item["translations"]["kk"]["district"]["name"] if item["translations"]["kk"]["district"] else ""
),
subdistrict=(
item["translations"]["kk"]["subdistrict"]["name"]
if "subdistrict" in item["translations"]["kk"]
else ""
),
)
)
return result
+1
View File
@@ -0,0 +1 @@
PROVIDER = "matchtv"
+17 -41
View File
@@ -1,57 +1,33 @@
import datetime
import json
import logging
from gallery.sketch.schedule.api import ScheduleApi
from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
from gallery.sketch.schedule.model import Channel, Schedule
from gallery.sketch.source import ApiSource
logger = logging.getLogger("matchtv")
from . import PROVIDER
from .parser import (
ChannelData,
MatchTVChannelParser,
MatchTVScheduleParser,
ScheduleData,
)
class MatchTvApi(ScheduleApi):
PROVIDER = "matchtv"
class MatchTVApi(ScheduleApi):
PROVIDER = PROVIDER
schedule_parser = MatchTVScheduleParser()
channel_parser = MatchTVChannelParser()
def _build_source(self) -> ApiSource:
return ApiSource("https://matchtv.ru")
async def find_channels(self, query: str) -> list[Channel]:
endpoint = "api/v1/channels"
data = json.loads(await self.source.request(endpoint))
result = []
query = query.lower()
for item in data["result"]:
if query in item["name"].lower() or query in item["alias"]:
channel_id = item["alias"]
name = item["name"].split("|")[0].strip()
result.append(Channel(id=channel_id, name=name, provider=self.provider))
return result
data = await self.source.request(endpoint)
return self.channel_parser.parse(ChannelData(query, data))
async def get_schedule(self, channel_id: str, date: datetime.date) -> Schedule:
endpoint = f"api/v1/channels/{channel_id}/tv-schedule?date={date:%Y%m%d}"
data = json.loads(await self.source.request(endpoint))
channel_data = data["result"]["channels"][0]
channel = Channel(id=channel_data["alias"], name=channel_data["name"], provider=self.provider)
values = []
current_day = datetime.datetime.combine(date.today(), datetime.datetime.min.time())
end = current_day + datetime.timedelta(days=1, hours=6)
prev_value: ScheduleValue | None = None
for item in channel_data["schedule"]:
title = item["title"]
time_str = item["time"]
hours, minutes = map(int, time_str.split(":"))
item_date = current_day.replace(hour=hours, minute=minutes)
if prev_value is not None and item_date.hour < prev_value.start.hour:
current_day += datetime.timedelta(days=1)
item_date += datetime.timedelta(days=1)
live = "Прямая трансляция" in title
value = ScheduleValue(start=item_date, end=end, label=title, live=live)
values.append(value)
if prev_value is not None:
prev_value.end = item_date
prev_value = value
return Schedule(
channel=channel,
date=date,
values=values,
)
data = await self.source.request(endpoint)
return self.schedule_parser.parse(ScheduleData(date, data))
+71
View File
@@ -0,0 +1,71 @@
import datetime
import re
from dataclasses import dataclass
from typing import Any
from gallery.sketch.parse.core import ParsedContext, Parser
from gallery.sketch.parse.json import JSONContextParser
from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
from . import PROVIDER
class DateParser(Parser[str, datetime.date]):
def parse(self, data: str) -> datetime.date:
matched = re.compile('{\\"date\\":(\\d+)}').search(data)
return datetime.date.strptime(matched.group(1), data)
@dataclass
class ScheduleData:
date: datetime.date
data: str
class MatchTVScheduleParser(JSONContextParser[ScheduleData, Schedule]):
date_parser = DateParser()
def extract(self, data: ParsedContext[ScheduleData, Any]) -> Schedule:
channel_data = data.data["result"]["channels"][0]
channel = Channel(id=channel_data["alias"], name=channel_data["name"], provider=PROVIDER)
values = []
start = datetime.datetime.combine(data.context.date, datetime.time.min)
end = start + datetime.timedelta(days=1, hours=6)
prev_value: ScheduleValue | None = None
for item in channel_data["schedule"]:
title = item["title"]
time_str = item["time"]
hours, minutes = map(int, time_str.split(":"))
item_date = start.replace(hour=hours, minute=minutes)
if prev_value is not None and item_date.hour < prev_value.start.hour:
start += datetime.timedelta(days=1)
item_date += datetime.timedelta(days=1)
live = "Прямая трансляция" in title
value = ScheduleValue(start=item_date, end=end, label=title, live=live)
values.append(value)
if prev_value is not None:
prev_value.end = item_date
prev_value = value
return Schedule(
channel=channel,
date=data.context.date,
values=values,
)
@dataclass
class ChannelData:
query: str
data: str
class MatchTVChannelParser(JSONContextParser[ChannelData, list[Channel]]):
def extract(self, data: ParsedContext[ChannelData, Any]) -> list[Channel]:
result = []
query = data.context.query.lower()
for item in data.data["result"]:
if query in item["name"].lower() or query in item["alias"]:
channel_id = item["alias"]
name = item["name"].split("|")[0].strip()
result.append(Channel(id=channel_id, name=name, provider=PROVIDER))
return result
+1
View File
@@ -0,0 +1 @@
PROVIDER = "openweather"
+20 -37
View File
@@ -1,6 +1,5 @@
import datetime
import logging
from collections import defaultdict
from itertools import groupby
from os import environ
from aiocache import cached
@@ -10,16 +9,17 @@ from gallery.sketch.weather.model import Location, WeatherResponse, WeatherValue
from gallery.sketch.weather.util import merge_weather_values
from gallery.util import TimeUnit
from .openweather import Forecast
from . import PROVIDER
from .openweather import Location as OpenWeatherLocation
from .openweather import OpenWeather
from .parser import FORECAST_ITEM_PARSER
logger = logging.getLogger("openweather")
from .parser import OpenWeatherForecastParser, OpenWeatherLocationParser
class OpenWeatherApi(WeatherApi[OpenWeather]):
PROVIDER = "openweather"
PROVIDER = PROVIDER
forecast_parser = OpenWeatherForecastParser()
location_parser = OpenWeatherLocationParser()
def _build_source(self) -> OpenWeather:
return OpenWeather(environ["OPENWEATHER_KEY"])
@@ -41,34 +41,17 @@ class OpenWeatherApi(WeatherApi[OpenWeather]):
alias="redis",
ttl=TimeUnit.HOUR,
)
async def _get_location_forecast(self, location_id: str) -> Forecast:
return await self.source.get_forecast(*self._parse_location(location_id))
async def _get_location_forecast(self, location_id: str) -> list[WeatherValue]:
content = await self.source.get_forecast(*self._parse_location(location_id))
return self.forecast_parser.parse(content)
async def find_locations(self, query: str) -> list[Location]:
result = await self.source.find_locations(query)
return [
Location(
id=f"{item.lat}:{item.lon}",
name=item.name,
provider=self.provider,
lat=item.lat,
lon=item.lon,
country=item.country,
country_code=item.country.lower(),
district=item.state or "",
subdistrict="",
)
for item in result
]
content = await self.source.find_locations(query)
return self.location_parser.parse(content)
async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
location: OpenWeatherLocation = await self._get_location(location_id)
data: Forecast = await self._get_location_forecast(location_id)
values = []
for item in data.list:
value = FORECAST_ITEM_PARSER.parse(item)
if value.date.date() == date:
values.append(value)
values = [value for value in await self._get_location_forecast(location_id) if value.date.date() == date]
return WeatherResponse(
location=location.name,
date=date,
@@ -78,13 +61,13 @@ class OpenWeatherApi(WeatherApi[OpenWeather]):
async def get_days(self, location_id: str, days: int) -> WeatherResponse:
location: OpenWeatherLocation = await self._get_location(location_id)
data: Forecast = await self._get_location_forecast(location_id)
values_by_date: dict[datetime.datetime, list[WeatherValue]] = defaultdict(list)
for item in data.list:
value = FORECAST_ITEM_PARSER.parse(item)
item_date = value.date.replace(hour=0, minute=0)
values_by_date[item_date].append(value)
values = [merge_weather_values(date, values) for date, values in values_by_date.items()]
values = [
merge_weather_values(datetime.datetime.combine(k, datetime.datetime.min.time()), list(g))
for k, g in groupby(
sorted(await self._get_location_forecast(location_id), key=lambda item: item.date.date()),
lambda item: item.date.date(),
)
]
return WeatherResponse(
location=location.name,
date=datetime.date.today(),
+62 -23
View File
@@ -1,13 +1,21 @@
import datetime
from gallery.sketch.weather.model import Cloudness, Precipitation, WeatherValue
from gallery.sketch.weather.util import build_weather_value
from gallery.sketch.parse.core import Parser
from gallery.sketch.weather.model import (
Cloudness,
Location,
Precipitation,
WeatherValue,
)
from gallery.sketch.weather.parse.wind import WIND_DIRECTION_PARSER
from gallery.sketch.weather.util import convert_hPa_to_mmHg
from ...sketch.weather.parser import parse_wind_direction
from .openweather import ForecastItem
from . import PROVIDER
from .openweather import Forecast, ForecastItem
from .openweather import Location as OpenWeatherLocation
class ForecastItemParser:
class ForecastItemParser(Parser[ForecastItem, list[WeatherValue]]):
CLOUDNESS_MAP: dict[str, Cloudness] = {
"clear sky": Cloudness.CLEAR,
"few clouds": Cloudness.PARTLY_CLOUDY,
@@ -23,23 +31,54 @@ class ForecastItemParser:
"heavy rain": Precipitation.SHOWER,
}
def parse(self, item: ForecastItem) -> WeatherValue:
item_date = datetime.datetime.fromtimestamp(item.dt, datetime.UTC)
item_date = item_date.replace(tzinfo=datetime.timezone.utc).astimezone(tz=None).replace(tzinfo=None)
value = build_weather_value(item_date)
# TODO parse temperature interval flag
value.temperature = [round(item.main.temp)]
# value.temperature = [round(item.main.temp_max), round(item.main.temp_min)]
value.pressure = [round(item.main.pressure * 0.75006)]
value.humidity = item.main.humidity
value.wind.speed = round(item.wind.speed)
value.wind.gust = round(item.wind.gust)
value.wind.direction = parse_wind_direction(item.wind.deg)
value.sky.cloudness = self.CLOUDNESS_MAP.get(item.weather[0].description, Cloudness.CLEAR)
value.sky.precipitation = self.PRECIPITATION_MAP.get(item.weather[0].description, Precipitation.NO)
if item.rain:
value.precipitation = round(item.rain.interval_3h, 1)
return value
def parse(self, data: ForecastItem) -> WeatherValue:
date = (
datetime.datetime.fromtimestamp(data.dt, datetime.UTC)
.replace(tzinfo=datetime.timezone.utc)
.astimezone(tz=None)
.replace(tzinfo=None)
)
return WeatherValue.model_validate(
{
"date": date,
"sky": {
"cloudness": self.CLOUDNESS_MAP.get(data.weather[0].description, Cloudness.CLEAR),
"precipitation": self.PRECIPITATION_MAP.get(data.weather[0].description, Precipitation.NO),
},
"wind": {
"speed": round(data.wind.speed),
"gust": round(data.wind.gust),
"direction": WIND_DIRECTION_PARSER.parse(data.wind.deg),
},
"temperature": [round(data.main.temp)], # [round(data.main.temp_max), round(data.main.temp_min)]
"precipitation": round(data.rain.interval_3h, 1) if data.rain else 0,
"pressure": [convert_hPa_to_mmHg(data.main.pressure)],
"humidity": data.main.humidity,
}
)
FORECAST_ITEM_PARSER = ForecastItemParser()
class OpenWeatherForecastParser(Parser[Forecast, list[WeatherValue]]):
item_parser = ForecastItemParser()
def parse(self, data: Forecast) -> list[WeatherValue]:
return [self.item_parser.parse(item) for item in data.list]
class OpenWeatherLocationParser(Parser[list[OpenWeatherLocation], list[Location]]):
def parse(self, data: list[OpenWeatherLocation]) -> list[Location]:
return [
Location(
id=f"{item.lat}:{item.lon}",
name=item.name,
provider=PROVIDER,
lat=item.lat,
lon=item.lon,
country=item.country,
country_code=item.country.lower(),
district=item.state or "",
subdistrict="",
)
for item in data
]
+1
View File
@@ -0,0 +1 @@
PROVIDER = "rp5"
+10 -34
View File
@@ -1,10 +1,9 @@
import datetime
import logging
import aiohttp
from aiocache import cached
from bs4 import BeautifulSoup
from gallery.sketch.headers import build_headers
from gallery.sketch.source import ApiSource
from gallery.sketch.weather.api import WeatherApi
from gallery.sketch.weather.model import (
@@ -14,14 +13,15 @@ from gallery.sketch.weather.model import (
from gallery.sketch.weather.util import merge_weather_values
from gallery.util import TimeUnit
from ...sketch.headers import build_headers
from .parser import parse_page
logger = logging.getLogger("rp5")
from . import PROVIDER
from .parser import RP5ForecastParser, RP5LocationParser
class RP5WeatherApi(WeatherApi):
PROVIDER = "rp5"
PROVIDER = PROVIDER
forecast_parser = RP5ForecastParser()
location_parser = RP5LocationParser()
def _build_source(self) -> ApiSource:
return ApiSource("https://rp5.ru", headers=build_headers("rp5.ru"))
@@ -29,31 +29,8 @@ class RP5WeatherApi(WeatherApi):
async def find_locations(self, query: str) -> list[Location]:
endpoint = "docs/search/ru"
data = aiohttp.FormData({"searchStr": query})
response = await self.source.request(endpoint, method="POST", data=data)
soup = BeautifulSoup(response, features="html.parser")
table = soup.select_one(".searchResults")
result = []
for row in table.select("tr")[1:]:
cells = row.select("td")
name = cells[0].text.strip()
location_id = cells[0].select_one("a").attrs["href"].lstrip("/")
country = cells[1].text.strip()
district = cells[2].text.strip()
subdistrict = cells[3].text.strip()
result.append(
Location(
id=location_id,
name=name,
provider=self.provider,
lat=0,
lon=0,
country=country,
country_code="ru", # TODO: contry_code from country
district=district,
subdistrict=subdistrict,
)
)
return result
content = await self.source.request(endpoint, method="POST", data=data)
return self.location_parser.parse(content)
@cached(
key_builder=lambda fun, self, location_id: f"api.weather.{self.provider}.source.{location_id}.forecast",
@@ -63,8 +40,7 @@ class RP5WeatherApi(WeatherApi):
async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]:
endpoint = f"{location_id}"
content = await self.source.request(endpoint)
soup = BeautifulSoup(content, features="html.parser")
return parse_page(soup)
return self.forecast_parser.parse(content)
async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
forecast = await self._get_location_forecast(location_id)
+180 -103
View File
@@ -1,18 +1,63 @@
import datetime
import re
from enum import Enum, auto
from typing import Iterable
import dateparser
from bs4 import BeautifulSoup, Tag
from bs4 import Tag
from gallery.sketch.parse.core import ParseError
from gallery.sketch.parse.date import DATE_PARSER
from gallery.sketch.parse.html import (
HTMLParser,
TableParser,
TableValueParser,
TagIterParser,
TagParser,
)
from gallery.sketch.weather.model import (
Cloudness,
Location,
Precipitation,
WeatherResponse,
WeatherValue,
WindDirection,
)
from gallery.sketch.weather.parser import ParseError, parse_wind_direction
from gallery.sketch.weather.util import build_weather_value
from gallery.sketch.weather.parse.wind import WIND_DIRECTION_PARSER
from gallery.sketch.weather.util import fix_dates_sequence
from . import PROVIDER
class LocationNameParser(TagParser[str]):
selector = ".intoLeftNavi > .verticalBottom > a"
def extract(self, data: Tag) -> str:
return data.text.strip()
class DatesParser(TagIterParser[tuple[datetime.datetime, int]]):
selector = "#forecastTable > tr.forecastDate > td"
def extract(self, data: Tag) -> tuple[datetime.datetime, int]:
colspan = int(data.attrs["colspan"])
index = data.parent.index(data)
if index == 1:
colspan -= 1
date_str = ",".join(data.text.strip().split(",")[-2:])
date = DATE_PARSER.parse(date_str)
return (date, colspan)
class TimeParser(TableValueParser[datetime.datetime]):
selector = "tr.forecastTime > td:not(:first-of-type)"
model_key = "date"
def extract(self, data: Tag) -> datetime.datetime:
hour = int(data.text.strip())
return datetime.datetime.min.replace(hour=hour)
class CloudnessParser(TableValueParser[Cloudness]):
selector = "tr:nth-of-type(3) > td:not(:first-of-type)"
model_key = "sky.cloudness"
CLOUDNESS_MAP = {
"Ясно": Cloudness.CLEAR,
@@ -25,122 +70,154 @@ CLOUDNESS_MAP = {
"Пасмурная погода": Cloudness.MAINLY_CLOUDY,
}
def extract(self, data: Tag) -> Cloudness:
onmouseover = data.select_one(".cc_0").next_element.attrs["onmouseover"]
cloudness_str = re.compile("<b>(.*?)</b>").search(onmouseover).group(1).strip()
if cloudness_str not in self.CLOUDNESS_MAP:
raise ParseError("Unknown cloudness value", cloudness_str)
return self.CLOUDNESS_MAP[cloudness_str]
class PrecipitationParser(TableValueParser[tuple[Precipitation, float]]):
selector = "tr:nth-of-type(4) > td:not(:first-of-type)"
model_key = ["sky.precipitation", "precipitation"]
PRECIPITATION_MAP = {
"Явления погоды отсутствуют": Precipitation.NO,
"Cлабый дождь": Precipitation.SMALL_RAIN,
"Умеренный дождь": Precipitation.RAIN,
}
def extract(self, data: Tag) -> tuple[Precipitation, float]:
onmouseover = data.select_one(".pr_0").attrs["onmouseover"]
precipitation_str = onmouseover.split(",")[1].strip(" '")
precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str)
precipitation_value = 0
if precipitation_value_matched:
# one hour value
precipitation_value = float(precipitation_value_matched.group(1)) * 6
if " (" in precipitation_str:
precipitation_str = precipitation_str.split(" (")[0]
if precipitation_str not in self.PRECIPITATION_MAP:
raise ParseError("Unknown precipitation value", precipitation_str)
return self.PRECIPITATION_MAP[precipitation_str], precipitation_value
def _parse_wind_speed(cell: Tag) -> int:
element = cell.select_one(".wv_0")
class TemperatureParser(TableValueParser[list[int]]):
selector = "tr:nth-of-type(5) > td:not(:first-of-type)"
model_key = "temperature"
def extract(self, data: Tag) -> list[int]:
return [int(data.select_one(".t_0").text.lstrip("+").strip())]
class PressureParser(TableValueParser[list[int]]):
selector = "tr:nth-last-of-type(8) > td:not(:first-of-type)"
model_key = "pressure"
def extract(self, data: Tag) -> list[int]:
return [int(data.select_one(".p_0").text.strip())]
class WindSpeedParser(TableValueParser[int]):
selector = "tr:nth-last-of-type(7) > td:not(:first-of-type)"
model_key = "wind.speed"
def extract(self, data: Tag) -> int:
element = data.select_one(".wv_0")
if element:
return int(element.text.strip() or "0")
return 0
class RowType(Enum):
DATE = auto()
TIME = auto()
CLOUDNESS = auto()
PRECIPITATION = auto()
TEMPERATURE = auto()
PRESSURE = auto()
WIND_SPEED = auto()
WIND_GUST = auto()
WIND_DIRECTION = auto()
HUMIDITY = auto()
class WindGustParser(WindSpeedParser):
selector = "tr:nth-last-of-type(6) > td:not(:first-of-type)"
model_key = "wind.gust"
def _iter_table_rows(table: Tag) -> Iterable[tuple[RowType, Tag]]:
for row in table.select("tr"):
title = row.select_one("td").text.strip().lower()
if "сегодня" in title or "завтра" in title:
yield RowType.DATE, row
elif "местное время" in title:
yield RowType.TIME, row
elif "облачность" in title:
yield RowType.CLOUDNESS, row
elif "осадки" in title:
yield RowType.PRECIPITATION, row
elif "температура" in title:
yield RowType.TEMPERATURE, row
elif "давление" in title:
yield RowType.PRESSURE, row
elif "скорость" in title:
yield RowType.WIND_SPEED, row
elif "порывы" in title:
yield RowType.WIND_GUST, row
elif "направление" in title:
yield RowType.WIND_DIRECTION, row
elif "влажность" in title:
yield RowType.HUMIDITY, row
class WindDirectionParser(TableValueParser[WindDirection | None]):
selector = "tr:nth-last-of-type(5) > td:not(:first-of-type)"
model_key = "wind.direction"
def extract(self, data: Tag) -> WindDirection | None:
return WIND_DIRECTION_PARSER.parse(data.text.strip())
# pylint:disable=too-many-branches
def parse_page(page: BeautifulSoup) -> list[WeatherResponse]:
location = page.select(".intoLeftNavi > .verticalBottom > a")[-1].text.strip()
table = page.select_one("#forecastTable")
if table is None:
raise ParseError("#forecastTable not found")
result: list[WeatherResponse] = []
values: list[WeatherValue] = []
for row_type, row in _iter_table_rows(table):
match row_type:
case RowType.DATE:
for index, cell in enumerate(row.select("td")):
colspan = int(cell.attrs["colspan"])
if index == 0:
colspan -= 1
date_str = ",".join(cell.text.strip().split(",")[-2:])
date = dateparser.parse(date_str, languages=["ru"])
if date is None:
raise ParseError.invalid_date(date_str)
item_values = [build_weather_value(date) for _ in range(colspan)]
result.append(
class HumidityParser(TableValueParser[int]):
selector = "tr:nth-last-of-type(4) > td:not(:first-of-type)"
model_key = "humidity"
def extract(self, data: Tag) -> int:
return int(data.text.strip())
class RP5TableParser(TableParser[WeatherValue]):
selector = "#forecastTable"
value_parsers = [
TimeParser(),
CloudnessParser(),
PrecipitationParser(),
TemperatureParser(),
PressureParser(),
WindSpeedParser(),
WindGustParser(),
WindDirectionParser(),
HumidityParser(),
]
def create_model(self, data: dict) -> WeatherValue:
return WeatherValue.model_validate(data)
class RP5ForecastParser(HTMLParser[list[WeatherResponse]]):
location_parser = LocationNameParser()
dates_parser = DatesParser()
table_parser = RP5TableParser()
def extract(self, data: Tag) -> list[WeatherResponse]:
location = self.location_parser.parse(data)
dates = self.dates_parser.parse(data)
values = self.table_parser.parse(data)
responses = []
for date, count in dates:
item_values, values = values[:count], values[count:]
for value in item_values:
value.date = date.replace(hour=value.date.hour)
responses.append(
WeatherResponse(
location=location,
date=date.date(),
date=date,
period="day",
values=item_values,
)
)
values += item_values
case RowType.TIME:
for index, cell in enumerate(row.select("td")[1:]):
hour = int(cell.text.strip())
values[index].date = values[index].date.replace(hour=hour)
case RowType.CLOUDNESS:
for index, cell in enumerate(row.select("td")[1:]):
onmouseover = cell.select_one(".cc_0").next_element.attrs["onmouseover"]
cloudness_str = re.compile("<b>(.*?)</b>").search(onmouseover).group(1).strip()
values[index].sky.cloudness = CLOUDNESS_MAP[cloudness_str]
case RowType.PRECIPITATION:
for index, cell in enumerate(row.select("td")[1:]):
onmouseover = cell.select_one(".pr_0").attrs["onmouseover"]
precipitation_str = onmouseover.split(",")[1].strip(" '")
precipitation_value_matched = re.compile("(\\d+\\.\\d+) мм").search(precipitation_str)
if precipitation_value_matched:
# one hour value
values[index].precipitation = float(precipitation_value_matched.group(1)) * 6
if "(" in precipitation_str:
precipitation_str = precipitation_str.split(" (")[0]
values[index].sky.precipitation = PRECIPITATION_MAP[precipitation_str]
case RowType.TEMPERATURE:
for index, cell in enumerate(row.select("td")[1:]):
values[index].temperature = [int(cell.select_one(".t_0").text.lstrip("+").strip())]
case RowType.PRESSURE:
for index, cell in enumerate(row.select("td")[1:]):
values[index].pressure = [int(cell.select_one(".p_0").text.strip())]
case RowType.WIND_SPEED:
for index, cell in enumerate(row.select("td")[1:]):
values[index].wind.speed = _parse_wind_speed(cell)
case RowType.WIND_GUST:
for index, cell in enumerate(row.select("td")[1:]):
values[index].wind.gust = _parse_wind_speed(cell)
case RowType.WIND_DIRECTION:
for index, cell in enumerate(row.select("td")[1:]):
values[index].wind.direction = parse_wind_direction(cell.text.strip())
case RowType.HUMIDITY:
for index, cell in enumerate(row.select("td")[1:]):
values[index].humidity = int(cell.text.strip())
fix_dates_sequence(responses)
return responses
class RP5LocationParser(HTMLParser[list[Location]]):
selector = ".searchResults"
def extract(self, data: Tag) -> list[Location]:
result = []
for row in data.select("tr")[1:]:
cells = row.select("td")
name = cells[0].text.strip()
location_id = cells[0].select_one("a").attrs["href"].lstrip("/")
country = cells[1].text.strip()
district = cells[2].text.strip()
subdistrict = cells[3].text.strip()
result.append(
Location(
id=location_id,
name=name,
provider=PROVIDER,
lat=0,
lon=0,
country=country,
country_code="ru", # TODO: contry_code from country
district=district,
subdistrict=subdistrict,
)
)
return result
+1
View File
@@ -0,0 +1 @@
PROVIDER = "yandex"
+17 -46
View File
@@ -1,20 +1,21 @@
import datetime
import json
import logging
from bs4 import BeautifulSoup
from gallery.sketch.headers import build_headers
from gallery.sketch.schedule.api import ScheduleApi
from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
from gallery.sketch.schedule.model import Channel, Schedule
from gallery.sketch.source import ApiSource
from ...sketch.headers import build_headers
logger = logging.getLogger("matchtv")
from . import PROVIDER
from .parser import ScheduleData, YandexTVChannelParser, YandexTVScheduleParser
class YandexTvApi(ScheduleApi):
PROVIDER = "yandextv"
class YandexTVApi(ScheduleApi):
PROVIDER = PROVIDER
schedule_parser = YandexTVScheduleParser()
channel_parser = YandexTVChannelParser()
lr = 213 # magic id
def _build_source(self) -> ApiSource:
return ApiSource("https://tv.yandex.ru", headers=build_headers("tv.yandex.ru"))
@@ -22,43 +23,13 @@ class YandexTvApi(ScheduleApi):
async def find_channels(self, query: str) -> list[Channel]:
url = (
"https://suggest-multi.yandex.ru/suggest-tv2?"
f"v=4&uil=ru&lr=10&count_channels=4&count_programs=4&sn=50&part={query}"
f"v=4&uil=ru&lr={self.lr}&count_channels=4&count_programs=4&sn=50&part={query}"
)
_, values = json.loads(await self.source.request(url))
result = []
for _, name, content in values:
if content["label"] == "Каналы":
channel_id = content["url"].split("/")[-1]
result.append(Channel(id=channel_id, name=name, provider=self.provider))
return result
data = await self.source.request(url)
return self.channel_parser.parse(data)
async def get_schedule(self, channel_id: str, date: datetime.date) -> Schedule:
endpoint = f"channels/{channel_id}?date={date:%Y-%m-%d}"
channel_path_part = "channel" if "-" in channel_id else "channels"
endpoint = f"{self.lr}/{channel_path_part}/{channel_id}?date={date:%Y-%m-%d}"
data = await self.source.request(endpoint)
soup = BeautifulSoup(data, features="html.parser")
if soup.select_one(".CheckboxCaptcha") is not None:
raise RuntimeError("Captcha")
values = []
channel_name = soup.select_one(".channel-header__text").text.strip()
current_day = datetime.datetime.combine(date.today(), datetime.datetime.min.time())
end = current_day + datetime.timedelta(days=1, hours=6)
prev_value: ScheduleValue | None = None
for item in soup.select(".channel-schedule .channel-schedule__event"):
title = item.select_one(".channel-schedule__title").text.strip()
time_str = item.select_one(".channel-schedule__time").text.strip()
hours, minutes = map(int, time_str.split(":"))
item_date = current_day.replace(hour=hours, minute=minutes)
if prev_value is not None and item_date.hour < prev_value.start.hour:
current_day += datetime.timedelta(days=1)
item_date += datetime.timedelta(days=1)
live = item.select_one(".channel-schedule__info .icon_live") is not None
value = ScheduleValue(start=item_date, end=end, label=title, live=live)
values.append(value)
if prev_value is not None:
prev_value.end = item_date
prev_value = value
return Schedule(
channel=Channel(id=channel_id, name=channel_name, provider=self.provider),
date=date,
values=values,
)
return self.schedule_parser.parse(ScheduleData(date, channel_id, data))
+60
View File
@@ -0,0 +1,60 @@
import datetime
from dataclasses import dataclass
from typing import Any
from bs4 import Tag
from gallery.sketch.parse.core import ParsedContext, ParseError
from gallery.sketch.parse.json import JSONParser
from gallery.sketch.schedule.model import Channel, Schedule, ScheduleValue
from ...sketch.parse.html import HTMLContextParser
from . import PROVIDER
@dataclass
class ScheduleData:
date: datetime.date
channel_id: str
data: str
class YandexTVScheduleParser(HTMLContextParser[ScheduleData, Schedule]):
def extract(self, data: ParsedContext[ScheduleData, Tag]) -> Schedule:
if data.data.select_one(".CheckboxCaptcha") is not None:
raise ParseError("Captcha")
values = []
channel_name = data.data.select_one(".channel-header__text").text.strip()
current_day = datetime.datetime.combine(data.context.date, datetime.datetime.min.time())
end = current_day + datetime.timedelta(days=1, hours=6)
prev_value: ScheduleValue | None = None
for item in data.data.select(".channel-schedule .channel-schedule__event"):
title = item.select_one(".channel-schedule__title").text.strip()
time_str = item.select_one(".channel-schedule__time").text.strip()
hours, minutes = map(int, time_str.split(":"))
item_date = current_day.replace(hour=hours, minute=minutes)
if prev_value is not None and item_date.hour < prev_value.start.hour:
current_day += datetime.timedelta(days=1)
item_date += datetime.timedelta(days=1)
live = item.select_one(".channel-schedule__info .icon_live") is not None
value = ScheduleValue(start=item_date, end=end, label=title, live=live)
values.append(value)
if prev_value is not None:
prev_value.end = item_date
prev_value = value
return Schedule(
channel=Channel(id=data.context.channel_id, name=channel_name, provider=PROVIDER),
date=data.context.date,
values=values,
)
class YandexTVChannelParser(JSONParser[list[Channel]]):
def extract(self, data: Any) -> list[Channel]:
_, values = data
result = []
for _, name, content in values:
if content["label"] == "Каналы":
channel_id = content["url"].split("/")[-1]
result.append(Channel(id=channel_id, name=name, provider=PROVIDER))
return result
@@ -0,0 +1 @@
PROVIDER = "yandex"
+8 -26
View File
@@ -1,9 +1,6 @@
import datetime
import json
import logging
from aiocache import cached
from bs4 import BeautifulSoup
from gallery.sketch.source import ApiSource
from gallery.sketch.weather.api import WeatherApi
@@ -14,13 +11,15 @@ from gallery.sketch.weather.model import (
from gallery.sketch.weather.util import merge_weather_values
from gallery.util import TimeUnit
from .parser import parse_article
logger = logging.getLogger("yandexweather")
from . import PROVIDER
from .parser import YandexForecastParser, YandexLocationParser
class YandexWeatherApi(WeatherApi):
PROVIDER = "yandex"
PROVIDER = PROVIDER
forecast_parser = YandexForecastParser()
location_parser = YandexLocationParser()
def _build_source(self) -> ApiSource:
return ApiSource("https://yandex.ru")
@@ -33,29 +32,12 @@ class YandexWeatherApi(WeatherApi):
async def _get_location_forecast(self, location_id: str) -> list[WeatherResponse]:
endpoint = f"pogoda/ru/{location_id}/details"
content = await self.source.request(endpoint, mode="browser")
soup = BeautifulSoup(content, features="html.parser")
location = soup.select_one("[class^=Suggest_placeholder]").text.strip()
articles = soup.select("article[class^=AppForecastDay]")
return [parse_article(article, location) for article in articles]
return self.forecast_parser.parse(content)
async def find_locations(self, query: str) -> list[Location]:
endpoint = f"weather/api/suggest?part={query}&lang=ru&type=weather"
response = await self.source.request(endpoint)
data = json.loads(response)
return [
Location(
id=str(item["geoId"]),
name=item["title"],
provider=self.provider,
lat=item["coords"]["lat"],
lon=item["coords"]["lon"],
country="ru",
country_code="ru",
district=item["description"],
subdistrict="",
)
for item in data
]
return self.location_parser.parse(response)
async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
forecast = await self._get_location_forecast(location_id)
+162 -66
View File
@@ -1,49 +1,98 @@
import logging
from enum import Enum
import datetime
from typing import Any
import dateparser
from bs4 import Tag
from gallery.sketch.weather.model import Cloudness, Precipitation, Sky, WeatherResponse
from gallery.sketch.weather.parser import ParseError, parse_wind_direction
from gallery.sketch.weather.util import build_weather_value
from gallery.sketch.parse.core import ParseError
from gallery.sketch.parse.date import DATE_PARSER
from gallery.sketch.parse.html import (
HTMLParser,
TableParser,
TableValueParser,
TagParser,
)
from gallery.sketch.parse.json import JSONParser
from gallery.sketch.weather.model import (
Cloudness,
Location,
Precipitation,
Sky,
WeatherResponse,
WeatherValue,
WindDirection,
)
from gallery.sketch.weather.parse.wind import WIND_DIRECTION_PARSER
logger = logging.getLogger("yandexweather")
from . import PROVIDER
class DayPart(Enum):
MORNING = ("Утром", 6)
AFTERNOON = ("Днём", 12)
EVENING = ("Вечером", 18)
NIGHT = ("Ночью", 0)
class LocationNameParser(TagParser[str]):
selector = "[class^=Suggest_placeholder]"
def __init__(self, title: str, hour: int):
super().__init__()
self.title = title
self.hour = hour
def extract(self, data: Tag) -> str:
return data.text.strip()
def parse_sky(sky: str) -> Sky:
class DateParser(TagParser[datetime.date]):
selector = "[class^=AppForecastDayHeader_dayTitle]"
def extract(self, data: Tag) -> datetime.date:
return DATE_PARSER.parse(data.text.split(",")[-1].strip())
class TimeParser(TableValueParser[datetime.datetime]):
selector = "[style$='-part']"
model_key = "date"
HOUR_MAP = {
"Утром": 6,
"Днём": 12,
"Вечером": 18,
"Ночью": 0,
}
def extract(self, data: Tag) -> datetime.datetime:
hour = self.HOUR_MAP[data.text.strip()]
return datetime.datetime.min.replace(hour=hour)
class TemperatureParser(TableValueParser[list[int]]):
selector = "[style$='-temp']"
model_key = "temperature"
def extract(self, data: Tag) -> list[int]:
return [int(data.text.strip().rstrip("°"))]
class SkyParser(TableValueParser[Sky]):
selector = "[style$='-text']"
model_key = "sky"
CLOUDNESS_MAP = {
"ясно": Cloudness.CLEAR,
"облачно с прояснениями": Cloudness.CLOUDY,
"малооблачно": Cloudness.CLOUDY,
"пасмурно": Cloudness.MAINLY_CLOUDY,
}
PRECIPITATION_MAP = {
"небольшой дождь": Precipitation.SMALL_RAIN,
"дождь": Precipitation.RAIN,
"ливень": Precipitation.SHOWER,
}
def extract(self, data: Tag) -> Sky:
sky = data.text.strip()
cloudness = Cloudness.CLEAR
precipitation = Precipitation.NO
thunder = False
fog = False
if sky == "ясно":
pass
elif sky == "облачно с прояснениями":
cloudness = Cloudness.PARTLY_CLOUDY
elif sky == "малооблачно":
cloudness = Cloudness.CLOUDY
elif sky == "пасмурно":
cloudness = Cloudness.MAINLY_CLOUDY
elif sky == "небольшой дождь":
precipitation = Precipitation.SMALL_RAIN
elif sky == "дождь":
precipitation = Precipitation.RAIN
elif sky == "ливень":
precipitation = Precipitation.SHOWER
if sky in self.CLOUDNESS_MAP:
cloudness = self.CLOUDNESS_MAP[sky]
elif sky in self.PRECIPITATION_MAP:
precipitation = self.PRECIPITATION_MAP[sky]
else:
logger.warning("unknown sky value: %s", sky)
raise ParseError("Unknown sky value", sky)
return Sky(
cloudness=cloudness,
precipitation=precipitation,
@@ -52,40 +101,87 @@ def parse_sky(sky: str) -> Sky:
)
def parse_article(article: Tag, location: str) -> WeatherResponse:
date_str = article.select_one("[class^=AppForecastDayHeader_dayTitle]").text.split(",")[-1].strip()
date = dateparser.parse(date_str, languages=["ru"])
if date is None:
raise ParseError.invalid_date(date_str)
values = []
for day_part in DayPart:
value = build_weather_value(date.replace(hour=day_part.hour))
day_part_tag = article.find(string=day_part.title)
temperature_tag = day_part_tag.next_element
temperature = int(temperature_tag.text.strip().rstrip("°"))
value.temperature.append(temperature)
sky_tag = temperature_tag.next_element.next_element.next_element
sky_str = sky_tag.text.strip()
value.sky = parse_sky(sky_str)
temperature_tag = sky_tag.next_element.next_element
temperature = int(temperature_tag.text.strip().rstrip("°"))
value.temperature.append(temperature)
wind_speed_tag = temperature_tag.next_element.next_element
wind_speed = int(wind_speed_tag.text.strip())
value.wind.speed = wind_speed
wind_direction_tag = wind_speed_tag.next_element.next_element.next_element.next_element
wind_direction_str = wind_direction_tag.text.strip()
value.wind.direction = parse_wind_direction(wind_direction_str)
humidity_tag = wind_direction_tag.next_element.next_element.next_element.next_element
humidity = int(humidity_tag.text.strip().rstrip("%"))
value.humidity = humidity
pressure_tag = humidity_tag.next_element.next_element
pressure = int(pressure_tag.text.strip())
value.pressure.append(pressure)
values.append(value)
return WeatherResponse(
class WindSpeedParser(TableValueParser[int]):
selector = "[style$='-wind']:not([class*=AppForecastDayPart_wind_formatted])"
model_key = "wind.speed"
def extract(self, data: Tag) -> int:
return int(data.text.strip())
class WindDirectionParser(TableValueParser[WindDirection | None]):
selector = "[style$='-dir']"
model_key = "wind.direction"
def extract(self, data: Tag) -> WindDirection | None:
return WIND_DIRECTION_PARSER.parse(data.text.strip())
class HumidityParser(TableValueParser[int]):
selector = "[style$='-hum']"
model_key = "humidity"
def extract(self, data: Tag) -> int:
return int(data.text.strip().rstrip("%"))
class PressureParser(TableValueParser[list[int]]):
selector = "[style$='-press']"
model_key = "pressure"
def extract(self, data: Tag) -> list[int]:
return [int(data.text.strip().rstrip("%"))]
class YandexTableParser(TableParser[WeatherValue]):
selector = ""
value_parsers = [
TimeParser(),
TemperatureParser(),
SkyParser(),
WindSpeedParser(),
WindDirectionParser(),
HumidityParser(),
PressureParser(),
]
def create_model(self, data: dict) -> WeatherValue:
return WeatherValue.model_validate(data)
class YandexForecastParser(HTMLParser[list[WeatherResponse]]):
table_parser = YandexTableParser()
date_parser = DateParser()
location_parser = LocationNameParser()
def extract(self, data: Tag) -> list[WeatherResponse]:
location = self.location_parser.parse(data)
articles = data.select("article[class^=AppForecastDay]")
return [
WeatherResponse(
location=location,
date=date,
date=self.date_parser.parse(item),
period="day",
values=values,
values=self.table_parser.parse(item),
)
for item in articles
]
class YandexLocationParser(JSONParser[list[Location]]):
def extract(self, data: Any) -> list[Location]:
return [
Location(
id=str(item["geoId"]),
name=item["title"],
provider=PROVIDER,
lat=item["coords"]["lat"],
lon=item["coords"]["lon"],
country="ru",
country_code="ru",
district=item["description"],
subdistrict="",
)
for item in data
]
View File
+49
View File
@@ -0,0 +1,49 @@
from abc import ABC, abstractmethod
from typing import Generic, NamedTuple, Protocol, TypeVar
class ParseError(Exception):
pass
D = TypeVar("D")
R = TypeVar("R")
class Parser[D, R](Protocol):
def parse(self, data: D) -> R: ...
P = TypeVar("P")
class PrepareParser(ABC, Parser[D, R], Generic[D, P, R]):
@abstractmethod
def prepare(self, data: D) -> P: ...
@abstractmethod
def extract(self, data: P) -> R: ...
def parse(self, data: D) -> R:
return self.extract(self.prepare(data))
class Context(Protocol):
data: str
C = TypeVar("C", bound=Context)
class ParsedContext(NamedTuple, Generic[C, D]):
context: C
data: D
class ContextParser(PrepareParser[C, ParsedContext[C, P], R], Generic[C, P, R]):
@abstractmethod
def prepare_context(self, data: str) -> P: ...
def prepare(self, data: C) -> ParsedContext[C, P]:
return ParsedContext(context=data, data=self.prepare_context(data.data))
+16
View File
@@ -0,0 +1,16 @@
import datetime
import dateparser
from .core import ParseError, Parser
class DateParser(Parser[str, datetime.datetime]):
def parse(self, data: str) -> datetime.datetime:
date = dateparser.parse(data, languages=["ru"])
if date is None:
raise ParseError("Invalid date", data)
return date
DATE_PARSER = DateParser()
+83
View File
@@ -0,0 +1,83 @@
from typing import Generic, Iterable
from bs4 import BeautifulSoup, Tag
from .core import C, ContextParser, ParseError, Parser, PrepareParser, R
from .util import set_dict_nested_value
def _resolve_tag(source: Tag, selector: str | None = None) -> Tag:
if selector:
tag = source.select_one(selector)
if tag is None:
raise ParseError("Tag not found", selector)
return tag
return source
class _HTMLParser(Parser[str, BeautifulSoup]):
def parse(self, data: str) -> BeautifulSoup:
return BeautifulSoup(data, features="html.parser")
HTML_PARSER = _HTMLParser()
class HTMLParser(PrepareParser[str, Tag, R], Generic[R]):
selector: str | None = None
def prepare(self, data: str) -> Tag:
return _resolve_tag(HTML_PARSER.parse(data), self.selector)
class HTMLContextParser(ContextParser[C, Tag, R], Generic[C, R]):
selector: str | None = None
def prepare_context(self, data: str) -> Tag:
return _resolve_tag(HTML_PARSER.parse(data), self.selector)
class TagParser(PrepareParser[Tag, Tag, R], Generic[R]):
selector: str | None = None
def prepare(self, data: Tag) -> Tag:
return _resolve_tag(data, self.selector)
class TagIterParser(Parser[Tag, Iterable[R]], Generic[R]):
selector: str
def extract(self, data: Tag) -> R:
raise NotImplementedError
def parse(self, data: Tag) -> Iterable[R]:
tags = data.select(self.selector)
for tag in tags:
yield self.extract(tag)
class TableValueParser(TagIterParser[R], Generic[R]):
model_key: str | list[str]
def extract(self, data: Tag) -> R:
raise NotImplementedError
class TableParser(TagParser[list[R]], Generic[R]):
value_parsers: list[TableValueParser]
def create_model(self, data: dict) -> R:
raise NotImplementedError
def extract(self, data: Tag) -> list[R]:
results = []
for parser in self.value_parsers:
for index, value in enumerate(parser.parse(data)):
while index > len(results) - 1:
results.append({})
if isinstance(parser.model_key, list):
for model_key_index, model_key in enumerate(parser.model_key):
set_dict_nested_value(results[index], model_key, value[model_key_index])
else:
set_dict_nested_value(results[index], parser.model_key, value)
return [self.create_model(item) for item in results]
+22
View File
@@ -0,0 +1,22 @@
import json
from typing import Any, Generic
from .core import C, ContextParser, Parser, PrepareParser, R
class _JSONParser(Parser[str, Any]):
def parse(self, data: str) -> Any:
return json.loads(data)
JSON_PARSER = _JSONParser()
class JSONParser(PrepareParser[str, Any, R], Generic[R]):
def prepare(self, data: str) -> Any:
return JSON_PARSER.parse(data)
class JSONContextParser(ContextParser[C, Any, R], Generic[C, R]):
def prepare_context(self, data: str) -> Any:
return JSON_PARSER.parse(data)
+17
View File
@@ -0,0 +1,17 @@
from typing import Any
def set_nested_attr(target: Any, key: str, value: Any):
parts = key.split(".")
for part in parts[:-1]:
target = getattr(target, part)
setattr(target, parts[-1], value)
def set_dict_nested_value(target: dict, key: str, value: Any):
parts = key.split(".")
for part in parts[:-1]:
if part not in target:
target[part] = {}
target = target[part]
target[parts[-1]] = value
+14 -14
View File
@@ -1,6 +1,6 @@
import datetime
from enum import StrEnum, auto
from typing import Self
from typing import Literal, Self
from pydantic import BaseModel, Field
@@ -51,10 +51,10 @@ Precipitation.values = list(Precipitation)
class Sky(Model):
cloudness: Cloudness
precipitation: Precipitation
thunder: bool
fog: bool
cloudness: Cloudness = Cloudness.CLEAR
precipitation: Precipitation = Precipitation.NO
thunder: bool = False
fog: bool = False
class WindDirection(StrEnum):
@@ -69,23 +69,23 @@ class WindDirection(StrEnum):
class Wind(Model):
speed: int
gust: int
speed: int = 0
gust: int = 0
direction: WindDirection | None = None
class WeatherValue(Model):
date: datetime.datetime
sky: Sky
wind: Wind
temperature: list[int]
precipitation: float
pressure: list[int] = Field(default=[])
humidity: int = Field(default=-1)
sky: Sky = Field(default_factory=Sky)
wind: Wind = Field(default_factory=Wind)
temperature: list[int] = Field(default_factory=list)
precipitation: float = 0
pressure: list[int] = Field(default_factory=list)
humidity: int = -1
class WeatherResponse(Model):
location: str
date: datetime.date
period: str
period: Literal["day", "days"] = "day"
values: list[WeatherValue]
+49
View File
@@ -0,0 +1,49 @@
from gallery.sketch.parse.core import Parser
from ..model import WindDirection
class WindDirectionParser(Parser[str | int | float, WindDirection | None]):
WIND_DIRECTION_MAP: dict[str, WindDirection | None] = {
"с": WindDirection.N,
"св": WindDirection.NE,
"в": WindDirection.E,
"юв": WindDirection.SE,
"ю": WindDirection.S,
"юз": WindDirection.SW,
"з": WindDirection.W,
"сз": WindDirection.NW,
}
# pylint:disable=too-many-return-statements
def _parse_number(self, value: int | float) -> WindDirection | None:
if value > 337.5 or value <= 22.25:
return WindDirection.N
elif value <= 67.5:
return WindDirection.NE
elif value <= 112.5:
return WindDirection.E
elif value <= 157.5:
return WindDirection.SE
elif value <= 202.5:
return WindDirection.S
elif value <= 247.5:
return WindDirection.SW
elif value <= 292.5:
return WindDirection.W
elif value <= 337.5:
return WindDirection.NW
else:
return None
def _parse_str(self, value: str) -> WindDirection | None:
return self.WIND_DIRECTION_MAP.get(value.replace("-", "").lower())
def parse(self, data: str | int | float) -> WindDirection | None:
if isinstance(data, str):
return self._parse_str(data)
else:
return self._parse_number(data)
WIND_DIRECTION_PARSER = WindDirectionParser()
-54
View File
@@ -1,54 +0,0 @@
from typing import Self
from gallery.sketch.weather.model import WindDirection
class ParseError(Exception):
@classmethod
def invalid_date(cls, date_str: str) -> Self:
return cls("Invalid date string", date_str)
WIND_DIRECTION_MAP: dict[str, WindDirection | None] = {
"с": WindDirection.N,
"св": WindDirection.NE,
"в": WindDirection.E,
"юв": WindDirection.SE,
"ю": WindDirection.S,
"юз": WindDirection.SW,
"з": WindDirection.W,
"сз": WindDirection.NW,
}
# pylint:disable=too-many-return-statements
def _parse_wind_direction_number(value: int | float) -> WindDirection | None:
if value > 337.5 or value <= 22.25:
return WindDirection.N
elif value <= 67.5:
return WindDirection.NE
elif value <= 112.5:
return WindDirection.E
elif value <= 157.5:
return WindDirection.SE
elif value <= 202.5:
return WindDirection.S
elif value <= 247.5:
return WindDirection.SW
elif value <= 292.5:
return WindDirection.W
elif value <= 337.5:
return WindDirection.NW
else:
return None
def _parse_wind_direction_str(value: str) -> WindDirection | None:
return WIND_DIRECTION_MAP.get(value.replace("-", "").lower())
def parse_wind_direction(value: str | int | float) -> WindDirection | None:
if isinstance(value, str):
return _parse_wind_direction_str(value)
else:
return _parse_wind_direction_number(value)
+23 -24
View File
@@ -1,33 +1,12 @@
import datetime
import statistics
from typing import Counter, Literal
from typing import Counter, Literal, overload
from .model import Cloudness, Precipitation, Sky, WeatherValue, Wind
from .model import Cloudness, Precipitation, WeatherResponse, WeatherValue
MergeMode = Literal["interval", "average"]
def build_weather_value(date: datetime.datetime) -> WeatherValue:
return WeatherValue(
date=date,
sky=Sky(
cloudness=Cloudness.CLEAR,
precipitation=Precipitation.NO,
thunder=False,
fog=False,
),
wind=Wind(
speed=0,
gust=0,
direction=None,
),
temperature=[],
precipitation=0,
pressure=[],
humidity=-1,
)
def _merge_values(values: list[int], mode: MergeMode) -> list[int]:
if mode == "interval":
if (max_value := max(values)) == (min_value := min(values)):
@@ -43,7 +22,7 @@ def merge_weather_values(
values: list[WeatherValue],
mode: MergeMode = "interval",
) -> WeatherValue:
result = build_weather_value(date)
result = WeatherValue(date=date)
temperatures = []
pressures = []
humidities = []
@@ -70,10 +49,30 @@ def merge_weather_values(
result.wind.gust = round(statistics.mean(wind_gusts))
result.wind.direction = Counter(wind_directions).most_common(1)[0][0]
for item in cloudnesses:
# TODO: Instance of 'FieldInfo' has no 'cloudness' member (no-member)
# pylint:disable=no-member
if Cloudness.values.index(item) > Cloudness.values.index(result.sky.cloudness):
result.sky.cloudness = item
for item in precipitations:
# TODO: Instance of 'FieldInfo' has no 'precipitation' member (no-member)
# pylint:disable=no-member
if Precipitation.values.index(item) > Precipitation.values.index(result.sky.precipitation):
result.sky.precipitation = item
result.precipitation = precipitation
return result
def convert_hPa_to_mmHg(value: int) -> int:
return round(value / 1.333)
@overload
def fix_dates_sequence(items: list[WeatherValue]): ...
@overload
def fix_dates_sequence(items: list[WeatherResponse]): ...
def fix_dates_sequence(items):
date = None
for item in items:
if date and (item.date - date).total_seconds() < 0:
item.date = date + datetime.timedelta(days=1)
date = item.date
Generated
+135 -1
View File
@@ -388,6 +388,140 @@ files = [
]
markers = {main = "extra == \"app\" and (platform_system == \"Windows\" or sys_platform == \"win32\")", dev = "platform_system == \"Windows\" or sys_platform == \"win32\"", test = "sys_platform == \"win32\""}
[[package]]
name = "coverage"
version = "7.16.1"
description = "Code coverage measurement for Python"
optional = false
python-versions = ">=3.10"
groups = ["test"]
files = [
{file = "coverage-7.16.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f12a9e27ca7b65e40a8475d27899b2d45064d9020e6a89148939e01987b5853"},
{file = "coverage-7.16.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f590d46c30d9e4c1fda3efefe5443f4c2f6a4192c5ca2ba403653e9ebadf097"},
{file = "coverage-7.16.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3df82f0a3cef4e1bcfc799436056f0b978dda319d0bfd4460c6e479b2802d98a"},
{file = "coverage-7.16.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb462d59146656e278d1e8ed913ce374d0ba68a4e081acde1867f6d3377fc881"},
{file = "coverage-7.16.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7db888dd0a1df1a653cae7f99d4047430d2187a3626eda51bc847b0fd6b9b43"},
{file = "coverage-7.16.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:38a7e16f061504ac2b45370bf5bf97e8250d8d3f25e37385bae884978166554b"},
{file = "coverage-7.16.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2b26b55b18e1a53e1a159dd743728c4ddbbef28ba19000a91aaff5ce023197ec"},
{file = "coverage-7.16.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fdb2f528b50953e29d22033b3256c396a700193c6e45b2490222ef9c333cbbf9"},
{file = "coverage-7.16.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3284754371dc78592aa3ae4d661d30ee20e02b2b6b0de3590a181a936d0d3b38"},
{file = "coverage-7.16.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:802d1246c540e07486d4ee1adfa19a797e4b33529ffc371dc140644e8f27da0a"},
{file = "coverage-7.16.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:6fc735d6fe6d57f803e7ba021be4dde48e43e6aff94e6954350555d5332b0594"},
{file = "coverage-7.16.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed5ade1bb18f62edace1bd198c66f9d4c75a8385d5fd24e87d917ea1a5958773"},
{file = "coverage-7.16.1-cp310-cp310-win32.whl", hash = "sha256:0c309096926b119543dc16438a11ef4c80783d2f4e59ff94f7f462651a944cdc"},
{file = "coverage-7.16.1-cp310-cp310-win_amd64.whl", hash = "sha256:7562f8067ed9360e8b9739e5703403a7686dd1b36bf0f89fc538047c54cdea90"},
{file = "coverage-7.16.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:72e013665e25cf9d44779f01f340af26319756f9a76822b7c94ce6b1d93813da"},
{file = "coverage-7.16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb05c0ff98b56ba6969adf35556bc43bcb8d094df8bc9cb403acff53460c4e07"},
{file = "coverage-7.16.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0d0ececb32090e3fbb03e0d352b973a0485879b4de6c58daf47227b9988b99e5"},
{file = "coverage-7.16.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f0ba3892d81aacf36996c52f16bca04e39af31a6c5de930b7688ab617f4a6475"},
{file = "coverage-7.16.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6410b75fe07d5271eaa95fc24bd0a9177ed588d9d1c10c0cf67829adb8f0567"},
{file = "coverage-7.16.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d1039cb2de093225d597109342ce1675626bd127565e80b3044f4eca07c15b2e"},
{file = "coverage-7.16.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cf047bc39fde5425be2628666d0f435ed8817859111c3aacc84b32d858069f5d"},
{file = "coverage-7.16.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2c05913d0d5badf7ac83200f35dcf9514cce5df16a7cc89e7d1d7fff0461813b"},
{file = "coverage-7.16.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4027bf6d7bc0a16df058ce913b69f10c5687f8e1ca668f08caa659ce101744bf"},
{file = "coverage-7.16.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4f48b345f831eaf4402ab6333c2dc3e2e2b5bc7b9c1b8fe12680dee3f0538f01"},
{file = "coverage-7.16.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8643baeb590726c558b2faed6cd59b0917480f9367fcc692026f1a86d824fd08"},
{file = "coverage-7.16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d06dcc420b570bf683cdb647cc8fe62b672d9e429ef711c3cbbb7a6880ca1572"},
{file = "coverage-7.16.1-cp311-cp311-win32.whl", hash = "sha256:946f58aa59b08bcd6afcc6a7bd0ff54ed5eee844f32ad69fe6814836d15856a2"},
{file = "coverage-7.16.1-cp311-cp311-win_amd64.whl", hash = "sha256:684c7ee9b4c04358fe6ac8b517ab51ec35fcd79d08ff0f105dd8bcd96885bbb7"},
{file = "coverage-7.16.1-cp311-cp311-win_arm64.whl", hash = "sha256:1b24f79e25bcf6c73931aeca7a3dfc7595c0cb5e9364aba3fdf387a3de4b1c22"},
{file = "coverage-7.16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b7f2c26ce6ce0b1e0ca0d5fae96ea510e3a2e78b7207f06e76b7f2c87fa3d0af"},
{file = "coverage-7.16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070acb9da788dff743a4d36fc015feee12d68f0349959017542017c79f59c21c"},
{file = "coverage-7.16.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e366587b370bc9b8b51b7b7272c610c56db5d5b4795b9e4a29d28ff2f440f809"},
{file = "coverage-7.16.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e82e10b9d290f60b63459cfb245a841aec347603997206296b93881463a93dcf"},
{file = "coverage-7.16.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3d73bb1f85c4150ac208fb0755beb04b2e44897bad81414de9380f98dd74729f"},
{file = "coverage-7.16.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3397b9032553d281ad6a9253b12675b65e0cc8cd7a3b0633cf48872c9eb13360"},
{file = "coverage-7.16.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:77890395cf37026a5907d3ad32376aa51f41c0f163b7477fdbd4f94966cc1d08"},
{file = "coverage-7.16.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b0944dc3bee3091039bf970d73caaf930c906013128a421bdc132e797494d941"},
{file = "coverage-7.16.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b89d22a89d5bc05dd95b64e08295b8394aa96dc88e08f8ba210c9ebfebbe0489"},
{file = "coverage-7.16.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:550a2a1faf7559f13d5344f12d1eb886ad87955155d7dfab2a3fe5c8ec8fe776"},
{file = "coverage-7.16.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:55eb268e5b81aefac759766c9162625b06c1bedb7b77d936225bafc4f038a6f6"},
{file = "coverage-7.16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65a8fc80898c9ce59f04349fe8b4849b1f9787f14e52ead990e5f849ff4727a0"},
{file = "coverage-7.16.1-cp312-cp312-win32.whl", hash = "sha256:528a61be40977c340cf201d23b69bd6a6bab507da60e9dbda85f8b30e935d70d"},
{file = "coverage-7.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:d0f02c633630e2b74522108ee95a84ad6e1204a8016a6cca5297f335ea27147e"},
{file = "coverage-7.16.1-cp312-cp312-win_arm64.whl", hash = "sha256:2959978f9d1d20a2c0c15d0a68baaeccf615ac1aa214cf4a05a10d6f568926c8"},
{file = "coverage-7.16.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ee5465db6e9152a7d09f3215309326878c6aa3ac509195a369f9d264ff4bfbd9"},
{file = "coverage-7.16.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2b8256f8b525ba233d2e4cdcdce0d6673c66fc9bf70df1fd5e67c54a74e2d245"},
{file = "coverage-7.16.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d57cc400275b9a2892e905fc893f732b21ddb95271bf96406c88e2f6367848b5"},
{file = "coverage-7.16.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6b3fd0f3435ebb7a7183b32a6062a8b755f08242ced1f3f22761d30b56b3c2a5"},
{file = "coverage-7.16.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5eb1762e7eb5fad34ef913e8107c7788a66f19d328e598ce95bf7217f9e5c8f"},
{file = "coverage-7.16.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46cd3a73e9140410de62cceb66214bce0e08fb3922b9176fbfc1522fec151b41"},
{file = "coverage-7.16.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d1ba5142d68dd2cb775cbd0ac8601819298152047803c8efe4eec6d7d7aa7878"},
{file = "coverage-7.16.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c389c6f9d1d518e1249ddcb8a7f158135644ce2c508fa6cc17b680777dad5bf2"},
{file = "coverage-7.16.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cdc57746c7ac0ea063351b4d651c3bb4dd4fd35e64dbb8e90c10e14eb03c4080"},
{file = "coverage-7.16.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5597180ed7670cc94c04c65347418a467d3a43d5f0cf52fcac647f5425f42037"},
{file = "coverage-7.16.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:a9647a0ac46255b8fef59a433a2161f03e5483f3a35e1cbd9dfe4600baff0c6b"},
{file = "coverage-7.16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e306e98186b9cd109121f3583aeb7978797ad21d948f22944c5c08845cd554d0"},
{file = "coverage-7.16.1-cp313-cp313-win32.whl", hash = "sha256:48a78a66fcce49d7f6156524bf979c0ac633d584199717c68c6ffa949fc14e6a"},
{file = "coverage-7.16.1-cp313-cp313-win_amd64.whl", hash = "sha256:7af03247d598a353bbbbe1b925deb735276e4d845e7197c4073dc89352b236fa"},
{file = "coverage-7.16.1-cp313-cp313-win_arm64.whl", hash = "sha256:166adae25b05b04c9a84135912066d9c97482115af38df1a419a38aacc6b6f5d"},
{file = "coverage-7.16.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:cc0b37fe6f5ce5f1ccc62ad4fa9b1ad201d8e9b6027fd5e0170877beee4b2d15"},
{file = "coverage-7.16.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6618f481053b63fc6121faf8fc676bd9b7163c2a19d9e984a2e850002c28ab57"},
{file = "coverage-7.16.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fa02d561eb1d8d2f8ba43ba6e3cef4c6c402a3b632a9460fa329fcadcd5df6a3"},
{file = "coverage-7.16.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bc5354a124799f1f87b7637bbe6f18cd4bc66a1f37f6aa2b5db40f9adad531dc"},
{file = "coverage-7.16.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34bafe9f4094315248573e6223e11af0ec1b25f9cbca43bf0e9a26a189ba2751"},
{file = "coverage-7.16.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:29c4d3e32a3b5efa420a3dc627c7e570deb80ef997def52c7686a474f5edc7ab"},
{file = "coverage-7.16.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f2066c447fdd0bca39a9633a082d8ce67bf9a539a203b85059a364a405dc9fe9"},
{file = "coverage-7.16.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fd8ac10cd2458b3c6343aac082fb9bd0e3fa806cb2c4975f2280153474b88412"},
{file = "coverage-7.16.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9d8c54ec32e5c102b9241f75d88ae26538b53662868ca491736611db448d9c7a"},
{file = "coverage-7.16.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6dd8dda3402a01a1a8fe8b753a282466f615128574a5590a9108acd07b1f8540"},
{file = "coverage-7.16.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:79afa9726438912e5cddd1fe541815cea9763c92935f594835e4c432565b68a9"},
{file = "coverage-7.16.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3db3978211c3cead5437a80136ca0556bab8bc7828de15a762884b0598c41361"},
{file = "coverage-7.16.1-cp314-cp314-win32.whl", hash = "sha256:49c39c7068a494f8eb427155f5682f44feee43f9b3107fd54b1e52465379c54b"},
{file = "coverage-7.16.1-cp314-cp314-win_amd64.whl", hash = "sha256:c510dad19552d912058e4c3e3cbec3fb155dbe8d0ce0ceb7e7dbf5c5822bae0b"},
{file = "coverage-7.16.1-cp314-cp314-win_arm64.whl", hash = "sha256:b7d4d7e6dcaf33e85f1919f03346403bdcc27437c420a78835f3805bca0ab71f"},
{file = "coverage-7.16.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3d0a3681c12d3e0bcdea3d9414b04087828d6c1a482802d6f7f42c37ed530152"},
{file = "coverage-7.16.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3f3b4469d3da3ecced775d1a8c9c5d9fc80f259e30b7b89f9fed0700d6035ecb"},
{file = "coverage-7.16.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c08ae35c1be2fe1ce4b4c628df5c6fc0dc9a87f8e5fe8e20238d249678984741"},
{file = "coverage-7.16.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8ee71a38c54bb2676bbe762b8b0943a79ccb1c2fd6a52054f66e63eda392f8c1"},
{file = "coverage-7.16.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76491917771f179f9772efe218c5ccc65950dbdb35f4439298d8a8dfc6ec1f72"},
{file = "coverage-7.16.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4aa0b0a6f81fa3deb211e643f6954e78b4376b62b9c218271236cfa757664e8"},
{file = "coverage-7.16.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:756ba2d96d073c5a2a55d67fa22784763710fadbe22c41adde2d9cfa4dd78a8c"},
{file = "coverage-7.16.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:99bf9ea435cefcefd220f8687c3ddbbf78dc2de0bd11b57c3ae9fbbdf8d5561a"},
{file = "coverage-7.16.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:35cbc81f937fc402971df45c897d2df2bfb2014efcd990360032aa0a651635da"},
{file = "coverage-7.16.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:8fae08e85b334ac6ac886002b5041396a31bcf805225bbe19847627203da99e2"},
{file = "coverage-7.16.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:83362b64e215ef00b0ba33fcf13655ace6c9fdd144d5ad2ab59ac86c2daf166e"},
{file = "coverage-7.16.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:33300f2e140ccf26af3d8152e62bff71993f9310cfc63ba7a20940b0d246a0ae"},
{file = "coverage-7.16.1-cp314-cp314t-win32.whl", hash = "sha256:5539304fdbb2cc144df684d35a33b81145334d23e1c2367b5a923d25107f70b2"},
{file = "coverage-7.16.1-cp314-cp314t-win_amd64.whl", hash = "sha256:715dcb72c3280c428c3a20134b87e42c29acec9669136e899ab2de69ca86218d"},
{file = "coverage-7.16.1-cp314-cp314t-win_arm64.whl", hash = "sha256:dac8b84c03e6029d272b8249c77018db83de59ca009a9adef7c144b4a62ee5e6"},
{file = "coverage-7.16.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:a337dc2d54c74430cd2febb8ee04f7c508ba8b3b412bf0463f077a66cfc73743"},
{file = "coverage-7.16.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:3acd1d78397dead78dd1b011b5fc19cc823c190349acd549e63856dff649c80e"},
{file = "coverage-7.16.1-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:73a32694603a34ad01d7e51a481a4023410d8099e1d0757e067945695c10f0ae"},
{file = "coverage-7.16.1-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fbbe8265736659a6be2e6042b6a35be13545d14b243cc1d7ecf65f90d788a370"},
{file = "coverage-7.16.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b0359eb4c62f9993e176bc8f50450fc736a6b90dbcc05bb8584e948812699ae"},
{file = "coverage-7.16.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:da506e669a8a851b59e122b4b219ea70996a6296f44f3a9348a852526ff961de"},
{file = "coverage-7.16.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:64a2a5985d81810ed605ff0dc4ccd6555efcb5700353825532a9a0aea65826e1"},
{file = "coverage-7.16.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:66d70132b69b861805dc1ca46cdd733e54c416890e8f1371d2fd103f70b59c9c"},
{file = "coverage-7.16.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:db651a9cf325a542bc2b7b8cc8f1b2bdc6492739bae3103731b2f1c85b96cff6"},
{file = "coverage-7.16.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:4184e78a4465dcda359fb403172b8951dd220929cb0984c02fabca1742fff06f"},
{file = "coverage-7.16.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:bc53c3f3adaa939b7a063533ffe0ae1259e7073393c618043a99a6970a87e3df"},
{file = "coverage-7.16.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b44308854ef210b9b78df9cdfd4e159513382a859f5ef8464306d14a54c2a040"},
{file = "coverage-7.16.1-cp315-cp315-win32.whl", hash = "sha256:dccc142614d3419ed71857deb43f1d757829a4c7fce9994464b71e7e38309827"},
{file = "coverage-7.16.1-cp315-cp315-win_amd64.whl", hash = "sha256:961fc424e9d5229a99f8f1189942d8e7f4e1519147c3af64842f944aca03914d"},
{file = "coverage-7.16.1-cp315-cp315-win_arm64.whl", hash = "sha256:681a9488c5a234397c4f013da065aa9e53eb7af4c78f1f80c6f15e7208acb855"},
{file = "coverage-7.16.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:8bb09a2d19b04db1fa0e087a7ca4f12458f7e0e7364cfcd838441d86fb1c61f6"},
{file = "coverage-7.16.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:2270a794600b635ca9452ce4c32e2fe81a35f9caa17ffac0eba99f14f275bd4d"},
{file = "coverage-7.16.1-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a4eff405b545dfcf79cf0d9d3ff750e5c5a887aa066114175193a81d249c5ee6"},
{file = "coverage-7.16.1-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ee1d5fc9e3bd6a217906929cc97880239a91d20dae7746f538eb0eefee705ab1"},
{file = "coverage-7.16.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4ec944947de098ad5a1738413f9364689a57067ecbc328e9de37218aa1e5cc1"},
{file = "coverage-7.16.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3f73ee3956fde2d461c9e2955dd48166e4821fc8587d135e8780fb84da2a098b"},
{file = "coverage-7.16.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ec9a4ee989c0d06ad95add0dbfdbb72b00ef53f43431ca0b612384e7878e5de"},
{file = "coverage-7.16.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7e5727b2508f817f3126d6c33327dda32fe69d15514badfcd61db8bc4209ecef"},
{file = "coverage-7.16.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:19a3ea2f364012ef06678118fffdc92442a16bef4a5c8ad4f4019dd8f9ac8876"},
{file = "coverage-7.16.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:996c2b891b441ec2b39725ee3e8386e2f11b4894b92be225fdfd54a3eeada2c8"},
{file = "coverage-7.16.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:a125fac1f6b1e88488d208a86b578e1790e3c4937f2e1568d23356141d236220"},
{file = "coverage-7.16.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b10095528b866d322d33d6bf1709b7f8cbf959f12e8cb2ba22fc59c8717866b0"},
{file = "coverage-7.16.1-cp315-cp315t-win32.whl", hash = "sha256:531d9be377fdcc05593b974656872eb82e808ebeb42a72515e3aaeb8bb7166f5"},
{file = "coverage-7.16.1-cp315-cp315t-win_amd64.whl", hash = "sha256:46a88f51770df7c9bc376bd57d3f86cdc7624b8e16ac4b585a655c22b7a1b4db"},
{file = "coverage-7.16.1-cp315-cp315t-win_arm64.whl", hash = "sha256:7580432cbe1e8b762660ae5806f04f869e1c02e519836a43f8094437e561e9f0"},
{file = "coverage-7.16.1-py3-none-any.whl", hash = "sha256:3d8bd4e58b6a5c2018d808f297905393c6c61da466a48c3f0596a76a4900ebe4"},
{file = "coverage-7.16.1.tar.gz", hash = "sha256:f83981779bcf9dfa06fa0a8d4cb43e0faec1706328ce07aa3e7b665b4ac0f210"},
]
[package.extras]
toml = ["tomli ; python_full_version <= \"3.11.0a6\""]
[[package]]
name = "dateparser"
version = "1.4.1"
@@ -3148,4 +3282,4 @@ app = ["babel", "fastapi", "jinja2"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.14,<4.0"
content-hash = "b0c7f9c6c9fbb20ada45e7d27601920b0ba42716a1a70b4e317e7295de230376"
content-hash = "1494c77fcc28f6899d0fd2101647cb880e793805898472a96e2a66b125dd6c72"
+10 -10
View File
@@ -1,16 +1,16 @@
[project]
name = "gallery"
description = ""
version = "0.8.4"
version = "0.10.5"
readme = "README.md"
authors = [{ name = "shmyga", email = "shmyga.z@gmail.com" }]
requires-python = '>=3.14,<4.0'
requires-python = ">=3.14,<4.0"
dependencies = [
'aiohttp (>=3.14,<4.0)',
'beautifulsoup4 (>=4.15,<5.0)',
'dateparser (>=1.4,<2.0)',
'pydantic (>=2.13,<3.0)',
'aiocache[redis] (>=0.12,<0.13)',
"aiohttp (>=3.14,<4.0)",
"beautifulsoup4 (>=4.15,<5.0)",
"dateparser (>=1.4,<2.0)",
"pydantic (>=2.13,<3.0)",
"aiocache[redis] (>=0.12,<0.13)",
"playwright (>=1.62.0,<2.0.0)",
]
@@ -22,7 +22,7 @@ app = [
]
[dependency-groups]
test = ["pytest (>=9.1,<10.0)", "pytest-asyncio (>=1.4,<2.0)"]
test = ["pytest (>=9.1,<10.0)", "pytest-asyncio (>=1.4,<2.0)", "coverage (>=7.16.1,<8.0.0)"]
dev = ["pylint (>=4.0.6,<5.0.0)", "black (>=26.5,<27.0)", "isort (>=8.0,<9.0)"]
[project.scripts]
@@ -30,10 +30,10 @@ gallery = "gallery.main:run"
[tool.poetry]
packages = [{ include = "gallery" }]
requires-poetry = '>=2.0.0,<3.0.0'
requires-poetry = ">=2.0.0,<3.0.0"
[build-system]
requires = ['poetry-core (>=2.0.0,<3.0.0)']
requires = ["poetry-core (>=2.0.0,<3.0.0)"]
build-backend = "poetry.core.masonry.api"
[tool.black]
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "gallery",
"version": "0.8.4",
"version": "0.10.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "gallery",
"version": "0.8.4",
"version": "0.10.5",
"license": "ISC",
"dependencies": {
"@popperjs/core": "^2.11.8",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "gallery",
"version": "0.8.4",
"version": "0.10.5",
"type": "module",
"scripts": {
"build": "webpack --mode production",
+3
View File
@@ -48,4 +48,7 @@
&.bi-cast {
mask-image: url(bootstrap-icons/icons/cast.svg);
}
&.bi-filetype-json {
mask-image: url(bootstrap-icons/icons/filetype-json.svg);
}
}
+5 -5
View File
@@ -2,22 +2,22 @@ import datetime
import pytest
from gallery.painting.matchtv.api import MatchTvApi
from gallery.painting.matchtv.api import MatchTVApi
from tests.common.mock import create_mock_api
from tests.data.matchtv import MATCHTV_MOCK_SOURCE
@pytest.fixture(name="matchtv_api", scope="module")
def matchtv_api_fixture() -> MatchTvApi:
return create_mock_api(MatchTvApi, MATCHTV_MOCK_SOURCE)
def matchtv_api_fixture() -> MatchTVApi:
return create_mock_api(MatchTVApi, MATCHTV_MOCK_SOURCE)
async def test_search(matchtv_api: MatchTvApi):
async def test_search(matchtv_api: MatchTVApi):
result = await matchtv_api.find_channels("матч")
assert len(result) == 6
async def test_channel(matchtv_api: MatchTvApi):
async def test_channel(matchtv_api: MatchTVApi):
result = await matchtv_api.get_schedule("test", datetime.date.today())
assert result is not None
assert len(result.values) > 0
+5 -5
View File
@@ -2,22 +2,22 @@ import datetime
import pytest
from gallery.painting.yandextv.api import YandexTvApi
from gallery.painting.yandextv.api import YandexTVApi
from tests.common.mock import create_mock_api
from tests.data.yandextv import YANDEXTV_MOCK_SOURCE
@pytest.fixture(name="yandextv_api", scope="module")
def yandextv_api_fixture() -> YandexTvApi:
return create_mock_api(YandexTvApi, YANDEXTV_MOCK_SOURCE)
def yandextv_api_fixture() -> YandexTVApi:
return create_mock_api(YandexTVApi, YANDEXTV_MOCK_SOURCE)
async def test_search(yandextv_api: YandexTvApi):
async def test_search(yandextv_api: YandexTVApi):
result = await yandextv_api.find_channels("матч")
assert len(result) == 4
async def test_channel(yandextv_api: YandexTvApi):
async def test_channel(yandextv_api: YandexTVApi):
result = await yandextv_api.get_schedule("test", datetime.date.today())
assert result is not None
assert len(result.values) > 0