InfernalGames/gallery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(api): add multiple days api

Browse Source
This commit is contained in:
shmyga
2024-07-29 21:42:44 +03:00
parent 22fab7a15a
commit 7f0e19fb5a
16 changed files with 5411 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
gismeteo/api.py
View File

@@ -9,15 +9,24 @@ from weather.model import WeatherResponse, WeatherValue
from . import datehelp
from .location import LOCATION_BUNDLE
from .parser import LOCATION_PARSER, ONE_DAY_PARSER, ROW_PARSERS
from .parser import DAYS_PARSER, LOCATION_PARSER, ONE_DAY_PARSER, ROW_PARSERS
class GismeteoApi(WeatherApi):
BASE_URL = "https://www.gismeteo.ru"
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
COOKIE = "cf_clearance=U28mYVC0ENu88vorlL_CWmWOoevvXp0vb4xCqfqYC9s-1722273367-1.0.1.1-IDV73azTHY0V.NAnmEvok3zf5HHEkvF098pmya7IiqRRB5nk3FhbLCb0AeWm_kpTFqi1niFk2mYN_ramGTSl0A"
async def _request(self, endpoint: str) -> str:
url = f"{self.BASE_URL}/{endpoint}"
async with aiohttp.ClientSession(raise_for_status=True) as session:
async with aiohttp.ClientSession(
headers={
"User-Agent": self.USER_AGENT,
"Cookie": self.COOKIE,
},
raise_for_status=True,
) as session:
async with session.request("GET", url) as response:
return await response.text()
@@ -39,7 +48,31 @@ class GismeteoApi(WeatherApi):
values=values,
)
def _parse_manydays(self, data: str) -> WeatherResponse:
result: List[Dict[str, Any]] = []
soup = BeautifulSoup(data, features="html.parser")
location = LOCATION_PARSER.parse_location(data)
widget = DAYS_PARSER.parse_widget(soup)
for parser in ROW_PARSERS:
for index, value in enumerate(parser.parse_row(widget)):
while len(result) < index + 1:
result.append({})
result[index][parser.KEY] = value
print(">", result)
values = [WeatherValue(**item) for item in result]
return WeatherResponse(
location=location or "n/a",
date=datetime.date.today(),
period="days",
values=values,
)
async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
location = LOCATION_BUNDLE.parse(location_id)
data = await self._request(f"weather-{location}/{datehelp.dump(date)}")
return self._parse_oneday(date, data)
async def get_days(self, location_id: str, days: int) -> WeatherResponse:
location = LOCATION_BUNDLE.parse(location_id)
data = await self._request(f"weather-{location}/{days}-days")
return self._parse_manydays(data)

10
gismeteo/mock/__init__.py
View File

@@ -6,13 +6,11 @@ from gismeteo.api import WeatherResponse
class MockData:
@property
def html(self) -> str:
return (Path(__file__).parent / "data/weather.html").read_text()
def get_html(self, key: str) -> str:
return (Path(__file__).parent / f"data/{key}.html").read_text()
@property
def response(self) -> WeatherResponse:
data = json.loads((Path(__file__).parent / "data/weather.json").read_text())
def get_response(self, key: str) -> WeatherResponse:
data = json.loads((Path(__file__).parent / f"data/{key}.json").read_text())
return WeatherResponse(**data)

5246
gismeteo/mock/data/10-days.html Normal file
View File

File diff suppressed because one or more lines are too long

1
gismeteo/mock/data/day.json Normal file
View File

@@ -0,0 +1 @@
{"location":"Орел","date":"2024-07-29","period":"day","values":[{"date":"2024-07-29T00:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[20],"wind_speed":1,"wind_gust":1,"wind_direction":"SW","precipitation":0.0,"pressure":[744],"humidity":85},{"date":"2024-07-29T03:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[18],"wind_speed":1,"wind_gust":1,"wind_direction":"W","precipitation":0.6,"pressure":[742],"humidity":96},{"date":"2024-07-29T06:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":false,"fog":false},"temperature":[19],"wind_speed":1,"wind_gust":2,"wind_direction":"S","precipitation":4.9,"pressure":[741],"humidity":95},{"date":"2024-07-29T09:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":false,"fog":false},"temperature":[19],"wind_speed":3,"wind_gust":7,"wind_direction":"S","precipitation":3.8,"pressure":[740],"humidity":83},{"date":"2024-07-29T12:00:00","sky":{"cloudness":"clear","precipitation":"no","thunder":false,"fog":false},"temperature":[21],"wind_speed":4,"wind_gust":11,"wind_direction":"W","precipitation":0.0,"pressure":[740],"humidity":54},{"date":"2024-07-29T15:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[21],"wind_speed":4,"wind_gust":10,"wind_direction":"SW","precipitation":0.0,"pressure":[738],"humidity":48},{"date":"2024-07-29T18:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[19],"wind_speed":3,"wind_gust":10,"wind_direction":"SW","precipitation":0.0,"pressure":[737],"humidity":63},{"date":"2024-07-29T21:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[17],"wind_speed":3,"wind_gust":7,"wind_direction":"SW","precipitation":0.0,"pressure":[737],"humidity":77}]}

1
gismeteo/mock/data/days.json Normal file
View File

@@ -0,0 +1 @@
{"location":"Орел","date":"2024-07-29","period":"days","values":[{"date":"2024-07-29T00:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":false,"fog":false},"temperature":[21,17],"wind_speed":4,"wind_gust":11,"wind_direction":"W","precipitation":9.3,"pressure":[744,737],"humidity":96},{"date":"2024-07-30T00:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":true,"fog":false},"temperature":[19,14],"wind_speed":2,"wind_gust":7,"wind_direction":"N","precipitation":11.0,"pressure":[737,733],"humidity":100},{"date":"2024-07-31T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[22,14],"wind_speed":3,"wind_gust":10,"wind_direction":"NW","precipitation":1.8,"pressure":[741,738],"humidity":99},{"date":"2024-07-01T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[24,14],"wind_speed":3,"wind_gust":10,"wind_direction":"W","precipitation":0.1,"pressure":[741,740],"humidity":97},{"date":"2024-07-02T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[24,17],"wind_speed":2,"wind_gust":8,"wind_direction":"W","precipitation":0.2,"pressure":[740],"humidity":84},{"date":"2024-07-03T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[25,14],"wind_speed":1,"wind_gust":4,"wind_direction":"N","precipitation":0.0,"pressure":[740,739],"humidity":99},{"date":"2024-07-04T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[25,14],"wind_speed":3,"wind_gust":6,"wind_direction":"N","precipitation":0.0,"pressure":[743,740],"humidity":92},{"date":"2024-07-05T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":true,"fog":false},"temperature":[25,15],"wind_speed":3,"wind_gust":7,"wind_direction":"NW","precipitation":2.1,"pressure":[744,743],"humidity":98},{"date":"2024-07-06T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[24,14],"wind_speed":3,"wind_gust":5,"wind_direction":"NW","precipitation":0.3,"pressure":[745,744],"humidity":98},{"date":"2024-07-07T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[26,14],"wind_speed":2,"wind_gust":5,"wind_direction":"NW","precipitation":0.2,"pressure":[747,745],"humidity":95}]}

0
gismeteo/mock/data/weather.html → gismeteo/mock/data/today.html
View File

1
gismeteo/mock/data/weather.json
View File

@@ -1 +0,0 @@
{"location":"Орел","date":"2024-07-29","period":"day","values":[{"date":"2024-07-29T00:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":20,"wind_speed":1,"wind_gust":1,"wind_direction":"SW","precipitation":0.0,"pressure":744,"humidity":85},{"date":"2024-07-29T03:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":19,"wind_speed":1,"wind_gust":1,"wind_direction":"W","precipitation":0.6,"pressure":743,"humidity":97},{"date":"2024-07-29T06:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":false,"fog":false},"temperature":19,"wind_speed":1,"wind_gust":3,"wind_direction":"S","precipitation":3.3,"pressure":741,"humidity":95},{"date":"2024-07-29T09:00:00","sky":{"cloudness":"cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":21,"wind_speed":3,"wind_gust":10,"wind_direction":"SW","precipitation":0.3,"pressure":740,"humidity":80},{"date":"2024-07-29T12:00:00","sky":{"cloudness":"party_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":21,"wind_speed":4,"wind_gust":12,"wind_direction":"W","precipitation":0.0,"pressure":740,"humidity":60},{"date":"2024-07-29T15:00:00","sky":{"cloudness":"cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":22,"wind_speed":4,"wind_gust":11,"wind_direction":"SW","precipitation":0.0,"pressure":739,"humidity":50},{"date":"2024-07-29T18:00:00","sky":{"cloudness":"cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":20,"wind_speed":4,"wind_gust":11,"wind_direction":"SW","precipitation":0.0,"pressure":738,"humidity":57},{"date":"2024-07-29T21:00:00","sky":{"cloudness":"cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":17,"wind_speed":2,"wind_gust":8,"wind_direction":"SW","precipitation":0.0,"pressure":737,"humidity":74}]}

46
gismeteo/parser.py
View File

@@ -1,6 +1,6 @@
import datetime
import re
from typing import Dict, Iterable, List, Optional
from typing import Iterable
import dateparser
from bs4 import Tag
@@ -10,12 +10,13 @@ from weather.model import Cloudness, Precipitation, Sky, WindDirection
from .core import BaseWidgetParser, RowParser
ONE_DAY_PARSER = BaseWidgetParser(".widget.widget-oneday .widget-items")
DAYS_PARSER = BaseWidgetParser(".widget.widget-days .widget-items")
class LocationParser:
PATTERN = re.compile('{"ru":{"city":{"name":"(.*?)"')
def parse_location(self, data: str) -> Optional[str]:
def parse_location(self, data: str) -> str | None:
match = self.PATTERN.search(data)
if match:
return match.group(1)
@@ -29,30 +30,35 @@ class DateParser(RowParser[datetime.datetime]):
KEY = "date"
def parse_row(self, tag: Tag) -> Iterable[datetime.datetime]:
date_str = (
tag.select_one(".widget-row.widget-row-datetime-date > .row-item")
.find(text=True, recursive=False)
.text
datetime_date_tag = tag.select_one(
".widget-row.widget-row-datetime-date > .row-item"
)
if datetime_date_tag:
date_str = datetime_date_tag.find(text=True, recursive=False).text
date = dateparser.parse(date_str, languages=["ru"])
for item in tag.select(".widget-row.widget-row-datetime-time > .row-item"):
time_str = item.text
time = dateparser.parse(time_str, languages=["ru"])
time = time.replace(year=date.year, month=date.month, day=date.day)
yield time
else:
for item in tag.select(".widget-row.widget-row-date > .row-item"):
date_str = item.text
date = dateparser.parse(date_str, languages=["ru"])
yield date
class SkyParser(RowParser[Sky]):
KEY = "sky"
CLOUDNESS_MAP: Dict[str, Cloudness] = {
CLOUDNESS_MAP: dict[str, Cloudness] = {
"ясно": Cloudness.CLEAR,
"малооблачно": Cloudness.PARTLY_CLOUDY,
"облачно": Cloudness.CLOUDY,
"пасмурно": Cloudness.MAINLY_CLOUDY,
}
PRECIPITATION_MAP: Dict[str, Precipitation] = {
PRECIPITATION_MAP: dict[str, Precipitation] = {
"без осадков": Precipitation.NO,
"небольшой дождь": Precipitation.SMALL_RAIN,
"дождь": Precipitation.RAIN,
@@ -83,14 +89,16 @@ class SkyParser(RowParser[Sky]):
)
class TemperatureParser(RowParser[int]):
class TemperatureParser(RowParser[list[int]]):
KEY = "temperature"
def parse_row(self, tag: Tag) -> Iterable[int]:
def parse_row(self, tag: Tag) -> Iterable[list[int]]:
for item in tag.select(
".widget-row-chart[data-row=temperature-air] > .chart > .values > .value > temperature-value"
".widget-row-chart[data-row=temperature-air] > .chart > .values > .value"
):
yield int(item.attrs["value"])
yield [
int(value.attrs["value"]) for value in item.select("temperature-value")
]
class WindSpeedParser(RowParser[int]):
@@ -115,7 +123,7 @@ class WindGustParser(RowParser[int]):
class WindDirectionParser(RowParser[WindDirection]):
KEY = "wind_direction"
WIND_DIRECTION_MAP: Dict[str, WindDirection] = {
WIND_DIRECTION_MAP: dict[str, WindDirection] = {
"штиль": WindDirection.CALM,
"с": WindDirection.N,
"св": WindDirection.NO,
@@ -145,14 +153,14 @@ class WindPrecipitationParser(RowParser[float]):
yield float(item.text.replace(",", "."))
class PressureParser(RowParser[int]):
class PressureParser(RowParser[list[int]]):
KEY = "pressure"
def parse_row(self, tag: Tag) -> Iterable[int]:
def parse_row(self, tag: Tag) -> Iterable[list[int]]:
for item in tag.select(
".widget-row-chart[data-row=pressure] > .chart > .values > .value > pressure-value"
".widget-row-chart[data-row=pressure] > .chart > .values > .value"
):
yield int(item.attrs["value"])
yield [int(value.attrs["value"]) for value in item.select("pressure-value")]
class HumidityParser(RowParser[int]):
@@ -163,7 +171,7 @@ class HumidityParser(RowParser[int]):
yield int(item.text)
ROW_PARSERS: List[RowParser] = [
ROW_PARSERS: list[RowParser] = [
DateParser(),
SkyParser(),
TemperatureParser(),
@@ -175,4 +183,4 @@ ROW_PARSERS: List[RowParser] = [
HumidityParser(),
]
ROW_PARSERS_MAP: Dict[str, RowParser] = {parser.KEY: parser for parser in ROW_PARSERS}
ROW_PARSERS_MAP: dict[str, RowParser] = {parser.KEY: parser for parser in ROW_PARSERS}

13
tests/test_gismeteo_api.py
View File

@@ -11,14 +11,17 @@ def gismeteo_api_fixture() -> GismeteoApi:
api = GismeteoApi()
async def _request(endpoint: str) -> str:
return MOCK_DATA.html
return MOCK_DATA.get_html(endpoint.split("/")[-1])
api._request = _request
return api
async def test_api(gismeteo_api: GismeteoApi):
result = await gismeteo_api.get_day(
"zmiyevka", datetime.date.today() + datetime.timedelta(days=1)
)
async def test_day(gismeteo_api: GismeteoApi):
result = await gismeteo_api.get_day("zmiyevka", datetime.date.today())
assert len(result.values) == 8
async def test_days(gismeteo_api: GismeteoApi):
result = await gismeteo_api.get_days("zmiyevka", 10)
assert len(result.values) == 10

3
weather/api.py
View File

@@ -6,3 +6,6 @@ from .model import WeatherResponse
class WeatherApi:
async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
raise NotImplementedError
async def get_days(self, location_id: str, days: int) -> WeatherResponse:
raise NotImplementedError

1
weather/app/__init__.py
View File

@@ -9,7 +9,6 @@ from .route import api, doc, view
def build_app(weather_api: WeatherApi) -> FastAPI:
locale.setlocale(locale.LC_TIME, "ru_RU.UTF-8")
app = FastAPI(
title="Weather",
docs_url=None,

15
weather/app/route/api.py
View File

@@ -2,12 +2,21 @@ import datetime
from fastapi import FastAPI, Request
from weather.api import WeatherApi
from weather.model import WeatherResponse
def mount(app: FastAPI):
@app.get("/api/weather/{location}/{date}")
async def get_weather_api(
@app.get("/api/weather/{location}/day/{date}")
async def get_api_weather_day(
request: Request, location: str, date: datetime.date
) -> WeatherResponse:
return await request.app.state.weather_api.get_day(location, date)
weather_api: WeatherApi = request.app.state.weather_api
return await weather_api.get_day(location, date)
@app.get("/api/weather/{location}/days/{days}")
async def get_api_weather_days(
request: Request, location: str, days: int
) -> WeatherResponse:
weather_api: WeatherApi = request.app.state.weather_api
return await weather_api.get_days(location, days)

29
weather/app/route/view/__init__.py
View File

@@ -9,6 +9,7 @@ from fastapi.templating import Jinja2Templates
from gismeteo import datehelp
from gismeteo.location import LOCATION_BUNDLE
from gismeteo.mock import MOCK_DATA
from weather.api import WeatherApi
from weather.model import WeatherResponse
from .filters import cloudness_icon, wind_direction_icon
@@ -47,20 +48,26 @@ def mount(app: FastAPI):
@app.get("/weather/{location}", response_class=RedirectResponse)
async def get_weather_default(location: str):
return RedirectResponse(f"{location}/{datetime.date.today()}")
return RedirectResponse(f"{location}/day/{datetime.date.today()}")
@app.get("/weather/{location}/mock", response_class=HTMLResponse)
async def get_weather_mock(request: Request):
response = MOCK_DATA.response
@app.get("/weather/{location}/day/mock", response_class=HTMLResponse)
async def get_weather_day_mock(request: Request):
response = MOCK_DATA.get_response("day")
return build_weather_response(request, response)
# @app.get("/weather/{location}/{day}", response_class=HTMLResponse)
async def get_weather_day(request: Request, location: str, day: datehelp.Day):
date = datehelp.parse(day)
response = await request.app.state.weather_api.get_day(location, date)
@app.get("/weather/{location}/days/mock", response_class=HTMLResponse)
async def get_weather_days_mock(request: Request):
response = MOCK_DATA.get_response("days")
return build_weather_response(request, response)
@app.get("/weather/{location}/{date}", response_class=HTMLResponse)
async def get_weather(request: Request, location: str, date: datetime.date):
response = await request.app.state.weather_api.get_day(location, date)
@app.get("/weather/{location}/day/{date}", response_class=HTMLResponse)
async def get_weather_day(request: Request, location: str, date: datetime.date):
weather_api: WeatherApi = request.app.state.weather_api
response = await weather_api.get_day(location, date)
return build_weather_response(request, response)
@app.get("/weather/{location}/days/{days}", response_class=HTMLResponse)
async def get_weather_days(request: Request, location: str, days: int):
weather_api: WeatherApi = request.app.state.weather_api
response = await weather_api.get_days(location, days)
return build_weather_response(request, response)

22
weather/app/route/view/static/style.css
View File

@@ -50,6 +50,11 @@ td {
background: rgba(0, 128, 255, 0.2);
}
.date .value a {
all: unset;
cursor: pointer;
}
.cloudness {
vertical-align: top;
}
@@ -62,11 +67,19 @@ td {
font-size: 2rem;
}
.temperature.positive .value {
.temperature {
padding: 0;
}
.temperature .value {
padding: 0.1rem 0.4rem;
}
.temperature .value.positive {
color: orangered;
}
.temperature.negative .value {
.temperature .value.negative {
color: blue;
}
@@ -82,7 +95,12 @@ td {
color: blue;
}
.pressure {
padding: 0;
}
.pressure .value {
padding: 0.1rem 0.4rem;
color: blueviolet;
}

34
weather/app/route/view/templates/weather.html
View File

@@ -17,21 +17,36 @@
<body>
<h3>
{% if response.period == 'day' %}
<a class="button {{'disabled' if response.date == datetime.date.today() else ''}}"
href="{{response.date - datetime.timedelta(days=1)}}">⬅️</a>
<a class="button"
href="../days/10">⬆️</a>
<span>{{response.location}} | {{response.date.strftime('%a, %d %B %Y')}}</span>
<a class="button"
href="{{response.date + datetime.timedelta(days=1)}}">➡️</a>
{% endif %}
{% if response.period == 'days' %}
<span>{{response.location}} | {{response.date.strftime('%a, %d %B %Y')}}</span>
{% endif %}
</h3>
<table>
<tbody>
<!-- date -->
<tr>
{% for value in response.values %}
{% if response.period == 'day' %}
<td
class="date {{'now' if value.date < datetime.datetime.now() and value.date + datetime.timedelta(hours=3) > datetime.datetime.now() else ''}}">
<span class="value">{{value.date.strftime('%H:%M')}}</span>
</td>
{% endif %}
{% if response.period == 'days' %}
<td
class="date {{'now' if value.date.date() == datetime.date.today() else ''}}">
<span class="value"><a href="../day/{{value.date.date()}}">{{value.date.strftime('%a %d')}}</a></span>
</td>
{% endif %}
{% endfor %}
</tr>
<!-- cloudness -->
@@ -59,9 +74,13 @@
</tr>
<tr>
{% for value in response.values %}
<td class="temperature {{'positive' if value.temperature > 0 else 'negative'}}"
style="background-color: rgba(255, 128, 128, {{(value.temperature - 10) * 0.015}});">
<span class="value">{{value.temperature}}</span>
<td class="temperature">
{% for temperature in value.temperature %}
<div class="value {{'positive' if temperature > 0 else 'negative'}}"
style="background-color: rgba(255, 128, 128, {{(temperature - 10) * 0.015}});">
{{temperature}}
</div>
{% endfor %}
</td>
{% endfor %}
</tr>
@@ -123,9 +142,12 @@
</tr>
<tr>
{% for value in response.values %}
<td class="pressure"
style="background-color: rgba(128, 0, 255, {{(value.pressure - 720) * 0.008}});">
<span class="value">{{value.pressure}}</span>
<td class="pressure">
{% for pressure in value.pressure %}
<div class="value"
style="background-color: rgba(128, 0, 255, {{(pressure - 720) * 0.008}});">
{{pressure}}</div>
{% endfor %}
</td>
{% endfor %}
</tr>

4
weather/model.py
View File

@@ -45,12 +45,12 @@ class WindDirection(str, Enum):
class WeatherValue(Model):
date: datetime.datetime
sky: Sky
temperature: int
temperature: list[int]
wind_speed: int
wind_gust: int
wind_direction: WindDirection
precipitation: float
pressure: int
pressure: list[int]
humidity: int