InfernalGames/gallery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(api): add multiple days api

Browse Source
This commit is contained in:
shmyga
2024-07-29 21:42:44 +03:00
parent 22fab7a15a
commit 7f0e19fb5a
16 changed files with 5411 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
gismeteo/api.py
View File

@@ -9,15 +9,24 @@ from weather.model import WeatherResponse, WeatherValue
from . import datehelp
from .location import LOCATION_BUNDLE
from .parser import LOCATION_PARSER, ONE_DAY_PARSER, ROW_PARSERS
from .parser import DAYS_PARSER, LOCATION_PARSER, ONE_DAY_PARSER, ROW_PARSERS
class GismeteoApi(WeatherApi):
BASE_URL = "https://www.gismeteo.ru"
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
COOKIE = "cf_clearance=U28mYVC0ENu88vorlL_CWmWOoevvXp0vb4xCqfqYC9s-1722273367-1.0.1.1-IDV73azTHY0V.NAnmEvok3zf5HHEkvF098pmya7IiqRRB5nk3FhbLCb0AeWm_kpTFqi1niFk2mYN_ramGTSl0A"
async def _request(self, endpoint: str) -> str:
url = f"{self.BASE_URL}/{endpoint}"
async with aiohttp.ClientSession(raise_for_status=True) as session:
async with aiohttp.ClientSession(
headers={
"User-Agent": self.USER_AGENT,
"Cookie": self.COOKIE,
},
raise_for_status=True,
) as session:
async with session.request("GET", url) as response:
return await response.text()
@@ -39,7 +48,31 @@ class GismeteoApi(WeatherApi):
values=values,
)
def _parse_manydays(self, data: str) -> WeatherResponse:
result: List[Dict[str, Any]] = []
soup = BeautifulSoup(data, features="html.parser")
location = LOCATION_PARSER.parse_location(data)
widget = DAYS_PARSER.parse_widget(soup)
for parser in ROW_PARSERS:
for index, value in enumerate(parser.parse_row(widget)):
while len(result) < index + 1:
result.append({})
result[index][parser.KEY] = value
print(">", result)
values = [WeatherValue(**item) for item in result]
return WeatherResponse(
location=location or "n/a",
date=datetime.date.today(),
period="days",
values=values,
)
async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
location = LOCATION_BUNDLE.parse(location_id)
data = await self._request(f"weather-{location}/{datehelp.dump(date)}")
return self._parse_oneday(date, data)
async def get_days(self, location_id: str, days: int) -> WeatherResponse:
location = LOCATION_BUNDLE.parse(location_id)
data = await self._request(f"weather-{location}/{days}-days")
return self._parse_manydays(data)

10
gismeteo/mock/__init__.py
View File

@@ -6,13 +6,11 @@ from gismeteo.api import WeatherResponse
class MockData:
@property
def html(self) -> str:
return (Path(__file__).parent / "data/weather.html").read_text()
def get_html(self, key: str) -> str:
return (Path(__file__).parent / f"data/{key}.html").read_text()
@property
def response(self) -> WeatherResponse:
data = json.loads((Path(__file__).parent / "data/weather.json").read_text())
def get_response(self, key: str) -> WeatherResponse:
data = json.loads((Path(__file__).parent / f"data/{key}.json").read_text())
return WeatherResponse(**data)

5246
gismeteo/mock/data/10-days.html Normal file
View File

File diff suppressed because one or more lines are too long

1
gismeteo/mock/data/day.json Normal file
View File

@@ -0,0 +1 @@
{"location":"Орел","date":"2024-07-29","period":"day","values":[{"date":"2024-07-29T00:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[20],"wind_speed":1,"wind_gust":1,"wind_direction":"SW","precipitation":0.0,"pressure":[744],"humidity":85},{"date":"2024-07-29T03:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[18],"wind_speed":1,"wind_gust":1,"wind_direction":"W","precipitation":0.6,"pressure":[742],"humidity":96},{"date":"2024-07-29T06:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":false,"fog":false},"temperature":[19],"wind_speed":1,"wind_gust":2,"wind_direction":"S","precipitation":4.9,"pressure":[741],"humidity":95},{"date":"2024-07-29T09:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":false,"fog":false},"temperature":[19],"wind_speed":3,"wind_gust":7,"wind_direction":"S","precipitation":3.8,"pressure":[740],"humidity":83},{"date":"2024-07-29T12:00:00","sky":{"cloudness":"clear","precipitation":"no","thunder":false,"fog":false},"temperature":[21],"wind_speed":4,"wind_gust":11,"wind_direction":"W","precipitation":0.0,"pressure":[740],"humidity":54},{"date":"2024-07-29T15:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[21],"wind_speed":4,"wind_gust":10,"wind_direction":"SW","precipitation":0.0,"pressure":[738],"humidity":48},{"date":"2024-07-29T18:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[19],"wind_speed":3,"wind_gust":10,"wind_direction":"SW","precipitation":0.0,"pressure":[737],"humidity":63},{"date":"2024-07-29T21:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[17],"wind_speed":3,"wind_gust":7,"wind_direction":"SW","precipitation":0.0,"pressure":[737],"humidity":77}]}

1
gismeteo/mock/data/days.json Normal file
View File

@@ -0,0 +1 @@
{"location":"Орел","date":"2024-07-29","period":"days","values":[{"date":"2024-07-29T00:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":false,"fog":false},"temperature":[21,17],"wind_speed":4,"wind_gust":11,"wind_direction":"W","precipitation":9.3,"pressure":[744,737],"humidity":96},{"date":"2024-07-30T00:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":true,"fog":false},"temperature":[19,14],"wind_speed":2,"wind_gust":7,"wind_direction":"N","precipitation":11.0,"pressure":[737,733],"humidity":100},{"date":"2024-07-31T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[22,14],"wind_speed":3,"wind_gust":10,"wind_direction":"NW","precipitation":1.8,"pressure":[741,738],"humidity":99},{"date":"2024-07-01T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[24,14],"wind_speed":3,"wind_gust":10,"wind_direction":"W","precipitation":0.1,"pressure":[741,740],"humidity":97},{"date":"2024-07-02T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[24,17],"wind_speed":2,"wind_gust":8,"wind_direction":"W","precipitation":0.2,"pressure":[740],"humidity":84},{"date":"2024-07-03T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[25,14],"wind_speed":1,"wind_gust":4,"wind_direction":"N","precipitation":0.0,"pressure":[740,739],"humidity":99},{"date":"2024-07-04T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":[25,14],"wind_speed":3,"wind_gust":6,"wind_direction":"N","precipitation":0.0,"pressure":[743,740],"humidity":92},{"date":"2024-07-05T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":true,"fog":false},"temperature":[25,15],"wind_speed":3,"wind_gust":7,"wind_direction":"NW","precipitation":2.1,"pressure":[744,743],"humidity":98},{"date":"2024-07-06T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[24,14],"wind_speed":3,"wind_gust":5,"wind_direction":"NW","precipitation":0.3,"pressure":[745,744],"humidity":98},{"date":"2024-07-07T00:00:00","sky":{"cloudness":"party_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":[26,14],"wind_speed":2,"wind_gust":5,"wind_direction":"NW","precipitation":0.2,"pressure":[747,745],"humidity":95}]}

0
gismeteo/mock/data/weather.html → gismeteo/mock/data/today.html
View File

1
gismeteo/mock/data/weather.json
View File

@@ -1 +0,0 @@
{"location":"Орел","date":"2024-07-29","period":"day","values":[{"date":"2024-07-29T00:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":20,"wind_speed":1,"wind_gust":1,"wind_direction":"SW","precipitation":0.0,"pressure":744,"humidity":85},{"date":"2024-07-29T03:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":19,"wind_speed":1,"wind_gust":1,"wind_direction":"W","precipitation":0.6,"pressure":743,"humidity":97},{"date":"2024-07-29T06:00:00","sky":{"cloudness":"mainly_cloudy","precipitation":"rain","thunder":false,"fog":false},"temperature":19,"wind_speed":1,"wind_gust":3,"wind_direction":"S","precipitation":3.3,"pressure":741,"humidity":95},{"date":"2024-07-29T09:00:00","sky":{"cloudness":"cloudy","precipitation":"small_rain","thunder":false,"fog":false},"temperature":21,"wind_speed":3,"wind_gust":10,"wind_direction":"SW","precipitation":0.3,"pressure":740,"humidity":80},{"date":"2024-07-29T12:00:00","sky":{"cloudness":"party_cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":21,"wind_speed":4,"wind_gust":12,"wind_direction":"W","precipitation":0.0,"pressure":740,"humidity":60},{"date":"2024-07-29T15:00:00","sky":{"cloudness":"cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":22,"wind_speed":4,"wind_gust":11,"wind_direction":"SW","precipitation":0.0,"pressure":739,"humidity":50},{"date":"2024-07-29T18:00:00","sky":{"cloudness":"cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":20,"wind_speed":4,"wind_gust":11,"wind_direction":"SW","precipitation":0.0,"pressure":738,"humidity":57},{"date":"2024-07-29T21:00:00","sky":{"cloudness":"cloudy","precipitation":"no","thunder":false,"fog":false},"temperature":17,"wind_speed":2,"wind_gust":8,"wind_direction":"SW","precipitation":0.0,"pressure":737,"humidity":74}]}

58
gismeteo/parser.py
View File

@@ -1,6 +1,6 @@
import datetime
import re
from typing import Dict, Iterable, List, Optional
from typing import Iterable
import dateparser
from bs4 import Tag
@@ -10,12 +10,13 @@ from weather.model import Cloudness, Precipitation, Sky, WindDirection
from .core import BaseWidgetParser, RowParser
ONE_DAY_PARSER = BaseWidgetParser(".widget.widget-oneday .widget-items")
DAYS_PARSER = BaseWidgetParser(".widget.widget-days .widget-items")
class LocationParser:
PATTERN = re.compile('{"ru":{"city":{"name":"(.*?)"')
def parse_location(self, data: str) -> Optional[str]:
def parse_location(self, data: str) -> str | None:
match = self.PATTERN.search(data)
if match:
return match.group(1)
@@ -29,30 +30,35 @@ class DateParser(RowParser[datetime.datetime]):
KEY = "date"
def parse_row(self, tag: Tag) -> Iterable[datetime.datetime]:
date_str = (
tag.select_one(".widget-row.widget-row-datetime-date > .row-item")
.find(text=True, recursive=False)
.text
datetime_date_tag = tag.select_one(
".widget-row.widget-row-datetime-date > .row-item"
)
date = dateparser.parse(date_str, languages=["ru"])
for item in tag.select(".widget-row.widget-row-datetime-time > .row-item"):
time_str = item.text
time = dateparser.parse(time_str, languages=["ru"])
time = time.replace(year=date.year, month=date.month, day=date.day)
yield time
if datetime_date_tag:
date_str = datetime_date_tag.find(text=True, recursive=False).text
date = dateparser.parse(date_str, languages=["ru"])
for item in tag.select(".widget-row.widget-row-datetime-time > .row-item"):
time_str = item.text
time = dateparser.parse(time_str, languages=["ru"])
time = time.replace(year=date.year, month=date.month, day=date.day)
yield time
else:
for item in tag.select(".widget-row.widget-row-date > .row-item"):
date_str = item.text
date = dateparser.parse(date_str, languages=["ru"])
yield date
class SkyParser(RowParser[Sky]):
KEY = "sky"
CLOUDNESS_MAP: Dict[str, Cloudness] = {
CLOUDNESS_MAP: dict[str, Cloudness] = {
"ясно": Cloudness.CLEAR,
"малооблачно": Cloudness.PARTLY_CLOUDY,
"облачно": Cloudness.CLOUDY,
"пасмурно": Cloudness.MAINLY_CLOUDY,
}
PRECIPITATION_MAP: Dict[str, Precipitation] = {
PRECIPITATION_MAP: dict[str, Precipitation] = {
"без осадков": Precipitation.NO,
"небольшой дождь": Precipitation.SMALL_RAIN,
"дождь": Precipitation.RAIN,
@@ -83,14 +89,16 @@ class SkyParser(RowParser[Sky]):
)
class TemperatureParser(RowParser[int]):
class TemperatureParser(RowParser[list[int]]):
KEY = "temperature"
def parse_row(self, tag: Tag) -> Iterable[int]:
def parse_row(self, tag: Tag) -> Iterable[list[int]]:
for item in tag.select(
".widget-row-chart[data-row=temperature-air] > .chart > .values > .value > temperature-value"
".widget-row-chart[data-row=temperature-air] > .chart > .values > .value"
):
yield int(item.attrs["value"])
yield [
int(value.attrs["value"]) for value in item.select("temperature-value")
]
class WindSpeedParser(RowParser[int]):
@@ -115,7 +123,7 @@ class WindGustParser(RowParser[int]):
class WindDirectionParser(RowParser[WindDirection]):
KEY = "wind_direction"
WIND_DIRECTION_MAP: Dict[str, WindDirection] = {
WIND_DIRECTION_MAP: dict[str, WindDirection] = {
"штиль": WindDirection.CALM,
"с": WindDirection.N,
"св": WindDirection.NO,
@@ -145,14 +153,14 @@ class WindPrecipitationParser(RowParser[float]):
yield float(item.text.replace(",", "."))
class PressureParser(RowParser[int]):
class PressureParser(RowParser[list[int]]):
KEY = "pressure"
def parse_row(self, tag: Tag) -> Iterable[int]:
def parse_row(self, tag: Tag) -> Iterable[list[int]]:
for item in tag.select(
".widget-row-chart[data-row=pressure] > .chart > .values > .value > pressure-value"
".widget-row-chart[data-row=pressure] > .chart > .values > .value"
):
yield int(item.attrs["value"])
yield [int(value.attrs["value"]) for value in item.select("pressure-value")]
class HumidityParser(RowParser[int]):
@@ -163,7 +171,7 @@ class HumidityParser(RowParser[int]):
yield int(item.text)
ROW_PARSERS: List[RowParser] = [
ROW_PARSERS: list[RowParser] = [
DateParser(),
SkyParser(),
TemperatureParser(),
@@ -175,4 +183,4 @@ ROW_PARSERS: List[RowParser] = [
HumidityParser(),
]
ROW_PARSERS_MAP: Dict[str, RowParser] = {parser.KEY: parser for parser in ROW_PARSERS}
ROW_PARSERS_MAP: dict[str, RowParser] = {parser.KEY: parser for parser in ROW_PARSERS}