diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..7b47975 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + redis: + container_name: gallery-redis + image: redis:alpine + stop_grace_period: 3s + volumes: + - redis_data:/data + command: [ "redis-server", "--bind", "0.0.0.0", "--port", "6379" ] + app: + container_name: gallery-app + build: . + # image: shmyga/gallery + environment: + - REDIS_HOST=redis + ports: + - 8000:80 + +volumes: + redis_data: diff --git a/gallery/__init__.py b/gallery/__init__.py index e69de29..31bb90e 100644 --- a/gallery/__init__.py +++ b/gallery/__init__.py @@ -0,0 +1,23 @@ +from os import environ + +from aiocache import caches + +REDIS_HOST = environ.get("REDIS_HOST", "0.0.0.0") +REDIS_PORT = int(environ.get("REDIS_PORT", 6379)) +REDIS_DB = int(environ.get("REDIS_DB", 1)) + +caches.set_config( + { + "default": { + "cache": "aiocache.SimpleMemoryCache", + "serializer": {"class": "aiocache.serializers.StringSerializer"}, + }, + "redis": { + "cache": "aiocache.RedisCache", + "endpoint": REDIS_HOST, + "port": REDIS_PORT, + "db": REDIS_DB, + "serializer": {"class": "aiocache.serializers.PickleSerializer"}, + }, + } +) diff --git a/gallery/main.py b/gallery/main.py index fad5ec1..b9b8fb7 100644 --- a/gallery/main.py +++ b/gallery/main.py @@ -6,8 +6,12 @@ import uvicorn from gallery.easel import build_app from gallery.painting.gismeteo.api import GismeteoApi from gallery.painting.matchtv.api import MatchTvApi +from gallery.sketch.schedule.cached import CachedScheduleApi +from gallery.sketch.weather.cached import CachedWeatherApi -app = build_app(GismeteoApi(), MatchTvApi()) +weather_api = CachedWeatherApi(GismeteoApi()) +schedule_api = CachedScheduleApi(MatchTvApi()) +app = build_app(weather_api, schedule_api) def run(): @@ -18,3 +22,7 @@ def run(): log_config=str(Path(__file__).parent / "logging.yaml"), reload="DEBUG" in environ, ) + + +if __name__ == "__main__": + run() diff --git a/gallery/painting/gismeteo/api.py b/gallery/painting/gismeteo/api.py index 9c0b444..46b9ea1 100644 --- a/gallery/painting/gismeteo/api.py +++ b/gallery/painting/gismeteo/api.py @@ -2,7 +2,6 @@ import datetime import logging from typing import Any, Dict, List -from aiocache import cached from bs4 import BeautifulSoup from gallery.sketch.source import ApiSource @@ -17,6 +16,7 @@ logger = logging.getLogger("gismeteo") class GismeteoApi(WeatherApi): + PROVIDER = "gismeteo" SOURCE = ApiSource( "https://www.gismeteo.ru", cookies={ @@ -32,7 +32,6 @@ class GismeteoApi(WeatherApi): ) }, ) - CACHE_TTL = 10 * 60 def _parse_oneday(self, date: datetime.date, data: str) -> WeatherResponse: result: List[Dict[str, Any]] = [] @@ -76,12 +75,10 @@ class GismeteoApi(WeatherApi): LocationId.ZMIYEVKA, ] - @cached(ttl=CACHE_TTL) async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse: data = await self.SOURCE.request(f"weather-{location_id}/{datehelp.dump(date)}") return self._parse_oneday(date, data) - @cached(ttl=CACHE_TTL) async def get_days(self, location_id: str, days: int) -> WeatherResponse: data = await self.SOURCE.request(f"weather-{location_id}/{days}-days") return self._parse_manydays(data) diff --git a/gallery/painting/matchtv/api.py b/gallery/painting/matchtv/api.py index ab3b64b..22c44d9 100644 --- a/gallery/painting/matchtv/api.py +++ b/gallery/painting/matchtv/api.py @@ -1,7 +1,6 @@ import datetime import logging -from aiocache import cached from bs4 import BeautifulSoup from gallery.sketch.schedule.api import ScheduleApi @@ -13,8 +12,8 @@ logger = logging.getLogger("matchtv") class MatchTvApi(ScheduleApi): + PROVIDER = "matchtv" SOURCE = ApiSource("https://matchtv.ru") - CACHE_TTL = 30 * 60 async def get_channels(self) -> list[str]: return [ @@ -27,7 +26,6 @@ class MatchTvApi(ScheduleApi): ChannelId.MATCH_STRANA, ] - @cached(ttl=CACHE_TTL) async def get_channel_schedule( self, channel_id: str, date: datetime.date ) -> Schedule: diff --git a/gallery/painting/matchtv/mock/data/matchtv.html b/gallery/painting/matchtv/mock/data/test.html similarity index 100% rename from gallery/painting/matchtv/mock/data/matchtv.html rename to gallery/painting/matchtv/mock/data/test.html diff --git a/gallery/sketch/__init__.py b/gallery/sketch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gallery/sketch/api.py b/gallery/sketch/api.py new file mode 100644 index 0000000..f7d26a5 --- /dev/null +++ b/gallery/sketch/api.py @@ -0,0 +1,6 @@ +class Api: + PROVIDER: str + + @property + def provider(self) -> str: + return self.PROVIDER diff --git a/gallery/sketch/cached.py b/gallery/sketch/cached.py new file mode 100644 index 0000000..624810f --- /dev/null +++ b/gallery/sketch/cached.py @@ -0,0 +1,20 @@ +from typing import Generic, TypeVar + +from gallery.util import TimeUnit + +from .api import Api + +API = TypeVar("API", bound=Api) + + +class CachedApi(Api, Generic[API]): + CACHE_TTL: int = TimeUnit.HOUR + CACHE_ALIAS: str = "redis" + CACHE_KEY: str + + def __init__(self, api: API): + self._api = api + + @property + def provider(self) -> str: + return self._api.provider diff --git a/gallery/sketch/schedule/api.py b/gallery/sketch/schedule/api.py index 2a17dd2..33ba0b0 100644 --- a/gallery/sketch/schedule/api.py +++ b/gallery/sketch/schedule/api.py @@ -1,9 +1,10 @@ import datetime +from ..api import Api from .model import Schedule -class ScheduleApi: +class ScheduleApi(Api): async def get_channels(self) -> list[str]: raise NotImplementedError diff --git a/gallery/sketch/schedule/cached.py b/gallery/sketch/schedule/cached.py new file mode 100644 index 0000000..ef6524f --- /dev/null +++ b/gallery/sketch/schedule/cached.py @@ -0,0 +1,32 @@ +import datetime + +from aiocache import cached + +from gallery.sketch.cached import CachedApi + +from .api import ScheduleApi +from .model import Schedule + + +class CachedScheduleApi(ScheduleApi, CachedApi[ScheduleApi]): + CACHE_KEY = "schedule" + + @cached( + key_builder=lambda fun, self: f"api.{self.CACHE_KEY}.{self.provider}.channels", + alias=CachedApi.CACHE_ALIAS, + ttl=CachedApi.CACHE_TTL, + ) + async def get_channels(self) -> list[str]: + return await self._api.get_channels() + + @cached( + key_builder=lambda fun, self, channel_id, date: ( + f"api.{self.CACHE_KEY}.{self.provider}.channel.{channel_id}.{date}" + ), + alias=CachedApi.CACHE_ALIAS, + ttl=CachedApi.CACHE_TTL, + ) + async def get_channel_schedule( + self, channel_id: str, date: datetime.date + ) -> Schedule: + return await self._api.get_channel_schedule(channel_id, date) diff --git a/gallery/sketch/weather/api.py b/gallery/sketch/weather/api.py index 1838b5d..dc9e1a0 100644 --- a/gallery/sketch/weather/api.py +++ b/gallery/sketch/weather/api.py @@ -1,9 +1,11 @@ import datetime +from ..api import Api from .model import WeatherResponse -class WeatherApi: +class WeatherApi(Api): + async def get_locations(self) -> list[str]: raise NotImplementedError diff --git a/gallery/sketch/weather/cached.py b/gallery/sketch/weather/cached.py new file mode 100644 index 0000000..7aab27a --- /dev/null +++ b/gallery/sketch/weather/cached.py @@ -0,0 +1,40 @@ +import datetime + +from aiocache import cached + +from gallery.sketch.cached import CachedApi + +from .api import WeatherApi +from .model import WeatherResponse + + +class CachedWeatherApi(WeatherApi, CachedApi[WeatherApi]): + CACHE_KEY = "weather" + + @cached( + key_builder=lambda fun, self: f"api.{self.CACHE_KEY}.{self.provider}.locations", + alias=CachedApi.CACHE_ALIAS, + ttl=CachedApi.CACHE_TTL, + ) + async def get_locations(self) -> list[str]: + return await self._api.get_locations() + + @cached( + key_builder=lambda fun, self, location_id, date: ( + f"api.{self.CACHE_KEY}.{self.provider}.day.{location_id}.{date}" + ), + alias=CachedApi.CACHE_ALIAS, + ttl=CachedApi.CACHE_TTL, + ) + async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse: + return await self._api.get_day(location_id, date) + + @cached( + key_builder=lambda fun, self, location_id, date: ( + f"api.{self.CACHE_KEY}.{self.provider}.day.{location_id}.{date}" + ), + alias=CachedApi.CACHE_ALIAS, + ttl=CachedApi.CACHE_TTL, + ) + async def get_days(self, location_id: str, days: int) -> WeatherResponse: + return await self._api.get_days(location_id, days) diff --git a/gallery/util.py b/gallery/util.py new file mode 100644 index 0000000..0967dfe --- /dev/null +++ b/gallery/util.py @@ -0,0 +1,5 @@ +class TimeUnit: + SECOND = 1 + MINUTE = 60 * SECOND + HOUR = 60 * MINUTE + DAY = 24 * HOUR diff --git a/poetry.lock b/poetry.lock index e79d893..e8c0418 100644 --- a/poetry.lock +++ b/poetry.lock @@ -11,6 +11,9 @@ files = [ {file = "aiocache-0.12.2.tar.gz", hash = "sha256:b41c9a145b050a5dcbae1599f847db6dd445193b1f3bd172d8e0fe0cb9e96684"}, ] +[package.dependencies] +redis = {version = ">=4.2.0", optional = true, markers = "extra == \"redis\""} + [package.extras] memcached = ["aiomcache (>=0.5.2)"] msgpack = ["msgpack (>=0.5.5)"] @@ -1234,6 +1237,21 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] +[[package]] +name = "redis" +version = "5.0.8" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.7" +files = [ + {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"}, + {file = "redis-5.0.8.tar.gz", hash = "sha256:0c5b10d387568dfe0698c6fad6615750c24170e548ca2deac10c649d463e9870"}, +] + +[package.extras] +hiredis = ["hiredis (>1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + [[package]] name = "regex" version = "2024.5.15" @@ -1811,4 +1829,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "61e9ac2e623a1f5f543705c252f34f8e15b58e164b0a8c14330b12d814a3c778" +content-hash = "7295e9ec7f7492017c5bbda489026f19bbf155f0ea82402d348b0aa4c03beaca" diff --git a/pyproject.toml b/pyproject.toml index 9702c22..9ae7bc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ aiohttp = "^3.9.5" beautifulsoup4 = "^4.12.3" dateparser = "^1.2.0" pydantic = "^2.8.2" -aiocache = "^0.12.2" +aiocache = {extras = ["redis"], version = "^0.12.2"} [tool.poetry.group.app.dependencies] fastapi = "^0.111.1" diff --git a/scripts/run b/scripts/run index 9f45915..65a7e50 100755 --- a/scripts/run +++ b/scripts/run @@ -2,4 +2,5 @@ set -e cd "$(dirname $(dirname "$0"))" || exit -docker run --rm -p 8000:80 shmyga/gallery +# docker run --rm -p 8000:80 shmyga/gallery +docker compose up --build diff --git a/tests/test_gismeteo_api.py b/tests/test_gismeteo_api.py index 3ead3ca..a588a8d 100644 --- a/tests/test_gismeteo_api.py +++ b/tests/test_gismeteo_api.py @@ -8,12 +8,12 @@ from gallery.painting.gismeteo.mock import GISMETEO_MOCK_DATA @pytest.fixture(name="gismeteo_api", scope="module") def gismeteo_api_fixture() -> GismeteoApi: + class MockSource: + async def request(self, endpoint: str): + return GISMETEO_MOCK_DATA.get_html(endpoint.split("/")[-1]) + api = GismeteoApi() - - async def _request(endpoint: str) -> str: - return GISMETEO_MOCK_DATA.get_html(endpoint.split("/")[-1]) - - api._request = _request + api.SOURCE = MockSource() return api diff --git a/tests/test_matchtv_api.py b/tests/test_matchtv_api.py index f234a82..2bd9cfb 100644 --- a/tests/test_matchtv_api.py +++ b/tests/test_matchtv_api.py @@ -8,16 +8,16 @@ from gallery.painting.matchtv.mock import MATCHTV_MOCK_DATA @pytest.fixture(name="matchtv_api", scope="module") def matchtv_api_fixture() -> MatchTvApi: + class MockSource: + async def request(self, endpoint: str): + return MATCHTV_MOCK_DATA.get_html(endpoint.split("/")[1]) + api = MatchTvApi() - - async def _request(endpoint: str) -> str: - return MATCHTV_MOCK_DATA.get_html(endpoint.split("/")[1]) - - api._request = _request + api.SOURCE = MockSource() return api async def test_channel(matchtv_api: MatchTvApi): - result = await matchtv_api.get_channel_schedule("matchtv", datetime.date.today()) + result = await matchtv_api.get_channel_schedule("test", datetime.date.today()) assert result is not None assert len(result.values) > 0