feat: add redis cache

docker-compose.yaml

@@ -0,0 +1,19 @@
+services:
+  redis:
+    container_name: gallery-redis
+    image: redis:alpine
+    stop_grace_period: 3s
+    volumes:
+      - redis_data:/data
+    command: [ "redis-server", "--bind", "0.0.0.0", "--port", "6379" ]
+  app:
+    container_name: gallery-app
+    build: .
+    # image: shmyga/gallery
+    environment:
+      - REDIS_HOST=redis
+    ports:
+      - 8000:80
+
+volumes:
+  redis_data:

gallery/__init__.py

@@ -0,0 +1,23 @@
+from os import environ
+
+from aiocache import caches
+
+REDIS_HOST = environ.get("REDIS_HOST", "0.0.0.0")
+REDIS_PORT = int(environ.get("REDIS_PORT", 6379))
+REDIS_DB = int(environ.get("REDIS_DB", 1))
+
+caches.set_config(
+    {
+        "default": {
+            "cache": "aiocache.SimpleMemoryCache",
+            "serializer": {"class": "aiocache.serializers.StringSerializer"},
+        },
+        "redis": {
+            "cache": "aiocache.RedisCache",
+            "endpoint": REDIS_HOST,
+            "port": REDIS_PORT,
+            "db": REDIS_DB,
+            "serializer": {"class": "aiocache.serializers.PickleSerializer"},
+        },
+    }
+)

gallery/main.py

@@ -6,8 +6,12 @@ import uvicorn
 from gallery.easel import build_app
 from gallery.painting.gismeteo.api import GismeteoApi
 from gallery.painting.matchtv.api import MatchTvApi
+from gallery.sketch.schedule.cached import CachedScheduleApi
+from gallery.sketch.weather.cached import CachedWeatherApi
 
-app = build_app(GismeteoApi(), MatchTvApi())
+weather_api = CachedWeatherApi(GismeteoApi())
+schedule_api = CachedScheduleApi(MatchTvApi())
+app = build_app(weather_api, schedule_api)
 
 
 def run():
@@ -18,3 +22,7 @@ def run():
         log_config=str(Path(__file__).parent / "logging.yaml"),
         reload="DEBUG" in environ,
     )
+
+
+if __name__ == "__main__":
+    run()

gallery/painting/gismeteo/api.py

@@ -2,7 +2,6 @@ import datetime
 import logging
 from typing import Any, Dict, List
 
-from aiocache import cached
 from bs4 import BeautifulSoup
 
 from gallery.sketch.source import ApiSource
@@ -17,6 +16,7 @@ logger = logging.getLogger("gismeteo")
 
 
 class GismeteoApi(WeatherApi):
+    PROVIDER = "gismeteo"
     SOURCE = ApiSource(
         "https://www.gismeteo.ru",
         cookies={
@@ -32,7 +32,6 @@ class GismeteoApi(WeatherApi):
             )
         },
     )
-    CACHE_TTL = 10 * 60
 
     def _parse_oneday(self, date: datetime.date, data: str) -> WeatherResponse:
         result: List[Dict[str, Any]] = []
@@ -76,12 +75,10 @@ class GismeteoApi(WeatherApi):
             LocationId.ZMIYEVKA,
         ]
 
-    @cached(ttl=CACHE_TTL)
     async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
         data = await self.SOURCE.request(f"weather-{location_id}/{datehelp.dump(date)}")
         return self._parse_oneday(date, data)
 
-    @cached(ttl=CACHE_TTL)
     async def get_days(self, location_id: str, days: int) -> WeatherResponse:
         data = await self.SOURCE.request(f"weather-{location_id}/{days}-days")
         return self._parse_manydays(data)

gallery/painting/matchtv/api.py

@@ -1,7 +1,6 @@
 import datetime
 import logging
 
-from aiocache import cached
 from bs4 import BeautifulSoup
 
 from gallery.sketch.schedule.api import ScheduleApi
@@ -13,8 +12,8 @@ logger = logging.getLogger("matchtv")
 
 
 class MatchTvApi(ScheduleApi):
+    PROVIDER = "matchtv"
     SOURCE = ApiSource("https://matchtv.ru")
-    CACHE_TTL = 30 * 60
 
     async def get_channels(self) -> list[str]:
         return [
@@ -27,7 +26,6 @@ class MatchTvApi(ScheduleApi):
             ChannelId.MATCH_STRANA,
         ]
 
-    @cached(ttl=CACHE_TTL)
     async def get_channel_schedule(
         self, channel_id: str, date: datetime.date
     ) -> Schedule:

gallery/sketch/api.py

@@ -0,0 +1,6 @@
+class Api:
+    PROVIDER: str
+
+    @property
+    def provider(self) -> str:
+        return self.PROVIDER

gallery/sketch/cached.py

@@ -0,0 +1,20 @@
+from typing import Generic, TypeVar
+
+from gallery.util import TimeUnit
+
+from .api import Api
+
+API = TypeVar("API", bound=Api)
+
+
+class CachedApi(Api, Generic[API]):
+    CACHE_TTL: int = TimeUnit.HOUR
+    CACHE_ALIAS: str = "redis"
+    CACHE_KEY: str
+
+    def __init__(self, api: API):
+        self._api = api
+
+    @property
+    def provider(self) -> str:
+        return self._api.provider

gallery/sketch/schedule/api.py

@@ -1,9 +1,10 @@
 import datetime
 
+from ..api import Api
 from .model import Schedule
 
 
-class ScheduleApi:
+class ScheduleApi(Api):
     async def get_channels(self) -> list[str]:
         raise NotImplementedError
 

gallery/sketch/schedule/cached.py

@@ -0,0 +1,32 @@
+import datetime
+
+from aiocache import cached
+
+from gallery.sketch.cached import CachedApi
+
+from .api import ScheduleApi
+from .model import Schedule
+
+
+class CachedScheduleApi(ScheduleApi, CachedApi[ScheduleApi]):
+    CACHE_KEY = "schedule"
+
+    @cached(
+        key_builder=lambda fun, self: f"api.{self.CACHE_KEY}.{self.provider}.channels",
+        alias=CachedApi.CACHE_ALIAS,
+        ttl=CachedApi.CACHE_TTL,
+    )
+    async def get_channels(self) -> list[str]:
+        return await self._api.get_channels()
+
+    @cached(
+        key_builder=lambda fun, self, channel_id, date: (
+            f"api.{self.CACHE_KEY}.{self.provider}.channel.{channel_id}.{date}"
+        ),
+        alias=CachedApi.CACHE_ALIAS,
+        ttl=CachedApi.CACHE_TTL,
+    )
+    async def get_channel_schedule(
+        self, channel_id: str, date: datetime.date
+    ) -> Schedule:
+        return await self._api.get_channel_schedule(channel_id, date)

gallery/sketch/weather/api.py

@@ -1,9 +1,11 @@
 import datetime
 
+from ..api import Api
 from .model import WeatherResponse
 
 
-class WeatherApi:
+class WeatherApi(Api):
+
     async def get_locations(self) -> list[str]:
         raise NotImplementedError
 

gallery/sketch/weather/cached.py

@@ -0,0 +1,40 @@
+import datetime
+
+from aiocache import cached
+
+from gallery.sketch.cached import CachedApi
+
+from .api import WeatherApi
+from .model import WeatherResponse
+
+
+class CachedWeatherApi(WeatherApi, CachedApi[WeatherApi]):
+    CACHE_KEY = "weather"
+
+    @cached(
+        key_builder=lambda fun, self: f"api.{self.CACHE_KEY}.{self.provider}.locations",
+        alias=CachedApi.CACHE_ALIAS,
+        ttl=CachedApi.CACHE_TTL,
+    )
+    async def get_locations(self) -> list[str]:
+        return await self._api.get_locations()
+
+    @cached(
+        key_builder=lambda fun, self, location_id, date: (
+            f"api.{self.CACHE_KEY}.{self.provider}.day.{location_id}.{date}"
+        ),
+        alias=CachedApi.CACHE_ALIAS,
+        ttl=CachedApi.CACHE_TTL,
+    )
+    async def get_day(self, location_id: str, date: datetime.date) -> WeatherResponse:
+        return await self._api.get_day(location_id, date)
+
+    @cached(
+        key_builder=lambda fun, self, location_id, date: (
+            f"api.{self.CACHE_KEY}.{self.provider}.day.{location_id}.{date}"
+        ),
+        alias=CachedApi.CACHE_ALIAS,
+        ttl=CachedApi.CACHE_TTL,
+    )
+    async def get_days(self, location_id: str, days: int) -> WeatherResponse:
+        return await self._api.get_days(location_id, days)

gallery/util.py

@@ -0,0 +1,5 @@
+class TimeUnit:
+    SECOND = 1
+    MINUTE = 60 * SECOND
+    HOUR = 60 * MINUTE
+    DAY = 24 * HOUR

poetry.lock

@@ -11,6 +11,9 @@ files = [
     {file = "aiocache-0.12.2.tar.gz", hash = "sha256:b41c9a145b050a5dcbae1599f847db6dd445193b1f3bd172d8e0fe0cb9e96684"},
 ]
 
+[package.dependencies]
+redis = {version = ">=4.2.0", optional = true, markers = "extra == \"redis\""}
+
 [package.extras]
 memcached = ["aiomcache (>=0.5.2)"]
 msgpack = ["msgpack (>=0.5.5)"]
@@ -1234,6 +1237,21 @@ files = [
     {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
 ]
 
+[[package]]
+name = "redis"
+version = "5.0.8"
+description = "Python client for Redis database and key-value store"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"},
+    {file = "redis-5.0.8.tar.gz", hash = "sha256:0c5b10d387568dfe0698c6fad6615750c24170e548ca2deac10c649d463e9870"},
+]
+
+[package.extras]
+hiredis = ["hiredis (>1.0.0)"]
+ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"]
+
 [[package]]
 name = "regex"
 version = "2024.5.15"
@@ -1811,4 +1829,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "61e9ac2e623a1f5f543705c252f34f8e15b58e164b0a8c14330b12d814a3c778"
+content-hash = "7295e9ec7f7492017c5bbda489026f19bbf155f0ea82402d348b0aa4c03beaca"

pyproject.toml

@@ -12,7 +12,7 @@ aiohttp = "^3.9.5"
 beautifulsoup4 = "^4.12.3"
 dateparser = "^1.2.0"
 pydantic = "^2.8.2"
-aiocache = "^0.12.2"
+aiocache = {extras = ["redis"], version = "^0.12.2"}
 
 [tool.poetry.group.app.dependencies]
 fastapi = "^0.111.1"

scripts/run

@@ -2,4 +2,5 @@
 set -e
 cd "$(dirname $(dirname "$0"))" || exit
 
-docker run --rm -p 8000:80 shmyga/gallery
+# docker run --rm -p 8000:80 shmyga/gallery
+docker compose up --build

tests/test_gismeteo_api.py

@@ -8,12 +8,12 @@ from gallery.painting.gismeteo.mock import GISMETEO_MOCK_DATA
 
 @pytest.fixture(name="gismeteo_api", scope="module")
 def gismeteo_api_fixture() -> GismeteoApi:
-    api = GismeteoApi()
-
-    async def _request(endpoint: str) -> str:
+    class MockSource:
+        async def request(self, endpoint: str):
             return GISMETEO_MOCK_DATA.get_html(endpoint.split("/")[-1])
 
-    api._request = _request
+    api = GismeteoApi()
+    api.SOURCE = MockSource()
     return api
 
 

tests/test_matchtv_api.py

@@ -8,16 +8,16 @@ from gallery.painting.matchtv.mock import MATCHTV_MOCK_DATA
 
 @pytest.fixture(name="matchtv_api", scope="module")
 def matchtv_api_fixture() -> MatchTvApi:
-    api = MatchTvApi()
-
-    async def _request(endpoint: str) -> str:
+    class MockSource:
+        async def request(self, endpoint: str):
             return MATCHTV_MOCK_DATA.get_html(endpoint.split("/")[1])
 
-    api._request = _request
+    api = MatchTvApi()
+    api.SOURCE = MockSource()
     return api
 
 
 async def test_channel(matchtv_api: MatchTvApi):
-    result = await matchtv_api.get_channel_schedule("matchtv", datetime.date.today())
+    result = await matchtv_api.get_channel_schedule("test", datetime.date.today())
     assert result is not None
     assert len(result.values) > 0