mirror of
https://github.com/shadoll/moma.git
synced 2026-08-28 11:33:25 +00:00
feat(cache): Implement unified caching subsystem with decorators, strategies, and management
- Added core caching functionality with `Cache` class supporting in-memory and file-based caching. - Introduced `CacheManager` for high-level cache operations and statistics. - Created various cache key generation strategies: `FilepathMethodStrategy`, `APIRequestStrategy`, `SimpleKeyStrategy`, and `CustomStrategy`. - Developed decorators for easy method caching: `cached`, `cached_method`, `cached_api`, and `cached_property`. - Implemented type definitions for cache entries and statistics. - Added comprehensive tests for cache operations, strategies, and decorators to ensure functionality and backward compatibility.
This commit is contained in:
Vendored
+107
@@ -0,0 +1,107 @@
|
||||
"""Unified caching subsystem for Renamer.
|
||||
|
||||
This module provides a flexible caching system with:
|
||||
- Multiple cache key generation strategies
|
||||
- Decorators for easy method caching
|
||||
- Cache management and statistics
|
||||
- Thread-safe operations
|
||||
- In-memory and file-based caching with TTL
|
||||
|
||||
Usage Examples:
|
||||
# Using decorators
|
||||
from renamer.cache import cached, cached_api
|
||||
|
||||
class MyExtractor:
|
||||
def __init__(self, file_path, cache, settings):
|
||||
self.file_path = file_path
|
||||
self.cache = cache
|
||||
self.settings = settings
|
||||
|
||||
@cached(ttl=3600)
|
||||
def extract_data(self):
|
||||
# Automatically cached using FilepathMethodStrategy
|
||||
return expensive_operation()
|
||||
|
||||
@cached_api("tmdb", ttl=21600)
|
||||
def fetch_movie_data(self, movie_id):
|
||||
# Cached API response
|
||||
return api_call(movie_id)
|
||||
|
||||
# Using cache manager
|
||||
from renamer.cache import Cache, CacheManager
|
||||
|
||||
cache = Cache()
|
||||
manager = CacheManager(cache)
|
||||
|
||||
# Get statistics
|
||||
stats = manager.get_stats()
|
||||
print(f"Total cache size: {stats['total_size_mb']} MB")
|
||||
|
||||
# Clear all cache
|
||||
manager.clear_all()
|
||||
|
||||
# Clear specific prefix
|
||||
manager.clear_by_prefix("tmdb_")
|
||||
"""
|
||||
|
||||
from .core import Cache
|
||||
from .managers import CacheManager
|
||||
from .strategies import (
|
||||
CacheKeyStrategy,
|
||||
FilepathMethodStrategy,
|
||||
APIRequestStrategy,
|
||||
SimpleKeyStrategy,
|
||||
CustomStrategy
|
||||
)
|
||||
from .decorators import (
|
||||
cached,
|
||||
cached_method,
|
||||
cached_api,
|
||||
cached_property
|
||||
)
|
||||
from .types import CacheEntry, CacheStats
|
||||
|
||||
__all__ = [
|
||||
# Core cache
|
||||
'Cache',
|
||||
'CacheManager',
|
||||
|
||||
# Strategies
|
||||
'CacheKeyStrategy',
|
||||
'FilepathMethodStrategy',
|
||||
'APIRequestStrategy',
|
||||
'SimpleKeyStrategy',
|
||||
'CustomStrategy',
|
||||
|
||||
# Decorators
|
||||
'cached',
|
||||
'cached_method',
|
||||
'cached_api',
|
||||
'cached_property',
|
||||
|
||||
# Types
|
||||
'CacheEntry',
|
||||
'CacheStats',
|
||||
|
||||
# Convenience functions
|
||||
'create_cache',
|
||||
]
|
||||
|
||||
|
||||
def create_cache(cache_dir=None):
|
||||
"""Create a Cache instance with Manager (convenience function).
|
||||
|
||||
Args:
|
||||
cache_dir: Optional cache directory path
|
||||
|
||||
Returns:
|
||||
tuple: (Cache instance, CacheManager instance)
|
||||
|
||||
Example:
|
||||
cache, manager = create_cache()
|
||||
stats = manager.get_stats()
|
||||
print(f"Cache has {stats['total_files']} files")
|
||||
"""
|
||||
cache = Cache(cache_dir)
|
||||
manager = CacheManager(cache)
|
||||
return cache, manager
|
||||
Vendored
+380
@@ -0,0 +1,380 @@
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import hashlib
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Dict
|
||||
|
||||
# Configure logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Cache:
|
||||
"""Thread-safe file-based cache with TTL support."""
|
||||
|
||||
def __init__(self, cache_dir: Optional[Path] = None):
|
||||
"""Initialize cache with optional custom directory.
|
||||
|
||||
Args:
|
||||
cache_dir: Optional cache directory path. Defaults to ~/.cache/renamer/
|
||||
"""
|
||||
# Always use the default cache dir to avoid creating cache in scan dir
|
||||
if cache_dir is None:
|
||||
cache_dir = Path.home() / ".cache" / "renamer"
|
||||
self.cache_dir = cache_dir
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._memory_cache: Dict[str, Dict[str, Any]] = {} # In-memory cache for faster access
|
||||
self._lock = threading.RLock() # Reentrant lock for thread safety
|
||||
|
||||
def _sanitize_key_component(self, component: str) -> str:
|
||||
"""Sanitize a key component to prevent filesystem escaping.
|
||||
|
||||
Args:
|
||||
component: Key component to sanitize
|
||||
|
||||
Returns:
|
||||
Sanitized component safe for filesystem use
|
||||
"""
|
||||
# Remove or replace dangerous characters
|
||||
dangerous_chars = ['/', '\\', '..', '\0']
|
||||
sanitized = component
|
||||
for char in dangerous_chars:
|
||||
sanitized = sanitized.replace(char, '_')
|
||||
return sanitized
|
||||
|
||||
def _get_cache_file(self, key: str) -> Path:
|
||||
"""Get cache file path with organized subdirectories.
|
||||
|
||||
Supports two key formats:
|
||||
1. Prefixed keys: "tmdb_id123", "poster_xyz" -> subdirectories
|
||||
2. Plain keys: "anykey" -> general subdirectory
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
|
||||
Returns:
|
||||
Path to cache file
|
||||
"""
|
||||
# Determine subdirectory and subkey based on prefix
|
||||
if key.startswith("tmdb_"):
|
||||
subdir = "tmdb"
|
||||
subkey = key[5:] # Remove "tmdb_" prefix
|
||||
elif key.startswith("poster_"):
|
||||
subdir = "posters"
|
||||
subkey = key[7:] # Remove "poster_" prefix
|
||||
elif key.startswith("extractor_"):
|
||||
subdir = "extractors"
|
||||
subkey = key[10:] # Remove "extractor_" prefix
|
||||
else:
|
||||
# Default to general subdirectory
|
||||
subdir = "general"
|
||||
subkey = key
|
||||
|
||||
# Sanitize subdirectory name
|
||||
subdir = self._sanitize_key_component(subdir)
|
||||
|
||||
# Create subdirectory
|
||||
cache_subdir = self.cache_dir / subdir
|
||||
cache_subdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Hash the subkey for filename (prevents filesystem issues with long/special names)
|
||||
key_hash = hashlib.md5(subkey.encode('utf-8')).hexdigest()
|
||||
|
||||
# Use .json extension for all cache files (simplifies logic)
|
||||
return cache_subdir / f"{key_hash}.json"
|
||||
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
"""Get cached value if not expired (thread-safe).
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
|
||||
Returns:
|
||||
Cached value or None if not found/expired
|
||||
"""
|
||||
with self._lock:
|
||||
# Check memory cache first
|
||||
if key in self._memory_cache:
|
||||
data = self._memory_cache[key]
|
||||
if time.time() <= data.get('expires', 0):
|
||||
return data.get('value')
|
||||
else:
|
||||
# Expired, remove from memory
|
||||
del self._memory_cache[key]
|
||||
logger.debug(f"Memory cache expired for key: {key}")
|
||||
|
||||
# Check file cache
|
||||
cache_file = self._get_cache_file(key)
|
||||
if not cache_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(cache_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if time.time() > data.get('expires', 0):
|
||||
# Expired, remove file
|
||||
cache_file.unlink(missing_ok=True)
|
||||
logger.debug(f"File cache expired for key: {key}, removed {cache_file}")
|
||||
return None
|
||||
|
||||
# Store in memory cache for faster future access
|
||||
self._memory_cache[key] = data
|
||||
return data.get('value')
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
# Corrupted JSON, remove file
|
||||
logger.warning(f"Corrupted cache file {cache_file}: {e}")
|
||||
cache_file.unlink(missing_ok=True)
|
||||
return None
|
||||
except IOError as e:
|
||||
# File read error
|
||||
logger.error(f"Failed to read cache file {cache_file}: {e}")
|
||||
return None
|
||||
|
||||
def set(self, key: str, value: Any, ttl_seconds: int) -> None:
|
||||
"""Set cached value with TTL (thread-safe).
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
value: Value to cache (must be JSON-serializable)
|
||||
ttl_seconds: Time-to-live in seconds
|
||||
"""
|
||||
with self._lock:
|
||||
data = {
|
||||
'value': value,
|
||||
'expires': time.time() + ttl_seconds
|
||||
}
|
||||
|
||||
# Store in memory cache
|
||||
self._memory_cache[key] = data
|
||||
|
||||
# Store in file cache
|
||||
cache_file = self._get_cache_file(key)
|
||||
try:
|
||||
with open(cache_file, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
logger.debug(f"Cached key: {key} to {cache_file} (TTL: {ttl_seconds}s)")
|
||||
except (IOError, TypeError) as e:
|
||||
logger.error(f"Failed to write cache file {cache_file}: {e}")
|
||||
|
||||
def invalidate(self, key: str) -> None:
|
||||
"""Remove cache entry (thread-safe).
|
||||
|
||||
Args:
|
||||
key: Cache key to invalidate
|
||||
"""
|
||||
with self._lock:
|
||||
# Remove from memory cache
|
||||
if key in self._memory_cache:
|
||||
del self._memory_cache[key]
|
||||
|
||||
# Remove from file cache
|
||||
cache_file = self._get_cache_file(key)
|
||||
if cache_file.exists():
|
||||
cache_file.unlink(missing_ok=True)
|
||||
logger.debug(f"Invalidated cache for key: {key}")
|
||||
|
||||
def get_image(self, key: str) -> Optional[Path]:
|
||||
"""Get cached image path if not expired (thread-safe).
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
|
||||
Returns:
|
||||
Path to cached image or None if not found/expired
|
||||
"""
|
||||
with self._lock:
|
||||
cache_file = self._get_cache_file(key)
|
||||
if not cache_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(cache_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if time.time() > data.get('expires', 0):
|
||||
# Expired, remove file and image
|
||||
image_path = data.get('image_path')
|
||||
if image_path and Path(image_path).exists():
|
||||
Path(image_path).unlink(missing_ok=True)
|
||||
cache_file.unlink(missing_ok=True)
|
||||
logger.debug(f"Image cache expired for key: {key}")
|
||||
return None
|
||||
|
||||
image_path = data.get('image_path')
|
||||
if image_path and Path(image_path).exists():
|
||||
return Path(image_path)
|
||||
else:
|
||||
logger.warning(f"Image path in cache but file missing: {image_path}")
|
||||
return None
|
||||
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
logger.warning(f"Failed to read image cache {cache_file}: {e}")
|
||||
cache_file.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
def set_image(self, key: str, image_data: bytes, ttl_seconds: int) -> Optional[Path]:
|
||||
"""Set cached image and return path (thread-safe).
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
image_data: Image binary data
|
||||
ttl_seconds: Time-to-live in seconds
|
||||
|
||||
Returns:
|
||||
Path to saved image or None if failed
|
||||
"""
|
||||
with self._lock:
|
||||
# Determine subdirectory for image storage
|
||||
if key.startswith("poster_"):
|
||||
subdir = "posters"
|
||||
subkey = key[7:]
|
||||
else:
|
||||
subdir = "images"
|
||||
subkey = key
|
||||
|
||||
# Create image directory
|
||||
image_dir = self.cache_dir / subdir
|
||||
image_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Hash for filename
|
||||
key_hash = hashlib.md5(subkey.encode('utf-8')).hexdigest()
|
||||
image_path = image_dir / f"{key_hash}.jpg"
|
||||
|
||||
try:
|
||||
# Write image data
|
||||
with open(image_path, 'wb') as f:
|
||||
f.write(image_data)
|
||||
|
||||
# Cache metadata
|
||||
data = {
|
||||
'image_path': str(image_path),
|
||||
'expires': time.time() + ttl_seconds
|
||||
}
|
||||
cache_file = self._get_cache_file(key)
|
||||
with open(cache_file, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
logger.debug(f"Cached image for key: {key} at {image_path} (TTL: {ttl_seconds}s)")
|
||||
return image_path
|
||||
|
||||
except IOError as e:
|
||||
logger.error(f"Failed to cache image for key {key}: {e}")
|
||||
return None
|
||||
|
||||
def get_object(self, key: str) -> Optional[Any]:
|
||||
"""Get pickled object from cache if not expired (thread-safe).
|
||||
|
||||
Note: This uses a separate .pkl file format for objects that can't be JSON-serialized.
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
|
||||
Returns:
|
||||
Cached object or None if not found/expired
|
||||
"""
|
||||
with self._lock:
|
||||
# Check memory cache first
|
||||
if key in self._memory_cache:
|
||||
data = self._memory_cache[key]
|
||||
if time.time() <= data.get('expires', 0):
|
||||
return data.get('value')
|
||||
else:
|
||||
del self._memory_cache[key]
|
||||
logger.debug(f"Memory cache expired for pickled object: {key}")
|
||||
|
||||
# Get cache file path but change extension to .pkl
|
||||
cache_file = self._get_cache_file(key).with_suffix('.pkl')
|
||||
if not cache_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(cache_file, 'rb') as f:
|
||||
data = pickle.load(f)
|
||||
|
||||
if time.time() > data.get('expires', 0):
|
||||
# Expired, remove file
|
||||
cache_file.unlink(missing_ok=True)
|
||||
logger.debug(f"Pickled cache expired for key: {key}")
|
||||
return None
|
||||
|
||||
# Store in memory cache
|
||||
self._memory_cache[key] = data
|
||||
return data.get('value')
|
||||
|
||||
except (pickle.PickleError, IOError) as e:
|
||||
# Corrupted or read error, remove
|
||||
logger.warning(f"Corrupted pickle cache {cache_file}: {e}")
|
||||
cache_file.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
def set_object(self, key: str, obj: Any, ttl_seconds: int) -> None:
|
||||
"""Pickle and cache object with TTL (thread-safe).
|
||||
|
||||
Note: This uses pickle format for objects that can't be JSON-serialized.
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
obj: Object to cache (must be picklable)
|
||||
ttl_seconds: Time-to-live in seconds
|
||||
"""
|
||||
with self._lock:
|
||||
data = {
|
||||
'value': obj,
|
||||
'expires': time.time() + ttl_seconds
|
||||
}
|
||||
|
||||
# Store in memory cache
|
||||
self._memory_cache[key] = data
|
||||
|
||||
# Get cache file path but change extension to .pkl
|
||||
cache_file = self._get_cache_file(key).with_suffix('.pkl')
|
||||
try:
|
||||
with open(cache_file, 'wb') as f:
|
||||
pickle.dump(data, f)
|
||||
logger.debug(f"Cached pickled object for key: {key} (TTL: {ttl_seconds}s)")
|
||||
except (IOError, pickle.PickleError) as e:
|
||||
logger.error(f"Failed to cache pickled object {cache_file}: {e}")
|
||||
|
||||
def clear_expired(self) -> int:
|
||||
"""Remove all expired cache entries.
|
||||
|
||||
Returns:
|
||||
Number of entries removed
|
||||
"""
|
||||
with self._lock:
|
||||
removed_count = 0
|
||||
current_time = time.time()
|
||||
|
||||
# Clear expired from memory cache
|
||||
expired_keys = [k for k, v in self._memory_cache.items()
|
||||
if current_time > v.get('expires', 0)]
|
||||
for key in expired_keys:
|
||||
del self._memory_cache[key]
|
||||
removed_count += 1
|
||||
|
||||
# Clear expired from file cache
|
||||
for cache_file in self.cache_dir.rglob('*'):
|
||||
if cache_file.is_file() and cache_file.suffix in ['.json', '.pkl']:
|
||||
try:
|
||||
if cache_file.suffix == '.json':
|
||||
with open(cache_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
else: # .pkl
|
||||
with open(cache_file, 'rb') as f:
|
||||
data = pickle.load(f)
|
||||
|
||||
if current_time > data.get('expires', 0):
|
||||
cache_file.unlink(missing_ok=True)
|
||||
removed_count += 1
|
||||
|
||||
except (json.JSONDecodeError, pickle.PickleError, IOError):
|
||||
# Corrupted file, remove it
|
||||
cache_file.unlink(missing_ok=True)
|
||||
removed_count += 1
|
||||
|
||||
logger.info(f"Cleared {removed_count} expired cache entries")
|
||||
return removed_count
|
||||
Vendored
+302
@@ -0,0 +1,302 @@
|
||||
"""Cache decorators for easy method caching.
|
||||
|
||||
Provides decorators that can be applied to methods for automatic caching
|
||||
with different strategies.
|
||||
"""
|
||||
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Any
|
||||
import logging
|
||||
import json
|
||||
|
||||
from .strategies import (
|
||||
CacheKeyStrategy,
|
||||
FilepathMethodStrategy,
|
||||
APIRequestStrategy,
|
||||
SimpleKeyStrategy
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cached(
|
||||
strategy: Optional[CacheKeyStrategy] = None,
|
||||
ttl: Optional[int] = None,
|
||||
key_prefix: Optional[str] = None
|
||||
):
|
||||
"""Generic cache decorator with strategy pattern.
|
||||
|
||||
This is the main caching decorator that supports different strategies
|
||||
for generating cache keys based on the use case.
|
||||
|
||||
Args:
|
||||
strategy: Cache key generation strategy (defaults to FilepathMethodStrategy)
|
||||
ttl: Time-to-live in seconds (defaults to settings value or 21600)
|
||||
key_prefix: Optional prefix for cache key
|
||||
|
||||
Returns:
|
||||
Decorated function with caching
|
||||
|
||||
Usage:
|
||||
@cached(strategy=FilepathMethodStrategy(), ttl=3600)
|
||||
def extract_title(self):
|
||||
# Expensive operation
|
||||
return title
|
||||
|
||||
@cached(strategy=APIRequestStrategy(), ttl=21600)
|
||||
def fetch_tmdb_data(self, movie_id):
|
||||
# API call
|
||||
return data
|
||||
|
||||
@cached(ttl=7200) # Uses FilepathMethodStrategy by default
|
||||
def extract_year(self):
|
||||
return year
|
||||
|
||||
Note:
|
||||
The instance must have a `cache` attribute for caching to work.
|
||||
If no cache is found, the function executes without caching.
|
||||
"""
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
# Get cache from instance
|
||||
cache = getattr(self, 'cache', None)
|
||||
if not cache:
|
||||
logger.debug(f"No cache found on {self.__class__.__name__}, executing uncached")
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
# Determine strategy
|
||||
actual_strategy = strategy or FilepathMethodStrategy()
|
||||
|
||||
# Generate cache key based on strategy type
|
||||
try:
|
||||
cache_key = _generate_cache_key(
|
||||
actual_strategy, self, func, args, kwargs, key_prefix
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to generate cache key: {e}, executing uncached")
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
# Check cache
|
||||
cached_value = cache.get(cache_key)
|
||||
if cached_value is not None:
|
||||
logger.debug(f"Cache hit for {func.__name__}: {cache_key}")
|
||||
return cached_value
|
||||
|
||||
# Execute function
|
||||
logger.debug(f"Cache miss for {func.__name__}: {cache_key}")
|
||||
result = func(self, *args, **kwargs)
|
||||
|
||||
# Determine TTL
|
||||
actual_ttl = _determine_ttl(self, ttl)
|
||||
|
||||
# Cache result (only if not None)
|
||||
if result is not None:
|
||||
cache.set(cache_key, result, actual_ttl)
|
||||
logger.debug(f"Cached {func.__name__}: {cache_key} (TTL: {actual_ttl}s)")
|
||||
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
def _generate_cache_key(
|
||||
strategy: CacheKeyStrategy,
|
||||
instance: Any,
|
||||
func: Callable,
|
||||
args: tuple,
|
||||
kwargs: dict,
|
||||
key_prefix: Optional[str]
|
||||
) -> str:
|
||||
"""Generate cache key based on strategy type.
|
||||
|
||||
Args:
|
||||
strategy: Cache key strategy
|
||||
instance: Instance the method is called on
|
||||
func: Function being cached
|
||||
args: Positional arguments
|
||||
kwargs: Keyword arguments
|
||||
key_prefix: Optional key prefix
|
||||
|
||||
Returns:
|
||||
Generated cache key
|
||||
"""
|
||||
if isinstance(strategy, FilepathMethodStrategy):
|
||||
# Extractor pattern: needs file_path attribute
|
||||
file_path = getattr(instance, 'file_path', None)
|
||||
if not file_path:
|
||||
raise ValueError(f"{instance.__class__.__name__} missing file_path attribute")
|
||||
|
||||
instance_id = str(id(instance))
|
||||
return strategy.generate_key(file_path, func.__name__, instance_id)
|
||||
|
||||
elif isinstance(strategy, APIRequestStrategy):
|
||||
# API pattern: expects service name in args or uses function name
|
||||
if args:
|
||||
service = str(args[0]) if len(args) >= 1 else func.__name__
|
||||
url = str(args[1]) if len(args) >= 2 else ""
|
||||
params = args[2] if len(args) >= 3 else kwargs
|
||||
else:
|
||||
service = func.__name__
|
||||
url = ""
|
||||
params = kwargs
|
||||
|
||||
return strategy.generate_key(service, url, params)
|
||||
|
||||
elif isinstance(strategy, SimpleKeyStrategy):
|
||||
# Simple pattern: uses prefix and first arg as identifier
|
||||
prefix = key_prefix or func.__name__
|
||||
identifier = str(args[0]) if args else str(kwargs.get('id', 'default'))
|
||||
return strategy.generate_key(prefix, identifier)
|
||||
|
||||
else:
|
||||
# Custom strategy: pass instance and all args
|
||||
return strategy.generate_key(instance, *args, **kwargs)
|
||||
|
||||
|
||||
def _determine_ttl(instance: Any, ttl: Optional[int]) -> int:
|
||||
"""Determine TTL from explicit value or instance settings.
|
||||
|
||||
Args:
|
||||
instance: Instance the method is called on
|
||||
ttl: Explicit TTL value (takes precedence)
|
||||
|
||||
Returns:
|
||||
TTL in seconds
|
||||
"""
|
||||
if ttl is not None:
|
||||
return ttl
|
||||
|
||||
# Try to get from settings
|
||||
settings = getattr(instance, 'settings', None)
|
||||
if settings:
|
||||
return settings.get('cache_ttl_extractors', 21600)
|
||||
|
||||
# Default to 6 hours
|
||||
return 21600
|
||||
|
||||
|
||||
def cached_method(ttl: Optional[int] = None):
|
||||
"""Decorator for extractor methods (legacy/convenience).
|
||||
|
||||
This is an alias for cached() with FilepathMethodStrategy.
|
||||
Provides backward compatibility with existing code.
|
||||
|
||||
Args:
|
||||
ttl: Time-to-live in seconds
|
||||
|
||||
Returns:
|
||||
Decorated function
|
||||
|
||||
Usage:
|
||||
@cached_method(ttl=3600)
|
||||
def extract_title(self):
|
||||
return title
|
||||
|
||||
Note:
|
||||
This is equivalent to:
|
||||
@cached(strategy=FilepathMethodStrategy(), ttl=3600)
|
||||
"""
|
||||
return cached(strategy=FilepathMethodStrategy(), ttl=ttl)
|
||||
|
||||
|
||||
def cached_api(service: str, ttl: Optional[int] = None):
|
||||
"""Decorator for API response caching.
|
||||
|
||||
Specialized decorator for caching API responses. Generates keys
|
||||
based on service name and request parameters.
|
||||
|
||||
Args:
|
||||
service: Service name (e.g., "tmdb", "imdb", "omdb")
|
||||
ttl: Time-to-live in seconds (defaults to cache_ttl_{service})
|
||||
|
||||
Returns:
|
||||
Decorated function
|
||||
|
||||
Usage:
|
||||
@cached_api("tmdb", ttl=21600)
|
||||
def search_movie(self, title, year=None):
|
||||
# Make API request
|
||||
response = requests.get(...)
|
||||
return response.json()
|
||||
|
||||
@cached_api("imdb")
|
||||
def get_movie_details(self, movie_id):
|
||||
return api_response
|
||||
|
||||
Note:
|
||||
The function args/kwargs are automatically included in the cache key.
|
||||
"""
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
cache = getattr(self, 'cache', None)
|
||||
if not cache:
|
||||
logger.debug(f"No cache on {self.__class__.__name__}, executing uncached")
|
||||
return func(self, *args, **kwargs)
|
||||
|
||||
# Build cache key from service + function name + args/kwargs
|
||||
args_repr = json.dumps({
|
||||
'args': [str(a) for a in args],
|
||||
'kwargs': {k: str(v) for k, v in sorted(kwargs.items())}
|
||||
}, sort_keys=True)
|
||||
|
||||
strategy = APIRequestStrategy()
|
||||
cache_key = strategy.generate_key(service, func.__name__, {'params': args_repr})
|
||||
|
||||
# Check cache
|
||||
cached_value = cache.get(cache_key)
|
||||
if cached_value is not None:
|
||||
logger.debug(f"API cache hit for {service}.{func.__name__}")
|
||||
return cached_value
|
||||
|
||||
# Execute function
|
||||
logger.debug(f"API cache miss for {service}.{func.__name__}")
|
||||
result = func(self, *args, **kwargs)
|
||||
|
||||
# Determine TTL (service-specific or default)
|
||||
actual_ttl = ttl
|
||||
if actual_ttl is None:
|
||||
settings = getattr(self, 'settings', None)
|
||||
if settings:
|
||||
# Try service-specific TTL first
|
||||
actual_ttl = settings.get(f'cache_ttl_{service}',
|
||||
settings.get('cache_ttl_api', 21600))
|
||||
else:
|
||||
actual_ttl = 21600 # Default 6 hours
|
||||
|
||||
# Cache result (only if not None)
|
||||
if result is not None:
|
||||
cache.set(cache_key, result, actual_ttl)
|
||||
logger.debug(f"API cached {service}.{func.__name__} (TTL: {actual_ttl}s)")
|
||||
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
def cached_property(ttl: Optional[int] = None):
|
||||
"""Decorator for caching property-like methods.
|
||||
|
||||
Similar to @property but with caching support.
|
||||
|
||||
Args:
|
||||
ttl: Time-to-live in seconds
|
||||
|
||||
Returns:
|
||||
Decorated function
|
||||
|
||||
Usage:
|
||||
@cached_property(ttl=3600)
|
||||
def metadata(self):
|
||||
# Expensive computation
|
||||
return complex_metadata
|
||||
|
||||
Note:
|
||||
Unlike @property, this still requires parentheses: obj.metadata()
|
||||
For true property behavior, use @property with manual caching.
|
||||
"""
|
||||
return cached(strategy=FilepathMethodStrategy(), ttl=ttl)
|
||||
Vendored
+241
@@ -0,0 +1,241 @@
|
||||
"""Cache management and operations.
|
||||
|
||||
Provides high-level cache management functionality including
|
||||
clearing, statistics, and maintenance operations.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
import logging
|
||||
import time
|
||||
import json
|
||||
import pickle
|
||||
|
||||
from .types import CacheStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CacheManager:
|
||||
"""High-level cache management and operations."""
|
||||
|
||||
def __init__(self, cache):
|
||||
"""Initialize manager with cache instance.
|
||||
|
||||
Args:
|
||||
cache: Core Cache instance
|
||||
"""
|
||||
self.cache = cache
|
||||
|
||||
def clear_all(self) -> int:
|
||||
"""Clear all cache entries (files and memory).
|
||||
|
||||
Returns:
|
||||
Number of entries removed
|
||||
"""
|
||||
count = 0
|
||||
|
||||
# Clear all cache files
|
||||
for cache_file in self.cache.cache_dir.rglob('*'):
|
||||
if cache_file.is_file():
|
||||
try:
|
||||
cache_file.unlink()
|
||||
count += 1
|
||||
except (OSError, PermissionError) as e:
|
||||
logger.warning(f"Failed to remove {cache_file}: {e}")
|
||||
|
||||
# Clear memory cache
|
||||
with self.cache._lock:
|
||||
mem_count = len(self.cache._memory_cache)
|
||||
self.cache._memory_cache.clear()
|
||||
count += mem_count
|
||||
|
||||
logger.info(f"Cleared all cache: {count} entries removed")
|
||||
return count
|
||||
|
||||
def clear_by_prefix(self, prefix: str) -> int:
|
||||
"""Clear cache entries matching prefix.
|
||||
|
||||
Args:
|
||||
prefix: Cache key prefix (e.g., "tmdb", "extractor", "poster")
|
||||
|
||||
Returns:
|
||||
Number of entries removed
|
||||
|
||||
Examples:
|
||||
clear_by_prefix("tmdb_") # Clear all TMDB cache
|
||||
clear_by_prefix("extractor_") # Clear all extractor cache
|
||||
"""
|
||||
count = 0
|
||||
|
||||
# Remove trailing underscore if present
|
||||
subdir = prefix.rstrip('_')
|
||||
cache_subdir = self.cache.cache_dir / subdir
|
||||
|
||||
# Clear files in subdirectory
|
||||
if cache_subdir.exists():
|
||||
for cache_file in cache_subdir.rglob('*'):
|
||||
if cache_file.is_file():
|
||||
try:
|
||||
cache_file.unlink()
|
||||
count += 1
|
||||
except (OSError, PermissionError) as e:
|
||||
logger.warning(f"Failed to remove {cache_file}: {e}")
|
||||
|
||||
# Clear from memory cache
|
||||
with self.cache._lock:
|
||||
keys_to_remove = [k for k in self.cache._memory_cache.keys()
|
||||
if k.startswith(prefix)]
|
||||
for key in keys_to_remove:
|
||||
del self.cache._memory_cache[key]
|
||||
count += 1
|
||||
|
||||
logger.info(f"Cleared cache with prefix '{prefix}': {count} entries removed")
|
||||
return count
|
||||
|
||||
def clear_expired(self) -> int:
|
||||
"""Clear all expired cache entries.
|
||||
|
||||
Delegates to Cache.clear_expired() for implementation.
|
||||
|
||||
Returns:
|
||||
Number of expired entries removed
|
||||
"""
|
||||
return self.cache.clear_expired()
|
||||
|
||||
def get_stats(self) -> CacheStats:
|
||||
"""Get comprehensive cache statistics.
|
||||
|
||||
Returns:
|
||||
Dictionary with cache statistics including:
|
||||
- cache_dir: Path to cache directory
|
||||
- subdirs: Per-subdirectory statistics
|
||||
- total_files: Total number of cached files
|
||||
- total_size_bytes: Total size in bytes
|
||||
- total_size_mb: Total size in megabytes
|
||||
- memory_cache_entries: Number of in-memory entries
|
||||
"""
|
||||
stats: CacheStats = {
|
||||
'cache_dir': str(self.cache.cache_dir),
|
||||
'subdirs': {},
|
||||
'total_files': 0,
|
||||
'total_size_bytes': 0,
|
||||
'total_size_mb': 0.0,
|
||||
'memory_cache_entries': len(self.cache._memory_cache)
|
||||
}
|
||||
|
||||
# Gather statistics for each subdirectory
|
||||
if self.cache.cache_dir.exists():
|
||||
for subdir in self.cache.cache_dir.iterdir():
|
||||
if subdir.is_dir():
|
||||
files = list(subdir.rglob('*'))
|
||||
file_list = [f for f in files if f.is_file()]
|
||||
file_count = len(file_list)
|
||||
size = sum(f.stat().st_size for f in file_list)
|
||||
|
||||
stats['subdirs'][subdir.name] = {
|
||||
'files': file_count,
|
||||
'size_bytes': size,
|
||||
'size_mb': round(size / (1024 * 1024), 2)
|
||||
}
|
||||
stats['total_files'] += file_count
|
||||
stats['total_size_bytes'] += size
|
||||
|
||||
stats['total_size_mb'] = round(stats['total_size_bytes'] / (1024 * 1024), 2)
|
||||
return stats
|
||||
|
||||
def clear_file_cache(self, file_path: Path) -> int:
|
||||
"""Clear all cache entries for a specific file.
|
||||
|
||||
Useful when file is renamed, moved, or modified.
|
||||
Removes all extractor cache entries associated with the file.
|
||||
|
||||
Args:
|
||||
file_path: Path to file whose cache should be cleared
|
||||
|
||||
Returns:
|
||||
Number of entries removed
|
||||
|
||||
Example:
|
||||
After renaming a file, clear its old cache:
|
||||
manager.clear_file_cache(old_path)
|
||||
"""
|
||||
count = 0
|
||||
import hashlib
|
||||
|
||||
# Generate the same hash used in FilepathMethodStrategy
|
||||
path_hash = hashlib.md5(str(file_path).encode()).hexdigest()[:12]
|
||||
|
||||
# Search in extractor subdirectory
|
||||
extractor_dir = self.cache.cache_dir / "extractors"
|
||||
if extractor_dir.exists():
|
||||
for cache_file in extractor_dir.rglob('*'):
|
||||
if cache_file.is_file() and path_hash in cache_file.name:
|
||||
try:
|
||||
cache_file.unlink()
|
||||
count += 1
|
||||
except (OSError, PermissionError) as e:
|
||||
logger.warning(f"Failed to remove {cache_file}: {e}")
|
||||
|
||||
# Clear from memory cache
|
||||
with self.cache._lock:
|
||||
keys_to_remove = [k for k in self.cache._memory_cache.keys()
|
||||
if path_hash in k]
|
||||
for key in keys_to_remove:
|
||||
del self.cache._memory_cache[key]
|
||||
count += 1
|
||||
|
||||
logger.info(f"Cleared cache for file {file_path}: {count} entries removed")
|
||||
return count
|
||||
|
||||
def get_cache_age(self, key: str) -> Optional[float]:
|
||||
"""Get the age of a cache entry in seconds.
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
|
||||
Returns:
|
||||
Age in seconds, or None if not cached
|
||||
"""
|
||||
cache_file = self.cache._get_cache_file(key)
|
||||
if not cache_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
# Check if it's a JSON or pickle file
|
||||
if cache_file.suffix == '.json':
|
||||
with open(cache_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
else: # .pkl
|
||||
with open(cache_file, 'rb') as f:
|
||||
data = pickle.load(f)
|
||||
|
||||
expires = data.get('expires', 0)
|
||||
age = time.time() - (expires - data.get('ttl', 0)) # Approximate
|
||||
return age if age >= 0 else None
|
||||
|
||||
except (json.JSONDecodeError, pickle.PickleError, IOError, KeyError):
|
||||
return None
|
||||
|
||||
def compact_cache(self) -> int:
|
||||
"""Remove empty subdirectories and organize cache.
|
||||
|
||||
Returns:
|
||||
Number of empty directories removed
|
||||
"""
|
||||
count = 0
|
||||
|
||||
if self.cache.cache_dir.exists():
|
||||
for subdir in self.cache.cache_dir.rglob('*'):
|
||||
if subdir.is_dir():
|
||||
try:
|
||||
# Try to remove if empty
|
||||
subdir.rmdir()
|
||||
count += 1
|
||||
logger.debug(f"Removed empty directory: {subdir}")
|
||||
except OSError:
|
||||
# Directory not empty or other error
|
||||
pass
|
||||
|
||||
logger.info(f"Compacted cache: removed {count} empty directories")
|
||||
return count
|
||||
Vendored
+152
@@ -0,0 +1,152 @@
|
||||
"""Cache key generation strategies.
|
||||
|
||||
Provides different strategies for generating cache keys based on use case.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Callable
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CacheKeyStrategy(ABC):
|
||||
"""Base class for cache key generation strategies."""
|
||||
|
||||
@abstractmethod
|
||||
def generate_key(self, *args, **kwargs) -> str:
|
||||
"""Generate cache key from arguments.
|
||||
|
||||
Returns:
|
||||
Cache key string
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class FilepathMethodStrategy(CacheKeyStrategy):
|
||||
"""Generate key from filepath + method name.
|
||||
|
||||
Format: extractor_{hash(filepath)}_{method_name}
|
||||
Usage: Extractor methods that operate on files
|
||||
|
||||
Examples:
|
||||
extractor_a1b2c3d4e5f6_extract_title
|
||||
extractor_a1b2c3d4e5f6_12345_extract_year (with instance_id)
|
||||
"""
|
||||
|
||||
def generate_key(
|
||||
self,
|
||||
file_path: Path,
|
||||
method_name: str,
|
||||
instance_id: str = ""
|
||||
) -> str:
|
||||
"""Generate cache key from file path and method name.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file being processed
|
||||
method_name: Name of the method being cached
|
||||
instance_id: Optional instance identifier for uniqueness
|
||||
|
||||
Returns:
|
||||
Cache key string
|
||||
"""
|
||||
# Hash the file path for consistent key length
|
||||
path_hash = hashlib.md5(str(file_path).encode()).hexdigest()[:12]
|
||||
|
||||
if instance_id:
|
||||
return f"extractor_{path_hash}_{instance_id}_{method_name}"
|
||||
return f"extractor_{path_hash}_{method_name}"
|
||||
|
||||
|
||||
class APIRequestStrategy(CacheKeyStrategy):
|
||||
"""Generate key from API request parameters.
|
||||
|
||||
Format: api_{service}_{hash(url+params)}
|
||||
Usage: API responses (TMDB, IMDB, etc.)
|
||||
|
||||
Examples:
|
||||
api_tmdb_a1b2c3d4e5f6
|
||||
api_imdb_b2c3d4e5f6a1
|
||||
"""
|
||||
|
||||
def generate_key(
|
||||
self,
|
||||
service: str,
|
||||
url: str,
|
||||
params: Optional[Dict] = None
|
||||
) -> str:
|
||||
"""Generate cache key from API request parameters.
|
||||
|
||||
Args:
|
||||
service: Service name (e.g., "tmdb", "imdb")
|
||||
url: API endpoint URL or path
|
||||
params: Optional request parameters dictionary
|
||||
|
||||
Returns:
|
||||
Cache key string
|
||||
"""
|
||||
# Sort params for consistent hashing
|
||||
params_str = json.dumps(params or {}, sort_keys=True)
|
||||
request_data = f"{url}{params_str}"
|
||||
request_hash = hashlib.md5(request_data.encode()).hexdigest()[:12]
|
||||
|
||||
return f"api_{service}_{request_hash}"
|
||||
|
||||
|
||||
class SimpleKeyStrategy(CacheKeyStrategy):
|
||||
"""Generate key from simple string prefix + identifier.
|
||||
|
||||
Format: {prefix}_{identifier}
|
||||
Usage: Posters, images, simple data
|
||||
|
||||
Examples:
|
||||
poster_movie_12345
|
||||
image_actor_67890
|
||||
"""
|
||||
|
||||
def generate_key(self, prefix: str, identifier: str) -> str:
|
||||
"""Generate cache key from prefix and identifier.
|
||||
|
||||
Args:
|
||||
prefix: Key prefix (e.g., "poster", "image")
|
||||
identifier: Unique identifier
|
||||
|
||||
Returns:
|
||||
Cache key string
|
||||
"""
|
||||
# Sanitize identifier for filesystem safety
|
||||
clean_id = identifier.replace('/', '_').replace('\\', '_').replace('..', '_')
|
||||
return f"{prefix}_{clean_id}"
|
||||
|
||||
|
||||
class CustomStrategy(CacheKeyStrategy):
|
||||
"""User-provided custom key generation.
|
||||
|
||||
Format: User-defined via callable
|
||||
Usage: Special cases requiring custom logic
|
||||
|
||||
Example:
|
||||
def my_key_generator(obj, *args):
|
||||
return f"custom_{obj.id}_{args[0]}"
|
||||
|
||||
strategy = CustomStrategy(my_key_generator)
|
||||
"""
|
||||
|
||||
def __init__(self, key_func: Callable[..., str]):
|
||||
"""Initialize with custom key generation function.
|
||||
|
||||
Args:
|
||||
key_func: Callable that returns cache key string
|
||||
"""
|
||||
self.key_func = key_func
|
||||
|
||||
def generate_key(self, *args, **kwargs) -> str:
|
||||
"""Generate cache key using custom function.
|
||||
|
||||
Returns:
|
||||
Cache key string from custom function
|
||||
"""
|
||||
return self.key_func(*args, **kwargs)
|
||||
Vendored
+33
@@ -0,0 +1,33 @@
|
||||
"""Type definitions for cache subsystem."""
|
||||
|
||||
from typing import TypedDict, Any, Dict
|
||||
|
||||
|
||||
class CacheEntry(TypedDict):
|
||||
"""Type definition for cache entry structure.
|
||||
|
||||
Attributes:
|
||||
value: The cached value (any JSON-serializable type)
|
||||
expires: Unix timestamp when entry expires
|
||||
"""
|
||||
value: Any
|
||||
expires: float
|
||||
|
||||
|
||||
class CacheStats(TypedDict):
|
||||
"""Type definition for cache statistics.
|
||||
|
||||
Attributes:
|
||||
cache_dir: Path to cache directory
|
||||
subdirs: Statistics for each subdirectory
|
||||
total_files: Total number of cache files
|
||||
total_size_bytes: Total size in bytes
|
||||
total_size_mb: Total size in megabytes
|
||||
memory_cache_entries: Number of entries in memory cache
|
||||
"""
|
||||
cache_dir: str
|
||||
subdirs: Dict[str, Dict[str, Any]]
|
||||
total_files: int
|
||||
total_size_bytes: int
|
||||
total_size_mb: float
|
||||
memory_cache_entries: int
|
||||
Reference in New Issue
Block a user