github-personal/moma
1
0
Fork 0
mirror of https://github.com/shadoll/moma.git synced 2026-08-28 11:33:25 +00:00
Code Issues Packages Projects Releases 4 Wiki Activity

Add comprehensive tests for formatter classes, services, and utilities

Browse Source 
- Introduced tests for various formatter classes including TextFormatter, DurationFormatter, SizeFormatter, DateFormatter, and more to ensure correct formatting behavior.
- Added tests for service classes such as FileTreeService, MetadataService, and RenameService, covering directory validation, metadata extraction, and file renaming functionalities.
- Implemented utility tests for LanguageCodeExtractor, PatternExtractor, and FrameClassMatcher to validate their extraction and matching capabilities.
- Updated test cases to use datasets for better maintainability and clarity.
- Enhanced error handling tests to ensure robustness against missing or invalid data.
This commit is contained in:
sha
2025-12-31 14:04:33 +00:00
parent c5fbd367fc
commit 262c0a7b7d
45 changed files with 3346 additions and 1057 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renamer/extractors/filename_extractor.py
+91 -130
View File
@@ -2,7 +2,12 @@ import re
import logging
from pathlib import Path
from collections import Counter
from ..constants import SOURCE_DICT, FRAME_CLASSES, MOVIE_DB_DICT, SPECIAL_EDITIONS
from ..constants import (
SOURCE_DICT, FRAME_CLASSES, MOVIE_DB_DICT, SPECIAL_EDITIONS, SKIP_WORDS,
NON_STANDARD_QUALITY_INDICATORS,
is_valid_year,
CYRILLIC_TO_ENGLISH
)
from ..decorators import cached_method
import langcodes
@@ -22,12 +27,7 @@ class FilenameExtractor:
def _normalize_cyrillic(self, text: str) -> str:
"""Normalize Cyrillic characters to English equivalents for parsing"""
replacements = {
'р': 'p',
'і': 'i',
# Add more as needed
}
for cyr, eng in replacements.items():
for cyr, eng in CYRILLIC_TO_ENGLISH.items():
text = text.replace(cyr, eng)
return text
@@ -60,10 +60,9 @@ class FilenameExtractor:
# Last resort: any 4-digit number
any_match = re.search(r'\b(\d{4})\b', self.file_name)
if any_match:
year = any_match.group(1)
# Basic sanity check
current_year = 2025
if 1900 <= int(year) <= current_year + 10:
year = int(any_match.group(1))
# Basic sanity check using constants
if is_valid_year(year):
year_pos = any_match.start() # Cut before the year for plain years
# Find source position
@@ -141,12 +140,11 @@ class FilenameExtractor:
# Last resort: any 4-digit number (but this is less reliable)
any_match = re.search(r'\b(\d{4})\b', self.file_name)
if any_match:
year = any_match.group(1)
# Basic sanity check: years should be between 1900 and current year + a few years
current_year = 2025 # Update this as needed
if 1900 <= int(year) <= current_year + 10:
year = int(any_match.group(1))
# Basic sanity check using constants
if is_valid_year(year):
year_pos = any_match.start()
return year
return str(year)
return None
@@ -201,9 +199,8 @@ class FilenameExtractor:
# Fallback to height-based if not in constants
return self._get_frame_class_from_height(height)
# If no specific resolution found, check for quality indicators
unclassified_indicators = ['SD', 'LQ', 'HD', 'QHD']
for indicator in unclassified_indicators:
# If no specific resolution found, check for non-standard quality indicators
for indicator in NON_STANDARD_QUALITY_INDICATORS:
if re.search(r'\b' + re.escape(indicator) + r'\b', self.file_name, re.IGNORECASE):
return None
@@ -312,8 +309,8 @@ class FilenameExtractor:
count = int(lang_match.group(1)) if lang_match.group(1) else 1
lang_code = lang_match.group(2)
# Skip if it's a quality/resolution indicator
if lang_code in ['sd', 'hd', 'lq', 'qhd', 'uhd', 'p', 'i', 'hdr', 'sdr']:
# Skip if it's a quality/resolution indicator or other skip word
if lang_code in SKIP_WORDS:
continue
# Skip if the language code is not at the end or if there are extra letters after
@@ -335,60 +332,38 @@ class FilenameExtractor:
# Second, look for standalone language codes outside brackets
# Remove bracketed content first
text_without_brackets = re.sub(r'\[([^\]]+)\]', '', self.file_name)
# Known language codes (2-3 letter ISO 639-1 or 639-3)
known_language_codes = {
'eng', 'ukr', 'rus', 'fra', 'deu', 'spa', 'ita', 'por', 'nor', 'swe', 'dan', 'fin', 'pol', 'cze', 'hun', 'tur', 'ara', 'heb', 'hin', 'jpn', 'kor', 'chi', 'tha', 'vie', 'und',
'dut', 'nld', 'bel', 'bul', 'hrv', 'ces', 'dan', 'nld', 'est', 'fin', 'fra', 'deu', 'ell', 'heb', 'hin', 'hrv', 'hun', 'ind', 'ita', 'jpn', 'kor', 'lav', 'lit', 'mkd', 'nor', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'srp', 'swe', 'tha', 'tur', 'ukr', 'vie', 'und', 'zho',
'arb', 'ben', 'hin', 'mar', 'tam', 'tel', 'urd', 'guj', 'kan', 'mal', 'ori', 'pan', 'asm', 'mai', 'bho', 'nep', 'sin', 'san', 'tib', 'mon', 'kaz', 'uzb', 'kir', 'tuk', 'aze', 'kat', 'hye', 'geo', 'ell', 'sqi', 'bos', 'hrv', 'srp', 'slv', 'mkd', 'bul', 'alb', 'ron', 'mol', 'hun',
'fin', 'swe', 'nor', 'dan', 'isl', 'fao', 'est', 'lav', 'lit', 'bel', 'ukr', 'rus', 'pol', 'cze', 'slk', 'slv', 'hrv', 'bos', 'srp', 'mkd', 'bul', 'ell', 'alb', 'ron', 'hun', 'tur', 'aze', 'geo', 'arm', 'kat', 'hye', 'per', 'kur', 'pus', 'urd', 'ara', 'heb', 'san', 'hin', 'ben', 'tam', 'tel', 'mar', 'guj', 'kan', 'mal', 'ori', 'pan', 'asm', 'mai', 'bho', 'awa', 'mag', 'nep', 'sin', 'div', 'tib', 'mon', 'kaz', 'kir', 'tuk', 'uzb', 'jpn', 'kor', 'chi', 'tha', 'vie', 'und', 'lao', 'khm', 'mya', 'vie', 'und', 'ind', 'msa', 'zho', 'yue', 'wuu', 'nan', 'hak', 'gan', 'hsn',
'spa', 'por', 'fra', 'ita', 'deu', 'nld', 'dut', 'swe', 'nor', 'dan', 'fin', 'est', 'lav', 'lit', 'pol', 'cze', 'slk', 'slv', 'hrv', 'bos', 'srp', 'mkd', 'bul', 'ell', 'alb', 'ron', 'hun', 'tur', 'aze', 'geo', 'arm', 'kat', 'hye', 'per', 'kur', 'pus', 'urd', 'ara', 'heb', 'san', 'hin', 'ben', 'tam', 'tel', 'mar', 'guj', 'kan', 'mal', 'ori', 'pan', 'asm', 'mai', 'bho', 'awa', 'mag', 'nep', 'sin', 'div', 'tib', 'mon', 'kaz', 'kir', 'tuk', 'uzb', 'jpn', 'kor', 'chi', 'tha', 'vie', 'und', 'lao', 'khm', 'mya', 'vie', 'und', 'ind', 'msa', 'zho', 'yue', 'wuu', 'nan', 'hak', 'gan', 'hsn'
}
allowed_title_case = {'ukr', 'nor', 'eng', 'rus', 'fra', 'deu', 'spa', 'ita', 'por', 'swe', 'dan', 'fin', 'pol', 'cze', 'hun', 'tur', 'ara', 'heb', 'hin', 'jpn', 'kor', 'chi', 'tha', 'vie', 'und'}
# Look for language codes in various formats:
# - Uppercase: ENG, UKR, NOR
# - Title case: Ukr, Nor, Eng
# - Lowercase: ukr, nor, eng
# - In dot-separated parts: .ukr. .eng.
# Split on dots, spaces, and underscores
parts = re.split(r'[.\s_]+', text_without_brackets)
for part in parts:
part = part.strip()
if not part or len(part) < 2:
continue
part_lower = part.lower()
# Check if this part is a 2-3 letter language code
if re.match(r'^[a-zA-Z]{2,3}$', part):
# Skip title case 2-letter words to avoid false positives like "In" -> "ind"
if part.istitle() and len(part) == 2:
continue
if part.istitle() and part_lower not in allowed_title_case:
continue
skip_words = [
'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had', 'her', 'was', 'one', 'our', 'out', 'day', 'get', 'has', 'him', 'his', 'how', 'its', 'may', 'new', 'now', 'old', 'see', 'two', 'way', 'who', 'boy', 'did', 'has', 'let', 'put', 'say', 'she', 'too', 'use',
'avi', 'mkv', 'mp4', 'mpg', 'mov', 'wmv', 'flv', 'webm', 'm4v', 'm2ts', 'ts', 'vob', 'iso', 'img',
'sd', 'hd', 'lq', 'qhd', 'uhd', 'p', 'i', 'hdr', 'sdr', '4k', '8k', '2160p', '1080p', '720p', '480p', '360p', '240p', '144p',
'web', 'dl', 'rip', 'bluray', 'dvd', 'hdtv', 'bdrip', 'dvdrip', 'xvid', 'divx', 'h264', 'h265', 'x264', 'x265', 'hevc', 'avc',
'ma', 'atmos', 'dts', 'aac', 'ac3', 'mp3', 'flac', 'wav', 'wma', 'ogg', 'opus'
]
if part_lower not in skip_words and part_lower in known_language_codes:
lang_code = part_lower
# Convert to 3-letter ISO code
try:
lang_obj = langcodes.Language.get(lang_code)
iso3_code = lang_obj.to_alpha3()
langs.append(iso3_code)
except (LookupError, ValueError, AttributeError) as e:
# Skip invalid language codes
logger.debug(f"Invalid language code '{lang_code}': {e}")
pass
part_lower = part.lower()
# Check if this part is a 2-3 letter code
if not re.match(r'^[a-zA-Z]{2,3}$', part):
continue
# Skip title case 2-letter words to avoid false positives like "In" -> "ind"
if part.istitle() and len(part) == 2:
continue
# Skip known non-language words
if part_lower in SKIP_WORDS:
continue
# Try to validate with langcodes library
try:
lang_obj = langcodes.Language.get(part_lower)
iso3_code = lang_obj.to_alpha3()
langs.append(iso3_code)
except (LookupError, ValueError, AttributeError) as e:
# Not a valid language code, skip
logger.debug(f"Invalid language code '{part_lower}': {e}")
pass
if not langs:
return ''
@@ -404,39 +379,47 @@ class FilenameExtractor:
audio_langs = [f"{count}{lang}" if count > 1 else lang for lang, count in lang_counts.items()]
return ','.join(audio_langs)
@cached_method()
def extract_extension(self) -> str | None:
"""Extract file extension from filename"""
# Use pathlib to extract extension properly
ext = self.file_path.suffix
# Remove leading dot and return
return ext[1:] if ext else None
@cached_method()
def extract_audio_tracks(self) -> list[dict]:
"""Extract audio track data from filename (simplified version with only language)"""
# Similar to extract_audio_langs but returns list of dicts
tracks = []
# First, look for languages inside brackets
bracket_pattern = r'\[([^\]]+)\]'
brackets = re.findall(bracket_pattern, self.file_name)
for bracket in brackets:
bracket_lower = bracket.lower()
# Skip brackets that contain movie database patterns
if any(db in bracket_lower for db in ['imdb', 'tmdb', 'tvdb']):
continue
# Parse items separated by commas or underscores
items = re.split(r'[,_]', bracket)
items = [item.strip() for item in items]
for item in items:
# Skip empty items or items that are clearly not languages
if not item or len(item) < 2:
continue
item_lower = item.lower()
# Skip subtitle indicators
if item_lower in ['sub', 'subs', 'subtitle']:
continue
# Check if item contains language codes (2-3 letter codes)
# Pattern: optional number + optional 'x' + language code
# Allow the language code to be at the end of the item
@@ -444,11 +427,11 @@ class FilenameExtractor:
if lang_match:
count = int(lang_match.group(1)) if lang_match.group(1) else 1
lang_code = lang_match.group(2)
# Skip if it's a quality/resolution indicator
if lang_code in ['sd', 'hd', 'lq', 'qhd', 'uhd', 'p', 'i', 'hdr', 'sdr']:
# Skip if it's a quality/resolution indicator or other skip word
if lang_code in SKIP_WORDS:
continue
# Skip if the language code is not at the end or if there are extra letters after
# But allow prefixes like numbers and 'x'
prefix = item_lower[:-len(lang_code)]
@@ -464,63 +447,41 @@ class FilenameExtractor:
# Skip invalid language codes
logger.debug(f"Invalid language code '{lang_code}': {e}")
pass
# Second, look for standalone language codes outside brackets
# Remove bracketed content first
text_without_brackets = re.sub(r'\[([^\]]+)\]', '', self.file_name)
# Known language codes (2-3 letter ISO 639-1 or 639-3)
known_language_codes = {
'eng', 'ukr', 'rus', 'fra', 'deu', 'spa', 'ita', 'por', 'nor', 'swe', 'dan', 'fin', 'pol', 'cze', 'hun', 'tur', 'ara', 'heb', 'hin', 'jpn', 'kor', 'chi', 'tha', 'vie', 'und',
'dut', 'nld', 'bel', 'bul', 'hrv', 'ces', 'dan', 'nld', 'est', 'fin', 'fra', 'deu', 'ell', 'heb', 'hin', 'hrv', 'hun', 'ind', 'ita', 'jpn', 'kor', 'lav', 'lit', 'mkd', 'nor', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'srp', 'swe', 'tha', 'tur', 'ukr', 'vie', 'und', 'zho',
'arb', 'ben', 'hin', 'mar', 'tam', 'tel', 'urd', 'guj', 'kan', 'mal', 'ori', 'pan', 'asm', 'mai', 'bho', 'nep', 'sin', 'san', 'tib', 'mon', 'kaz', 'uzb', 'kir', 'tuk', 'aze', 'kat', 'hye', 'geo', 'ell', 'sqi', 'bos', 'hrv', 'srp', 'slv', 'mkd', 'bul', 'alb', 'ron', 'mol', 'hun',
'fin', 'swe', 'nor', 'dan', 'isl', 'fao', 'est', 'lav', 'lit', 'bel', 'ukr', 'rus', 'pol', 'cze', 'slk', 'slv', 'hrv', 'bos', 'srp', 'mkd', 'bul', 'ell', 'alb', 'ron', 'hun', 'tur', 'aze', 'geo', 'arm', 'kat', 'hye', 'per', 'kur', 'pus', 'urd', 'ara', 'heb', 'san', 'hin', 'ben', 'tam', 'tel', 'mar', 'guj', 'kan', 'mal', 'ori', 'pan', 'asm', 'mai', 'bho', 'awa', 'mag', 'nep', 'sin', 'div', 'tib', 'mon', 'kaz', 'kir', 'tuk', 'uzb', 'jpn', 'kor', 'chi', 'tha', 'vie', 'und', 'lao', 'khm', 'mya', 'vie', 'und', 'ind', 'msa', 'zho', 'yue', 'wuu', 'nan', 'hak', 'gan', 'hsn',
'spa', 'por', 'fra', 'ita', 'deu', 'nld', 'dut', 'swe', 'nor', 'dan', 'fin', 'est', 'lav', 'lit', 'pol', 'cze', 'slk', 'slv', 'hrv', 'bos', 'srp', 'mkd', 'bul', 'ell', 'alb', 'ron', 'hun', 'tur', 'aze', 'geo', 'arm', 'kat', 'hye', 'per', 'kur', 'pus', 'urd', 'ara', 'heb', 'san', 'hin', 'ben', 'tam', 'tel', 'mar', 'guj', 'kan', 'mal', 'ori', 'pan', 'asm', 'mai', 'bho', 'awa', 'mag', 'nep', 'sin', 'div', 'tib', 'mon', 'kaz', 'kir', 'tuk', 'uzb', 'jpn', 'kor', 'chi', 'tha', 'vie', 'und', 'lao', 'khm', 'mya', 'vie', 'und', 'ind', 'msa', 'zho', 'yue', 'wuu', 'nan', 'hak', 'gan', 'hsn'
}
allowed_title_case = {'ukr', 'nor', 'eng', 'rus', 'fra', 'deu', 'spa', 'ita', 'por', 'swe', 'dan', 'fin', 'pol', 'cze', 'hun', 'tur', 'ara', 'heb', 'hin', 'jpn', 'kor', 'chi', 'tha', 'vie', 'und'}
# Look for language codes in various formats:
# - Uppercase: ENG, UKR, NOR
# - Title case: Ukr, Nor, Eng
# - Lowercase: ukr, nor, eng
# - In dot-separated parts: .ukr. .eng.
# Split on dots, spaces, and underscores
parts = re.split(r'[.\s_]+', text_without_brackets)
for part in parts:
part = part.strip()
if not part or len(part) < 2:
continue
part_lower = part.lower()
# Check if this part is a 2-3 letter language code
if re.match(r'^[a-zA-Z]{2,3}$', part):
# Skip title case 2-letter words to avoid false positives like "In" -> "ind"
if part.istitle() and len(part) == 2:
continue
if part.istitle() and part_lower not in allowed_title_case:
continue
skip_words = [
'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had', 'her', 'was', 'one', 'our', 'out', 'day', 'get', 'has', 'him', 'his', 'how', 'its', 'may', 'new', 'now', 'old', 'see', 'two', 'way', 'who', 'boy', 'did', 'has', 'let', 'put', 'say', 'she', 'too', 'use',
'avi', 'mkv', 'mp4', 'mpg', 'mov', 'wmv', 'flv', 'webm', 'm4v', 'm2ts', 'ts', 'vob', 'iso', 'img',
'sd', 'hd', 'lq', 'qhd', 'uhd', 'p', 'i', 'hdr', 'sdr', '4k', '8k', '2160p', '1080p', '720p', '480p', '360p', '240p', '144p',
'web', 'dl', 'rip', 'bluray', 'dvd', 'hdtv', 'bdrip', 'dvdrip', 'xvid', 'divx', 'h264', 'h265', 'x264', 'x265', 'hevc', 'avc',
'ma', 'atmos', 'dts', 'aac', 'ac3', 'mp3', 'flac', 'wav', 'wma', 'ogg', 'opus'
]
if part_lower not in skip_words and part_lower in known_language_codes:
lang_code = part_lower
# Convert to 3-letter ISO code
try:
lang_obj = langcodes.Language.get(lang_code)
iso3_code = lang_obj.to_alpha3()
tracks.append({'language': iso3_code})
except (LookupError, ValueError, AttributeError) as e:
# Skip invalid language codes
logger.debug(f"Invalid language code '{lang_code}': {e}")
pass
part_lower = part.lower()
# Check if this part is a 2-3 letter code
if not re.match(r'^[a-zA-Z]{2,3}$', part):
continue
# Skip title case 2-letter words to avoid false positives like "In" -> "ind"
if part.istitle() and len(part) == 2:
continue
# Skip known non-language words
if part_lower in SKIP_WORDS:
continue
# Try to validate with langcodes library
try:
lang_obj = langcodes.Language.get(part_lower)
iso3_code = lang_obj.to_alpha3()
tracks.append({'language': iso3_code})
except (LookupError, ValueError, AttributeError) as e:
# Not a valid language code, skip
logger.debug(f"Invalid language code '{part_lower}': {e}")
pass
return tracks