diff --git a/cleanup_commands.txt b/cleanup_commands.txt new file mode 100644 index 0000000..9524acb --- /dev/null +++ b/cleanup_commands.txt @@ -0,0 +1,65 @@ +# Cleanup Commands for Stuck Celery Tasks + +## Quick Commands + +### 1. Restart Celery Worker (to apply code changes) +```bash +docker restart mix_celery_worker +docker logs -f --tail 50 mix_celery_worker +``` + +### 2. Run Cleanup Script (Interactive) +```bash +# Copy script to container +docker cp cleanup_stuck_photos.py mix_web:/app/ + +# Run interactively +docker exec -it mix_web python manage.py shell +>>> exec(open('cleanup_stuck_photos.py').read()) +``` + +### 3. Manual Cleanup (Django Shell) +```bash +docker exec -it mix_web python manage.py shell +``` + +Then run: +```python +from django.db import connection +from products.models import ProductPhoto, PhotoProcessingStatus +from django.core.files.storage import default_storage + +# Activate schema +connection.set_schema('mixflowers') + +# Find photo #6 +photo = ProductPhoto.objects.get(id=6) +print(f"Photo: {photo}") +print(f"File path: {photo.image.name}") +print(f"File exists: {default_storage.exists(photo.image.name)}") + +# Option 1: Delete the record +photo.delete() + +# Option 2: Clear the image field (keep record) +# photo.image = None +# photo.save() + +# Clean up stuck processing statuses +stuck = PhotoProcessingStatus.objects.filter(status__in=['pending', 'processing']) +print(f"Found {stuck.count()} stuck statuses") +stuck.update(status='failed', error_message='File was deleted before processing') +``` + +## What Was Fixed + +1. **FileNotFoundError handling**: Task now fails immediately instead of retrying when file is missing +2. **File existence check**: Added check before processing to catch missing files early +3. **Status updates**: PhotoProcessingStatus is properly updated to 'failed' when file not found + +## Testing + +After restarting Celery worker: +1. Upload a new photo - should process normally +2. Check logs - should not see retry loops for missing files +3. Verify stuck tasks are cleaned up diff --git a/cleanup_stuck_photos.py b/cleanup_stuck_photos.py new file mode 100644 index 0000000..f84b8db --- /dev/null +++ b/cleanup_stuck_photos.py @@ -0,0 +1,124 @@ +""" +Скрипт для очистки застрявших записей фото в БД. +Используется когда файлы были удалены вручную до обработки Celery. + +Использование: + docker exec -it mix_web python manage.py shell < cleanup_stuck_photos.py + +Или интерактивно: + docker exec -it mix_web python manage.py shell + >>> exec(open('cleanup_stuck_photos.py').read()) +""" + +from django.db import connection +from products.models import ProductPhoto, ProductKitPhoto, ProductCategoryPhoto, PhotoProcessingStatus +from django.core.files.storage import default_storage + +# Укажите schema_name вашего тенанта +SCHEMA_NAME = 'mixflowers' + +# Активируем схему +connection.set_schema(SCHEMA_NAME) +print(f"✓ Activated schema: {SCHEMA_NAME}") + +# Функция для проверки и очистки фото +def cleanup_photo_model(model_class, model_name): + """Очищает записи с несуществующими файлами для указанной модели""" + print(f"\n{'='*60}") + print(f"Checking {model_name}...") + print(f"{'='*60}") + + photos = model_class.objects.all() + total = photos.count() + print(f"Total {model_name} records: {total}") + + if total == 0: + print(f"No {model_name} records found.") + return + + missing_files = [] + + for photo in photos: + if photo.image: + file_path = photo.image.name + exists = default_storage.exists(file_path) + + if not exists: + missing_files.append({ + 'id': photo.id, + 'path': file_path, + 'entity': photo.get_entity(), + 'photo': photo + }) + print(f" ✗ Photo #{photo.id}: File NOT found: {file_path}") + else: + print(f" ✓ Photo #{photo.id}: File exists: {file_path}") + + if missing_files: + print(f"\n{'='*60}") + print(f"Found {len(missing_files)} {model_name} with missing files:") + print(f"{'='*60}") + + for item in missing_files: + print(f"\nPhoto ID: {item['id']}") + print(f" Entity: {item['entity']}") + print(f" Missing file: {item['path']}") + + # Спрашиваем подтверждение + print(f"\n{'='*60}") + response = input(f"Delete these {len(missing_files)} {model_name} records? (yes/no): ").strip().lower() + + if response == 'yes': + deleted_count = 0 + for item in missing_files: + try: + item['photo'].delete() + deleted_count += 1 + print(f" ✓ Deleted Photo #{item['id']}") + except Exception as e: + print(f" ✗ Error deleting Photo #{item['id']}: {e}") + + print(f"\n✓ Deleted {deleted_count} {model_name} records") + else: + print(f"\nSkipped deletion for {model_name}") + else: + print(f"\n✓ All {model_name} files exist. No cleanup needed.") + +# Очищаем каждую модель +cleanup_photo_model(ProductPhoto, "ProductPhoto") +cleanup_photo_model(ProductKitPhoto, "ProductKitPhoto") +cleanup_photo_model(ProductCategoryPhoto, "ProductCategoryPhoto") + +# Проверяем застрявшие статусы обработки +print(f"\n{'='*60}") +print(f"Checking PhotoProcessingStatus...") +print(f"{'='*60}") + +stuck_statuses = PhotoProcessingStatus.objects.filter( + status__in=['pending', 'processing'] +).order_by('-created_at') + +if stuck_statuses.exists(): + print(f"Found {stuck_statuses.count()} stuck processing statuses:") + for status in stuck_statuses: + print(f"\n Photo ID: {status.photo_id}") + print(f" Model: {status.photo_model}") + print(f" Status: {status.status}") + print(f" Created: {status.created_at}") + print(f" Task ID: {status.task_id}") + + response = input(f"\nMark these as 'failed'? (yes/no): ").strip().lower() + if response == 'yes': + updated = stuck_statuses.update( + status='failed', + error_message='Marked as failed during cleanup (file was deleted before processing)' + ) + print(f"\n✓ Updated {updated} processing statuses to 'failed'") + else: + print("\nSkipped updating processing statuses") +else: + print("✓ No stuck processing statuses found") + +print(f"\n{'='*60}") +print("Cleanup complete!") +print(f"{'='*60}") diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d193682..021a651 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -151,6 +151,9 @@ EOF if [ ! -f "manage.py" ] && [ -d "myproject" ]; then echo "Changing directory to myproject..." cd myproject + # Устанавливаем PYTHONPATH чтобы Python мог найти модуль myproject + export PYTHONPATH=$(pwd):$PYTHONPATH + echo "PYTHONPATH set to: $PYTHONPATH" fi case "$1" in diff --git a/myproject/myproject/__init__.py b/myproject/myproject/__init__.py index a58cbd9..b3c181b 100644 --- a/myproject/myproject/__init__.py +++ b/myproject/myproject/__init__.py @@ -7,4 +7,4 @@ from .celery import app as celery_app # Для совместимости с командами типа 'celery -A myproject' celery = celery_app -__all__ = ('celery_app',) +__all__ = ('celery_app', 'celery') diff --git a/myproject/products/tasks.py b/myproject/products/tasks.py index 3b0d1dc..d3da793 100644 --- a/myproject/products/tasks.py +++ b/myproject/products/tasks.py @@ -16,6 +16,7 @@ from django.db import connection from django.apps import apps from django.conf import settings from django.core.files.storage import default_storage +from django.utils import timezone logger = logging.getLogger(__name__) @@ -73,6 +74,12 @@ def process_product_photo_async(self, photo_id, photo_model_class, schema_name): # Сохраняем путь к временному файлу до перезаписи поля image temp_path = photo_obj.image.name + # КРИТИЧНО: Проверяем что файл существует перед обработкой + # Это предотвращает бесполезные retry если файл был удален вручную + if not default_storage.exists(temp_path): + logger.error(f"[Celery] File does not exist: {temp_path}") + raise FileNotFoundError(f"File not found: {temp_path}") + # Получаем entity type для правильного пути сохранения entity_type = photo_obj.get_entity_type() @@ -138,6 +145,35 @@ def process_product_photo_async(self, photo_id, photo_model_class, schema_name): logger.warning(f"[Celery] Could not delete temp file for photo {photo_id} on not_found: {del_exc}") return {'status': 'error', 'reason': 'not_found', 'photo_id': photo_id} + except FileNotFoundError as exc: + # Файл не найден - не имеет смысла повторять task + # Это обычно означает что файл был удален до обработки + logger.error(f"[Celery] File not found for photo {photo_id} in {schema_name}: {str(exc)}") + logger.error(f"[Celery] This usually means the file was deleted before processing. Marking as failed.") + + # Обновляем статус обработки без retry + try: + from .models.photos import PhotoProcessingStatus + status = (PhotoProcessingStatus.objects + .filter(photo_id=photo_id, photo_model=photo_model_class) + .order_by('-created_at') + .first()) + if status: + status.status = 'failed' + status.error_message = f"File not found: {str(exc)}" + status.completed_at = timezone.now() + status.save() + logger.info(f"[Celery] Updated PhotoProcessingStatus to 'failed' for photo {photo_id}") + except Exception as status_exc: + logger.warning(f"[Celery] Could not update PhotoProcessingStatus: {status_exc}") + + return { + 'status': 'error', + 'reason': 'file_not_found', + 'photo_id': photo_id, + 'error': str(exc) + } + except Exception as exc: logger.error(f"[Celery] Error processing photo {photo_id} in {schema_name}: {str(exc)}", exc_info=True)