feat: Add Docker entrypoint script for application orchestration and implement cleanup for stuck photo processing tasks with improved error handling.
This commit is contained in:
65
cleanup_commands.txt
Normal file
65
cleanup_commands.txt
Normal file
@@ -0,0 +1,65 @@
|
||||
# Cleanup Commands for Stuck Celery Tasks
|
||||
|
||||
## Quick Commands
|
||||
|
||||
### 1. Restart Celery Worker (to apply code changes)
|
||||
```bash
|
||||
docker restart mix_celery_worker
|
||||
docker logs -f --tail 50 mix_celery_worker
|
||||
```
|
||||
|
||||
### 2. Run Cleanup Script (Interactive)
|
||||
```bash
|
||||
# Copy script to container
|
||||
docker cp cleanup_stuck_photos.py mix_web:/app/
|
||||
|
||||
# Run interactively
|
||||
docker exec -it mix_web python manage.py shell
|
||||
>>> exec(open('cleanup_stuck_photos.py').read())
|
||||
```
|
||||
|
||||
### 3. Manual Cleanup (Django Shell)
|
||||
```bash
|
||||
docker exec -it mix_web python manage.py shell
|
||||
```
|
||||
|
||||
Then run:
|
||||
```python
|
||||
from django.db import connection
|
||||
from products.models import ProductPhoto, PhotoProcessingStatus
|
||||
from django.core.files.storage import default_storage
|
||||
|
||||
# Activate schema
|
||||
connection.set_schema('mixflowers')
|
||||
|
||||
# Find photo #6
|
||||
photo = ProductPhoto.objects.get(id=6)
|
||||
print(f"Photo: {photo}")
|
||||
print(f"File path: {photo.image.name}")
|
||||
print(f"File exists: {default_storage.exists(photo.image.name)}")
|
||||
|
||||
# Option 1: Delete the record
|
||||
photo.delete()
|
||||
|
||||
# Option 2: Clear the image field (keep record)
|
||||
# photo.image = None
|
||||
# photo.save()
|
||||
|
||||
# Clean up stuck processing statuses
|
||||
stuck = PhotoProcessingStatus.objects.filter(status__in=['pending', 'processing'])
|
||||
print(f"Found {stuck.count()} stuck statuses")
|
||||
stuck.update(status='failed', error_message='File was deleted before processing')
|
||||
```
|
||||
|
||||
## What Was Fixed
|
||||
|
||||
1. **FileNotFoundError handling**: Task now fails immediately instead of retrying when file is missing
|
||||
2. **File existence check**: Added check before processing to catch missing files early
|
||||
3. **Status updates**: PhotoProcessingStatus is properly updated to 'failed' when file not found
|
||||
|
||||
## Testing
|
||||
|
||||
After restarting Celery worker:
|
||||
1. Upload a new photo - should process normally
|
||||
2. Check logs - should not see retry loops for missing files
|
||||
3. Verify stuck tasks are cleaned up
|
||||
124
cleanup_stuck_photos.py
Normal file
124
cleanup_stuck_photos.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
Скрипт для очистки застрявших записей фото в БД.
|
||||
Используется когда файлы были удалены вручную до обработки Celery.
|
||||
|
||||
Использование:
|
||||
docker exec -it mix_web python manage.py shell < cleanup_stuck_photos.py
|
||||
|
||||
Или интерактивно:
|
||||
docker exec -it mix_web python manage.py shell
|
||||
>>> exec(open('cleanup_stuck_photos.py').read())
|
||||
"""
|
||||
|
||||
from django.db import connection
|
||||
from products.models import ProductPhoto, ProductKitPhoto, ProductCategoryPhoto, PhotoProcessingStatus
|
||||
from django.core.files.storage import default_storage
|
||||
|
||||
# Укажите schema_name вашего тенанта
|
||||
SCHEMA_NAME = 'mixflowers'
|
||||
|
||||
# Активируем схему
|
||||
connection.set_schema(SCHEMA_NAME)
|
||||
print(f"✓ Activated schema: {SCHEMA_NAME}")
|
||||
|
||||
# Функция для проверки и очистки фото
|
||||
def cleanup_photo_model(model_class, model_name):
|
||||
"""Очищает записи с несуществующими файлами для указанной модели"""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Checking {model_name}...")
|
||||
print(f"{'='*60}")
|
||||
|
||||
photos = model_class.objects.all()
|
||||
total = photos.count()
|
||||
print(f"Total {model_name} records: {total}")
|
||||
|
||||
if total == 0:
|
||||
print(f"No {model_name} records found.")
|
||||
return
|
||||
|
||||
missing_files = []
|
||||
|
||||
for photo in photos:
|
||||
if photo.image:
|
||||
file_path = photo.image.name
|
||||
exists = default_storage.exists(file_path)
|
||||
|
||||
if not exists:
|
||||
missing_files.append({
|
||||
'id': photo.id,
|
||||
'path': file_path,
|
||||
'entity': photo.get_entity(),
|
||||
'photo': photo
|
||||
})
|
||||
print(f" ✗ Photo #{photo.id}: File NOT found: {file_path}")
|
||||
else:
|
||||
print(f" ✓ Photo #{photo.id}: File exists: {file_path}")
|
||||
|
||||
if missing_files:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Found {len(missing_files)} {model_name} with missing files:")
|
||||
print(f"{'='*60}")
|
||||
|
||||
for item in missing_files:
|
||||
print(f"\nPhoto ID: {item['id']}")
|
||||
print(f" Entity: {item['entity']}")
|
||||
print(f" Missing file: {item['path']}")
|
||||
|
||||
# Спрашиваем подтверждение
|
||||
print(f"\n{'='*60}")
|
||||
response = input(f"Delete these {len(missing_files)} {model_name} records? (yes/no): ").strip().lower()
|
||||
|
||||
if response == 'yes':
|
||||
deleted_count = 0
|
||||
for item in missing_files:
|
||||
try:
|
||||
item['photo'].delete()
|
||||
deleted_count += 1
|
||||
print(f" ✓ Deleted Photo #{item['id']}")
|
||||
except Exception as e:
|
||||
print(f" ✗ Error deleting Photo #{item['id']}: {e}")
|
||||
|
||||
print(f"\n✓ Deleted {deleted_count} {model_name} records")
|
||||
else:
|
||||
print(f"\nSkipped deletion for {model_name}")
|
||||
else:
|
||||
print(f"\n✓ All {model_name} files exist. No cleanup needed.")
|
||||
|
||||
# Очищаем каждую модель
|
||||
cleanup_photo_model(ProductPhoto, "ProductPhoto")
|
||||
cleanup_photo_model(ProductKitPhoto, "ProductKitPhoto")
|
||||
cleanup_photo_model(ProductCategoryPhoto, "ProductCategoryPhoto")
|
||||
|
||||
# Проверяем застрявшие статусы обработки
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Checking PhotoProcessingStatus...")
|
||||
print(f"{'='*60}")
|
||||
|
||||
stuck_statuses = PhotoProcessingStatus.objects.filter(
|
||||
status__in=['pending', 'processing']
|
||||
).order_by('-created_at')
|
||||
|
||||
if stuck_statuses.exists():
|
||||
print(f"Found {stuck_statuses.count()} stuck processing statuses:")
|
||||
for status in stuck_statuses:
|
||||
print(f"\n Photo ID: {status.photo_id}")
|
||||
print(f" Model: {status.photo_model}")
|
||||
print(f" Status: {status.status}")
|
||||
print(f" Created: {status.created_at}")
|
||||
print(f" Task ID: {status.task_id}")
|
||||
|
||||
response = input(f"\nMark these as 'failed'? (yes/no): ").strip().lower()
|
||||
if response == 'yes':
|
||||
updated = stuck_statuses.update(
|
||||
status='failed',
|
||||
error_message='Marked as failed during cleanup (file was deleted before processing)'
|
||||
)
|
||||
print(f"\n✓ Updated {updated} processing statuses to 'failed'")
|
||||
else:
|
||||
print("\nSkipped updating processing statuses")
|
||||
else:
|
||||
print("✓ No stuck processing statuses found")
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print("Cleanup complete!")
|
||||
print(f"{'='*60}")
|
||||
@@ -151,6 +151,9 @@ EOF
|
||||
if [ ! -f "manage.py" ] && [ -d "myproject" ]; then
|
||||
echo "Changing directory to myproject..."
|
||||
cd myproject
|
||||
# Устанавливаем PYTHONPATH чтобы Python мог найти модуль myproject
|
||||
export PYTHONPATH=$(pwd):$PYTHONPATH
|
||||
echo "PYTHONPATH set to: $PYTHONPATH"
|
||||
fi
|
||||
|
||||
case "$1" in
|
||||
|
||||
@@ -7,4 +7,4 @@ from .celery import app as celery_app
|
||||
# Для совместимости с командами типа 'celery -A myproject'
|
||||
celery = celery_app
|
||||
|
||||
__all__ = ('celery_app',)
|
||||
__all__ = ('celery_app', 'celery')
|
||||
|
||||
@@ -16,6 +16,7 @@ from django.db import connection
|
||||
from django.apps import apps
|
||||
from django.conf import settings
|
||||
from django.core.files.storage import default_storage
|
||||
from django.utils import timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -73,6 +74,12 @@ def process_product_photo_async(self, photo_id, photo_model_class, schema_name):
|
||||
# Сохраняем путь к временному файлу до перезаписи поля image
|
||||
temp_path = photo_obj.image.name
|
||||
|
||||
# КРИТИЧНО: Проверяем что файл существует перед обработкой
|
||||
# Это предотвращает бесполезные retry если файл был удален вручную
|
||||
if not default_storage.exists(temp_path):
|
||||
logger.error(f"[Celery] File does not exist: {temp_path}")
|
||||
raise FileNotFoundError(f"File not found: {temp_path}")
|
||||
|
||||
# Получаем entity type для правильного пути сохранения
|
||||
entity_type = photo_obj.get_entity_type()
|
||||
|
||||
@@ -138,6 +145,35 @@ def process_product_photo_async(self, photo_id, photo_model_class, schema_name):
|
||||
logger.warning(f"[Celery] Could not delete temp file for photo {photo_id} on not_found: {del_exc}")
|
||||
return {'status': 'error', 'reason': 'not_found', 'photo_id': photo_id}
|
||||
|
||||
except FileNotFoundError as exc:
|
||||
# Файл не найден - не имеет смысла повторять task
|
||||
# Это обычно означает что файл был удален до обработки
|
||||
logger.error(f"[Celery] File not found for photo {photo_id} in {schema_name}: {str(exc)}")
|
||||
logger.error(f"[Celery] This usually means the file was deleted before processing. Marking as failed.")
|
||||
|
||||
# Обновляем статус обработки без retry
|
||||
try:
|
||||
from .models.photos import PhotoProcessingStatus
|
||||
status = (PhotoProcessingStatus.objects
|
||||
.filter(photo_id=photo_id, photo_model=photo_model_class)
|
||||
.order_by('-created_at')
|
||||
.first())
|
||||
if status:
|
||||
status.status = 'failed'
|
||||
status.error_message = f"File not found: {str(exc)}"
|
||||
status.completed_at = timezone.now()
|
||||
status.save()
|
||||
logger.info(f"[Celery] Updated PhotoProcessingStatus to 'failed' for photo {photo_id}")
|
||||
except Exception as status_exc:
|
||||
logger.warning(f"[Celery] Could not update PhotoProcessingStatus: {status_exc}")
|
||||
|
||||
return {
|
||||
'status': 'error',
|
||||
'reason': 'file_not_found',
|
||||
'photo_id': photo_id,
|
||||
'error': str(exc)
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(f"[Celery] Error processing photo {photo_id} in {schema_name}: {str(exc)}",
|
||||
exc_info=True)
|
||||
|
||||
Reference in New Issue
Block a user