from __future__ import annotations from typing import List, Dict, Any from datetime import datetime, timedelta from django.core.management.base import BaseCommand, CommandParser from django.contrib.auth import get_user_model from django.utils import timezone as dj_tz from django.db.models import Q from bazi.models import Person as BaziPerson from bazi.models_group import GroupRelation class Command(BaseCommand): help = "Monitor the status of background relation calculations and diagnose hanging processes" def add_arguments(self, parser: CommandParser) -> None: parser.add_argument( "--status", choices=['all', 'processing', 'error', 'completed', 'idle'], default='all', help="Filter by status (default: all)" ) parser.add_argument( "--timeout-minutes", type=int, default=5, help="Minutes after which a processing state is considered hanging (default: 5)" ) parser.add_argument( "--show-details", action="store_true", help="Show detailed information for each user" ) parser.add_argument( "--fix-hanging", action="store_true", help="Automatically fix hanging processes by setting them to error state" ) def handle(self, *args, **options): status_filter = options["status"] timeout_minutes = options["timeout_minutes"] show_details = options["show_details"] fix_hanging = options["fix_hanging"] User = get_user_model() # Build query based on status filter if status_filter == 'all': users = User.objects.all() else: users = User.objects.filter(group_relations_state=status_filter) # Get users with relation state users_with_state = users.exclude(group_relations_state__isnull=True) self.stdout.write(f"=== BaZi Relations Status Monitor ===") self.stdout.write(f"Status Filter: {status_filter}") self.stdout.write(f"Timeout Threshold: {timeout_minutes} minutes") self.stdout.write(f"Total Users: {users_with_state.count()}") self.stdout.write("") # Analyze each status status_counts = {} hanging_users = [] error_users = [] for user in users_with_state: state = user.group_relations_state or 'idle' status_counts[state] = status_counts.get(state, 0) + 1 if state == 'processing': # Check if hanging if user.group_relations_started_at: time_since_start = dj_tz.now() - user.group_relations_started_at if time_since_start > timedelta(minutes=timeout_minutes): hanging_users.append(user) elif state == 'error': error_users.append(user) # Display summary self.stdout.write("=== Status Summary ===") for status, count in sorted(status_counts.items()): self.stdout.write(f"{status.capitalize()}: {count}") self.stdout.write("") # Show hanging processes if hanging_users: self.stdout.write(f"=== ⚠️ HANGING PROCESSES ({len(hanging_users)}) ===") self.stdout.write(f"These users have been in 'processing' state for >{timeout_minutes} minutes:") for user in hanging_users: time_since_start = dj_tz.now() - user.group_relations_started_at self.stdout.write(f" User {user.id} ({user.email or user.username})") self.stdout.write(f" Started: {user.group_relations_started_at}") self.stdout.write(f" Hanging for: {time_since_start}") if fix_hanging: user.group_relations_state = 'error' user.group_relations_error_at = dj_tz.now() user.save(update_fields=['group_relations_state', 'group_relations_error_at']) self.stdout.write(f" → Fixed: Set to 'error' state") self.stdout.write("") # Show error states if error_users: self.stdout.write(f"=== ❌ ERROR STATES ({len(error_users)}) ===") for user in error_users: self.stdout.write(f" User {user.id} ({user.email or user.username})") if user.group_relations_error_at: self.stdout.write(f" Error at: {user.group_relations_error_at}") if show_details: # Show related data owner = BaziPerson.objects.filter(created_by=user, owner=True).first() if owner: self.stdout.write(f" Has owner BaZi: {'Yes' if owner.bazi_result else 'No'}") person_count = BaziPerson.objects.filter(created_by=user).count() self.stdout.write(f" Total persons: {person_count}") self.stdout.write("") # Show processing states processing_users = [u for u in users_with_state if u.group_relations_state == 'processing'] if processing_users: self.stdout.write(f"=== 🔄 PROCESSING ({len(processing_users)}) ===") for user in processing_users: if user.group_relations_started_at: time_since_start = dj_tz.now() - user.group_relations_started_at self.stdout.write(f" User {user.id} ({user.email or user.username})") self.stdout.write(f" Started: {user.group_relations_started_at}") self.stdout.write(f" Running for: {time_since_start}") if show_details: # Show progress info owner = BaziPerson.objects.filter(created_by=user, owner=True).first() if owner and owner.bazi_result: person_count = BaziPerson.objects.filter(created_by=user).exclude(id=owner.id).count() self.stdout.write(f" Persons to process: {person_count}") self.stdout.write("") # Show completed states completed_users = [u for u in users_with_state if u.group_relations_state == 'completed'] if completed_users: self.stdout.write(f"=== ✅ COMPLETED ({len(completed_users)}) ===") if show_details: for user in completed_users[:10]: # Show first 10 self.stdout.write(f" User {user.id} ({user.email or user.username})") if user.group_relations_updated_at: self.stdout.write(f" Completed at: {user.group_relations_updated_at}") # Show results group_count = GroupRelation.objects.filter(owner_user_id=user.id).count() self.stdout.write(f" Group relations: {group_count}") self.stdout.write("") if len(completed_users) > 10: self.stdout.write(f" ... and {len(completed_users) - 10} more users") else: self.stdout.write(f" {len(completed_users)} users completed successfully") # Recommendations self.stdout.write("") self.stdout.write("=== 💡 RECOMMENDATIONS ===") if hanging_users: self.stdout.write(f"• {len(hanging_users)} hanging processes detected") if not fix_hanging: self.stdout.write(f"• Run with --fix-hanging to automatically fix hanging processes") self.stdout.write(f"• Check server logs for these user IDs: {[u.id for u in hanging_users]}") if error_users: self.stdout.write(f"• {len(error_users)} users in error state") self.stdout.write(f"• Run: python manage.py recalc_bazi_relations --user --force") if processing_users: self.stdout.write(f"• {len(processing_users)} users currently processing") self.stdout.write(f"• Monitor with: python manage.py monitor_relations_status --status processing") self.stdout.write("") self.stdout.write("=== COMMANDS ===") self.stdout.write("• Monitor all: python manage.py monitor_relations_status") self.stdout.write("• Monitor processing: python manage.py monitor_relations_status --status processing") self.stdout.write("• Fix hanging: python manage.py monitor_relations_status --fix-hanging") self.stdout.write("• Force recalculation: python manage.py recalc_bazi_relations --user --force") self.stdout.write("• Recalc all: python manage.py recalc_bazi_relations --all-users --force")