Monitoring & Troubleshooting¶

# View all logs
docker-compose logs -f

# View gateway logs only
docker-compose logs -f inference-gateway

# Last 100 lines
docker-compose logs --tail=100 inference-gateway

# Application logs
tail -f logs/django_info.log

# Gunicorn logs
tail -f logs/backend_gateway.error.log
tail -f logs/backend_gateway.access.log

# Systemd service logs
sudo journalctl -u inference-gateway -f

# Connection stats
psql -h localhost -U inferencedev -d inferencegateway -c "SELECT * FROM pg_stat_activity;"

# Table sizes
psql -h localhost -U inferencedev -d inferencegateway -c "
SELECT schemaname,tablename,pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename))
FROM pg_tables WHERE schemaname='public' ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;"

# Connect to Redis CLI
redis-cli

# In Redis CLI:
INFO
DBSIZE
MONITOR  # Watch commands in real-time

# List endpoints
globus-compute-endpoint list

# Check status
globus-compute-endpoint status my-endpoint

# View logs
globus-compute-endpoint log my-endpoint -n 100

# Follow logs
tail -f ~/.globus_compute/my-endpoint/endpoint.log

# Docker
docker-compose logs inference-gateway

# Bare metal
sudo journalctl -u inference-gateway -n 50
python manage.py check

# Verify PostgreSQL is running
sudo systemctl status postgresql
docker-compose ps postgres

# Test connection
psql -h localhost -U inferencedev -d inferencegateway

# Check pg_hba.conf
sudo nano /etc/postgresql/*/main/pg_hba.conf

# Restart PostgreSQL
sudo systemctl restart postgresql

# Check endpoint is running
globus-compute-endpoint list

# Restart endpoint
globus-compute-endpoint restart my-endpoint

# View detailed logs
globus-compute-endpoint log my-endpoint -n 200

# Verify function UUID is allowed
cat ~/.globus_compute/my-endpoint/config.yaml

# vLLM: Reduce GPU memory usage
--gpu-memory-utilization 0.7

# vLLM: Use quantization
--quantization awq

# vLLM: Reduce context length
--max-model-len 2048

# System: Add swap space
sudo fallocate -l 32G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile

# Gateway health
curl http://localhost:8000/

# vLLM health
curl http://localhost:8001/health

# Database connectivity
python manage.py dbshell

# Redis connectivity
redis-cli ping

#!/bin/bash
# health_check.sh

# Check gateway
if curl -s http://localhost:8000/ > /dev/null; then
    echo "✓ Gateway is healthy"
else
    echo "✗ Gateway is down"
    systemctl restart inference-gateway
fi

# Check database
if psql -h localhost -U inferencedev -d inferencegateway -c "SELECT 1;" > /dev/null 2>&1; then
    echo "✓ Database is healthy"
else
    echo "✗ Database is down"
fi

*/5 * * * * /path/to/health_check.sh >> /var/log/health_check.log 2>&1

# Request metrics
http_requests_total
http_request_duration_seconds

# Globus Compute metrics
globus_compute_tasks_submitted
globus_compute_tasks_completed
globus_compute_tasks_failed

# System metrics
process_cpu_seconds_total
process_resident_memory_bytes