freeleaps-ops/apps/gitea-webhook-ambassador-python/app/tasks/jenkins_tasks.py

"""
Jenkins 任务处理
使用 Celery 处理异步 Jenkins 任务触发
"""

import asyncio
import time
from typing import Dict, Any
from datetime import datetime
import structlog
from celery import Celery, Task
import httpx

from app.config import get_settings
from app.services.jenkins_service import JenkinsService

logger = structlog.get_logger()
settings = get_settings()

# 创建 Celery 应用
celery_app = Celery(
    "gitea_webhook_ambassador",
    broker=settings.redis.url,
    backend=settings.redis.url,
    include=["app.tasks.jenkins_tasks"]
)

# Celery 配置
celery_app.conf.update(
    task_serializer="json",
    accept_content=["json"],
    result_serializer="json",
    timezone="UTC",
    enable_utc=True,
    task_track_started=True,
    task_time_limit=300,  # 5分钟超时
    task_soft_time_limit=240,  # 4分钟软超时
    worker_prefetch_multiplier=1,
    worker_max_tasks_per_child=1000,
    worker_max_memory_per_child=200000,  # 200MB
    task_acks_late=True,
    task_reject_on_worker_lost=True,
    task_always_eager=False,  # 生产环境设为 False
    result_expires=3600,  # 结果缓存1小时
)


class JenkinsTask(Task):
    """Jenkins 任务基类"""
    
    abstract = True
    
    def __init__(self):
        self.jenkins_service = None
    
    def __call__(self, *args, **kwargs):
        if self.jenkins_service is None:
            self.jenkins_service = JenkinsService()
        return self.run(*args, **kwargs)
    
    def on_failure(self, exc, task_id, args, kwargs, einfo):
        """任务失败回调"""
        logger.error("Task failed",
                    task_id=task_id,
                    task_name=self.name,
                    error=str(exc),
                    args=args,
                    kwargs=kwargs)
    
    def on_retry(self, exc, task_id, args, kwargs, einfo):
        """任务重试回调"""
        logger.warning("Task retrying",
                      task_id=task_id,
                      task_name=self.name,
                      error=str(exc),
                      retry_count=self.request.retries)
    
    def on_success(self, retval, task_id, args, kwargs):
        """任务成功回调"""
        logger.info("Task completed successfully",
                   task_id=task_id,
                   task_name=self.name,
                   result=retval)


@celery_app.task(
    bind=True,
    base=JenkinsTask,
    max_retries=3,
    default_retry_delay=60,
    autoretry_for=(Exception,),
    retry_backoff=True,
    retry_jitter=True
)
def trigger_jenkins_job(
    self,
    job_name: str,
    jenkins_url: str,
    parameters: Dict[str, str],
    event_id: str,
    repository: str,
    branch: str,
    commit_hash: str,
    priority: int = 1
) -> Dict[str, Any]:
    """
    触发 Jenkins 任务
    
    Args:
        job_name: Jenkins 任务名
        jenkins_url: Jenkins URL
        parameters: 任务参数
        event_id: 事件 ID
        repository: 仓库名
        branch: 分支名
        commit_hash: 提交哈希
        priority: 优先级
        
    Returns:
        Dict: 任务执行结果
    """
    start_time = time.time()
    
    try:
        logger.info("Starting Jenkins job trigger",
                   task_id=self.request.id,
                   job_name=job_name,
                   jenkins_url=jenkins_url,
                   repository=repository,
                   branch=branch,
                   commit_hash=commit_hash,
                   priority=priority)
        
        # 创建 Jenkins 服务实例
        jenkins_service = JenkinsService()
        
        # 触发 Jenkins 任务
        result = asyncio.run(jenkins_service.trigger_job(
            job_name=job_name,
            jenkins_url=jenkins_url,
            parameters=parameters
        ))
        
        execution_time = time.time() - start_time
        
        if result["success"]:
            logger.info("Jenkins job triggered successfully",
                       task_id=self.request.id,
                       job_name=job_name,
                       build_number=result.get("build_number"),
                       execution_time=execution_time)
            
            return {
                "success": True,
                "task_id": self.request.id,
                "job_name": job_name,
                "jenkins_url": jenkins_url,
                "build_number": result.get("build_number"),
                "build_url": result.get("build_url"),
                "event_id": event_id,
                "repository": repository,
                "branch": branch,
                "commit_hash": commit_hash,
                "execution_time": execution_time,
                "timestamp": datetime.utcnow().isoformat()
            }
        else:
            logger.error("Jenkins job trigger failed",
                        task_id=self.request.id,
                        job_name=job_name,
                        error=result.get("error"),
                        execution_time=execution_time)
            
            # 重试任务
            raise self.retry(
                countdown=settings.queue.retry_delay * (2 ** self.request.retries),
                max_retries=settings.queue.max_retries
            )
            
    except Exception as e:
        execution_time = time.time() - start_time
        logger.error("Unexpected error in Jenkins task",
                    task_id=self.request.id,
                    job_name=job_name,
                    error=str(e),
                    execution_time=execution_time)
        
        # 重试任务
        raise self.retry(
            countdown=settings.queue.retry_delay * (2 ** self.request.retries),
            max_retries=settings.queue.max_retries
        )


@celery_app.task(
    bind=True,
    base=JenkinsTask,
    max_retries=2,
    default_retry_delay=30
)
def check_jenkins_health(
    self,
    jenkins_url: str
) -> Dict[str, Any]:
    """
    检查 Jenkins 健康状态
    
    Args:
        jenkins_url: Jenkins URL
        
    Returns:
        Dict: 健康检查结果
    """
    try:
        logger.info("Checking Jenkins health", jenkins_url=jenkins_url)
        
        jenkins_service = JenkinsService()
        result = asyncio.run(jenkins_service.check_health(jenkins_url))
        
        return {
            "success": True,
            "jenkins_url": jenkins_url,
            "healthy": result.get("healthy", False),
            "response_time": result.get("response_time"),
            "timestamp": datetime.utcnow().isoformat()
        }
        
    except Exception as e:
        logger.error("Jenkins health check failed",
                    jenkins_url=jenkins_url,
                    error=str(e))
        
        return {
            "success": False,
            "jenkins_url": jenkins_url,
            "error": str(e),
            "timestamp": datetime.utcnow().isoformat()
        }


@celery_app.task(
    bind=True,
    base=JenkinsTask
)
def cleanup_expired_tasks(self) -> Dict[str, Any]:
    """
    清理过期任务
    
    Returns:
        Dict: 清理结果
    """
    try:
        logger.info("Starting task cleanup")
        
        # 获取所有任务
        inspect = self.app.control.inspect()
        
        # 清理过期的结果
        cleaned_count = 0
        current_time = time.time()
        
        # 这里可以添加更复杂的清理逻辑
        # 比如清理超过一定时间的任务结果
        
        logger.info("Task cleanup completed", cleaned_count=cleaned_count)
        
        return {
            "success": True,
            "cleaned_count": cleaned_count,
            "timestamp": datetime.utcnow().isoformat()
        }
        
    except Exception as e:
        logger.error("Task cleanup failed", error=str(e))
        
        return {
            "success": False,
            "error": str(e),
            "timestamp": datetime.utcnow().isoformat()
        }


# 定时任务
@celery_app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
    """设置定时任务"""
    
    # 每小时清理过期任务
    sender.add_periodic_task(
        3600.0,  # 1小时
        cleanup_expired_tasks.s(),
        name="cleanup-expired-tasks"
    )
    
    # 每5分钟检查 Jenkins 健康状态
    for env_name, env_config in settings.environments.items():
        sender.add_periodic_task(
            300.0,  # 5分钟
            check_jenkins_health.s(env_config.jenkins_url),
            name=f"check-jenkins-health-{env_name}"
        )


def get_celery_app() -> Celery:
    """获取 Celery 应用实例"""
    return celery_app