f6c515157c
- 新增完整的 Python 实现，替代 Go 版本 - 添加 Web 登录界面和仪表板 - 实现 JWT 认证和 API 密钥管理 - 添加数据库存储功能 - 保持与 Go 版本一致的目录结构和启动脚本 - 包含完整的文档和测试脚本
280 lines
10 KiB
Python
280 lines
10 KiB
Python
"""
|
|
Webhook 处理服务
|
|
实现智能分发、任务排队和防抖策略
|
|
"""
|
|
|
|
import asyncio
|
|
from typing import Optional, Dict, Any
|
|
from datetime import datetime
|
|
import structlog
|
|
from celery import Celery
|
|
|
|
from app.config import get_settings
|
|
from app.models.gitea import GiteaWebhook, WebhookResponse
|
|
from app.services.dedup_service import DeduplicationService
|
|
from app.services.jenkins_service import JenkinsService
|
|
from app.services.database_service import get_database_service
|
|
from app.tasks.jenkins_tasks import trigger_jenkins_job
|
|
|
|
logger = structlog.get_logger()
|
|
|
|
|
|
class WebhookService:
|
|
"""Webhook 处理服务"""
|
|
|
|
def __init__(
|
|
self,
|
|
dedup_service: DeduplicationService,
|
|
jenkins_service: JenkinsService,
|
|
celery_app: Celery
|
|
):
|
|
self.dedup_service = dedup_service
|
|
self.jenkins_service = jenkins_service
|
|
self.celery_app = celery_app
|
|
self.settings = get_settings()
|
|
self.db_service = get_database_service()
|
|
|
|
async def process_webhook(self, webhook: GiteaWebhook) -> WebhookResponse:
|
|
"""
|
|
处理 Webhook 事件
|
|
|
|
Args:
|
|
webhook: Gitea Webhook 数据
|
|
|
|
Returns:
|
|
WebhookResponse: 处理结果
|
|
"""
|
|
try:
|
|
# 1. 验证事件类型
|
|
if not webhook.is_push_event():
|
|
return WebhookResponse(
|
|
success=True,
|
|
message="Non-push event ignored",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
# 2. 提取关键信息
|
|
branch = webhook.get_branch_name()
|
|
commit_hash = webhook.get_commit_hash()
|
|
repository = webhook.repository.full_name
|
|
|
|
logger.info("Processing webhook",
|
|
repository=repository,
|
|
branch=branch,
|
|
commit_hash=commit_hash)
|
|
|
|
# 3. 防抖检查
|
|
dedup_key = self.dedup_service.generate_dedup_key(commit_hash, branch)
|
|
if await self.dedup_service.is_duplicate(dedup_key):
|
|
return WebhookResponse(
|
|
success=True,
|
|
message="Duplicate event ignored",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
# 4. 获取项目映射和任务名
|
|
job_name = await self._determine_job_name(repository, branch)
|
|
if not job_name:
|
|
return WebhookResponse(
|
|
success=True,
|
|
message=f"No Jenkins job mapping for repository: {repository}, branch: {branch}",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
# 5. 准备任务参数
|
|
job_params = self._prepare_job_parameters(webhook, job_name)
|
|
|
|
# 6. 提交任务到队列
|
|
task_result = await self._submit_job_to_queue(
|
|
webhook, job_name, job_params
|
|
)
|
|
|
|
if task_result:
|
|
return WebhookResponse(
|
|
success=True,
|
|
message="Job queued successfully",
|
|
event_id=webhook.get_event_id(),
|
|
job_name=job_name
|
|
)
|
|
else:
|
|
return WebhookResponse(
|
|
success=False,
|
|
message="Failed to queue job",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error("Error processing webhook",
|
|
repository=webhook.repository.full_name,
|
|
error=str(e))
|
|
return WebhookResponse(
|
|
success=False,
|
|
message=f"Internal server error: {str(e)}",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
async def _determine_job_name(self, repository: str, branch: str) -> Optional[str]:
|
|
"""根据仓库和分支确定任务名"""
|
|
# 首先尝试从数据库获取项目映射
|
|
job_name = await self.db_service.determine_job_name(repository, branch)
|
|
if job_name:
|
|
return job_name
|
|
|
|
# 如果数据库中没有映射，使用配置文件中的环境分发
|
|
environment = self.settings.get_environment_for_branch(branch)
|
|
if environment:
|
|
return environment.jenkins_job
|
|
|
|
return None
|
|
|
|
def _prepare_job_parameters(self, webhook: GiteaWebhook, job_name: str) -> Dict[str, str]:
|
|
"""准备 Jenkins 任务参数"""
|
|
author_info = webhook.get_author_info()
|
|
|
|
return {
|
|
"BRANCH_NAME": webhook.get_branch_name(),
|
|
"COMMIT_SHA": webhook.get_commit_hash(),
|
|
"REPOSITORY_URL": webhook.repository.clone_url,
|
|
"REPOSITORY_NAME": webhook.repository.full_name,
|
|
"PUSHER_NAME": author_info["name"],
|
|
"PUSHER_EMAIL": author_info["email"],
|
|
"PUSHER_USERNAME": author_info["username"],
|
|
"COMMIT_MESSAGE": webhook.get_commit_message(),
|
|
"JOB_NAME": job_name,
|
|
"WEBHOOK_EVENT_ID": webhook.get_event_id(),
|
|
"TRIGGER_TIME": datetime.utcnow().isoformat()
|
|
}
|
|
|
|
async def _submit_job_to_queue(
|
|
self,
|
|
webhook: GiteaWebhook,
|
|
job_name: str,
|
|
job_params: Dict[str, str]
|
|
) -> bool:
|
|
"""提交任务到 Celery 队列"""
|
|
try:
|
|
# 创建任务
|
|
task_kwargs = {
|
|
"job_name": job_name,
|
|
"jenkins_url": self.settings.jenkins.url,
|
|
"parameters": job_params,
|
|
"event_id": webhook.get_event_id(),
|
|
"repository": webhook.repository.full_name,
|
|
"branch": webhook.get_branch_name(),
|
|
"commit_hash": webhook.get_commit_hash(),
|
|
"priority": 1 # 默认优先级
|
|
}
|
|
|
|
# 提交到 Celery 队列
|
|
task = self.celery_app.send_task(
|
|
"app.tasks.jenkins_tasks.trigger_jenkins_job",
|
|
kwargs=task_kwargs,
|
|
priority=environment.priority
|
|
)
|
|
|
|
logger.info("Job submitted to queue",
|
|
task_id=task.id,
|
|
job_name=job_name,
|
|
repository=webhook.repository.full_name,
|
|
branch=webhook.get_branch_name())
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to submit job to queue",
|
|
job_name=job_name,
|
|
error=str(e))
|
|
return False
|
|
|
|
async def get_webhook_stats(self) -> Dict[str, Any]:
|
|
"""获取 Webhook 处理统计"""
|
|
try:
|
|
# 获取队列统计
|
|
queue_stats = await self._get_queue_stats()
|
|
|
|
# 获取防抖统计
|
|
dedup_stats = await self.dedup_service.get_stats()
|
|
|
|
# 获取环境配置
|
|
environments = {}
|
|
for name, config in self.settings.environments.items():
|
|
environments[name] = {
|
|
"branches": config.branches,
|
|
"jenkins_job": config.jenkins_job,
|
|
"jenkins_url": config.jenkins_url,
|
|
"priority": config.priority
|
|
}
|
|
|
|
return {
|
|
"queue": queue_stats,
|
|
"deduplication": dedup_stats,
|
|
"environments": environments,
|
|
"config": {
|
|
"max_concurrent": self.settings.queue.max_concurrent,
|
|
"max_retries": self.settings.queue.max_retries,
|
|
"retry_delay": self.settings.queue.retry_delay
|
|
},
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Error getting webhook stats", error=str(e))
|
|
return {"error": str(e)}
|
|
|
|
async def _get_queue_stats(self) -> Dict[str, Any]:
|
|
"""获取队列统计信息"""
|
|
try:
|
|
# 获取 Celery 队列统计
|
|
inspect = self.celery_app.control.inspect()
|
|
|
|
# 活跃任务
|
|
active = inspect.active()
|
|
active_count = sum(len(tasks) for tasks in active.values()) if active else 0
|
|
|
|
# 等待任务
|
|
reserved = inspect.reserved()
|
|
reserved_count = sum(len(tasks) for tasks in reserved.values()) if reserved else 0
|
|
|
|
# 注册的 worker
|
|
registered = inspect.registered()
|
|
worker_count = len(registered) if registered else 0
|
|
|
|
return {
|
|
"active_tasks": active_count,
|
|
"queued_tasks": reserved_count,
|
|
"worker_count": worker_count,
|
|
"queue_length": active_count + reserved_count
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Error getting queue stats", error=str(e))
|
|
return {"error": str(e)}
|
|
|
|
async def clear_queue(self) -> Dict[str, Any]:
|
|
"""清空队列"""
|
|
try:
|
|
# 撤销所有活跃任务
|
|
inspect = self.celery_app.control.inspect()
|
|
active = inspect.active()
|
|
|
|
revoked_count = 0
|
|
if active:
|
|
for worker, tasks in active.items():
|
|
for task in tasks:
|
|
self.celery_app.control.revoke(task["id"], terminate=True)
|
|
revoked_count += 1
|
|
|
|
logger.info("Queue cleared", revoked_count=revoked_count)
|
|
|
|
return {
|
|
"success": True,
|
|
"revoked_count": revoked_count,
|
|
"message": f"Cleared {revoked_count} tasks from queue"
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Error clearing queue", error=str(e))
|
|
return {
|
|
"success": False,
|
|
"error": str(e)
|
|
} |