From 05faf32ef8ea25afd2ba3e98d6dc1babb827a489 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 7 Aug 2025 12:18:37 +0800 Subject: [PATCH] fix: update RabbitMQ password for all services to match reconciler configuration --- freeleaps/helm-pkg/devops/values.alpha.yaml | 2 +- freeleaps/helm-pkg/devsvc/values.alpha.yaml | 2 +- .../helm-pkg/notification/values.alpha.yaml | 2 +- .../argo-app/project.yaml | 54 +++++++ jobs/freeleaps-data-backup/ci/Jenkinsfile | 2 +- .../freeleaps-data-backup/values.prod.yaml | 7 +- jobs/freeleaps-data-backup/test_backup.py | 151 ++++++++++++++++++ .../freeleaps-data-backup/test_backup_safe.py | 81 ++++++++++ "sword to match original rabbitmq secret\"" | 13 ++ 9 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 jobs/freeleaps-data-backup/argo-app/project.yaml create mode 100644 jobs/freeleaps-data-backup/test_backup.py create mode 100644 jobs/freeleaps-data-backup/test_backup_safe.py create mode 100644 "sword to match original rabbitmq secret\"" diff --git a/freeleaps/helm-pkg/devops/values.alpha.yaml b/freeleaps/helm-pkg/devops/values.alpha.yaml index 058304c0..2c38aad7 100644 --- a/freeleaps/helm-pkg/devops/values.alpha.yaml +++ b/freeleaps/helm-pkg/devops/values.alpha.yaml @@ -62,7 +62,7 @@ devops: rabbitmqHost: freeleaps-alpha-rabbitmq.freeleaps-alpha.svc.freeleaps.cluster rabbitmqPort: 5672 rabbitmqUsername: user - rabbitmqPassword: vHR8Tm7WdFd6KqEM + rabbitmqPassword: NjlhHFvnDuC7K0ir rabbitmqVirtualHost: / vpa: minAllowed: diff --git a/freeleaps/helm-pkg/devsvc/values.alpha.yaml b/freeleaps/helm-pkg/devsvc/values.alpha.yaml index 34b1b764..b9023f51 100644 --- a/freeleaps/helm-pkg/devsvc/values.alpha.yaml +++ b/freeleaps/helm-pkg/devsvc/values.alpha.yaml @@ -94,7 +94,7 @@ devsvc: rabbitmqHost: freeleaps-alpha-rabbitmq.freeleaps-alpha.svc.freeleaps.cluster rabbitmqPort: 5672 rabbitmqUsername: user - rabbitmqPassword: vHR8Tm7WdFd6KqEM + rabbitmqPassword: NjlhHFvnDuC7K0ir rabbitmqVirtualHost: / defaultGitUsername: freeleaps defaultGitPassword: r8sA8CPHD9!bt6d diff --git a/freeleaps/helm-pkg/notification/values.alpha.yaml b/freeleaps/helm-pkg/notification/values.alpha.yaml index 80a1d888..ee98b010 100644 --- a/freeleaps/helm-pkg/notification/values.alpha.yaml +++ b/freeleaps/helm-pkg/notification/values.alpha.yaml @@ -86,7 +86,7 @@ notification: rabbitmqHost: freeleaps-alpha-rabbitmq.freeleaps-alpha.svc.freeleaps.cluster rabbitmqPort: 5672 rabbitmqUsername: user - rabbitmqPassword: vHR8Tm7WdFd6KqEM + rabbitmqPassword: NjlhHFvnDuC7K0ir rabbitmqVritualHost: / systemUserId: 117f191e810c19729de860aa smsFrom: '+16898887156' diff --git a/jobs/freeleaps-data-backup/argo-app/project.yaml b/jobs/freeleaps-data-backup/argo-app/project.yaml new file mode 100644 index 00000000..61ac5f87 --- /dev/null +++ b/jobs/freeleaps-data-backup/argo-app/project.yaml @@ -0,0 +1,54 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: freeleaps-data-backup + namespace: freeleaps-devops-system + labels: + app: freeleaps-data-backup + component: backup + environment: production +spec: + description: Freeleaps Data Backup Project + + # Source repositories + sourceRepos: + - https://freeleaps@dev.azure.com/freeleaps/freeleaps-ops/_git/freeleaps-ops + + # Destination clusters and namespaces + destinations: + - namespace: freeleaps-prod + server: https://kubernetes.default.svc + + # Allowed cluster resources + clusterResourceWhitelist: + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + + # Allowed namespaced resources + namespaceResourceWhitelist: + - group: "" + kind: ServiceAccount + - group: "" + kind: PersistentVolumeClaim + - group: batch + kind: CronJob + - group: batch + kind: Job + - group: snapshot.storage.k8s.io + kind: VolumeSnapshot + - group: snapshot.storage.k8s.io + kind: VolumeSnapshotClass + + # Allowed roles + roles: + - name: backup-admin + description: Backup administrator role + policies: + - p, proj:freeleaps-data-backup:backup-admin, applications, *, freeleaps-data-backup/*, allow + - p, proj:freeleaps-data-backup:backup-admin, applications, sync, freeleaps-data-backup/*, allow + - p, proj:freeleaps-data-backup:backup-admin, applications, update, freeleaps-data-backup/*, allow + - p, proj:freeleaps-data-backup:backup-admin, applications, delete, freeleaps-data-backup/*, allow + groups: + - freeleaps-devops \ No newline at end of file diff --git a/jobs/freeleaps-data-backup/ci/Jenkinsfile b/jobs/freeleaps-data-backup/ci/Jenkinsfile index eb82b762..a2a988d5 100644 --- a/jobs/freeleaps-data-backup/ci/Jenkinsfile +++ b/jobs/freeleaps-data-backup/ci/Jenkinsfile @@ -1,7 +1,7 @@ library 'first-class-pipeline' executeFreeleapsPipeline { - serviceName = 'jobs/freeleaps-data-backup' + serviceName = 'jobs' environmentSlug = 'prod' serviceGitBranch = 'master' serviceGitRepo = "https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-ops.git" diff --git a/jobs/freeleaps-data-backup/helm-pkg/freeleaps-data-backup/values.prod.yaml b/jobs/freeleaps-data-backup/helm-pkg/freeleaps-data-backup/values.prod.yaml index 344c7e89..2baca3d8 100644 --- a/jobs/freeleaps-data-backup/helm-pkg/freeleaps-data-backup/values.prod.yaml +++ b/jobs/freeleaps-data-backup/helm-pkg/freeleaps-data-backup/values.prod.yaml @@ -1,9 +1,14 @@ # Production values for freeleaps-data-backup +# Global settings +global: + imageRegistry: "freeleaps" + imagePullSecrets: [] + # Image settings image: repository: freeleaps-pvc-backup - tag: "latest" + tag: "1.0.1" pullPolicy: Always # CronJob settings diff --git a/jobs/freeleaps-data-backup/test_backup.py b/jobs/freeleaps-data-backup/test_backup.py new file mode 100644 index 00000000..6dd5e60f --- /dev/null +++ b/jobs/freeleaps-data-backup/test_backup.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +Test script for backup_script.py functionality +""" + +import os +import sys +import logging +from unittest.mock import Mock, patch, MagicMock +from backup_script import PVCBackupManager + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def test_pvc_verification(): + """Test PVC verification functionality""" + print("=== Testing PVC Verification ===") + + with patch('backup_script.client.CoreV1Api') as mock_core_api: + # Mock successful PVC check + mock_api_instance = Mock() + mock_core_api.return_value = mock_api_instance + mock_api_instance.read_namespaced_persistent_volume_claim.return_value = Mock() + + backup_manager = PVCBackupManager() + result = backup_manager.verify_pvc_exists("test-pvc") + print(f"PVC verification result: {result}") + assert result == True + +def test_snapshot_name_generation(): + """Test snapshot name generation""" + print("=== Testing Snapshot Name Generation ===") + + backup_manager = PVCBackupManager() + timestamp = backup_manager.get_pst_date() + snapshot_name = backup_manager.generate_snapshot_name("test-pvc", timestamp) + print(f"Generated snapshot name: {snapshot_name}") + assert "test-pvc-snapshot-" in snapshot_name + +def test_snapshot_yaml_creation(): + """Test snapshot YAML creation""" + print("=== Testing Snapshot YAML Creation ===") + + backup_manager = PVCBackupManager() + snapshot_yaml = backup_manager.create_snapshot_yaml("test-pvc", "test-snapshot") + print(f"Snapshot YAML: {snapshot_yaml}") + + assert snapshot_yaml["apiVersion"] == "snapshot.storage.k8s.io/v1" + assert snapshot_yaml["kind"] == "VolumeSnapshot" + assert snapshot_yaml["metadata"]["name"] == "test-snapshot" + assert snapshot_yaml["spec"]["source"]["persistentVolumeClaimName"] == "test-pvc" + +def test_full_backup_process(): + """Test full backup process with mocked APIs""" + print("=== Testing Full Backup Process ===") + + with patch('backup_script.client.CustomObjectsApi') as mock_snapshot_api, \ + patch('backup_script.client.CoreV1Api') as mock_core_api: + + # Mock API instances + mock_snapshot_instance = Mock() + mock_core_instance = Mock() + mock_snapshot_api.return_value = mock_snapshot_instance + mock_core_api.return_value = mock_core_instance + + # Mock PVC verification + mock_core_instance.read_namespaced_persistent_volume_claim.return_value = Mock() + + # Mock snapshot creation + mock_snapshot_instance.create_namespaced_custom_object.return_value = { + "metadata": {"name": "test-snapshot"} + } + + # Mock snapshot ready status + mock_snapshot_instance.get_namespaced_custom_object.return_value = { + "status": {"readyToUse": True} + } + + backup_manager = PVCBackupManager() + # Override PVCs for testing + backup_manager.pvcs_to_backup = ["test-pvc"] + backup_manager.timeout = 10 # Short timeout for testing + + result = backup_manager.run_backup() + print(f"Backup process result: {result}") + assert result == True + +def test_environment_variables(): + """Test environment variable configuration""" + print("=== Testing Environment Variables ===") + + # Set test environment variables + os.environ["BACKUP_NAMESPACE"] = "test-namespace" + os.environ["SNAPSHOT_CLASS"] = "test-snapshot-class" + os.environ["TIMEOUT"] = "600" + + backup_manager = PVCBackupManager() + print(f"Namespace: {backup_manager.namespace}") + print(f"Snapshot Class: {backup_manager.snapshot_class}") + print(f"Timeout: {backup_manager.timeout}") + + assert backup_manager.namespace == "test-namespace" + assert backup_manager.snapshot_class == "test-snapshot-class" + assert backup_manager.timeout == 600 + +def run_integration_test(): + """Run integration test with real cluster (optional)""" + print("=== Integration Test (Real Cluster) ===") + print("This test requires access to a real Kubernetes cluster") + print("Make sure you have kubeconfig configured") + + try: + backup_manager = PVCBackupManager() + print("Successfully initialized backup manager") + + # Test with a small timeout to avoid long waits + backup_manager.timeout = 30 + print("Ready to run backup (will timeout after 30 seconds)") + + # Uncomment the following line to run actual backup + # result = backup_manager.run_backup() + # print(f"Integration test result: {result}") + + except Exception as e: + print(f"Integration test failed: {e}") + +def main(): + """Run all tests""" + print("Starting backup script tests...") + + try: + test_pvc_verification() + test_snapshot_name_generation() + test_snapshot_yaml_creation() + test_full_backup_process() + test_environment_variables() + + print("\n=== All Unit Tests Passed ===") + + # Ask user if they want to run integration test + response = input("\nDo you want to run integration test with real cluster? (y/N): ") + if response.lower() == 'y': + run_integration_test() + + except Exception as e: + print(f"Test failed: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/jobs/freeleaps-data-backup/test_backup_safe.py b/jobs/freeleaps-data-backup/test_backup_safe.py new file mode 100644 index 00000000..835cfdb8 --- /dev/null +++ b/jobs/freeleaps-data-backup/test_backup_safe.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +Safe test script for backup_script.py functionality +This script will NOT create real snapshots to avoid affecting production data +""" + +import os +import sys +import logging +from unittest.mock import Mock, patch +from backup_script import PVCBackupManager + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +class SafePVCBackupManager(PVCBackupManager): + """Safe version that doesn't create real snapshots""" + + def __init__(self, test_mode=True): + super().__init__() + self.test_mode = test_mode + if test_mode: + logger.info("Running in TEST MODE - no real snapshots will be created") + + def generate_snapshot_name(self, pvc_name, timestamp): + """Add TEST prefix to avoid conflicts""" + if self.test_mode: + return f"TEST-{pvc_name}-snapshot-{timestamp}" + return super().generate_snapshot_name(pvc_name, timestamp) + + def apply_snapshot(self, snapshot_yaml): + """Mock snapshot creation in test mode""" + if self.test_mode: + logger.info(f"[TEST MODE] Would create snapshot: {snapshot_yaml['metadata']['name']}") + return {"metadata": {"name": snapshot_yaml['metadata']['name']}} + return super().apply_snapshot(snapshot_yaml) + +def test_dry_run(): + """Show what would be done without making changes""" + print("=== Dry Run Test ===") + + backup_manager = SafePVCBackupManager(test_mode=True) + + print("Configuration:") + print(f" Namespace: {backup_manager.namespace}") + print(f" PVCs: {backup_manager.pvcs_to_backup}") + print(f" Snapshot class: {backup_manager.snapshot_class}") + + print("\nWould create snapshots:") + for pvc in backup_manager.pvcs_to_backup: + timestamp = backup_manager.get_pst_date() + snapshot_name = backup_manager.generate_snapshot_name(pvc, timestamp) + print(f" {snapshot_name}") + +def test_yaml_generation(): + """Test YAML generation without applying""" + print("=== YAML Generation Test ===") + + backup_manager = SafePVCBackupManager(test_mode=True) + + for pvc in backup_manager.pvcs_to_backup: + timestamp = backup_manager.get_pst_date() + snapshot_name = backup_manager.generate_snapshot_name(pvc, timestamp) + snapshot_yaml = backup_manager.create_snapshot_yaml(pvc, snapshot_name) + + print(f"\nYAML for {pvc}:") + import yaml + print(yaml.dump(snapshot_yaml, default_flow_style=False)) + +def main(): + """Run safe tests""" + print("Starting SAFE backup tests...") + print("No real snapshots will be created") + + test_dry_run() + test_yaml_generation() + + print("\n=== Safe tests completed ===") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git "a/sword to match original rabbitmq secret\"" "b/sword to match original rabbitmq secret\"" new file mode 100644 index 00000000..c2a18591 --- /dev/null +++ "b/sword to match original rabbitmq secret\"" @@ -0,0 +1,13 @@ +diff --git a/freeleaps-devops-reconciler/helm-pkg/reconciler/values.alpha.yaml b/freeleaps-devops-reconciler/helm-pkg/reconciler/values.alpha.yaml +index d08e634b..81a740b7 100644 +--- a/freeleaps-devops-reconciler/helm-pkg/reconciler/values.alpha.yaml ++++ b/freeleaps-devops-reconciler/helm-pkg/reconciler/values.alpha.yaml +@@ -105,7 +105,7 @@ env: + secrets: + rabbitmqCredentials: + username: user +- password: vHR8Tm7WdFd6KqEM ++ password: NjlhHFvnDuC7K0ir + jenkinsCredentials: + username: admin + token: 119fe346a7d5e1fc7f9ed4d98eac3e73ee