diff --git a/.github/workflows/sync-db.yml b/.github/workflows/sync-db.yml index 419bfee..8146519 100644 --- a/.github/workflows/sync-db.yml +++ b/.github/workflows/sync-db.yml @@ -139,29 +139,48 @@ jobs: sleep 15 # Find the job created by k8up for this restore - for i in {1..30}; do - JOB_NAME=$(kubectl get jobs -n default -l k8up.io/owned-by=restore -o jsonpath='{.items[?(@.metadata.ownerReferences[0].name=="'$RESTORE_NAME'")].metadata.name}' 2>/dev/null) + # k8up creates jobs with name pattern "restore-" + # Since our restore is named "restore-from-prod-*", the job will be "restore-restore-from-prod-*" + for i in {1..60}; do + JOB_NAME=$(kubectl get jobs -n default --no-headers 2>/dev/null | grep "^restore-$RESTORE_NAME" | awk '{print $1}' | head -1) if [ -n "$JOB_NAME" ]; then echo "Found restore job: $JOB_NAME" break fi - echo "Waiting for job to be created... ($i/30)" + echo "Waiting for job to be created... ($i/60)" sleep 2 done if [ -z "$JOB_NAME" ]; then - echo "ERROR: Restore job not found" - kubectl get restore $RESTORE_NAME -n default -o yaml + echo "ERROR: Restore job not found after 120 seconds" + echo "Checking restore resource status:" + kubectl get restore $RESTORE_NAME -n default + kubectl describe restore $RESTORE_NAME -n default + + echo "Checking for any restore jobs:" + kubectl get jobs -n default | grep restore || echo "No restore jobs found" + + echo "Checking k8up operator logs:" + kubectl logs deployment/k8up -n default --tail=50 | grep -i restore || echo "No restore logs found" exit 1 fi # Wait for the restore job to complete (max 15 minutes) + echo "Waiting for restore job to complete..." kubectl wait --for=condition=complete \ job/$JOB_NAME \ --timeout=900s -n default || { echo "Restore job failed or timed out" + echo "Job status:" + kubectl get job/$JOB_NAME -n default + echo "Job details:" kubectl describe job/$JOB_NAME -n default - kubectl logs job/$JOB_NAME -n default --tail=100 + echo "Job logs (if available):" + kubectl logs job/$JOB_NAME -n default --tail=100 || echo "No logs available" + + # Check if it's a credential issue + echo "Checking if credentials exist:" + kubectl get secret prod-to-staging-sync-credentials -n default || echo "Credentials secret missing!" exit 1 } @@ -370,11 +389,36 @@ jobs: - name: Cleanup if: always() run: | + # Wait for any restore jobs to finish before cleanup (max 5 minutes for large database restores) + echo "Checking for running restore jobs..." + RESTORE_NAME="${{ steps.restore.outputs.restore_name }}" + + for i in {1..60}; do + # Check specifically for our restore job in Running or Pending state + RUNNING_JOBS=$(kubectl get jobs -n default --no-headers 2>/dev/null | grep "^restore-${RESTORE_NAME}" | grep -E "Running|Pending" | wc -l) + if [ "$RUNNING_JOBS" -eq 0 ]; then + echo "No running restore jobs found" + break + fi + echo "Waiting for $RUNNING_JOBS restore job(s) to finish... ($i/60)" + sleep 5 + done + # Clean up jobs first if [ -n "${{ steps.copy-job.outputs.job_name }}" ]; then kubectl delete job ${{ steps.copy-job.outputs.job_name }} -n default || true fi + # Clean up the restore job if it exists (with proper error handling) + if [ -n "$RESTORE_NAME" ]; then + if kubectl get job "restore-$RESTORE_NAME" -n default >/dev/null 2>&1; then + echo "Deleting restore job: restore-$RESTORE_NAME" + kubectl delete job "restore-$RESTORE_NAME" -n default || echo "Failed to delete job, may have already been cleaned up" + else + echo "Restore job restore-$RESTORE_NAME not found, skipping deletion" + fi + fi + # Remove restore PVC (will wait for jobs to finish) kubectl delete pvc restore-data-pvc -n default || true @@ -384,5 +428,8 @@ jobs: # Clean up old restore resources (keep last 3) kubectl get restore -n default --sort-by=.metadata.creationTimestamp -o name | head -n -3 | xargs -r kubectl delete || true + # Clean up old restore jobs (keep last 3) + kubectl get jobs -n default --sort-by=.metadata.creationTimestamp -o name | grep '^job.batch/restore-' | head -n -3 | xargs -r kubectl delete -n default || true + # Clean up old copy jobs (keep last 3) kubectl get jobs -n default --sort-by=.metadata.creationTimestamp -o name | grep 'copy-pgdata-' | head -n -3 | xargs -r kubectl delete -n default || true