diff --git a/jenkins/scripts/cleanup-resources.sh b/jenkins/scripts/cleanup-resources.sh new file mode 100755 index 00000000..06f53a93 --- /dev/null +++ b/jenkins/scripts/cleanup-resources.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash +# Script to cleanup temporary files and resources + +set -o errexit +set -o nounset +set -o pipefail + +# Configuration +WORK_DIR="${WORK_DIR:-/data/centos-images}" +KEEP_METADATA="${KEEP_METADATA:-true}" +KEEP_LOGS="${KEEP_LOGS:-true}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Display disk usage before cleanup +show_disk_usage_before() { + log_info "Disk usage before cleanup:" + if [ -d "$WORK_DIR" ]; then + du -sh "$WORK_DIR" 2>/dev/null || log_warn "Could not calculate disk usage" + log_info "Files in work directory:" + ls -lh "$WORK_DIR" 2>/dev/null || log_warn "Could not list files" + else + log_info "Work directory does not exist: $WORK_DIR" + fi +} + +# Cleanup qcow2 files +cleanup_qcow2_files() { + log_info "Cleaning up qcow2 files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + for file in "$WORK_DIR"/*.qcow2; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + count=$((count + 1)) + fi + done + + # Also remove checksum files + for file in "$WORK_DIR"/*.SHA256SUM; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + count=$((count + 1)) + fi + done + fi + + log_info "Removed $count qcow2 and checksum files" +} + +# Cleanup OVA files +cleanup_ova_files() { + log_info "Cleaning up OVA files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + for file in "$WORK_DIR"/*.ova.gz "$WORK_DIR"/*.ova; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + count=$((count + 1)) + fi + done + fi + + log_info "Removed $count OVA files" +} + +# Cleanup temporary files +cleanup_temp_files() { + log_info "Cleaning up temporary files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + # Remove temp directories created by pvsadm + for dir in "$WORK_DIR"/tmp* "$WORK_DIR"/pvsadm-*; do + if [ -d "$dir" ]; then + log_info "Removing directory: $(basename "$dir")" + rm -rf "$dir" + count=$((count + 1)) + fi + done + + # Remove other temporary files + for file in "$WORK_DIR"/*.tmp "$WORK_DIR"/*.temp; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + count=$((count + 1)) + fi + done + fi + + log_info "Removed $count temporary files/directories" +} + +# Cleanup logs (optional) +cleanup_logs() { + if [ "$KEEP_LOGS" == "false" ]; then + log_info "Cleaning up log files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + for file in "$WORK_DIR"/*.log; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + count=$((count + 1)) + fi + done + fi + + log_info "Removed $count log files" + else + log_info "Keeping log files (KEEP_LOGS=true)" + fi +} + +# Cleanup metadata (optional) +cleanup_metadata() { + if [ "$KEEP_METADATA" == "false" ]; then + log_info "Cleaning up metadata files..." + + local count=0 + if [ -d "$WORK_DIR" ]; then + for file in "$WORK_DIR"/*.json; do + if [ -f "$file" ]; then + log_info "Removing: $(basename "$file")" + rm -f "$file" + count=$((count + 1)) + fi + done + fi + + log_info "Removed $count metadata files" + else + log_info "Keeping metadata files (KEEP_METADATA=true)" + fi +} + +# Archive important files before cleanup +archive_important_files() { + log_info "Archiving important files..." + + if [ -d "$WORK_DIR" ]; then + local archive_dir="${WORK_DIR}/archive" + mkdir -p "$archive_dir" + + # Archive metadata + for file in "$WORK_DIR"/*.json; do + if [ -f "$file" ]; then + cp "$file" "$archive_dir/" 2>/dev/null || true + fi + done + + # Archive logs + for file in "$WORK_DIR"/*.log; do + if [ -f "$file" ]; then + cp "$file" "$archive_dir/" 2>/dev/null || true + fi + done + + # Create tarball + if [ -d "$archive_dir" ] && [ "$(ls -A "$archive_dir")" ]; then + local timestamp=$(date +%Y%m%d-%H%M%S) + local archive_file="${WORK_DIR}/centos-image-automation-${timestamp}.tar.gz" + + tar -czf "$archive_file" -C "$WORK_DIR" archive/ + log_info "Created archive: $archive_file" + + # Remove archive directory + rm -rf "$archive_dir" + fi + fi +} + +# Display disk usage after cleanup +show_disk_usage_after() { + log_info "Disk usage after cleanup:" + if [ -d "$WORK_DIR" ]; then + du -sh "$WORK_DIR" 2>/dev/null || log_warn "Could not calculate disk usage" + + if [ "$(ls -A "$WORK_DIR" 2>/dev/null)" ]; then + log_info "Remaining files:" + ls -lh "$WORK_DIR" 2>/dev/null || true + else + log_info "Work directory is empty" + fi + else + log_info "Work directory does not exist: $WORK_DIR" + fi +} + +# Remove work directory completely (optional) +remove_work_dir() { + if [ "${REMOVE_WORK_DIR:-false}" == "true" ]; then + log_warn "Removing entire work directory: $WORK_DIR" + rm -rf "$WORK_DIR" + log_info "Work directory removed" + fi +} + +# Remove sibling work directories under /data that are older than 30 days. +# Pattern: /data/centos-images-* (created by this pipeline for each version/env cell) +cleanup_stale_work_dirs() { + local base_dir + base_dir=$(dirname "$WORK_DIR") + local stale_days=30 + + log_info "Checking for stale work directories older than ${stale_days} days under ${base_dir}..." + + local count=0 + while IFS= read -r -d '' stale_dir; do + log_warn "Removing stale work directory (>${stale_days} days old): $stale_dir" + rm -rf "$stale_dir" + count=$((count + 1)) + done < <(find "$base_dir" -maxdepth 1 -type d -name "centos-images-*" \ + -not -path "$WORK_DIR" \ + -mtime "+${stale_days}" \ + -print0 2>/dev/null) + + if [ "$count" -gt 0 ]; then + log_info "Removed $count stale work director(ies)" + else + log_info "No stale work directories found" + fi +} + +# Main execution +main() { + log_info "Starting cleanup process" + log_info "Work directory: $WORK_DIR" + log_info "Keep metadata: $KEEP_METADATA" + log_info "Keep logs: $KEEP_LOGS" + + # Show disk usage before + show_disk_usage_before + + echo "" + + # Archive important files first + archive_important_files + + # Cleanup in order + cleanup_qcow2_files + cleanup_ova_files + cleanup_temp_files + cleanup_logs + cleanup_metadata + + # Remove other pipeline work dirs older than 30 days + cleanup_stale_work_dirs + + echo "" + + # Show disk usage after + show_disk_usage_after + + # Optionally remove entire work directory + remove_work_dir + + log_info "==========================================" + log_info "Cleanup completed successfully!" + log_info "==========================================" +} + +# Run main function +main "$@" + + diff --git a/jenkins/scripts/download-and-convert-centos.sh b/jenkins/scripts/download-and-convert-centos.sh new file mode 100755 index 00000000..c6c19937 --- /dev/null +++ b/jenkins/scripts/download-and-convert-centos.sh @@ -0,0 +1,481 @@ +#!/usr/bin/env bash +# Script to download latest CentOS Stream image and convert to OVA using pvsadm +# This script runs directly on a PowerVM with ppc64le architecture + +set -o errexit +set -o nounset +set -o pipefail + +# Configuration +CENTOS_VERSION="${CENTOS_VERSION:-10}" +WORK_DIR="${WORK_DIR:-/data/centos-images}" +IMAGE_PREP_TEMPLATE="${IMAGE_PREP_TEMPLATE:-./image-prep.template.static}" +IMAGE_SIZE="${IMAGE_SIZE:-120}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" >&2 +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Check if running on ppc64le +check_architecture() { + local arch=$(uname -m) + if [[ "$arch" != "ppc64le" ]]; then + log_error "This script must run on ppc64le architecture. Current: $arch" + exit 1 + fi + log_info "Architecture check passed: $arch" +} + +# Check required tools +check_requirements() { + local missing_tools=() + local need_install=() + + # Check which tools are missing + for tool in curl jq; do + if ! command -v $tool &> /dev/null; then + missing_tools+=($tool) + need_install+=($tool) + fi + done + + # Check wget or curl (at least one needed) + if ! command -v wget &> /dev/null && ! command -v curl &> /dev/null; then + missing_tools+=(wget) + need_install+=(wget) + fi + + # Check qemu-img (required by pvsadm) + if ! command -v qemu-img &> /dev/null; then + missing_tools+=(qemu-img) + need_install+=(qemu-img) + fi + + # Check pvsadm separately + if ! command -v pvsadm &> /dev/null; then + missing_tools+=(pvsadm) + fi + + if [ ${#missing_tools[@]} -ne 0 ]; then + log_warn "Missing tools: ${missing_tools[*]}" + log_info "Installing missing tools..." + + # Install system packages first + if [ ${#need_install[@]} -ne 0 ]; then + log_info "Installing system packages: ${need_install[*]}" + if command -v dnf &> /dev/null; then + sudo dnf install -y "${need_install[@]}" || log_warn "Some packages failed to install" + elif command -v yum &> /dev/null; then + sudo yum install -y "${need_install[@]}" || log_warn "Some packages failed to install" + else + log_error "No package manager found (yum/dnf). Please install: ${need_install[*]}" + exit 1 + fi + fi + + # Install pvsadm if missing + if [[ " ${missing_tools[*]} " =~ " pvsadm " ]]; then + install_pvsadm + fi + fi + + log_info "All required tools are available" +} + +# Install pvsadm +install_pvsadm() { + log_info "Installing pvsadm..." + local PVSADM_VERSION="v0.1.13" + local ARCH="ppc64le" + local DOWNLOAD_URL="https://github.com/ppc64le-cloud/pvsadm/releases/download/${PVSADM_VERSION}/pvsadm-linux-${ARCH}.tar.gz" + local TAR_FILE="pvsadm-linux-${ARCH}.tar.gz" + + # Try wget first, fall back to curl + if command -v wget &> /dev/null; then + wget -q "$DOWNLOAD_URL" -O "$TAR_FILE" + elif command -v curl &> /dev/null; then + curl -sL "$DOWNLOAD_URL" -o "$TAR_FILE" + else + log_error "Neither wget nor curl is available. Please install one of them." + exit 1 + fi + + tar -xzf "$TAR_FILE" + sudo mv pvsadm /usr/local/bin/ 2>/dev/null || mv pvsadm /usr/local/bin/ + rm -f "$TAR_FILE" + + pvsadm version + log_info "pvsadm installed successfully" +} + +# Check disk space +check_disk_space() { + local required_space_gb=170 + local available_space_gb=$(df -BG "$WORK_DIR" | awk 'NR==2 {print $4}' | sed 's/G//') + + if [ "$available_space_gb" -lt "$required_space_gb" ]; then + log_error "Insufficient disk space. Required: ${required_space_gb}GB, Available: ${available_space_gb}GB" + exit 1 + fi + + log_info "Disk space check passed: ${available_space_gb}GB available" +} + +# Get latest CentOS Stream image URL +get_latest_image_url() { + log_info "Detecting latest CentOS Stream ${CENTOS_VERSION} image..." + + local base_url="https://cloud.centos.org/centos/${CENTOS_VERSION}-stream/ppc64le/images" + + # Get the latest image filename + local latest_image=$(curl -s "$base_url/" | \ + grep -oP 'CentOS-Stream-GenericCloud-'"${CENTOS_VERSION}"'-[0-9]{8}\.[0-9]+\.ppc64le\.qcow2' | \ + sort -V | tail -1) + + if [ -z "$latest_image" ]; then + log_error "Could not detect latest CentOS Stream ${CENTOS_VERSION} image" + exit 1 + fi + + echo "${base_url}/${latest_image}" +} + +# Download CentOS image +download_image() { + local image_url="$1" + local image_name=$(basename "$image_url") + local image_path="${WORK_DIR}/${image_name}" + + # Extract date from image name to check for existing OVA + local date_part=$(echo "$image_name" | grep -oP '\d{8}' | head -1) + local ova_pattern="centos-${CENTOS_VERSION}-stream-${date_part}*.ova.gz" + + # Check if OVA already exists - if so, skip download entirely + local existing_ova=$(find "$WORK_DIR" -name "$ova_pattern" -type f 2>/dev/null | head -1) + if [ -n "$existing_ova" ] && [ -f "$existing_ova" ]; then + log_info "==========================================" + log_info "OVA already exists: $(basename "$existing_ova")" + log_info "Skipping download and conversion" + log_info "To force re-download, delete: $existing_ova" + log_info "==========================================" + # Return a dummy path since we won't use it + echo "$image_path" + return 0 + fi + + # Check if qcow2 image already exists and is valid + if [ -f "$image_path" ]; then + local file_size=$(stat -f%z "$image_path" 2>/dev/null || stat -c%s "$image_path" 2>/dev/null) + if [ "$file_size" -gt 100000000 ]; then # > 100MB, likely valid + log_info "==========================================" + log_info "Image already exists: $image_name" + log_info "Size: $(du -h "$image_path" | cut -f1)" + log_info "Skipping download to save time" + log_info "To force re-download, delete: $image_path" + log_info "==========================================" + echo "$image_path" + return 0 + else + log_warn "Existing file is too small, re-downloading..." + rm -f "$image_path" + fi + fi + + log_info "Downloading image: $image_name" + log_info "URL: $image_url" + + # Download with progress - try wget first, fall back to curl + if command -v wget &> /dev/null; then + log_info "Using wget for download..." + wget -c -O "$image_path" "$image_url" 2>&1 | \ + grep --line-buffered -oP '\d+%' | \ + awk 'BEGIN {last=""} {if ($1 ~ /^(20|40|60|80|100)%$/ && $1 != last) {print "[INFO] Download progress: " $1; last=$1}}' >&2 + elif command -v curl &> /dev/null; then + log_info "Using curl for download (this may take 5-10 minutes)..." + curl -L -C - -o "$image_path" "$image_url" 2>&1 | \ + grep --line-buffered -oP '\d+\.\d+' | \ + awk 'BEGIN {last=-1} {p=int($1); if (p>=20 && p%20==0 && p!=last) {print "[INFO] Download progress: " p "%"; last=p}}' >&2 + else + log_error "Neither wget nor curl is available" + exit 1 + fi + + # Check if download succeeded + if [ ! -f "$image_path" ]; then + log_error "Download failed: $image_path does not exist" + exit 1 + fi + + log_info "Download completed: $(du -h "$image_path" | cut -f1)" + + # Download checksum if available + local checksum_url="${image_url}.SHA256SUM" + local checksum_exists=false + + if command -v wget &> /dev/null; then + wget -q --spider "$checksum_url" 2>&1 && checksum_exists=true + elif command -v curl &> /dev/null; then + curl -s -I "$checksum_url" 2>&1 | grep -q "200 OK" && checksum_exists=true + fi + + if [ "$checksum_exists" = true ]; then + log_info "Downloading checksum..." + if command -v wget &> /dev/null; then + wget -q -O "${image_path}.SHA256SUM" "$checksum_url" >&2 + else + curl -sL -o "${image_path}.SHA256SUM" "$checksum_url" >&2 + fi + + # Verify checksum + log_info "Verifying checksum..." + cd "$WORK_DIR" + if sha256sum -c "${image_name}.SHA256SUM" 2>&1 | grep -q "OK"; then + log_info "Checksum verification passed" + else + log_warn "Checksum verification failed or not available" + fi + cd - >/dev/null + else + log_warn "Checksum file not available, skipping verification" + fi + + echo "$image_path" +} + +# Convert qcow2 to OVA using pvsadm +convert_to_ova() { + local qcow2_path="$1" + local image_name=$(basename "$qcow2_path" .qcow2) + + # Extract date from image name (e.g., CentOS-Stream-GenericCloud-10-20251215.0.ppc64le.qcow2) + local date_part=$(echo "$image_name" | grep -oP '\d{8}' | head -1) + local ova_name="centos-${CENTOS_VERSION}-stream-${date_part}" + + # Check if OVA already exists + local existing_ova=$(find "$WORK_DIR" -name "${ova_name}*.ova.gz" -type f 2>/dev/null | head -1) + if [ -n "$existing_ova" ] && [ -f "$existing_ova" ]; then + log_info "==========================================" + log_info "OVA already exists: $(basename "$existing_ova")" + log_info "Skipping conversion to save time" + log_info "To force reconversion, delete: $existing_ova" + log_info "==========================================" + echo "$existing_ova" + return 0 + fi + + log_info "Converting image to OVA format..." + log_info "Image name: $ova_name" + log_info "Source: $qcow2_path" + + # Check if prep template exists + if [ ! -f "$IMAGE_PREP_TEMPLATE" ]; then + log_error "Image prep template not found: $IMAGE_PREP_TEMPLATE" + exit 1 + fi + + # Ensure loop devices and /tmp are clean before calling pvsadm. + # pvsadm uses /tmp by default and calls losetup without sudo โ€” this worked + # fine before; don't change either of those behaviours. + sudo losetup -D 2>/dev/null || true + sudo find /tmp -maxdepth 1 -type d -name "qcow2ova*" -exec rm -rf {} + 2>/dev/null || true + + # Verify at least one free loop device exists before handing off to pvsadm + if ! losetup -f &>/dev/null; then + log_error "No free loop device available even after losetup -D. Check kernel loop module." + losetup -a >&2 || true + exit 1 + fi + log_info "Loop device check passed: $(losetup -f)" + + # Run pvsadm conversion (no --temp-dir: use /tmp default, same as before) + cd "$WORK_DIR" + + log_info "Running pvsadm qcow2ova..." + log_info "This may take 30-60 minutes..." + + set +e # Temporarily disable errexit to handle error properly + pvsadm image qcow2ova \ + --image-name "$ova_name" \ + --image-url "$qcow2_path" \ + --image-dist centos \ + --prep-template "$IMAGE_PREP_TEMPLATE" \ + --skip-os-password \ + --image-size "$IMAGE_SIZE" \ + 2>&1 | tee "${WORK_DIR}/conversion.log" >&2 + + local pvsadm_exit_code=${PIPESTATUS[0]} + set -e # Re-enable errexit + + if [ $pvsadm_exit_code -ne 0 ]; then + log_error "==========================================" + log_error "pvsadm conversion FAILED with exit code: $pvsadm_exit_code" + log_error "==========================================" + + if [ -f "${WORK_DIR}/conversion.log" ]; then + log_error "Last 30 lines of conversion log:" + tail -30 "${WORK_DIR}/conversion.log" >&2 + fi + + log_error "Common issues:" + log_error " - Loop device exhaustion: Run 'sudo losetup -D' and 'sudo rm -rf /tmp/qcow2ova*'" + log_error " - Disk space on /tmp: Run 'df -h /tmp'" + log_error " - Permissions: Ensure the jenkins user can run losetup" + + exit 1 + fi + + log_info "pvsadm conversion completed successfully" + + # pvsadm writes the OVA into its /tmp/qcow2ova/ scratch dir, + # then moves it into WORK_DIR. Search both to be safe. + local ova_file + ova_file=$(find "$WORK_DIR" /tmp -maxdepth 3 -name "${ova_name}*.ova.gz" -type f 2>/dev/null | head -1) + + if [ -z "$ova_file" ] || [ ! -f "$ova_file" ]; then + log_error "OVA file not found after conversion" + log_error "Expected pattern: ${ova_name}*.ova.gz" + log_error "Files in work directory:" + ls -lh "$WORK_DIR" >&2 + exit 1 + fi + + # If pvsadm left the OVA in /tmp rather than WORK_DIR, move it over + if [[ "$ova_file" != "$WORK_DIR"/* ]]; then + log_info "Moving OVA from /tmp to WORK_DIR..." + mv "$ova_file" "$WORK_DIR/" + ova_file="${WORK_DIR}/$(basename "$ova_file")" + fi + + log_info "Conversion completed successfully" + log_info "OVA file: $ova_file" + + # Clean up pvsadm /tmp scratch dirs now that OVA is safely in WORK_DIR + log_info "Cleaning up pvsadm /tmp scratch directories..." + sleep 2 + sudo find /tmp -maxdepth 1 -type d -name "qcow2ova*" -exec rm -rf {} + 2>/dev/null || true + + # Clean up qcow2 file if OVA exists (saves ~1-2GB) + if [ -f "$ova_file" ] && [ -f "$qcow2_path" ]; then + log_info "Removing qcow2 file to save space (OVA already created)..." + rm -f "$qcow2_path" + fi + + # Save metadata + cat > "${WORK_DIR}/image-metadata.json" </dev/null | grep -q '[p]vsadm'; then + ps aux | grep '[p]vsadm' | awk '{print $2}' | while read pid; do + if [ -n "$pid" ]; then + local start_time + start_time=$(ps -p "$pid" -o etimes= 2>/dev/null | tr -d ' ') + if [ -n "$start_time" ] && [ "$start_time" -gt 14400 ]; then + log_warn "Killing old pvsadm process: $pid (running for ${start_time}s)" + kill -9 "$pid" 2>/dev/null || true + fi + fi + done + fi + + # Remove any stale pvsadm /tmp scratch dirs from prior runs + local stale_dirs + stale_dirs=$(find /tmp -maxdepth 1 -type d -name "qcow2ova*" 2>/dev/null | wc -l) + if [ "$stale_dirs" -gt 0 ]; then + log_info "Removing $stale_dirs stale /tmp/qcow2ova* directories..." + sudo find /tmp -maxdepth 1 -type d -name "qcow2ova*" -exec rm -rf {} \; 2>/dev/null || true + fi + + # Release any stuck loop devices + log_info "Cleaning up loop devices..." + if command -v losetup &>/dev/null; then + sudo losetup -D 2>/dev/null || losetup -D 2>/dev/null || true + log_info "Loop devices released" + fi + + sleep 2 +} + +# Main execution +main() { + log_info "Starting CentOS Stream ${CENTOS_VERSION} image download and conversion" + log_info "Work directory: $WORK_DIR" + + # Create work directory + mkdir -p "$WORK_DIR" + cd "$WORK_DIR" + + # Cleanup old pvsadm artifacts first + cleanup_old_pvsadm + + # Pre-flight checks + check_architecture + check_requirements + check_disk_space + + # Get latest image URL + local image_url + image_url=$(get_latest_image_url) + log_info "Latest image URL: $image_url" + + # Download image + local qcow2_path + qcow2_path=$(download_image "$image_url") + log_info "Downloaded to: $qcow2_path" + + # Convert to OVA + local ova_file + ova_file=$(convert_to_ova "$qcow2_path") + + if [ -z "$ova_file" ] || [ ! -f "$ova_file" ]; then + log_error "Conversion returned empty or missing OVA path: '$ova_file'" + exit 1 + fi + + log_info "==========================================" + log_info "Process completed successfully!" + log_info "OVA file: $ova_file" + log_info "Metadata: ${WORK_DIR}/image-metadata.json" + log_info "==========================================" + + # Display file sizes (only show files that exist) + log_info "File sizes:" + if [ -f "$qcow2_path" ]; then + ls -lh "$qcow2_path" 2>/dev/null || true + fi + ls -lh "$ova_file" 2>/dev/null || true +} + +# Run main function +main "$@" + + diff --git a/jenkins/scripts/import-to-powervs.sh b/jenkins/scripts/import-to-powervs.sh new file mode 100755 index 00000000..16ee089a --- /dev/null +++ b/jenkins/scripts/import-to-powervs.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# Script to import OVA image from COS to PowerVS workspace + +set -o errexit +set -o nounset + +# Configuration +WORK_DIR="${WORK_DIR:-/data/centos-images}" +IBM_API_KEY="${IBM_API_KEY:?IBM_API_KEY environment variable is required}" # pragma: allowlist secret +POWERVS_WORKSPACE_CRN="${POWERVS_WORKSPACE_CRN:?POWERVS_WORKSPACE_CRN environment variable is required}" +COS_BUCKET_NAME="${COS_BUCKET_NAME:?COS_BUCKET_NAME environment variable is required}" +COS_REGION="${COS_REGION:-us-south}" +COS_HMAC_ACCESS_KEY="${COS_HMAC_ACCESS_KEY:?COS_HMAC_ACCESS_KEY environment variable is required}" +COS_HMAC_SECRET_KEY="${COS_HMAC_SECRET_KEY:?COS_HMAC_SECRET_KEY environment variable is required}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" >&2 +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Check required tools +check_requirements() { + local missing_tools=() + + for tool in pvsadm jq; do + if ! command -v $tool &> /dev/null; then + missing_tools+=($tool) + fi + done + + if [ ${#missing_tools[@]} -ne 0 ]; then + log_error "Missing required tools: ${missing_tools[*]}" + exit 1 + fi + + log_info "All required tools are available" +} + +# Get image metadata +get_image_metadata() { + local metadata_file="${WORK_DIR}/image-metadata.json" + + if [ ! -f "$metadata_file" ]; then + log_error "Metadata file not found: $metadata_file" + exit 1 + fi + + cat "$metadata_file" +} + +# Import image to PowerVS +import_image() { + local image_name="$1" + local cos_object_name="$2" + + # Extract instance GUID from CRN for --pvs-instance-id + # CRN format: crn:v1:bluemix:public:power-iaas:region:a/account:GUID:: + local pvs_instance_id + pvs_instance_id=$(echo "$POWERVS_WORKSPACE_CRN" | awk -F: '{print $8}') + + export IBMCLOUD_API_KEY="$IBM_API_KEY" + + log_info "Importing image to PowerVS workspace" + log_info "Workspace CRN : $POWERVS_WORKSPACE_CRN" + log_info "Instance GUID : $pvs_instance_id" + log_info "Image name : $image_name" + log_info "COS object : $cos_object_name" + log_info "COS bucket : $COS_BUCKET_NAME" + log_info "COS region : $COS_REGION" + + log_info "Running: pvsadm image import --pvs-instance-id $pvs_instance_id --bucket $COS_BUCKET_NAME --bucket-region $COS_REGION --object $cos_object_name --pvs-image-name $image_name --accesskey --secretkey --watch" + + local import_raw="${WORK_DIR}/import_attempt.log" + + pvsadm image import \ + --pvs-instance-id "$pvs_instance_id" \ + --bucket "$COS_BUCKET_NAME" \ + --bucket-region "$COS_REGION" \ + --object "$cos_object_name" \ + --pvs-image-name "$image_name" \ + --accesskey "$COS_HMAC_ACCESS_KEY" \ + --secretkey "$COS_HMAC_SECRET_KEY" \ + --watch \ + 2>&1 | tee "$import_raw" | tee "${WORK_DIR}/import.log" + + local import_status=${PIPESTATUS[0]} + + if [ $import_status -ne 0 ]; then + # PowerVS returns 400 when the image name already exists or was recently + # deleted and is in a cooling-off period. Both mean the image is already + # present โ€” treat as success, not a pipeline failure. + if grep -qiE "already exists|recently deleted" "$import_raw"; then + log_warn "==========================================" + log_warn "Image '$image_name' already exists in PowerVS โ€” skipping import" + log_warn "To re-import: delete it from the PowerVS console, wait a few minutes," + log_warn "then re-run the pipeline." + log_warn "==========================================" + echo "SKIPPED: image already exists: $image_name" >> "${WORK_DIR}/import.log" + return 0 + fi + + log_error "Import failed with status: $import_status" + cat "$import_raw" >&2 + exit 1 + fi + + log_info "Import completed successfully" +} + +# Get imported image ID from the import.log (pvsadm prints it on success) +get_image_id() { + local image_name="$1" + + log_info "Retrieving image ID from import log..." + + # pvsadm prints: "Successfully imported the image: with ID: within ..." + local image_id=$(grep -oP 'with ID: \K[a-f0-9-]+' "${WORK_DIR}/import.log" 2>/dev/null | head -1) + + if [ -z "$image_id" ]; then + log_warn "Could not parse image ID from import log" + return 1 + fi + + log_info "Image ID: $image_id" + echo "$image_id" +} + +# Update metadata with PowerVS information +update_metadata() { + local image_id="$1" + local metadata_file="${WORK_DIR}/image-metadata.json" + local temp_file="${metadata_file}.tmp" + + jq --arg workspace "$POWERVS_WORKSPACE_CRN" \ + --arg image_id "$image_id" \ + '. + { + powervs_workspace_crn: $workspace, + powervs_image_id: $image_id, + import_date: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) + }' "$metadata_file" > "$temp_file" + + mv "$temp_file" "$metadata_file" + + log_info "Metadata updated with PowerVS information" +} + +# Main execution +main() { + log_info "Starting image import to PowerVS workspace" + log_info "Work directory: $WORK_DIR" + + # Check requirements + check_requirements + + # Get metadata + local metadata=$(get_image_metadata) + local image_name=$(echo "$metadata" | jq -r '.image_name') + local cos_object_name=$(echo "$metadata" | jq -r '.cos_object_name') + + if [ -z "$image_name" ] || [ "$image_name" == "null" ]; then + log_error "Image name not found in metadata" + exit 1 + fi + + if [ -z "$cos_object_name" ] || [ "$cos_object_name" == "null" ]; then + log_error "COS object name not found in metadata" + exit 1 + fi + + log_info "Image name: $image_name" + log_info "COS object: $cos_object_name" + + # Import image + import_image "$image_name" "$cos_object_name" + + # Get image ID + local image_id="" + if image_id=$(get_image_id "$image_name"); then + # Update metadata + update_metadata "$image_id" + else + log_warn "Could not retrieve image ID, but import may have succeeded" + fi + + log_info "==========================================" + log_info "Import completed successfully!" + log_info "Workspace: $POWERVS_WORKSPACE_CRN" + log_info "Image name: $image_name" + if [ -n "$image_id" ]; then + log_info "Image ID: $image_id" + fi + log_info "==========================================" + + # Display next steps + log_info "" + log_info "Next steps:" + log_info "1. Verify the image in PowerVS console" + log_info "2. Test deploy a VM with the new image" + log_info "3. Update Catalog CRD with new image ID" +} + +# Run main function +main "$@" + + diff --git a/jenkins/scripts/send-notification.sh b/jenkins/scripts/send-notification.sh new file mode 100755 index 00000000..ff85d997 --- /dev/null +++ b/jenkins/scripts/send-notification.sh @@ -0,0 +1,336 @@ +#!/usr/bin/env bash +# Script to send Slack notifications about the automation status + +set -o nounset +set -o pipefail + +# Configuration +WORK_DIR="${WORK_DIR:-/data/centos-images}" +SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL:-}" +SLACK_BOT_TOKEN="${SLACK_BOT_TOKEN:-}" +SLACK_CHANNEL_ID="${SLACK_CHANNEL_ID:-}" +JOB_STATUS="${JOB_STATUS:-unknown}" +ENVIRONMENT="${ENVIRONMENT:-production}" +GITHUB_RUN_ID="${GITHUB_RUN_ID:-}" +GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-}" +GITHUB_SERVER_URL="${GITHUB_SERVER_URL:-https://github.com}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if Slack is configured (webhook or bot token + channel) +check_slack_config() { + if [ -n "$SLACK_WEBHOOK_URL" ]; then + log_info "Using Slack Webhook URL" + return 0 + elif [ -n "$SLACK_BOT_TOKEN" ] && [ -n "$SLACK_CHANNEL_ID" ]; then + log_info "Using Slack Bot Token with Channel ID: $SLACK_CHANNEL_ID" + return 0 + else + log_warn "Slack not configured. Set either:" + log_warn " - SLACK_WEBHOOK_URL, or" + log_warn " - SLACK_BOT_TOKEN + SLACK_CHANNEL_ID" + return 1 + fi +} + +# Get image metadata +get_metadata() { + local metadata_file="${WORK_DIR}/image-metadata.json" + + if [ -f "$metadata_file" ]; then + cat "$metadata_file" + else + echo "{}" + fi +} + +# Build success message +build_success_message() { + local metadata=$(get_metadata) + local image_name=$(echo "$metadata" | jq -r '.image_name // "N/A"') + local centos_version=$(echo "$metadata" | jq -r '.centos_version // "N/A"') + local powervs_image_id=$(echo "$metadata" | jq -r '.powervs_image_id // "N/A"') + local cos_bucket=$(echo "$metadata" | jq -r '.cos_bucket // "N/A"') + local cos_object=$(echo "$metadata" | jq -r '.cos_object_name // "N/A"') + + local workflow_url="" + if [ -n "$GITHUB_RUN_ID" ] && [ -n "$GITHUB_REPOSITORY" ]; then + workflow_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + fi + + # Determine environment emoji + local env_emoji="๐Ÿš€" + if [ "$ENVIRONMENT" = "staging" ]; then + env_emoji="๐Ÿงช" + fi + + cat <" + } + } +EOF + fi + + cat <" + } + } +EOF + fi + + cat <&2 +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Check required tools +check_requirements() { + local missing_tools=() + + for tool in pvsadm jq; do + if ! command -v $tool &> /dev/null; then + missing_tools+=($tool) + fi + done + + if [ ${#missing_tools[@]} -ne 0 ]; then + log_error "Missing required tools: ${missing_tools[*]}" + exit 1 + fi + + log_info "All required tools are available" +} + +# Get OVA file โ€” prefer metadata, fall back to glob search in WORK_DIR +get_ova_file() { + local metadata_file="${WORK_DIR}/image-metadata.json" + local ova_file="" + + # Try metadata first + if [ -f "$metadata_file" ]; then + ova_file=$(jq -r '.ova_file // empty' "$metadata_file" 2>/dev/null || true) + fi + + # If metadata path is missing or stale, search by glob + if [ -z "$ova_file" ] || [ ! -f "$ova_file" ]; then + log_warn "OVA path from metadata not usable, searching ${WORK_DIR} for *.ova.gz..." + ova_file=$(find "$WORK_DIR" -maxdepth 1 -name "*.ova.gz" -type f 2>/dev/null | sort -V | tail -1) + fi + + if [ -z "$ova_file" ] || [ ! -f "$ova_file" ]; then + log_error "No OVA file found in ${WORK_DIR}. Run the Download & Convert stage first." + exit 1 + fi + + log_info "Using OVA file: $ova_file" + echo "$ova_file" +} + +# Upload OVA to COS using pvsadm +upload_to_cos() { + local ova_file="$1" + local ova_filename=$(basename "$ova_file") + + log_info "Uploading OVA to IBM Cloud Object Storage" + log_info "File : $ova_filename" + log_info "Bucket : $COS_BUCKET_NAME" + log_info "Region : $COS_REGION" + + log_info "Running: pvsadm image upload --bucket $COS_BUCKET_NAME --bucket-region $COS_REGION --accesskey --secretkey -f $ova_file" + + # Capture output and check for duplicate error + local upload_output="${WORK_DIR}/upload_attempt.log" + + if pvsadm image upload \ + --bucket "$COS_BUCKET_NAME" \ + --bucket-region "$COS_REGION" \ + --accesskey "$COS_HMAC_ACCESS_KEY" \ + --secretkey "$COS_HMAC_SECRET_KEY" \ + -f "$ova_file" \ + 2>&1 | tee "$upload_output" | awk ' + /[0-9]+%/ { + match($0, /([0-9]+)%/, arr) + percent = arr[1] + if (percent >= 20 && percent % 20 == 0 && percent != last) { + print "[INFO] Upload progress: " percent "%" + last = percent + } + next + } + { print } + ' | tee "${WORK_DIR}/upload.log"; then + log_info "Upload completed successfully" + else + # Check if error is due to file already existing + if grep -q "object already exists" "$upload_output"; then + log_warn "==========================================" + log_warn "File already exists in COS bucket" + log_warn "This is expected on retry - continuing..." + log_warn "==========================================" + # Not a fatal error, continue + else + log_error "Upload failed with unexpected error" + log_error "Full pvsadm output:" + cat "$upload_output" >&2 + exit 1 + fi + fi + + # Update metadata with COS information + local metadata_file="${WORK_DIR}/image-metadata.json" + local temp_file="${metadata_file}.tmp" + + jq --arg bucket "$COS_BUCKET_NAME" \ + --arg region "$COS_REGION" \ + --arg object "$ova_filename" \ + '. + { + cos_bucket: $bucket, + cos_region: $region, + cos_object_name: $object, + upload_date: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) + }' "$metadata_file" > "$temp_file" + + mv "$temp_file" "$metadata_file" + + log_info "Metadata updated with COS information" +} + +# Verify upload using s3cmd/curl with HMAC keys +verify_upload() { + local ova_filename="$1" + + log_info "Verifying upload..." + + # Use curl with HMAC signature to list bucket objects + local endpoint="https://s3.${COS_REGION}.cloud-object-storage.appdomain.cloud" + local date_val=$(date -u +"%a, %d %b %Y %H:%M:%S +0000") + local string_to_sign="GET\n\n\n${date_val}\n/${COS_BUCKET_NAME}/?prefix=${ova_filename}" + local signature=$(echo -en "$string_to_sign" | openssl dgst -sha1 -hmac "$COS_HMAC_SECRET_KEY" -binary | base64) + + local http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Date: ${date_val}" \ + -H "Authorization: AWS ${COS_HMAC_ACCESS_KEY}:${signature}" \ + "${endpoint}/${COS_BUCKET_NAME}/?prefix=${ova_filename}") + + if [ "$http_code" = "200" ]; then + log_info "Upload verification successful: $ova_filename found in bucket" + return 0 + else + log_warn "Could not verify upload (HTTP $http_code) โ€“ upload may still have succeeded" + return 1 + fi +} + +# Main execution +main() { + log_info "Starting OVA upload to IBM Cloud Object Storage" + log_info "Work directory: $WORK_DIR" + + # Check requirements + check_requirements + + # Get OVA file path + local ova_file=$(get_ova_file) + log_info "OVA file: $ova_file" + + # Get file size + local file_size=$(du -h "$ova_file" | cut -f1) + log_info "File size: $file_size" + + # Upload to COS + upload_to_cos "$ova_file" + + # Verify upload + local ova_filename=$(basename "$ova_file") + verify_upload "$ova_filename" || log_warn "Verification skipped or failed" + + log_info "==========================================" + log_info "Upload completed successfully!" + log_info "Bucket: $COS_BUCKET_NAME" + log_info "Region: $COS_REGION" + log_info "Object: $ova_filename" + log_info "==========================================" +} + +# Run main function +main "$@" + + diff --git a/jenkins/test/Jenkinsfile.centos-image-automation b/jenkins/test/Jenkinsfile.centos-image-automation new file mode 100644 index 00000000..b56c17d2 --- /dev/null +++ b/jenkins/test/Jenkinsfile.centos-image-automation @@ -0,0 +1,500 @@ +// CentOS Image Automation Pipeline +// Runs on pac-agent (ppc64le Power VM node) which has pvsadm installed. +// Downloads the latest CentOS Stream qcow2 image, converts it to OVA using +// pvsadm, uploads the OVA to IBM Cloud Object Storage, and imports it into +// a PowerVS workspace. +// +// Scheduled flow (convert once per version, upload+import per environment): +// 1. Convert centos-9 โ†’ /data/centos-images-9 +// 2. Upload+Import centos-9 โ†’ staging +// 3. Upload+Import centos-9 โ†’ production +// 4. Convert centos-10 โ†’ /data/centos-images-10 +// 5. Upload+Import centos-10 โ†’ staging +// 6. Upload+Import centos-10 โ†’ production +// +// Manual: set SCHEDULED=false to build a single version + environment. + +@Library('pac-shared-lib') _ + +// โ”€โ”€ Helper: download + convert one CentOS version (shared by both environments) โ”€ +// workDir = /data/centos-images- (no env suffix โ€” OVA is env-agnostic) +def convertCentosVersion(String centosVersion) { + def workDir = "/data/centos-images-${centosVersion}" + + echo "=== [centos-${centosVersion}] Convert: starting ===" + + sh """ + mkdir -p ${workDir} + cp jenkins/scripts/download-and-convert-centos.sh ${workDir}/ + cp jenkins/scripts/cleanup-resources.sh ${workDir}/ + chmod +x ${workDir}/*.sh + [ -f hack/scripts/image-prep.template.static ] && \\ + cp hack/scripts/image-prep.template.static ${workDir}/ || true + """ + + sh """ + cd ${workDir} + export CENTOS_VERSION=${centosVersion} + export WORK_DIR=${workDir} + export IMAGE_PREP_TEMPLATE=${workDir}/image-prep.template.static + export IMAGE_SIZE=${params.IMAGE_SIZE} + + if [ -f image-metadata.json ]; then + OVA_PATH=\$(jq -r '.ova_file // empty' image-metadata.json 2>/dev/null || true) + if [ -n "\$OVA_PATH" ] && [ ! -f "\$OVA_PATH" ]; then + echo "Stale metadata detected โ€” removing to force reconversion" + rm -f image-metadata.json + fi + fi + + ./download-and-convert-centos.sh + """ + + sh "cp ${workDir}/conversion.log conversion-${centosVersion}.log 2>/dev/null || true" + archiveArtifacts(artifacts: "conversion-${centosVersion}.log", allowEmptyArchive: true) + + echo "=== [centos-${centosVersion}] Convert: done ===" + return workDir +} + +// โ”€โ”€ Helper: upload + import for one environment, using the shared converted workDir โ”€ +def uploadAndImport(String centosVersion, String environment, String workDir) { + def cosCredId = (environment == 'staging') ? 'COS_STAGING_BUCKET_NAME' : 'COS_PROD_BUCKET_NAME' + def pvsCrnCredId = (environment == 'staging') ? 'POWERVS_STAGING_WORKSPACE_CRN' : 'POWERVS_PROD_WORKSPACE_CRN' + + echo "=== [${centosVersion}/${environment}] Upload+Import: starting ===" + + // Copy upload/import scripts into the shared workDir + sh """ + cp jenkins/scripts/upload-to-cos.sh ${workDir}/ + cp jenkins/scripts/import-to-powervs.sh ${workDir}/ + chmod +x ${workDir}/upload-to-cos.sh ${workDir}/import-to-powervs.sh + """ + + // Upload + Import wrapped in try/finally: + // - logs are always archived (even on failure) + // - OVA cleanup only on success (preserve for retry on failure) + def cellSuccess = false + try { + withCredentials([ + string(credentialsId: 'IBM_API_KEY', variable: 'IBM_API_KEY'), + string(credentialsId: cosCredId, variable: 'COS_BUCKET_NAME'), + string(credentialsId: 'COS_REGION', variable: 'COS_REGION'), + string(credentialsId: 'COS_HMAC_ACCESS_KEY', variable: 'COS_HMAC_ACCESS_KEY'), + string(credentialsId: 'COS_HMAC_SECRET_KEY', variable: 'COS_HMAC_SECRET_KEY'), + string(credentialsId: pvsCrnCredId, variable: 'POWERVS_WORKSPACE_CRN') + ]) { + sh """ + cd ${workDir} + export WORK_DIR=${workDir} + ./upload-to-cos.sh + """ + sh """ + cd ${workDir} + export WORK_DIR=${workDir} + ./import-to-powervs.sh + """ + } + cellSuccess = true + } finally { + // Always archive logs + sh """ + cp ${workDir}/upload.log upload-${centosVersion}-${environment}.log 2>/dev/null || true + cp ${workDir}/import.log import-${centosVersion}-${environment}.log 2>/dev/null || true + cp ${workDir}/image-metadata.json image-metadata-${centosVersion}-${environment}.json 2>/dev/null || true + """ + archiveArtifacts( + artifacts: "upload-${centosVersion}-${environment}.log, import-${centosVersion}-${environment}.log, image-metadata-${centosVersion}-${environment}.json", + allowEmptyArchive: true + ) + + if (!cellSuccess) { + echo "=== [${centosVersion}/${environment}] FAILED โ€” preserving ${workDir} for next run ===" + sh "sudo find /tmp -maxdepth 1 -type d -name 'qcow2ova*' -exec rm -rf {} + 2>/dev/null || true" + } + } + + echo "=== [${centosVersion}/${environment}] Upload+Import: done ===" +} + +// โ”€โ”€ Helper: cleanup one version's shared workDir (called after both envs succeed) โ”€ +def cleanupVersion(String centosVersion, String workDir) { + sh """ + if [ -d ${workDir} ] && [ -f ${workDir}/cleanup-resources.sh ]; then + cd ${workDir} + export WORK_DIR=${workDir} + KEEP_METADATA=true KEEP_LOGS=true ./cleanup-resources.sh + fi + sudo find /tmp -maxdepth 1 -type d -name "qcow2ova*" -exec rm -rf {} + 2>/dev/null || true + sudo losetup -D 2>/dev/null || true + """ +} + +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +pipeline { + agent { label 'pac-agent' } + + triggers { + // 1st and 16th of every month at 02:00 UTC โ‰ˆ every 15 days + cron('0 2 1,16 * *') + } + + options { + timestamps() + // 8 h: 2 conversions (~1 h each) + 4 upload+import (~30 min each) + timeout(time: 480, unit: 'MINUTES') + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '10')) + disableConcurrentBuilds() + } + + parameters { + choice( + name: 'ENVIRONMENT', + choices: ['production', 'staging'], + description: 'Target environment (ignored for scheduled runs โ€” both are built)' + ) + choice( + name: 'CENTOS_VERSION', + choices: ['10', '9'], + description: 'CentOS Stream version (ignored for scheduled runs โ€” both are built)' + ) + string( + name: 'IMAGE_SIZE', + defaultValue: '120', + description: 'Image disk size in GB' + ) + booleanParam( + name: 'SCHEDULED', + defaultValue: false, + description: 'Set to true by cron โ€” converts each version once, uploads to both envs' + ) + } + + environment { + WORK_DIR = '/data/centos-images' + CENTOS_VERSION = "${params.CENTOS_VERSION}" + IMAGE_SIZE = "${params.IMAGE_SIZE}" + } + + stages { + + stage('Checkout') { + steps { + echo "=== Checkout repository ===" + checkout scm + } + } + + stage('Pre-flight checks') { + steps { + sh ''' + echo "Hostname : $(hostname)" + echo "Arch : $(uname -m)" + echo "Kernel : $(uname -r)" + df -h + echo "" + + if [ "$(uname -m)" != "ppc64le" ]; then + echo "ERROR: This pipeline must run on a ppc64le node (pac-agent)." + exit 1 + fi + + if ! command -v pvsadm &>/dev/null; then + echo "pvsadm not found โ€“ installing..." + PVSADM_VERSION="v0.1.13" + PVSADM_URL="https://github.com/ppc64le-cloud/pvsadm/releases/download/${PVSADM_VERSION}/pvsadm-linux-ppc64le.tar.gz" + curl -sL "${PVSADM_URL}" -o /tmp/pvsadm.tar.gz + tar -xzf /tmp/pvsadm.tar.gz -C /tmp + sudo mv /tmp/pvsadm /usr/local/bin/pvsadm + rm -f /tmp/pvsadm.tar.gz + fi + pvsadm version + + MISSING=() + for tool in curl jq qemu-img; do + command -v "$tool" &>/dev/null || MISSING+=("$tool") + done + if [ ${#MISSING[@]} -gt 0 ]; then + echo "Installing missing tools: ${MISSING[*]}" + if command -v dnf &>/dev/null; then + sudo dnf install -y "${MISSING[@]}" + else + sudo yum install -y "${MISSING[@]}" + fi + fi + + echo "Pre-flight checks passed." + ''' + } + } + + // โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + // SCHEDULED PATH + // Convert each version once โ†’ upload+import to staging then production + // โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + stage('centos-9: Convert') { + when { + anyOf { + triggeredBy 'TimerTrigger' + expression { return params.SCHEDULED == true } + } + } + steps { + script { convertCentosVersion('9') } + } + } + + stage('centos-9: staging') { + when { + anyOf { + triggeredBy 'TimerTrigger' + expression { return params.SCHEDULED == true } + } + } + steps { + script { uploadAndImport('9', 'staging', '/data/centos-images-9') } + } + } + + stage('centos-9: production') { + when { + anyOf { + triggeredBy 'TimerTrigger' + expression { return params.SCHEDULED == true } + } + } + steps { + script { uploadAndImport('9', 'production', '/data/centos-images-9') } + } + } + + stage('centos-9: Cleanup') { + when { + anyOf { + triggeredBy 'TimerTrigger' + expression { return params.SCHEDULED == true } + } + } + steps { + script { cleanupVersion('9', '/data/centos-images-9') } + } + } + + stage('centos-10: Convert') { + when { + anyOf { + triggeredBy 'TimerTrigger' + expression { return params.SCHEDULED == true } + } + } + steps { + script { convertCentosVersion('10') } + } + } + + stage('centos-10: staging') { + when { + anyOf { + triggeredBy 'TimerTrigger' + expression { return params.SCHEDULED == true } + } + } + steps { + script { uploadAndImport('10', 'staging', '/data/centos-images-10') } + } + } + + stage('centos-10: production') { + when { + anyOf { + triggeredBy 'TimerTrigger' + expression { return params.SCHEDULED == true } + } + } + steps { + script { uploadAndImport('10', 'production', '/data/centos-images-10') } + } + } + + stage('centos-10: Cleanup') { + when { + anyOf { + triggeredBy 'TimerTrigger' + expression { return params.SCHEDULED == true } + } + } + steps { + script { cleanupVersion('10', '/data/centos-images-10') } + } + } + + // โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + // MANUAL PATH โ€” single version + environment from build parameters + // โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + stage('Prepare work directory') { + when { + allOf { + not { triggeredBy 'TimerTrigger' } + expression { return params.SCHEDULED == false } + } + } + steps { + sh ''' + mkdir -p "${WORK_DIR}" + df -h "${WORK_DIR}" + cp jenkins/scripts/download-and-convert-centos.sh "${WORK_DIR}/" + cp jenkins/scripts/upload-to-cos.sh "${WORK_DIR}/" + cp jenkins/scripts/import-to-powervs.sh "${WORK_DIR}/" + cp jenkins/scripts/cleanup-resources.sh "${WORK_DIR}/" + chmod +x "${WORK_DIR}"/*.sh + if [ -f hack/scripts/image-prep.template.static ]; then + cp hack/scripts/image-prep.template.static "${WORK_DIR}/" + else + echo "WARNING: image-prep.template.static not found" + fi + ls -lh "${WORK_DIR}/" + ''' + } + } + + stage('Download & Convert') { + when { + allOf { + not { triggeredBy 'TimerTrigger' } + expression { return params.SCHEDULED == false } + } + } + steps { + sh ''' + cd "${WORK_DIR}" + export IMAGE_PREP_TEMPLATE="${WORK_DIR}/image-prep.template.static" + if [ -f image-metadata.json ]; then + OVA_PATH=$(jq -r '.ova_file // empty' image-metadata.json 2>/dev/null || true) + if [ -n "$OVA_PATH" ] && [ ! -f "$OVA_PATH" ]; then + echo "Stale metadata detected โ€” removing to force reconversion" + rm -f image-metadata.json + fi + fi + ./download-and-convert-centos.sh + ''' + } + post { + always { + sh 'cp "${WORK_DIR}/conversion.log" conversion.log 2>/dev/null || true' + archiveArtifacts(artifacts: 'conversion.log', allowEmptyArchive: true) + } + } + } + + stage('Upload to COS') { + when { + allOf { + not { triggeredBy 'TimerTrigger' } + expression { return params.SCHEDULED == false } + } + } + steps { + script { + def cosCredId = (params.ENVIRONMENT == 'staging') ? 'COS_STAGING_BUCKET_NAME' : 'COS_PROD_BUCKET_NAME' + withCredentials([ + string(credentialsId: 'IBM_API_KEY', variable: 'IBM_API_KEY'), + string(credentialsId: cosCredId, variable: 'COS_BUCKET_NAME'), + string(credentialsId: 'COS_REGION', variable: 'COS_REGION'), + string(credentialsId: 'COS_HMAC_ACCESS_KEY', variable: 'COS_HMAC_ACCESS_KEY'), + string(credentialsId: 'COS_HMAC_SECRET_KEY', variable: 'COS_HMAC_SECRET_KEY') + ]) { + sh ''' + cd "${WORK_DIR}" + export WORK_DIR="${WORK_DIR}" + ./upload-to-cos.sh + ''' + } + } + } + post { + always { + sh 'cp "${WORK_DIR}/upload.log" upload.log 2>/dev/null || true' + archiveArtifacts(artifacts: 'upload.log', allowEmptyArchive: true) + } + } + } + + stage('Import to PowerVS') { + when { + allOf { + not { triggeredBy 'TimerTrigger' } + expression { return params.SCHEDULED == false } + } + } + steps { + script { + def cosCredId = (params.ENVIRONMENT == 'staging') ? 'COS_STAGING_BUCKET_NAME' : 'COS_PROD_BUCKET_NAME' + def pvsCrnCredId = (params.ENVIRONMENT == 'staging') ? 'POWERVS_STAGING_WORKSPACE_CRN' : 'POWERVS_PROD_WORKSPACE_CRN' + withCredentials([ + string(credentialsId: 'IBM_API_KEY', variable: 'IBM_API_KEY'), + string(credentialsId: pvsCrnCredId, variable: 'POWERVS_WORKSPACE_CRN'), + string(credentialsId: cosCredId, variable: 'COS_BUCKET_NAME'), + string(credentialsId: 'COS_REGION', variable: 'COS_REGION'), + string(credentialsId: 'COS_HMAC_ACCESS_KEY', variable: 'COS_HMAC_ACCESS_KEY'), + string(credentialsId: 'COS_HMAC_SECRET_KEY', variable: 'COS_HMAC_SECRET_KEY') + ]) { + sh ''' + cd "${WORK_DIR}" + export WORK_DIR="${WORK_DIR}" + ./import-to-powervs.sh + ''' + } + } + } + post { + always { + sh ''' + cp "${WORK_DIR}/import.log" import.log 2>/dev/null || true + cp "${WORK_DIR}/image-metadata.json" image-metadata.json 2>/dev/null || true + ''' + archiveArtifacts(artifacts: 'import.log, image-metadata.json', allowEmptyArchive: true) + } + } + } + + stage('Cleanup') { + when { + allOf { + not { triggeredBy 'TimerTrigger' } + expression { return params.SCHEDULED == false } + } + } + steps { + sh ''' + cd "${WORK_DIR}" + export WORK_DIR="${WORK_DIR}" + KEEP_METADATA=true KEEP_LOGS=true ./cleanup-resources.sh + sudo find /tmp -maxdepth 1 -type d -name "qcow2ova*" -exec rm -rf {} + 2>/dev/null || true + sudo losetup -D 2>/dev/null || true + ''' + } + } + } + + post { + success { + echo "=== CentOS Image Automation completed successfully ===" + } + failure { + echo "=== CentOS Image Automation FAILED ===" + echo "Check archived logs above for details." + echo "" + echo "Common causes:" + echo " - Loop device exhaustion โ†’ sudo losetup -D" + echo " - Stale /tmp/qcow2ova dirs โ†’ sudo rm -rf /tmp/qcow2ova*" + echo " - Insufficient disk space โ†’ df -h /data" + echo " - IBM API key missing PowerVS Editor role" + echo " - Expired IBM API key" + } + always { + pipelineTeardown() + } + } +} diff --git a/jenkins/test/vars/notifyGitHub.groovy b/jenkins/test/vars/notifyGitHub.groovy index e2427238..dadcaac1 100644 --- a/jenkins/test/vars/notifyGitHub.groovy +++ b/jenkins/test/vars/notifyGitHub.groovy @@ -27,7 +27,6 @@ def call(String buildResult) { def state = (result == 'SUCCESS') ? 'success' : 'failure' def desc = (result == 'SUCCESS') ? 'Build passed' : 'Build failed' - // JOB_NAME = "pac/pac-test/pac-go-lint/PR-110" // Second-to-last segment is always the Multibranch pipeline name e.g. "pac-go-lint" def parts = env.JOB_NAME?.split('/') def jobName = (parts && parts.size() >= 2) ? parts[parts.size() - 2] : (parts ? parts[0] : 'jenkins')