Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions api/v1alpha1/release_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ const (
ConditionApplied = "Applied"
// ConditionManifestResolved indicates whether the release manifest was successfully retrieved.
ConditionManifestResolved = "ManifestResolved"
// ConditionLCMUpgraded indicates the states of LCM upgrade.
// Pending -> InProgress -> Succeeded/Failed
ConditionLCMUpgraded = "LCMUpgraded"
// ConditionOSUpgraded indicates the status of the OS upgrade.
// Pending -> InProgress -> Succeeded/Failed
ConditionOSUpgraded = "OSUpgraded"
Expand Down
2 changes: 1 addition & 1 deletion api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ func main() {
Scheme: mgr.GetScheme(),
RetrieveManifest: release.RetrieveManifest,
Pipeline: upgrade.NewPipeline(
reconcilers.NewLCMReconciler(k8sClient, helmClient),
reconcilers.NewOSReconciler(k8sClient, sucPlanReconciler),
reconcilers.NewKubernetesReconciler(
k8sClient,
Expand Down
13 changes: 13 additions & 0 deletions docs/monitor-and-troubleshoot.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ The table below summarizes the conditions LCM reports during an upgrade:
| Condition Type | Description |
|----------------------|-------------|
| `ManifestResolved` | Indicates whether LCM retrieved and resolved the [release manifest](https://github.com/SUSE/elemental/blob/main/docs/release-manifest.md) for the requested version. |
| `LCMUpgraded` | Tracks the LCM upgrade phase which upgrades the Helm charts for LCM itself and its CRDs if found in the manifest under upgrade |
| `OSUpgraded` | Tracks the operating system upgrade phase, including the related System Upgrade Controller Plans for `control-plane` and `worker` nodes. |
| `KubernetesUpgraded` | Tracks the Kubernetes upgrade phase, including related System Upgrade Controller Plans and the availability of packaged Kubernetes components after node upgrades complete. |
| `HelmChartsUpgraded` | Tracks the Helm chart upgrade phase for additional chart components defined in the release manifest. |
Expand All @@ -37,6 +38,18 @@ If this phase fails or stops progressing, inspect the following resources:
| Manifest Cache ConfigMap | LCM namespace | `release-manifest-cache` | Use it to confirm whether the release manifest was retrieved and cached. |
| LCM Pod | LCM namespace | LCM Pod name | Inspect LCM's logs for errors while retrieving, parsing, or caching the release manifest. |

### Lifecycle Manager Upgrade

Condition type: `LCMUpgraded`

If this phase fails or stops progressing, inspect the following resources:


| Resource | Namespace | Name | Description |
|-------------------------|---------------------------|-------------|------------- |
| Helm Chart Pod | `kube-system` | `helm-install-<chart-name>` | Inspect the Pod logs for Helm chart errors. |
| LCM Pod | `elemental-system` | LCM Pod name | Inspect LCM's logs for LCM charts upgrade reconciliation errors. |

### Operating System Upgrade

Condition type: `OSUpgraded`
Expand Down
1 change: 1 addition & 0 deletions internal/upgrade/phase.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ type Phase string

// Phase constants derived from condition types.
var (
PhaseLCM = Phase(strings.TrimSuffix(lifecyclev1alpha1.ConditionLCMUpgraded, "Upgraded"))
PhaseOS = Phase(strings.TrimSuffix(lifecyclev1alpha1.ConditionOSUpgraded, "Upgraded"))
PhaseKubernetes = Phase(strings.TrimSuffix(lifecyclev1alpha1.ConditionKubernetesUpgraded, "Upgraded"))
PhaseHelmCharts = Phase(strings.TrimSuffix(lifecyclev1alpha1.ConditionHelmChartsUpgraded, "Upgraded"))
Expand Down
14 changes: 11 additions & 3 deletions internal/upgrade/reconcilers/helm.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,15 @@ func (r *HelmReconciler) reconcileHelmCharts(ctx context.Context, releaseName, r
r.releaseName = releaseName
r.releaseVersion = releaseVersion

orderedChartConfigs, err := sortChartConfigsByDependencies(chartConfigs)
var workloadCharts []*upgrade.HelmChartConfig
for _, chartCfg := range chartConfigs {
name := chartCfg.Chart.GetName()
if !isLCMChart(name) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: do we actually need to prevent LCMchart to be processed here? Up to my understanding it should not hurt attempting to upgrade it again, the upgrade if it is already at the desired version should do nothing.
If so, then we can probably keep the helm reconciler unmodified and if we ever set an upgrade pipe without LCMreconciler the helm reconciler would still run the full upgrade.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see any technical issue in removing isLCMChart check from here. Except it being a brain thing where it feels weird to keep it in spite of having a dedicated phase for it earlier in the flow.

if we ever set an upgrade pipe without LCMreconciler the helm reconciler would still run the full upgrade.

If we do this, it would be for the purpose of prevent LCM charts' upgrade, right? But removing isLCMChart check from Helm reconciler would still cause the LCM charts' upgrade. Just presenting a counter point.

I'm fine by removing the check from here if it makes more sense. Keeping this open for others' feedback/suggestions.

workloadCharts = append(workloadCharts, chartCfg)
}
}

orderedChartConfigs, err := sortChartConfigsByDependencies(workloadCharts)
if err != nil {
return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeFailed,
Expand Down Expand Up @@ -130,7 +138,7 @@ func (r *HelmReconciler) reconcileHelmCharts(ctx context.Context, releaseName, r
}
}

return r.aggregateResults(results, len(orderedChartConfigs)), nil
return aggregateResults(results, len(orderedChartConfigs)), nil
}

// sortChartConfigsByDependencies returns a sorted slice of chart configurations,
Expand Down Expand Up @@ -405,7 +413,7 @@ func (r *HelmReconciler) evaluateHelmChartJobStatus(ctx context.Context, chart *
}

// aggregateResults aggregates chart upgrade results into a single PhaseStatus.
func (r *HelmReconciler) aggregateResults(results []chartUpgradeResult, totalCharts int) *upgrade.PhaseStatus {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we unbound this from the HelmController? While it does not use any values from it, IMO bounding it shows the intention that this function is supposed to be called only from the HelmController itself and not be used anywhere else without it.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we unbound this from the HelmController? While it does not use any values from it IMO bounding it shows the intention that this function is supposed to be called only from the HelmController itself and not be used anywhere else without it.

Sure, I'll revert it.

func aggregateResults(results []chartUpgradeResult, totalCharts int) *upgrade.PhaseStatus {
if len(results) == 0 {
return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeSucceeded,
Expand Down
36 changes: 35 additions & 1 deletion internal/upgrade/reconcilers/helm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ var _ = Describe("HelmReconciler", func() {
var chart1 *api.HelmChart

BeforeEach(func() {
chart1 = testutil.NewTestHelmChart(testChart1Name, "1.0.0")
chart1 = testutil.NewTestHelmChart(testChart1Name, testChartVersion)
config = testutil.NewTestConfig(testutil.WithHelmChartConfig([]*upgrade.HelmChartConfig{{Chart: chart1}}))
})

Expand Down Expand Up @@ -134,6 +134,36 @@ var _ = Describe("HelmReconciler", func() {
Expect(status.Message).To(ContainSubstring("skipped"))
})

It("should skip LCM charts as they are handled by LCMReconciler", func() {
lcmChart := testutil.NewTestHelmChart("elemental-lifecycle-manager", "0.2.1")
config = testutil.NewTestConfig(testutil.WithHelmChartConfig([]*upgrade.HelmChartConfig{{Chart: lcmChart}}))
status, err := reconciler.Reconcile(ctx, config)

Expect(err).ToNot(HaveOccurred())
Expect(status).ToNot(BeNil())
Expect(status.Message).To(Equal("No Helm charts to reconcile"))
})

It("should skip LCM chart and reconcile other charts successfully", func() {
lcmChart := testutil.NewTestHelmChart("elemental-lifecycle-manager", "0.2.1")
config = testutil.NewTestConfig(testutil.WithHelmChartConfig([]*upgrade.HelmChartConfig{{Chart: lcmChart}, {Chart: chart1}}))

mockHelm.RetrieveReleaseFn = func(name string) (*helm.ReleaseInfo, error) {
return &helm.ReleaseInfo{
ChartVersion: testChartVersion,
Namespace: testNamespace,
Config: map[string]any{},
Revisions: 1,
}, nil
}

status, err := reconciler.Reconcile(ctx, config)

Expect(err).NotTo(HaveOccurred())
Expect(status).NotTo(BeNil())
Expect(status.Message).To(Equal("All 1 Helm charts upgraded successfully (0 skipped)"))
})

It("should return error on helm client failure", func() {
mockHelm.RetrieveReleaseFn = func(name string) (*helm.ReleaseInfo, error) {
return nil, fmt.Errorf("helm client error")
Expand Down Expand Up @@ -362,6 +392,10 @@ var _ = Describe("HelmReconciler", func() {
// Ensure that the chart version was correctly updated.
Expect(helmChart.Spec.Version).To(Equal("2.0.0"))

// Ensure the labels are applied
Expect(helmChart.Labels).To(HaveKeyWithValue(lifecyclev1alpha1.ReleaseNameLabel, config.ReleaseNamespacedName.Name))
Expect(helmChart.Labels).To(HaveKeyWithValue(lifecyclev1alpha1.ReleaseVersionLabel, lifecyclev1alpha1.SanitizeVersion(config.ReleaseVersion)))

// Ensure install time custom values are not corrupted.
expectedInstallValues, err := yaml.Marshal(installTimeValues)
Expect(err).ToNot(HaveOccurred())
Expand Down
160 changes: 160 additions & 0 deletions internal/upgrade/reconcilers/lcm.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
/*
Copyright © 2026 SUSE LLC
SPDX-License-Identifier: Apache-2.0

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reconcilers

import (
"context"
"fmt"

lifecyclev1alpha1 "github.com/suse/elemental-lifecycle-manager/api/v1alpha1"
"github.com/suse/elemental-lifecycle-manager/internal/helm"
"github.com/suse/elemental-lifecycle-manager/internal/upgrade"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)

const (
ElementalLifecycleManagerChart = "elemental-lifecycle-manager"
ElementalLifecycleManagerCRDsChart = "elemental-lifecycle-manager-crds"
)

type LCMReconciler struct {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not completely convinced that we need a separate reconciler for LCM. Right now this reconciler is a wrapper over the HelmController so that it can call reconcileChart and immediately fail on chart failure.

I wonder, can't we just extend the HelmController to support a failure policy (e.g. lenient and strict), where lenient will be the current logic and strict will fail immediately on helm chart failure. Then we can have two HelmController instances:

  1. For the LCM itself - configured with failure policy strict; defined before any RM / upgrade config parsing. Will be passed an upgrade config that only has the two LCM charts that need to be upgraded.
  2. For all other Helm charts - This is the already existing HelmController with a failure policy defined as lenient. This controller would also be passed an upgrade config that does not hold any LCM chart details, as those would be handled by the first controller earlier.

I think this is worth considering mainly because:

  1. It reuses the existing logic without defining a separate wrapper reconciler, which reduces the testing matrix and code included.
  2. It fits very nicely with what we have discussed during our upgrade recovery discussion sessions - this would tie to the users configuring their recovery mode and whether they want an immediate recovery, or want a dirty cluster. I imagine for immediate recovery we would have both HelmControllers configured with a strict failure policy. On the other hand dirty clusters would required HelmControllers configured with lenient policy.

Do you see value in this?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It fits very nicely with what we have discussed during our upgrade recovery discussion sessions - this would tie to the users configuring their recovery mode and whether they want an immediate recovery, or want a dirty cluster. I imagine for immediate recovery we would have both HelmControllers configured with a strict failure policy. On the other hand dirty clusters would required HelmControllers configured with lenient policy.

Makes sense! This could likely simplify things. I can modify the PR.

On the other hand dirty clusters would required HelmControllers configured with lenient policy.

Not a topic for this PR, but even for dirty clusters configuration, we should stick to being strict for LCM charts because that's Elemental project requirement, not user's.

@ipetrov117 ipetrov117 Sep 17, 2026 •

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree, for specific charts we should always fail. This goes back to how we handle different charts failing, but that is a topic for a different discussion 😄

helm *HelmReconciler
}

// NewLCMReconciler creates a new LCM reconciler.
func NewLCMReconciler(c client.Client, h helm.Client) *LCMReconciler {
return &LCMReconciler{
helm: NewHelmReconciler(c, h),
}
}

func (r *LCMReconciler) Phase() upgrade.Phase {
return upgrade.PhaseLCM
}

func (r *LCMReconciler) Reconcile(ctx context.Context, config *upgrade.Config) (*upgrade.PhaseStatus, error) {
if config == nil || config.HelmCharts == nil {
return r.Phase().SkippedStatus(), nil
}
logger := log.FromContext(ctx)

r.helm.releaseName = config.ReleaseNamespacedName.Name
r.helm.releaseVersion = config.ReleaseVersion

var lcmCharts []*upgrade.HelmChartConfig
for _, chartConfig := range config.HelmCharts {
name := chartConfig.Chart.GetName()
if isLCMChart(name) {
lcmCharts = append(lcmCharts, chartConfig)
}
}
if len(lcmCharts) == 0 {
return r.Phase().SkippedStatus(), nil
}

orderedChartConfigs, err := sortChartConfigsByDependencies(lcmCharts)
if err != nil {
return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeFailed,
Message: fmt.Sprintf("Failed to resolve chart dependencies: %v", err),
}, err
}

logger.Info("Reconciling LCM charts and CRDs", "count", len(orderedChartConfigs))

var results []chartUpgradeResult
for _, chartConfig := range orderedChartConfigs {
name := chartConfig.Chart.GetName()
state, err := r.helm.reconcileChart(ctx, chartConfig)
if err != nil {
return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeFailed,
Message: fmt.Sprintf("Failed to reconcile LCM chart %s: %v", name, err),
}, err
}

results = append(results, chartUpgradeResult{
chartName: name,
state: state,
})

if state == helm.ChartStateInProgress {
logger.Info("LCM chart upgrade in progress, waiting", "chart", name)
break
}

if state == helm.ChartStateFailed {
// Failure in upgrading either LCM CRD or LCM chart should result in overall upgrade failure,
// and prevent moving forward to any other upgrade phase
return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeFailed,
Message: fmt.Sprintf("Failed to upgrade LCM chart %q", name),
}, fmt.Errorf("upgrading LCM chart %q", name)
}
}

return aggregateLCMResults(results, len(orderedChartConfigs)), nil
}

// aggregateLCMResults aggregates chart upgrade results into a single PhaseStatus.
func aggregateLCMResults(results []chartUpgradeResult, totalCharts int) *upgrade.PhaseStatus {
if len(results) == 0 {
return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeSucceeded,
Message: "No LCM charts to reconcile",
}
}

var inProgress, succeeded, skipped int

for _, result := range results {
switch result.state {
case helm.ChartStateInProgress:
inProgress++
case helm.ChartStateSucceeded, helm.ChartStateVersionAlreadyInstalled:
succeeded++
case helm.ChartStateNotInstalled:
skipped++
}
}

if inProgress > 0 {
return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeInProgress,
Message: fmt.Sprintf("LCM charts in progress (%d/%d completed, %d skipped)", succeeded, totalCharts-skipped, skipped),
}
}

if succeeded == 0 && skipped == totalCharts {
return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeSucceeded,
Message: "All LCM charts skipped (not installed on cluster)",
}
}

return &upgrade.PhaseStatus{
State: lifecyclev1alpha1.UpgradeSucceeded,
Message: fmt.Sprintf("All %d LCM charts upgraded successfully (%d skipped)", succeeded, skipped),
}
}

// isLCMChart reports whether name is one of LCM's own charts. These are upgraded by the LCM phase and skipped by the Helm chart phase
func isLCMChart(name string) bool {
return name == ElementalLifecycleManagerCRDsChart || name == ElementalLifecycleManagerChart
}
Loading