Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add wait polling for retrieving redfish storages and systems #174

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Add wait polling for retrieving redfish storages and systems #174

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f48584a
adds wait polling for retrieving redfish storages
stefanhipfel Nov 8, 2024
c7bf550
adds polling options to bmcClient
stefanhipfel Nov 8, 2024
6781b30
creates BMCPollingOptions type
stefanhipfel Nov 8, 2024
73a8e65
fixes lint errors
stefanhipfel Nov 11, 2024
7d6697d
adds bmc polling defaults
stefanhipfel Nov 12, 2024
40802a1
removes ctx error check
stefanhipfel Nov 14, 2024
ce22db2
switches back to generic bmc client options struct
stefanhipfel Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion bmc/bmc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
package bmc

import (
"context"
"time"

"github.com/stmcginnis/gofish/common"
"github.com/stmcginnis/gofish/redfish"
)
Expand Down Expand Up @@ -47,7 +50,9 @@ type BMC interface {

SetBootOrder(systemUUID string, order []string) error

GetStorages(systemUUID string) ([]Storage, error)
GetStorages(ctx context.Context, systemUUID string) ([]Storage, error)

WaitForServerPowerState(ctx context.Context, systemUUID string, interval, timeout time.Duration, powerState redfish.PowerState) error
}

type Entity struct {
Expand Down
42 changes: 39 additions & 3 deletions bmc/redfish.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ import (
"fmt"
"strconv"
"strings"
"time"

"github.com/stmcginnis/gofish"
"github.com/stmcginnis/gofish/redfish"
"k8s.io/apimachinery/pkg/util/wait"
)

var _ BMC = (*RedfishBMC)(nil)
Expand Down Expand Up @@ -342,14 +344,29 @@ func (r *RedfishBMC) checkBiosAttributes(attrs map[string]string) (reset bool, e
return
}

func (r *RedfishBMC) GetStorages(systemUUID string) ([]Storage, error) {
func (r *RedfishBMC) GetStorages(ctx context.Context, systemUUID string) ([]Storage, error) {
system, err := r.getSystemByUUID(systemUUID)
if err != nil {
return nil, err
}
systemStorage, err := system.Storage()
var systemStorage []*redfish.Storage
err = wait.PollUntilContextTimeout(
ctx,
10*time.Second,
5*time.Minute,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we make interval and timeout configurable like in bmcClient.WaitForServerPowerState?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking about it. We just have so many config options already :)

true,
func(ctx context.Context) (bool, error) {
if ctx.Err() != nil {
return false, ctx.Err()
}
stefanhipfel marked this conversation as resolved.
Show resolved Hide resolved
systemStorage, err = system.Storage()
if err != nil {
return false, nil
}
return len(systemStorage) > 0, nil
})
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to wait for for server storages to be ready: %w", err)
}
result := make([]Storage, 0, len(systemStorage))
for _, s := range systemStorage {
Expand Down Expand Up @@ -429,3 +446,22 @@ func (r *RedfishBMC) getSystemByUUID(systemUUID string) (*redfish.ComputerSystem
}
return nil, errors.New("no system found")
}

func (r *RedfishBMC) WaitForServerPowerState(
ctx context.Context,
systemUUID string,
interval,
timeout time.Duration,
powerState redfish.PowerState,
) error {
if err := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (done bool, err error) {
sysInfo, err := r.getSystemByUUID(systemUUID)
if err != nil {
return false, fmt.Errorf("failed to get system info: %w", err)
}
return sysInfo.PowerState == powerState, nil
}); err != nil {
return fmt.Errorf("failed to wait for for server power state: %w", err)
}
return nil
}
43 changes: 21 additions & 22 deletions internal/controller/server_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@ import (
"sort"
"time"

"github.com/ironcore-dev/metal-operator/bmc"
"k8s.io/apimachinery/pkg/util/wait"

"github.com/go-logr/logr"
"github.com/ironcore-dev/controller-utils/clientutils"
metalv1alpha1 "github.com/ironcore-dev/metal-operator/api/v1alpha1"
Expand Down Expand Up @@ -279,7 +276,7 @@ func (r *ServerReconciler) handleDiscoveryState(ctx context.Context, log logr.Lo
if err != nil {
return false, fmt.Errorf("failed to create BMC client: %w", err)
}
storages, err := bmcClient.GetStorages(server.Spec.UUID)
storages, err := bmcClient.GetStorages(ctx, server.Spec.UUID)
if err != nil {
return false, fmt.Errorf("failed to get storages for Server: %w", err)
}
Expand Down Expand Up @@ -722,7 +719,12 @@ func (r *ServerReconciler) ensureServerPowerState(ctx context.Context, log logr.
if err := bmcClient.PowerOn(server.Spec.UUID); err != nil {
return fmt.Errorf("failed to power on server: %w", err)
}
if err := r.waitForServerPowerState(ctx, log, bmcClient, server, redfish.OnPowerState); err != nil {
if err := bmcClient.WaitForServerPowerState(
ctx, server.Spec.UUID,
r.PowerPollingInterval,
r.PowerPollingTimeout,
redfish.OnPowerState,
); err != nil {
return fmt.Errorf("failed to wait for server power on server: %w", err)
}
case powerOpOff:
Expand All @@ -731,14 +733,26 @@ func (r *ServerReconciler) ensureServerPowerState(ctx context.Context, log logr.
if err := powerOffType(server.Spec.UUID); err != nil {
return fmt.Errorf("failed to power off server: %w", err)
}
if err := r.waitForServerPowerState(ctx, log, bmcClient, server, redfish.OffPowerState); err != nil {
if err := bmcClient.WaitForServerPowerState(
ctx,
server.Spec.UUID,
r.PowerPollingInterval,
r.PowerPollingTimeout,
redfish.OffPowerState,
); err != nil {
if r.EnforcePowerOff {
log.V(1).Info("Failed to wait for server graceful shutdown, retrying with force power off")
powerOffType = bmcClient.ForcePowerOff
if err := powerOffType(server.Spec.UUID); err != nil {
return fmt.Errorf("failed to power off server: %w", err)
}
if err := r.waitForServerPowerState(ctx, log, bmcClient, server, redfish.OffPowerState); err != nil {
if err := bmcClient.WaitForServerPowerState(
ctx,
server.Spec.UUID,
r.PowerPollingInterval,
r.PowerPollingTimeout,
redfish.OffPowerState,
); err != nil {
return fmt.Errorf("failed to wait for server force power off: %w", err)
}
} else {
Expand All @@ -751,21 +765,6 @@ func (r *ServerReconciler) ensureServerPowerState(ctx context.Context, log logr.
return nil
}

func (r *ServerReconciler) waitForServerPowerState(ctx context.Context, log logr.Logger, bmcClient bmc.BMC, server *metalv1alpha1.Server, powerState redfish.PowerState) error {
if err := wait.PollUntilContextTimeout(ctx, r.PowerPollingInterval, r.PowerPollingTimeout, true, func(ctx context.Context) (done bool, err error) {
log.V(1).Info("Waiting for Server to reach target power state", "TargetPowerState", powerState)
sysInfo, err := bmcClient.GetSystemInfo(server.Spec.UUID)
if err != nil {
return false, fmt.Errorf("failed to get system info: %w", err)
}
log.V(1).Info("Read Server power state", "PowerState", sysInfo.PowerState, "TargetPowerState", powerState)
return sysInfo.PowerState == powerState, nil
}); err != nil {
return fmt.Errorf("failed to wait for for server power state: %w", err)
}
return nil
}

func (r *ServerReconciler) ensureIndicatorLED(ctx context.Context, log logr.Logger, server *metalv1alpha1.Server) error {
// TODO: implement
return nil
Expand Down
Loading