Skip to content

Commit

Permalink
Fixed node being taken repeatedly causing queue block
Browse files Browse the repository at this point in the history
  • Loading branch information
wzshiming committed Oct 29, 2024
1 parent 45e59c0 commit f0ca9e8
Showing 1 changed file with 9 additions and 6 deletions.
15 changes: 9 additions & 6 deletions pkg/kwok/controllers/node_lease_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,12 +182,15 @@ func (c *NodeLeaseController) sync(ctx context.Context, nodeName string) (lease
return nil, nil
}
logger.Info("Syncing lease")
lease, err := c.renewLease(ctx, lease)
lease, transitions, err := c.renewLease(ctx, lease)
if err != nil {
return nil, fmt.Errorf("failed to update lease using lease: %w", err)
}

c.onNodeManaged(nodeName)
// Only the lease have been transitions and then retry to manage the node.
if transitions {
c.onNodeManaged(nodeName)
}
return lease, nil
}

Expand Down Expand Up @@ -249,7 +252,7 @@ func (c *NodeLeaseController) ensureLease(ctx context.Context, leaseName string)
}

// renewLease attempts to update the lease for maxUpdateRetries, call this once you're sure the lease has been created
func (c *NodeLeaseController) renewLease(ctx context.Context, base *coordinationv1.Lease) (*coordinationv1.Lease, error) {
func (c *NodeLeaseController) renewLease(ctx context.Context, base *coordinationv1.Lease) (*coordinationv1.Lease, bool, error) {
lease := base.DeepCopy()

transitions := format.ElemOrDefault(lease.Spec.HolderIdentity) != c.holderIdentity
Expand All @@ -263,15 +266,15 @@ func (c *NodeLeaseController) renewLease(ctx context.Context, base *coordination
if c.mutateLeaseFunc != nil {
err := c.mutateLeaseFunc(lease)
if err != nil {
return nil, err
return nil, false, err
}
}

lease, err := c.typedClient.CoordinationV1().Leases(lease.Namespace).Update(ctx, lease, metav1.UpdateOptions{})
if err != nil {
return nil, err
return nil, false, err
}
return lease, nil
return lease, transitions, nil
}

// setNodeOwnerFunc helps construct a mutateLeaseFunc which sets a node OwnerReference to the given lease object
Expand Down

0 comments on commit f0ca9e8

Please sign in to comment.