Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

pkg/services: ErrorBuffer.Flush fix race #998

Merged
merged 1 commit into from
Jan 15, 2025
Merged

pkg/services: ErrorBuffer.Flush fix race #998

merged 1 commit into from
Jan 15, 2025

Conversation

jmank88
Copy link
Collaborator

@jmank88 jmank88 commented Jan 14, 2025

race 
WARNING: DATA RACE,
Read at 0x00c002871120 by goroutine 35490:,
  github.com/smartcontractkit/chainlink-common/pkg/services.(*ErrorBuffer).Flush(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/services/state.go:217 +0xc4,
  github.com/smartcontractkit/chainlink-common/pkg/services.(*StateMachine).Healthy(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/services/state.go:196 +0x44,
  github.com/smartcontractkit/chainlink-common/pkg/services.(*service).Healthy(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/services/service.go:229 +0x28,
  github.com/smartcontractkit/chainlink-common/pkg/services.(*service).HealthReport(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/services/service.go:237 +0x75,
  github.com/smartcontractkit/chainlink/v2/common/headtracker.(*headTracker[*github.com/smartcontractkit/chainlink/v2/core/chains/evm/types.Head,github.com/ethereum/go-ethereum.Subscription,*math/big.Int,github.com/ethereum/go-ethereum/common.Hash]).HealthReport(),
      \u003cautogenerated\u003e:1 +0x43,
  github.com/smartcontractkit/chainlink/v2/core/services/relay/evm.(*chainReader).HealthReport(),
      /home/runner/work/chainlink/chainlink/core/services/relay/evm/chain_reader.go:188 +0x12d,
  github.com/smartcontractkit/chainlink-ccip/pkg/contractreader.(*Observed).HealthReport(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/contractreader/observed.go:95 +0x3a,
  github.com/smartcontractkit/chainlink-ccip/pkg/contractreader.(*extendedContractReader).hasFinalityViolation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/contractreader/extended.go:324 +0x52,
  github.com/smartcontractkit/chainlink-ccip/pkg/contractreader.(*extendedContractReader).GetLatestValue(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/contractreader/extended.go:193 +0xfa,
  github.com/smartcontractkit/chainlink-ccip/pkg/contractreader.(*extendedContractReader).ExtendedGetLatestValue(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/contractreader/extended.go:212 +0x25a,
  github.com/smartcontractkit/chainlink-ccip/pkg/reader.(*ccipChainReader).getDestinationData(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/reader/ccip.go:1225 +0x2d2,
  github.com/smartcontractkit/chainlink-ccip/pkg/reader.(*ccipChainReader).discoverOffRampContracts(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/reader/ccip.go:832 +0x1044,
  github.com/smartcontractkit/chainlink-ccip/pkg/reader.(*ccipChainReader).DiscoverContracts(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/reader/ccip.go:853 +0x110,
  github.com/smartcontractkit/chainlink-ccip/internal/plugincommon/discovery.(*ContractDiscoveryProcessor).Observation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/internal/plugincommon/discovery/processor.go:69 +0x149,
  github.com/smartcontractkit/chainlink-ccip/execute.(*Plugin).Observation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/execute/observation.go:46 +0x3a4,
  github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper.(*reportingPlugin[go.shape.[]uint8]).Observation.func1(),
      /home/runner/work/chainlink/chainlink/core/services/ocr3/promwrapper/plugin.go:58 +0x11d,
  github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper.withObservedExecution[go.shape.[]uint8,go.shape.[]uint8](),
      /home/runner/work/chainlink/chainlink/core/services/ocr3/promwrapper/plugin.go:143 +0x72,
  github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper.(*reportingPlugin[go.shape.[]uint8]).Observation(),
      /home/runner/work/chainlink/chainlink/core/services/ocr3/promwrapper/plugin.go:57 +0x1a4,
  github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper.(*reportingPlugin[[]uint8]).Observation(),
      /home/runner/work/chainlink/chainlink/core/services/ocr3/promwrapper/plugin.go:56 +0xe4,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/shim.LimitCheckOCR3ReportingPlugin[go.shape.[]uint8].Observation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/shim/ocr3_reporting_plugin.go:39 +0x10d,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/shim.(*LimitCheckOCR3ReportingPlugin[[]uint8]).Observation(),
      \u003cautogenerated\u003e:1 +0x21b,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).backgroundObservation.func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:241 +0x158,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.callPluginFromOutcomeGenerationBackground[go.shape.[]uint8,go.shape.[]uint8].func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:391 +0xb5,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.callPluginFromBackground[go.shape.[]uint8](),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/common.go:72 +0x227,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.callPluginFromOutcomeGenerationBackground[go.shape.[]uint8,go.shape.[]uint8](),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:381 +0x297,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).backgroundObservation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:234 +0x1d1,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).tryProcessRoundStartPool.func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:223 +0x14d,
  github.com/smartcontractkit/libocr/subprocesses.(*Subprocesses).Go.func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/subprocesses/subprocesses.go:29 +0x86,

Previous write at 0x00c002871120 by goroutine 34343:,
  github.com/smartcontractkit/chainlink-common/pkg/services.(*ErrorBuffer).Flush(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/services/state.go:218 +0x256,
  github.com/smartcontractkit/chainlink-common/pkg/services.(*StateMachine).Healthy(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/services/state.go:196 +0x44,
  github.com/smartcontractkit/chainlink-common/pkg/services.(*service).Healthy(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/services/service.go:229 +0x28,
  github.com/smartcontractkit/chainlink-common/pkg/services.(*service).HealthReport(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/services/service.go:237 +0x75,
  github.com/smartcontractkit/chainlink/v2/common/headtracker.(*headTracker[*github.com/smartcontractkit/chainlink/v2/core/chains/evm/types.Head,github.com/ethereum/go-ethereum.Subscription,*math/big.Int,github.com/ethereum/go-ethereum/common.Hash]).HealthReport(),
      \u003cautogenerated\u003e:1 +0x43,
  github.com/smartcontractkit/chainlink/v2/core/services/relay/evm.(*chainReader).HealthReport(),
      /home/runner/work/chainlink/chainlink/core/services/relay/evm/chain_reader.go:188 +0x12d,
  github.com/smartcontractkit/chainlink-ccip/pkg/contractreader.(*Observed).HealthReport(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/contractreader/observed.go:95 +0x3a,
  github.com/smartcontractkit/chainlink-ccip/pkg/contractreader.(*extendedContractReader).hasFinalityViolation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/contractreader/extended.go:324 +0x52,
  github.com/smartcontractkit/chainlink-ccip/pkg/contractreader.(*extendedContractReader).GetLatestValue(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/contractreader/extended.go:193 +0xfa,
  github.com/smartcontractkit/chainlink-ccip/pkg/reader.(*priceReader).GetFeeQuoterTokenUpdates(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/pkg/reader/price_reader.go:123 +0xea7,
  github.com/smartcontractkit/chainlink-ccip/commit/tokenprice.(*processor).ObserveFeeQuoterTokenUpdates(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/commit/tokenprice/observation.go:114 +0x673,
  github.com/smartcontractkit/chainlink-ccip/commit/tokenprice.(*processor).Observation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/commit/tokenprice/observation.go:26 +0xf4,
  github.com/smartcontractkit/chainlink-ccip/commit.(*Plugin).getPriceRelatedObservations(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/commit/plugin.go:321 +0x86a,
  github.com/smartcontractkit/chainlink-ccip/commit.(*Plugin).Observation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/commit/plugin.go:259 +0xf76,
  github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper.(*reportingPlugin[go.shape.[]uint8]).Observation.func1(),
      /home/runner/work/chainlink/chainlink/core/services/ocr3/promwrapper/plugin.go:58 +0x11d,
  github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper.withObservedExecution[go.shape.[]uint8,go.shape.[]uint8](),
      /home/runner/work/chainlink/chainlink/core/services/ocr3/promwrapper/plugin.go:143 +0x72,
  github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper.(*reportingPlugin[go.shape.[]uint8]).Observation(),
      /home/runner/work/chainlink/chainlink/core/services/ocr3/promwrapper/plugin.go:57 +0x1a4,
  github.com/smartcontractkit/chainlink/v2/core/services/ocr3/promwrapper.(*reportingPlugin[[]uint8]).Observation(),
      /home/runner/work/chainlink/chainlink/core/services/ocr3/promwrapper/plugin.go:56 +0xe4,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/shim.LimitCheckOCR3ReportingPlugin[go.shape.[]uint8].Observation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/shim/ocr3_reporting_plugin.go:39 +0x10d,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/shim.(*LimitCheckOCR3ReportingPlugin[[]uint8]).Observation(),
      \u003cautogenerated\u003e:1 +0x21b,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).backgroundObservation.func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:241 +0x158,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.callPluginFromOutcomeGenerationBackground[go.shape.[]uint8,go.shape.[]uint8].func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:391 +0xb5,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.callPluginFromBackground[go.shape.[]uint8](),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/common.go:72 +0x227,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.callPluginFromOutcomeGenerationBackground[go.shape.[]uint8,go.shape.[]uint8](),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:381 +0x297,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).backgroundObservation(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:234 +0x1d1,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).tryProcessRoundStartPool.func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:223 +0x14d,
  github.com/smartcontractkit/libocr/subprocesses.(*Subprocesses).Go.func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/subprocesses/subprocesses.go:29 +0x86,

Goroutine 35490 (running) created at:,
  github.com/smartcontractkit/libocr/subprocesses.(*Subprocesses).Go(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/subprocesses/subprocesses.go:27 +0xe4,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).tryProcessRoundStartPool(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:222 +0xe8b,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).messageRoundStart(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:175 +0xe51,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.MessageRoundStart[go.shape.[]uint8].processOutcomeGeneration(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/message.go:275 +0x116,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*MessageRoundStart[[]uint8]).processOutcomeGeneration(),
      \u003cautogenerated\u003e:1 +0x29,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).messageToOutcomeGeneration(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:245 +0x2a4,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).run(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:204 +0x8d5,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.RunOutcomeGeneration[go.shape.[]uint8](),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:68 +0x3cc,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*oracleState[go.shape.[]uint8]).run.func2(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/oracle.go:188 +0x50f,
  github.com/smartcontractkit/libocr/subprocesses.(*Subprocesses).Go.func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/subprocesses/subprocesses.go:29 +0x86,

Goroutine 34343 (running) created at:,
  github.com/smartcontractkit/libocr/subprocesses.(*Subprocesses).Go(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/subprocesses/subprocesses.go:27 +0xe4,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).tryProcessRoundStartPool(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:222 +0xe8b,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).messageRoundStart(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation_follower.go:175 +0xe51,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.MessageRoundStart[go.shape.[]uint8].processOutcomeGeneration(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/message.go:275 +0x116,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*MessageRoundStart[[]uint8]).processOutcomeGeneration(),
      \u003cautogenerated\u003e:1 +0x29,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).messageToOutcomeGeneration(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:245 +0x2a4,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*outcomeGenerationState[go.shape.[]uint8]).run(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:204 +0x8d5,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.RunOutcomeGeneration[go.shape.[]uint8](),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/outcome_generation.go:68 +0x3cc,
  github.com/smartcontractkit/libocr/offchainreporting2plus/internal/ocr3/protocol.(*oracleState[go.shape.[]uint8]).run.func2(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/offchainreporting2plus/internal/ocr3/protocol/oracle.go:188 +0x50f,
  github.com/smartcontractkit/libocr/subprocesses.(*Subprocesses).Go.func1(),
      /home/runner/go/pkg/mod/github.com/smartcontractkit/[email protected]/subprocesses/subprocesses.go:29 +0x86,

@jmank88 jmank88 requested a review from a team as a code owner January 14, 2025 20:26
@jmank88 jmank88 requested a review from krehermann January 14, 2025 20:26
@jmank88 jmank88 requested a review from makramkd January 14, 2025 20:26
makramkd
makramkd approved these changes Jan 14, 2025
View reviewed changes
Copy link
Contributor

@makramkd makramkd left a comment

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚡

@pavel-raykov pavel-raykov merged commit 4e61572 into main Jan 15, 2025
11 checks passed
@pavel-raykov pavel-raykov deleted the error-buffer-race branch January 15, 2025 09:43
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Reviewers

@pavel-raykov pavel-raykov pavel-raykov approved these changes

@makramkd makramkd makramkd approved these changes

@krehermann krehermann Awaiting requested review from krehermann krehermann is a code owner automatically assigned from smartcontractkit/foundations

Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

Successfully merging this pull request may close these issues.
3 participants
@jmank88 @makramkd @pavel-raykov