From 80fd2e872d1aee2254cef8e24bb6edd18805f67a Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Mon, 15 Jun 2026 14:49:57 -0400 Subject: [PATCH 1/6] WIP: adding resource managers for triggers --- chain_capabilities/evm/go.mod | 1 + chain_capabilities/evm/main.go | 73 ++- .../evm/trigger/physical_filter_id.go | 63 +++ chain_capabilities/evm/trigger/store.go | 20 +- chain_capabilities/evm/trigger/trigger.go | 216 ++++++-- .../evm/trigger/trigger_metering_test.go | 490 ++++++++++++++++++ .../evm/trigger/trigger_test.go | 19 +- cron/go.mod | 12 +- cron/go.sum | 10 +- cron/main.go | 34 +- cron/trigger/metering_test.go | 375 ++++++++++++++ cron/trigger/trigger.go | 186 ++++++- cron/trigger/trigger_test.go | 26 +- http_trigger/trigger/connector_handler.go | 161 +++++- .../trigger/connector_handler_test.go | 398 +++++++++++++- .../gateway_metadata_publisher_test.go | 4 +- http_trigger/trigger/trigger.go | 81 ++- http_trigger/trigger/workflow.go | 20 +- http_trigger/trigger/workflow_test.go | 39 +- 19 files changed, 2077 insertions(+), 151 deletions(-) create mode 100644 chain_capabilities/evm/trigger/physical_filter_id.go create mode 100644 chain_capabilities/evm/trigger/trigger_metering_test.go create mode 100644 cron/trigger/metering_test.go diff --git a/chain_capabilities/evm/go.mod b/chain_capabilities/evm/go.mod index 753e68deb..beba4586b 100644 --- a/chain_capabilities/evm/go.mod +++ b/chain_capabilities/evm/go.mod @@ -13,6 +13,7 @@ require ( github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260410144512-ca02ad6ed16a github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.uber.org/zap v1.27.1 diff --git a/chain_capabilities/evm/main.go b/chain_capabilities/evm/main.go index 38faa09d5..e57d7a881 100644 --- a/chain_capabilities/evm/main.go +++ b/chain_capabilities/evm/main.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "strconv" "time" @@ -28,11 +29,13 @@ import ( "github.com/smartcontractkit/capabilities/chain_capabilities/evm/trigger" "github.com/smartcontractkit/capabilities/libs/loopserver" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" evmcappb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm" evmcapserver "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm/server" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/types/core" @@ -160,9 +163,19 @@ func (c *capabilityGRPCService) Initialise(ctx context.Context, dependencies cor // TODO: add org resolver capabilityID := fmt.Sprintf("%s (%d)", c.id, cfg.ChainID) - c.triggerService, err = trigger.NewLogTriggerService(evmRelayer, trigger.NewLogTriggerStore(), c.lggr, capabilityID, capabilityDonID, processor, messageBuilder, + // The ResourceManager owns the snapshot tick; the LogTriggerService starts it + // as a sub-service and registers itself, so it must be configured with a + // snapshot interval here. Identity/snapshots are gated by the same metering + // env flag as MeterRecords. + resourceManager := resourcemanager.NewResourceManager(c.lggr, resourcemanager.ResourceManagerConfig{ + Enabled: meterRecordsEnabled(c.lggr), + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + }) + baseIdentity := newBaseMeteringIdentity(dependencies) + c.triggerService, err = trigger.NewLogTriggerService(evmRelayer, trigger.NewLogTriggerStore(), c.lggr, capabilityID, processor, messageBuilder, cfg.LogTriggerPollInterval, cfg.LogTriggerSendChannelBufferSize, cfg.LogTriggerLimitQueryLogSize, c.limitsFactory, - dependencies.OrgResolver, dependencies.TriggerEventStore) + dependencies.OrgResolver, dependencies.TriggerEventStore, resourceManager, baseIdentity, c.chainSelector) if err != nil { return fmt.Errorf("error when creating trigger: %w", err) } @@ -197,6 +210,62 @@ func (c *capabilityGRPCService) Initialise(ctx context.Context, dependencies cor return nil } +// defaultMeteringProduct is the fallback metering product dimension used when +// the host did not inject one via the standardized Initialise channel (a legacy +// node or a boot path not yet updated). The other deployment dimensions +// (environment, zone, node_id) have no meaningful constant and are left empty +// in that case, as documented on StandardCapabilitiesDependencies. +const defaultMeteringProduct = "cre" + +// newBaseMeteringIdentity builds the EVM log trigger's base metering identity +// from the host-injected dependencies. It carries the six coarse dimensions +// plus the service-level resource/resource_type; the per-resource ResourceID is +// set per emit/snapshot. DONID is the capability DON when the host injected one +// (deps.CapabilityDonID); when 0, it is left empty here and resolved per emit +// from the consumer's WorkflowDonID (see LogTriggerService.resolveDONID). This +// reads deps.CapabilityDonID at the Initialise layer so the change is orthogonal +// to capabilities#619's NewLogTriggerService signature edit. +func newBaseMeteringIdentity(deps core.StandardCapabilitiesDependencies) resourcemanager.ResourceIdentity { + product := deps.Product + if product == "" { + product = defaultMeteringProduct + } + var donID string + if deps.CapabilityDonID != 0 { + donID = strconv.FormatUint(uint64(deps.CapabilityDonID), 10) + } + return resourcemanager.ResourceIdentity{ + Product: product, + Environment: deps.Environment, + Zone: deps.Zone, + DONID: donID, + NodeID: deps.NodeID, + Service: trigger.MeteringService, + Resource: trigger.MeteringResource, + ResourceType: trigger.MeteringResourceType, + } +} + +// meterRecordsEnabledEnvVar gates MeterRecord emission; the name is the +// cross-producer convention for the metering rollout (SHARED-2718). +const meterRecordsEnabledEnvVar = "CL_METER_RECORDS_ENABLED" + +// meterRecordsEnabled reads the metering gate from the environment. Unset or +// unparseable values disable emission; metering config must never prevent the +// capability from starting. +func meterRecordsEnabled(lggr logger.Logger) bool { + v := os.Getenv(meterRecordsEnabledEnvVar) + if v == "" { + return false + } + enabled, err := strconv.ParseBool(v) + if err != nil { + lggr.Warnw("Invalid value for "+meterRecordsEnabledEnvVar+", meter record emission disabled", "value", v, "error", err) + return false + } + return enabled +} + func (c *capabilityGRPCService) unmarshalConfig(configStr string) (*config.Config, error) { var cfg config.Config if err := json.Unmarshal([]byte(configStr), &cfg); err != nil { diff --git a/chain_capabilities/evm/trigger/physical_filter_id.go b/chain_capabilities/evm/trigger/physical_filter_id.go new file mode 100644 index 000000000..0d587739d --- /dev/null +++ b/chain_capabilities/evm/trigger/physical_filter_id.go @@ -0,0 +1,63 @@ +package trigger + +import ( + "crypto/sha256" + "encoding/hex" + "sort" + "strings" + + evmtypes "github.com/smartcontractkit/chainlink-common/pkg/types/chains/evm" +) + +// physicalFilterID returns the workflow-independent content identity of an EVM +// log filter: the lowercase hex SHA-256 over a canonical encoding of the +// filter's physical matching criteria (chain selector, addresses, event +// signatures, and positional topic slots). Two filters that match exactly the +// same on-chain logs hash to the same ID regardless of which workflow or +// trigger registered them, or of the order their addresses/sigs/topics were +// supplied. It is used as ResourceIdentity.ResourceID and as the +// RESERVE/RELEASE event identity so identical filters share one billable +// physical resource (R4). +// +// Canonicalization rules (each rule defeats a source of non-determinism): +// - addresses and event sigs are lowercased 0x-prefixed hex and sorted +// ascending: the matching set is order-independent; +// - topic2/topic3/topic4 are POSITIONAL — a value in topic2 is a different +// filter than the same value in topic3 — so each slot is encoded under its +// own positional tag, and within a slot the values are sorted ascending; +// - the chain selector scopes the hash so identical filters on different +// chains stay distinct. +// +// The preimage uses "|" as a top-level separator and "," within a set; the +// per-element hex encodings are fixed-width and contain neither, so the +// encoding is unambiguous. +func physicalFilterID(chainSelector string, addresses []evmtypes.Address, eventSigs, topic2, topic3, topic4 []evmtypes.Hash) string { + sortedAddrs := make([]string, len(addresses)) + for i, a := range addresses { + sortedAddrs[i] = "0x" + hex.EncodeToString(a[:]) + } + sort.Strings(sortedAddrs) + + canonHashes := func(hs []evmtypes.Hash) string { + out := make([]string, len(hs)) + for i, h := range hs { + out[i] = "0x" + hex.EncodeToString(h[:]) + } + sort.Strings(out) + return strings.Join(out, ",") + } + + // Topic slots are encoded positionally so the same value in different slots + // produces a different identity. + preimage := strings.Join([]string{ + "cs=" + chainSelector, + "addrs=" + strings.Join(sortedAddrs, ","), + "sigs=" + canonHashes(eventSigs), + "t2=" + canonHashes(topic2), + "t3=" + canonHashes(topic3), + "t4=" + canonHashes(topic4), + }, "|") + + sum := sha256.Sum256([]byte(preimage)) + return hex.EncodeToString(sum[:]) +} diff --git a/chain_capabilities/evm/trigger/store.go b/chain_capabilities/evm/trigger/store.go index ee781ea63..5658110b2 100644 --- a/chain_capabilities/evm/trigger/store.go +++ b/chain_capabilities/evm/trigger/store.go @@ -12,7 +12,25 @@ import ( ) type filter struct { - filterID string + filterID string + // physicalFilterID is the workflow-independent content hash of the filter's + // physical matching criteria (chain selector + canonicalized addresses, + // event sigs, and positional topics). It is the metering ResourceID and the + // RESERVE/RELEASE event identity, so the unregister, cleanup, snapshot, and + // graceful-close paths all reuse it from here without the request input. + physicalFilterID string + // reservedAddressCount is the number of filter addresses metered in the + // RESERVE record when the filter was registered. The matching RELEASE + // must carry the same value, and UnregisterLogTrigger ignores its request + // input, so the count is stashed here at registration. + reservedAddressCount int64 + // donID is stashed from the registration RequestMetadata so the + // unregister/cleanup/snapshot/close paths can emit a metering record with + // the same identity as the RESERVE, without the original request. It is the + // resolved metering DON ID string (capability DON, or the consumer + // WorkflowDonID fallback when the host did not inject a capability DON); + // empty when neither is known. + donID string expressions []query.Expression confidence primitives.ConfidenceLevel } diff --git a/chain_capabilities/evm/trigger/trigger.go b/chain_capabilities/evm/trigger/trigger.go index 5f11cfcef..de602411f 100644 --- a/chain_capabilities/evm/trigger/trigger.go +++ b/chain_capabilities/evm/trigger/trigger.go @@ -12,8 +12,6 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - capcommon "github.com/smartcontractkit/capabilities/chain_capabilities/common" - commoncfg "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink-common/pkg/beholder" @@ -23,6 +21,7 @@ import ( evmservice "github.com/smartcontractkit/chainlink-common/pkg/chains/evm" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" @@ -34,6 +33,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives/evm" "github.com/smartcontractkit/chainlink-common/pkg/workflows" "github.com/smartcontractkit/chainlink-common/pkg/workflows/events" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" "github.com/smartcontractkit/capabilities/chain_capabilities/evm/monitoring" ) @@ -44,6 +44,22 @@ const ( defaultLimitQueryLogSize = 1000 ) +// Metering identity constants for the EVM log trigger (SHARED-2711). These are +// the service-level dimensions of the base ResourceIdentity: Service is the +// stable service constant (it must not encode deployment environment or zone, +// which ride on the structured identity's coarse dimensions), Resource is the +// resource pool, and ResourceType is the billing unit converted to credits. +const ( + MeteringService = "evm-log-trigger" + MeteringResource = "log_filters" + MeteringResourceType = "log_filter_addresses" +) + +// LogTriggerService is a resourcemanager.Meterable: it is registered with the +// ResourceManager at start so the manager's snapshot tick polls its active +// filters. +var _ resourcemanager.Meterable = (*LogTriggerService)(nil) + type LogTriggerService struct { services.Service @@ -55,12 +71,17 @@ type LogTriggerService struct { lggr logger.Logger beholderProcessor beholder.ProtoProcessor messageBuilder *monitoring.MessageBuilder - - // capabilityDonID is the on-chain DON ID of this capability DON. - // Used to label emitted events with the sending DON ID, distinct from the - // consumer workflow's DON ID carried in RequestMetadata.WorkflowDonID. Zero - // means unknown; the labeler then falls back to WorkflowDonID. - capabilityDonID uint32 + resourceManager *resourcemanager.ResourceManager + // baseIdentity is the producer's base metering identity: the six coarse + // dimensions plus service/resource/resource_type, built once at Initialise. + // The per-resource ResourceID is set per emit/snapshot via WithResourceID. + // When the host did not inject a capability DON ID, baseIdentity.DONID is + // empty and is filled per-emit from the consumer's WorkflowDonID. + baseIdentity resourcemanager.ResourceIdentity + chainSelector string // decimal chain selector, the chain label on meter records + // rmUnregister removes this service from the ResourceManager's snapshot + // registry; it is set when the RM is started in start and invoked in close. + rmUnregister func() triggers LogTriggerStore logTriggerPollInterval time.Duration @@ -76,13 +97,15 @@ type LogTriggerService struct { // NewLogTriggerService creates a new instance of logTriggerService. func NewLogTriggerService(evmService types.EVMService, store LogTriggerStore, lggr logger.Logger, capabilityID string, - capabilityDonID uint32, beholderProcessor beholder.ProtoProcessor, messageBuilder *monitoring.MessageBuilder, logTriggerPollInterval time.Duration, logTriggerSendChannelBufferSize uint64, logTriggerLimitQueryLogSize uint64, limitsFactory limits.Factory, orgResolver orgresolver.OrgResolver, - triggerEventStore capabilities.EventStore) (*LogTriggerService, error) { + triggerEventStore capabilities.EventStore, + resourceManager *resourcemanager.ResourceManager, + baseIdentity resourcemanager.ResourceIdentity, + chainSelector uint64) (*LogTriggerService, error) { if capabilityID == "" { return nil, fmt.Errorf("capabilityID must be non-empty") } @@ -112,7 +135,9 @@ func NewLogTriggerService(evmService types.EVMService, store LogTriggerStore, lg lggr: lggr, beholderProcessor: beholderProcessor, messageBuilder: messageBuilder, - capabilityDonID: capabilityDonID, + resourceManager: resourceManager, + baseIdentity: baseIdentity, + chainSelector: strconv.FormatUint(chainSelector, 10), triggers: store, logTriggerPollInterval: logTriggerPollInterval, logTriggerSendChannelBufferSize: currentSendChannelBufferSize, @@ -122,6 +147,9 @@ func NewLogTriggerService(evmService types.EVMService, store LogTriggerStore, lg if lts.orgResolver == nil { lts.lggr.Warn("OrgResolver is nil, EVM log trigger capability will not be able to fetch organization ID") } + if lts.resourceManager == nil { + lts.lggr.Warn("ResourceManager is nil, EVM log trigger capability will not emit meter records") + } if err := lts.initLimiters(limitsFactory); err != nil { return nil, err } @@ -173,14 +201,49 @@ func (lts *LogTriggerService) start(ctx context.Context) error { ticker := services.NewTicker(duration) lts.lggr.Infof("Starting clean up of failed log poller filters every %s seconds", duration) lts.srvcEng.GoTick(ticker, lts.cleanUpStaleFilters) + + // The ResourceManager owns the snapshot tick: start it as a sub-service of + // this service and Register ourselves so its tick polls GetUtilization. We + // never run our own snapshot loop. The RM is fail-open and starting it must + // not gate the trigger service, so a start error is logged, not returned. + if lts.resourceManager != nil { + if err := lts.resourceManager.Start(ctx); err != nil { + lts.lggr.Errorw("failed to start metering ResourceManager; snapshots disabled", "err", err) + } else { + lts.rmUnregister = lts.resourceManager.Register(lts) + } + } return nil } func (lts *LogTriggerService) close() error { lts.baseTrigger.Stop() + // On graceful shutdown, release every still-active filter so a clean stop is + // not seen downstream as a leaked reservation. These releases reuse each + // filter's stashed physicalFilterID + reserved count, so they dedup against + // any user-facing RELEASE on the identical idempotency key. + lts.releaseActiveFiltersOnClose(context.Background()) + if lts.rmUnregister != nil { + lts.rmUnregister() + } + if lts.resourceManager != nil { + return lts.resourceManager.Close() + } return nil } +// releaseActiveFiltersOnClose emits a RELEASE MeterRecord for every filter still +// present in the store at shutdown, carrying the reserved address count. It runs +// before the ResourceManager is unregistered/closed. +func (lts *LogTriggerService) releaseActiveFiltersOnClose(ctx context.Context) { + if lts.resourceManager == nil { + return + } + for _, state := range lts.triggers.ReadAll() { + lts.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, state.filter, state.reservedAddressCount) + } +} + func (lts *LogTriggerService) cleanUpStaleFilters(ctx context.Context) { lts.lggr.Debugf("Starting cleanUpStaleFilters") telemetryContext := monitoring.TelemetryContext{TsStart: time.Now().UnixMilli(), RequestMetadata: capabilities.RequestMetadata{ @@ -215,7 +278,14 @@ func (lts *LogTriggerService) cleanUpStaleFilters(ctx context.Context) { if err := lts.EVMService.UnregisterLogTracking(ctx, filterID); err != nil { summary := fmt.Sprintf("failed to unregister log-tracking from the clean up thread: '%v' source triggerID: %s", err, filterID) monitoring.LogAndEmitError(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerCleanUpError(telemetryContext, summary, err.Error())) + continue } + // This is log-poller filter hygiene only; it emits no MeterRecord. An + // orphaned filter has no trigger state, so it is already absent from + // GetUtilization and therefore from subsequent Snapshots. Billing + // reconciles the lost reservation by that absence (the snapshot liveness + // mechanism), not by a synthetic cleanup RELEASE. We never expect, nor + // design around, orphan cleanup as a metering event. } } @@ -295,6 +365,21 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID Topic4: t4, } + expressions, confidence := lts.createLogRequest(ctx, addresses, sigs, t2, t3, t4, input.GetConfidence()) + + // Build the filter's metering identity once from the already-converted + // inputs: a workflow-independent content hash and the resolved DON ID. It is + // stashed on the trigger state so every later path (unregister, cleanup, + // snapshot, close) reproduces the same identity without the request input. + loggedFilter := filter{ + filterID: filterID, + physicalFilterID: physicalFilterID(lts.chainSelector, addresses, sigs, t2, t3, t4), + reservedAddressCount: int64(len(addresses)), + donID: lts.resolveDONID(meta.WorkflowDonID), + expressions: expressions, + confidence: confidence, + } + if err = lts.EVMService.RegisterLogTracking(ctx, filterQuery); err != nil { registerError := fmt.Errorf("failed to register log-tracking: '%w' for triggerID: %s, addresses: %v, eventSig: %v, topic2: %v, topic3: %v, topic4: %v", err, triggerID, filterQuery.Addresses, filterQuery.EventSigs, filterQuery.Topic2, filterQuery.Topic3, filterQuery.Topic4) @@ -309,7 +394,9 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID monitoring.LogAndEmitError(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerError(telemetryContext, triggerID, summary, err.Error())) return nil, caperrors.NewPublicSystemError(registerError, caperrors.Unavailable) } - expressions, confidence := lts.createLogRequest(ctx, addresses, sigs, t2, t3, t4, input.GetConfidence()) + // RESERVE only after RegisterLogTracking succeeds: a failed registration + // holds no addresses, so it must not be billed (see the no-reserve test). + lts.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, loggedFilter, loggedFilter.reservedAddressCount) monitoring.EmitInitiated(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerInitiated(telemetryContext, input)) @@ -323,11 +410,7 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID cancelFunc: cancel, lastBlock: fromBlock, unfinalizedSentEventIDs: make(map[string]*big.Int), - filter: filter{ - filterID: filterID, - expressions: expressions, - confidence: confidence, - }, + filter: loggedFilter, }) ctx = meta.ContextWithCRE(ctx) lts.startPolling(ctx, telemetryContext, triggerID, input, logCh) @@ -378,6 +461,75 @@ func (lts *LogTriggerService) generateFilterID(triggerID string) string { return triggerID + SuffixLogTriggerFilterID } +// resolveDONID returns the metering DON ID for an emit, applying the +// capabilities#619 0->WorkflowDonID rule: when the host injected a capability +// DON ID, baseIdentity.DONID is non-empty and used as-is; otherwise the +// consumer workflow's DON ID (from the request metadata) is the documented +// fallback. The result is stashed on the filter at registration so the +// unregister/cleanup/snapshot/close paths reproduce the same identity without +// the request. Empty when neither source is known. +func (lts *LogTriggerService) resolveDONID(workflowDonID uint32) string { + if lts.baseIdentity.DONID != "" { + return lts.baseIdentity.DONID + } + if workflowDonID != 0 { + return strconv.FormatUint(uint64(workflowDonID), 10) + } + return "" +} + +// identity returns the base metering identity with its DON ID and ResourceID +// set for one resource. donID is the value stashed on the filter at +// registration (see resolveDONID); resourceID is the physical filter content +// hash (empty when unrecoverable, e.g. an orphaned filter). +func (lts *LogTriggerService) identity(donID, resourceID string) resourcemanager.ResourceIdentity { + id := lts.baseIdentity + id.DONID = donID + id.ResourceID = resourceID + return id +} + +// emitMeterRecord emits a MeterRecord for a filter whose full state is known +// (register/unregister/close). The physical filter content hash is both the +// ResourceID and the RESERVE/RELEASE event identity, so a register retry and a +// later unregister/cleanup of the same physical filter dedup on an identical +// idempotency key. The resource is fully identified by its ResourceIdentity; +// there is no separate label metadata. Emission is fail-open and must never +// gate the path that calls it. +func (lts *LogTriggerService) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, f filter, value int64) { + if lts.resourceManager == nil { + return + } + identity := lts.identity(f.donID, f.physicalFilterID) + lts.resourceManager.EmitMeterRecord(ctx, identity, action, + resourcemanager.NewUtilization(identity, action, value, f.physicalFilterID)) +} + +// ResourceIdentity implements resourcemanager.Meterable: it returns the +// producer's base identity (the six coarse dimensions plus +// service/resource/resource_type). The per-resource DON ID and ResourceID are +// populated per active filter by GetUtilization. +func (lts *LogTriggerService) ResourceIdentity() resourcemanager.ResourceIdentity { + return lts.baseIdentity +} + +// GetUtilization implements resourcemanager.Meterable: it returns one snapshot +// entry per currently active log filter for the snapshot tick. It is a cheap +// in-memory read — triggers.ReadAll already returns a copy — with no I/O and no +// lock held across the loop, as the snapshot contract requires (R6). +func (lts *LogTriggerService) GetUtilization(_ context.Context) []resourcemanager.SnapshotEntry { + triggers := lts.triggers.ReadAll() + entries := make([]resourcemanager.SnapshotEntry, 0, len(triggers)) + for _, state := range triggers { + f := state.filter + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: lts.identity(f.donID, f.physicalFilterID), + Value: f.reservedAddressCount, + }) + } + return entries +} + func (lts *LogTriggerService) startPolling(ctx context.Context, telemetryContext monitoring.TelemetryContext, triggerID string, input *evmcappb.FilterLogTriggerRequest, logCh chan capabilities.TriggerAndId[*evmcappb.Log]) { lts.lggr.Infof("Starting polling for triggerID: %s, interval: %d", triggerID, lts.logTriggerPollInterval) ticker := defaultTickerFactory.NewTicker(lts.logTriggerPollInterval) @@ -496,19 +648,14 @@ func (lts *LogTriggerService) sendLogsToWorkflows(ctx context.Context, telemetry events.KeyWorkflowName, displayWorkflowName, ) - // Emit the *sending* capability DON ID. The trigger plugin runs on a capability - // DON (e.g. chain_capabilities_zone-a), separate from the consumer workflow's - // DON carried in RequestMetadata.WorkflowDonID. The workflow service needs the - // sender's DON to resolve on-chain quorum params (N, F). See CRE-4409. - // capabilityDonID is 0 when the host could not resolve it authoritatively - // (a multi-DON job-spec node, or a core node that pre-dates CRE-4409); in - // that case we fall back to WorkflowDonID. This fallback is permanent, not - // transitional, since the job-spec boot path is still supported. - switch { - case lts.capabilityDonID != 0: - labeler = labeler.With(events.KeyDonID, strconv.Itoa(int(lts.capabilityDonID))) - case telemetryContext.WorkflowDonID != 0: - labeler = labeler.With(events.KeyDonID, strconv.Itoa(int(telemetryContext.WorkflowDonID))) + // Emit the *sending* capability DON ID (CRE-4409). resolveDONID applies + // contract rule 8: the authoritative host-injected CapDONID (carried on + // baseIdentity) wins, and the consumer workflow's WorkflowDonID is used + // only when CapDONID is 0. This is the SAME resolver the metering + // identity uses (filter.donID is set from resolveDONID at registration), + // so the event label and the meter record cannot diverge. + if donID := lts.resolveDONID(telemetryContext.WorkflowDonID); donID != "" { + labeler = labeler.With(events.KeyDonID, donID) } if telemetryContext.WorkflowDonConfigVersion != 0 { labeler = labeler.With(events.KeyDonVersion, strconv.Itoa(int(telemetryContext.WorkflowDonConfigVersion))) @@ -589,7 +736,7 @@ func (lts *LogTriggerService) deliverLogReliably( } lts.lggr.Infow("Sending log event to pipe", "triggerID", triggerID, "eventID", eventID, "blockNumber", log.BlockNumber, "txHash", log.TxHash) - deliverCtx := capcommon.ContextWithOrgForDelivery(ctx, lts.lggr, lts.orgResolver, telemetryContext.RequestMetadata) + deliverCtx := lts.contextWithOrgForDelivery(ctx, telemetryContext.RequestMetadata) if err := lts.baseTrigger.DeliverEvent(deliverCtx, te, triggerID); err != nil { summary := fmt.Sprintf("failed to persist/deliver event (triggerID=%s, eventID=%s): %v", triggerID, eventID, err) lts.lggr.Error(summary) @@ -743,6 +890,13 @@ func (lts *LogTriggerService) UnregisterLogTrigger(ctx context.Context, triggerI trigger.cancelFunc() lts.triggers.Delete(triggerID) lts.baseTrigger.UnregisterTrigger(triggerID) + // The reservation ends with the trigger state, which is the only holder of + // the reserved address count and the physical filter identity. Both are + // reused from the stashed filter so this RELEASE carries the same value and + // identity as its RESERVE. If UnregisterLogTracking fails below, the filter + // is orphaned at the log poller and the stale-filter cleanup unregisters it + // later (silently — metering already emitted this RELEASE here). + lts.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, trigger.filter, trigger.reservedAddressCount) err := lts.EVMService.UnregisterLogTracking(ctx, lts.generateFilterID(triggerID)) if err != nil { diff --git a/chain_capabilities/evm/trigger/trigger_metering_test.go b/chain_capabilities/evm/trigger/trigger_metering_test.go new file mode 100644 index 000000000..1a02523d1 --- /dev/null +++ b/chain_capabilities/evm/trigger/trigger_metering_test.go @@ -0,0 +1,490 @@ +package trigger + +import ( + "bytes" + "context" + "errors" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + evmcappb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm" + evmservice "github.com/smartcontractkit/chainlink-common/pkg/chains/evm" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + evmtypes "github.com/smartcontractkit/chainlink-common/pkg/types/chains/evm" + evmmock "github.com/smartcontractkit/chainlink-common/pkg/types/mocks" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" +) + +const testChainSelector = "5009297550715157269" + +// testBaseIdentity is the producer base identity the metering tests build their +// LogTriggerService with. It carries every coarse dimension so the tests can +// assert each one is populated on the emitted records (the host-injected +// identity contract). DONID is the capability DON; an empty DONID exercises the +// WorkflowDonID fallback. +func testBaseIdentity() resourcemanager.ResourceIdentity { + return resourcemanager.ResourceIdentity{ + Product: "cre", + Environment: "staging", + Zone: "wf-zone-a", + DONID: "42", + NodeID: "csa-pubkey-hex", + Service: MeteringService, + Resource: MeteringResource, + ResourceType: MeteringResourceType, + } +} + +// fakeMeterEmitter captures MeterRecords and MeterSnapshots emitted through the +// ResourceManager. The two message types are distinguished by the entity +// attribute the emitter is called with. The manager now emits one MeterSnapshot +// per active resource (no single Snapshot envelope), so snapshots accumulates +// one message per resource. +type fakeMeterEmitter struct { + err error + emitCalls int + records []*meteringpb.MeterRecord + snapshots []*meteringpb.MeterSnapshot +} + +func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) error { + f.emitCalls++ + if f.err != nil { + return f.err + } + if isSnapshotEmit(attrKVs) { + var snapshot meteringpb.MeterSnapshot + if err := proto.Unmarshal(body, &snapshot); err != nil { + return err + } + f.snapshots = append(f.snapshots, &snapshot) + return nil + } + var record meteringpb.MeterRecord + if err := proto.Unmarshal(body, &record); err != nil { + return err + } + f.records = append(f.records, &record) + return nil +} + +// isSnapshotEmit reports whether the emitter attributes name the MeterSnapshot +// entity, so the fake can demux the two message types off the same Emit method. +// The key is beholder.AttrKeyEntity ("beholder_entity") and the value is the +// snapshot entity constant the ResourceManager emits with. +func isSnapshotEmit(attrKVs []any) bool { + for i := 0; i+1 < len(attrKVs); i += 2 { + if attrKVs[i] == beholder.AttrKeyEntity && attrKVs[i+1] == "metering.v1.MeterSnapshot" { + return true + } + } + return false +} + +// newMeteredTriggerObject builds a LogTriggerService whose ResourceManager is +// enabled and wired to a fake emitter. The poll interval is stretched so the +// polling goroutine stays quiet; metering happens on the register, unregister, +// cleanup, snapshot, and close paths only. +func newMeteredTriggerObject(t *testing.T, mockEVM *evmmock.EVMService, store LogTriggerStore) (*LogTriggerService, *fakeMeterEmitter, *clockwork.FakeClock) { + t.Helper() + lts := createTriggerObject(t, mockEVM, store) + lts.logTriggerPollInterval = time.Hour + emitter := &fakeMeterEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + lts.resourceManager = resourcemanager.NewResourceManager(logger.Test(t), + resourcemanager.ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + lts.baseIdentity = testBaseIdentity() + lts.chainSelector = testChainSelector + return lts, emitter, clock +} + +// meteringTestInput is a registration request with two filter addresses, so +// tests can tell an address count apart from a hardcoded 1. +func meteringTestInput() *evmcappb.FilterLogTriggerRequest { + return &evmcappb.FilterLogTriggerRequest{ + Addresses: [][]byte{expectedAddress, bytes.Repeat([]byte{0x42}, evmtypes.AddressLength)}, + Topics: topicsWithEventSig0, + } +} + +// assertBaseIdentity checks the six coarse dimensions + service/resource on the +// emitted record identity, proving the host-injected identity is carried. +func assertBaseIdentity(t *testing.T, id *meteringpb.ResourceIdentity) { + t.Helper() + require.NotNil(t, id) + require.Equal(t, "cre", id.GetProduct()) + require.Equal(t, "staging", id.GetEnvironment()) + require.Equal(t, "wf-zone-a", id.GetZone()) + require.Equal(t, "42", id.GetDonId()) + require.Equal(t, "csa-pubkey-hex", id.GetNodeId()) + require.Equal(t, MeteringService, id.GetService()) + require.Equal(t, MeteringResource, id.GetResource()) + require.Equal(t, MeteringResourceType, id.GetResourceType()) +} + +// expectedPhysicalFilterID recomputes the physical filter id for the metering +// test input via the production helper, so the tests assert against the real +// canonicalization rather than a frozen literal. +func expectedPhysicalFilterID(t *testing.T, input *evmcappb.FilterLogTriggerRequest) string { + t.Helper() + svc := &LogTriggerService{} + eventSigs, t2, t3, t4 := svc.getTopics(input) + addrs, err := evmservice.ConvertAddressesFromProto(input.GetAddresses()) + require.NoError(t, err) + sigs, err := evmservice.ConvertHashesFromProto(eventSigs) + require.NoError(t, err) + h2, err := evmservice.ConvertHashesFromProto(t2) + require.NoError(t, err) + h3, err := evmservice.ConvertHashesFromProto(t3) + require.NoError(t, err) + h4, err := evmservice.ConvertHashesFromProto(t4) + require.NoError(t, err) + return physicalFilterID(testChainSelector, addrs, sigs, h2, h3, h4) +} + +func TestLogTriggerMetering_ReserveOnRegister(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + meta := capabilities.RequestMetadata{WorkflowID: "wf-id", WorkflowOwner: "0xOwner"} + _, err := service.RegisterLogTrigger(t.Context(), triggerID, meta, meteringTestInput()) + require.NoError(t, err) + + require.Len(t, emitter.records, 1) + record := emitter.records[0] + assertBaseIdentity(t, record.GetIdentity()) + physID := expectedPhysicalFilterID(t, meteringTestInput()) + require.Equal(t, physID, record.GetIdentity().GetResourceId(), "resource_id must be the physical filter content hash") + require.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, record.GetAction()) + require.NotNil(t, record.GetUtilization()) + require.Equal(t, int64(2), record.GetUtilization().GetValue(), "RESERVE value must equal the filter address count") + expectedID := service.identity("42", physID) + require.Equal(t, + resourcemanager.IdempotencyKey(expectedID, meteringpb.MeterAction_METER_ACTION_RESERVE, physID), + record.GetUtilization().GetIdempotencyKey()) +} + +func TestLogTriggerMetering_DonIDFallbackToWorkflowDon(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + // Host did not inject a capability DON; the consumer's WorkflowDonID is the + // documented fallback resolved at emit time. + service.baseIdentity.DONID = "" + + meta := capabilities.RequestMetadata{WorkflowID: "wf-id", WorkflowOwner: "0xOwner", WorkflowDonID: 7} + _, err := service.RegisterLogTrigger(t.Context(), triggerID, meta, meteringTestInput()) + require.NoError(t, err) + + require.Len(t, emitter.records, 1) + require.Equal(t, "7", emitter.records[0].GetIdentity().GetDonId(), "empty capability DON must fall back to WorkflowDonID") +} + +func TestLogTriggerMetering_NoReserveOnRegisterFailure(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(errors.New("mocked register failure")).Once() + service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + _, err := service.RegisterLogTrigger(t.Context(), triggerID, capabilities.RequestMetadata{WorkflowID: "wf-id"}, meteringTestInput()) + require.Error(t, err) + require.Zero(t, emitter.emitCalls, "no RESERVE may be emitted for a failed registration") +} + +func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { + meta := capabilities.RequestMetadata{WorkflowID: "wf-id", WorkflowOwner: "0xOwner"} + + registerTrigger := func(t *testing.T, service *LogTriggerService) { + t.Helper() + _, err := service.RegisterLogTrigger(t.Context(), triggerID, meta, meteringTestInput()) + require.NoError(t, err) + // The trigger state (holding the reserved address count) is written by + // the polling goroutine; wait for it before unregistering. + require.Eventually(t, func() bool { + _, ok := service.triggers.Read(triggerID) + return ok + }, time.Second, time.Millisecond) + } + + assertRelease := func(t *testing.T, service *LogTriggerService, record *meteringpb.MeterRecord) { + t.Helper() + assertBaseIdentity(t, record.GetIdentity()) + require.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, record.GetAction()) + require.NotNil(t, record.GetUtilization()) + require.Equal(t, int64(2), record.GetUtilization().GetValue(), "RELEASE must carry the same value that was reserved") + physID := expectedPhysicalFilterID(t, meteringTestInput()) + require.Equal(t, physID, record.GetIdentity().GetResourceId()) + expectedID := service.identity("42", physID) + require.Equal(t, + resourcemanager.IdempotencyKey(expectedID, meteringpb.MeterAction_METER_ACTION_RELEASE, physID), + record.GetUtilization().GetIdempotencyKey()) + } + + t.Run("release pairs the reserve", func(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + evmService.On("UnregisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + registerTrigger(t, service) + require.NoError(t, service.UnregisterLogTrigger(t.Context(), triggerID, meta, &evmcappb.FilterLogTriggerRequest{})) + + require.Len(t, emitter.records, 2) + require.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, emitter.records[0].GetAction()) + assertRelease(t, service, emitter.records[1]) + require.Equal(t, emitter.records[0].GetUtilization().GetValue(), emitter.records[1].GetUtilization().GetValue()) + require.Equal(t, emitter.records[0].GetIdentity().GetResourceId(), emitter.records[1].GetIdentity().GetResourceId(), + "RESERVE and RELEASE must share one physical resource_id") + }) + + t.Run("release emitted even when UnregisterLogTracking fails", func(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + evmService.On("UnregisterLogTracking", mock.Anything, mock.Anything).Return(errors.New("mocked unregister failure")).Once() + service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + registerTrigger(t, service) + // The reservation is released here (from the stashed count) before the + // UnregisterLogTracking RPC. If the RPC fails the filter is orphaned at + // the log poller; the cleanup thread unregisters it silently, emitting + // no further metering record. + require.Error(t, service.UnregisterLogTrigger(t.Context(), triggerID, meta, &evmcappb.FilterLogTriggerRequest{})) + + require.Len(t, emitter.records, 2) + assertRelease(t, service, emitter.records[1]) + }) +} + +func TestLogTriggerMetering_OrphanCleanupEmitsNothing(t *testing.T) { + // Orphan cleanup is log-poller filter hygiene, never a metering event. A + // lost reservation is reconciled by the resource's absence from subsequent + // Snapshots (the liveness mechanism), not by a synthetic cleanup RELEASE. + t.Run("stale filter cleanup emits no meter record", func(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + store := NewLogTriggerStore() + service, emitter := newMeteredTriggerObject(t, mockEVM, store) + + liveFilterID := service.generateFilterID("live-trigger") + staleFilterID := service.generateFilterID("stale-trigger") + mockEVM.On("GetFiltersNames", mock.Anything).Return([]string{liveFilterID, staleFilterID}, nil).Once() + mockEVM.On("UnregisterLogTracking", mock.Anything, staleFilterID).Return(nil).Once() + // mimicking there's a live trigger with the filter registered to log poller + store.Write("live-trigger", logTriggerState{filter: filter{filterID: liveFilterID}}) + + service.cleanUpStaleFilters(t.Context()) + + require.Zero(t, emitter.emitCalls, "orphan cleanup must not emit any MeterRecord") + }) + + t.Run("emits nothing when cleanup unregister fails", func(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + service, emitter := newMeteredTriggerObject(t, mockEVM, NewLogTriggerStore()) + + staleFilterID := service.generateFilterID("stale-trigger") + mockEVM.On("GetFiltersNames", mock.Anything).Return([]string{staleFilterID}, nil).Once() + mockEVM.On("UnregisterLogTracking", mock.Anything, staleFilterID).Return(errors.New("mocked cleanup failure")).Once() + + service.cleanUpStaleFilters(t.Context()) + require.Zero(t, emitter.emitCalls, "orphan cleanup never emits a meter record") + }) +} + +func TestLogTriggerMetering_FailOpen(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + emitter.err = errors.New("mocked emitter failure") + + _, err := service.RegisterLogTrigger(t.Context(), triggerID, capabilities.RequestMetadata{WorkflowID: "wf-id"}, meteringTestInput()) + require.NoError(t, err, "a metering emit failure must never fail registration") + require.Equal(t, 1, emitter.emitCalls, "the emit was attempted and its failure swallowed") +} + +// TestPhysicalFilterID_Canonicalization proves the content hash is independent +// of the order addresses / event sigs / per-slot topic values are supplied, and +// independent of which workflow or trigger registered the filter, while staying +// sensitive to the positional topic slot. +func TestPhysicalFilterID_Canonicalization(t *testing.T) { + addrA := evmtypes.Address(expectedAddress) + addrB := evmtypes.Address(bytes.Repeat([]byte{0x42}, evmtypes.AddressLength)) + sig1 := evmtypes.Hash(eventSig0Example) + sig2 := evmtypes.Hash(bytes.Repeat([]byte{0x11}, evmtypes.HashLength)) + none := []evmtypes.Hash{} + + t.Run("address order does not change the id", func(t *testing.T) { + id1 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA, addrB}, []evmtypes.Hash{sig1}, none, none, none) + id2 := physicalFilterID(testChainSelector, []evmtypes.Address{addrB, addrA}, []evmtypes.Hash{sig1}, none, none, none) + require.Equal(t, id1, id2) + }) + + t.Run("event sig order does not change the id", func(t *testing.T) { + id1 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1, sig2}, none, none, none) + id2 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig2, sig1}, none, none, none) + require.Equal(t, id1, id2) + }) + + t.Run("topic values within a slot are order-independent", func(t *testing.T) { + id1 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, []evmtypes.Hash{sig1, sig2}, none, none) + id2 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, []evmtypes.Hash{sig2, sig1}, none, none) + require.Equal(t, id1, id2) + }) + + t.Run("topic slots are positional", func(t *testing.T) { + inSlot2 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, []evmtypes.Hash{sig2}, none, none) + inSlot3 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, none, []evmtypes.Hash{sig2}, none) + require.NotEqual(t, inSlot2, inSlot3, "the same value in topic2 vs topic3 is a different filter") + }) + + t.Run("different chain selector changes the id", func(t *testing.T) { + id1 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, none, none, none) + id2 := physicalFilterID("999", []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, none, none, none) + require.NotEqual(t, id1, id2) + }) + + t.Run("identical filters from different workflows/triggers share one id", func(t *testing.T) { + // physicalFilterID takes only physical criteria; workflow/trigger are not + // inputs. Two registrations with identical criteria therefore collide by + // construction, which this asserts end to end through the register path. + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Twice() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Twice() + service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + _, err := service.RegisterLogTrigger(t.Context(), "trigger-A", + capabilities.RequestMetadata{WorkflowID: "wf-1", WorkflowOwner: "0xOwner"}, meteringTestInput()) + require.NoError(t, err) + _, err = service.RegisterLogTrigger(t.Context(), "trigger-B", + capabilities.RequestMetadata{WorkflowID: "wf-2", WorkflowOwner: "0xOther"}, meteringTestInput()) + require.NoError(t, err) + + require.Len(t, emitter.records, 2) + require.Equal(t, + emitter.records[0].GetIdentity().GetResourceId(), + emitter.records[1].GetIdentity().GetResourceId(), + "identical physical filters must share one resource_id across workflows/triggers") + }) +} + +// TestLogTriggerMetering_Snapshot drives one snapshot tick and asserts one +// MeterSnapshot per active filter, each fully identified by its +// ResourceIdentity (physical resource_id) with the right value. The manager +// emits one MeterSnapshot message per resource; there is no label metadata, so +// snapshots are keyed by their physical resource_id. +func TestLogTriggerMetering_Snapshot(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + store := NewLogTriggerStore() + service, emitter, clock := newMeteredTriggerObject(t, mockEVM, store) + + physA := expectedPhysicalFilterID(t, meteringTestInput()) + store.Write("trigger-A", logTriggerState{filter: filter{ + filterID: service.generateFilterID("trigger-A"), + physicalFilterID: physA, + reservedAddressCount: 2, + donID: "42", + }}) + store.Write("trigger-B", logTriggerState{filter: filter{ + filterID: service.generateFilterID("trigger-B"), + physicalFilterID: "physB", + reservedAddressCount: 5, + donID: "42", + }}) + + unregister := service.resourceManager.Register(service) + t.Cleanup(unregister) + servicetest.Run(t, service.resourceManager) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(emitter.snapshots) == 2 + }, time.Second, time.Millisecond) + + require.Len(t, emitter.snapshots, 2, "one MeterSnapshot per active filter") + + byResourceID := map[string]*meteringpb.MeterSnapshot{} + for _, s := range emitter.snapshots { + assertBaseIdentity(t, s.GetIdentity()) + byResourceID[s.GetIdentity().GetResourceId()] = s + } + + a := byResourceID[physA] + require.NotNil(t, a) + require.Equal(t, int64(2), a.GetUtilization().GetValue()) + + b := byResourceID["physB"] + require.NotNil(t, b) + require.Equal(t, int64(5), b.GetUtilization().GetValue()) +} + +// TestLogTriggerMetering_Snapshot_NothingActive asserts an empty store emits no +// snapshots: billing zeroes a resource out by its absence from later snapshots. +func TestLogTriggerMetering_Snapshot_NothingActive(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + service, emitter, clock := newMeteredTriggerObject(t, mockEVM, NewLogTriggerStore()) + + unregister := service.resourceManager.Register(service) + t.Cleanup(unregister) + servicetest.Run(t, service.resourceManager) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Empty(t, emitter.snapshots, "an empty store emits no MeterSnapshot") +} + +// TestLogTriggerMetering_ReleaseOnGracefulClose asserts that closing the service +// releases every still-active filter so a clean shutdown is not seen as a leak. +func TestLogTriggerMetering_ReleaseOnGracefulClose(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + store := NewLogTriggerStore() + service, emitter := newMeteredTriggerObject(t, mockEVM, store) + + physA := expectedPhysicalFilterID(t, meteringTestInput()) + store.Write("trigger-A", logTriggerState{filter: filter{ + filterID: service.generateFilterID("trigger-A"), + physicalFilterID: physA, + reservedAddressCount: 2, + donID: "42", + }}) + store.Write("trigger-B", logTriggerState{filter: filter{ + filterID: service.generateFilterID("trigger-B"), + physicalFilterID: "physB", + reservedAddressCount: 5, + donID: "42", + }}) + + service.releaseActiveFiltersOnClose(t.Context()) + + require.Len(t, emitter.records, 2, "one RELEASE per active filter on graceful close") + for _, record := range emitter.records { + require.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, record.GetAction()) + assertBaseIdentity(t, record.GetIdentity()) + } + // The records carry no label metadata, so pair them by their physical + // resource_id (the only per-filter discriminator on the record). + byResourceID := map[string]*meteringpb.MeterRecord{} + for _, record := range emitter.records { + byResourceID[record.GetIdentity().GetResourceId()] = record + } + require.Equal(t, int64(2), byResourceID[physA].GetUtilization().GetValue()) + require.Equal(t, int64(5), byResourceID["physB"].GetUtilization().GetValue()) +} diff --git a/chain_capabilities/evm/trigger/trigger_test.go b/chain_capabilities/evm/trigger/trigger_test.go index 9cccae9a0..f91b28d79 100644 --- a/chain_capabilities/evm/trigger/trigger_test.go +++ b/chain_capabilities/evm/trigger/trigger_test.go @@ -30,6 +30,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" evmcappb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" evmtypes "github.com/smartcontractkit/chainlink-common/pkg/types/chains/evm" evmmock "github.com/smartcontractkit/chainlink-common/pkg/types/mocks" @@ -1261,33 +1262,33 @@ func TestNewLogTriggerService(t *testing.T) { t.Run("empty capability id", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, "", 0, beholderProcessor, messageBuilder, time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, capabilities.NewMemEventStore()) + _, err := NewLogTriggerService(evmService, store, lggr, "", beholderProcessor, messageBuilder, time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "capabilityID must be non-empty") }) t.Run("ok initialize interval", func(t *testing.T) { - trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, 10*time.Second, 0, 0, testLimitsFactory(t), nil, capabilities.NewMemEventStore()) + trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, beholderProcessor, messageBuilder, 10*time.Second, 0, 0, testLimitsFactory(t), nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.NoError(t, err) require.Equal(t, 10*time.Second, trigger.logTriggerPollInterval) require.Equal(t, uint64(1000), trigger.logTriggerSendChannelBufferSize) require.Equal(t, uint64(1000), trigger.limitAndSort.Limit.Count) }) t.Run("ok initialize all params", func(t *testing.T) { - trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, 10*time.Second, 100, 50, testLimitsFactory(t), nil, capabilities.NewMemEventStore()) + trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, beholderProcessor, messageBuilder, 10*time.Second, 100, 50, testLimitsFactory(t), nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.NoError(t, err) require.Equal(t, 10*time.Second, trigger.logTriggerPollInterval) require.Equal(t, uint64(100), trigger.logTriggerSendChannelBufferSize) require.Equal(t, uint64(50), trigger.limitAndSort.Limit.Count) }) t.Run("ok initialize buffer only", func(t *testing.T) { - trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, 10*time.Second, 10000, 0, testLimitsFactory(t), nil, capabilities.NewMemEventStore()) + trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, beholderProcessor, messageBuilder, 10*time.Second, 10000, 0, testLimitsFactory(t), nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.NoError(t, err) require.Equal(t, 10*time.Second, trigger.logTriggerPollInterval) require.Equal(t, uint64(10000), trigger.logTriggerSendChannelBufferSize) require.Equal(t, uint64(defaultLimitQueryLogSize), trigger.limitAndSort.Limit.Count) //default value for limit as 0 was provided }) t.Run("ok initialize query limit only", func(t *testing.T) { - trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, 10*time.Second, 0, 100, testLimitsFactory(t), nil, capabilities.NewMemEventStore()) + trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, beholderProcessor, messageBuilder, 10*time.Second, 0, 100, testLimitsFactory(t), nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.NoError(t, err) require.Equal(t, 10*time.Second, trigger.logTriggerPollInterval) require.Equal(t, uint64(defaultSendChannelBufferSize), trigger.logTriggerSendChannelBufferSize) //default value for buffer size as 0 was provided @@ -1296,25 +1297,25 @@ func TestNewLogTriggerService(t *testing.T) { // negative tests t.Run("negative poll interval", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, -1*time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, nil) + _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, beholderProcessor, messageBuilder, -1*time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, nil, nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "logTriggerPollInterval must be positive, got: -1s") }) t.Run("limit query log size >= send channel buffer size", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, time.Second, 5, 10, limits.Factory{Logger: lggr}, nil, nil) + _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, beholderProcessor, messageBuilder, time.Second, 5, 10, limits.Factory{Logger: lggr}, nil, nil, nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "logTriggerLimitQueryLogSize (10) must be less than logTriggerSendChannelBufferSize (5)") }) t.Run("limit query log size >= default send channel buffer size", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, time.Second, 0, defaultSendChannelBufferSize+1, limits.Factory{Logger: lggr}, nil, nil) + _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, beholderProcessor, messageBuilder, time.Second, 0, defaultSendChannelBufferSize+1, limits.Factory{Logger: lggr}, nil, nil, nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "logTriggerLimitQueryLogSize (1001) must be less than logTriggerSendChannelBufferSize (1000)") }) t.Run("nil trigger event store", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, nil) + _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, beholderProcessor, messageBuilder, time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, nil, nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "no trigger event store provided") }) diff --git a/cron/go.mod b/cron/go.mod index d76d4aae6..51aaac6a0 100644 --- a/cron/go.mod +++ b/cron/go.mod @@ -2,13 +2,21 @@ module github.com/smartcontractkit/capabilities/cron go 1.26.2 +// Unpublished local stack for SHARED-2709; drop once chainlink-common and +// chainlink-protos/metering/go are tagged. +replace ( + github.com/smartcontractkit/chainlink-common => ../../chainlink-common + github.com/smartcontractkit/chainlink-protos/metering/go => ../../chainlink-protos/metering/go +) + require ( github.com/go-co-op/gocron/v2 v2.18.0 github.com/google/uuid v1.6.0 github.com/jonboulle/clockwork v0.5.0 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 github.com/smartcontractkit/chainlink-common v0.11.2-0.20260529092756-a94bc8ce96d6 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260526195338-adcf8013a1b7 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260611123141-db97012a6c32 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 @@ -85,7 +93,7 @@ require ( github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect - github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260323124644-faea187e6997 // indirect + github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect github.com/smartcontractkit/libocr v0.0.0-20250912173940-f3ab0246e23d // indirect diff --git a/cron/go.sum b/cron/go.sum index f6f3aad2a..d07775fc9 100644 --- a/cron/go.sum +++ b/cron/go.sum @@ -213,18 +213,16 @@ github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924/go.mod h1:v0O0Au8RE00Z89QxBE6I2q9bR9r3+RO1gLD3oaO2WB0= github.com/smartcontractkit/chain-selectors v1.0.100 h1:wpiSpmI/eFjY+wx/nPr5VuNF4hki0prIBMKEaQWn3g4= github.com/smartcontractkit/chain-selectors v1.0.100/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260529092756-a94bc8ce96d6 h1:hms02zQQ0BPcp9CBwh/xda5KwJWdU0IIA/yjtwyRoA4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260529092756-a94bc8ce96d6/go.mod h1:jueIfDkkRexwGgLbVB7vGCZlNtd383zuwi4uHHwcbqc= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 h1:ucHu2bPDT/58AzSgnPDyp4IjnjVbrVWYD3bG5jCbXMY= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260526195338-adcf8013a1b7 h1:iljEJss3WOwcsMkWy72Yn2zvjw7Gyxc+RXL7r8YKM6g= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260526195338-adcf8013a1b7/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260611123141-db97012a6c32 h1:GNl+lLK0QCakqA1J1i7FoOai2JrOGOzNzSniMijaCjA= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260611123141-db97012a6c32/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260323124644-faea187e6997 h1:W0HKHO8eE8BckTRnhSdqjHKbJcnk068nEWYnWRu6tJY= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260323124644-faea187e6997/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e h1:Hv9Mww35LrufCdM9wtS9yVi/rEWGI1UnjHbcKKU0nVY= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e/go.mod h1:T4zH9R8R8lVWKfU7tUvYz2o2jMv1OpGCdpY2j2QZXzU= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= diff --git a/cron/main.go b/cron/main.go index a085c0391..c2f40b391 100644 --- a/cron/main.go +++ b/cron/main.go @@ -1,16 +1,48 @@ package main import ( + "os" + "strconv" + "github.com/smartcontractkit/capabilities/cron/trigger" "github.com/smartcontractkit/capabilities/libs/loopserver" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/cron/server" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" ) +// meterRecordsEnabledEnvVar gates MeterRecord emission; the name is the +// cross-producer convention for the metering rollout (SHARED-2718). +const meterRecordsEnabledEnvVar = "CL_METER_RECORDS_ENABLED" + +// meterRecordsEnabled reads the metering gate from the environment. Unset or +// unparseable values disable emission; metering config must never prevent the +// capability from starting. +func meterRecordsEnabled(lggr logger.Logger) bool { + v := os.Getenv(meterRecordsEnabledEnvVar) + if v == "" { + return false + } + enabled, err := strconv.ParseBool(v) + if err != nil { + lggr.Warnw("Invalid value for "+meterRecordsEnabledEnvVar+", meter record emission disabled", "value", v, "error", err) + return false + } + return enabled +} + func main() { loopserver.ServeNew(trigger.ServiceName, func(s *loop.Server) loop.StandardCapabilities { - triggerService, err := trigger.NewTriggerService(s.Logger, nil, s.LimitsFactory) + meters := resourcemanager.NewResourceManager(s.Logger, resourcemanager.ResourceManagerConfig{ + Enabled: meterRecordsEnabled(s.Logger), + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + }) + + triggerService, err := trigger.NewTriggerService(s.Logger, nil, s.LimitsFactory, meters) if err != nil { s.Logger.Fatalw("Failed to create cron trigger service", "error", err) } diff --git a/cron/trigger/metering_test.go b/cron/trigger/metering_test.go new file mode 100644 index 000000000..44efd13f6 --- /dev/null +++ b/cron/trigger/metering_test.go @@ -0,0 +1,375 @@ +package trigger + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + crontypedapi "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/cron" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" +) + +// fakeMeterEmitter captures metering records delivered through a real +// ResourceManager, so tests assert on exactly the bytes production would emit. +// It demultiplexes MeterRecord and MeterSnapshot messages by their beholder +// entity attribute. A non-nil err simulates delivery failure: nothing is +// recorded. +type fakeMeterEmitter struct { + mu sync.Mutex + err error + records []*meteringpb.MeterRecord + snapshots []*meteringpb.MeterSnapshot +} + +func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.err != nil { + return f.err + } + if attrEntity(attrKVs) == "metering.v1.MeterSnapshot" { + snapshot := &meteringpb.MeterSnapshot{} + if err := proto.Unmarshal(body, snapshot); err != nil { + return err + } + f.snapshots = append(f.snapshots, snapshot) + return nil + } + record := &meteringpb.MeterRecord{} + if err := proto.Unmarshal(body, record); err != nil { + return err + } + f.records = append(f.records, record) + return nil +} + +// attrEntity extracts the beholder entity attribute value from the variadic +// key/value attrs the ResourceManager passes to Emit. +func attrEntity(attrKVs []any) string { + for i := 0; i+1 < len(attrKVs); i += 2 { + if k, ok := attrKVs[i].(string); ok && k == "beholder_entity" { + if v, ok := attrKVs[i+1].(string); ok { + return v + } + } + } + return "" +} + +func (f *fakeMeterEmitter) Records() []*meteringpb.MeterRecord { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*meteringpb.MeterRecord(nil), f.records...) +} + +func (f *fakeMeterEmitter) Snapshots() []*meteringpb.MeterSnapshot { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*meteringpb.MeterSnapshot(nil), f.snapshots...) +} + +// meteredTestDeps are the host-injected identity dimensions used by metering +// tests. They mirror what the host populates via the Initialise channel. +var meteredTestDeps = core.StandardCapabilitiesDependencies{ + Product: "cre-mainline", + Environment: "staging", + Zone: "wf-zone-a", + NodeID: "csa-pubkey-1", + CapabilityDonID: 7, +} + +// expectedBaseIdentity is the base identity the Service builds from +// meteredTestDeps (resource_id left empty; set per trigger). +var expectedBaseIdentity = resourcemanager.ResourceIdentity{ + Product: "cre-mainline", + Environment: "staging", + Zone: "wf-zone-a", + DONID: "7", + NodeID: "csa-pubkey-1", + Service: "cron-trigger", + Resource: "trigger_registrations", + ResourceType: "operations", +} + +// newMeteredTriggerService builds an initialised trigger service whose +// ResourceManager is enabled and wired to emitter, with identity sourced from +// meteredTestDeps. Snapshots use a fake clock so tests advance the tick +// deterministically. +func newMeteredTriggerService(t *testing.T, clock clockwork.Clock, emitter resourcemanager.Emitter) (*Service, *resourcemanager.ResourceManager, *clockwork.FakeClock) { + t.Helper() + + fakeClock, ok := clock.(*clockwork.FakeClock) + if !ok { + fakeClock = clockwork.NewFakeClockAt(clock.Now()) + clock = fakeClock + } + + meters := resourcemanager.NewResourceManager(logger.Nop(), resourcemanager.ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + ts, err := NewTriggerService(logger.Nop(), clock, limits.Factory{}, meters) + require.NoError(t, err) + + config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) + require.NoError(t, err) + + deps := meteredTestDeps + deps.Config = string(config) + require.NoError(t, ts.Initialise(t.Context(), deps)) + + return ts, meters, fakeClock +} + +func TestCronTrigger_Metering_ReserveAndRelease(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + emitter := &fakeMeterEmitter{} + ts, _, _ := newMeteredTriggerService(t, fakeClock, emitter) + + metadata := capabilities.RequestMetadata{ + WorkflowID: workflowID1, + WorkflowOwner: "0xOwner-1", + } + ch, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + records := emitter.Records() + require.Len(t, records, 1, "expected exactly one RESERVE on successful registration") + reserve := records[0] + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, reserve.GetAction()) + + // Identity is populated from the host-injected deps, with resource_id set + // to the trigger_id (cron is workflow-scoped). + id := reserve.GetIdentity() + require.NotNil(t, id) + assert.Equal(t, "cre-mainline", id.GetProduct()) + assert.Equal(t, "staging", id.GetEnvironment()) + assert.Equal(t, "wf-zone-a", id.GetZone()) + assert.Equal(t, "7", id.GetDonId()) + assert.Equal(t, "csa-pubkey-1", id.GetNodeId()) + assert.Equal(t, "cron-trigger", id.GetService()) + assert.Equal(t, "trigger_registrations", id.GetResource()) + assert.Equal(t, "operations", id.GetResourceType()) + assert.Equal(t, triggerID1, id.GetResourceId()) + + require.NotNil(t, reserve.GetUtilization()) + assert.Equal(t, int64(1), reserve.GetUtilization().GetValue()) + assert.Equal(t, + resourcemanager.IdempotencyKey(expectedBaseIdentity.WithResourceID(triggerID1), meteringpb.MeterAction_METER_ACTION_RESERVE, triggerID1), + reserve.GetUtilization().GetIdempotencyKey()) + + // Each cron tick re-Writes the trigger to reschedule it; the Write + // happens before the channel send, so after receiving the event the + // callback path has fully run. It must not emit. + for range 3 { + fakeClock.Advance(time.Second) + <-ch + } + require.Len(t, emitter.Records(), 1, "cron tick callbacks must not emit meter records") + + require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) + records = emitter.Records() + require.Len(t, records, 2, "expected exactly one RELEASE on unregistration") + release := records[1] + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, release.GetAction()) + assert.Equal(t, reserve.GetIdentity().GetResourceId(), release.GetIdentity().GetResourceId()) + require.NotNil(t, release.GetUtilization()) + assert.Equal(t, int64(1), release.GetUtilization().GetValue()) + assert.NotEqual(t, reserve.GetUtilization().GetIdempotencyKey(), release.GetUtilization().GetIdempotencyKey(), + "RESERVE and RELEASE must not share an idempotency key") + + require.NoError(t, ts.Close()) +} + +func TestCronTrigger_Metering_NoEmitOnFailedPaths(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClock() + emitter := &fakeMeterEmitter{} + ts, _, _ := newMeteredTriggerService(t, fakeClock, emitter) + + metadata := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "owner-1"} + + // Invalid schedule: registration fails before allocation, nothing emitted. + _, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: "not-a-schedule"}) + require.NotNil(t, capErr) + require.Empty(t, emitter.Records()) + + // Unregistering a trigger that was never registered releases nothing. + require.Nil(t, ts.UnregisterTrigger(t.Context(), "missing", metadata, &crontypedapi.Config{Schedule: everySecond})) + require.Empty(t, emitter.Records()) + + // Duplicate registration fails and must not double-RESERVE. + _, capErr = ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + _, capErr = ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.NotNil(t, capErr) + require.Len(t, emitter.Records(), 1) + + // Unregister to avoid a graceful-close RELEASE from interfering with the + // single-RESERVE assertion intent. + require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) + require.NoError(t, ts.Close()) +} + +func TestCronTrigger_Metering_FailOpen(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClock() + emitter := &fakeMeterEmitter{err: errors.New("collector unavailable")} + ts, _, _ := newMeteredTriggerService(t, fakeClock, emitter) + + metadata := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "owner-1"} + + // Registration and unregistration succeed even though every emission fails. + ch, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + fakeClock.Advance(time.Second) + <-ch // trigger still fires + + require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) + require.Empty(t, emitter.Records()) + + require.NoError(t, ts.Close()) +} + +// TestCronTrigger_Metering_Snapshot asserts the Service implements Meterable +// such that a forced snapshot emits one MeterSnapshot per active trigger, each +// carrying the full per-resource identity (resource_id set to the trigger_id). +func TestCronTrigger_Metering_Snapshot(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + emitter := &fakeMeterEmitter{} + ts, rm, clock := newMeteredTriggerService(t, fakeClock, emitter) + _ = rm + + metadata1 := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "0xOwner-1"} + _, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata1, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + metadata2 := capabilities.RequestMetadata{WorkflowID: "workflow-id-2", WorkflowOwner: "owner-2"} + const triggerID2 = "test-id-2" + _, capErr = ts.RegisterTrigger(t.Context(), triggerID2, metadata2, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + // One MeterSnapshot per active trigger, value 1, full per-resource identity. + require.Eventually(t, func() bool { + return len(emitter.Snapshots()) == 2 + }, time.Second, time.Millisecond) + snapshots := emitter.Snapshots() + require.Len(t, snapshots, 2, "one MeterSnapshot per active trigger per tick") + + byTrigger := map[string]*meteringpb.MeterSnapshot{} + for _, s := range snapshots { + byTrigger[s.GetIdentity().GetResourceId()] = s + } + + s1 := byTrigger[triggerID1] + require.NotNil(t, s1) + assert.Equal(t, int64(1), s1.GetUtilization().GetValue()) + assert.Equal(t, "cron-trigger", s1.GetIdentity().GetService()) + assert.Equal(t, "trigger_registrations", s1.GetIdentity().GetResource()) + assert.Equal(t, "operations", s1.GetIdentity().GetResourceType()) + + s2 := byTrigger[triggerID2] + require.NotNil(t, s2) + assert.Equal(t, int64(1), s2.GetUtilization().GetValue()) + assert.Equal(t, triggerID2, s2.GetIdentity().GetResourceId()) + + require.NoError(t, ts.Close()) +} + +// TestCronTrigger_Metering_GracefulCloseReleases asserts that Close drains a +// RELEASE for every still-active registration, so a graceful shutdown does not +// leak reservations in billing. +func TestCronTrigger_Metering_GracefulCloseReleases(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + emitter := &fakeMeterEmitter{} + ts, _, _ := newMeteredTriggerService(t, fakeClock, emitter) + + metadata1 := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "0xOwner-1"} + _, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata1, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + metadata2 := capabilities.RequestMetadata{WorkflowID: "workflow-id-2", WorkflowOwner: "owner-2"} + const triggerID2 = "test-id-2" + _, capErr = ts.RegisterTrigger(t.Context(), triggerID2, metadata2, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + // Two RESERVEs so far. + require.Len(t, emitter.Records(), 2) + + require.NoError(t, ts.Close()) + + // Close drained a RELEASE for each active trigger. + records := emitter.Records() + require.Len(t, records, 4, "two RESERVEs + one RELEASE per active trigger on graceful close") + + releases := map[string]*meteringpb.MeterRecord{} + for _, r := range records[2:] { + require.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, r.GetAction()) + releases[r.GetIdentity().GetResourceId()] = r + } + require.Contains(t, releases, triggerID1) + require.Contains(t, releases, triggerID2) + assert.Equal(t, int64(1), releases[triggerID1].GetUtilization().GetValue()) +} + +// TestCronTrigger_Metering_DonIDFallback asserts the DON ID falls back to the +// consumer workflow's DON when the host has not injected a capability DON ID. +func TestCronTrigger_Metering_DonIDFallback(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + emitter := &fakeMeterEmitter{} + + meters := resourcemanager.NewResourceManager(logger.Nop(), resourcemanager.ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + }) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, meters) + require.NoError(t, err) + + config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) + require.NoError(t, err) + + // No CapabilityDonID injected (zero) → fall back to WorkflowDonID at emit. + require.NoError(t, ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{Config: string(config)})) + + metadata := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "owner-1", WorkflowDonID: 42} + _, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + records := emitter.Records() + require.Len(t, records, 1) + assert.Equal(t, "42", records[0].GetIdentity().GetDonId(), "DON ID falls back to WorkflowDonID") + // Product falls back to the cron constant when the host injects none. + assert.Equal(t, "cre", records[0].GetIdentity().GetProduct()) + + require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) + require.NoError(t, ts.Close()) +} diff --git a/cron/trigger/trigger.go b/cron/trigger/trigger.go index 3afd24602..5cd6142b9 100644 --- a/cron/trigger/trigger.go +++ b/cron/trigger/trigger.go @@ -21,6 +21,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/cron/server" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" @@ -28,6 +29,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types/core" "github.com/smartcontractkit/chainlink-common/pkg/workflows" "github.com/smartcontractkit/chainlink-common/pkg/workflows/events" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" ) const ServiceName = "CronCapabilities" @@ -40,6 +42,21 @@ var cronTriggerInfo = capabilities.MustNewCapabilityInfo( "A trigger that uses a cron schedule to run periodically at fixed times, dates, or intervals.", ) +const ( + // meteringService is the stable service constant for cron trigger + // registrations on emitted MeterRecords and Snapshots. It must not encode + // deployment environment or zone: those are discrete identity dimensions + // sourced from deps at Initialise. + meteringService = "cron-trigger" + // meteringResource is the resource pool cron records apply to. + meteringResource = "trigger_registrations" + // meteringResourceType is the billing unit for cron registrations. + meteringResourceType = "operations" + // meteringProductFallback is used when the host has not injected a Product + // (a legacy node or a boot path not yet updated to populate deps.Product). + meteringProductFallback = "cre" +) + type Config struct { FastestScheduleIntervalSeconds int `json:"fastestScheduleIntervalSeconds"` } @@ -57,6 +74,9 @@ type cronTrigger struct { } type Service struct { + services.Service + srvcEng *services.Engine + capabilities.CapabilityInfo limitsFactory limits.Factory fastestScheduleInterval limits.TimeLimiter @@ -66,7 +86,15 @@ type Service struct { triggers *cronStore labeler custmsg.MessageEmitter metrics *Metrics - orgResolver orgresolver.OrgResolver + meters *resourcemanager.ResourceManager + // unregisterMeterable removes this Service from the ResourceManager's + // snapshot registry; set at start, called at close. Nil until started. + unregisterMeterable func() + // base is the resourcemanager identity for cron registrations, built from + // the host-injected deployment/node/DON dimensions at Initialise. ResourceID + // is left empty here and set per trigger via base.WithResourceID(triggerID). + base resourcemanager.ResourceIdentity + orgResolver orgresolver.OrgResolver } func (s *Service) RegisterLegacyTrigger(ctx context.Context, triggerID string, metadata capabilities.RequestMetadata, input *crontypedapi.Config) (<-chan capabilities.TriggerAndId[*crontypedapi.LegacyPayload], caperrors.Error) { //nolint:staticcheck @@ -101,13 +129,21 @@ func (s *Service) UnregisterLegacyTrigger(ctx context.Context, triggerID string, return s.UnregisterTrigger(ctx, triggerID, metadata, input) } -var _ services.Service = &Service{} +var ( + _ services.Service = &Service{} + _ resourcemanager.Meterable = &Service{} +) // NewTriggerService creates a new trigger service. Optionally, a clock can be passed in for testing, if nil // the system clock will be used. The orgResolver is optional and can be nil, but should be set in live environments. -func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFactory limits.Factory) (*Service, error) { +// meters reports trigger registrations for billing; if nil, a disabled no-op manager is used. +func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFactory limits.Factory, meters *resourcemanager.ResourceManager) (*Service, error) { lggr := logger.Named(parentLggr, "CRONTrigger") + if meters == nil { + meters = resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{}) + } + metrics, err := NewMetrics() if err != nil { return nil, fmt.Errorf("error creating metrics: %w", err) @@ -131,7 +167,7 @@ func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFa return nil, fmt.Errorf("error creating scheduler: %w", err) } - return &Service{ + s := &Service{ lggr: lggr, CapabilityInfo: cronTriggerInfo, limitsFactory: limitsFactory, @@ -144,7 +180,44 @@ func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFa "capabilityName", cronTriggerInfo.ID, ), metrics: metrics, - }, nil + meters: meters, + } + + // Adopt services.Engine so the trigger can host the ResourceManager as a + // sub-service (the RM owns the snapshot tick) and shut down cleanly. The + // scheduler is started/stopped in s.start / s.close. + s.Service, s.srvcEng = services.Config{ + Name: "CronTrigger", + NewSubServices: func(logger.Logger) []services.Service { return []services.Service{meters} }, + Start: s.start, + Close: s.close, + }.NewServiceEngine(lggr) + + return s, nil +} + +// identityFor returns the per-trigger metering identity: the base identity +// with ResourceID set to triggerID. resource_id is workflow-scoped (the +// trigger_id) for cron, which has no shared physical resource. The DON ID +// falls back to the consumer workflow's DON when the host has not injected a +// capability DON ID (deps.CapabilityDonID == 0). +func (s *Service) identityFor(triggerID string, workflowDonID uint32) resourcemanager.ResourceIdentity { + id := s.base.WithResourceID(triggerID) + if id.DONID == "" { + id.DONID = strconv.FormatUint(uint64(workflowDonID), 10) + } + return id +} + +// emitMeterRecord reports a change to this trigger's registration reservation +// for billing. The triggerID doubles as the idempotency event identity: a +// triggerID is registered at most once at a time, so retried emissions for the +// same registration dedup downstream. Emission is fail-open and never affects +// the registration itself. +func (s *Service) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, metadata capabilities.RequestMetadata, triggerID string) { + id := s.identityFor(triggerID, metadata.WorkflowDonID) + s.meters.EmitMeterRecord(ctx, id, action, + resourcemanager.NewUtilization(id, action, 1, triggerID)) } func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapabilitiesDependencies) error { @@ -173,6 +246,30 @@ func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapa s.lggr.Warn("OrgResolver is nil, cron capability will not be able to fetch organization ID") } + // Build the base metering identity from the host-injected deployment, node, + // and DON dimensions. These arrive via the standardized Initialise channel + // (mirroring how capabilities#619 injects CapabilityDonID). Any may be + // empty/zero until the host is updated to populate them; DONID falls back to + // the consumer workflow DON at emit time (see identityFor). + product := dependencies.Product + if product == "" { + product = meteringProductFallback + } + var donID string + if dependencies.CapabilityDonID != 0 { + donID = strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10) + } + s.base = resourcemanager.ResourceIdentity{ + Product: product, + Environment: dependencies.Environment, + Zone: dependencies.Zone, + DONID: donID, + NodeID: dependencies.NodeID, + Service: meteringService, + Resource: meteringResource, + ResourceType: meteringResourceType, + } + err = s.Start(ctx) if err != nil { return fmt.Errorf("error when starting trigger service: %w", err) @@ -356,6 +453,8 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat close: closeCh, }) + s.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, metadata, triggerID) + s.lggr.Debugw("Trigger registered", "workflowId", metadata.WorkflowID, "triggerId", triggerID, "jobId", job.ID()) s.metrics.IncActiveTriggersGauge(ctx) return callbackCh, nil @@ -407,17 +506,26 @@ func (s *Service) UnregisterTrigger(ctx context.Context, triggerID string, metad // Remove from triggers context s.triggers.Delete(triggerID) + s.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, metadata, triggerID) + s.lggr.Debugw("UnregisterTrigger", "triggerId", triggerID, "jobId", jobID) s.metrics.DecActiveTriggersGauge(ctx) return nil } -// Start the service. -func (s *Service) Start(ctx context.Context) error { +// start is the services.Engine start hook. The ResourceManager sub-service has +// already been started by the engine, so start registers this Service as a +// Meterable (the RM polls it once per snapshot tick) and starts the scheduler, +// refreshing next-run times for any registrations that survived a restart. +func (s *Service) start(_ context.Context) error { if s.scheduler == nil { return errors.New("service has shutdown, it must be built again to restart") } + // Register for snapshots. The RM owns the tick; we only supply state via + // the Meterable interface. unregisterMeterable is called in close. + s.unregisterMeterable = s.meters.Register(s) + s.scheduler.Start() for triggerID, trigger := range s.triggers.ReadAll() { @@ -433,19 +541,35 @@ func (s *Service) Start(ctx context.Context) error { } } - s.lggr.Info(s.Name() + " started") - return nil } -// Close stops the Service. -// After this call the Service cannot be started again, -// The service will need to be re-built to start scheduling again. -func (s *Service) Close() error { +// close is the services.Engine close hook. After this the Service cannot be +// started again; it must be re-built to schedule again. close drains a RELEASE +// for every still-active registration (so a graceful shutdown does not leak +// reservations in billing), unregisters from the snapshot registry, then shuts +// the scheduler down. The ResourceManager sub-service is closed by the engine +// afterwards. +func (s *Service) close() error { if s.scheduler == nil { return errors.New("service has shutdown, it must be built again to restart") } + // Graceful-close RELEASEs. Use a background context: the engine's start + // context is already cancelled by the time close runs. Emission is + // fail-open, so a metering failure never blocks shutdown. + ctx := context.Background() + for triggerID := range s.triggers.ReadAll() { + id := s.base.WithResourceID(triggerID) + s.meters.EmitMeterRecord(ctx, id, meteringpb.MeterAction_METER_ACTION_RELEASE, + resourcemanager.NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RELEASE, 1, triggerID)) + } + + if s.unregisterMeterable != nil { + s.unregisterMeterable() + s.unregisterMeterable = nil + } + err := s.scheduler.Shutdown() if err != nil { return fmt.Errorf("scheduler shutdown encountered a problem: %s", err) @@ -455,23 +579,35 @@ func (s *Service) Close() error { // but calling .Start() on it will not error. Set to nil to mark closed. s.scheduler = nil - s.lggr.Info(s.Name() + " closed") - - return nil -} - -func (s *Service) Ready() error { return nil } -func (s *Service) HealthReport() map[string]error { - return map[string]error{s.Name(): nil} +func (s *Service) Description() string { + return "Cron Trigger Capability" } -func (s *Service) Name() string { - return s.lggr.Name() +// ResourceIdentity implements resourcemanager.Meterable: it returns the base +// six-dimension identity (resource_id left empty; set per active trigger in +// GetUtilization). +func (s *Service) ResourceIdentity() resourcemanager.ResourceIdentity { + return s.base } -func (s *Service) Description() string { - return "Cron Trigger Capability" +// GetUtilization implements resourcemanager.Meterable: it returns the absolute +// state of every currently active cron registration, one SnapshotEntry per +// trigger, each at value 1 (a registration is a single reserved unit). It is a +// cheap in-memory read of the store snapshot and tolerates ctx cancellation. +func (s *Service) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { + if ctx.Err() != nil { + return nil + } + triggers := s.triggers.ReadAll() + entries := make([]resourcemanager.SnapshotEntry, 0, len(triggers)) + for triggerID := range triggers { + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: s.base.WithResourceID(triggerID), + Value: 1, + }) + } + return entries } diff --git a/cron/trigger/trigger_test.go b/cron/trigger/trigger_test.go index 3eca57abd..bba0a4b38 100644 --- a/cron/trigger/trigger_test.go +++ b/cron/trigger/trigger_test.go @@ -252,7 +252,7 @@ func successWithStandardCronIntervals(t *testing.T, useTypedAPI bool) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -333,7 +333,7 @@ func TestCronTrigger_Load(t *testing.T) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -480,7 +480,7 @@ func testCronTriggerRegisterTriggerBeforeStart(t *testing.T, useTypedAPI bool) { fakeClock := clockwork.NewRealClock() config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -553,7 +553,7 @@ func testCronTriggerTimeWindows(t *testing.T, useTypedAPI bool) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -629,7 +629,7 @@ func testCronTriggerMultipleDifferentSchedules(t *testing.T, useTypedAPI bool) { } config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -752,7 +752,7 @@ func testCronTriggerTimeZone(t *testing.T, useTypedAPI bool) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -865,7 +865,7 @@ func testCronTriggerRegisterTrigger(t *testing.T, useTypedAPI bool) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { fakeClock := clockwork.NewRealClock() - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{}) require.NoError(t, err) @@ -904,7 +904,7 @@ func TestCronTrigger_RegisterTriggerDuplicateError(t *testing.T) { triggerConfig, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) fakeClock := clockwork.NewRealClock() - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(triggerConfig), @@ -939,7 +939,7 @@ func TestCronTrigger_UnregisterTriggerError(t *testing.T) { triggerConfig, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) fakeClock := clockwork.NewRealClock() - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(triggerConfig), @@ -1018,7 +1018,7 @@ func TestCronTrigger_UnregisterTriggerError(t *testing.T) { }) t.Run("NOK fails to unregister if closed", func(t *testing.T) { - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(triggerConfig), @@ -1056,7 +1056,7 @@ func TestCronTrigger_UnregisterTriggerError(t *testing.T) { func TestCronTrigger_CloseStartErrors(t *testing.T) { fakeClock := clockwork.NewRealClock() - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) ctx := t.Context() @@ -1082,7 +1082,7 @@ func TestGocronNewTaskPanic(t *testing.T) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) logger, observedLogs := logger.TestObserved(t, zap.ErrorLevel) - ts, err := NewTriggerService(logger, fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger, fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -1176,7 +1176,7 @@ func TestCronTrigger_ExecutionIDWithTriggerIndex(t *testing.T) { triggerConfig, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(lggr, fakeClock, limits.Factory{}) + ts, err := NewTriggerService(lggr, fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{Config: string(triggerConfig)}) require.NoError(t, err) diff --git a/http_trigger/trigger/connector_handler.go b/http_trigger/trigger/connector_handler.go index 48c680c0d..217dbd6ce 100644 --- a/http_trigger/trigger/connector_handler.go +++ b/http_trigger/trigger/connector_handler.go @@ -16,13 +16,14 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/custmsg" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" - "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types/core" gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" "github.com/smartcontractkit/chainlink-common/pkg/workflows" "github.com/smartcontractkit/chainlink-common/pkg/workflows/events" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" ) const ( @@ -32,12 +33,17 @@ const ( var _ core.GatewayConnectorHandler = &connectorHandler{} +// connectorHandler implements resourcemanager.Meterable: it owns the +// workflowStore and the base metering identity, so it both emits lifecycle +// edges inline and reports the absolute state of active registrations on the +// ResourceManager's snapshot tick. +var _ resourcemanager.Meterable = &connectorHandler{} + type connectorHandler struct { services.StateMachine lggr logger.Logger gatewayConnector core.GatewayConnector config ServiceConfig - capabilityDonID uint32 // authoritative sending DON ID; 0 = unknown, falls back to WorkflowDONID requestCache *requestCache workflowStore *workflowStore gatewayMetadataPublisher GatewayMetadataPublisher @@ -45,22 +51,34 @@ type connectorHandler struct { wg sync.WaitGroup stopChan services.StopChan orgResolver orgresolver.OrgResolver // Optional org resolver for fetching organization IDs + resourceManager *resourcemanager.ResourceManager + // baseIdentity is the six-dimension + resource/resource_type metering + // identity for this trigger LOOP, built once at Initialise. The + // per-workflow resource_id is derived per emission via WithResourceID. + baseIdentity resourcemanager.ResourceIdentity + // unregisterMeterable removes this handler from the ResourceManager's + // snapshot registry; set on Start, called on Close. + unregisterMeterable func() } -func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config ServiceConfig, capabilityDonID uint32, - workflowStore *workflowStore, gatewayMetadataPublisher GatewayMetadataPublisher, requestCache *requestCache, metrics *Metrics, - orgResolver orgresolver.OrgResolver, limitsFactory limits.Factory) (*connectorHandler, error) { +func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config ServiceConfig, + workflowStore *workflowStore, gatewayMetadataPublisher GatewayMetadataPublisher, requestCache *requestCache, metrics *Metrics, orgResolver orgresolver.OrgResolver, + resourceManager *resourcemanager.ResourceManager, baseIdentity resourcemanager.ResourceIdentity) (*connectorHandler, error) { + if resourceManager == nil { + resourceManager = resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{}) + } return &connectorHandler{ lggr: logger.Named(lggr, HandlerName), gatewayConnector: gc, config: config, - capabilityDonID: capabilityDonID, workflowStore: workflowStore, gatewayMetadataPublisher: gatewayMetadataPublisher, requestCache: requestCache, metrics: metrics, stopChan: make(chan struct{}), orgResolver: orgResolver, + resourceManager: resourceManager, + baseIdentity: baseIdentity, }, nil } @@ -69,6 +87,13 @@ func (h *connectorHandler) Start(ctx context.Context) error { h.wg.Add(1) go h.startRequestCacheCleanup(ctx) return h.StartOnce(HandlerName, func() error { + // Start the ResourceManager as a sub-service (it owns the snapshot + // tick) and register this handler as the snapshotted Meterable. The RM + // is fail-open and disabled by default, so this never gates startup. + if err := h.resourceManager.Start(ctx); err != nil { + return err + } + h.unregisterMeterable = h.resourceManager.Register(h) return h.gatewayConnector.AddHandler(ctx, []string{ gateway_common.MethodWorkflowExecute, gateway_common.MethodPullWorkflowMetadata, @@ -103,10 +128,27 @@ func (h *connectorHandler) Close() error { return h.StopOnce(HandlerName, func() error { close(h.stopChan) h.wg.Wait() - return nil + // Drain RELEASEs for every still-active workflow so reservations do not + // leak past shutdown, then unregister from the snapshot tick and stop + // the ResourceManager. Order matters: release before unregister/close so + // the final edges are emitted while emission is still wired up. + h.releaseActiveWorkflows(context.Background()) + if h.unregisterMeterable != nil { + h.unregisterMeterable() + } + return h.resourceManager.Close() }) } +// releaseActiveWorkflows emits a RELEASE for each workflow still active in the +// store at shutdown. It is fail-open: emission never blocks or fails close. +func (h *connectorHandler) releaseActiveWorkflows(ctx context.Context) { + for _, w := range h.workflowStore.getWorkflows() { + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, + w.workflowSelector.WorkflowID, w.metadata.WorkflowDONID) + } +} + func (h *connectorHandler) HealthReport() map[string]error { return map[string]error{h.Name(): h.Healthy()} } @@ -146,14 +188,90 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR h.metrics.RecordBroadcastMetadataLatency(ctx, latencyMs, h.lggr) workflow := newWorkflowWithMetadata(input.WorkflowSelector, authorizedKeys, sendCh, input.Metadata) - if err := h.workflowStore.upsertWorkflow(workflow); err != nil { + prevWorkflowID, replaced, err := h.workflowStore.upsertWorkflow(workflow) + if err != nil { return fmt.Errorf("failed to register workflow (ID: %s, Owner: %s, Name: %s): %w", input.WorkflowSelector.WorkflowID, input.WorkflowSelector.WorkflowOwner, input.WorkflowSelector.WorkflowName, err) } + newWorkflowID := input.WorkflowSelector.WorkflowID + workflowDONID := input.Metadata.WorkflowDONID + switch { + case !replaced: + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, newWorkflowID, workflowDONID) + case prevWorkflowID == newWorkflowID: + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_UPDATE, newWorkflowID, workflowDONID) + default: + // Version update: the same owner/name/tag reference now resolves to a + // new workflow ID. Release the previous workflow's reservation before + // reserving the new one so the old reservation does not leak. + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, prevWorkflowID, workflowDONID) + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, newWorkflowID, workflowDONID) + } h.lggr.Debugw("Registered workflow", "workflowID", input.WorkflowSelector.WorkflowID, "workflowOwner", input.WorkflowSelector.WorkflowOwner, "workflowName", input.WorkflowSelector.WorkflowName, "workflowTag", input.WorkflowSelector.WorkflowTag) return nil } +// emitMeterRecord emits a meter record for one workflow registration +// operation. resource_id is the workflow ID (HTTP registrations are +// workflow-scoped, so there is no shared physical resource); the workflow ID +// also doubles as the event identity, so a repeated emission for the same +// workflow and action dedups downstream. The workflow_id is recoverable from +// resource_id and the owner is resolved downstream, so no label metadata is +// attached. Emission is fail-open and never affects the registration outcome. +func (h *connectorHandler) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, workflowID string, workflowDONID uint32) { + identity := h.identityForWorkflow(workflowID, workflowDONID) + h.resourceManager.EmitMeterRecord(ctx, identity, action, + resourcemanager.NewUtilization(identity, action, 1, workflowID)) +} + +// identityForWorkflow derives the per-workflow metering identity: the base +// identity with resource_id set to the workflow ID, and DONID resolved per +// registration when the host did not inject a capability DON. +func (h *connectorHandler) identityForWorkflow(workflowID string, workflowDONID uint32) resourcemanager.ResourceIdentity { + identity := h.baseIdentity.WithResourceID(workflowID) + identity.DONID = h.donID(workflowDONID) + return identity +} + +// donID returns the DON identifier for an emission. It prefers the +// host-injected capability DON captured in the base identity (capabilities#619) +// and falls back to the per-registration workflow DON when the host did not +// populate one (CapabilityDonID == 0 at Initialise). +func (h *connectorHandler) donID(workflowDONID uint32) string { + if h.baseIdentity.DONID != "" { + return h.baseIdentity.DONID + } + if workflowDONID != 0 { + return strconv.FormatUint(uint64(workflowDONID), 10) + } + return "" +} + +// ResourceIdentity returns the HTTP trigger's base metering identity (six +// dimensions + resource / resource_type). The per-workflow resource_id is +// populated by GetUtilization. It implements resourcemanager.Meterable. +func (h *connectorHandler) ResourceIdentity() resourcemanager.ResourceIdentity { + return h.baseIdentity +} + +// GetUtilization returns the absolute state of currently active HTTP workflow +// registrations, one SnapshotEntry per workflow, for the ResourceManager's +// snapshot tick. It is a cheap read-snapshot of in-memory state (a read-locked +// copy from the workflow store) and holds no lock across I/O. It implements +// resourcemanager.Meterable. +func (h *connectorHandler) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { + workflows := h.workflowStore.getWorkflows() + entries := make([]resourcemanager.SnapshotEntry, 0, len(workflows)) + for _, w := range workflows { + workflowID := w.workflowSelector.WorkflowID + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: h.identityForWorkflow(workflowID, w.metadata.WorkflowDONID), + Value: 1, + }) + } + return entries +} + func (h *connectorHandler) validateAuthorizedKeys(inputKeys []*http.AuthorizedKey) ([]gateway_common.AuthorizedKey, error) { if len(inputKeys) == 0 { return nil, fmt.Errorf("HTTP trigger requires at least one authorized key to sign JSON-RPC requests. Add AuthorizedKeys to your http.Trigger configuration with ECDSA EVM public keys (0x-prefixed hex strings)") @@ -181,10 +299,17 @@ func (h *connectorHandler) validateAuthorizedKeys(inputKeys []*http.AuthorizedKe } func (h *connectorHandler) UnregisterWorkflow(ctx context.Context, workflowID string) error { + // Snapshot the workflow DON before removal; it is needed for the meter + // record's identity DON fallback. + var workflowDONID uint32 + if w, ok := h.workflowStore.getWorkflowByID(workflowID); ok { + workflowDONID = w.metadata.WorkflowDONID + } err := h.workflowStore.removeWorkflow(workflowID) if err != nil { return fmt.Errorf("failed to unregister workflow %s: %w", workflowID, err) } + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, workflowID, workflowDONID) h.lggr.Debugw("Unregistered workflow", "workflowID", workflowID) return nil } @@ -294,17 +419,13 @@ func (h *connectorHandler) processTrigger(ctx context.Context, gatewayID string, displayWorkflowName = workflowMetadata.WorkflowName } - // Emit the *sending* capability DON ID. The HTTP trigger plugin runs on a - // capability DON, separate from the consumer workflow's DON. The workflow - // service needs the sender's DON to resolve on-chain quorum params (N, F). - // See CRE-4409. capabilityDonID is 0 when the host could not resolve it - // authoritatively (a multi-DON job-spec node, or a core node that pre-dates - // CRE-4409); in that case we fall back to WorkflowDONID. This fallback is - // permanent, not transitional, since the job-spec boot path is still supported. - donIDForEvent := h.capabilityDonID - if donIDForEvent == 0 { - donIDForEvent = workflowMetadata.WorkflowDONID - } + // Resolve the *sending* capability DON ID once (CRE-4409). h.donID applies + // contract rule 8: the authoritative host-injected CapDONID (carried on + // baseIdentity) wins, and the consumer workflow's WorkflowDONID is used only + // when CapDONID is 0. This is the SAME resolver the metering identity uses + // (identityForWorkflow -> donID), so the event label and the meter record + // cannot diverge. + donIDForEvent := h.donID(workflowMetadata.WorkflowDONID) labeler := custmsg.NewLabeler().With( events.KeyTriggerID, req.ID, @@ -315,7 +436,7 @@ func (h *connectorHandler) processTrigger(ctx context.Context, gatewayID string, events.KeyWorkflowRegistryChainSelector, workflowMetadata.WorkflowRegistryChainSelector, events.KeyWorkflowRegistryAddress, workflowMetadata.WorkflowRegistryAddress, events.KeyEngineVersion, workflowMetadata.EngineVersion, - events.KeyDonID, strconv.Itoa(int(donIDForEvent)), + events.KeyDonID, donIDForEvent, ) // Try to fetch organization ID if org resolver is available diff --git a/http_trigger/trigger/connector_handler_test.go b/http_trigger/trigger/connector_handler_test.go index c5c964f0a..e60f1c218 100644 --- a/http_trigger/trigger/connector_handler_test.go +++ b/http_trigger/trigger/connector_handler_test.go @@ -4,21 +4,27 @@ import ( "context" "database/sql" "encoding/json" + "errors" "strings" "sync" "testing" "time" + "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/types/core" gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" "github.com/smartcontractkit/chainlink-common/pkg/workflows" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" ) const ( @@ -190,13 +196,13 @@ func setupWithTriggerChannelBuffer(t *testing.T, lggr logger.Logger, triggerChBu lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) sdkCfg := &http.Config{ @@ -596,13 +602,13 @@ func TestRegisterWorkflow_TooManyAuthorizedKeys(t *testing.T) { lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) @@ -715,13 +721,13 @@ func TestConnectorHandler_Start_HealthReport_Ready_Name_Close(t *testing.T) { lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) @@ -876,13 +882,13 @@ func TestHandleGatewayMessage_PullAuthMetadata_EmptyWorkflows(t *testing.T) { lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) @@ -1054,13 +1060,13 @@ func TestConnectorHandler_StartRequestCacheCleanup(t *testing.T) { lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) @@ -1122,6 +1128,376 @@ func TestHandleGatewayMessage_NilRequest(t *testing.T) { require.Contains(t, err.Error(), "request cannot be nil") } +// fakeMeterEmitter decodes and records the MeterRecords and MeterSnapshots +// passed to Emit and can be configured to fail, for asserting fail-open +// behavior. It dispatches on the beholder entity attribute so MeterRecord and +// MeterSnapshot bodies are decoded with the correct type. Each MeterSnapshot +// covers exactly one resource. +type fakeMeterEmitter struct { + err error + records []*meteringpb.MeterRecord + snapshots []*meteringpb.MeterSnapshot +} + +func (f *fakeMeterEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { + if f.entity(attrKVs) == "metering.v1.MeterSnapshot" { + var snapshot meteringpb.MeterSnapshot + if err := proto.Unmarshal(body, &snapshot); err != nil { + return err + } + f.snapshots = append(f.snapshots, &snapshot) + return f.err + } + var record meteringpb.MeterRecord + if err := proto.Unmarshal(body, &record); err != nil { + return err + } + f.records = append(f.records, &record) + return f.err +} + +// entity returns the value of the beholder entity attribute from the +// alternating key/value attrKVs slice, or "" if absent. +func (f *fakeMeterEmitter) entity(attrKVs []any) string { + for i := 0; i+1 < len(attrKVs); i += 2 { + if k, ok := attrKVs[i].(string); ok && k == beholder.AttrKeyEntity { + if v, ok := attrKVs[i+1].(string); ok { + return v + } + } + } + return "" +} + +func (f *fakeMeterEmitter) actions() []meteringpb.MeterAction { + actions := make([]meteringpb.MeterAction, len(f.records)) + for i, r := range f.records { + actions[i] = r.GetAction() + } + return actions +} + +// testBaseIdentity is the base metering identity used by metering tests. It +// carries the six coarse dimensions plus the service-level resource / +// resource_type; per-workflow resource_id is derived per emission. +var testBaseIdentity = resourcemanager.ResourceIdentity{ + Product: "cre-test", + Environment: "staging", + Zone: "wf-zone-a", + DONID: "7", + NodeID: "node-csa-pubkey", + Service: meterService, + Resource: meterResource, + ResourceType: meterResourceType, +} + +// setupWithMeterEmitter builds a handler with metering enabled and a fake +// emitter capturing emitted MeterRecords. The ResourceManager is started (so +// the snapshot tick is wired) and the handler is registered as the snapshotted +// Meterable; both are torn down on test cleanup. No workflows are registered. +func setupWithMeterEmitter(t *testing.T, lggr logger.Logger, emitErr error) (*connectorHandler, *fakeMeterEmitter) { + t.Helper() + emitter := &fakeMeterEmitter{err: emitErr} + cfg := ServiceConfig{ + MetadataBatchSize: 10, + MaxAuthorizedKeysPerWorkflow: 3, + } + store := newWorkflowStore(lggr) + metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) + requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) + resourceManager := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + }) + handler, err := NewConnectorHandler( + lggr, + &mockGatewayConnector{}, + cfg, + store, + metadataPublisher, + requestCache, + newMetrics(t), + nil, + resourceManager, + testBaseIdentity, + ) + require.NoError(t, err) + require.NoError(t, handler.Start(t.Context())) + t.Cleanup(func() { require.NoError(t, handler.Close()) }) + return handler, emitter +} + +func meterTestRegistrationInput() WorkflowRegistrationInput { + return WorkflowRegistrationInput{ + WorkflowSelector: gateway_common.WorkflowSelector{ + WorkflowID: testWorkflowID, + WorkflowOwner: testWorkflowOwner, + WorkflowName: testWorkflowName, + WorkflowTag: testWorkflowTag, + }, + Config: &http.Config{ + AuthorizedKeys: []*http.AuthorizedKey{ + { + PublicKey: publicKey, + Type: http.KeyType_KEY_TYPE_ECDSA_EVM, + }, + }, + }, + Metadata: WorkflowRegistrationMetadata{}, + } +} + +func TestRegisterWorkflow_MetersReserveThenUpdate(t *testing.T) { + lggr := logger.Test(t) + handler, emitter := setupWithMeterEmitter(t, lggr, nil) + input := meterTestRegistrationInput() + + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + err := handler.RegisterWorkflow(t.Context(), input, sendCh) + require.NoError(t, err) + + // First registration reserves exactly once, with the full structured + // identity populated on the record. + require.Equal(t, []meteringpb.MeterAction{meteringpb.MeterAction_METER_ACTION_RESERVE}, emitter.actions()) + record := emitter.records[0] + id := record.GetIdentity() + require.Equal(t, testBaseIdentity.Product, id.GetProduct()) + require.Equal(t, testBaseIdentity.Environment, id.GetEnvironment()) + require.Equal(t, testBaseIdentity.Zone, id.GetZone()) + require.Equal(t, testBaseIdentity.DONID, id.GetDonId()) + require.Equal(t, testBaseIdentity.NodeID, id.GetNodeId()) + require.Equal(t, meterService, id.GetService()) + require.Equal(t, meterResource, id.GetResource()) + require.Equal(t, meterResourceType, id.GetResourceType()) + // resource_id is the workflow ID (HTTP registrations are workflow-scoped). + require.Equal(t, testWorkflowID, id.GetResourceId()) + require.Equal(t, int64(1), record.GetUtilization().GetValue()) + require.Equal(t, + resourcemanager.IdempotencyKey(testBaseIdentity.WithResourceID(testWorkflowID), meteringpb.MeterAction_METER_ACTION_RESERVE, testWorkflowID), + record.GetUtilization().GetIdempotencyKey()) + + // Re-registering the same workflow emits UPDATE, not a second RESERVE. + sendCh2 := make(chan capabilities.TriggerAndId[*http.Payload], 1) + err = handler.RegisterWorkflow(t.Context(), input, sendCh2) + require.NoError(t, err) + require.Equal(t, []meteringpb.MeterAction{ + meteringpb.MeterAction_METER_ACTION_RESERVE, + meteringpb.MeterAction_METER_ACTION_UPDATE, + }, emitter.actions()) +} + +func TestRegisterWorkflow_VersionUpdate_MetersReleaseThenReserve(t *testing.T) { + lggr := logger.Test(t) + handler, emitter := setupWithMeterEmitter(t, lggr, nil) + + inputA := meterTestRegistrationInput() + inputA.WorkflowSelector.WorkflowID = testWorkflowID1 + sendChA := make(chan capabilities.TriggerAndId[*http.Payload], 1) + require.NoError(t, handler.RegisterWorkflow(t.Context(), inputA, sendChA)) + + // Re-registering the same owner/name/tag reference with a NEW workflow ID + // is a version update: the previous workflow's reservation is released + // before the new one is reserved, so the old reservation cannot leak. + inputB := meterTestRegistrationInput() + inputB.WorkflowSelector.WorkflowID = testWorkflowID2 + sendChB := make(chan capabilities.TriggerAndId[*http.Payload], 1) + require.NoError(t, handler.RegisterWorkflow(t.Context(), inputB, sendChB)) + + require.Equal(t, []meteringpb.MeterAction{ + meteringpb.MeterAction_METER_ACTION_RESERVE, + meteringpb.MeterAction_METER_ACTION_RELEASE, + meteringpb.MeterAction_METER_ACTION_RESERVE, + }, emitter.actions()) + + // RESERVE(A) anchors the old workflow ID via its resource_id. + reserveA := emitter.records[0] + require.Equal(t, testWorkflowID1, reserveA.GetIdentity().GetResourceId()) + + // RELEASE targets the PREVIOUS workflow ID under the same owner; its + // resource_id is that previous workflow ID. + release := emitter.records[1] + require.Equal(t, testWorkflowID1, release.GetIdentity().GetResourceId()) + require.Equal(t, + resourcemanager.IdempotencyKey(testBaseIdentity.WithResourceID(testWorkflowID1), meteringpb.MeterAction_METER_ACTION_RELEASE, testWorkflowID1), + release.GetUtilization().GetIdempotencyKey()) + + // The trailing RESERVE anchors the new workflow ID. + reserveB := emitter.records[2] + require.Equal(t, testWorkflowID2, reserveB.GetIdentity().GetResourceId()) + require.Equal(t, + resourcemanager.IdempotencyKey(testBaseIdentity.WithResourceID(testWorkflowID2), meteringpb.MeterAction_METER_ACTION_RESERVE, testWorkflowID2), + reserveB.GetUtilization().GetIdempotencyKey()) +} + +func TestUnregisterWorkflow_MetersRelease(t *testing.T) { + lggr := logger.Test(t) + handler, emitter := setupWithMeterEmitter(t, lggr, nil) + + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + err := handler.RegisterWorkflow(t.Context(), meterTestRegistrationInput(), sendCh) + require.NoError(t, err) + + err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) + require.NoError(t, err) + require.Equal(t, []meteringpb.MeterAction{ + meteringpb.MeterAction_METER_ACTION_RESERVE, + meteringpb.MeterAction_METER_ACTION_RELEASE, + }, emitter.actions()) + release := emitter.records[1] + require.Equal(t, testWorkflowID, release.GetIdentity().GetResourceId()) + + // Unregistering an absent workflow fails and must not emit RELEASE. + err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) + require.Error(t, err) + require.Len(t, emitter.records, 2) +} + +func TestRegisterWorkflow_MeteringFailOpen(t *testing.T) { + lggr := logger.Test(t) + handler, emitter := setupWithMeterEmitter(t, lggr, errors.New("emit failed")) + + // Registration and unregistration succeed even though every emit fails. + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + err := handler.RegisterWorkflow(t.Context(), meterTestRegistrationInput(), sendCh) + require.NoError(t, err) + err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) + require.NoError(t, err) + require.Len(t, emitter.records, 2) +} + +// registerMeterWorkflow registers a workflow with the given ID/owner under the +// metering test config (each registration uses a distinct reference so they +// coexist). +func registerMeterWorkflow(t *testing.T, handler *connectorHandler, workflowID, owner string) { + t.Helper() + input := meterTestRegistrationInput() + input.WorkflowSelector.WorkflowID = workflowID + input.WorkflowSelector.WorkflowOwner = owner + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + require.NoError(t, handler.RegisterWorkflow(t.Context(), input, sendCh)) +} + +// TestSnapshot_EmitsOneEntryPerActiveWorkflow starts the ResourceManager tick +// and asserts one MeterSnapshot per active workflow, each carrying the full +// per-workflow identity. +func TestSnapshot_EmitsOneEntryPerActiveWorkflow(t *testing.T) { + lggr := logger.Test(t) + emitter := &fakeMeterEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + cfg := ServiceConfig{MetadataBatchSize: 10, MaxAuthorizedKeysPerWorkflow: 3} + store := newWorkflowStore(lggr) + metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) + requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) + rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, testBaseIdentity) + require.NoError(t, err) + unregister := rm.Register(handler) + t.Cleanup(unregister) + + registerMeterWorkflow(t, handler, testWorkflowID1, testWorkflowOwner1) + registerMeterWorkflow(t, handler, testWorkflowID2, testWorkflowOwner2) + + // Drop the lifecycle RESERVE records; we assert only on the snapshot tick. + emitter.records = nil + servicetest.Run(t, rm) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(emitter.snapshots) == 2 + }, time.Second, time.Millisecond) + + // One MeterSnapshot per active workflow, each fully identified by its own + // per-workflow identity (resource_id is the workflow ID). + require.Len(t, emitter.snapshots, 2) + byWorkflowID := map[string]*meteringpb.MeterSnapshot{} + for _, s := range emitter.snapshots { + byWorkflowID[s.GetIdentity().GetResourceId()] = s + } + require.Len(t, byWorkflowID, 2) + + r1 := byWorkflowID[testWorkflowID1] + require.NotNil(t, r1) + require.Equal(t, testBaseIdentity.Product, r1.GetIdentity().GetProduct()) + require.Equal(t, meterResource, r1.GetIdentity().GetResource()) + require.Equal(t, meterResourceType, r1.GetIdentity().GetResourceType()) + require.Equal(t, int64(1), r1.GetUtilization().GetValue()) + + r2 := byWorkflowID[testWorkflowID2] + require.NotNil(t, r2) + require.Equal(t, int64(1), r2.GetUtilization().GetValue()) +} + +// TestClose_EmitsReleasePerActiveWorkflow asserts graceful close drains a +// RELEASE for every still-active workflow so reservations do not leak. +func TestClose_EmitsReleasePerActiveWorkflow(t *testing.T) { + lggr := logger.Test(t) + emitter := &fakeMeterEmitter{} + cfg := ServiceConfig{MetadataBatchSize: 10, MaxAuthorizedKeysPerWorkflow: 3} + store := newWorkflowStore(lggr) + metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) + requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) + rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + }) + handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, testBaseIdentity) + require.NoError(t, err) + require.NoError(t, handler.Start(t.Context())) + + registerMeterWorkflow(t, handler, testWorkflowID1, testWorkflowOwner1) + registerMeterWorkflow(t, handler, testWorkflowID2, testWorkflowOwner2) + + // Drop the lifecycle RESERVE records; assert only on the close drain. + emitter.records = nil + require.NoError(t, handler.Close()) + + require.Equal(t, []meteringpb.MeterAction{ + meteringpb.MeterAction_METER_ACTION_RELEASE, + meteringpb.MeterAction_METER_ACTION_RELEASE, + }, emitter.actions()) + + released := map[string]bool{} + for _, r := range emitter.records { + released[r.GetIdentity().GetResourceId()] = true + } + require.True(t, released[testWorkflowID1]) + require.True(t, released[testWorkflowID2]) +} + +// TestDONIDFallback_UsesWorkflowDON asserts that when the host did not inject a +// capability DON (base DONID empty), records fall back to the per-registration +// workflow DON. +func TestDONIDFallback_UsesWorkflowDON(t *testing.T) { + lggr := logger.Test(t) + emitter := &fakeMeterEmitter{} + cfg := ServiceConfig{MetadataBatchSize: 10, MaxAuthorizedKeysPerWorkflow: 3} + store := newWorkflowStore(lggr) + metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) + requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) + rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{Enabled: true, Emitter: emitter}) + // Base identity WITHOUT a capability DON (host did not inject one). + base := testBaseIdentity + base.DONID = "" + handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, base) + require.NoError(t, err) + + input := meterTestRegistrationInput() + input.Metadata.WorkflowDONID = 99 + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + require.NoError(t, handler.RegisterWorkflow(t.Context(), input, sendCh)) + + require.Len(t, emitter.records, 1) + require.Equal(t, "99", emitter.records[0].GetIdentity().GetDonId()) +} + // TestResolveWorkflowMetadata_PreservesStoredWorkflowOwner tests that the workflowOwner // from the stored workflow is used, even if the incoming request has zeros or missing values. // This is a regression test for the bug where workflowOwner was being set to zeros. diff --git a/http_trigger/trigger/gateway_metadata_publisher_test.go b/http_trigger/trigger/gateway_metadata_publisher_test.go index bedf22bdf..2b0e0a435 100644 --- a/http_trigger/trigger/gateway_metadata_publisher_test.go +++ b/http_trigger/trigger/gateway_metadata_publisher_test.go @@ -221,9 +221,9 @@ func TestSendWorkflows_Success(t *testing.T) { wf1 := newWorkflow(selector1, authorizedKeys1, sendCh1) wf2 := newWorkflow(selector2, authorizedKeys2, sendCh2) - err := workflowStore.upsertWorkflow(wf1) + _, _, err := workflowStore.upsertWorkflow(wf1) require.NoError(t, err) - err = workflowStore.upsertWorkflow(wf2) + _, _, err = workflowStore.upsertWorkflow(wf2) require.NoError(t, err) gatewayID := "gateway1" diff --git a/http_trigger/trigger/trigger.go b/http_trigger/trigger/trigger.go index abb164be9..5e1952711 100644 --- a/http_trigger/trigger/trigger.go +++ b/http_trigger/trigger/trigger.go @@ -4,14 +4,18 @@ import ( "context" "encoding/json" "fmt" + "os" + "strconv" "strings" "time" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" caperrors "github.com/smartcontractkit/chainlink-common/pkg/capabilities/errors" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http/server" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -21,6 +25,39 @@ import ( const ServiceName = "HTTPTriggerCapability" +// Metering identity constants for the HTTP trigger. Service is the stable +// service constant (it must not encode environment or zone); Resource and +// ResourceType identify the HTTP workflow-registration pool and its billing +// unit. +const ( + meterService = "http-trigger" + meterResource = "http_workflows" + meterResourceType = "operations" + // meterProductFallback is used when the host did not inject a Product + // dimension (legacy node or a boot path not yet updated). + meterProductFallback = "cre" +) + +// meterRecordsEnabledEnvVar gates MeterRecord emission; the name is the +// cross-producer convention for the metering rollout (SHARED-2718). +const meterRecordsEnabledEnvVar = "CL_METER_RECORDS_ENABLED" + +// meterRecordsEnabled reads the metering gate from the environment. Unset or +// unparseable values disable emission; metering config must never prevent the +// capability from starting. +func meterRecordsEnabled(lggr logger.Logger) bool { + v := os.Getenv(meterRecordsEnabledEnvVar) + if v == "" { + return false + } + enabled, err := strconv.ParseBool(v) + if err != nil { + lggr.Warnw("Invalid value for "+meterRecordsEnabledEnvVar+", meter record emission disabled", "value", v, "error", err) + return false + } + return enabled +} + var _ server.HTTPCapability = &service{} type WorkflowRegistrationInput struct { @@ -85,18 +122,50 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa } metadataPublisher := NewGatewayMetadataPublisher(s.lggr, dependencies.GatewayConnector, workflowStore, s.cfg, s.metrics) requestCache := newRequestCache(s.lggr, dependencies.Store, time.Duration(s.cfg.RequestCacheTTL)*time.Second) - // dependencies.CapabilityDonID is the on-chain DON ID this plugin process - // serves, used to label emitted events with the *sending* DON. Zero means the - // host could not resolve it authoritatively (a multi-DON job-spec node, or a - // core node that pre-dates CRE-4409); the handler then falls back to - // RequestMetadata.WorkflowDONID. See CRE-4409. - s.connectorHandler, err = NewConnectorHandler(s.lggr, dependencies.GatewayConnector, s.cfg, dependencies.CapabilityDonID, workflowStore, metadataPublisher, requestCache, s.metrics, s.orgResolver, s.limitsFactory) + resourceManager := resourcemanager.NewResourceManager(s.lggr, resourcemanager.ResourceManagerConfig{ + Enabled: meterRecordsEnabled(s.lggr), + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + }) + baseIdentity := baseMeterIdentity(dependencies) + s.connectorHandler, err = NewConnectorHandler(s.lggr, dependencies.GatewayConnector, s.cfg, workflowStore, metadataPublisher, requestCache, s.metrics, s.orgResolver, resourceManager, baseIdentity) if err != nil { return err } return s.Start(ctx) } +// baseMeterIdentity builds the HTTP trigger's base metering identity from the +// host-injected dependencies. The six coarse dimensions plus the service-level +// resource / resource_type are fixed here; the per-workflow resource_id is set +// per emission via ResourceIdentity.WithResourceID. +// +// DONID is the capability DON the trigger LOOP was spawned for +// (deps.CapabilityDonID, host-injected via capabilities#619). When the host has +// not populated it (0), DONID is left empty here and resolved per registration +// from the workflow DON at emit time (see connectorHandler.donID). Product +// falls back to a constant when the host did not inject one. +func baseMeterIdentity(deps core.StandardCapabilitiesDependencies) resourcemanager.ResourceIdentity { + product := deps.Product + if product == "" { + product = meterProductFallback + } + var donID string + if deps.CapabilityDonID != 0 { + donID = strconv.FormatUint(uint64(deps.CapabilityDonID), 10) + } + return resourcemanager.ResourceIdentity{ + Product: product, + Environment: deps.Environment, + Zone: deps.Zone, + DONID: donID, + NodeID: deps.NodeID, + Service: meterService, + Resource: meterResource, + ResourceType: meterResourceType, + } +} + func (s *service) Start(ctx context.Context) error { s.lggr.Debug("Service starting...") return s.StartOnce(ServiceName, func() error { diff --git a/http_trigger/trigger/workflow.go b/http_trigger/trigger/workflow.go index 67daefa28..8b7be4945 100644 --- a/http_trigger/trigger/workflow.go +++ b/http_trigger/trigger/workflow.go @@ -52,26 +52,30 @@ func newWorkflowStore(lggr logger.Logger) *workflowStore { // workflow reference (owner/name/tag combination) with new workflow instance. // upsertWorkflow should be invoked in the order of workflow registration, so that // the latest workflow instance is always used for the given reference. -func (s *workflowStore) upsertWorkflow(w *workflow) error { +// The returned replaced flag reports whether an existing registration was +// replaced (true) rather than a new one inserted (false); when replaced, +// prevWorkflowID is the workflow ID the reference pointed to before the +// upsert (it may equal the new workflow ID, or differ on a version update). +func (s *workflowStore) upsertWorkflow(w *workflow) (prevWorkflowID string, replaced bool, err error) { // Validate workflow fields if err := validateWorkflowSelector(w.workflowSelector); err != nil { - return fmt.Errorf("invalid workflow selector: %w", err) + return "", false, fmt.Errorf("invalid workflow selector: %w", err) } s.mu.Lock() defer s.mu.Unlock() - workflowID, exists := s.workflowReferenceToID[workflowReference{ + prevWorkflowID, replaced = s.workflowReferenceToID[workflowReference{ workflowOwner: w.workflowSelector.WorkflowOwner, workflowName: w.workflowSelector.WorkflowName, workflowTag: w.workflowSelector.WorkflowTag, }] - if exists { + if replaced { reference := fmt.Sprintf("%s/%s/%s", w.workflowSelector.WorkflowOwner, w.workflowSelector.WorkflowName, w.workflowSelector.WorkflowTag) - s.lggr.Debugw("Updating existing workflow reference and removing previous workflow", "reference", reference, "prevWorkflowID", workflowID) - if oldW, ok := s.workflows[workflowID]; ok { + s.lggr.Debugw("Updating existing workflow reference and removing previous workflow", "reference", reference, "prevWorkflowID", prevWorkflowID) + if oldW, ok := s.workflows[prevWorkflowID]; ok { oldW.close() } - delete(s.workflows, workflowID) + delete(s.workflows, prevWorkflowID) } s.workflows[w.workflowSelector.WorkflowID] = w s.workflowReferenceToID[workflowReference{ @@ -79,7 +83,7 @@ func (s *workflowStore) upsertWorkflow(w *workflow) error { workflowName: w.workflowSelector.WorkflowName, workflowTag: w.workflowSelector.WorkflowTag, }] = w.workflowSelector.WorkflowID - return nil + return prevWorkflowID, replaced, nil } // validateWorkflowSelector validates the workflow selector fields diff --git a/http_trigger/trigger/workflow_test.go b/http_trigger/trigger/workflow_test.go index 189ed0d39..62edf5ef5 100644 --- a/http_trigger/trigger/workflow_test.go +++ b/http_trigger/trigger/workflow_test.go @@ -134,8 +134,10 @@ func TestWorkflowStore_upsertWorkflow(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - err := store.upsertWorkflow(wf) + prevWorkflowID, replaced, err := store.upsertWorkflow(wf) require.NoError(t, err) + require.False(t, replaced) + require.Empty(t, prevWorkflowID) w, exists := store.getWorkflowByID(wf.workflowSelector.WorkflowID) require.True(t, exists) @@ -257,7 +259,7 @@ func TestWorkflowStore_upsertWorkflow_ValidationErrors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { wf := newWorkflow(tt.selector, authorizedKeys, sendCh) - err := store.upsertWorkflow(wf) + _, _, err := store.upsertWorkflow(wf) require.Error(t, err) require.Contains(t, err.Error(), tt.wantErr) }) @@ -273,12 +275,16 @@ func TestWorkflowStore_upsertWorkflow_Duplicate(t *testing.T) { w2, _ := testWorkflow() // Add first workflow - err := store.upsertWorkflow(w1) + prevWorkflowID, replaced, err := store.upsertWorkflow(w1) require.NoError(t, err) + require.False(t, replaced) + require.Empty(t, prevWorkflowID) // Add second workflow with same ID (this should replace the first) - err = store.upsertWorkflow(w2) + prevWorkflowID, replaced, err = store.upsertWorkflow(w2) require.NoError(t, err) + require.True(t, replaced) + require.Equal(t, w1.workflowSelector.WorkflowID, prevWorkflowID) // Verify the workflow was replaced - since both have same ID/reference, // the second one should be present @@ -294,7 +300,7 @@ func TestWorkflowStore_removeWorkflow_Success(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) w, _ := testWorkflow() - err := store.upsertWorkflow(w) + _, _, err := store.upsertWorkflow(w) require.NoError(t, err) wf, exists := store.getWorkflowByID(w.workflowSelector.WorkflowID) @@ -393,11 +399,11 @@ func TestWorkflowStore_GetWorkflows_Multiple(t *testing.T) { wf2 := newWorkflow(wfSelector2, authorizedKeys, sendCh2) wf3 := newWorkflow(wfSelector3, authorizedKeys, sendCh3) - err := store.upsertWorkflow(wf1) + _, _, err := store.upsertWorkflow(wf1) require.NoError(t, err) - err = store.upsertWorkflow(wf2) + _, _, err = store.upsertWorkflow(wf2) require.NoError(t, err) - err = store.upsertWorkflow(wf3) + _, _, err = store.upsertWorkflow(wf3) require.NoError(t, err) // Get all workflows @@ -421,7 +427,7 @@ func TestWorkflowStore_getWorkflowIDByReference_Success(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - err := store.upsertWorkflow(wf) + _, _, err := store.upsertWorkflow(wf) require.NoError(t, err) workflowID, exists := store.getWorkflowIDByReference( @@ -538,8 +544,10 @@ func TestWorkflowStore_upsertWorkflow_ReplaceWithSameReference(t *testing.T) { wf2 := newWorkflow(selector2, authorizedKeys, sendCh2) // Add first workflow - err := store.upsertWorkflow(wf1) + prevWorkflowID, replaced, err := store.upsertWorkflow(wf1) require.NoError(t, err) + require.False(t, replaced) + require.Empty(t, prevWorkflowID) // Verify first workflow is there workflow, exists := store.getWorkflowByID(testWorkflowID1) @@ -551,9 +559,12 @@ func TestWorkflowStore_upsertWorkflow_ReplaceWithSameReference(t *testing.T) { require.True(t, exists) require.Equal(t, testWorkflowID1, workflowID) - // Add second workflow with same reference - err = store.upsertWorkflow(wf2) + // Add second workflow with same reference; the previous workflow ID is + // surfaced so callers can release its reservation. + prevWorkflowID, replaced, err = store.upsertWorkflow(wf2) require.NoError(t, err) + require.True(t, replaced) + require.Equal(t, testWorkflowID1, prevWorkflowID) // First workflow should be removed _, exists = store.getWorkflowByID(testWorkflowID1) @@ -576,7 +587,7 @@ func TestWorkflowStore_removeWorkflow_RemovesReference(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - err := store.upsertWorkflow(wf) + _, _, err := store.upsertWorkflow(wf) require.NoError(t, err) // Verify workflow and reference exist @@ -735,7 +746,7 @@ func TestWorkflowStore_getWorkflowIDByReference_PartialMatch(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - err := store.upsertWorkflow(wf) + _, _, err := store.upsertWorkflow(wf) require.NoError(t, err) // Test with wrong owner From 100e42d552757d7b6a9b8529f5ef0d22ac0b1de0 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Tue, 7 Jul 2026 14:13:29 -0400 Subject: [PATCH 2/6] numericTenantID + naming changes to MeterRecord + MeterSnapshot proto --- chain_capabilities/evm/go.mod | 10 +- chain_capabilities/evm/go.sum | 14 +- chain_capabilities/evm/main.go | 119 ++++++------ chain_capabilities/evm/trigger/store.go | 1 + chain_capabilities/evm/trigger/trigger.go | 76 +++++--- .../evm/trigger/trigger_metering_test.go | 102 +++++------ cron/go.mod | 8 +- cron/go.sum | 8 +- cron/main.go | 54 +++--- cron/trigger/metering_test.go | 95 +++++----- cron/trigger/trigger.go | 169 ++++++++++++------ http_trigger/go.mod | 10 +- http_trigger/go.sum | 16 +- http_trigger/main.go | 20 ++- http_trigger/trigger/connector_handler.go | 93 +++++++--- .../trigger/connector_handler_test.go | 98 +++++----- http_trigger/trigger/trigger.go | 98 +++++----- http_trigger/trigger/trigger_test.go | 8 +- integration_tests/go.mod | 7 +- integration_tests/go.sum | 14 +- 20 files changed, 607 insertions(+), 413 deletions(-) diff --git a/chain_capabilities/evm/go.mod b/chain_capabilities/evm/go.mod index beba4586b..da3bd6fac 100644 --- a/chain_capabilities/evm/go.mod +++ b/chain_capabilities/evm/go.mod @@ -5,15 +5,16 @@ go 1.26.2 require ( github.com/ethereum/go-ethereum v1.17.0 github.com/google/go-cmp v0.7.0 + github.com/jonboulle/clockwork v0.5.0 github.com/smartcontractkit/capabilities/chain_capabilities/common v0.0.0-20260615195421-fb87220e503f github.com/smartcontractkit/capabilities/libs v0.0.0-20260609124022-2749e4a32bfb github.com/smartcontractkit/chain-selectors v1.0.104 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260410162948-2dca02f24e98 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260410144512-ca02ad6ed16a - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-00010101000000-000000000000 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.uber.org/zap v1.27.1 @@ -70,7 +71,6 @@ require ( github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jonboulle/clockwork v0.5.0 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -91,7 +91,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/smartcontractkit/chainlink-common/keystore v1.1.1-0.20260529092756-a94bc8ce96d6 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260326122810-b657beadfb57 // indirect github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260401162955-be2bc6b5264b // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect diff --git a/chain_capabilities/evm/go.sum b/chain_capabilities/evm/go.sum index 02eed31f7..fe8c010e8 100644 --- a/chain_capabilities/evm/go.sum +++ b/chain_capabilities/evm/go.sum @@ -473,12 +473,12 @@ github.com/smartcontractkit/capabilities/libs v0.0.0-20260609124022-2749e4a32bfb github.com/smartcontractkit/capabilities/libs v0.0.0-20260609124022-2749e4a32bfb/go.mod h1:LS7F8U2YZNc0Vt8f6SVWUUigGLxdxZMpyC7VCcUTagg= github.com/smartcontractkit/chain-selectors v1.0.104 h1:/n9pPGM5W/+r1eHoWZv4VwX9LNS1af4+ICyhM8zKRNM= github.com/smartcontractkit/chain-selectors v1.0.104/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37 h1:ZZlU2e+hVC1Y8VAczVNGZBM3rU3HSqkOCn2KZHHV7gc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37/go.mod h1:paOB/6dy57owHtOGzhgaRBWRDT5BEWfnJF5M7sgkcro= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.1.1-0.20260529092756-a94bc8ce96d6 h1:fWsYxxj35fp1/6YZngoTsOTMLqDie4N5X0osAOdhUTE= github.com/smartcontractkit/chainlink-common/keystore v1.1.1-0.20260529092756-a94bc8ce96d6/go.mod h1:6JexOOhPhknQ0QMuppFIlOpm6wCp54yZMxai+tWugwY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0 h1:NExKM/D0HneOq/N5LGTbkV4VOa0UHCvfTNEb4GqYpto= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260410162948-2dca02f24e98 h1:h/L6wrXYLQalI/vHm6qg/KBv6d7kMb3geMHV5hCM1t4= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260410162948-2dca02f24e98/go.mod h1:6vCMfxz7cMW0wWseNKtct+b1JJbbRVJJhh/t6pQWN3M= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1 h1:NTODgwAil7BLoijS7y6KnEuNbQ9v60VUhIR9FcAzIhg= @@ -491,10 +491,12 @@ github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260410144512- github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260410144512-ca02ad6ed16a/go.mod h1:7ketk4ischPQW/JQgmyHz6zdzLUJv1VC29SiSgosydQ= github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 h1:GCzrxDWn3b7jFfEA+WiYRi8CKoegsayiDoJBCjYkneE= github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4/go.mod h1:HHGeDUpAsPa0pmOx7wrByCitjQ0mbUxf0R9v+g67uCA= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 h1:iRFmfMFQtcnhGDjCuARQG4MPbcmbbJDDw7MUH3GcGy8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b h1:VDgJWDipihV9f7M5+d21d1RzSsg5rEv+iI12oN1VQbo= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= diff --git a/chain_capabilities/evm/main.go b/chain_capabilities/evm/main.go index e57d7a881..e7b70397a 100644 --- a/chain_capabilities/evm/main.go +++ b/chain_capabilities/evm/main.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "os" "strconv" "time" @@ -49,6 +48,40 @@ type capabilityGRPCService struct { capability lggr logger.Logger limitsFactory limits.Factory + // metering carries emission toggles and deployment/node identity dimensions + // delivered to the plugin process via loop.EnvConfig (set at startup). The + // zero value is valid and leaves those dimensions empty/disabled. + metering meteringConfig +} + +type meteringConfig struct { + meterRecordsEnabled bool + meterSnapshotsEnabled bool + deployment resourcemanager.DeploymentIdentity +} + +func newMeteringConfig(env loop.EnvConfig) meteringConfig { + return meteringConfig{ + meterRecordsEnabled: env.MeterRecordsEnabled, + meterSnapshotsEnabled: env.MeterSnapshotsEnabled, + deployment: resourcemanager.DeploymentIdentity{ + Product: env.MeterProduct, + Tenant: env.MeterTenant, + NumericTenantID: env.MeterNumericTenantID, + Environment: env.MeterEnvironment, + Zone: env.MeterZone, + NodeID: env.MeterNodeID, + }, + } +} + +func (m meteringConfig) resourceManagerConfig() resourcemanager.ResourceManagerConfig { + return resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: m.meterRecordsEnabled, + MeterSnapshotsEnabled: m.meterSnapshotsEnabled, + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + } } type capability struct { @@ -65,7 +98,12 @@ var _ evmcapserver.ClientCapability = &capabilityGRPCService{} func main() { loopserver.ServeNew(CapabilityName, func(s *loop.Server) loop.StandardCapabilities { - return evmcapserver.NewClientServer(&capabilityGRPCService{lggr: s.Logger, limitsFactory: s.LimitsFactory}) + meteringCfg := newMeteringConfig(s.EnvConfig) + return evmcapserver.NewClientServer(&capabilityGRPCService{ + lggr: s.Logger, + limitsFactory: s.LimitsFactory, + metering: meteringCfg, + }) }, loop.WithOtelViews(append(consMetrics.MetricViews(), monitoring.MetricViews()...))) } @@ -167,12 +205,8 @@ func (c *capabilityGRPCService) Initialise(ctx context.Context, dependencies cor // as a sub-service and registers itself, so it must be configured with a // snapshot interval here. Identity/snapshots are gated by the same metering // env flag as MeterRecords. - resourceManager := resourcemanager.NewResourceManager(c.lggr, resourcemanager.ResourceManagerConfig{ - Enabled: meterRecordsEnabled(c.lggr), - Emitter: beholder.GetEmitter(), - SnapshotInterval: resourcemanager.DefaultSnapshotInterval, - }) - baseIdentity := newBaseMeteringIdentity(dependencies) + resourceManager := resourcemanager.NewResourceManager(c.lggr, c.metering.resourceManagerConfig()) + baseIdentity := newBaseMeteringIdentity(dependencies, c.metering.deployment) c.triggerService, err = trigger.NewLogTriggerService(evmRelayer, trigger.NewLogTriggerStore(), c.lggr, capabilityID, processor, messageBuilder, cfg.LogTriggerPollInterval, cfg.LogTriggerSendChannelBufferSize, cfg.LogTriggerLimitQueryLogSize, c.limitsFactory, dependencies.OrgResolver, dependencies.TriggerEventStore, resourceManager, baseIdentity, c.chainSelector) @@ -211,22 +245,20 @@ func (c *capabilityGRPCService) Initialise(ctx context.Context, dependencies cor } // defaultMeteringProduct is the fallback metering product dimension used when -// the host did not inject one via the standardized Initialise channel (a legacy -// node or a boot path not yet updated). The other deployment dimensions -// (environment, zone, node_id) have no meaningful constant and are left empty -// in that case, as documented on StandardCapabilitiesDependencies. +// the host did not provide one via loop.EnvConfig (a legacy node or a boot path +// not yet updated). The other deployment dimensions (environment, zone, +// node_id) have no meaningful constant and are left empty in that case. const defaultMeteringProduct = "cre" -// newBaseMeteringIdentity builds the EVM log trigger's base metering identity -// from the host-injected dependencies. It carries the six coarse dimensions -// plus the service-level resource/resource_type; the per-resource ResourceID is -// set per emit/snapshot. DONID is the capability DON when the host injected one -// (deps.CapabilityDonID); when 0, it is left empty here and resolved per emit -// from the consumer's WorkflowDonID (see LogTriggerService.resolveDONID). This -// reads deps.CapabilityDonID at the Initialise layer so the change is orthogonal -// to capabilities#619's NewLogTriggerService signature edit. -func newBaseMeteringIdentity(deps core.StandardCapabilitiesDependencies) resourcemanager.ResourceIdentity { - product := deps.Product +// newBaseMeteringIdentity builds the EVM log trigger's base metering identity. +// The deployment/node dimensions come from deployment (delivered via +// loop.EnvConfig); the DON dimension comes from the host-injected +// CapabilityDonID. It carries the six coarse dimensions plus the service-level +// resource/resource_type; the per-resource ResourceID is set per emit/snapshot. +// When CapabilityDonID is 0, the DON identifier is left empty here and resolved per emit from +// the consumer's WorkflowDonID (see LogTriggerService.resolveDONID). +func newBaseMeteringIdentity(deps core.StandardCapabilitiesDependencies, deployment resourcemanager.DeploymentIdentity) resourcemanager.ResourceIdentity { + product := deployment.Product if product == "" { product = defaultMeteringProduct } @@ -234,36 +266,23 @@ func newBaseMeteringIdentity(deps core.StandardCapabilitiesDependencies) resourc if deps.CapabilityDonID != 0 { donID = strconv.FormatUint(uint64(deps.CapabilityDonID), 10) } - return resourcemanager.ResourceIdentity{ - Product: product, - Environment: deps.Environment, - Zone: deps.Zone, - DONID: donID, - NodeID: deps.NodeID, - Service: trigger.MeteringService, - Resource: trigger.MeteringResource, - ResourceType: trigger.MeteringResourceType, - } -} - -// meterRecordsEnabledEnvVar gates MeterRecord emission; the name is the -// cross-producer convention for the metering rollout (SHARED-2718). -const meterRecordsEnabledEnvVar = "CL_METER_RECORDS_ENABLED" - -// meterRecordsEnabled reads the metering gate from the environment. Unset or -// unparseable values disable emission; metering config must never prevent the -// capability from starting. -func meterRecordsEnabled(lggr logger.Logger) bool { - v := os.Getenv(meterRecordsEnabledEnvVar) - if v == "" { - return false + var donIdentity *resourcemanager.DonIdentity + if donID != "" || deployment.NodeID != "" { + donIdentity = &resourcemanager.DonIdentity{ + DonID: donID, + NodeID: deployment.NodeID, + } } - enabled, err := strconv.ParseBool(v) - if err != nil { - lggr.Warnw("Invalid value for "+meterRecordsEnabledEnvVar+", meter record emission disabled", "value", v, "error", err) - return false + return resourcemanager.ResourceIdentity{ + Product: product, + Tenant: deployment.Tenant, + NumericTenantID: deployment.NumericTenantID, + Environment: deployment.Environment, + Zone: deployment.Zone, + Don: donIdentity, + Service: trigger.MeteringService, + ResourcePool: trigger.MeteringResource, } - return enabled } func (c *capabilityGRPCService) unmarshalConfig(configStr string) (*config.Config, error) { diff --git a/chain_capabilities/evm/trigger/store.go b/chain_capabilities/evm/trigger/store.go index 5658110b2..4362d0bb2 100644 --- a/chain_capabilities/evm/trigger/store.go +++ b/chain_capabilities/evm/trigger/store.go @@ -31,6 +31,7 @@ type filter struct { // WorkflowDonID fallback when the host did not inject a capability DON); // empty when neither is known. donID string + orgID string expressions []query.Expression confidence primitives.ConfidenceLevel } diff --git a/chain_capabilities/evm/trigger/trigger.go b/chain_capabilities/evm/trigger/trigger.go index de602411f..e0ee3d19e 100644 --- a/chain_capabilities/evm/trigger/trigger.go +++ b/chain_capabilities/evm/trigger/trigger.go @@ -35,6 +35,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/workflows/events" meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" + capcommon "github.com/smartcontractkit/capabilities/chain_capabilities/common" "github.com/smartcontractkit/capabilities/chain_capabilities/evm/monitoring" ) @@ -47,8 +48,8 @@ const ( // Metering identity constants for the EVM log trigger (SHARED-2711). These are // the service-level dimensions of the base ResourceIdentity: Service is the // stable service constant (it must not encode deployment environment or zone, -// which ride on the structured identity's coarse dimensions), Resource is the -// resource pool, and ResourceType is the billing unit converted to credits. +// which ride on the structured identity's coarse dimensions). Resource pool +// lives on ResourceIdentity; billing unit lives on Utilization.resource_type. const ( MeteringService = "evm-log-trigger" MeteringResource = "log_filters" @@ -73,10 +74,11 @@ type LogTriggerService struct { messageBuilder *monitoring.MessageBuilder resourceManager *resourcemanager.ResourceManager // baseIdentity is the producer's base metering identity: the six coarse - // dimensions plus service/resource/resource_type, built once at Initialise. - // The per-resource ResourceID is set per emit/snapshot via WithResourceID. - // When the host did not inject a capability DON ID, baseIdentity.DONID is - // empty and is filled per-emit from the consumer's WorkflowDonID. + // dimensions plus service/resource_pool, built once at Initialise. + // Per-resource billing fields are set on Utilization. + // When the host did not inject a capability DON ID, baseIdentity has an + // empty DON identifier and is filled per-emit from the consumer's + // WorkflowDonID. baseIdentity resourcemanager.ResourceIdentity chainSelector string // decimal chain selector, the chain label on meter records // rmUnregister removes this service from the ResourceManager's snapshot @@ -376,6 +378,7 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID physicalFilterID: physicalFilterID(lts.chainSelector, addresses, sigs, t2, t3, t4), reservedAddressCount: int64(len(addresses)), donID: lts.resolveDONID(meta.WorkflowDonID), + orgID: lts.resolveOrgID(ctx, meta.WorkflowOwner), expressions: expressions, confidence: confidence, } @@ -463,14 +466,14 @@ func (lts *LogTriggerService) generateFilterID(triggerID string) string { // resolveDONID returns the metering DON ID for an emit, applying the // capabilities#619 0->WorkflowDonID rule: when the host injected a capability -// DON ID, baseIdentity.DONID is non-empty and used as-is; otherwise the +// DON ID, baseIdentity's DON identifier is non-empty and used as-is; otherwise the // consumer workflow's DON ID (from the request metadata) is the documented // fallback. The result is stashed on the filter at registration so the // unregister/cleanup/snapshot/close paths reproduce the same identity without // the request. Empty when neither source is known. func (lts *LogTriggerService) resolveDONID(workflowDonID uint32) string { - if lts.baseIdentity.DONID != "" { - return lts.baseIdentity.DONID + if lts.baseIdentity.DonID() != "" { + return lts.baseIdentity.DonID() } if workflowDonID != 0 { return strconv.FormatUint(uint64(workflowDonID), 10) @@ -478,14 +481,29 @@ func (lts *LogTriggerService) resolveDONID(workflowDonID uint32) string { return "" } -// identity returns the base metering identity with its DON ID and ResourceID -// set for one resource. donID is the value stashed on the filter at -// registration (see resolveDONID); resourceID is the physical filter content -// hash (empty when unrecoverable, e.g. an orphaned filter). -func (lts *LogTriggerService) identity(donID, resourceID string) resourcemanager.ResourceIdentity { +func (lts *LogTriggerService) resolveOrgID(ctx context.Context, workflowOwner string) string { + if lts.orgResolver == nil || workflowOwner == "" { + return "" + } + orgID, err := lts.orgResolver.Get(ctx, workflowOwner) + if err != nil { + lts.lggr.Warnw("failed to fetch organization ID from org resolver", "workflowOwner", workflowOwner, "error", err) + return "" + } + return orgID +} + +// identity returns the base metering identity with DON ID resolved for one +// resource. +func (lts *LogTriggerService) identity(donID string) resourcemanager.ResourceIdentity { id := lts.baseIdentity - id.DONID = donID - id.ResourceID = resourceID + if donID == "" { + return id + } + id.Don = &resourcemanager.DonIdentity{ + DonID: donID, + NodeID: id.NodeID(), + } return id } @@ -500,14 +518,21 @@ func (lts *LogTriggerService) emitMeterRecord(ctx context.Context, action meteri if lts.resourceManager == nil { return } - identity := lts.identity(f.donID, f.physicalFilterID) + identity := lts.identity(f.donID) lts.resourceManager.EmitMeterRecord(ctx, identity, action, - resourcemanager.NewUtilization(identity, action, value, f.physicalFilterID)) + []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(value, resourcemanager.UtilizationFields{ + ResourceType: MeteringResourceType, + ResourceID: f.physicalFilterID, + EventID: f.physicalFilterID, + OrgID: f.orgID, + }), + }) } // ResourceIdentity implements resourcemanager.Meterable: it returns the // producer's base identity (the six coarse dimensions plus -// service/resource/resource_type). The per-resource DON ID and ResourceID are +// service/resource_pool). The per-resource DON ID and billing fields are // populated per active filter by GetUtilization. func (lts *LogTriggerService) ResourceIdentity() resourcemanager.ResourceIdentity { return lts.baseIdentity @@ -523,8 +548,15 @@ func (lts *LogTriggerService) GetUtilization(_ context.Context) []resourcemanage for _, state := range triggers { f := state.filter entries = append(entries, resourcemanager.SnapshotEntry{ - Identity: lts.identity(f.donID, f.physicalFilterID), - Value: f.reservedAddressCount, + Identity: lts.identity(f.donID), + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(f.reservedAddressCount, resourcemanager.UtilizationFields{ + ResourceType: MeteringResourceType, + ResourceID: f.physicalFilterID, + EventID: f.physicalFilterID, + OrgID: f.orgID, + }), + }, }) } return entries @@ -736,7 +768,7 @@ func (lts *LogTriggerService) deliverLogReliably( } lts.lggr.Infow("Sending log event to pipe", "triggerID", triggerID, "eventID", eventID, "blockNumber", log.BlockNumber, "txHash", log.TxHash) - deliverCtx := lts.contextWithOrgForDelivery(ctx, telemetryContext.RequestMetadata) + deliverCtx := capcommon.ContextWithOrgForDelivery(ctx, lts.lggr, lts.orgResolver, telemetryContext.RequestMetadata) if err := lts.baseTrigger.DeliverEvent(deliverCtx, te, triggerID); err != nil { summary := fmt.Sprintf("failed to persist/deliver event (triggerID=%s, eventID=%s): %v", triggerID, eventID, err) lts.lggr.Error(summary) diff --git a/chain_capabilities/evm/trigger/trigger_metering_test.go b/chain_capabilities/evm/trigger/trigger_metering_test.go index 1a02523d1..ab80eb277 100644 --- a/chain_capabilities/evm/trigger/trigger_metering_test.go +++ b/chain_capabilities/evm/trigger/trigger_metering_test.go @@ -33,14 +33,14 @@ const testChainSelector = "5009297550715157269" // WorkflowDonID fallback. func testBaseIdentity() resourcemanager.ResourceIdentity { return resourcemanager.ResourceIdentity{ - Product: "cre", - Environment: "staging", - Zone: "wf-zone-a", - DONID: "42", - NodeID: "csa-pubkey-hex", - Service: MeteringService, - Resource: MeteringResource, - ResourceType: MeteringResourceType, + Product: "cre", + Tenant: "mainline", + NumericTenantID: "42", + Environment: "staging", + Zone: "wf-zone-a", + Don: &resourcemanager.DonIdentity{DonID: "42", NodeID: "csa-pubkey-hex"}, + Service: MeteringService, + ResourcePool: MeteringResource, } } @@ -102,10 +102,11 @@ func newMeteredTriggerObject(t *testing.T, mockEVM *evmmock.EVMService, store Lo clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) lts.resourceManager = resourcemanager.NewResourceManager(logger.Test(t), resourcemanager.ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: time.Minute, - Clock: clock, + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, }) lts.baseIdentity = testBaseIdentity() lts.chainSelector = testChainSelector @@ -121,19 +122,20 @@ func meteringTestInput() *evmcappb.FilterLogTriggerRequest { } } -// assertBaseIdentity checks the six coarse dimensions + service/resource on the +// assertBaseIdentity checks the six coarse dimensions + service/resource_pool on the // emitted record identity, proving the host-injected identity is carried. func assertBaseIdentity(t *testing.T, id *meteringpb.ResourceIdentity) { t.Helper() require.NotNil(t, id) require.Equal(t, "cre", id.GetProduct()) + require.Equal(t, "mainline", id.GetTenant()) + require.Equal(t, "42", id.GetNumericTenantId()) require.Equal(t, "staging", id.GetEnvironment()) require.Equal(t, "wf-zone-a", id.GetZone()) - require.Equal(t, "42", id.GetDonId()) - require.Equal(t, "csa-pubkey-hex", id.GetNodeId()) + require.Equal(t, "42", id.GetDon().GetDonId()) + require.Equal(t, "csa-pubkey-hex", id.GetDon().GetNodeId()) require.Equal(t, MeteringService, id.GetService()) - require.Equal(t, MeteringResource, id.GetResource()) - require.Equal(t, MeteringResourceType, id.GetResourceType()) + require.Equal(t, MeteringResource, id.GetResourcePool()) } // expectedPhysicalFilterID recomputes the physical filter id for the metering @@ -160,7 +162,7 @@ func TestLogTriggerMetering_ReserveOnRegister(t *testing.T) { evmService := initMocks(t) evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() - service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) meta := capabilities.RequestMetadata{WorkflowID: "wf-id", WorkflowOwner: "0xOwner"} _, err := service.RegisterLogTrigger(t.Context(), triggerID, meta, meteringTestInput()) @@ -170,38 +172,35 @@ func TestLogTriggerMetering_ReserveOnRegister(t *testing.T) { record := emitter.records[0] assertBaseIdentity(t, record.GetIdentity()) physID := expectedPhysicalFilterID(t, meteringTestInput()) - require.Equal(t, physID, record.GetIdentity().GetResourceId(), "resource_id must be the physical filter content hash") + require.Equal(t, physID, record.GetUtilizations()[0].GetResourceId(), "resource_id must be the physical filter content hash") require.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, record.GetAction()) - require.NotNil(t, record.GetUtilization()) - require.Equal(t, int64(2), record.GetUtilization().GetValue(), "RESERVE value must equal the filter address count") - expectedID := service.identity("42", physID) - require.Equal(t, - resourcemanager.IdempotencyKey(expectedID, meteringpb.MeterAction_METER_ACTION_RESERVE, physID), - record.GetUtilization().GetIdempotencyKey()) + require.Len(t, record.GetUtilizations(), 1) + require.Equal(t, "2", record.GetUtilizations()[0].GetValue(), "RESERVE value must equal the filter address count") + require.Equal(t, MeteringResourceType, record.GetUtilizations()[0].GetResourceType()) } func TestLogTriggerMetering_DonIDFallbackToWorkflowDon(t *testing.T) { evmService := initMocks(t) evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() - service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) // Host did not inject a capability DON; the consumer's WorkflowDonID is the // documented fallback resolved at emit time. - service.baseIdentity.DONID = "" + service.baseIdentity.Don = &resourcemanager.DonIdentity{NodeID: "csa-pubkey-hex"} meta := capabilities.RequestMetadata{WorkflowID: "wf-id", WorkflowOwner: "0xOwner", WorkflowDonID: 7} _, err := service.RegisterLogTrigger(t.Context(), triggerID, meta, meteringTestInput()) require.NoError(t, err) require.Len(t, emitter.records, 1) - require.Equal(t, "7", emitter.records[0].GetIdentity().GetDonId(), "empty capability DON must fall back to WorkflowDonID") + require.Equal(t, "7", emitter.records[0].GetIdentity().GetDon().GetDonId(), "empty capability DON must fall back to WorkflowDonID") } func TestLogTriggerMetering_NoReserveOnRegisterFailure(t *testing.T) { evmService := initMocks(t) evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(errors.New("mocked register failure")).Once() - service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) _, err := service.RegisterLogTrigger(t.Context(), triggerID, capabilities.RequestMetadata{WorkflowID: "wf-id"}, meteringTestInput()) require.Error(t, err) @@ -227,14 +226,10 @@ func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { t.Helper() assertBaseIdentity(t, record.GetIdentity()) require.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, record.GetAction()) - require.NotNil(t, record.GetUtilization()) - require.Equal(t, int64(2), record.GetUtilization().GetValue(), "RELEASE must carry the same value that was reserved") + require.Len(t, record.GetUtilizations(), 1) + require.Equal(t, "2", record.GetUtilizations()[0].GetValue(), "RELEASE must carry the same value that was reserved") physID := expectedPhysicalFilterID(t, meteringTestInput()) - require.Equal(t, physID, record.GetIdentity().GetResourceId()) - expectedID := service.identity("42", physID) - require.Equal(t, - resourcemanager.IdempotencyKey(expectedID, meteringpb.MeterAction_METER_ACTION_RELEASE, physID), - record.GetUtilization().GetIdempotencyKey()) + require.Equal(t, physID, record.GetUtilizations()[0].GetResourceId()) } t.Run("release pairs the reserve", func(t *testing.T) { @@ -242,7 +237,7 @@ func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() evmService.On("UnregisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() - service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) registerTrigger(t, service) require.NoError(t, service.UnregisterLogTrigger(t.Context(), triggerID, meta, &evmcappb.FilterLogTriggerRequest{})) @@ -250,8 +245,8 @@ func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { require.Len(t, emitter.records, 2) require.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, emitter.records[0].GetAction()) assertRelease(t, service, emitter.records[1]) - require.Equal(t, emitter.records[0].GetUtilization().GetValue(), emitter.records[1].GetUtilization().GetValue()) - require.Equal(t, emitter.records[0].GetIdentity().GetResourceId(), emitter.records[1].GetIdentity().GetResourceId(), + require.Equal(t, emitter.records[0].GetUtilizations()[0].GetValue(), emitter.records[1].GetUtilizations()[0].GetValue()) + require.Equal(t, emitter.records[0].GetUtilizations()[0].GetResourceId(), emitter.records[1].GetUtilizations()[0].GetResourceId(), "RESERVE and RELEASE must share one physical resource_id") }) @@ -260,7 +255,7 @@ func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() evmService.On("UnregisterLogTracking", mock.Anything, mock.Anything).Return(errors.New("mocked unregister failure")).Once() - service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) registerTrigger(t, service) // The reservation is released here (from the stashed count) before the @@ -281,7 +276,7 @@ func TestLogTriggerMetering_OrphanCleanupEmitsNothing(t *testing.T) { t.Run("stale filter cleanup emits no meter record", func(t *testing.T) { mockEVM := evmmock.NewEVMService(t) store := NewLogTriggerStore() - service, emitter := newMeteredTriggerObject(t, mockEVM, store) + service, emitter, _ := newMeteredTriggerObject(t, mockEVM, store) liveFilterID := service.generateFilterID("live-trigger") staleFilterID := service.generateFilterID("stale-trigger") @@ -297,7 +292,7 @@ func TestLogTriggerMetering_OrphanCleanupEmitsNothing(t *testing.T) { t.Run("emits nothing when cleanup unregister fails", func(t *testing.T) { mockEVM := evmmock.NewEVMService(t) - service, emitter := newMeteredTriggerObject(t, mockEVM, NewLogTriggerStore()) + service, emitter, _ := newMeteredTriggerObject(t, mockEVM, NewLogTriggerStore()) staleFilterID := service.generateFilterID("stale-trigger") mockEVM.On("GetFiltersNames", mock.Anything).Return([]string{staleFilterID}, nil).Once() @@ -312,7 +307,7 @@ func TestLogTriggerMetering_FailOpen(t *testing.T) { evmService := initMocks(t) evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() - service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) emitter.err = errors.New("mocked emitter failure") _, err := service.RegisterLogTrigger(t.Context(), triggerID, capabilities.RequestMetadata{WorkflowID: "wf-id"}, meteringTestInput()) @@ -368,7 +363,7 @@ func TestPhysicalFilterID_Canonicalization(t *testing.T) { evmService := initMocks(t) evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Twice() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Twice() - service, emitter := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) _, err := service.RegisterLogTrigger(t.Context(), "trigger-A", capabilities.RequestMetadata{WorkflowID: "wf-1", WorkflowOwner: "0xOwner"}, meteringTestInput()) @@ -379,8 +374,8 @@ func TestPhysicalFilterID_Canonicalization(t *testing.T) { require.Len(t, emitter.records, 2) require.Equal(t, - emitter.records[0].GetIdentity().GetResourceId(), - emitter.records[1].GetIdentity().GetResourceId(), + emitter.records[0].GetUtilizations()[0].GetResourceId(), + emitter.records[1].GetUtilizations()[0].GetResourceId(), "identical physical filters must share one resource_id across workflows/triggers") }) } @@ -424,16 +419,17 @@ func TestLogTriggerMetering_Snapshot(t *testing.T) { byResourceID := map[string]*meteringpb.MeterSnapshot{} for _, s := range emitter.snapshots { assertBaseIdentity(t, s.GetIdentity()) - byResourceID[s.GetIdentity().GetResourceId()] = s + byResourceID[s.GetUtilization()[0].GetResourceId()] = s } a := byResourceID[physA] require.NotNil(t, a) - require.Equal(t, int64(2), a.GetUtilization().GetValue()) + require.Equal(t, "2", a.GetUtilization()[0].GetValue()) + require.Equal(t, MeteringResourceType, a.GetUtilization()[0].GetResourceType()) b := byResourceID["physB"] require.NotNil(t, b) - require.Equal(t, int64(5), b.GetUtilization().GetValue()) + require.Equal(t, "5", b.GetUtilization()[0].GetValue()) } // TestLogTriggerMetering_Snapshot_NothingActive asserts an empty store emits no @@ -456,7 +452,7 @@ func TestLogTriggerMetering_Snapshot_NothingActive(t *testing.T) { func TestLogTriggerMetering_ReleaseOnGracefulClose(t *testing.T) { mockEVM := evmmock.NewEVMService(t) store := NewLogTriggerStore() - service, emitter := newMeteredTriggerObject(t, mockEVM, store) + service, emitter, _ := newMeteredTriggerObject(t, mockEVM, store) physA := expectedPhysicalFilterID(t, meteringTestInput()) store.Write("trigger-A", logTriggerState{filter: filter{ @@ -483,8 +479,8 @@ func TestLogTriggerMetering_ReleaseOnGracefulClose(t *testing.T) { // resource_id (the only per-filter discriminator on the record). byResourceID := map[string]*meteringpb.MeterRecord{} for _, record := range emitter.records { - byResourceID[record.GetIdentity().GetResourceId()] = record + byResourceID[record.GetUtilizations()[0].GetResourceId()] = record } - require.Equal(t, int64(2), byResourceID[physA].GetUtilization().GetValue()) - require.Equal(t, int64(5), byResourceID["physB"].GetUtilization().GetValue()) + require.Equal(t, "2", byResourceID[physA].GetUtilizations()[0].GetValue()) + require.Equal(t, "5", byResourceID["physB"].GetUtilizations()[0].GetValue()) } diff --git a/cron/go.mod b/cron/go.mod index 51aaac6a0..e0b034ccb 100644 --- a/cron/go.mod +++ b/cron/go.mod @@ -14,9 +14,9 @@ require ( github.com/google/uuid v1.6.0 github.com/jonboulle/clockwork v0.5.0 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260529092756-a94bc8ce96d6 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260611123141-db97012a6c32 - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-00010101000000-000000000000 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 @@ -90,7 +90,7 @@ require ( github.com/scylladb/go-reflectx v1.0.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/smartcontractkit/chain-selectors v1.0.100 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect diff --git a/cron/go.sum b/cron/go.sum index d07775fc9..05cab8559 100644 --- a/cron/go.sum +++ b/cron/go.sum @@ -213,10 +213,10 @@ github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924/go.mod h1:v0O0Au8RE00Z89QxBE6I2q9bR9r3+RO1gLD3oaO2WB0= github.com/smartcontractkit/chain-selectors v1.0.100 h1:wpiSpmI/eFjY+wx/nPr5VuNF4hki0prIBMKEaQWn3g4= github.com/smartcontractkit/chain-selectors v1.0.100/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 h1:ucHu2bPDT/58AzSgnPDyp4IjnjVbrVWYD3bG5jCbXMY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260611123141-db97012a6c32 h1:GNl+lLK0QCakqA1J1i7FoOai2JrOGOzNzSniMijaCjA= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260611123141-db97012a6c32/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b h1:VDgJWDipihV9f7M5+d21d1RzSsg5rEv+iI12oN1VQbo= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= diff --git a/cron/main.go b/cron/main.go index c2f40b391..e61f006a5 100644 --- a/cron/main.go +++ b/cron/main.go @@ -1,51 +1,55 @@ package main import ( - "os" - "strconv" - "github.com/smartcontractkit/capabilities/cron/trigger" "github.com/smartcontractkit/capabilities/libs/loopserver" "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/cron/server" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" ) -// meterRecordsEnabledEnvVar gates MeterRecord emission; the name is the -// cross-producer convention for the metering rollout (SHARED-2718). -const meterRecordsEnabledEnvVar = "CL_METER_RECORDS_ENABLED" - -// meterRecordsEnabled reads the metering gate from the environment. Unset or -// unparseable values disable emission; metering config must never prevent the -// capability from starting. -func meterRecordsEnabled(lggr logger.Logger) bool { - v := os.Getenv(meterRecordsEnabledEnvVar) - if v == "" { - return false +type meteringConfig struct { + meterRecordsEnabled bool + meterSnapshotsEnabled bool + deployment resourcemanager.DeploymentIdentity +} + +func newMeteringConfig(env loop.EnvConfig) meteringConfig { + return meteringConfig{ + meterRecordsEnabled: env.MeterRecordsEnabled, + meterSnapshotsEnabled: env.MeterSnapshotsEnabled, + deployment: resourcemanager.DeploymentIdentity{ + Product: env.MeterProduct, + Tenant: env.MeterTenant, + NumericTenantID: env.MeterNumericTenantID, + Environment: env.MeterEnvironment, + Zone: env.MeterZone, + NodeID: env.MeterNodeID, + }, } - enabled, err := strconv.ParseBool(v) - if err != nil { - lggr.Warnw("Invalid value for "+meterRecordsEnabledEnvVar+", meter record emission disabled", "value", v, "error", err) - return false +} + +func (m meteringConfig) resourceManagerConfig() resourcemanager.ResourceManagerConfig { + return resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: m.meterRecordsEnabled, + MeterSnapshotsEnabled: m.meterSnapshotsEnabled, + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, } - return enabled } func main() { loopserver.ServeNew(trigger.ServiceName, func(s *loop.Server) loop.StandardCapabilities { - meters := resourcemanager.NewResourceManager(s.Logger, resourcemanager.ResourceManagerConfig{ - Enabled: meterRecordsEnabled(s.Logger), - Emitter: beholder.GetEmitter(), - SnapshotInterval: resourcemanager.DefaultSnapshotInterval, - }) + meteringCfg := newMeteringConfig(s.EnvConfig) + meters := resourcemanager.NewResourceManager(s.Logger, meteringCfg.resourceManagerConfig()) triggerService, err := trigger.NewTriggerService(s.Logger, nil, s.LimitsFactory, meters) if err != nil { s.Logger.Fatalw("Failed to create cron trigger service", "error", err) } + triggerService.Deployment = meteringCfg.deployment return server.NewCronServer(triggerService) }, loop.WithOtelViews(trigger.MetricViews())) diff --git a/cron/trigger/metering_test.go b/cron/trigger/metering_test.go index 44efd13f6..9f3ecc4b0 100644 --- a/cron/trigger/metering_test.go +++ b/cron/trigger/metering_test.go @@ -81,27 +81,35 @@ func (f *fakeMeterEmitter) Snapshots() []*meteringpb.MeterSnapshot { return append([]*meteringpb.MeterSnapshot(nil), f.snapshots...) } -// meteredTestDeps are the host-injected identity dimensions used by metering -// tests. They mirror what the host populates via the Initialise channel. +// meteredTestDeps are the host-injected dependencies used by metering tests. +// The DON dimension still arrives via the Initialise channel; the +// deployment/node dimensions now arrive via loop.EnvConfig (meteredTestDeployment). var meteredTestDeps = core.StandardCapabilitiesDependencies{ + CapabilityDonID: 7, +} + +// meteredTestDeployment is the deployment/node identity that main would source +// from loop.EnvConfig and set on the service before Initialise. +var meteredTestDeployment = resourcemanager.DeploymentIdentity{ Product: "cre-mainline", + Tenant: "mainline", + NumericTenantID: "42", Environment: "staging", Zone: "wf-zone-a", NodeID: "csa-pubkey-1", - CapabilityDonID: 7, } // expectedBaseIdentity is the base identity the Service builds from // meteredTestDeps (resource_id left empty; set per trigger). var expectedBaseIdentity = resourcemanager.ResourceIdentity{ - Product: "cre-mainline", - Environment: "staging", - Zone: "wf-zone-a", - DONID: "7", - NodeID: "csa-pubkey-1", - Service: "cron-trigger", - Resource: "trigger_registrations", - ResourceType: "operations", + Product: "cre-mainline", + Tenant: "mainline", + NumericTenantID: "42", + Environment: "staging", + Zone: "wf-zone-a", + Don: &resourcemanager.DonIdentity{DonID: "7", NodeID: "csa-pubkey-1"}, + Service: "cron-trigger", + ResourcePool: "trigger_registrations", } // newMeteredTriggerService builds an initialised trigger service whose @@ -118,13 +126,15 @@ func newMeteredTriggerService(t *testing.T, clock clockwork.Clock, emitter resou } meters := resourcemanager.NewResourceManager(logger.Nop(), resourcemanager.ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: time.Minute, - Clock: clock, + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, }) ts, err := NewTriggerService(logger.Nop(), clock, limits.Factory{}, meters) require.NoError(t, err) + ts.Deployment = meteredTestDeployment config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) @@ -155,25 +165,24 @@ func TestCronTrigger_Metering_ReserveAndRelease(t *testing.T) { reserve := records[0] assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, reserve.GetAction()) - // Identity is populated from the host-injected deps, with resource_id set - // to the trigger_id (cron is workflow-scoped). + // Identity is populated from host-injected deps and points at the cron + // resource pool. Per-trigger fields are carried on utilization. id := reserve.GetIdentity() require.NotNil(t, id) assert.Equal(t, "cre-mainline", id.GetProduct()) + assert.Equal(t, "mainline", id.GetTenant()) + assert.Equal(t, "42", id.GetNumericTenantId()) assert.Equal(t, "staging", id.GetEnvironment()) assert.Equal(t, "wf-zone-a", id.GetZone()) - assert.Equal(t, "7", id.GetDonId()) - assert.Equal(t, "csa-pubkey-1", id.GetNodeId()) + assert.Equal(t, "7", id.GetDon().GetDonId()) + assert.Equal(t, "csa-pubkey-1", id.GetDon().GetNodeId()) assert.Equal(t, "cron-trigger", id.GetService()) - assert.Equal(t, "trigger_registrations", id.GetResource()) - assert.Equal(t, "operations", id.GetResourceType()) - assert.Equal(t, triggerID1, id.GetResourceId()) + assert.Equal(t, "trigger_registrations", id.GetResourcePool()) - require.NotNil(t, reserve.GetUtilization()) - assert.Equal(t, int64(1), reserve.GetUtilization().GetValue()) - assert.Equal(t, - resourcemanager.IdempotencyKey(expectedBaseIdentity.WithResourceID(triggerID1), meteringpb.MeterAction_METER_ACTION_RESERVE, triggerID1), - reserve.GetUtilization().GetIdempotencyKey()) + require.Len(t, reserve.GetUtilizations(), 1) + assert.Equal(t, "1", reserve.GetUtilizations()[0].GetValue()) + assert.Equal(t, "operations", reserve.GetUtilizations()[0].GetResourceType()) + assert.Equal(t, triggerID1, reserve.GetUtilizations()[0].GetResourceId()) // Each cron tick re-Writes the trigger to reschedule it; the Write // happens before the channel send, so after receiving the event the @@ -189,11 +198,9 @@ func TestCronTrigger_Metering_ReserveAndRelease(t *testing.T) { require.Len(t, records, 2, "expected exactly one RELEASE on unregistration") release := records[1] assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, release.GetAction()) - assert.Equal(t, reserve.GetIdentity().GetResourceId(), release.GetIdentity().GetResourceId()) - require.NotNil(t, release.GetUtilization()) - assert.Equal(t, int64(1), release.GetUtilization().GetValue()) - assert.NotEqual(t, reserve.GetUtilization().GetIdempotencyKey(), release.GetUtilization().GetIdempotencyKey(), - "RESERVE and RELEASE must not share an idempotency key") + assert.Equal(t, reserve.GetUtilizations()[0].GetResourceId(), release.GetUtilizations()[0].GetResourceId()) + require.Len(t, release.GetUtilizations(), 1) + assert.Equal(t, "1", release.GetUtilizations()[0].GetValue()) require.NoError(t, ts.Close()) } @@ -253,7 +260,7 @@ func TestCronTrigger_Metering_FailOpen(t *testing.T) { // TestCronTrigger_Metering_Snapshot asserts the Service implements Meterable // such that a forced snapshot emits one MeterSnapshot per active trigger, each -// carrying the full per-resource identity (resource_id set to the trigger_id). +// carrying the full base identity and per-trigger utilization. func TestCronTrigger_Metering_Snapshot(t *testing.T) { t.Parallel() @@ -283,20 +290,20 @@ func TestCronTrigger_Metering_Snapshot(t *testing.T) { byTrigger := map[string]*meteringpb.MeterSnapshot{} for _, s := range snapshots { - byTrigger[s.GetIdentity().GetResourceId()] = s + byTrigger[s.GetUtilization()[0].GetResourceId()] = s } s1 := byTrigger[triggerID1] require.NotNil(t, s1) - assert.Equal(t, int64(1), s1.GetUtilization().GetValue()) + assert.Equal(t, "1", s1.GetUtilization()[0].GetValue()) assert.Equal(t, "cron-trigger", s1.GetIdentity().GetService()) - assert.Equal(t, "trigger_registrations", s1.GetIdentity().GetResource()) - assert.Equal(t, "operations", s1.GetIdentity().GetResourceType()) + assert.Equal(t, "trigger_registrations", s1.GetIdentity().GetResourcePool()) + assert.Equal(t, "operations", s1.GetUtilization()[0].GetResourceType()) s2 := byTrigger[triggerID2] require.NotNil(t, s2) - assert.Equal(t, int64(1), s2.GetUtilization().GetValue()) - assert.Equal(t, triggerID2, s2.GetIdentity().GetResourceId()) + assert.Equal(t, "1", s2.GetUtilization()[0].GetValue()) + assert.Equal(t, triggerID2, s2.GetUtilization()[0].GetResourceId()) require.NoError(t, ts.Close()) } @@ -332,11 +339,11 @@ func TestCronTrigger_Metering_GracefulCloseReleases(t *testing.T) { releases := map[string]*meteringpb.MeterRecord{} for _, r := range records[2:] { require.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, r.GetAction()) - releases[r.GetIdentity().GetResourceId()] = r + releases[r.GetUtilizations()[0].GetResourceId()] = r } require.Contains(t, releases, triggerID1) require.Contains(t, releases, triggerID2) - assert.Equal(t, int64(1), releases[triggerID1].GetUtilization().GetValue()) + assert.Equal(t, "1", releases[triggerID1].GetUtilizations()[0].GetValue()) } // TestCronTrigger_Metering_DonIDFallback asserts the DON ID falls back to the @@ -348,8 +355,8 @@ func TestCronTrigger_Metering_DonIDFallback(t *testing.T) { emitter := &fakeMeterEmitter{} meters := resourcemanager.NewResourceManager(logger.Nop(), resourcemanager.ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, + MeterRecordsEnabled: true, + Emitter: emitter, }) ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, meters) require.NoError(t, err) @@ -366,7 +373,7 @@ func TestCronTrigger_Metering_DonIDFallback(t *testing.T) { records := emitter.Records() require.Len(t, records, 1) - assert.Equal(t, "42", records[0].GetIdentity().GetDonId(), "DON ID falls back to WorkflowDonID") + assert.Equal(t, "42", records[0].GetIdentity().GetDon().GetDonId(), "DON ID falls back to WorkflowDonID") // Product falls back to the cron constant when the host injects none. assert.Equal(t, "cre", records[0].GetIdentity().GetProduct()) diff --git a/cron/trigger/trigger.go b/cron/trigger/trigger.go index 5cd6142b9..2ec43f6dc 100644 --- a/cron/trigger/trigger.go +++ b/cron/trigger/trigger.go @@ -46,14 +46,14 @@ const ( // meteringService is the stable service constant for cron trigger // registrations on emitted MeterRecords and Snapshots. It must not encode // deployment environment or zone: those are discrete identity dimensions - // sourced from deps at Initialise. + // delivered via loop.EnvConfig (see Service.Deployment). meteringService = "cron-trigger" // meteringResource is the resource pool cron records apply to. meteringResource = "trigger_registrations" // meteringResourceType is the billing unit for cron registrations. meteringResourceType = "operations" - // meteringProductFallback is used when the host has not injected a Product - // (a legacy node or a boot path not yet updated to populate deps.Product). + // meteringProductFallback is used when the host has not provided a Product + // via loop.EnvConfig (a legacy node or a boot path not yet updated). meteringProductFallback = "cre" ) @@ -67,10 +67,12 @@ type Response struct { } type cronTrigger struct { - job gocron.Job - nextRun time.Time - workflowID string - close func() + job gocron.Job + nextRun time.Time + workflowID string + workflowDonID uint32 + orgID string + close func() } type Service struct { @@ -91,9 +93,14 @@ type Service struct { // snapshot registry; set at start, called at close. Nil until started. unregisterMeterable func() // base is the resourcemanager identity for cron registrations, built from - // the host-injected deployment/node/DON dimensions at Initialise. ResourceID - // is left empty here and set per trigger via base.WithResourceID(triggerID). - base resourcemanager.ResourceIdentity + // the deployment/node dimensions (Deployment) and DON dimension at Initialise. + base resourcemanager.ResourceIdentity + // Deployment carries the static deployment/node identity dimensions + // delivered to the plugin process via loop.EnvConfig. It is set once at + // startup (by main, before Initialise) and read when building the base + // metering identity. The zero value is valid and leaves those dimensions + // empty. + Deployment resourcemanager.DeploymentIdentity orgResolver orgresolver.OrgResolver } @@ -196,28 +203,50 @@ func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFa return s, nil } -// identityFor returns the per-trigger metering identity: the base identity -// with ResourceID set to triggerID. resource_id is workflow-scoped (the -// trigger_id) for cron, which has no shared physical resource. The DON ID +// identityFor returns the per-trigger metering identity. resource_id is +// workflow-scoped (the trigger_id) for cron and is carried on Utilization. +// The DON ID // falls back to the consumer workflow's DON when the host has not injected a // capability DON ID (deps.CapabilityDonID == 0). -func (s *Service) identityFor(triggerID string, workflowDonID uint32) resourcemanager.ResourceIdentity { - id := s.base.WithResourceID(triggerID) - if id.DONID == "" { - id.DONID = strconv.FormatUint(uint64(workflowDonID), 10) +func (s *Service) identityFor(workflowDonID uint32) resourcemanager.ResourceIdentity { + id := s.base + if id.DonID() == "" && workflowDonID != 0 { + id.Don = &resourcemanager.DonIdentity{ + DonID: strconv.FormatUint(uint64(workflowDonID), 10), + NodeID: s.Deployment.NodeID, + } } return id } +func (s *Service) resolveOrgID(ctx context.Context, workflowOwner string) string { + if s.orgResolver == nil || workflowOwner == "" { + return "" + } + orgID, err := s.orgResolver.Get(ctx, workflowOwner) + if err != nil { + s.lggr.Warnw("failed to fetch organization ID from org resolver", "workflowOwner", workflowOwner, "error", err) + return "" + } + return orgID +} + // emitMeterRecord reports a change to this trigger's registration reservation // for billing. The triggerID doubles as the idempotency event identity: a // triggerID is registered at most once at a time, so retried emissions for the // same registration dedup downstream. Emission is fail-open and never affects // the registration itself. -func (s *Service) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, metadata capabilities.RequestMetadata, triggerID string) { - id := s.identityFor(triggerID, metadata.WorkflowDonID) +func (s *Service) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, metadata capabilities.RequestMetadata, triggerID string, orgID string) { + id := s.identityFor(metadata.WorkflowDonID) s.meters.EmitMeterRecord(ctx, id, action, - resourcemanager.NewUtilization(id, action, 1, triggerID)) + []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meteringResourceType, + ResourceID: triggerID, + EventID: triggerID, + OrgID: orgID, + }), + }) } func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapabilitiesDependencies) error { @@ -246,12 +275,12 @@ func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapa s.lggr.Warn("OrgResolver is nil, cron capability will not be able to fetch organization ID") } - // Build the base metering identity from the host-injected deployment, node, - // and DON dimensions. These arrive via the standardized Initialise channel - // (mirroring how capabilities#619 injects CapabilityDonID). Any may be - // empty/zero until the host is updated to populate them; DONID falls back to - // the consumer workflow DON at emit time (see identityFor). - product := dependencies.Product + // Build the base metering identity. The deployment/node dimensions come from + // s.Deployment (delivered via loop.EnvConfig, set by main before + // Initialise); the DON dimension comes from the host-injected + // CapabilityDonID. Any may be empty/zero; the DON identifier falls back to the consumer + // workflow DON at emit time (see identityFor). + product := s.Deployment.Product if product == "" { product = meteringProductFallback } @@ -259,15 +288,22 @@ func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapa if dependencies.CapabilityDonID != 0 { donID = strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10) } + var donIdentity *resourcemanager.DonIdentity + if donID != "" || s.Deployment.NodeID != "" { + donIdentity = &resourcemanager.DonIdentity{ + DonID: donID, + NodeID: s.Deployment.NodeID, + } + } s.base = resourcemanager.ResourceIdentity{ - Product: product, - Environment: dependencies.Environment, - Zone: dependencies.Zone, - DONID: donID, - NodeID: dependencies.NodeID, - Service: meteringService, - Resource: meteringResource, - ResourceType: meteringResourceType, + Product: product, + Tenant: s.Deployment.Tenant, + NumericTenantID: s.Deployment.NumericTenantID, + Environment: s.Deployment.Environment, + Zone: s.Deployment.Zone, + Don: donIdentity, + Service: meteringService, + ResourcePool: meteringResource, } err = s.Start(ctx) @@ -404,10 +440,12 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat return // unregistered already } s.triggers.Write(triggerID, cronTrigger{ - job: job, - nextRun: nextExecutionTime, - workflowID: metadata.WorkflowID, - close: closeCh, + job: job, + nextRun: nextExecutionTime, + workflowID: metadata.WorkflowID, + workflowDonID: metadata.WorkflowDonID, + orgID: trigger.orgID, + close: closeCh, }) select { @@ -446,14 +484,17 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat return nil, caperrors.NewPublicSystemError(fmt.Errorf("RegisterTrigger failed to remove job: %s", err), caperrors.Internal) } + orgID := s.resolveOrgID(ctx, metadata.WorkflowOwner) s.triggers.Write(triggerID, cronTrigger{ - job: job, - nextRun: firstRunTime, - workflowID: metadata.WorkflowID, - close: closeCh, + job: job, + nextRun: firstRunTime, + workflowID: metadata.WorkflowID, + workflowDonID: metadata.WorkflowDonID, + orgID: orgID, + close: closeCh, }) - s.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, metadata, triggerID) + s.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, metadata, triggerID, orgID) s.lggr.Debugw("Trigger registered", "workflowId", metadata.WorkflowID, "triggerId", triggerID, "jobId", job.ID()) s.metrics.IncActiveTriggersGauge(ctx) @@ -506,7 +547,7 @@ func (s *Service) UnregisterTrigger(ctx context.Context, triggerID string, metad // Remove from triggers context s.triggers.Delete(triggerID) - s.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, metadata, triggerID) + s.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, metadata, triggerID, trigger.orgID) s.lggr.Debugw("UnregisterTrigger", "triggerId", triggerID, "jobId", jobID) s.metrics.DecActiveTriggersGauge(ctx) @@ -531,10 +572,12 @@ func (s *Service) start(_ context.Context) error { for triggerID, trigger := range s.triggers.ReadAll() { nextExecutionTime, err := trigger.job.NextRun() s.triggers.Write(triggerID, cronTrigger{ - job: trigger.job, - nextRun: nextExecutionTime, - workflowID: trigger.workflowID, - close: trigger.close, + job: trigger.job, + nextRun: nextExecutionTime, + workflowID: trigger.workflowID, + workflowDonID: trigger.workflowDonID, + orgID: trigger.orgID, + close: trigger.close, }) if err != nil { s.lggr.Errorw("Unable to get next run time", "err", err, "triggerID", triggerID) @@ -559,10 +602,17 @@ func (s *Service) close() error { // context is already cancelled by the time close runs. Emission is // fail-open, so a metering failure never blocks shutdown. ctx := context.Background() - for triggerID := range s.triggers.ReadAll() { - id := s.base.WithResourceID(triggerID) + for triggerID, trigger := range s.triggers.ReadAll() { + id := s.identityFor(trigger.workflowDonID) s.meters.EmitMeterRecord(ctx, id, meteringpb.MeterAction_METER_ACTION_RELEASE, - resourcemanager.NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RELEASE, 1, triggerID)) + []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meteringResourceType, + ResourceID: triggerID, + EventID: triggerID, + OrgID: trigger.orgID, + }), + }) } if s.unregisterMeterable != nil { @@ -587,7 +637,8 @@ func (s *Service) Description() string { } // ResourceIdentity implements resourcemanager.Meterable: it returns the base -// six-dimension identity (resource_id left empty; set per active trigger in +// six-dimension identity (per-resource billing fields are set per active +// trigger in // GetUtilization). func (s *Service) ResourceIdentity() resourcemanager.ResourceIdentity { return s.base @@ -603,10 +654,18 @@ func (s *Service) GetUtilization(ctx context.Context) []resourcemanager.Snapshot } triggers := s.triggers.ReadAll() entries := make([]resourcemanager.SnapshotEntry, 0, len(triggers)) - for triggerID := range triggers { + for triggerID, trigger := range triggers { + id := s.identityFor(trigger.workflowDonID) entries = append(entries, resourcemanager.SnapshotEntry{ - Identity: s.base.WithResourceID(triggerID), - Value: 1, + Identity: id, + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meteringResourceType, + ResourceID: triggerID, + EventID: triggerID, + OrgID: trigger.orgID, + }), + }, }) } return entries diff --git a/http_trigger/go.mod b/http_trigger/go.mod index 069b1f7e8..986e0f37d 100644 --- a/http_trigger/go.mod +++ b/http_trigger/go.mod @@ -3,8 +3,9 @@ module github.com/smartcontractkit/capabilities/http_trigger go 1.26.2 require ( + github.com/jonboulle/clockwork v0.5.0 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260603162703-43f4fa4bd88c + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 @@ -75,9 +76,10 @@ require ( github.com/scylladb/go-reflectx v1.0.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/smartcontractkit/chain-selectors v1.0.100 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 // indirect - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260602131523-5168ac1ba014 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e // indirect @@ -119,7 +121,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/grpc v1.80.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/http_trigger/go.sum b/http_trigger/go.sum index 719469218..193964b5b 100644 --- a/http_trigger/go.sum +++ b/http_trigger/go.sum @@ -130,6 +130,8 @@ github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5Xum github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -207,14 +209,16 @@ github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924/go.mod h1:v0O0Au8RE00Z89QxBE6I2q9bR9r3+RO1gLD3oaO2WB0= github.com/smartcontractkit/chain-selectors v1.0.100 h1:wpiSpmI/eFjY+wx/nPr5VuNF4hki0prIBMKEaQWn3g4= github.com/smartcontractkit/chain-selectors v1.0.100/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260603162703-43f4fa4bd88c h1:Khlxg3kMFJ5DNbZlREoVD5KZ77DKYMOz5qPj9MdrTD4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260603162703-43f4fa4bd88c/go.mod h1:zC5csAXmmn2FZbZ78Rrfc4AvmEKJzKQLMEY/w2SRwDo= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 h1:ucHu2bPDT/58AzSgnPDyp4IjnjVbrVWYD3bG5jCbXMY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260602131523-5168ac1ba014 h1:4rxcbbe1qe1yR+HcclvOi/e0CFLcBLfx2fgiWxBMMZ4= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260602131523-5168ac1ba014/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b h1:VDgJWDipihV9f7M5+d21d1RzSsg5rEv+iI12oN1VQbo= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= diff --git a/http_trigger/main.go b/http_trigger/main.go index 337e478c3..7dcfc4417 100644 --- a/http_trigger/main.go +++ b/http_trigger/main.go @@ -6,10 +6,28 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http/server" "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" ) +func newMeteringConfig(env loop.EnvConfig) trigger.MeteringConfig { + return trigger.MeteringConfig{ + MeterRecordsEnabled: env.MeterRecordsEnabled, + MeterSnapshotsEnabled: env.MeterSnapshotsEnabled, + Deployment: resourcemanager.DeploymentIdentity{ + Product: env.MeterProduct, + Tenant: env.MeterTenant, + NumericTenantID: env.MeterNumericTenantID, + Environment: env.MeterEnvironment, + Zone: env.MeterZone, + NodeID: env.MeterNodeID, + }, + } +} + func main() { loopserver.ServeNew(trigger.ServiceName, func(s *loop.Server) loop.StandardCapabilities { - return server.NewHTTPServer(trigger.NewService(s.Logger, s.LimitsFactory)) + meteringCfg := newMeteringConfig(s.EnvConfig) + svc := trigger.NewService(s.Logger, s.LimitsFactory, meteringCfg) + return server.NewHTTPServer(svc) }) } diff --git a/http_trigger/trigger/connector_handler.go b/http_trigger/trigger/connector_handler.go index 217dbd6ce..c5de95010 100644 --- a/http_trigger/trigger/connector_handler.go +++ b/http_trigger/trigger/connector_handler.go @@ -52,9 +52,9 @@ type connectorHandler struct { stopChan services.StopChan orgResolver orgresolver.OrgResolver // Optional org resolver for fetching organization IDs resourceManager *resourcemanager.ResourceManager - // baseIdentity is the six-dimension + resource/resource_type metering - // identity for this trigger LOOP, built once at Initialise. The - // per-workflow resource_id is derived per emission via WithResourceID. + // baseIdentity is the six-dimension + resource_pool metering identity for + // this trigger LOOP, built once at Initialise. Per-workflow billing fields + // are carried by Utilization. baseIdentity resourcemanager.ResourceIdentity // unregisterMeterable removes this handler from the ResourceManager's // snapshot registry; set on Start, called on Close. @@ -145,7 +145,7 @@ func (h *connectorHandler) Close() error { func (h *connectorHandler) releaseActiveWorkflows(ctx context.Context) { for _, w := range h.workflowStore.getWorkflows() { h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, - w.workflowSelector.WorkflowID, w.metadata.WorkflowDONID) + w.workflowSelector.WorkflowID, w.metadata.WorkflowDONID, w.metadata.OrganizationID) } } @@ -187,6 +187,18 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR latencyMs := time.Since(startTime).Milliseconds() h.metrics.RecordBroadcastMetadataLatency(ctx, latencyMs, h.lggr) + orgID := h.resolveOrgID(ctx, input.WorkflowSelector.WorkflowOwner) + input.Metadata.OrganizationID = orgID + + var prevWorkflowOrgID string + var prevWorkflowDONID uint32 + if prevID, ok := h.workflowStore.getWorkflowIDByReference(input.WorkflowSelector.WorkflowOwner, input.WorkflowSelector.WorkflowName, input.WorkflowSelector.WorkflowTag); ok { + if prevWorkflow, found := h.workflowStore.getWorkflowByID(prevID); found { + prevWorkflowOrgID = prevWorkflow.metadata.OrganizationID + prevWorkflowDONID = prevWorkflow.metadata.WorkflowDONID + } + } + workflow := newWorkflowWithMetadata(input.WorkflowSelector, authorizedKeys, sendCh, input.Metadata) prevWorkflowID, replaced, err := h.workflowStore.upsertWorkflow(workflow) if err != nil { @@ -197,15 +209,15 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR workflowDONID := input.Metadata.WorkflowDONID switch { case !replaced: - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, newWorkflowID, workflowDONID) + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, newWorkflowID, workflowDONID, orgID) case prevWorkflowID == newWorkflowID: - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_UPDATE, newWorkflowID, workflowDONID) + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_UPDATE, newWorkflowID, workflowDONID, orgID) default: // Version update: the same owner/name/tag reference now resolves to a // new workflow ID. Release the previous workflow's reservation before // reserving the new one so the old reservation does not leak. - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, prevWorkflowID, workflowDONID) - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, newWorkflowID, workflowDONID) + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, prevWorkflowID, prevWorkflowDONID, prevWorkflowOrgID) + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, newWorkflowID, workflowDONID, orgID) } h.lggr.Debugw("Registered workflow", "workflowID", input.WorkflowSelector.WorkflowID, "workflowOwner", input.WorkflowSelector.WorkflowOwner, "workflowName", input.WorkflowSelector.WorkflowName, "workflowTag", input.WorkflowSelector.WorkflowTag) return nil @@ -218,18 +230,32 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR // workflow and action dedups downstream. The workflow_id is recoverable from // resource_id and the owner is resolved downstream, so no label metadata is // attached. Emission is fail-open and never affects the registration outcome. -func (h *connectorHandler) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, workflowID string, workflowDONID uint32) { - identity := h.identityForWorkflow(workflowID, workflowDONID) +func (h *connectorHandler) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, workflowID string, workflowDONID uint32, orgID string) { + identity := h.identityForWorkflow(workflowDONID) h.resourceManager.EmitMeterRecord(ctx, identity, action, - resourcemanager.NewUtilization(identity, action, 1, workflowID)) + []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + EventID: workflowID, + OrgID: orgID, + }), + }) } -// identityForWorkflow derives the per-workflow metering identity: the base -// identity with resource_id set to the workflow ID, and DONID resolved per +// identityForWorkflow derives per-workflow metering identity: base identity +// with DON identifier resolved per // registration when the host did not inject a capability DON. -func (h *connectorHandler) identityForWorkflow(workflowID string, workflowDONID uint32) resourcemanager.ResourceIdentity { - identity := h.baseIdentity.WithResourceID(workflowID) - identity.DONID = h.donID(workflowDONID) +func (h *connectorHandler) identityForWorkflow(workflowDONID uint32) resourcemanager.ResourceIdentity { + identity := h.baseIdentity + donID := h.donID(workflowDONID) + if donID == "" { + return identity + } + identity.Don = &resourcemanager.DonIdentity{ + DonID: donID, + NodeID: identity.NodeID(), + } return identity } @@ -238,8 +264,8 @@ func (h *connectorHandler) identityForWorkflow(workflowID string, workflowDONID // and falls back to the per-registration workflow DON when the host did not // populate one (CapabilityDonID == 0 at Initialise). func (h *connectorHandler) donID(workflowDONID uint32) string { - if h.baseIdentity.DONID != "" { - return h.baseIdentity.DONID + if h.baseIdentity.DonID() != "" { + return h.baseIdentity.DonID() } if workflowDONID != 0 { return strconv.FormatUint(uint64(workflowDONID), 10) @@ -247,9 +273,21 @@ func (h *connectorHandler) donID(workflowDONID uint32) string { return "" } +func (h *connectorHandler) resolveOrgID(ctx context.Context, workflowOwner string) string { + if h.orgResolver == nil || workflowOwner == "" { + return "" + } + orgID, err := h.orgResolver.Get(ctx, workflowOwner) + if err != nil { + h.lggr.Warnw("failed to fetch organization ID from org resolver", "workflowOwner", workflowOwner, "error", err) + return "" + } + return orgID +} + // ResourceIdentity returns the HTTP trigger's base metering identity (six -// dimensions + resource / resource_type). The per-workflow resource_id is -// populated by GetUtilization. It implements resourcemanager.Meterable. +// dimensions + resource_pool). Per-workflow billing fields are populated on +// Utilization in GetUtilization. It implements resourcemanager.Meterable. func (h *connectorHandler) ResourceIdentity() resourcemanager.ResourceIdentity { return h.baseIdentity } @@ -265,8 +303,15 @@ func (h *connectorHandler) GetUtilization(ctx context.Context) []resourcemanager for _, w := range workflows { workflowID := w.workflowSelector.WorkflowID entries = append(entries, resourcemanager.SnapshotEntry{ - Identity: h.identityForWorkflow(workflowID, w.metadata.WorkflowDONID), - Value: 1, + Identity: h.identityForWorkflow(w.metadata.WorkflowDONID), + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + EventID: workflowID, + OrgID: w.metadata.OrganizationID, + }), + }, }) } return entries @@ -302,14 +347,16 @@ func (h *connectorHandler) UnregisterWorkflow(ctx context.Context, workflowID st // Snapshot the workflow DON before removal; it is needed for the meter // record's identity DON fallback. var workflowDONID uint32 + var orgID string if w, ok := h.workflowStore.getWorkflowByID(workflowID); ok { workflowDONID = w.metadata.WorkflowDONID + orgID = w.metadata.OrganizationID } err := h.workflowStore.removeWorkflow(workflowID) if err != nil { return fmt.Errorf("failed to unregister workflow %s: %w", workflowID, err) } - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, workflowID, workflowDONID) + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, workflowID, workflowDONID, orgID) h.lggr.Debugw("Unregistered workflow", "workflowID", workflowID) return nil } diff --git a/http_trigger/trigger/connector_handler_test.go b/http_trigger/trigger/connector_handler_test.go index e60f1c218..641bb3373 100644 --- a/http_trigger/trigger/connector_handler_test.go +++ b/http_trigger/trigger/connector_handler_test.go @@ -1178,17 +1178,16 @@ func (f *fakeMeterEmitter) actions() []meteringpb.MeterAction { } // testBaseIdentity is the base metering identity used by metering tests. It -// carries the six coarse dimensions plus the service-level resource / -// resource_type; per-workflow resource_id is derived per emission. +// carries the six coarse dimensions plus the service-level resource_pool. var testBaseIdentity = resourcemanager.ResourceIdentity{ - Product: "cre-test", - Environment: "staging", - Zone: "wf-zone-a", - DONID: "7", - NodeID: "node-csa-pubkey", - Service: meterService, - Resource: meterResource, - ResourceType: meterResourceType, + Product: "cre-test", + Tenant: "mainline", + NumericTenantID: "42", + Environment: "staging", + Zone: "wf-zone-a", + Don: &resourcemanager.DonIdentity{DonID: "7", NodeID: "node-csa-pubkey"}, + Service: meterService, + ResourcePool: meterResource, } // setupWithMeterEmitter builds a handler with metering enabled and a fake @@ -1206,9 +1205,10 @@ func setupWithMeterEmitter(t *testing.T, lggr logger.Logger, emitErr error) (*co metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) resourceManager := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, }) handler, err := NewConnectorHandler( lggr, @@ -1263,19 +1263,18 @@ func TestRegisterWorkflow_MetersReserveThenUpdate(t *testing.T) { record := emitter.records[0] id := record.GetIdentity() require.Equal(t, testBaseIdentity.Product, id.GetProduct()) + require.Equal(t, testBaseIdentity.Tenant, id.GetTenant()) + require.Equal(t, testBaseIdentity.NumericTenantID, id.GetNumericTenantId()) require.Equal(t, testBaseIdentity.Environment, id.GetEnvironment()) require.Equal(t, testBaseIdentity.Zone, id.GetZone()) - require.Equal(t, testBaseIdentity.DONID, id.GetDonId()) - require.Equal(t, testBaseIdentity.NodeID, id.GetNodeId()) + require.Equal(t, testBaseIdentity.DonID(), id.GetDon().GetDonId()) + require.Equal(t, testBaseIdentity.NodeID(), id.GetDon().GetNodeId()) require.Equal(t, meterService, id.GetService()) - require.Equal(t, meterResource, id.GetResource()) - require.Equal(t, meterResourceType, id.GetResourceType()) + require.Equal(t, meterResource, id.GetResourcePool()) + require.Equal(t, meterResourceType, record.GetUtilizations()[0].GetResourceType()) // resource_id is the workflow ID (HTTP registrations are workflow-scoped). - require.Equal(t, testWorkflowID, id.GetResourceId()) - require.Equal(t, int64(1), record.GetUtilization().GetValue()) - require.Equal(t, - resourcemanager.IdempotencyKey(testBaseIdentity.WithResourceID(testWorkflowID), meteringpb.MeterAction_METER_ACTION_RESERVE, testWorkflowID), - record.GetUtilization().GetIdempotencyKey()) + require.Equal(t, testWorkflowID, record.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "1", record.GetUtilizations()[0].GetValue()) // Re-registering the same workflow emits UPDATE, not a second RESERVE. sendCh2 := make(chan capabilities.TriggerAndId[*http.Payload], 1) @@ -1310,24 +1309,18 @@ func TestRegisterWorkflow_VersionUpdate_MetersReleaseThenReserve(t *testing.T) { meteringpb.MeterAction_METER_ACTION_RESERVE, }, emitter.actions()) - // RESERVE(A) anchors the old workflow ID via its resource_id. + // RESERVE(A) anchors the old workflow ID via utilization.resource_id. reserveA := emitter.records[0] - require.Equal(t, testWorkflowID1, reserveA.GetIdentity().GetResourceId()) + require.Equal(t, testWorkflowID1, reserveA.GetUtilizations()[0].GetResourceId()) // RELEASE targets the PREVIOUS workflow ID under the same owner; its - // resource_id is that previous workflow ID. + // utilization.resource_id is that previous workflow ID. release := emitter.records[1] - require.Equal(t, testWorkflowID1, release.GetIdentity().GetResourceId()) - require.Equal(t, - resourcemanager.IdempotencyKey(testBaseIdentity.WithResourceID(testWorkflowID1), meteringpb.MeterAction_METER_ACTION_RELEASE, testWorkflowID1), - release.GetUtilization().GetIdempotencyKey()) + require.Equal(t, testWorkflowID1, release.GetUtilizations()[0].GetResourceId()) // The trailing RESERVE anchors the new workflow ID. reserveB := emitter.records[2] - require.Equal(t, testWorkflowID2, reserveB.GetIdentity().GetResourceId()) - require.Equal(t, - resourcemanager.IdempotencyKey(testBaseIdentity.WithResourceID(testWorkflowID2), meteringpb.MeterAction_METER_ACTION_RESERVE, testWorkflowID2), - reserveB.GetUtilization().GetIdempotencyKey()) + require.Equal(t, testWorkflowID2, reserveB.GetUtilizations()[0].GetResourceId()) } func TestUnregisterWorkflow_MetersRelease(t *testing.T) { @@ -1345,7 +1338,7 @@ func TestUnregisterWorkflow_MetersRelease(t *testing.T) { meteringpb.MeterAction_METER_ACTION_RELEASE, }, emitter.actions()) release := emitter.records[1] - require.Equal(t, testWorkflowID, release.GetIdentity().GetResourceId()) + require.Equal(t, testWorkflowID, release.GetUtilizations()[0].GetResourceId()) // Unregistering an absent workflow fails and must not emit RELEASE. err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) @@ -1390,10 +1383,11 @@ func TestSnapshot_EmitsOneEntryPerActiveWorkflow(t *testing.T) { metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: time.Minute, - Clock: clock, + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, }) handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, testBaseIdentity) require.NoError(t, err) @@ -1413,25 +1407,24 @@ func TestSnapshot_EmitsOneEntryPerActiveWorkflow(t *testing.T) { return len(emitter.snapshots) == 2 }, time.Second, time.Millisecond) - // One MeterSnapshot per active workflow, each fully identified by its own - // per-workflow identity (resource_id is the workflow ID). + // One MeterSnapshot per active workflow, keyed by utilization.resource_id. require.Len(t, emitter.snapshots, 2) byWorkflowID := map[string]*meteringpb.MeterSnapshot{} for _, s := range emitter.snapshots { - byWorkflowID[s.GetIdentity().GetResourceId()] = s + byWorkflowID[s.GetUtilization()[0].GetResourceId()] = s } require.Len(t, byWorkflowID, 2) r1 := byWorkflowID[testWorkflowID1] require.NotNil(t, r1) require.Equal(t, testBaseIdentity.Product, r1.GetIdentity().GetProduct()) - require.Equal(t, meterResource, r1.GetIdentity().GetResource()) - require.Equal(t, meterResourceType, r1.GetIdentity().GetResourceType()) - require.Equal(t, int64(1), r1.GetUtilization().GetValue()) + require.Equal(t, meterResource, r1.GetIdentity().GetResourcePool()) + require.Equal(t, meterResourceType, r1.GetUtilization()[0].GetResourceType()) + require.Equal(t, "1", r1.GetUtilization()[0].GetValue()) r2 := byWorkflowID[testWorkflowID2] require.NotNil(t, r2) - require.Equal(t, int64(1), r2.GetUtilization().GetValue()) + require.Equal(t, "1", r2.GetUtilization()[0].GetValue()) } // TestClose_EmitsReleasePerActiveWorkflow asserts graceful close drains a @@ -1444,9 +1437,10 @@ func TestClose_EmitsReleasePerActiveWorkflow(t *testing.T) { metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, }) handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, testBaseIdentity) require.NoError(t, err) @@ -1466,7 +1460,7 @@ func TestClose_EmitsReleasePerActiveWorkflow(t *testing.T) { released := map[string]bool{} for _, r := range emitter.records { - released[r.GetIdentity().GetResourceId()] = true + released[r.GetUtilizations()[0].GetResourceId()] = true } require.True(t, released[testWorkflowID1]) require.True(t, released[testWorkflowID2]) @@ -1482,10 +1476,10 @@ func TestDONIDFallback_UsesWorkflowDON(t *testing.T) { store := newWorkflowStore(lggr) metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) - rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{Enabled: true, Emitter: emitter}) + rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{MeterRecordsEnabled: true, Emitter: emitter}) // Base identity WITHOUT a capability DON (host did not inject one). base := testBaseIdentity - base.DONID = "" + base.Don = &resourcemanager.DonIdentity{NodeID: "node-csa-pubkey"} handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, base) require.NoError(t, err) @@ -1495,7 +1489,7 @@ func TestDONIDFallback_UsesWorkflowDON(t *testing.T) { require.NoError(t, handler.RegisterWorkflow(t.Context(), input, sendCh)) require.Len(t, emitter.records, 1) - require.Equal(t, "99", emitter.records[0].GetIdentity().GetDonId()) + require.Equal(t, "99", emitter.records[0].GetIdentity().GetDon().GetDonId()) } // TestResolveWorkflowMetadata_PreservesStoredWorkflowOwner tests that the workflowOwner diff --git a/http_trigger/trigger/trigger.go b/http_trigger/trigger/trigger.go index 5e1952711..28e4fbac4 100644 --- a/http_trigger/trigger/trigger.go +++ b/http_trigger/trigger/trigger.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "os" "strconv" "strings" "time" @@ -26,8 +25,9 @@ import ( const ServiceName = "HTTPTriggerCapability" // Metering identity constants for the HTTP trigger. Service is the stable -// service constant (it must not encode environment or zone); Resource and -// ResourceType identify the HTTP workflow-registration pool and its billing +// service constant (it must not encode environment or zone); resource pool and +// utilization resource_type identify the HTTP workflow-registration pool and +// its billing // unit. const ( meterService = "http-trigger" @@ -38,26 +38,6 @@ const ( meterProductFallback = "cre" ) -// meterRecordsEnabledEnvVar gates MeterRecord emission; the name is the -// cross-producer convention for the metering rollout (SHARED-2718). -const meterRecordsEnabledEnvVar = "CL_METER_RECORDS_ENABLED" - -// meterRecordsEnabled reads the metering gate from the environment. Unset or -// unparseable values disable emission; metering config must never prevent the -// capability from starting. -func meterRecordsEnabled(lggr logger.Logger) bool { - v := os.Getenv(meterRecordsEnabledEnvVar) - if v == "" { - return false - } - enabled, err := strconv.ParseBool(v) - if err != nil { - lggr.Warnw("Invalid value for "+meterRecordsEnabledEnvVar+", meter record emission disabled", "value", v, "error", err) - return false - } - return enabled -} - var _ server.HTTPCapability = &service{} type WorkflowRegistrationInput struct { @@ -72,6 +52,7 @@ type WorkflowRegistrationMetadata struct { EngineVersion string WorkflowDONID uint32 ReferenceID string + OrganizationID string // DecodedWorkflowName is the human-readable workflow name DecodedWorkflowName string } @@ -82,6 +63,23 @@ type ConnectorHandler interface { UnregisterWorkflow(ctx context.Context, workflowID string) error } +// MeteringConfig carries emission toggles and deployment/node identity +// dimensions for the HTTP trigger's ResourceManager. +type MeteringConfig struct { + MeterRecordsEnabled bool + MeterSnapshotsEnabled bool + Deployment resourcemanager.DeploymentIdentity +} + +func (m MeteringConfig) resourceManagerConfig() resourcemanager.ResourceManagerConfig { + return resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: m.MeterRecordsEnabled, + MeterSnapshotsEnabled: m.MeterSnapshotsEnabled, + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + } +} + type service struct { services.StateMachine lggr logger.SugaredLogger @@ -90,12 +88,16 @@ type service struct { metrics *Metrics limitsFactory limits.Factory orgResolver orgresolver.OrgResolver + // metering carries static deployment/node identity dimensions plus metering + // emission toggles delivered via loop.EnvConfig. + metering MeteringConfig } -func NewService(lggr logger.Logger, limitsFactory limits.Factory) *service { +func NewService(lggr logger.Logger, limitsFactory limits.Factory, metering MeteringConfig) *service { return &service{ lggr: logger.Sugared(logger.Named(lggr, ServiceName)), limitsFactory: limitsFactory, + metering: metering, } } @@ -122,12 +124,8 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa } metadataPublisher := NewGatewayMetadataPublisher(s.lggr, dependencies.GatewayConnector, workflowStore, s.cfg, s.metrics) requestCache := newRequestCache(s.lggr, dependencies.Store, time.Duration(s.cfg.RequestCacheTTL)*time.Second) - resourceManager := resourcemanager.NewResourceManager(s.lggr, resourcemanager.ResourceManagerConfig{ - Enabled: meterRecordsEnabled(s.lggr), - Emitter: beholder.GetEmitter(), - SnapshotInterval: resourcemanager.DefaultSnapshotInterval, - }) - baseIdentity := baseMeterIdentity(dependencies) + resourceManager := resourcemanager.NewResourceManager(s.lggr, s.metering.resourceManagerConfig()) + baseIdentity := baseMeterIdentity(dependencies, s.metering.Deployment) s.connectorHandler, err = NewConnectorHandler(s.lggr, dependencies.GatewayConnector, s.cfg, workflowStore, metadataPublisher, requestCache, s.metrics, s.orgResolver, resourceManager, baseIdentity) if err != nil { return err @@ -135,18 +133,19 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa return s.Start(ctx) } -// baseMeterIdentity builds the HTTP trigger's base metering identity from the -// host-injected dependencies. The six coarse dimensions plus the service-level -// resource / resource_type are fixed here; the per-workflow resource_id is set -// per emission via ResourceIdentity.WithResourceID. +// baseMeterIdentity builds the HTTP trigger's base metering identity. The +// deployment/node dimensions come from deployment (delivered via +// loop.EnvConfig); the DON dimension comes from the host-injected +// CapabilityDonID. The service-level resource_pool is fixed here; the +// per-workflow billing fields are set on each Utilization. // -// DONID is the capability DON the trigger LOOP was spawned for +// The DON identifier is the capability DON the trigger LOOP was spawned for // (deps.CapabilityDonID, host-injected via capabilities#619). When the host has -// not populated it (0), DONID is left empty here and resolved per registration +// not populated it (0), the DON identifier is left empty here and resolved per registration // from the workflow DON at emit time (see connectorHandler.donID). Product // falls back to a constant when the host did not inject one. -func baseMeterIdentity(deps core.StandardCapabilitiesDependencies) resourcemanager.ResourceIdentity { - product := deps.Product +func baseMeterIdentity(deps core.StandardCapabilitiesDependencies, deployment resourcemanager.DeploymentIdentity) resourcemanager.ResourceIdentity { + product := deployment.Product if product == "" { product = meterProductFallback } @@ -154,15 +153,22 @@ func baseMeterIdentity(deps core.StandardCapabilitiesDependencies) resourcemanag if deps.CapabilityDonID != 0 { donID = strconv.FormatUint(uint64(deps.CapabilityDonID), 10) } + var donIdentity *resourcemanager.DonIdentity + if donID != "" || deployment.NodeID != "" { + donIdentity = &resourcemanager.DonIdentity{ + DonID: donID, + NodeID: deployment.NodeID, + } + } return resourcemanager.ResourceIdentity{ - Product: product, - Environment: deps.Environment, - Zone: deps.Zone, - DONID: donID, - NodeID: deps.NodeID, - Service: meterService, - Resource: meterResource, - ResourceType: meterResourceType, + Product: product, + Tenant: deployment.Tenant, + NumericTenantID: deployment.NumericTenantID, + Environment: deployment.Environment, + Zone: deployment.Zone, + Don: donIdentity, + Service: meterService, + ResourcePool: meterResource, } } diff --git a/http_trigger/trigger/trigger_test.go b/http_trigger/trigger/trigger_test.go index 6d0413547..25180b82c 100644 --- a/http_trigger/trigger/trigger_test.go +++ b/http_trigger/trigger/trigger_test.go @@ -56,7 +56,7 @@ func TestService_RegisterTrigger(t *testing.T) { mockHandler := &mockConnectorHandler{ registerErr: tc.registerErr, } - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, MeteringConfig{}) cfgStr := fmt.Sprintf(`{"sendChannelBufferSize": %d}`, tc.sendChannelBufSize) gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ @@ -118,7 +118,7 @@ func TestService_UnregisterTrigger(t *testing.T) { mockHandler := &mockConnectorHandler{ unregisterErr: tt.handlerErr, } - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, MeteringConfig{}) cfg := "{}" gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ @@ -141,7 +141,7 @@ func TestService_UnregisterTrigger(t *testing.T) { } func TestService_Initialise_EmptyConfig(t *testing.T) { - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, MeteringConfig{}) gc := mockedGatewayConnector(t) err := svc.Initialise(context.Background(), core.StandardCapabilitiesDependencies{ @@ -157,7 +157,7 @@ func TestService_Initialise_EmptyConfig(t *testing.T) { func TestService_Start_HealthReport_Ready_Close(t *testing.T) { mockHandler := &mockConnectorHandler{} - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, MeteringConfig{}) cfg := "{}" gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ diff --git a/integration_tests/go.mod b/integration_tests/go.mod index ecd6d51c2..72fe52bb1 100644 --- a/integration_tests/go.mod +++ b/integration_tests/go.mod @@ -26,10 +26,10 @@ require ( github.com/smartcontractkit/capabilities/http_trigger v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/capabilities/loadtestwritetarget v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chain-selectors v1.0.104 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260609161557-8ceae53b8ab1 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260512150409-b4068bf735e6 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528221400-84746b70eeeb github.com/smartcontractkit/chainlink/v2 v2.29.1-cre-beta.0.0.20260609174137-e2407e0bdd98 github.com/smartcontractkit/cre-sdk-go v1.5.0 @@ -314,7 +314,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc // indirect github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260428133800-3b1484e8b1fd // indirect github.com/smartcontractkit/chainlink-common/keystore v1.2.0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect @@ -329,6 +329,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.10.1-0.20260528221400-84746b70eeeb // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/integration_tests/go.sum b/integration_tests/go.sum index ce890a667..493a16f69 100644 --- a/integration_tests/go.sum +++ b/integration_tests/go.sum @@ -1159,12 +1159,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260428133800-3b1484e8b1fd h1:IMopuENFVS63AerRELdfWo6o60UNUidcldJOxJLmk24= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260428133800-3b1484e8b1fd/go.mod h1:SBN8Urnh5sQvrQRbSo1Nr8coWatHg8LZoPw3R/42sho= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37 h1:ZZlU2e+hVC1Y8VAczVNGZBM3rU3HSqkOCn2KZHHV7gc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37/go.mod h1:paOB/6dy57owHtOGzhgaRBWRDT5BEWfnJF5M7sgkcro= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0 h1:NExKM/D0HneOq/N5LGTbkV4VOa0UHCvfTNEb4GqYpto= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a h1:8bIqv4r7SgDWkXL2Qz/Ijw+YjZY1uroIte3E2v2keVk= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260609161557-8ceae53b8ab1 h1:VdJBtNmasHzISQQF0k0LHFh44WDKO7S00VyaT7qykuc= @@ -1193,12 +1193,14 @@ github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0. github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:ATjAPIVJibHRcIfiG47rEQkUIOoYa6KDvWj3zwCAw6g= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d h1:AJy55QJ/pBhXkZjc7N+ATnWfxrcjq9BI9DmdtdjwDUQ= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:5JdppgngCOUS76p61zCinSCgOhPeYQ+OcDUuome5THQ= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 h1:iRFmfMFQtcnhGDjCuARQG4MPbcmbbJDDw7MUH3GcGy8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b h1:VDgJWDipihV9f7M5+d21d1RzSsg5rEv+iI12oN1VQbo= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 h1:SG+wAsNyAcA6Kk19ljuxi3HK9Ll2lpHik8OKoY4x7A0= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/orchestrator v0.10.1-0.20260528221400-84746b70eeeb h1:G8X3SR21VYAHWkDkNGZCjsrWrLJoVmXMpYBa2KKK3GU= From 1de77a554b4f0e6ad405443ea16362c76855bb92 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Thu, 9 Jul 2026 12:54:09 -0400 Subject: [PATCH 3/6] build: add local chainlink-common/metering replaces to sibling modules cron already carried the SHARED-2709 local `replace` stack for chainlink-common and chainlink-protos/metering/go; its sibling modules (http_trigger, chain_capabilities/evm, integration_tests) pinned the same versions but lacked the replaces, so they resolved a different module state than cron. Add the same replace directives (depth-adjusted) to keep the local stack consistent across all four modules. No version pins bumped. --- chain_capabilities/evm/go.mod | 7 +++++++ http_trigger/go.mod | 7 +++++++ integration_tests/go.mod | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/chain_capabilities/evm/go.mod b/chain_capabilities/evm/go.mod index da3bd6fac..ab31491bf 100644 --- a/chain_capabilities/evm/go.mod +++ b/chain_capabilities/evm/go.mod @@ -2,6 +2,13 @@ module github.com/smartcontractkit/capabilities/chain_capabilities/evm go 1.26.2 +// Unpublished local stack for SHARED-2709; drop once chainlink-common and +// chainlink-protos/metering/go are tagged. +replace ( + github.com/smartcontractkit/chainlink-common => ../../../chainlink-common + github.com/smartcontractkit/chainlink-protos/metering/go => ../../../chainlink-protos/metering/go +) + require ( github.com/ethereum/go-ethereum v1.17.0 github.com/google/go-cmp v0.7.0 diff --git a/http_trigger/go.mod b/http_trigger/go.mod index 986e0f37d..4a10c1246 100644 --- a/http_trigger/go.mod +++ b/http_trigger/go.mod @@ -2,6 +2,13 @@ module github.com/smartcontractkit/capabilities/http_trigger go 1.26.2 +// Unpublished local stack for SHARED-2709; drop once chainlink-common and +// chainlink-protos/metering/go are tagged. +replace ( + github.com/smartcontractkit/chainlink-common => ../../chainlink-common + github.com/smartcontractkit/chainlink-protos/metering/go => ../../chainlink-protos/metering/go +) + require ( github.com/jonboulle/clockwork v0.5.0 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 diff --git a/integration_tests/go.mod b/integration_tests/go.mod index 72fe52bb1..70f1fcb86 100644 --- a/integration_tests/go.mod +++ b/integration_tests/go.mod @@ -2,6 +2,13 @@ module github.com/smartcontractkit/capabilities/integration_tests go 1.26.4 +// Unpublished local stack for SHARED-2709; drop once chainlink-common and +// chainlink-protos/metering/go are tagged. +replace ( + github.com/smartcontractkit/chainlink-common => ../../chainlink-common + github.com/smartcontractkit/chainlink-protos/metering/go => ../../chainlink-protos/metering/go +) + replace github.com/smartcontractkit/capabilities/loadtestwritetarget => ../loadtestwritetarget replace github.com/smartcontractkit/capabilities/http_action => ../http_action From 2a36b71a9c1ac95e076b3a4221a97161f75b1315 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Thu, 9 Jul 2026 14:21:44 -0400 Subject: [PATCH 4/6] feat: delta-based metering for cron, http, and evm trigger producers Phase 3 of the delta-based metering redesign. Replaces the RESERVE/RELEASE emitter model with signed-delta UPDATE records built on the new chainlink-common resourcemanager API. All three producers (cron, http_trigger, chain_capabilities/evm): - Emit only METER_ACTION_UPDATE via EmitDelta (register=+N, unregister=-N); RESERVE/RELEASE are never emitted. - Delete all process-lifecycle (graceful-close) emissions; the Meterable is deregistered from the ResourceManager first on shutdown, and billing releases a resource by its absence from the next snapshot. - Store the workflow OWNER (never a resolved org) in durable state; org is resolved at emit time via orgresolver.ResolveOrEmpty and in snapshots via a shared CachingOrgResolver whose Start/Close are wired into each service. - Adopt the Phase 2 helpers: ConfigFromEnv, NewBaseIdentity, WithWorkflowDonFallback, ResolveOrEmpty, NewCaching. event_id is stamped by the ResourceManager per emission (producers never set it). - Resolve the authoritative CapDONID once and feed the single value to both the metering identity and the events.KeyDonID labeler. EVM: shared physical filters are billed once via a derived 0<->1 refcount (+addressCount on activation, -addressCount on release), computed atomically under the store lock; snapshots dedup to one entry per physicalFilterID with "lowest triggerID's owner" attribution; the store write is synchronous before the delta emission and cleanUpStaleFilters gains a minimum-age guard. Cron: emitMeterRecord derives don_id/owner from stored trigger state and shares utilization construction with GetUtilization; the callback closes the resurrection window with an atomic WriteIfPresent. HTTP: upsertWorkflow atomically returns the evicted workflow, removing the RegisterWorkflow pre-read TOCTOU; same-ID re-register emits nothing, version update emits -1/+1; RM start failure now logs-and-continues (fail-open, uniform with EVM). Tests updated for delta semantics (signed values, no shutdown emissions, EVM 0<->1 dedup, unique event_ids, cll-meter domain, shared CapDONID). go.mod/go.sum pick up the forced transitive node-platform resolution required to build against the local chainlink-common replace. --- chain_capabilities/evm/go.mod | 2 +- chain_capabilities/evm/go.sum | 2 + chain_capabilities/evm/main.go | 106 ++------ chain_capabilities/evm/trigger/store.go | 82 ++++-- chain_capabilities/evm/trigger/trigger.go | 243 ++++++++++++------ .../evm/trigger/trigger_metering_test.go | 178 +++++++++---- cron/go.mod | 2 +- cron/go.sum | 2 + cron/trigger/metering_test.go | 93 +++---- cron/trigger/store.go | 18 ++ cron/trigger/trigger.go | 214 ++++++++------- http_trigger/go.mod | 2 +- http_trigger/go.sum | 2 + http_trigger/main.go | 20 +- http_trigger/trigger/connector_handler.go | 172 ++++++------- .../trigger/connector_handler_test.go | 112 ++++---- .../gateway_metadata_publisher_test.go | 4 +- http_trigger/trigger/trigger.go | 91 ++----- http_trigger/trigger/trigger_test.go | 9 +- http_trigger/trigger/workflow.go | 30 +-- http_trigger/trigger/workflow_test.go | 47 ++-- 21 files changed, 765 insertions(+), 666 deletions(-) diff --git a/chain_capabilities/evm/go.mod b/chain_capabilities/evm/go.mod index ab31491bf..e2e846c33 100644 --- a/chain_capabilities/evm/go.mod +++ b/chain_capabilities/evm/go.mod @@ -102,7 +102,7 @@ require ( github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260326122810-b657beadfb57 // indirect github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260401162955-be2bc6b5264b // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe // indirect diff --git a/chain_capabilities/evm/go.sum b/chain_capabilities/evm/go.sum index fe8c010e8..2ba3254c2 100644 --- a/chain_capabilities/evm/go.sum +++ b/chain_capabilities/evm/go.sum @@ -499,6 +499,8 @@ github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-8 github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e h1:Hv9Mww35LrufCdM9wtS9yVi/rEWGI1UnjHbcKKU0nVY= diff --git a/chain_capabilities/evm/main.go b/chain_capabilities/evm/main.go index e7b70397a..304e368d9 100644 --- a/chain_capabilities/evm/main.go +++ b/chain_capabilities/evm/main.go @@ -28,13 +28,13 @@ import ( "github.com/smartcontractkit/capabilities/chain_capabilities/evm/trigger" "github.com/smartcontractkit/capabilities/libs/loopserver" - "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" evmcappb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm" evmcapserver "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm/server" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/types/core" @@ -48,40 +48,11 @@ type capabilityGRPCService struct { capability lggr logger.Logger limitsFactory limits.Factory - // metering carries emission toggles and deployment/node identity dimensions - // delivered to the plugin process via loop.EnvConfig (set at startup). The - // zero value is valid and leaves those dimensions empty/disabled. - metering meteringConfig -} - -type meteringConfig struct { - meterRecordsEnabled bool - meterSnapshotsEnabled bool - deployment resourcemanager.DeploymentIdentity -} - -func newMeteringConfig(env loop.EnvConfig) meteringConfig { - return meteringConfig{ - meterRecordsEnabled: env.MeterRecordsEnabled, - meterSnapshotsEnabled: env.MeterSnapshotsEnabled, - deployment: resourcemanager.DeploymentIdentity{ - Product: env.MeterProduct, - Tenant: env.MeterTenant, - NumericTenantID: env.MeterNumericTenantID, - Environment: env.MeterEnvironment, - Zone: env.MeterZone, - NodeID: env.MeterNodeID, - }, - } -} - -func (m meteringConfig) resourceManagerConfig() resourcemanager.ResourceManagerConfig { - return resourcemanager.ResourceManagerConfig{ - MeterRecordsEnabled: m.meterRecordsEnabled, - MeterSnapshotsEnabled: m.meterSnapshotsEnabled, - Emitter: beholder.GetEmitter(), - SnapshotInterval: resourcemanager.DefaultSnapshotInterval, - } + // metering is the resolved metering Config (ResourceManagerConfig + + // DeploymentIdentity) produced from loop.EnvConfig by + // resourcemanager.ConfigFromEnv at startup. The zero value is valid and + // leaves those dimensions empty/disabled. + metering resourcemanager.Config } type capability struct { @@ -98,7 +69,10 @@ var _ evmcapserver.ClientCapability = &capabilityGRPCService{} func main() { loopserver.ServeNew(CapabilityName, func(s *loop.Server) loop.StandardCapabilities { - meteringCfg := newMeteringConfig(s.EnvConfig) + // ConfigFromEnv is the single, canonical loop-env -> metering mapping + // (enable flags, beholder emitter, snapshot interval, deployment + // identity); no per-main copy of that mapping. + meteringCfg := resourcemanager.ConfigFromEnv(&s.EnvConfig) return evmcapserver.NewClientServer(&capabilityGRPCService{ lggr: s.Logger, limitsFactory: s.LimitsFactory, @@ -199,17 +173,28 @@ func (c *capabilityGRPCService) Initialise(ctx context.Context, dependencies cor return fmt.Errorf("failed to init evm relayer for chainID %d from relayer: %w", cfg.ChainID, err) } - // TODO: add org resolver capabilityID := fmt.Sprintf("%s (%d)", c.id, cfg.ChainID) // The ResourceManager owns the snapshot tick; the LogTriggerService starts it // as a sub-service and registers itself, so it must be configured with a // snapshot interval here. Identity/snapshots are gated by the same metering // env flag as MeterRecords. - resourceManager := resourcemanager.NewResourceManager(c.lggr, c.metering.resourceManagerConfig()) - baseIdentity := newBaseMeteringIdentity(dependencies, c.metering.deployment) + resourceManager := resourcemanager.NewResourceManager(c.lggr, c.metering.ResourceManagerConfig) + // The authoritative DON ID is the host-injected CapabilityDonID; when 0, + // NewBaseIdentity leaves don_id empty and each emission falls back to the + // consumer workflow's DON via the LogTriggerService's resolveDONID. + baseIdentity := resourcemanager.NewBaseIdentity(c.metering.DeploymentIdentity, dependencies.CapabilityDonID, trigger.MeteringService, trigger.MeteringResource) + // Wrap the injected org resolver in a CachingOrgResolver so the snapshot + // path (GetUtilization, contractually no-network) resolves org from memory. + // We own its Start/Close (wired into the start loop and Close below). + // The LogTriggerService owns the caching resolver's Start/Close lifecycle + // (it type-asserts the *CachingOrgResolver), so we only construct it here. + var orgResolver orgresolver.OrgResolver = dependencies.OrgResolver + if dependencies.OrgResolver != nil { + orgResolver = orgresolver.NewCaching(dependencies.OrgResolver, resourcemanager.DefaultSnapshotInterval) + } c.triggerService, err = trigger.NewLogTriggerService(evmRelayer, trigger.NewLogTriggerStore(), c.lggr, capabilityID, processor, messageBuilder, cfg.LogTriggerPollInterval, cfg.LogTriggerSendChannelBufferSize, cfg.LogTriggerLimitQueryLogSize, c.limitsFactory, - dependencies.OrgResolver, dependencies.TriggerEventStore, resourceManager, baseIdentity, c.chainSelector) + orgResolver, dependencies.TriggerEventStore, resourceManager, baseIdentity, c.chainSelector) if err != nil { return fmt.Errorf("error when creating trigger: %w", err) } @@ -244,47 +229,6 @@ func (c *capabilityGRPCService) Initialise(ctx context.Context, dependencies cor return nil } -// defaultMeteringProduct is the fallback metering product dimension used when -// the host did not provide one via loop.EnvConfig (a legacy node or a boot path -// not yet updated). The other deployment dimensions (environment, zone, -// node_id) have no meaningful constant and are left empty in that case. -const defaultMeteringProduct = "cre" - -// newBaseMeteringIdentity builds the EVM log trigger's base metering identity. -// The deployment/node dimensions come from deployment (delivered via -// loop.EnvConfig); the DON dimension comes from the host-injected -// CapabilityDonID. It carries the six coarse dimensions plus the service-level -// resource/resource_type; the per-resource ResourceID is set per emit/snapshot. -// When CapabilityDonID is 0, the DON identifier is left empty here and resolved per emit from -// the consumer's WorkflowDonID (see LogTriggerService.resolveDONID). -func newBaseMeteringIdentity(deps core.StandardCapabilitiesDependencies, deployment resourcemanager.DeploymentIdentity) resourcemanager.ResourceIdentity { - product := deployment.Product - if product == "" { - product = defaultMeteringProduct - } - var donID string - if deps.CapabilityDonID != 0 { - donID = strconv.FormatUint(uint64(deps.CapabilityDonID), 10) - } - var donIdentity *resourcemanager.DonIdentity - if donID != "" || deployment.NodeID != "" { - donIdentity = &resourcemanager.DonIdentity{ - DonID: donID, - NodeID: deployment.NodeID, - } - } - return resourcemanager.ResourceIdentity{ - Product: product, - Tenant: deployment.Tenant, - NumericTenantID: deployment.NumericTenantID, - Environment: deployment.Environment, - Zone: deployment.Zone, - Don: donIdentity, - Service: trigger.MeteringService, - ResourcePool: trigger.MeteringResource, - } -} - func (c *capabilityGRPCService) unmarshalConfig(configStr string) (*config.Config, error) { var cfg config.Config if err := json.Unmarshal([]byte(configStr), &cfg); err != nil { diff --git a/chain_capabilities/evm/trigger/store.go b/chain_capabilities/evm/trigger/store.go index 4362d0bb2..f00c31512 100644 --- a/chain_capabilities/evm/trigger/store.go +++ b/chain_capabilities/evm/trigger/store.go @@ -16,24 +16,30 @@ type filter struct { // physicalFilterID is the workflow-independent content hash of the filter's // physical matching criteria (chain selector + canonicalized addresses, // event sigs, and positional topics). It is the metering ResourceID and the - // RESERVE/RELEASE event identity, so the unregister, cleanup, snapshot, and - // graceful-close paths all reuse it from here without the request input. + // shared-resource refcount key, so the unregister, cleanup, and snapshot + // paths all reuse it from here without the request input. Identical filters + // registered by different triggers share one physicalFilterID and are billed + // once (a +delta on the 0->1 activation, a -delta on the 1->0 release). physicalFilterID string - // reservedAddressCount is the number of filter addresses metered in the - // RESERVE record when the filter was registered. The matching RELEASE - // must carry the same value, and UnregisterLogTrigger ignores its request - // input, so the count is stashed here at registration. + // reservedAddressCount is the number of filter addresses this filter bills: + // the +delta emitted on the physical filter's 0->1 activation, and the + // -delta on its 1->0 release, both carry this value. UnregisterLogTrigger + // ignores its request input, so the count is stashed here at registration. reservedAddressCount int64 // donID is stashed from the registration RequestMetadata so the - // unregister/cleanup/snapshot/close paths can emit a metering record with - // the same identity as the RESERVE, without the original request. It is the - // resolved metering DON ID string (capability DON, or the consumer - // WorkflowDonID fallback when the host did not inject a capability DON); - // empty when neither is known. - donID string - orgID string - expressions []query.Expression - confidence primitives.ConfidenceLevel + // unregister/cleanup/snapshot paths reproduce the same identity as the + // activation delta without the original request. It is the resolved metering + // DON ID string (capability DON, or the consumer WorkflowDonID fallback when + // the host did not inject a capability DON); empty when neither is known. + donID string + // workflowOwner is the durable attribution key. We store the workflow OWNER, + // never a resolved org ID: the org is resolved fresh at each emission (via + // orgresolver.ResolveOrEmpty) and at snapshot time (via the shared + // CachingOrgResolver), so a re-linked owner is picked up without rewriting + // durable state. + workflowOwner string + expressions []query.Expression + confidence primitives.ConfidenceLevel } type logTriggerState struct { @@ -57,6 +63,8 @@ type LogTriggerStore interface { Read(triggerID string) (value logTriggerState, ok bool) ReadAll() (values map[string]logTriggerState) Write(triggerID string, value logTriggerState) + WriteAndIsFirstForPhysical(triggerID string, value logTriggerState) (firstForPhysical bool) + DeleteAndIsLastForPhysical(triggerID, physicalFilterID string) (found, lastForPhysical bool) Update(triggerID string, lastBlock *big.Int, unfinalizedSentEventIDs map[string]*big.Int) error Delete(triggerID string) } @@ -90,6 +98,50 @@ func (cs *logTriggerStore) Write(triggerID string, value logTriggerState) { cs.triggers[triggerID] = value } +// WriteAndIsFirstForPhysical writes value under triggerID and reports whether, +// at the instant of the write, no OTHER trigger already held +// value.physicalFilterID. A true result is the 0->1 activation of that shared +// physical filter — the only transition that bills a +delta. Deriving the +// transition from owned state at operation time keeps the emitter stateless +// (no ledger). The scan and write are atomic under the store lock so two +// concurrent registrations of the same physical filter can never both observe +// zero and double-bill. +func (cs *logTriggerStore) WriteAndIsFirstForPhysical(triggerID string, value logTriggerState) (firstForPhysical bool) { + cs.mu.Lock() + defer cs.mu.Unlock() + firstForPhysical = true + for id, existing := range cs.triggers { + if id == triggerID { + continue + } + if existing.physicalFilterID == value.physicalFilterID { + firstForPhysical = false + break + } + } + cs.triggers[triggerID] = value + return firstForPhysical +} + +// DeleteAndIsLastForPhysical deletes triggerID and reports whether any remaining +// trigger still holds physicalFilterID. lastForPhysical is true when none +// remain — the 1->0 deactivation that bills a -delta. found reports whether the +// trigger existed. Scan and delete are atomic under the store lock. +func (cs *logTriggerStore) DeleteAndIsLastForPhysical(triggerID, physicalFilterID string) (found, lastForPhysical bool) { + cs.mu.Lock() + defer cs.mu.Unlock() + _, found = cs.triggers[triggerID] + delete(cs.triggers, triggerID) + lastForPhysical = true + for _, existing := range cs.triggers { + if existing.physicalFilterID == physicalFilterID { + lastForPhysical = false + break + } + } + return found, lastForPhysical +} + func (cs *logTriggerStore) Update(triggerID string, lastBlock *big.Int, unfinalizedSentEventIDs map[string]*big.Int) error { cs.mu.Lock() defer cs.mu.Unlock() diff --git a/chain_capabilities/evm/trigger/trigger.go b/chain_capabilities/evm/trigger/trigger.go index e0ee3d19e..3a0144596 100644 --- a/chain_capabilities/evm/trigger/trigger.go +++ b/chain_capabilities/evm/trigger/trigger.go @@ -7,6 +7,7 @@ import ( "math/big" "strconv" "strings" + "sync" "time" "google.golang.org/protobuf/proto" @@ -45,6 +46,19 @@ const ( defaultLimitQueryLogSize = 1000 ) +// cleanupInterval is how often stale log-poller filters are swept AND the +// minimum age a filter must reach before it is eligible for that sweep. The +// min-age guard closes a race the cleanup previously assumed away: a filter can +// be live at the log poller for a brief window before its store entry is +// observable, and cleaning it in that window would bill-then-kill a live +// filter. +const cleanupInterval = 30 * time.Second + +// orgResolverRefreshInterval bounds owner->org cache staleness in the snapshot +// path. It matches the snapshot cadence so a re-linked org is reflected within +// one snapshot interval. +const orgResolverRefreshInterval = resourcemanager.DefaultSnapshotInterval + // Metering identity constants for the EVM log trigger (SHARED-2711). These are // the service-level dimensions of the base ResourceIdentity: Service is the // stable service constant (it must not encode deployment environment or zone, @@ -95,6 +109,15 @@ type LogTriggerService struct { eventRateLimit limits.RateLimiter eventPayloadSizeLimiter limits.BoundLimiter[commoncfg.Size] orgResolver orgresolver.OrgResolver // Optional org resolver for fetching organization IDs + // orgResolverSvc is the CachingOrgResolver's service handle when one was + // injected; its Start/Close are wired into the service lifecycle. Nil when + // no resolver was injected. + orgResolverSvc services.Service + // filterRegisteredAt records, per log-poller filter name, the time it was + // registered at the log poller. cleanUpStaleFilters uses it to skip filters + // younger than one cleanup interval, closing the register-time window where + // a filter is live at the poller before its store entry is observable. + filterRegisteredAt sync.Map // filterID (string) -> time.Time } // NewLogTriggerService creates a new instance of logTriggerService. @@ -149,6 +172,11 @@ func NewLogTriggerService(evmService types.EVMService, store LogTriggerStore, lg if lts.orgResolver == nil { lts.lggr.Warn("OrgResolver is nil, EVM log trigger capability will not be able to fetch organization ID") } + // Own the caching resolver's lifecycle when one was injected, so its + // background refresh runs while the service is up. + if caching, ok := orgResolver.(*orgresolver.CachingOrgResolver); ok { + lts.orgResolverSvc = caching + } if lts.resourceManager == nil { lts.lggr.Warn("ResourceManager is nil, EVM log trigger capability will not emit meter records") } @@ -199,11 +227,18 @@ func (lts *LogTriggerService) start(ctx context.Context) error { if err != nil { return err } - duration := 30 * time.Second - ticker := services.NewTicker(duration) - lts.lggr.Infof("Starting clean up of failed log poller filters every %s seconds", duration) + ticker := services.NewTicker(cleanupInterval) + lts.lggr.Infof("Starting clean up of failed log poller filters every %s", cleanupInterval) lts.srvcEng.GoTick(ticker, lts.cleanUpStaleFilters) + // Start the caching org resolver (if any) so its background refresh runs + // and the snapshot path is served from memory. Fail-open. + if lts.orgResolverSvc != nil { + if err := lts.orgResolverSvc.Start(ctx); err != nil { + lts.lggr.Errorw("failed to start caching org resolver; org attribution may be empty", "err", err) + } + } + // The ResourceManager owns the snapshot tick: start it as a sub-service of // this service and Register ourselves so its tick polls GetUtilization. We // never run our own snapshot loop. The RM is fail-open and starting it must @@ -218,15 +253,22 @@ func (lts *LogTriggerService) start(ctx context.Context) error { return nil } +// close performs an orderly shutdown. There are NO process-lifecycle metering +// emissions: a graceful stop emits nothing, and billing releases each +// still-active filter by its absence from the next snapshot. The Meterable is +// deregistered from the ResourceManager FIRST so no snapshot tick can run +// against a half-torn-down service, then the base trigger, caching resolver, +// and ResourceManager are closed. func (lts *LogTriggerService) close() error { - lts.baseTrigger.Stop() - // On graceful shutdown, release every still-active filter so a clean stop is - // not seen downstream as a leaked reservation. These releases reuse each - // filter's stashed physicalFilterID + reserved count, so they dedup against - // any user-facing RELEASE on the identical idempotency key. - lts.releaseActiveFiltersOnClose(context.Background()) if lts.rmUnregister != nil { lts.rmUnregister() + lts.rmUnregister = nil + } + lts.baseTrigger.Stop() + if lts.orgResolverSvc != nil { + if err := lts.orgResolverSvc.Close(); err != nil { + lts.lggr.Errorw("failed to close caching org resolver", "err", err) + } } if lts.resourceManager != nil { return lts.resourceManager.Close() @@ -234,18 +276,6 @@ func (lts *LogTriggerService) close() error { return nil } -// releaseActiveFiltersOnClose emits a RELEASE MeterRecord for every filter still -// present in the store at shutdown, carrying the reserved address count. It runs -// before the ResourceManager is unregistered/closed. -func (lts *LogTriggerService) releaseActiveFiltersOnClose(ctx context.Context) { - if lts.resourceManager == nil { - return - } - for _, state := range lts.triggers.ReadAll() { - lts.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, state.filter, state.reservedAddressCount) - } -} - func (lts *LogTriggerService) cleanUpStaleFilters(ctx context.Context) { lts.lggr.Debugf("Starting cleanUpStaleFilters") telemetryContext := monitoring.TelemetryContext{TsStart: time.Now().UnixMilli(), RequestMetadata: capabilities.RequestMetadata{ @@ -276,18 +306,31 @@ func (lts *LogTriggerService) cleanUpStaleFilters(ctx context.Context) { lts.lggr.Debugf("Found %d filters to clean up that are not live", len(toCleanUp)) for filterID := range toCleanUp { + // Min-age guard: a filter that was registered at the log poller less + // than one cleanup interval ago may simply not have its store entry + // observable yet (the register path writes the store right after the + // RegisterLogTracking RPC returns). Skip it this round so we never + // bill-then-kill a filter that is actually live. Filters with no + // recorded registration time (e.g. orphaned from a previous process) + // are always eligible. + if registeredAt, ok := lts.filterRegisteredAt.Load(filterID); ok { + if time.Since(registeredAt.(time.Time)) < cleanupInterval { + lts.lggr.Debugf("Skipping filter %s: younger than one cleanup interval", filterID) + continue + } + } lts.lggr.Debugf("Cleaning up filter %s", filterID) if err := lts.EVMService.UnregisterLogTracking(ctx, filterID); err != nil { summary := fmt.Sprintf("failed to unregister log-tracking from the clean up thread: '%v' source triggerID: %s", err, filterID) monitoring.LogAndEmitError(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerCleanUpError(telemetryContext, summary, err.Error())) continue } + lts.filterRegisteredAt.Delete(filterID) // This is log-poller filter hygiene only; it emits no MeterRecord. An // orphaned filter has no trigger state, so it is already absent from // GetUtilization and therefore from subsequent Snapshots. Billing - // reconciles the lost reservation by that absence (the snapshot liveness - // mechanism), not by a synthetic cleanup RELEASE. We never expect, nor - // design around, orphan cleanup as a metering event. + // reconciles the lost level by that absence (the snapshot liveness + // mechanism), not by a synthetic cleanup emission. } } @@ -372,13 +415,14 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID // Build the filter's metering identity once from the already-converted // inputs: a workflow-independent content hash and the resolved DON ID. It is // stashed on the trigger state so every later path (unregister, cleanup, - // snapshot, close) reproduces the same identity without the request input. + // snapshot) reproduces the same identity without the request input. We store + // the workflow OWNER, never a resolved org, and resolve org at emit time. loggedFilter := filter{ filterID: filterID, physicalFilterID: physicalFilterID(lts.chainSelector, addresses, sigs, t2, t3, t4), reservedAddressCount: int64(len(addresses)), donID: lts.resolveDONID(meta.WorkflowDonID), - orgID: lts.resolveOrgID(ctx, meta.WorkflowOwner), + workflowOwner: meta.WorkflowOwner, expressions: expressions, confidence: confidence, } @@ -397,9 +441,30 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID monitoring.LogAndEmitError(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerError(telemetryContext, triggerID, summary, err.Error())) return nil, caperrors.NewPublicSystemError(registerError, caperrors.Unavailable) } - // RESERVE only after RegisterLogTracking succeeds: a failed registration - // holds no addresses, so it must not be billed (see the no-reserve test). - lts.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, loggedFilter, loggedFilter.reservedAddressCount) + // The filter is now live at the log poller. Record when so the stale-filter + // cleanup skips it until it is at least one interval old (see + // cleanUpStaleFilters). + lts.filterRegisteredAt.Store(filterID, time.Now()) + + // Create the polling context up front (cancelled on unregister or service + // stop) so the store write is synchronous and carries a working cancelFunc. + pollCtx, cancel := lts.srvcEng.NewCtx() + + // Write the trigger state SYNCHRONOUSLY, before emitting any delta, and let + // the store report whether this is the physical filter's 0->1 activation. + // Writing first means the orphan-cleanup thread can never observe the live + // log-poller filter without its store entry and bill-then-kill it; deriving + // the transition under the store lock keeps identical filters billed once. + firstForPhysical := lts.triggers.WriteAndIsFirstForPhysical(triggerID, logTriggerState{ + cancelFunc: cancel, + lastBlock: fromBlock, + unfinalizedSentEventIDs: make(map[string]*big.Int), + filter: loggedFilter, + }) + if firstForPhysical { + // 0->1 activation of a shared physical filter: bill +addressCount once. + lts.emitDelta(ctx, loggedFilter.reservedAddressCount, loggedFilter) + } monitoring.EmitInitiated(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerInitiated(telemetryContext, input)) @@ -407,14 +472,7 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID lts.baseTrigger.RegisterTrigger(triggerID, logCh) - lts.srvcEng.Go(func(ctx context.Context) { - ctx, cancel := context.WithCancel(ctx) - lts.triggers.Write(triggerID, logTriggerState{ - cancelFunc: cancel, - lastBlock: fromBlock, - unfinalizedSentEventIDs: make(map[string]*big.Int), - filter: loggedFilter, - }) + lts.srvcEng.GoCtx(pollCtx, func(ctx context.Context) { ctx = meta.ContextWithCRE(ctx) lts.startPolling(ctx, telemetryContext, triggerID, input, logCh) }) @@ -469,8 +527,10 @@ func (lts *LogTriggerService) generateFilterID(triggerID string) string { // DON ID, baseIdentity's DON identifier is non-empty and used as-is; otherwise the // consumer workflow's DON ID (from the request metadata) is the documented // fallback. The result is stashed on the filter at registration so the -// unregister/cleanup/snapshot/close paths reproduce the same identity without -// the request. Empty when neither source is known. +// unregister/cleanup/snapshot paths reproduce the same identity without the +// request. Empty when neither source is known. This is the SAME value stamped +// on the events.KeyDonID label (see startPolling), so metering identity and the +// event label cannot diverge. func (lts *LogTriggerService) resolveDONID(workflowDonID uint32) string { if lts.baseIdentity.DonID() != "" { return lts.baseIdentity.DonID() @@ -481,18 +541,6 @@ func (lts *LogTriggerService) resolveDONID(workflowDonID uint32) string { return "" } -func (lts *LogTriggerService) resolveOrgID(ctx context.Context, workflowOwner string) string { - if lts.orgResolver == nil || workflowOwner == "" { - return "" - } - orgID, err := lts.orgResolver.Get(ctx, workflowOwner) - if err != nil { - lts.lggr.Warnw("failed to fetch organization ID from org resolver", "workflowOwner", workflowOwner, "error", err) - return "" - } - return orgID -} - // identity returns the base metering identity with DON ID resolved for one // resource. func (lts *LogTriggerService) identity(donID string) resourcemanager.ResourceIdentity { @@ -507,27 +555,24 @@ func (lts *LogTriggerService) identity(donID string) resourcemanager.ResourceIde return id } -// emitMeterRecord emits a MeterRecord for a filter whose full state is known -// (register/unregister/close). The physical filter content hash is both the -// ResourceID and the RESERVE/RELEASE event identity, so a register retry and a -// later unregister/cleanup of the same physical filter dedup on an identical -// idempotency key. The resource is fully identified by its ResourceIdentity; -// there is no separate label metadata. Emission is fail-open and must never -// gate the path that calls it. -func (lts *LogTriggerService) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, f filter, value int64) { +// emitDelta emits a signed delta MeterRecord (METER_ACTION_UPDATE) for a shared +// physical log filter: +addressCount on the physical filter's 0->1 activation, +// -addressCount on its 1->0 release. The physical filter content hash is the +// ResourceID, so all triggers sharing it bill against one resource. The org is +// resolved fresh from the stored owner at emit time; event_id is stamped by the +// ResourceManager per emission. Emission is fail-open and must never gate the +// path that calls it. +func (lts *LogTriggerService) emitDelta(ctx context.Context, delta int64, f filter) { if lts.resourceManager == nil { return } identity := lts.identity(f.donID) - lts.resourceManager.EmitMeterRecord(ctx, identity, action, - []*meteringpb.Utilization{ - resourcemanager.NewUtilizationInt(value, resourcemanager.UtilizationFields{ - ResourceType: MeteringResourceType, - ResourceID: f.physicalFilterID, - EventID: f.physicalFilterID, - OrgID: f.orgID, - }), - }) + orgID := orgresolver.ResolveOrEmpty(ctx, lts.orgResolver, f.workflowOwner, lts.lggr) + lts.resourceManager.EmitDelta(ctx, identity, delta, resourcemanager.UtilizationFields{ + ResourceType: MeteringResourceType, + ResourceID: f.physicalFilterID, + OrgID: orgID, + }) } // ResourceIdentity implements resourcemanager.Meterable: it returns the @@ -539,22 +584,47 @@ func (lts *LogTriggerService) ResourceIdentity() resourcemanager.ResourceIdentit } // GetUtilization implements resourcemanager.Meterable: it returns one snapshot -// entry per currently active log filter for the snapshot tick. It is a cheap -// in-memory read — triggers.ReadAll already returns a copy — with no I/O and no -// lock held across the loop, as the snapshot contract requires (R6). -func (lts *LogTriggerService) GetUtilization(_ context.Context) []resourcemanager.SnapshotEntry { +// entry per distinct physical filter (NOT one per trigger registration), since +// identical filters registered by many triggers share one billable physical +// resource. The value is the shared filter's address count. Org attribution for +// a shared filter uses the deterministic "lowest triggerID's owner" rule so all +// nodes agree on the same org without coordination. It is a cheap in-memory +// read — triggers.ReadAll already returns a copy — with no I/O and no lock held +// across the loop, and org is served from the caching resolver's memory, as the +// snapshot contract requires (R6). +func (lts *LogTriggerService) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { triggers := lts.triggers.ReadAll() - entries := make([]resourcemanager.SnapshotEntry, 0, len(triggers)) - for _, state := range triggers { - f := state.filter + + // Dedup by physicalFilterID, keeping the filter owned by the lowest + // triggerID for deterministic org attribution of a shared filter. + type physicalAgg struct { + f filter + lowestTriggerID string + } + byPhysical := make(map[string]*physicalAgg, len(triggers)) + for triggerID, state := range triggers { + agg, ok := byPhysical[state.physicalFilterID] + if !ok { + byPhysical[state.physicalFilterID] = &physicalAgg{f: state.filter, lowestTriggerID: triggerID} + continue + } + if triggerID < agg.lowestTriggerID { + agg.lowestTriggerID = triggerID + agg.f = state.filter + } + } + + entries := make([]resourcemanager.SnapshotEntry, 0, len(byPhysical)) + for _, agg := range byPhysical { + f := agg.f + orgID := orgresolver.ResolveOrEmpty(ctx, lts.orgResolver, f.workflowOwner, lts.lggr) entries = append(entries, resourcemanager.SnapshotEntry{ Identity: lts.identity(f.donID), Utilizations: []*meteringpb.Utilization{ resourcemanager.NewUtilizationInt(f.reservedAddressCount, resourcemanager.UtilizationFields{ ResourceType: MeteringResourceType, ResourceID: f.physicalFilterID, - EventID: f.physicalFilterID, - OrgID: f.orgID, + OrgID: orgID, }), }, }) @@ -920,15 +990,18 @@ func (lts *LogTriggerService) UnregisterLogTrigger(ctx context.Context, triggerI } lts.lggr.Infof("UnregisterLogTrigger triggerID: %s", triggerID) trigger.cancelFunc() - lts.triggers.Delete(triggerID) + // Delete under the store lock and learn whether this was the physical + // filter's 1->0 release (no other trigger still holds it). Only that + // transition bills a -delta, mirroring the +delta billed on 0->1 activation. + _, lastForPhysical := lts.triggers.DeleteAndIsLastForPhysical(triggerID, trigger.physicalFilterID) + lts.filterRegisteredAt.Delete(lts.generateFilterID(triggerID)) lts.baseTrigger.UnregisterTrigger(triggerID) - // The reservation ends with the trigger state, which is the only holder of - // the reserved address count and the physical filter identity. Both are - // reused from the stashed filter so this RELEASE carries the same value and - // identity as its RESERVE. If UnregisterLogTracking fails below, the filter - // is orphaned at the log poller and the stale-filter cleanup unregisters it - // later (silently — metering already emitted this RELEASE here). - lts.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, trigger.filter, trigger.reservedAddressCount) + if lastForPhysical { + // 1->0 release of the shared physical filter: bill -addressCount once. + // The value and identity are reused from the stashed filter so this + // -delta reverses the exact +delta the activation billed. + lts.emitDelta(ctx, -trigger.reservedAddressCount, trigger.filter) + } err := lts.EVMService.UnregisterLogTracking(ctx, lts.generateFilterID(triggerID)) if err != nil { diff --git a/chain_capabilities/evm/trigger/trigger_metering_test.go b/chain_capabilities/evm/trigger/trigger_metering_test.go index ab80eb277..d4cffc54f 100644 --- a/chain_capabilities/evm/trigger/trigger_metering_test.go +++ b/chain_capabilities/evm/trigger/trigger_metering_test.go @@ -50,10 +50,11 @@ func testBaseIdentity() resourcemanager.ResourceIdentity { // per active resource (no single Snapshot envelope), so snapshots accumulates // one message per resource. type fakeMeterEmitter struct { - err error - emitCalls int - records []*meteringpb.MeterRecord - snapshots []*meteringpb.MeterSnapshot + err error + emitCalls int + records []*meteringpb.MeterRecord + recordDomains []string + snapshots []*meteringpb.MeterSnapshot } func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) error { @@ -74,9 +75,23 @@ func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) return err } f.records = append(f.records, &record) + f.recordDomains = append(f.recordDomains, attrString(attrKVs, beholder.AttrKeyDomain)) return nil } +// attrString returns the string value for key in the alternating key/value +// attrs the ResourceManager passes to Emit, or "" if absent. +func attrString(attrKVs []any, key string) string { + for i := 0; i+1 < len(attrKVs); i += 2 { + if attrKVs[i] == key { + if v, ok := attrKVs[i+1].(string); ok { + return v + } + } + } + return "" +} + // isSnapshotEmit reports whether the emitter attributes name the MeterSnapshot // entity, so the fake can demux the two message types off the same Emit method. // The key is beholder.AttrKeyEntity ("beholder_entity") and the value is the @@ -158,7 +173,7 @@ func expectedPhysicalFilterID(t *testing.T, input *evmcappb.FilterLogTriggerRequ return physicalFilterID(testChainSelector, addrs, sigs, h2, h3, h4) } -func TestLogTriggerMetering_ReserveOnRegister(t *testing.T) { +func TestLogTriggerMetering_RegisterEmitsPositiveDelta(t *testing.T) { evmService := initMocks(t) evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() @@ -173,10 +188,18 @@ func TestLogTriggerMetering_ReserveOnRegister(t *testing.T) { assertBaseIdentity(t, record.GetIdentity()) physID := expectedPhysicalFilterID(t, meteringTestInput()) require.Equal(t, physID, record.GetUtilizations()[0].GetResourceId(), "resource_id must be the physical filter content hash") - require.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, record.GetAction()) + // Producers emit only signed-delta UPDATE records; a fresh registration is + // the physical filter's 0->1 activation and bills +addressCount. + require.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, record.GetAction()) require.Len(t, record.GetUtilizations(), 1) - require.Equal(t, "2", record.GetUtilizations()[0].GetValue(), "RESERVE value must equal the filter address count") + require.Equal(t, "2", record.GetUtilizations()[0].GetValue(), "activation delta must equal the filter address count") require.Equal(t, MeteringResourceType, record.GetUtilizations()[0].GetResourceType()) + require.NotEmpty(t, record.GetUtilizations()[0].GetEventId(), "event_id is stamped per emission") + // The record carries the cll-meter billing domain. + require.Equal(t, "cll-meter", emitter.recordDomains[0]) + // The metering identity DON and the events.KeyDonID label derive from the + // same resolveDONID, so they cannot diverge. + require.Equal(t, service.resolveDONID(meta.WorkflowDonID), record.GetIdentity().GetDon().GetDonId()) } func TestLogTriggerMetering_DonIDFallbackToWorkflowDon(t *testing.T) { @@ -225,14 +248,14 @@ func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { assertRelease := func(t *testing.T, service *LogTriggerService, record *meteringpb.MeterRecord) { t.Helper() assertBaseIdentity(t, record.GetIdentity()) - require.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, record.GetAction()) + require.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, record.GetAction()) require.Len(t, record.GetUtilizations(), 1) - require.Equal(t, "2", record.GetUtilizations()[0].GetValue(), "RELEASE must carry the same value that was reserved") + require.Equal(t, "-2", record.GetUtilizations()[0].GetValue(), "the 1->0 release delta negates the activation value") physID := expectedPhysicalFilterID(t, meteringTestInput()) require.Equal(t, physID, record.GetUtilizations()[0].GetResourceId()) } - t.Run("release pairs the reserve", func(t *testing.T) { + t.Run("release negates the activation", func(t *testing.T) { evmService := initMocks(t) evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() @@ -243,11 +266,13 @@ func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { require.NoError(t, service.UnregisterLogTrigger(t.Context(), triggerID, meta, &evmcappb.FilterLogTriggerRequest{})) require.Len(t, emitter.records, 2) - require.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, emitter.records[0].GetAction()) + require.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, emitter.records[0].GetAction()) + require.Equal(t, "2", emitter.records[0].GetUtilizations()[0].GetValue()) assertRelease(t, service, emitter.records[1]) - require.Equal(t, emitter.records[0].GetUtilizations()[0].GetValue(), emitter.records[1].GetUtilizations()[0].GetValue()) require.Equal(t, emitter.records[0].GetUtilizations()[0].GetResourceId(), emitter.records[1].GetUtilizations()[0].GetResourceId(), - "RESERVE and RELEASE must share one physical resource_id") + "activation and release must share one physical resource_id") + require.NotEqual(t, emitter.records[0].GetUtilizations()[0].GetEventId(), emitter.records[1].GetUtilizations()[0].GetEventId(), + "each emission gets a distinct event_id") }) t.Run("release emitted even when UnregisterLogTracking fails", func(t *testing.T) { @@ -258,7 +283,7 @@ func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) registerTrigger(t, service) - // The reservation is released here (from the stashed count) before the + // The -delta is emitted here (from the stashed count) before the // UnregisterLogTracking RPC. If the RPC fails the filter is orphaned at // the log poller; the cleanup thread unregisters it silently, emitting // no further metering record. @@ -356,10 +381,11 @@ func TestPhysicalFilterID_Canonicalization(t *testing.T) { require.NotEqual(t, id1, id2) }) - t.Run("identical filters from different workflows/triggers share one id", func(t *testing.T) { + t.Run("identical filters from different workflows/triggers share one billed resource", func(t *testing.T) { // physicalFilterID takes only physical criteria; workflow/trigger are not - // inputs. Two registrations with identical criteria therefore collide by - // construction, which this asserts end to end through the register path. + // inputs. Two registrations with identical criteria collide by + // construction, so only the first (the 0->1 activation) bills a delta; + // the second shares the already-active physical filter and emits nothing. evmService := initMocks(t) evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Twice() evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Twice() @@ -372,14 +398,53 @@ func TestPhysicalFilterID_Canonicalization(t *testing.T) { capabilities.RequestMetadata{WorkflowID: "wf-2", WorkflowOwner: "0xOther"}, meteringTestInput()) require.NoError(t, err) - require.Len(t, emitter.records, 2) - require.Equal(t, - emitter.records[0].GetUtilizations()[0].GetResourceId(), - emitter.records[1].GetUtilizations()[0].GetResourceId(), - "identical physical filters must share one resource_id across workflows/triggers") + require.Len(t, emitter.records, 1, "the shared physical filter is billed once (only the 0->1 activation)") + require.Equal(t, expectedPhysicalFilterID(t, meteringTestInput()), emitter.records[0].GetUtilizations()[0].GetResourceId()) }) } +// TestLogTriggerMetering_SharedFilterRefcount asserts the derived 0<->1 refcount +// billing for a physical filter shared by two triggers: the first register bills +// +addressCount (0->1), the second register bills nothing (1->2), the first +// unregister bills nothing (2->1), and the last unregister bills -addressCount +// (1->0). +func TestLogTriggerMetering_SharedFilterRefcount(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Twice() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Twice() + evmService.On("UnregisterLogTracking", mock.Anything, mock.Anything).Return(nil) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + physID := expectedPhysicalFilterID(t, meteringTestInput()) + + // trigger-A: 0->1 activation bills +2. + _, err := service.RegisterLogTrigger(t.Context(), "trigger-A", + capabilities.RequestMetadata{WorkflowID: "wf-1", WorkflowOwner: "0xOwner"}, meteringTestInput()) + require.NoError(t, err) + require.Len(t, emitter.records, 1) + require.Equal(t, "2", emitter.records[0].GetUtilizations()[0].GetValue()) + + // trigger-B shares the same physical filter: 1->2, bills nothing. + _, err = service.RegisterLogTrigger(t.Context(), "trigger-B", + capabilities.RequestMetadata{WorkflowID: "wf-2", WorkflowOwner: "0xOther"}, meteringTestInput()) + require.NoError(t, err) + require.Len(t, emitter.records, 1, "a second holder of the same physical filter bills nothing") + + // Unregister trigger-A: 2->1, still held by trigger-B, bills nothing. + require.NoError(t, service.UnregisterLogTrigger(t.Context(), "trigger-A", capabilities.RequestMetadata{}, &evmcappb.FilterLogTriggerRequest{})) + require.Len(t, emitter.records, 1, "releasing one of two holders bills nothing") + + // Unregister trigger-B: 1->0, bills -2. + require.NoError(t, service.UnregisterLogTrigger(t.Context(), "trigger-B", capabilities.RequestMetadata{}, &evmcappb.FilterLogTriggerRequest{})) + require.Len(t, emitter.records, 2) + require.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, emitter.records[1].GetAction()) + require.Equal(t, "-2", emitter.records[1].GetUtilizations()[0].GetValue()) + require.Equal(t, physID, emitter.records[1].GetUtilizations()[0].GetResourceId()) + + // All emitted event_ids are distinct. + require.NotEqual(t, emitter.records[0].GetUtilizations()[0].GetEventId(), emitter.records[1].GetUtilizations()[0].GetEventId()) +} + // TestLogTriggerMetering_Snapshot drives one snapshot tick and asserts one // MeterSnapshot per active filter, each fully identified by its // ResourceIdentity (physical resource_id) with the right value. The manager @@ -447,40 +512,55 @@ func TestLogTriggerMetering_Snapshot_NothingActive(t *testing.T) { require.Empty(t, emitter.snapshots, "an empty store emits no MeterSnapshot") } -// TestLogTriggerMetering_ReleaseOnGracefulClose asserts that closing the service -// releases every still-active filter so a clean shutdown is not seen as a leak. -func TestLogTriggerMetering_ReleaseOnGracefulClose(t *testing.T) { +// TestLogTriggerMetering_NoShutdownEmissions asserts that a graceful Close emits +// NO meter records. Process-lifecycle emissions are deleted by design: an active +// filter is released by its absence from the next snapshot, not by a close-time +// drain. +func TestLogTriggerMetering_NoShutdownEmissions(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + evmService.EXPECT().GetFiltersNames(mock.Anything).Return([]string{}, nil).Maybe() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + require.NoError(t, service.Start(t.Context())) + + _, err := service.RegisterLogTrigger(t.Context(), triggerID, + capabilities.RequestMetadata{WorkflowID: "wf", WorkflowOwner: "0xOwner"}, meteringTestInput()) + require.NoError(t, err) + require.Len(t, emitter.records, 1, "registration bills a +delta") + + recordsBefore := len(emitter.records) + require.NoError(t, service.Close()) + require.Len(t, emitter.records, recordsBefore, "graceful close must emit no meter records") +} + +// TestLogTriggerMetering_SnapshotDedup asserts GetUtilization emits one entry +// per DISTINCT physical filter (not per trigger registration): two triggers +// sharing one physicalFilterID snapshot as a single resource. +func TestLogTriggerMetering_SnapshotDedup(t *testing.T) { mockEVM := evmmock.NewEVMService(t) store := NewLogTriggerStore() - service, emitter, _ := newMeteredTriggerObject(t, mockEVM, store) + service, emitter, clock := newMeteredTriggerObject(t, mockEVM, store) - physA := expectedPhysicalFilterID(t, meteringTestInput()) + physShared := expectedPhysicalFilterID(t, meteringTestInput()) + // Two triggers share one physical filter. store.Write("trigger-A", logTriggerState{filter: filter{ - filterID: service.generateFilterID("trigger-A"), - physicalFilterID: physA, - reservedAddressCount: 2, - donID: "42", + filterID: service.generateFilterID("trigger-A"), physicalFilterID: physShared, reservedAddressCount: 2, donID: "42", }}) store.Write("trigger-B", logTriggerState{filter: filter{ - filterID: service.generateFilterID("trigger-B"), - physicalFilterID: "physB", - reservedAddressCount: 5, - donID: "42", + filterID: service.generateFilterID("trigger-B"), physicalFilterID: physShared, reservedAddressCount: 2, donID: "42", }}) - service.releaseActiveFiltersOnClose(t.Context()) + unregister := service.resourceManager.Register(service) + t.Cleanup(unregister) + servicetest.Run(t, service.resourceManager) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) - require.Len(t, emitter.records, 2, "one RELEASE per active filter on graceful close") - for _, record := range emitter.records { - require.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, record.GetAction()) - assertBaseIdentity(t, record.GetIdentity()) - } - // The records carry no label metadata, so pair them by their physical - // resource_id (the only per-filter discriminator on the record). - byResourceID := map[string]*meteringpb.MeterRecord{} - for _, record := range emitter.records { - byResourceID[record.GetUtilizations()[0].GetResourceId()] = record - } - require.Equal(t, "2", byResourceID[physA].GetUtilizations()[0].GetValue()) - require.Equal(t, "5", byResourceID["physB"].GetUtilizations()[0].GetValue()) + require.Eventually(t, func() bool { + return len(emitter.snapshots) == 1 + }, time.Second, time.Millisecond) + require.Len(t, emitter.snapshots, 1, "two triggers sharing one physical filter snapshot once") + require.Equal(t, physShared, emitter.snapshots[0].GetUtilization()[0].GetResourceId()) + require.Equal(t, "2", emitter.snapshots[0].GetUtilization()[0].GetValue()) } diff --git a/cron/go.mod b/cron/go.mod index e0b034ccb..0e28d3521 100644 --- a/cron/go.mod +++ b/cron/go.mod @@ -92,7 +92,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.100 // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect diff --git a/cron/go.sum b/cron/go.sum index 05cab8559..6b0f0b36a 100644 --- a/cron/go.sum +++ b/cron/go.sum @@ -221,6 +221,8 @@ github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-202510021 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e h1:Hv9Mww35LrufCdM9wtS9yVi/rEWGI1UnjHbcKKU0nVY= diff --git a/cron/trigger/metering_test.go b/cron/trigger/metering_test.go index 9f3ecc4b0..619776fd5 100644 --- a/cron/trigger/metering_test.go +++ b/cron/trigger/metering_test.go @@ -28,10 +28,11 @@ import ( // entity attribute. A non-nil err simulates delivery failure: nothing is // recorded. type fakeMeterEmitter struct { - mu sync.Mutex - err error - records []*meteringpb.MeterRecord - snapshots []*meteringpb.MeterSnapshot + mu sync.Mutex + err error + records []*meteringpb.MeterRecord + recordDomains []string + snapshots []*meteringpb.MeterSnapshot } func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) error { @@ -40,7 +41,7 @@ func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) if f.err != nil { return f.err } - if attrEntity(attrKVs) == "metering.v1.MeterSnapshot" { + if attrValue(attrKVs, "beholder_entity") == "metering.v1.MeterSnapshot" { snapshot := &meteringpb.MeterSnapshot{} if err := proto.Unmarshal(body, snapshot); err != nil { return err @@ -53,14 +54,15 @@ func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) return err } f.records = append(f.records, record) + f.recordDomains = append(f.recordDomains, attrValue(attrKVs, "beholder_domain")) return nil } -// attrEntity extracts the beholder entity attribute value from the variadic +// attrValue extracts a beholder attribute value by key from the variadic // key/value attrs the ResourceManager passes to Emit. -func attrEntity(attrKVs []any) string { +func attrValue(attrKVs []any, key string) string { for i := 0; i+1 < len(attrKVs); i += 2 { - if k, ok := attrKVs[i].(string); ok && k == "beholder_entity" { + if k, ok := attrKVs[i].(string); ok && k == key { if v, ok := attrKVs[i+1].(string); ok { return v } @@ -146,7 +148,7 @@ func newMeteredTriggerService(t *testing.T, clock clockwork.Clock, emitter resou return ts, meters, fakeClock } -func TestCronTrigger_Metering_ReserveAndRelease(t *testing.T) { +func TestCronTrigger_Metering_RegisterUnregisterDeltas(t *testing.T) { t.Parallel() fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) @@ -161,13 +163,16 @@ func TestCronTrigger_Metering_ReserveAndRelease(t *testing.T) { require.Nil(t, capErr) records := emitter.Records() - require.Len(t, records, 1, "expected exactly one RESERVE on successful registration") - reserve := records[0] - assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, reserve.GetAction()) + require.Len(t, records, 1, "expected exactly one +1 UPDATE on successful registration") + register := records[0] + // Producers emit only signed-delta UPDATE records; register is a +1 delta. + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, register.GetAction()) + // The record carries the cll-meter billing domain. + assert.Equal(t, "cll-meter", emitter.recordDomains[0]) // Identity is populated from host-injected deps and points at the cron // resource pool. Per-trigger fields are carried on utilization. - id := reserve.GetIdentity() + id := register.GetIdentity() require.NotNil(t, id) assert.Equal(t, "cre-mainline", id.GetProduct()) assert.Equal(t, "mainline", id.GetTenant()) @@ -178,11 +183,15 @@ func TestCronTrigger_Metering_ReserveAndRelease(t *testing.T) { assert.Equal(t, "csa-pubkey-1", id.GetDon().GetNodeId()) assert.Equal(t, "cron-trigger", id.GetService()) assert.Equal(t, "trigger_registrations", id.GetResourcePool()) + // The metering identity DON and the events.KeyDonID label derive from the + // same resolver, so they cannot diverge. + assert.Equal(t, ts.donID(metadata.WorkflowDonID), id.GetDon().GetDonId()) - require.Len(t, reserve.GetUtilizations(), 1) - assert.Equal(t, "1", reserve.GetUtilizations()[0].GetValue()) - assert.Equal(t, "operations", reserve.GetUtilizations()[0].GetResourceType()) - assert.Equal(t, triggerID1, reserve.GetUtilizations()[0].GetResourceId()) + require.Len(t, register.GetUtilizations(), 1) + assert.Equal(t, "1", register.GetUtilizations()[0].GetValue()) + assert.Equal(t, "operations", register.GetUtilizations()[0].GetResourceType()) + assert.Equal(t, triggerID1, register.GetUtilizations()[0].GetResourceId()) + require.NotEmpty(t, register.GetUtilizations()[0].GetEventId(), "event_id is stamped per emission") // Each cron tick re-Writes the trigger to reschedule it; the Write // happens before the channel send, so after receiving the event the @@ -195,12 +204,17 @@ func TestCronTrigger_Metering_ReserveAndRelease(t *testing.T) { require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) records = emitter.Records() - require.Len(t, records, 2, "expected exactly one RELEASE on unregistration") - release := records[1] - assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, release.GetAction()) - assert.Equal(t, reserve.GetUtilizations()[0].GetResourceId(), release.GetUtilizations()[0].GetResourceId()) - require.Len(t, release.GetUtilizations(), 1) - assert.Equal(t, "1", release.GetUtilizations()[0].GetValue()) + require.Len(t, records, 2, "expected exactly one -1 UPDATE on unregistration") + unregister := records[1] + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, unregister.GetAction()) + assert.Equal(t, register.GetUtilizations()[0].GetResourceId(), unregister.GetUtilizations()[0].GetResourceId()) + require.Len(t, unregister.GetUtilizations(), 1) + assert.Equal(t, "-1", unregister.GetUtilizations()[0].GetValue(), "unregister is a signed -1 delta") + + // event_id is unique per emission: register and unregister must differ. + require.NotEmpty(t, unregister.GetUtilizations()[0].GetEventId()) + assert.NotEqual(t, register.GetUtilizations()[0].GetEventId(), unregister.GetUtilizations()[0].GetEventId(), + "each emission gets a distinct event_id") require.NoError(t, ts.Close()) } @@ -219,19 +233,17 @@ func TestCronTrigger_Metering_NoEmitOnFailedPaths(t *testing.T) { require.NotNil(t, capErr) require.Empty(t, emitter.Records()) - // Unregistering a trigger that was never registered releases nothing. + // Unregistering a trigger that was never registered emits no delta. require.Nil(t, ts.UnregisterTrigger(t.Context(), "missing", metadata, &crontypedapi.Config{Schedule: everySecond})) require.Empty(t, emitter.Records()) - // Duplicate registration fails and must not double-RESERVE. + // Duplicate registration fails and must not emit a second +1 delta. _, capErr = ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) require.Nil(t, capErr) _, capErr = ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) require.NotNil(t, capErr) require.Len(t, emitter.Records(), 1) - // Unregister to avoid a graceful-close RELEASE from interfering with the - // single-RESERVE assertion intent. require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) require.NoError(t, ts.Close()) } @@ -308,10 +320,11 @@ func TestCronTrigger_Metering_Snapshot(t *testing.T) { require.NoError(t, ts.Close()) } -// TestCronTrigger_Metering_GracefulCloseReleases asserts that Close drains a -// RELEASE for every still-active registration, so a graceful shutdown does not -// leak reservations in billing. -func TestCronTrigger_Metering_GracefulCloseReleases(t *testing.T) { +// TestCronTrigger_Metering_NoShutdownEmissions asserts that a graceful Close +// emits NO meter records. Process-lifecycle emissions are deleted by design: +// billing releases each still-active registration by its absence from the next +// snapshot, not by a shutdown drain. +func TestCronTrigger_Metering_NoShutdownEmissions(t *testing.T) { t.Parallel() fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) @@ -327,23 +340,13 @@ func TestCronTrigger_Metering_GracefulCloseReleases(t *testing.T) { _, capErr = ts.RegisterTrigger(t.Context(), triggerID2, metadata2, &crontypedapi.Config{Schedule: everySecond}) require.Nil(t, capErr) - // Two RESERVEs so far. + // Two +1 registration deltas so far. require.Len(t, emitter.Records(), 2) require.NoError(t, ts.Close()) - // Close drained a RELEASE for each active trigger. - records := emitter.Records() - require.Len(t, records, 4, "two RESERVEs + one RELEASE per active trigger on graceful close") - - releases := map[string]*meteringpb.MeterRecord{} - for _, r := range records[2:] { - require.Equal(t, meteringpb.MeterAction_METER_ACTION_RELEASE, r.GetAction()) - releases[r.GetUtilizations()[0].GetResourceId()] = r - } - require.Contains(t, releases, triggerID1) - require.Contains(t, releases, triggerID2) - assert.Equal(t, "1", releases[triggerID1].GetUtilizations()[0].GetValue()) + // Close emitted nothing: no shutdown drain. + require.Len(t, emitter.Records(), 2, "graceful close must emit no meter records") } // TestCronTrigger_Metering_DonIDFallback asserts the DON ID falls back to the @@ -374,6 +377,8 @@ func TestCronTrigger_Metering_DonIDFallback(t *testing.T) { records := emitter.Records() require.Len(t, records, 1) assert.Equal(t, "42", records[0].GetIdentity().GetDon().GetDonId(), "DON ID falls back to WorkflowDonID") + // Metering identity DON and events.KeyDonID label share the same resolver. + assert.Equal(t, ts.donID(metadata.WorkflowDonID), records[0].GetIdentity().GetDon().GetDonId()) // Product falls back to the cron constant when the host injects none. assert.Equal(t, "cre", records[0].GetIdentity().GetProduct()) diff --git a/cron/trigger/store.go b/cron/trigger/store.go index 9484c2970..ee9ed1a90 100644 --- a/cron/trigger/store.go +++ b/cron/trigger/store.go @@ -14,6 +14,7 @@ type CronStore interface { Read(triggerID string) (value cronTrigger, ok bool) ReadAll() (values map[string]cronTrigger) Write(triggerID string, value cronTrigger) + WriteIfPresent(triggerID string, value cronTrigger) (written bool) Delete(triggerID string) } @@ -46,6 +47,23 @@ func (cs *cronStore) Write(triggerID string, value cronTrigger) { cs.triggers[triggerID] = value } +// WriteIfPresent updates triggerID only when it currently exists, performing +// the existence check and the write atomically under the store lock. It returns +// false without writing when the trigger has already been deleted (e.g. by a +// concurrent UnregisterTrigger). The cron task callback uses this so a tick that +// began before an unregister cannot re-insert ("resurrect") a trigger that was +// just removed: snapshot absence is the release signal, so a resurrected entry +// would keep the resource billed after the caller stopped it. +func (cs *cronStore) WriteIfPresent(triggerID string, value cronTrigger) (written bool) { + cs.mu.Lock() + defer cs.mu.Unlock() + if _, ok := cs.triggers[triggerID]; !ok { + return false + } + cs.triggers[triggerID] = value + return true +} + func (cs *cronStore) Delete(triggerID string) { cs.mu.Lock() defer cs.mu.Unlock() diff --git a/cron/trigger/trigger.go b/cron/trigger/trigger.go index 2ec43f6dc..36b3c2d13 100644 --- a/cron/trigger/trigger.go +++ b/cron/trigger/trigger.go @@ -52,11 +52,13 @@ const ( meteringResource = "trigger_registrations" // meteringResourceType is the billing unit for cron registrations. meteringResourceType = "operations" - // meteringProductFallback is used when the host has not provided a Product - // via loop.EnvConfig (a legacy node or a boot path not yet updated). - meteringProductFallback = "cre" ) +// orgResolverRefreshInterval bounds how stale a cached owner->org mapping may +// get in the snapshot path. It matches the snapshot cadence so a re-linked org +// is reflected within one snapshot interval. +const orgResolverRefreshInterval = resourcemanager.DefaultSnapshotInterval + type Config struct { FastestScheduleIntervalSeconds int `json:"fastestScheduleIntervalSeconds"` } @@ -71,7 +73,12 @@ type cronTrigger struct { nextRun time.Time workflowID string workflowDonID uint32 - orgID string + // workflowOwner is the durable attribution key. We store the workflow OWNER, + // never a resolved org ID: the org is resolved fresh at each record emission + // (via orgresolver.ResolveOrEmpty) and at snapshot time (via the shared + // CachingOrgResolver), so a re-linked owner is picked up without rewriting + // durable state. + workflowOwner string close func() } @@ -102,6 +109,11 @@ type Service struct { // empty. Deployment resourcemanager.DeploymentIdentity orgResolver orgresolver.OrgResolver + // orgResolverSvc is the CachingOrgResolver's service handle when this + // service owns its lifecycle (set in Initialise when an OrgResolver was + // injected). Its Start/Close are wired into start/close. Nil when no + // resolver was injected. + orgResolverSvc services.Service } func (s *Service) RegisterLegacyTrigger(ctx context.Context, triggerID string, metadata capabilities.RequestMetadata, input *crontypedapi.Config) (<-chan capabilities.TriggerAndId[*crontypedapi.LegacyPayload], caperrors.Error) { //nolint:staticcheck @@ -203,50 +215,38 @@ func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFa return s, nil } -// identityFor returns the per-trigger metering identity. resource_id is -// workflow-scoped (the trigger_id) for cron and is carried on Utilization. -// The DON ID -// falls back to the consumer workflow's DON when the host has not injected a -// capability DON ID (deps.CapabilityDonID == 0). -func (s *Service) identityFor(workflowDonID uint32) resourcemanager.ResourceIdentity { - id := s.base - if id.DonID() == "" && workflowDonID != 0 { - id.Don = &resourcemanager.DonIdentity{ - DonID: strconv.FormatUint(uint64(workflowDonID), 10), - NodeID: s.Deployment.NodeID, - } - } - return id +// donID resolves the single DON identifier stamped on BOTH the metering +// identity and the events.KeyDonID label, so the two can never diverge: the +// host-injected CapabilityDonID (carried on base) wins, otherwise the consumer +// workflow's DON is the documented fallback. +func (s *Service) donID(workflowDonID uint32) string { + return s.base.WithWorkflowDonFallback(workflowDonID).DonID() } -func (s *Service) resolveOrgID(ctx context.Context, workflowOwner string) string { - if s.orgResolver == nil || workflowOwner == "" { - return "" - } - orgID, err := s.orgResolver.Get(ctx, workflowOwner) - if err != nil { - s.lggr.Warnw("failed to fetch organization ID from org resolver", "workflowOwner", workflowOwner, "error", err) - return "" +// utilizationFields builds the per-trigger billing fields shared by the record +// and snapshot paths. resource_id is the workflow-scoped trigger_id; org_id is +// resolved fresh from the workflow owner by the caller (never stored). event_id +// is intentionally absent: the ResourceManager stamps a unique UUID per +// emission. +func (s *Service) utilizationFields(triggerID, orgID string) resourcemanager.UtilizationFields { + return resourcemanager.UtilizationFields{ + ResourceType: meteringResourceType, + ResourceID: triggerID, + OrgID: orgID, } - return orgID } -// emitMeterRecord reports a change to this trigger's registration reservation -// for billing. The triggerID doubles as the idempotency event identity: a -// triggerID is registered at most once at a time, so retried emissions for the -// same registration dedup downstream. Emission is fail-open and never affects -// the registration itself. -func (s *Service) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, metadata capabilities.RequestMetadata, triggerID string, orgID string) { - id := s.identityFor(metadata.WorkflowDonID) - s.meters.EmitMeterRecord(ctx, id, action, - []*meteringpb.Utilization{ - resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ - ResourceType: meteringResourceType, - ResourceID: triggerID, - EventID: triggerID, - OrgID: orgID, - }), - }) +// emitMeterRecord emits a signed delta MeterRecord (METER_ACTION_UPDATE) for a +// change to the durable cron-registration level: register bills +1, unregister +// bills -1. don_id is derived from the STORED workflow DON via +// WithWorkflowDonFallback so a register and its later unregister resolve the +// same identity regardless of the unregister request's metadata. The org is +// resolved fresh from owner at emit time. Emission is fail-open and never +// affects the registration itself. +func (s *Service) emitMeterRecord(ctx context.Context, delta int64, workflowDonID uint32, triggerID, owner string) { + id := s.base.WithWorkflowDonFallback(workflowDonID) + orgID := orgresolver.ResolveOrEmpty(ctx, s.orgResolver, owner, s.lggr) + s.meters.EmitDelta(ctx, id, delta, s.utilizationFields(triggerID, orgID)) } func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapabilitiesDependencies) error { @@ -270,41 +270,23 @@ func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapa } s.fastestScheduleInterval = limiter - s.orgResolver = dependencies.OrgResolver - if s.orgResolver == nil { + if dependencies.OrgResolver == nil { s.lggr.Warn("OrgResolver is nil, cron capability will not be able to fetch organization ID") + } else { + // Wrap the injected resolver in a CachingOrgResolver so the snapshot + // path (GetUtilization, contractually no-network) resolves org from + // memory. We own this wrapper's lifecycle (see start/close). + caching := orgresolver.NewCaching(dependencies.OrgResolver, orgResolverRefreshInterval) + s.orgResolver = caching + s.orgResolverSvc = caching } // Build the base metering identity. The deployment/node dimensions come from // s.Deployment (delivered via loop.EnvConfig, set by main before - // Initialise); the DON dimension comes from the host-injected - // CapabilityDonID. Any may be empty/zero; the DON identifier falls back to the consumer - // workflow DON at emit time (see identityFor). - product := s.Deployment.Product - if product == "" { - product = meteringProductFallback - } - var donID string - if dependencies.CapabilityDonID != 0 { - donID = strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10) - } - var donIdentity *resourcemanager.DonIdentity - if donID != "" || s.Deployment.NodeID != "" { - donIdentity = &resourcemanager.DonIdentity{ - DonID: donID, - NodeID: s.Deployment.NodeID, - } - } - s.base = resourcemanager.ResourceIdentity{ - Product: product, - Tenant: s.Deployment.Tenant, - NumericTenantID: s.Deployment.NumericTenantID, - Environment: s.Deployment.Environment, - Zone: s.Deployment.Zone, - Don: donIdentity, - Service: meteringService, - ResourcePool: meteringResource, - } + // Initialise); the authoritative DON ID is the host-injected CapabilityDonID. + // When it is 0, NewBaseIdentity leaves don_id empty and each emission falls + // back to the consumer workflow's DON via WithWorkflowDonFallback. + s.base = resourcemanager.NewBaseIdentity(s.Deployment, dependencies.CapabilityDonID, meteringService, meteringResource) err = s.Start(ctx) if err != nil { @@ -412,7 +394,7 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat events.KeyWorkflowExecutionID, workflowExecutionID, events.KeyWorkflowOwner, metadata.WorkflowOwner, events.KeyWorkflowName, displayWorkflowName, - events.KeyDonID, strconv.Itoa(int(metadata.WorkflowDonID)), + events.KeyDonID, s.donID(metadata.WorkflowDonID), events.KeyDonVersion, strconv.Itoa(int(metadata.WorkflowDonConfigVersion)), events.KeyOrganizationID, orgID, events.KeyWorkflowRegistryChainSelector, metadata.WorkflowRegistryChainSelector, @@ -439,14 +421,21 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat if callbackCh == nil { return // unregistered already } - s.triggers.Write(triggerID, cronTrigger{ + // Re-check existence atomically with the write: an unregister that + // ran during this callback (after the Read above) deletes the + // trigger, and resurrecting it here would keep the resource billed + // via snapshots after the caller stopped it. WriteIfPresent skips + // the write when the trigger is already gone. + if written := s.triggers.WriteIfPresent(triggerID, cronTrigger{ job: job, nextRun: nextExecutionTime, workflowID: metadata.WorkflowID, workflowDonID: metadata.WorkflowDonID, - orgID: trigger.orgID, + workflowOwner: metadata.WorkflowOwner, close: closeCh, - }) + }); !written { + return // unregistered concurrently; do not resurrect or send + } select { case callbackCh <- response: @@ -484,17 +473,17 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat return nil, caperrors.NewPublicSystemError(fmt.Errorf("RegisterTrigger failed to remove job: %s", err), caperrors.Internal) } - orgID := s.resolveOrgID(ctx, metadata.WorkflowOwner) s.triggers.Write(triggerID, cronTrigger{ job: job, nextRun: firstRunTime, workflowID: metadata.WorkflowID, workflowDonID: metadata.WorkflowDonID, - orgID: orgID, + workflowOwner: metadata.WorkflowOwner, close: closeCh, }) - s.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, metadata, triggerID, orgID) + // Register bills a +1 delta to the durable trigger-registration level. + s.emitMeterRecord(ctx, 1, metadata.WorkflowDonID, triggerID, metadata.WorkflowOwner) s.lggr.Debugw("Trigger registered", "workflowId", metadata.WorkflowID, "triggerId", triggerID, "jobId", job.ID()) s.metrics.IncActiveTriggersGauge(ctx) @@ -547,7 +536,10 @@ func (s *Service) UnregisterTrigger(ctx context.Context, triggerID string, metad // Remove from triggers context s.triggers.Delete(triggerID) - s.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, metadata, triggerID, trigger.orgID) + // Unregister bills a -1 delta. don_id and owner come from the STORED trigger + // (not the unregister request's metadata) so the delta reverses the exact + // identity the register +1 billed. + s.emitMeterRecord(ctx, -1, trigger.workflowDonID, triggerID, trigger.workflowOwner) s.lggr.Debugw("UnregisterTrigger", "triggerId", triggerID, "jobId", jobID) s.metrics.DecActiveTriggersGauge(ctx) @@ -558,11 +550,20 @@ func (s *Service) UnregisterTrigger(ctx context.Context, triggerID string, metad // already been started by the engine, so start registers this Service as a // Meterable (the RM polls it once per snapshot tick) and starts the scheduler, // refreshing next-run times for any registrations that survived a restart. -func (s *Service) start(_ context.Context) error { +func (s *Service) start(ctx context.Context) error { if s.scheduler == nil { return errors.New("service has shutdown, it must be built again to restart") } + // Start the caching org resolver so its background refresh runs and the + // snapshot path is served from memory. Fail-open: a resolver start failure + // must not gate the trigger service. + if s.orgResolverSvc != nil { + if err := s.orgResolverSvc.Start(ctx); err != nil { + s.lggr.Errorw("failed to start caching org resolver; org attribution may be empty", "err", err) + } + } + // Register for snapshots. The RM owns the tick; we only supply state via // the Meterable interface. unregisterMeterable is called in close. s.unregisterMeterable = s.meters.Register(s) @@ -576,7 +577,7 @@ func (s *Service) start(_ context.Context) error { nextRun: nextExecutionTime, workflowID: trigger.workflowID, workflowDonID: trigger.workflowDonID, - orgID: trigger.orgID, + workflowOwner: trigger.workflowOwner, close: trigger.close, }) if err != nil { @@ -588,38 +589,31 @@ func (s *Service) start(_ context.Context) error { } // close is the services.Engine close hook. After this the Service cannot be -// started again; it must be re-built to schedule again. close drains a RELEASE -// for every still-active registration (so a graceful shutdown does not leak -// reservations in billing), unregisters from the snapshot registry, then shuts -// the scheduler down. The ResourceManager sub-service is closed by the engine -// afterwards. +// started again; it must be re-built to schedule again. There are NO +// process-lifecycle metering emissions: a graceful shutdown emits nothing, and +// billing releases each still-active registration by its absence from the next +// snapshot. close deregisters the Meterable from the ResourceManager FIRST (so +// no snapshot can run after the store is torn down), closes the caching org +// resolver, then shuts the scheduler down. The ResourceManager sub-service is +// closed by the engine afterwards. func (s *Service) close() error { if s.scheduler == nil { return errors.New("service has shutdown, it must be built again to restart") } - // Graceful-close RELEASEs. Use a background context: the engine's start - // context is already cancelled by the time close runs. Emission is - // fail-open, so a metering failure never blocks shutdown. - ctx := context.Background() - for triggerID, trigger := range s.triggers.ReadAll() { - id := s.identityFor(trigger.workflowDonID) - s.meters.EmitMeterRecord(ctx, id, meteringpb.MeterAction_METER_ACTION_RELEASE, - []*meteringpb.Utilization{ - resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ - ResourceType: meteringResourceType, - ResourceID: triggerID, - EventID: triggerID, - OrgID: trigger.orgID, - }), - }) - } - + // Deregister from the snapshot registry before anything else so no snapshot + // tick can observe a half-torn-down service. if s.unregisterMeterable != nil { s.unregisterMeterable() s.unregisterMeterable = nil } + if s.orgResolverSvc != nil { + if err := s.orgResolverSvc.Close(); err != nil { + s.lggr.Errorw("failed to close caching org resolver", "err", err) + } + } + err := s.scheduler.Shutdown() if err != nil { return fmt.Errorf("scheduler shutdown encountered a problem: %s", err) @@ -655,16 +649,12 @@ func (s *Service) GetUtilization(ctx context.Context) []resourcemanager.Snapshot triggers := s.triggers.ReadAll() entries := make([]resourcemanager.SnapshotEntry, 0, len(triggers)) for triggerID, trigger := range triggers { - id := s.identityFor(trigger.workflowDonID) + id := s.base.WithWorkflowDonFallback(trigger.workflowDonID) + orgID := orgresolver.ResolveOrEmpty(ctx, s.orgResolver, trigger.workflowOwner, s.lggr) entries = append(entries, resourcemanager.SnapshotEntry{ Identity: id, Utilizations: []*meteringpb.Utilization{ - resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ - ResourceType: meteringResourceType, - ResourceID: triggerID, - EventID: triggerID, - OrgID: trigger.orgID, - }), + resourcemanager.NewUtilizationInt(1, s.utilizationFields(triggerID, orgID)), }, }) } diff --git a/http_trigger/go.mod b/http_trigger/go.mod index 4a10c1246..64812be24 100644 --- a/http_trigger/go.mod +++ b/http_trigger/go.mod @@ -87,7 +87,7 @@ require ( github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect diff --git a/http_trigger/go.sum b/http_trigger/go.sum index 193964b5b..e3e97ee06 100644 --- a/http_trigger/go.sum +++ b/http_trigger/go.sum @@ -221,6 +221,8 @@ github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-8 github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e h1:Hv9Mww35LrufCdM9wtS9yVi/rEWGI1UnjHbcKKU0nVY= diff --git a/http_trigger/main.go b/http_trigger/main.go index 7dcfc4417..e2bbe7392 100644 --- a/http_trigger/main.go +++ b/http_trigger/main.go @@ -9,24 +9,12 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" ) -func newMeteringConfig(env loop.EnvConfig) trigger.MeteringConfig { - return trigger.MeteringConfig{ - MeterRecordsEnabled: env.MeterRecordsEnabled, - MeterSnapshotsEnabled: env.MeterSnapshotsEnabled, - Deployment: resourcemanager.DeploymentIdentity{ - Product: env.MeterProduct, - Tenant: env.MeterTenant, - NumericTenantID: env.MeterNumericTenantID, - Environment: env.MeterEnvironment, - Zone: env.MeterZone, - NodeID: env.MeterNodeID, - }, - } -} - func main() { loopserver.ServeNew(trigger.ServiceName, func(s *loop.Server) loop.StandardCapabilities { - meteringCfg := newMeteringConfig(s.EnvConfig) + // ConfigFromEnv is the single, canonical loop-env -> metering mapping + // (enable flags, beholder emitter, snapshot interval, deployment + // identity); no per-main copy of that mapping. + meteringCfg := resourcemanager.ConfigFromEnv(&s.EnvConfig) svc := trigger.NewService(s.Logger, s.LimitsFactory, meteringCfg) return server.NewHTTPServer(svc) }) diff --git a/http_trigger/trigger/connector_handler.go b/http_trigger/trigger/connector_handler.go index c5de95010..2b1fa99de 100644 --- a/http_trigger/trigger/connector_handler.go +++ b/http_trigger/trigger/connector_handler.go @@ -59,6 +59,10 @@ type connectorHandler struct { // unregisterMeterable removes this handler from the ResourceManager's // snapshot registry; set on Start, called on Close. unregisterMeterable func() + // orgResolverSvc is the CachingOrgResolver's service handle when one was + // injected; its Start/Close are wired into the handler lifecycle. Nil when + // no resolver was injected. + orgResolverSvc services.Service } func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config ServiceConfig, @@ -67,7 +71,7 @@ func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config Se if resourceManager == nil { resourceManager = resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{}) } - return &connectorHandler{ + h := &connectorHandler{ lggr: logger.Named(lggr, HandlerName), gatewayConnector: gc, config: config, @@ -79,7 +83,13 @@ func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config Se orgResolver: orgResolver, resourceManager: resourceManager, baseIdentity: baseIdentity, - }, nil + } + // Own the caching resolver's lifecycle when one was injected, so its + // background refresh runs while the handler is up. + if caching, ok := orgResolver.(*orgresolver.CachingOrgResolver); ok { + h.orgResolverSvc = caching + } + return h, nil } func (h *connectorHandler) Start(ctx context.Context) error { @@ -87,13 +97,22 @@ func (h *connectorHandler) Start(ctx context.Context) error { h.wg.Add(1) go h.startRequestCacheCleanup(ctx) return h.StartOnce(HandlerName, func() error { + // Start the caching org resolver (if any) so its background refresh + // runs and the snapshot path is served from memory. Fail-open. + if h.orgResolverSvc != nil { + if err := h.orgResolverSvc.Start(ctx); err != nil { + h.lggr.Errorw("failed to start caching org resolver; org attribution may be empty", "err", err) + } + } // Start the ResourceManager as a sub-service (it owns the snapshot // tick) and register this handler as the snapshotted Meterable. The RM - // is fail-open and disabled by default, so this never gates startup. + // is fail-open: a start failure logs and continues (uniform with the + // other trigger producers) rather than gating the handler. if err := h.resourceManager.Start(ctx); err != nil { - return err + h.lggr.Errorw("failed to start metering ResourceManager; snapshots disabled", "err", err) + } else { + h.unregisterMeterable = h.resourceManager.Register(h) } - h.unregisterMeterable = h.resourceManager.Register(h) return h.gatewayConnector.AddHandler(ctx, []string{ gateway_common.MethodWorkflowExecute, gateway_common.MethodPullWorkflowMetadata, @@ -128,27 +147,24 @@ func (h *connectorHandler) Close() error { return h.StopOnce(HandlerName, func() error { close(h.stopChan) h.wg.Wait() - // Drain RELEASEs for every still-active workflow so reservations do not - // leak past shutdown, then unregister from the snapshot tick and stop - // the ResourceManager. Order matters: release before unregister/close so - // the final edges are emitted while emission is still wired up. - h.releaseActiveWorkflows(context.Background()) + // No process-lifecycle metering emissions: a graceful shutdown emits + // nothing, and billing releases each still-active workflow by its + // absence from the next snapshot. Deregister the Meterable from the + // ResourceManager FIRST so no snapshot tick can run after teardown, + // then close the caching org resolver and the ResourceManager. if h.unregisterMeterable != nil { h.unregisterMeterable() + h.unregisterMeterable = nil + } + if h.orgResolverSvc != nil { + if err := h.orgResolverSvc.Close(); err != nil { + h.lggr.Errorw("failed to close caching org resolver", "err", err) + } } return h.resourceManager.Close() }) } -// releaseActiveWorkflows emits a RELEASE for each workflow still active in the -// store at shutdown. It is fail-open: emission never blocks or fails close. -func (h *connectorHandler) releaseActiveWorkflows(ctx context.Context) { - for _, w := range h.workflowStore.getWorkflows() { - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, - w.workflowSelector.WorkflowID, w.metadata.WorkflowDONID, w.metadata.OrganizationID) - } -} - func (h *connectorHandler) HealthReport() map[string]error { return map[string]error{h.Name(): h.Healthy()} } @@ -187,76 +203,52 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR latencyMs := time.Since(startTime).Milliseconds() h.metrics.RecordBroadcastMetadataLatency(ctx, latencyMs, h.lggr) - orgID := h.resolveOrgID(ctx, input.WorkflowSelector.WorkflowOwner) - input.Metadata.OrganizationID = orgID - - var prevWorkflowOrgID string - var prevWorkflowDONID uint32 - if prevID, ok := h.workflowStore.getWorkflowIDByReference(input.WorkflowSelector.WorkflowOwner, input.WorkflowSelector.WorkflowName, input.WorkflowSelector.WorkflowTag); ok { - if prevWorkflow, found := h.workflowStore.getWorkflowByID(prevID); found { - prevWorkflowOrgID = prevWorkflow.metadata.OrganizationID - prevWorkflowDONID = prevWorkflow.metadata.WorkflowDONID - } - } - workflow := newWorkflowWithMetadata(input.WorkflowSelector, authorizedKeys, sendCh, input.Metadata) - prevWorkflowID, replaced, err := h.workflowStore.upsertWorkflow(workflow) + // upsertWorkflow returns the evicted workflow (if any) atomically under the + // store lock, so we never need a separate pre-read (which would be a TOCTOU + // race against a concurrent register/unregister). + evicted, err := h.workflowStore.upsertWorkflow(workflow) if err != nil { return fmt.Errorf("failed to register workflow (ID: %s, Owner: %s, Name: %s): %w", input.WorkflowSelector.WorkflowID, input.WorkflowSelector.WorkflowOwner, input.WorkflowSelector.WorkflowName, err) } newWorkflowID := input.WorkflowSelector.WorkflowID workflowDONID := input.Metadata.WorkflowDONID + owner := input.WorkflowSelector.WorkflowOwner switch { - case !replaced: - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, newWorkflowID, workflowDONID, orgID) - case prevWorkflowID == newWorkflowID: - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_UPDATE, newWorkflowID, workflowDONID, orgID) + case evicted == nil: + // Brand-new registration: bill +1 for the new durable resource. + h.emitMeterRecord(ctx, 1, newWorkflowID, workflowDONID, owner) + case evicted.workflowSelector.WorkflowID == newWorkflowID: + // Same-ID re-register: the durable resource is unchanged, so there is + // no level delta to bill. Emit nothing. default: // Version update: the same owner/name/tag reference now resolves to a - // new workflow ID. Release the previous workflow's reservation before - // reserving the new one so the old reservation does not leak. - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, prevWorkflowID, prevWorkflowDONID, prevWorkflowOrgID) - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, newWorkflowID, workflowDONID, orgID) + // new workflow ID. Bill -1 against the evicted workflow's resource_id + // and +1 for the new, both derived from the atomically returned + // eviction so the old reservation cannot leak. + h.emitMeterRecord(ctx, -1, evicted.workflowSelector.WorkflowID, evicted.metadata.WorkflowDONID, evicted.workflowSelector.WorkflowOwner) + h.emitMeterRecord(ctx, 1, newWorkflowID, workflowDONID, owner) } h.lggr.Debugw("Registered workflow", "workflowID", input.WorkflowSelector.WorkflowID, "workflowOwner", input.WorkflowSelector.WorkflowOwner, "workflowName", input.WorkflowSelector.WorkflowName, "workflowTag", input.WorkflowSelector.WorkflowTag) return nil } -// emitMeterRecord emits a meter record for one workflow registration -// operation. resource_id is the workflow ID (HTTP registrations are -// workflow-scoped, so there is no shared physical resource); the workflow ID -// also doubles as the event identity, so a repeated emission for the same -// workflow and action dedups downstream. The workflow_id is recoverable from -// resource_id and the owner is resolved downstream, so no label metadata is -// attached. Emission is fail-open and never affects the registration outcome. -func (h *connectorHandler) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, workflowID string, workflowDONID uint32, orgID string) { - identity := h.identityForWorkflow(workflowDONID) - h.resourceManager.EmitMeterRecord(ctx, identity, action, - []*meteringpb.Utilization{ - resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ - ResourceType: meterResourceType, - ResourceID: workflowID, - EventID: workflowID, - OrgID: orgID, - }), - }) -} - -// identityForWorkflow derives per-workflow metering identity: base identity -// with DON identifier resolved per -// registration when the host did not inject a capability DON. -func (h *connectorHandler) identityForWorkflow(workflowDONID uint32) resourcemanager.ResourceIdentity { - identity := h.baseIdentity - donID := h.donID(workflowDONID) - if donID == "" { - return identity - } - identity.Don = &resourcemanager.DonIdentity{ - DonID: donID, - NodeID: identity.NodeID(), - } - return identity +// emitMeterRecord emits a signed delta MeterRecord (METER_ACTION_UPDATE) for a +// change to the durable HTTP-workflow-registration level: register bills +1, +// unregister/version-eviction bills -1. resource_id is the workflow ID (HTTP +// registrations are workflow-scoped, so there is no shared physical resource). +// The org is resolved fresh from owner at emit time; event_id is stamped by the +// ResourceManager per emission. Emission is fail-open and never affects the +// registration outcome. +func (h *connectorHandler) emitMeterRecord(ctx context.Context, delta int64, workflowID string, workflowDONID uint32, owner string) { + identity := h.baseIdentity.WithWorkflowDonFallback(workflowDONID) + orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, owner, h.lggr) + h.resourceManager.EmitDelta(ctx, identity, delta, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + OrgID: orgID, + }) } // donID returns the DON identifier for an emission. It prefers the @@ -273,18 +265,6 @@ func (h *connectorHandler) donID(workflowDONID uint32) string { return "" } -func (h *connectorHandler) resolveOrgID(ctx context.Context, workflowOwner string) string { - if h.orgResolver == nil || workflowOwner == "" { - return "" - } - orgID, err := h.orgResolver.Get(ctx, workflowOwner) - if err != nil { - h.lggr.Warnw("failed to fetch organization ID from org resolver", "workflowOwner", workflowOwner, "error", err) - return "" - } - return orgID -} - // ResourceIdentity returns the HTTP trigger's base metering identity (six // dimensions + resource_pool). Per-workflow billing fields are populated on // Utilization in GetUtilization. It implements resourcemanager.Meterable. @@ -302,14 +282,17 @@ func (h *connectorHandler) GetUtilization(ctx context.Context) []resourcemanager entries := make([]resourcemanager.SnapshotEntry, 0, len(workflows)) for _, w := range workflows { workflowID := w.workflowSelector.WorkflowID + // Org is resolved from the stored owner via the shared caching resolver + // (a cache hit populated at registration), keeping GetUtilization + // no-network as the snapshot contract requires. + orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, w.workflowSelector.WorkflowOwner, h.lggr) entries = append(entries, resourcemanager.SnapshotEntry{ - Identity: h.identityForWorkflow(w.metadata.WorkflowDONID), + Identity: h.baseIdentity.WithWorkflowDonFallback(w.metadata.WorkflowDONID), Utilizations: []*meteringpb.Utilization{ resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ ResourceType: meterResourceType, ResourceID: workflowID, - EventID: workflowID, - OrgID: w.metadata.OrganizationID, + OrgID: orgID, }), }, }) @@ -344,19 +327,20 @@ func (h *connectorHandler) validateAuthorizedKeys(inputKeys []*http.AuthorizedKe } func (h *connectorHandler) UnregisterWorkflow(ctx context.Context, workflowID string) error { - // Snapshot the workflow DON before removal; it is needed for the meter - // record's identity DON fallback. + // Snapshot the workflow DON and owner before removal; the DON feeds the + // meter record's identity fallback and the owner resolves org at emit time. var workflowDONID uint32 - var orgID string + var owner string if w, ok := h.workflowStore.getWorkflowByID(workflowID); ok { workflowDONID = w.metadata.WorkflowDONID - orgID = w.metadata.OrganizationID + owner = w.workflowSelector.WorkflowOwner } err := h.workflowStore.removeWorkflow(workflowID) if err != nil { return fmt.Errorf("failed to unregister workflow %s: %w", workflowID, err) } - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, workflowID, workflowDONID, orgID) + // Unregister bills a -1 delta against the workflow's resource_id. + h.emitMeterRecord(ctx, -1, workflowID, workflowDONID, owner) h.lggr.Debugw("Unregistered workflow", "workflowID", workflowID) return nil } diff --git a/http_trigger/trigger/connector_handler_test.go b/http_trigger/trigger/connector_handler_test.go index 641bb3373..4d3003787 100644 --- a/http_trigger/trigger/connector_handler_test.go +++ b/http_trigger/trigger/connector_handler_test.go @@ -1134,13 +1134,14 @@ func TestHandleGatewayMessage_NilRequest(t *testing.T) { // MeterSnapshot bodies are decoded with the correct type. Each MeterSnapshot // covers exactly one resource. type fakeMeterEmitter struct { - err error - records []*meteringpb.MeterRecord - snapshots []*meteringpb.MeterSnapshot + err error + records []*meteringpb.MeterRecord + recordDomains []string + snapshots []*meteringpb.MeterSnapshot } func (f *fakeMeterEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { - if f.entity(attrKVs) == "metering.v1.MeterSnapshot" { + if f.attr(attrKVs, beholder.AttrKeyEntity) == "metering.v1.MeterSnapshot" { var snapshot meteringpb.MeterSnapshot if err := proto.Unmarshal(body, &snapshot); err != nil { return err @@ -1153,14 +1154,15 @@ func (f *fakeMeterEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any return err } f.records = append(f.records, &record) + f.recordDomains = append(f.recordDomains, f.attr(attrKVs, beholder.AttrKeyDomain)) return f.err } -// entity returns the value of the beholder entity attribute from the -// alternating key/value attrKVs slice, or "" if absent. -func (f *fakeMeterEmitter) entity(attrKVs []any) string { +// attr returns the value of a beholder attribute by key from the alternating +// key/value attrKVs slice, or "" if absent. +func (f *fakeMeterEmitter) attr(attrKVs []any, key string) string { for i := 0; i+1 < len(attrKVs); i += 2 { - if k, ok := attrKVs[i].(string); ok && k == beholder.AttrKeyEntity { + if k, ok := attrKVs[i].(string); ok && k == key { if v, ok := attrKVs[i+1].(string); ok { return v } @@ -1248,7 +1250,7 @@ func meterTestRegistrationInput() WorkflowRegistrationInput { } } -func TestRegisterWorkflow_MetersReserveThenUpdate(t *testing.T) { +func TestRegisterWorkflow_RegisterThenSameIDReRegister(t *testing.T) { lggr := logger.Test(t) handler, emitter := setupWithMeterEmitter(t, lggr, nil) input := meterTestRegistrationInput() @@ -1257,10 +1259,11 @@ func TestRegisterWorkflow_MetersReserveThenUpdate(t *testing.T) { err := handler.RegisterWorkflow(t.Context(), input, sendCh) require.NoError(t, err) - // First registration reserves exactly once, with the full structured - // identity populated on the record. - require.Equal(t, []meteringpb.MeterAction{meteringpb.MeterAction_METER_ACTION_RESERVE}, emitter.actions()) + // First registration bills a single +1 UPDATE delta with the full + // structured identity populated on the record. + require.Equal(t, []meteringpb.MeterAction{meteringpb.MeterAction_METER_ACTION_UPDATE}, emitter.actions()) record := emitter.records[0] + require.Equal(t, "cll-meter", emitter.recordDomains[0]) id := record.GetIdentity() require.Equal(t, testBaseIdentity.Product, id.GetProduct()) require.Equal(t, testBaseIdentity.Tenant, id.GetTenant()) @@ -1271,22 +1274,25 @@ func TestRegisterWorkflow_MetersReserveThenUpdate(t *testing.T) { require.Equal(t, testBaseIdentity.NodeID(), id.GetDon().GetNodeId()) require.Equal(t, meterService, id.GetService()) require.Equal(t, meterResource, id.GetResourcePool()) + // The metering identity DON and events.KeyDonID label derive from the same + // donID resolver, so they cannot diverge. + require.Equal(t, handler.donID(input.Metadata.WorkflowDONID), id.GetDon().GetDonId()) require.Equal(t, meterResourceType, record.GetUtilizations()[0].GetResourceType()) // resource_id is the workflow ID (HTTP registrations are workflow-scoped). require.Equal(t, testWorkflowID, record.GetUtilizations()[0].GetResourceId()) require.Equal(t, "1", record.GetUtilizations()[0].GetValue()) + require.NotEmpty(t, record.GetUtilizations()[0].GetEventId(), "event_id is stamped per emission") - // Re-registering the same workflow emits UPDATE, not a second RESERVE. + // Re-registering the SAME workflow ID is not a level change and emits + // nothing (no RESERVE/UPDATE): the durable resource is unchanged. sendCh2 := make(chan capabilities.TriggerAndId[*http.Payload], 1) err = handler.RegisterWorkflow(t.Context(), input, sendCh2) require.NoError(t, err) - require.Equal(t, []meteringpb.MeterAction{ - meteringpb.MeterAction_METER_ACTION_RESERVE, - meteringpb.MeterAction_METER_ACTION_UPDATE, - }, emitter.actions()) + require.Equal(t, []meteringpb.MeterAction{meteringpb.MeterAction_METER_ACTION_UPDATE}, emitter.actions(), + "same-ID re-register emits no additional delta") } -func TestRegisterWorkflow_VersionUpdate_MetersReleaseThenReserve(t *testing.T) { +func TestRegisterWorkflow_VersionUpdate_MetersNegativeThenPositiveDelta(t *testing.T) { lggr := logger.Test(t) handler, emitter := setupWithMeterEmitter(t, lggr, nil) @@ -1296,34 +1302,43 @@ func TestRegisterWorkflow_VersionUpdate_MetersReleaseThenReserve(t *testing.T) { require.NoError(t, handler.RegisterWorkflow(t.Context(), inputA, sendChA)) // Re-registering the same owner/name/tag reference with a NEW workflow ID - // is a version update: the previous workflow's reservation is released - // before the new one is reserved, so the old reservation cannot leak. + // is a version update: the previous workflow's resource is billed -1 before + // the new one is billed +1, so the old resource cannot leak. inputB := meterTestRegistrationInput() inputB.WorkflowSelector.WorkflowID = testWorkflowID2 sendChB := make(chan capabilities.TriggerAndId[*http.Payload], 1) require.NoError(t, handler.RegisterWorkflow(t.Context(), inputB, sendChB)) require.Equal(t, []meteringpb.MeterAction{ - meteringpb.MeterAction_METER_ACTION_RESERVE, - meteringpb.MeterAction_METER_ACTION_RELEASE, - meteringpb.MeterAction_METER_ACTION_RESERVE, + meteringpb.MeterAction_METER_ACTION_UPDATE, + meteringpb.MeterAction_METER_ACTION_UPDATE, + meteringpb.MeterAction_METER_ACTION_UPDATE, }, emitter.actions()) - // RESERVE(A) anchors the old workflow ID via utilization.resource_id. - reserveA := emitter.records[0] - require.Equal(t, testWorkflowID1, reserveA.GetUtilizations()[0].GetResourceId()) + // +1 for the old workflow ID (via utilization.resource_id). + registerA := emitter.records[0] + require.Equal(t, testWorkflowID1, registerA.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "1", registerA.GetUtilizations()[0].GetValue()) - // RELEASE targets the PREVIOUS workflow ID under the same owner; its - // utilization.resource_id is that previous workflow ID. + // -1 targets the PREVIOUS (evicted) workflow ID. release := emitter.records[1] require.Equal(t, testWorkflowID1, release.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "-1", release.GetUtilizations()[0].GetValue()) + + // +1 for the new workflow ID. + registerB := emitter.records[2] + require.Equal(t, testWorkflowID2, registerB.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "1", registerB.GetUtilizations()[0].GetValue()) - // The trailing RESERVE anchors the new workflow ID. - reserveB := emitter.records[2] - require.Equal(t, testWorkflowID2, reserveB.GetUtilizations()[0].GetResourceId()) + // event_ids are unique across all three emissions. + ids := map[string]struct{}{} + for _, r := range emitter.records { + ids[r.GetUtilizations()[0].GetEventId()] = struct{}{} + } + require.Len(t, ids, 3, "each emission gets a distinct event_id") } -func TestUnregisterWorkflow_MetersRelease(t *testing.T) { +func TestUnregisterWorkflow_MetersNegativeDelta(t *testing.T) { lggr := logger.Test(t) handler, emitter := setupWithMeterEmitter(t, lggr, nil) @@ -1334,13 +1349,15 @@ func TestUnregisterWorkflow_MetersRelease(t *testing.T) { err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) require.NoError(t, err) require.Equal(t, []meteringpb.MeterAction{ - meteringpb.MeterAction_METER_ACTION_RESERVE, - meteringpb.MeterAction_METER_ACTION_RELEASE, + meteringpb.MeterAction_METER_ACTION_UPDATE, + meteringpb.MeterAction_METER_ACTION_UPDATE, }, emitter.actions()) release := emitter.records[1] require.Equal(t, testWorkflowID, release.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "-1", release.GetUtilizations()[0].GetValue()) + require.NotEqual(t, emitter.records[0].GetUtilizations()[0].GetEventId(), release.GetUtilizations()[0].GetEventId()) - // Unregistering an absent workflow fails and must not emit RELEASE. + // Unregistering an absent workflow fails and must not emit a delta. err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) require.Error(t, err) require.Len(t, emitter.records, 2) @@ -1427,9 +1444,11 @@ func TestSnapshot_EmitsOneEntryPerActiveWorkflow(t *testing.T) { require.Equal(t, "1", r2.GetUtilization()[0].GetValue()) } -// TestClose_EmitsReleasePerActiveWorkflow asserts graceful close drains a -// RELEASE for every still-active workflow so reservations do not leak. -func TestClose_EmitsReleasePerActiveWorkflow(t *testing.T) { +// TestClose_EmitsNoShutdownRecords asserts a graceful close emits NO meter +// records. Process-lifecycle emissions are deleted by design: an active +// workflow is released by its absence from the next snapshot, not by a +// close-time drain. +func TestClose_EmitsNoShutdownRecords(t *testing.T) { lggr := logger.Test(t) emitter := &fakeMeterEmitter{} cfg := ServiceConfig{MetadataBatchSize: 10, MaxAuthorizedKeysPerWorkflow: 3} @@ -1449,21 +1468,10 @@ func TestClose_EmitsReleasePerActiveWorkflow(t *testing.T) { registerMeterWorkflow(t, handler, testWorkflowID1, testWorkflowOwner1) registerMeterWorkflow(t, handler, testWorkflowID2, testWorkflowOwner2) - // Drop the lifecycle RESERVE records; assert only on the close drain. + // Drop the register deltas; assert the close path emits nothing. emitter.records = nil require.NoError(t, handler.Close()) - - require.Equal(t, []meteringpb.MeterAction{ - meteringpb.MeterAction_METER_ACTION_RELEASE, - meteringpb.MeterAction_METER_ACTION_RELEASE, - }, emitter.actions()) - - released := map[string]bool{} - for _, r := range emitter.records { - released[r.GetUtilizations()[0].GetResourceId()] = true - } - require.True(t, released[testWorkflowID1]) - require.True(t, released[testWorkflowID2]) + require.Empty(t, emitter.records, "graceful close must emit no meter records") } // TestDONIDFallback_UsesWorkflowDON asserts that when the host did not inject a @@ -1490,6 +1498,8 @@ func TestDONIDFallback_UsesWorkflowDON(t *testing.T) { require.Len(t, emitter.records, 1) require.Equal(t, "99", emitter.records[0].GetIdentity().GetDon().GetDonId()) + // Metering identity DON and events.KeyDonID label share the same resolver. + require.Equal(t, handler.donID(input.Metadata.WorkflowDONID), emitter.records[0].GetIdentity().GetDon().GetDonId()) } // TestResolveWorkflowMetadata_PreservesStoredWorkflowOwner tests that the workflowOwner diff --git a/http_trigger/trigger/gateway_metadata_publisher_test.go b/http_trigger/trigger/gateway_metadata_publisher_test.go index 2b0e0a435..2c6a8c10c 100644 --- a/http_trigger/trigger/gateway_metadata_publisher_test.go +++ b/http_trigger/trigger/gateway_metadata_publisher_test.go @@ -221,9 +221,9 @@ func TestSendWorkflows_Success(t *testing.T) { wf1 := newWorkflow(selector1, authorizedKeys1, sendCh1) wf2 := newWorkflow(selector2, authorizedKeys2, sendCh2) - _, _, err := workflowStore.upsertWorkflow(wf1) + _, err := workflowStore.upsertWorkflow(wf1) require.NoError(t, err) - _, _, err = workflowStore.upsertWorkflow(wf2) + _, err = workflowStore.upsertWorkflow(wf2) require.NoError(t, err) gatewayID := "gateway1" diff --git a/http_trigger/trigger/trigger.go b/http_trigger/trigger/trigger.go index 28e4fbac4..f04050872 100644 --- a/http_trigger/trigger/trigger.go +++ b/http_trigger/trigger/trigger.go @@ -4,11 +4,9 @@ import ( "context" "encoding/json" "fmt" - "strconv" "strings" "time" - "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" caperrors "github.com/smartcontractkit/chainlink-common/pkg/capabilities/errors" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http" @@ -33,11 +31,13 @@ const ( meterService = "http-trigger" meterResource = "http_workflows" meterResourceType = "operations" - // meterProductFallback is used when the host did not inject a Product - // dimension (legacy node or a boot path not yet updated). - meterProductFallback = "cre" ) +// orgResolverRefreshInterval bounds owner->org cache staleness in the snapshot +// path. It matches the snapshot cadence so a re-linked org is reflected within +// one snapshot interval. +const orgResolverRefreshInterval = resourcemanager.DefaultSnapshotInterval + var _ server.HTTPCapability = &service{} type WorkflowRegistrationInput struct { @@ -52,7 +52,6 @@ type WorkflowRegistrationMetadata struct { EngineVersion string WorkflowDONID uint32 ReferenceID string - OrganizationID string // DecodedWorkflowName is the human-readable workflow name DecodedWorkflowName string } @@ -63,23 +62,6 @@ type ConnectorHandler interface { UnregisterWorkflow(ctx context.Context, workflowID string) error } -// MeteringConfig carries emission toggles and deployment/node identity -// dimensions for the HTTP trigger's ResourceManager. -type MeteringConfig struct { - MeterRecordsEnabled bool - MeterSnapshotsEnabled bool - Deployment resourcemanager.DeploymentIdentity -} - -func (m MeteringConfig) resourceManagerConfig() resourcemanager.ResourceManagerConfig { - return resourcemanager.ResourceManagerConfig{ - MeterRecordsEnabled: m.MeterRecordsEnabled, - MeterSnapshotsEnabled: m.MeterSnapshotsEnabled, - Emitter: beholder.GetEmitter(), - SnapshotInterval: resourcemanager.DefaultSnapshotInterval, - } -} - type service struct { services.StateMachine lggr logger.SugaredLogger @@ -88,12 +70,13 @@ type service struct { metrics *Metrics limitsFactory limits.Factory orgResolver orgresolver.OrgResolver - // metering carries static deployment/node identity dimensions plus metering - // emission toggles delivered via loop.EnvConfig. - metering MeteringConfig + // metering is the resolved metering Config (ResourceManagerConfig + + // DeploymentIdentity) produced from loop.EnvConfig by + // resourcemanager.ConfigFromEnv in main. + metering resourcemanager.Config } -func NewService(lggr logger.Logger, limitsFactory limits.Factory, metering MeteringConfig) *service { +func NewService(lggr logger.Logger, limitsFactory limits.Factory, metering resourcemanager.Config) *service { return &service{ lggr: logger.Sugared(logger.Named(lggr, ServiceName)), limitsFactory: limitsFactory, @@ -112,9 +95,13 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa } } s.cfg = applyDefaults(serviceConfig) - s.orgResolver = dependencies.OrgResolver - if s.orgResolver == nil { + if dependencies.OrgResolver == nil { s.lggr.Warn("OrgResolver is nil, HTTP trigger capability will not be able to fetch organization ID") + s.orgResolver = nil + } else { + // Wrap in a CachingOrgResolver so the snapshot path (no-network) is + // served from memory. The handler owns its Start/Close lifecycle. + s.orgResolver = orgresolver.NewCaching(dependencies.OrgResolver, orgResolverRefreshInterval) } workflowStore := newWorkflowStore(s.lggr) var err error @@ -124,8 +111,11 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa } metadataPublisher := NewGatewayMetadataPublisher(s.lggr, dependencies.GatewayConnector, workflowStore, s.cfg, s.metrics) requestCache := newRequestCache(s.lggr, dependencies.Store, time.Duration(s.cfg.RequestCacheTTL)*time.Second) - resourceManager := resourcemanager.NewResourceManager(s.lggr, s.metering.resourceManagerConfig()) - baseIdentity := baseMeterIdentity(dependencies, s.metering.Deployment) + resourceManager := resourcemanager.NewResourceManager(s.lggr, s.metering.ResourceManagerConfig) + // The authoritative DON ID is the host-injected CapabilityDonID; when 0, + // NewBaseIdentity leaves don_id empty and each emission falls back to the + // consumer workflow's DON via WithWorkflowDonFallback. + baseIdentity := resourcemanager.NewBaseIdentity(s.metering.DeploymentIdentity, dependencies.CapabilityDonID, meterService, meterResource) s.connectorHandler, err = NewConnectorHandler(s.lggr, dependencies.GatewayConnector, s.cfg, workflowStore, metadataPublisher, requestCache, s.metrics, s.orgResolver, resourceManager, baseIdentity) if err != nil { return err @@ -133,45 +123,6 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa return s.Start(ctx) } -// baseMeterIdentity builds the HTTP trigger's base metering identity. The -// deployment/node dimensions come from deployment (delivered via -// loop.EnvConfig); the DON dimension comes from the host-injected -// CapabilityDonID. The service-level resource_pool is fixed here; the -// per-workflow billing fields are set on each Utilization. -// -// The DON identifier is the capability DON the trigger LOOP was spawned for -// (deps.CapabilityDonID, host-injected via capabilities#619). When the host has -// not populated it (0), the DON identifier is left empty here and resolved per registration -// from the workflow DON at emit time (see connectorHandler.donID). Product -// falls back to a constant when the host did not inject one. -func baseMeterIdentity(deps core.StandardCapabilitiesDependencies, deployment resourcemanager.DeploymentIdentity) resourcemanager.ResourceIdentity { - product := deployment.Product - if product == "" { - product = meterProductFallback - } - var donID string - if deps.CapabilityDonID != 0 { - donID = strconv.FormatUint(uint64(deps.CapabilityDonID), 10) - } - var donIdentity *resourcemanager.DonIdentity - if donID != "" || deployment.NodeID != "" { - donIdentity = &resourcemanager.DonIdentity{ - DonID: donID, - NodeID: deployment.NodeID, - } - } - return resourcemanager.ResourceIdentity{ - Product: product, - Tenant: deployment.Tenant, - NumericTenantID: deployment.NumericTenantID, - Environment: deployment.Environment, - Zone: deployment.Zone, - Don: donIdentity, - Service: meterService, - ResourcePool: meterResource, - } -} - func (s *service) Start(ctx context.Context) error { s.lggr.Debug("Service starting...") return s.StartOnce(ServiceName, func() error { diff --git a/http_trigger/trigger/trigger_test.go b/http_trigger/trigger/trigger_test.go index 25180b82c..5ab96ac4f 100644 --- a/http_trigger/trigger/trigger_test.go +++ b/http_trigger/trigger/trigger_test.go @@ -13,6 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types/core" gcmocks "github.com/smartcontractkit/chainlink-common/pkg/types/core/mocks" @@ -56,7 +57,7 @@ func TestService_RegisterTrigger(t *testing.T) { mockHandler := &mockConnectorHandler{ registerErr: tc.registerErr, } - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, MeteringConfig{}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, resourcemanager.Config{}) cfgStr := fmt.Sprintf(`{"sendChannelBufferSize": %d}`, tc.sendChannelBufSize) gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ @@ -118,7 +119,7 @@ func TestService_UnregisterTrigger(t *testing.T) { mockHandler := &mockConnectorHandler{ unregisterErr: tt.handlerErr, } - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, MeteringConfig{}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, resourcemanager.Config{}) cfg := "{}" gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ @@ -141,7 +142,7 @@ func TestService_UnregisterTrigger(t *testing.T) { } func TestService_Initialise_EmptyConfig(t *testing.T) { - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, MeteringConfig{}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, resourcemanager.Config{}) gc := mockedGatewayConnector(t) err := svc.Initialise(context.Background(), core.StandardCapabilitiesDependencies{ @@ -157,7 +158,7 @@ func TestService_Initialise_EmptyConfig(t *testing.T) { func TestService_Start_HealthReport_Ready_Close(t *testing.T) { mockHandler := &mockConnectorHandler{} - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, MeteringConfig{}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, resourcemanager.Config{}) cfg := "{}" gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ diff --git a/http_trigger/trigger/workflow.go b/http_trigger/trigger/workflow.go index 8b7be4945..0e3b13d99 100644 --- a/http_trigger/trigger/workflow.go +++ b/http_trigger/trigger/workflow.go @@ -52,38 +52,38 @@ func newWorkflowStore(lggr logger.Logger) *workflowStore { // workflow reference (owner/name/tag combination) with new workflow instance. // upsertWorkflow should be invoked in the order of workflow registration, so that // the latest workflow instance is always used for the given reference. -// The returned replaced flag reports whether an existing registration was -// replaced (true) rather than a new one inserted (false); when replaced, -// prevWorkflowID is the workflow ID the reference pointed to before the -// upsert (it may equal the new workflow ID, or differ on a version update). -func (s *workflowStore) upsertWorkflow(w *workflow) (prevWorkflowID string, replaced bool, err error) { +// +// It returns the evicted workflow: the registration that previously held this +// owner/name/tag reference (nil when the reference was new). The eviction is +// determined atomically under the store lock so the caller can meter the +// replaced resource without a separate, racy pre-read. The evicted workflow's +// WorkflowID may equal the new one (a same-ID re-register, no level change) or +// differ (a version update whose old resource_id must be released). +func (s *workflowStore) upsertWorkflow(w *workflow) (evicted *workflow, err error) { // Validate workflow fields if err := validateWorkflowSelector(w.workflowSelector); err != nil { - return "", false, fmt.Errorf("invalid workflow selector: %w", err) + return nil, fmt.Errorf("invalid workflow selector: %w", err) } s.mu.Lock() defer s.mu.Unlock() - prevWorkflowID, replaced = s.workflowReferenceToID[workflowReference{ + ref := workflowReference{ workflowOwner: w.workflowSelector.WorkflowOwner, workflowName: w.workflowSelector.WorkflowName, workflowTag: w.workflowSelector.WorkflowTag, - }] - if replaced { + } + if prevWorkflowID, replaced := s.workflowReferenceToID[ref]; replaced { reference := fmt.Sprintf("%s/%s/%s", w.workflowSelector.WorkflowOwner, w.workflowSelector.WorkflowName, w.workflowSelector.WorkflowTag) s.lggr.Debugw("Updating existing workflow reference and removing previous workflow", "reference", reference, "prevWorkflowID", prevWorkflowID) if oldW, ok := s.workflows[prevWorkflowID]; ok { + evicted = oldW oldW.close() } delete(s.workflows, prevWorkflowID) } s.workflows[w.workflowSelector.WorkflowID] = w - s.workflowReferenceToID[workflowReference{ - workflowOwner: w.workflowSelector.WorkflowOwner, - workflowName: w.workflowSelector.WorkflowName, - workflowTag: w.workflowSelector.WorkflowTag, - }] = w.workflowSelector.WorkflowID - return prevWorkflowID, replaced, nil + s.workflowReferenceToID[ref] = w.workflowSelector.WorkflowID + return evicted, nil } // validateWorkflowSelector validates the workflow selector fields diff --git a/http_trigger/trigger/workflow_test.go b/http_trigger/trigger/workflow_test.go index 62edf5ef5..e45a16f75 100644 --- a/http_trigger/trigger/workflow_test.go +++ b/http_trigger/trigger/workflow_test.go @@ -134,10 +134,9 @@ func TestWorkflowStore_upsertWorkflow(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - prevWorkflowID, replaced, err := store.upsertWorkflow(wf) + evicted, err := store.upsertWorkflow(wf) require.NoError(t, err) - require.False(t, replaced) - require.Empty(t, prevWorkflowID) + require.Nil(t, evicted, "a new reference evicts nothing") w, exists := store.getWorkflowByID(wf.workflowSelector.WorkflowID) require.True(t, exists) @@ -259,7 +258,7 @@ func TestWorkflowStore_upsertWorkflow_ValidationErrors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { wf := newWorkflow(tt.selector, authorizedKeys, sendCh) - _, _, err := store.upsertWorkflow(wf) + _, err := store.upsertWorkflow(wf) require.Error(t, err) require.Contains(t, err.Error(), tt.wantErr) }) @@ -275,16 +274,15 @@ func TestWorkflowStore_upsertWorkflow_Duplicate(t *testing.T) { w2, _ := testWorkflow() // Add first workflow - prevWorkflowID, replaced, err := store.upsertWorkflow(w1) + evicted, err := store.upsertWorkflow(w1) require.NoError(t, err) - require.False(t, replaced) - require.Empty(t, prevWorkflowID) + require.Nil(t, evicted) // Add second workflow with same ID (this should replace the first) - prevWorkflowID, replaced, err = store.upsertWorkflow(w2) + evicted, err = store.upsertWorkflow(w2) require.NoError(t, err) - require.True(t, replaced) - require.Equal(t, w1.workflowSelector.WorkflowID, prevWorkflowID) + require.NotNil(t, evicted, "same-reference re-register evicts the previous workflow") + require.Equal(t, w1.workflowSelector.WorkflowID, evicted.workflowSelector.WorkflowID) // Verify the workflow was replaced - since both have same ID/reference, // the second one should be present @@ -300,7 +298,7 @@ func TestWorkflowStore_removeWorkflow_Success(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) w, _ := testWorkflow() - _, _, err := store.upsertWorkflow(w) + _, err := store.upsertWorkflow(w) require.NoError(t, err) wf, exists := store.getWorkflowByID(w.workflowSelector.WorkflowID) @@ -399,11 +397,11 @@ func TestWorkflowStore_GetWorkflows_Multiple(t *testing.T) { wf2 := newWorkflow(wfSelector2, authorizedKeys, sendCh2) wf3 := newWorkflow(wfSelector3, authorizedKeys, sendCh3) - _, _, err := store.upsertWorkflow(wf1) + _, err := store.upsertWorkflow(wf1) require.NoError(t, err) - _, _, err = store.upsertWorkflow(wf2) + _, err = store.upsertWorkflow(wf2) require.NoError(t, err) - _, _, err = store.upsertWorkflow(wf3) + _, err = store.upsertWorkflow(wf3) require.NoError(t, err) // Get all workflows @@ -427,7 +425,7 @@ func TestWorkflowStore_getWorkflowIDByReference_Success(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - _, _, err := store.upsertWorkflow(wf) + _, err := store.upsertWorkflow(wf) require.NoError(t, err) workflowID, exists := store.getWorkflowIDByReference( @@ -544,10 +542,9 @@ func TestWorkflowStore_upsertWorkflow_ReplaceWithSameReference(t *testing.T) { wf2 := newWorkflow(selector2, authorizedKeys, sendCh2) // Add first workflow - prevWorkflowID, replaced, err := store.upsertWorkflow(wf1) + evicted, err := store.upsertWorkflow(wf1) require.NoError(t, err) - require.False(t, replaced) - require.Empty(t, prevWorkflowID) + require.Nil(t, evicted) // Verify first workflow is there workflow, exists := store.getWorkflowByID(testWorkflowID1) @@ -559,12 +556,12 @@ func TestWorkflowStore_upsertWorkflow_ReplaceWithSameReference(t *testing.T) { require.True(t, exists) require.Equal(t, testWorkflowID1, workflowID) - // Add second workflow with same reference; the previous workflow ID is - // surfaced so callers can release its reservation. - prevWorkflowID, replaced, err = store.upsertWorkflow(wf2) + // Add second workflow with same reference; the evicted workflow is + // surfaced so callers can release its resource. + evicted, err = store.upsertWorkflow(wf2) require.NoError(t, err) - require.True(t, replaced) - require.Equal(t, testWorkflowID1, prevWorkflowID) + require.NotNil(t, evicted) + require.Equal(t, testWorkflowID1, evicted.workflowSelector.WorkflowID) // First workflow should be removed _, exists = store.getWorkflowByID(testWorkflowID1) @@ -587,7 +584,7 @@ func TestWorkflowStore_removeWorkflow_RemovesReference(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - _, _, err := store.upsertWorkflow(wf) + _, err := store.upsertWorkflow(wf) require.NoError(t, err) // Verify workflow and reference exist @@ -746,7 +743,7 @@ func TestWorkflowStore_getWorkflowIDByReference_PartialMatch(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - _, _, err := store.upsertWorkflow(wf) + _, err := store.upsertWorkflow(wf) require.NoError(t, err) // Test with wrong owner From 4aaa7bc92467e5e40ac7d562ac42c3f0dffb71f3 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Fri, 10 Jul 2026 14:10:10 -0400 Subject: [PATCH 5/6] feat(metering): supply deterministic cross-node event_id from trigger producers The common EmitDelta/EmitUsage now require a producer-supplied event_id. Each trigger producer derives it via resourcemanager.EventID from the DON-aggregated request that drove the delta, so every capability-DON node emits the identical value for the same logical (un)register. The remote trigger publisher invokes RegisterTrigger/UnregisterTrigger with the mode-aggregated request (byte-identical on every node), so the request fields are DON-consistent. - cron: EventID("cron-register"/"cron-unregister", workflowID, triggerID). - http: EventID("http-register"/"http-unregister", workflowID); version updates change the workflowID so their +1/-1 pairs are distinct; unregister hashes symmetrically with register. - evm: EventID("evm-activate"/"evm-release", workflowID, triggerID) from the RegisterLogTrigger/UnregisterLogTrigger request; physicalFilterID stays the resource_id (not the event_id). Also scrubs stale "UUID" wording from the event_id docs and aligns each module's metering pin to the merged chainlink-common pin (local replace intact). --- chain_capabilities/evm/go.mod | 2 +- chain_capabilities/evm/trigger/trigger.go | 21 ++++++++++----- cron/go.mod | 2 +- cron/trigger/metering_test.go | 31 ++++++++++++++++++----- cron/trigger/trigger.go | 27 +++++++++++++------- http_trigger/go.mod | 2 +- http_trigger/trigger/connector_handler.go | 29 +++++++++++++-------- 7 files changed, 78 insertions(+), 36 deletions(-) diff --git a/chain_capabilities/evm/go.mod b/chain_capabilities/evm/go.mod index e2e846c33..3b67611a1 100644 --- a/chain_capabilities/evm/go.mod +++ b/chain_capabilities/evm/go.mod @@ -21,7 +21,7 @@ require ( github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260410144512-ca02ad6ed16a github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.uber.org/zap v1.27.1 diff --git a/chain_capabilities/evm/trigger/trigger.go b/chain_capabilities/evm/trigger/trigger.go index 3a0144596..245c7f126 100644 --- a/chain_capabilities/evm/trigger/trigger.go +++ b/chain_capabilities/evm/trigger/trigger.go @@ -463,7 +463,7 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID }) if firstForPhysical { // 0->1 activation of a shared physical filter: bill +addressCount once. - lts.emitDelta(ctx, loggedFilter.reservedAddressCount, loggedFilter) + lts.emitDelta(ctx, loggedFilter.reservedAddressCount, "evm-activate", meta.WorkflowID, triggerID, loggedFilter) } monitoring.EmitInitiated(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerInitiated(telemetryContext, input)) @@ -559,16 +559,23 @@ func (lts *LogTriggerService) identity(donID string) resourcemanager.ResourceIde // physical log filter: +addressCount on the physical filter's 0->1 activation, // -addressCount on its 1->0 release. The physical filter content hash is the // ResourceID, so all triggers sharing it bill against one resource. The org is -// resolved fresh from the stored owner at emit time; event_id is stamped by the -// ResourceManager per emission. Emission is fail-open and must never gate the -// path that calls it. -func (lts *LogTriggerService) emitDelta(ctx context.Context, delta int64, f filter) { +// resolved fresh from the stored owner at emit time. Emission is fail-open and +// must never gate the path that calls it. +// +// event_id is derived from the DON-aggregated request that drove the transition +// (workflowID + triggerID of the RegisterLogTrigger / UnregisterLogTrigger call), +// namespaced per action. The remote trigger publisher invokes those methods with +// the mode-aggregated request, byte-identical on every capability node, so the +// parts are DON-consistent. physicalFilterID is intentionally NOT the event_id +// (it stays the resource_id): it would collide across activate/release cycles. +func (lts *LogTriggerService) emitDelta(ctx context.Context, delta int64, namespace, workflowID, triggerID string, f filter) { if lts.resourceManager == nil { return } identity := lts.identity(f.donID) orgID := orgresolver.ResolveOrEmpty(ctx, lts.orgResolver, f.workflowOwner, lts.lggr) - lts.resourceManager.EmitDelta(ctx, identity, delta, resourcemanager.UtilizationFields{ + eventID := resourcemanager.EventID(namespace, workflowID, triggerID) + lts.resourceManager.EmitDelta(ctx, identity, eventID, delta, resourcemanager.UtilizationFields{ ResourceType: MeteringResourceType, ResourceID: f.physicalFilterID, OrgID: orgID, @@ -1000,7 +1007,7 @@ func (lts *LogTriggerService) UnregisterLogTrigger(ctx context.Context, triggerI // 1->0 release of the shared physical filter: bill -addressCount once. // The value and identity are reused from the stashed filter so this // -delta reverses the exact +delta the activation billed. - lts.emitDelta(ctx, -trigger.reservedAddressCount, trigger.filter) + lts.emitDelta(ctx, -trigger.reservedAddressCount, "evm-release", meta.WorkflowID, triggerID, trigger.filter) } err := lts.EVMService.UnregisterLogTracking(ctx, lts.generateFilterID(triggerID)) diff --git a/cron/go.mod b/cron/go.mod index 0e28d3521..9a1eeae18 100644 --- a/cron/go.mod +++ b/cron/go.mod @@ -16,7 +16,7 @@ require ( github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 diff --git a/cron/trigger/metering_test.go b/cron/trigger/metering_test.go index 619776fd5..6783dd9bc 100644 --- a/cron/trigger/metering_test.go +++ b/cron/trigger/metering_test.go @@ -98,7 +98,7 @@ var meteredTestDeployment = resourcemanager.DeploymentIdentity{ NumericTenantID: "42", Environment: "staging", Zone: "wf-zone-a", - NodeID: "csa-pubkey-1", + NodeID: "clp-cre-wf-zone-a-1", } // expectedBaseIdentity is the base identity the Service builds from @@ -109,7 +109,7 @@ var expectedBaseIdentity = resourcemanager.ResourceIdentity{ NumericTenantID: "42", Environment: "staging", Zone: "wf-zone-a", - Don: &resourcemanager.DonIdentity{DonID: "7", NodeID: "csa-pubkey-1"}, + Don: &resourcemanager.DonIdentity{DonID: "7", NodeID: "clp-cre-wf-zone-a-1"}, Service: "cron-trigger", ResourcePool: "trigger_registrations", } @@ -180,7 +180,7 @@ func TestCronTrigger_Metering_RegisterUnregisterDeltas(t *testing.T) { assert.Equal(t, "staging", id.GetEnvironment()) assert.Equal(t, "wf-zone-a", id.GetZone()) assert.Equal(t, "7", id.GetDon().GetDonId()) - assert.Equal(t, "csa-pubkey-1", id.GetDon().GetNodeId()) + assert.Equal(t, "clp-cre-wf-zone-a-1", id.GetDon().GetNodeId()) assert.Equal(t, "cron-trigger", id.GetService()) assert.Equal(t, "trigger_registrations", id.GetResourcePool()) // The metering identity DON and the events.KeyDonID label derive from the @@ -191,7 +191,21 @@ func TestCronTrigger_Metering_RegisterUnregisterDeltas(t *testing.T) { assert.Equal(t, "1", register.GetUtilizations()[0].GetValue()) assert.Equal(t, "operations", register.GetUtilizations()[0].GetResourceType()) assert.Equal(t, triggerID1, register.GetUtilizations()[0].GetResourceId()) - require.NotEmpty(t, register.GetUtilizations()[0].GetEventId(), "event_id is stamped per emission") + // event_id is the deterministic, cross-node-identical id derived from the + // DON-aggregated request identity (workflowID + triggerID). + wantRegisterID := resourcemanager.EventID("cron-register", workflowID1, triggerID1) + assert.Equal(t, wantRegisterID, register.GetUtilizations()[0].GetEventId()) + + // Cross-node determinism: a second node fielding the identical register + // request derives the identical event_id. + emitter2 := &fakeMeterEmitter{} + ts2, _, _ := newMeteredTriggerService(t, clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), emitter2) + _, capErr2 := ts2.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr2) + require.Len(t, emitter2.Records(), 1) + assert.Equal(t, register.GetUtilizations()[0].GetEventId(), emitter2.Records()[0].GetUtilizations()[0].GetEventId(), + "two nodes fielding the same register must emit the identical event_id") + require.NoError(t, ts2.Close()) // Each cron tick re-Writes the trigger to reschedule it; the Write // happens before the channel send, so after receiving the event the @@ -211,10 +225,13 @@ func TestCronTrigger_Metering_RegisterUnregisterDeltas(t *testing.T) { require.Len(t, unregister.GetUtilizations(), 1) assert.Equal(t, "-1", unregister.GetUtilizations()[0].GetValue(), "unregister is a signed -1 delta") - // event_id is unique per emission: register and unregister must differ. - require.NotEmpty(t, unregister.GetUtilizations()[0].GetEventId()) + // event_id: unregister is namespaced distinctly from register (so the paired + // +1/-1 deltas never dedup into each other) but is otherwise the same + // deterministic derivation over the DON-consistent workflowID+triggerID. + wantUnregisterID := resourcemanager.EventID("cron-unregister", workflowID1, triggerID1) + assert.Equal(t, wantUnregisterID, unregister.GetUtilizations()[0].GetEventId()) assert.NotEqual(t, register.GetUtilizations()[0].GetEventId(), unregister.GetUtilizations()[0].GetEventId(), - "each emission gets a distinct event_id") + "register and unregister must have distinct event_ids") require.NoError(t, ts.Close()) } diff --git a/cron/trigger/trigger.go b/cron/trigger/trigger.go index 36b3c2d13..7fe5b8dcd 100644 --- a/cron/trigger/trigger.go +++ b/cron/trigger/trigger.go @@ -226,8 +226,9 @@ func (s *Service) donID(workflowDonID uint32) string { // utilizationFields builds the per-trigger billing fields shared by the record // and snapshot paths. resource_id is the workflow-scoped trigger_id; org_id is // resolved fresh from the workflow owner by the caller (never stored). event_id -// is intentionally absent: the ResourceManager stamps a unique UUID per -// emission. +// is intentionally absent from the fields: for records the caller passes it +// explicitly (a deterministic cross-node id), and for snapshots the +// ResourceManager derives it from the bucket/resource/node key. func (s *Service) utilizationFields(triggerID, orgID string) resourcemanager.UtilizationFields { return resourcemanager.UtilizationFields{ ResourceType: meteringResourceType, @@ -243,10 +244,17 @@ func (s *Service) utilizationFields(triggerID, orgID string) resourcemanager.Uti // same identity regardless of the unregister request's metadata. The org is // resolved fresh from owner at emit time. Emission is fail-open and never // affects the registration itself. -func (s *Service) emitMeterRecord(ctx context.Context, delta int64, workflowDonID uint32, triggerID, owner string) { +// emitMeterRecord derives the cross-node-deterministic event_id from the +// registration namespace + the DON-aggregated request identity (workflowID + +// triggerID). RegisterTrigger/UnregisterTrigger are invoked by the remote +// trigger publisher with the mode-aggregated request, byte-identical on every +// capability-DON node, so these parts are DON-consistent and every node emits +// the identical event_id for the same logical (un)register. +func (s *Service) emitMeterRecord(ctx context.Context, delta int64, namespace, workflowID, triggerID string, workflowDonID uint32, owner string) { id := s.base.WithWorkflowDonFallback(workflowDonID) orgID := orgresolver.ResolveOrEmpty(ctx, s.orgResolver, owner, s.lggr) - s.meters.EmitDelta(ctx, id, delta, s.utilizationFields(triggerID, orgID)) + eventID := resourcemanager.EventID(namespace, workflowID, triggerID) + s.meters.EmitDelta(ctx, id, eventID, delta, s.utilizationFields(triggerID, orgID)) } func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapabilitiesDependencies) error { @@ -483,7 +491,7 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat }) // Register bills a +1 delta to the durable trigger-registration level. - s.emitMeterRecord(ctx, 1, metadata.WorkflowDonID, triggerID, metadata.WorkflowOwner) + s.emitMeterRecord(ctx, 1, "cron-register", metadata.WorkflowID, triggerID, metadata.WorkflowDonID, metadata.WorkflowOwner) s.lggr.Debugw("Trigger registered", "workflowId", metadata.WorkflowID, "triggerId", triggerID, "jobId", job.ID()) s.metrics.IncActiveTriggersGauge(ctx) @@ -536,10 +544,11 @@ func (s *Service) UnregisterTrigger(ctx context.Context, triggerID string, metad // Remove from triggers context s.triggers.Delete(triggerID) - // Unregister bills a -1 delta. don_id and owner come from the STORED trigger - // (not the unregister request's metadata) so the delta reverses the exact - // identity the register +1 billed. - s.emitMeterRecord(ctx, -1, trigger.workflowDonID, triggerID, trigger.workflowOwner) + // Unregister bills a -1 delta. workflowID, don_id and owner come from the + // STORED trigger (not the unregister request's metadata) so the delta + // reverses the exact identity the register +1 billed, and the event_id is + // derived from the same DON-consistent workflowID+triggerID. + s.emitMeterRecord(ctx, -1, "cron-unregister", trigger.workflowID, triggerID, trigger.workflowDonID, trigger.workflowOwner) s.lggr.Debugw("UnregisterTrigger", "triggerId", triggerID, "jobId", jobID) s.metrics.DecActiveTriggersGauge(ctx) diff --git a/http_trigger/go.mod b/http_trigger/go.mod index 64812be24..f23e5212a 100644 --- a/http_trigger/go.mod +++ b/http_trigger/go.mod @@ -86,7 +86,7 @@ require ( github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e // indirect diff --git a/http_trigger/trigger/connector_handler.go b/http_trigger/trigger/connector_handler.go index 2b1fa99de..5ffb96471 100644 --- a/http_trigger/trigger/connector_handler.go +++ b/http_trigger/trigger/connector_handler.go @@ -218,7 +218,7 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR switch { case evicted == nil: // Brand-new registration: bill +1 for the new durable resource. - h.emitMeterRecord(ctx, 1, newWorkflowID, workflowDONID, owner) + h.emitMeterRecord(ctx, 1, "http-register", newWorkflowID, workflowDONID, owner) case evicted.workflowSelector.WorkflowID == newWorkflowID: // Same-ID re-register: the durable resource is unchanged, so there is // no level delta to bill. Emit nothing. @@ -227,8 +227,8 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR // new workflow ID. Bill -1 against the evicted workflow's resource_id // and +1 for the new, both derived from the atomically returned // eviction so the old reservation cannot leak. - h.emitMeterRecord(ctx, -1, evicted.workflowSelector.WorkflowID, evicted.metadata.WorkflowDONID, evicted.workflowSelector.WorkflowOwner) - h.emitMeterRecord(ctx, 1, newWorkflowID, workflowDONID, owner) + h.emitMeterRecord(ctx, -1, "http-unregister", evicted.workflowSelector.WorkflowID, evicted.metadata.WorkflowDONID, evicted.workflowSelector.WorkflowOwner) + h.emitMeterRecord(ctx, 1, "http-register", newWorkflowID, workflowDONID, owner) } h.lggr.Debugw("Registered workflow", "workflowID", input.WorkflowSelector.WorkflowID, "workflowOwner", input.WorkflowSelector.WorkflowOwner, "workflowName", input.WorkflowSelector.WorkflowName, "workflowTag", input.WorkflowSelector.WorkflowTag) return nil @@ -238,13 +238,20 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR // change to the durable HTTP-workflow-registration level: register bills +1, // unregister/version-eviction bills -1. resource_id is the workflow ID (HTTP // registrations are workflow-scoped, so there is no shared physical resource). -// The org is resolved fresh from owner at emit time; event_id is stamped by the -// ResourceManager per emission. Emission is fail-open and never affects the -// registration outcome. -func (h *connectorHandler) emitMeterRecord(ctx context.Context, delta int64, workflowID string, workflowDONID uint32, owner string) { +// The org is resolved fresh from owner at emit time. Emission is fail-open and +// never affects the registration outcome. +// +// event_id is derived from the action namespace + the workflow ID, which is +// DON-consistent: the (un)register requests are delivered to every capability +// node as the mode-aggregated request (see the remote trigger publisher), and a +// version update changes the workflow ID so its +1/-1 pair is distinct from the +// prior version. The unregister path passes the same workflowID so its -1 hashes +// symmetrically with the register +1 it reverses. +func (h *connectorHandler) emitMeterRecord(ctx context.Context, delta int64, namespace, workflowID string, workflowDONID uint32, owner string) { identity := h.baseIdentity.WithWorkflowDonFallback(workflowDONID) orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, owner, h.lggr) - h.resourceManager.EmitDelta(ctx, identity, delta, resourcemanager.UtilizationFields{ + eventID := resourcemanager.EventID(namespace, workflowID) + h.resourceManager.EmitDelta(ctx, identity, eventID, delta, resourcemanager.UtilizationFields{ ResourceType: meterResourceType, ResourceID: workflowID, OrgID: orgID, @@ -339,8 +346,10 @@ func (h *connectorHandler) UnregisterWorkflow(ctx context.Context, workflowID st if err != nil { return fmt.Errorf("failed to unregister workflow %s: %w", workflowID, err) } - // Unregister bills a -1 delta against the workflow's resource_id. - h.emitMeterRecord(ctx, -1, workflowID, workflowDONID, owner) + // Unregister bills a -1 delta against the workflow's resource_id. It hashes + // symmetrically with the register +1 (same workflowID, "http-unregister" + // namespace) so the consumer pairs them by workflowID. + h.emitMeterRecord(ctx, -1, "http-unregister", workflowID, workflowDONID, owner) h.lggr.Debugw("Unregistered workflow", "workflowID", workflowID) return nil } From 07b548bd1855b99748f68aeabda9747c6f2ed00b Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Thu, 16 Jul 2026 17:58:18 -0400 Subject: [PATCH 6/6] wip - with api updates from common --- chain_capabilities/evm/main.go | 34 +++---- chain_capabilities/evm/trigger/store.go | 10 +- chain_capabilities/evm/trigger/trigger.go | 67 ++++--------- cron/trigger/metering_test.go | 4 +- cron/trigger/trigger.go | 117 ++++++++-------------- http_trigger/main.go | 15 ++- http_trigger/trigger/connector_handler.go | 60 ++++------- http_trigger/trigger/trigger.go | 20 ++-- 8 files changed, 117 insertions(+), 210 deletions(-) diff --git a/chain_capabilities/evm/main.go b/chain_capabilities/evm/main.go index 304e368d9..eff20ef35 100644 --- a/chain_capabilities/evm/main.go +++ b/chain_capabilities/evm/main.go @@ -31,10 +31,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" evmcappb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm" evmcapserver "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm/server" + "github.com/smartcontractkit/chainlink-common/pkg/durableemitter" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" - "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/types/core" @@ -50,7 +50,7 @@ type capabilityGRPCService struct { limitsFactory limits.Factory // metering is the resolved metering Config (ResourceManagerConfig + // DeploymentIdentity) produced from loop.EnvConfig by - // resourcemanager.ConfigFromEnv at startup. The zero value is valid and + // EnvConfig.MeteringConfig at startup. The zero value is valid and // leaves those dimensions empty/disabled. metering resourcemanager.Config } @@ -69,10 +69,16 @@ var _ evmcapserver.ClientCapability = &capabilityGRPCService{} func main() { loopserver.ServeNew(CapabilityName, func(s *loop.Server) loop.StandardCapabilities { - // ConfigFromEnv is the single, canonical loop-env -> metering mapping - // (enable flags, beholder emitter, snapshot interval, deployment - // identity); no per-main copy of that mapping. - meteringCfg := resourcemanager.ConfigFromEnv(&s.EnvConfig) + // EnvConfig.MeteringConfig is the single, canonical loop-env -> metering + // mapping (enable flags, snapshot interval, deployment identity); no + // per-main copy of that mapping. The durable emitter is resolved here + // and injected, since resourcemanager itself must not reach for the + // process-global emitter. + var emitter resourcemanager.Emitter + if de := durableemitter.GetGlobalEmitter(); de != nil { + emitter = de + } + meteringCfg := s.EnvConfig.MeteringConfig(emitter) return evmcapserver.NewClientServer(&capabilityGRPCService{ lggr: s.Logger, limitsFactory: s.LimitsFactory, @@ -179,19 +185,11 @@ func (c *capabilityGRPCService) Initialise(ctx context.Context, dependencies cor // snapshot interval here. Identity/snapshots are gated by the same metering // env flag as MeterRecords. resourceManager := resourcemanager.NewResourceManager(c.lggr, c.metering.ResourceManagerConfig) - // The authoritative DON ID is the host-injected CapabilityDonID; when 0, - // NewBaseIdentity leaves don_id empty and each emission falls back to the - // consumer workflow's DON via the LogTriggerService's resolveDONID. - baseIdentity := resourcemanager.NewBaseIdentity(c.metering.DeploymentIdentity, dependencies.CapabilityDonID, trigger.MeteringService, trigger.MeteringResource) - // Wrap the injected org resolver in a CachingOrgResolver so the snapshot - // path (GetUtilization, contractually no-network) resolves org from memory. - // We own its Start/Close (wired into the start loop and Close below). - // The LogTriggerService owns the caching resolver's Start/Close lifecycle - // (it type-asserts the *CachingOrgResolver), so we only construct it here. - var orgResolver orgresolver.OrgResolver = dependencies.OrgResolver - if dependencies.OrgResolver != nil { - orgResolver = orgresolver.NewCaching(dependencies.OrgResolver, resourcemanager.DefaultSnapshotInterval) + baseIdentity := resourcemanager.NewBaseIdentity(c.metering.DeploymentIdentity, trigger.MeteringService, trigger.MeteringResource) + if dependencies.CapabilityDonID != 0 { + baseIdentity = baseIdentity.WithDonID(strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10)) } + orgResolver := dependencies.OrgResolver c.triggerService, err = trigger.NewLogTriggerService(evmRelayer, trigger.NewLogTriggerStore(), c.lggr, capabilityID, processor, messageBuilder, cfg.LogTriggerPollInterval, cfg.LogTriggerSendChannelBufferSize, cfg.LogTriggerLimitQueryLogSize, c.limitsFactory, orgResolver, dependencies.TriggerEventStore, resourceManager, baseIdentity, c.chainSelector) diff --git a/chain_capabilities/evm/trigger/store.go b/chain_capabilities/evm/trigger/store.go index f00c31512..2a4978953 100644 --- a/chain_capabilities/evm/trigger/store.go +++ b/chain_capabilities/evm/trigger/store.go @@ -32,12 +32,12 @@ type filter struct { // DON ID string (capability DON, or the consumer WorkflowDonID fallback when // the host did not inject a capability DON); empty when neither is known. donID string - // workflowOwner is the durable attribution key. We store the workflow OWNER, - // never a resolved org ID: the org is resolved fresh at each emission (via - // orgresolver.ResolveOrEmpty) and at snapshot time (via the shared - // CachingOrgResolver), so a re-linked owner is picked up without rewriting - // durable state. + // workflowOwner is stored for attribution. workflowOwner string + // orgID is the organization ID resolved from workflowOwner at registration + // time and stored alongside so that emit and snapshot paths can use it + // without a network call. + orgID string expressions []query.Expression confidence primitives.ConfidenceLevel } diff --git a/chain_capabilities/evm/trigger/trigger.go b/chain_capabilities/evm/trigger/trigger.go index 245c7f126..fe0a2dd42 100644 --- a/chain_capabilities/evm/trigger/trigger.go +++ b/chain_capabilities/evm/trigger/trigger.go @@ -54,11 +54,6 @@ const ( // filter. const cleanupInterval = 30 * time.Second -// orgResolverRefreshInterval bounds owner->org cache staleness in the snapshot -// path. It matches the snapshot cadence so a re-linked org is reflected within -// one snapshot interval. -const orgResolverRefreshInterval = resourcemanager.DefaultSnapshotInterval - // Metering identity constants for the EVM log trigger (SHARED-2711). These are // the service-level dimensions of the base ResourceIdentity: Service is the // stable service constant (it must not encode deployment environment or zone, @@ -108,11 +103,7 @@ type LogTriggerService struct { filterTopicsPerSlotLimiter limits.BoundLimiter[int] eventRateLimit limits.RateLimiter eventPayloadSizeLimiter limits.BoundLimiter[commoncfg.Size] - orgResolver orgresolver.OrgResolver // Optional org resolver for fetching organization IDs - // orgResolverSvc is the CachingOrgResolver's service handle when one was - // injected; its Start/Close are wired into the service lifecycle. Nil when - // no resolver was injected. - orgResolverSvc services.Service + orgResolver orgresolver.OrgResolver // Optional org resolver for fetching organization IDs // filterRegisteredAt records, per log-poller filter name, the time it was // registered at the log poller. cleanUpStaleFilters uses it to skip filters // younger than one cleanup interval, closing the register-time window where @@ -172,11 +163,6 @@ func NewLogTriggerService(evmService types.EVMService, store LogTriggerStore, lg if lts.orgResolver == nil { lts.lggr.Warn("OrgResolver is nil, EVM log trigger capability will not be able to fetch organization ID") } - // Own the caching resolver's lifecycle when one was injected, so its - // background refresh runs while the service is up. - if caching, ok := orgResolver.(*orgresolver.CachingOrgResolver); ok { - lts.orgResolverSvc = caching - } if lts.resourceManager == nil { lts.lggr.Warn("ResourceManager is nil, EVM log trigger capability will not emit meter records") } @@ -231,14 +217,6 @@ func (lts *LogTriggerService) start(ctx context.Context) error { lts.lggr.Infof("Starting clean up of failed log poller filters every %s", cleanupInterval) lts.srvcEng.GoTick(ticker, lts.cleanUpStaleFilters) - // Start the caching org resolver (if any) so its background refresh runs - // and the snapshot path is served from memory. Fail-open. - if lts.orgResolverSvc != nil { - if err := lts.orgResolverSvc.Start(ctx); err != nil { - lts.lggr.Errorw("failed to start caching org resolver; org attribution may be empty", "err", err) - } - } - // The ResourceManager owns the snapshot tick: start it as a sub-service of // this service and Register ourselves so its tick polls GetUtilization. We // never run our own snapshot loop. The RM is fail-open and starting it must @@ -265,11 +243,6 @@ func (lts *LogTriggerService) close() error { lts.rmUnregister = nil } lts.baseTrigger.Stop() - if lts.orgResolverSvc != nil { - if err := lts.orgResolverSvc.Close(); err != nil { - lts.lggr.Errorw("failed to close caching org resolver", "err", err) - } - } if lts.resourceManager != nil { return lts.resourceManager.Close() } @@ -415,14 +388,23 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID // Build the filter's metering identity once from the already-converted // inputs: a workflow-independent content hash and the resolved DON ID. It is // stashed on the trigger state so every later path (unregister, cleanup, - // snapshot) reproduces the same identity without the request input. We store - // the workflow OWNER, never a resolved org, and resolve org at emit time. + // snapshot) reproduces the same identity without the request input. The orgID + // is resolved at registration and stored so emit/snapshot paths avoid network. + var orgID string + if lts.orgResolver != nil && meta.WorkflowOwner != "" { + if resolved, err := lts.orgResolver.Get(ctx, meta.WorkflowOwner); err != nil { + lts.lggr.Warnw("failed to resolve org ID for metering", "owner", meta.WorkflowOwner, "err", err) + } else { + orgID = resolved + } + } loggedFilter := filter{ filterID: filterID, physicalFilterID: physicalFilterID(lts.chainSelector, addresses, sigs, t2, t3, t4), reservedAddressCount: int64(len(addresses)), donID: lts.resolveDONID(meta.WorkflowDonID), workflowOwner: meta.WorkflowOwner, + orgID: orgID, expressions: expressions, confidence: confidence, } @@ -522,23 +504,11 @@ func (lts *LogTriggerService) generateFilterID(triggerID string) string { return triggerID + SuffixLogTriggerFilterID } -// resolveDONID returns the metering DON ID for an emit, applying the -// capabilities#619 0->WorkflowDonID rule: when the host injected a capability -// DON ID, baseIdentity's DON identifier is non-empty and used as-is; otherwise the -// consumer workflow's DON ID (from the request metadata) is the documented -// fallback. The result is stashed on the filter at registration so the -// unregister/cleanup/snapshot paths reproduce the same identity without the -// request. Empty when neither source is known. This is the SAME value stamped -// on the events.KeyDonID label (see startPolling), so metering identity and the -// event label cannot diverge. +// resolveDONID returns the base identity's DON ID. The DON ID is stamped on +// the identity at construction (via WithDonID) and is the single source of +// truth for metering and event labels. func (lts *LogTriggerService) resolveDONID(workflowDonID uint32) string { - if lts.baseIdentity.DonID() != "" { - return lts.baseIdentity.DonID() - } - if workflowDonID != 0 { - return strconv.FormatUint(uint64(workflowDonID), 10) - } - return "" + return lts.baseIdentity.DonID() } // identity returns the base metering identity with DON ID resolved for one @@ -573,12 +543,11 @@ func (lts *LogTriggerService) emitDelta(ctx context.Context, delta int64, namesp return } identity := lts.identity(f.donID) - orgID := orgresolver.ResolveOrEmpty(ctx, lts.orgResolver, f.workflowOwner, lts.lggr) eventID := resourcemanager.EventID(namespace, workflowID, triggerID) lts.resourceManager.EmitDelta(ctx, identity, eventID, delta, resourcemanager.UtilizationFields{ ResourceType: MeteringResourceType, ResourceID: f.physicalFilterID, - OrgID: orgID, + OrgID: f.orgID, }) } @@ -624,7 +593,7 @@ func (lts *LogTriggerService) GetUtilization(ctx context.Context) []resourcemana entries := make([]resourcemanager.SnapshotEntry, 0, len(byPhysical)) for _, agg := range byPhysical { f := agg.f - orgID := orgresolver.ResolveOrEmpty(ctx, lts.orgResolver, f.workflowOwner, lts.lggr) + orgID := f.orgID entries = append(entries, resourcemanager.SnapshotEntry{ Identity: lts.identity(f.donID), Utilizations: []*meteringpb.Utilization{ diff --git a/cron/trigger/metering_test.go b/cron/trigger/metering_test.go index 6783dd9bc..c4df305bd 100644 --- a/cron/trigger/metering_test.go +++ b/cron/trigger/metering_test.go @@ -185,7 +185,7 @@ func TestCronTrigger_Metering_RegisterUnregisterDeltas(t *testing.T) { assert.Equal(t, "trigger_registrations", id.GetResourcePool()) // The metering identity DON and the events.KeyDonID label derive from the // same resolver, so they cannot diverge. - assert.Equal(t, ts.donID(metadata.WorkflowDonID), id.GetDon().GetDonId()) + assert.Equal(t, ts.donID(), id.GetDon().GetDonId()) require.Len(t, register.GetUtilizations(), 1) assert.Equal(t, "1", register.GetUtilizations()[0].GetValue()) @@ -395,7 +395,7 @@ func TestCronTrigger_Metering_DonIDFallback(t *testing.T) { require.Len(t, records, 1) assert.Equal(t, "42", records[0].GetIdentity().GetDon().GetDonId(), "DON ID falls back to WorkflowDonID") // Metering identity DON and events.KeyDonID label share the same resolver. - assert.Equal(t, ts.donID(metadata.WorkflowDonID), records[0].GetIdentity().GetDon().GetDonId()) + assert.Equal(t, ts.donID(), records[0].GetIdentity().GetDon().GetDonId()) // Product falls back to the cron constant when the host injects none. assert.Equal(t, "cre", records[0].GetIdentity().GetProduct()) diff --git a/cron/trigger/trigger.go b/cron/trigger/trigger.go index 7fe5b8dcd..ae9d63ea8 100644 --- a/cron/trigger/trigger.go +++ b/cron/trigger/trigger.go @@ -54,11 +54,6 @@ const ( meteringResourceType = "operations" ) -// orgResolverRefreshInterval bounds how stale a cached owner->org mapping may -// get in the snapshot path. It matches the snapshot cadence so a re-linked org -// is reflected within one snapshot interval. -const orgResolverRefreshInterval = resourcemanager.DefaultSnapshotInterval - type Config struct { FastestScheduleIntervalSeconds int `json:"fastestScheduleIntervalSeconds"` } @@ -73,12 +68,8 @@ type cronTrigger struct { nextRun time.Time workflowID string workflowDonID uint32 - // workflowOwner is the durable attribution key. We store the workflow OWNER, - // never a resolved org ID: the org is resolved fresh at each record emission - // (via orgresolver.ResolveOrEmpty) and at snapshot time (via the shared - // CachingOrgResolver), so a re-linked owner is picked up without rewriting - // durable state. workflowOwner string + orgID string close func() } @@ -109,11 +100,6 @@ type Service struct { // empty. Deployment resourcemanager.DeploymentIdentity orgResolver orgresolver.OrgResolver - // orgResolverSvc is the CachingOrgResolver's service handle when this - // service owns its lifecycle (set in Initialise when an OrgResolver was - // injected). Its Start/Close are wired into start/close. Nil when no - // resolver was injected. - orgResolverSvc services.Service } func (s *Service) RegisterLegacyTrigger(ctx context.Context, triggerID string, metadata capabilities.RequestMetadata, input *crontypedapi.Config) (<-chan capabilities.TriggerAndId[*crontypedapi.LegacyPayload], caperrors.Error) { //nolint:staticcheck @@ -215,18 +201,15 @@ func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFa return s, nil } -// donID resolves the single DON identifier stamped on BOTH the metering -// identity and the events.KeyDonID label, so the two can never diverge: the -// host-injected CapabilityDonID (carried on base) wins, otherwise the consumer -// workflow's DON is the documented fallback. -func (s *Service) donID(workflowDonID uint32) string { - return s.base.WithWorkflowDonFallback(workflowDonID).DonID() +// donID returns the DON identifier stamped on metering identity and event labels. +func (s *Service) donID() string { + return s.base.DonID() } // utilizationFields builds the per-trigger billing fields shared by the record // and snapshot paths. resource_id is the workflow-scoped trigger_id; org_id is -// resolved fresh from the workflow owner by the caller (never stored). event_id -// is intentionally absent from the fields: for records the caller passes it +// resolved at registration time and stored on the cronTrigger. event_id is +// intentionally absent from the fields: for records the caller passes it // explicitly (a deterministic cross-node id), and for snapshots the // ResourceManager derives it from the bucket/resource/node key. func (s *Service) utilizationFields(triggerID, orgID string) resourcemanager.UtilizationFields { @@ -239,22 +222,11 @@ func (s *Service) utilizationFields(triggerID, orgID string) resourcemanager.Uti // emitMeterRecord emits a signed delta MeterRecord (METER_ACTION_UPDATE) for a // change to the durable cron-registration level: register bills +1, unregister -// bills -1. don_id is derived from the STORED workflow DON via -// WithWorkflowDonFallback so a register and its later unregister resolve the -// same identity regardless of the unregister request's metadata. The org is -// resolved fresh from owner at emit time. Emission is fail-open and never -// affects the registration itself. -// emitMeterRecord derives the cross-node-deterministic event_id from the -// registration namespace + the DON-aggregated request identity (workflowID + -// triggerID). RegisterTrigger/UnregisterTrigger are invoked by the remote -// trigger publisher with the mode-aggregated request, byte-identical on every -// capability-DON node, so these parts are DON-consistent and every node emits -// the identical event_id for the same logical (un)register. -func (s *Service) emitMeterRecord(ctx context.Context, delta int64, namespace, workflowID, triggerID string, workflowDonID uint32, owner string) { - id := s.base.WithWorkflowDonFallback(workflowDonID) - orgID := orgresolver.ResolveOrEmpty(ctx, s.orgResolver, owner, s.lggr) +// bills -1. orgID is resolved by the caller before invoking this method. +// Emission is fail-open and never affects the registration itself. +func (s *Service) emitMeterRecord(ctx context.Context, delta int64, namespace, workflowID, triggerID, orgID string) { eventID := resourcemanager.EventID(namespace, workflowID, triggerID) - s.meters.EmitDelta(ctx, id, eventID, delta, s.utilizationFields(triggerID, orgID)) + s.meters.EmitDelta(ctx, s.base, eventID, delta, s.utilizationFields(triggerID, orgID)) } func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapabilitiesDependencies) error { @@ -281,20 +253,15 @@ func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapa if dependencies.OrgResolver == nil { s.lggr.Warn("OrgResolver is nil, cron capability will not be able to fetch organization ID") } else { - // Wrap the injected resolver in a CachingOrgResolver so the snapshot - // path (GetUtilization, contractually no-network) resolves org from - // memory. We own this wrapper's lifecycle (see start/close). - caching := orgresolver.NewCaching(dependencies.OrgResolver, orgResolverRefreshInterval) - s.orgResolver = caching - s.orgResolverSvc = caching + s.orgResolver = dependencies.OrgResolver } // Build the base metering identity. The deployment/node dimensions come from - // s.Deployment (delivered via loop.EnvConfig, set by main before - // Initialise); the authoritative DON ID is the host-injected CapabilityDonID. - // When it is 0, NewBaseIdentity leaves don_id empty and each emission falls - // back to the consumer workflow's DON via WithWorkflowDonFallback. - s.base = resourcemanager.NewBaseIdentity(s.Deployment, dependencies.CapabilityDonID, meteringService, meteringResource) + // s.Deployment (delivered via loop.EnvConfig, set by main before Initialise). + s.base = resourcemanager.NewBaseIdentity(s.Deployment, meteringService, meteringResource) + if dependencies.CapabilityDonID != 0 { + s.base = s.base.WithDonID(strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10)) + } err = s.Start(ctx) if err != nil { @@ -402,7 +369,7 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat events.KeyWorkflowExecutionID, workflowExecutionID, events.KeyWorkflowOwner, metadata.WorkflowOwner, events.KeyWorkflowName, displayWorkflowName, - events.KeyDonID, s.donID(metadata.WorkflowDonID), + events.KeyDonID, s.donID(), events.KeyDonVersion, strconv.Itoa(int(metadata.WorkflowDonConfigVersion)), events.KeyOrganizationID, orgID, events.KeyWorkflowRegistryChainSelector, metadata.WorkflowRegistryChainSelector, @@ -440,6 +407,7 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat workflowID: metadata.WorkflowID, workflowDonID: metadata.WorkflowDonID, workflowOwner: metadata.WorkflowOwner, + orgID: trigger.orgID, close: closeCh, }); !written { return // unregistered concurrently; do not resurrect or send @@ -481,17 +449,27 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat return nil, caperrors.NewPublicSystemError(fmt.Errorf("RegisterTrigger failed to remove job: %s", err), caperrors.Internal) } + var orgID string + if s.orgResolver != nil && metadata.WorkflowOwner != "" { + if resolved, err := s.orgResolver.Get(ctx, metadata.WorkflowOwner); err != nil { + logger.Sugared(s.lggr).Warnw("failed to resolve org ID for metering", "owner", metadata.WorkflowOwner, "err", err) + } else { + orgID = resolved + } + } + s.triggers.Write(triggerID, cronTrigger{ job: job, nextRun: firstRunTime, workflowID: metadata.WorkflowID, workflowDonID: metadata.WorkflowDonID, workflowOwner: metadata.WorkflowOwner, + orgID: orgID, close: closeCh, }) // Register bills a +1 delta to the durable trigger-registration level. - s.emitMeterRecord(ctx, 1, "cron-register", metadata.WorkflowID, triggerID, metadata.WorkflowDonID, metadata.WorkflowOwner) + s.emitMeterRecord(ctx, 1, "cron-register", metadata.WorkflowID, triggerID, orgID) s.lggr.Debugw("Trigger registered", "workflowId", metadata.WorkflowID, "triggerId", triggerID, "jobId", job.ID()) s.metrics.IncActiveTriggersGauge(ctx) @@ -544,11 +522,10 @@ func (s *Service) UnregisterTrigger(ctx context.Context, triggerID string, metad // Remove from triggers context s.triggers.Delete(triggerID) - // Unregister bills a -1 delta. workflowID, don_id and owner come from the - // STORED trigger (not the unregister request's metadata) so the delta - // reverses the exact identity the register +1 billed, and the event_id is - // derived from the same DON-consistent workflowID+triggerID. - s.emitMeterRecord(ctx, -1, "cron-unregister", trigger.workflowID, triggerID, trigger.workflowDonID, trigger.workflowOwner) + // Unregister bills a -1 delta. workflowID and orgID come from the STORED + // trigger (not the unregister request's metadata) so the delta reverses + // the exact identity the register +1 billed. + s.emitMeterRecord(ctx, -1, "cron-unregister", trigger.workflowID, triggerID, trigger.orgID) s.lggr.Debugw("UnregisterTrigger", "triggerId", triggerID, "jobId", jobID) s.metrics.DecActiveTriggersGauge(ctx) @@ -564,15 +541,6 @@ func (s *Service) start(ctx context.Context) error { return errors.New("service has shutdown, it must be built again to restart") } - // Start the caching org resolver so its background refresh runs and the - // snapshot path is served from memory. Fail-open: a resolver start failure - // must not gate the trigger service. - if s.orgResolverSvc != nil { - if err := s.orgResolverSvc.Start(ctx); err != nil { - s.lggr.Errorw("failed to start caching org resolver; org attribution may be empty", "err", err) - } - } - // Register for snapshots. The RM owns the tick; we only supply state via // the Meterable interface. unregisterMeterable is called in close. s.unregisterMeterable = s.meters.Register(s) @@ -587,6 +555,7 @@ func (s *Service) start(ctx context.Context) error { workflowID: trigger.workflowID, workflowDonID: trigger.workflowDonID, workflowOwner: trigger.workflowOwner, + orgID: trigger.orgID, close: trigger.close, }) if err != nil { @@ -602,9 +571,8 @@ func (s *Service) start(ctx context.Context) error { // process-lifecycle metering emissions: a graceful shutdown emits nothing, and // billing releases each still-active registration by its absence from the next // snapshot. close deregisters the Meterable from the ResourceManager FIRST (so -// no snapshot can run after the store is torn down), closes the caching org -// resolver, then shuts the scheduler down. The ResourceManager sub-service is -// closed by the engine afterwards. +// no snapshot can run after the store is torn down), then shuts the scheduler +// down. The ResourceManager sub-service is closed by the engine afterwards. func (s *Service) close() error { if s.scheduler == nil { return errors.New("service has shutdown, it must be built again to restart") @@ -617,12 +585,6 @@ func (s *Service) close() error { s.unregisterMeterable = nil } - if s.orgResolverSvc != nil { - if err := s.orgResolverSvc.Close(); err != nil { - s.lggr.Errorw("failed to close caching org resolver", "err", err) - } - } - err := s.scheduler.Shutdown() if err != nil { return fmt.Errorf("scheduler shutdown encountered a problem: %s", err) @@ -658,12 +620,11 @@ func (s *Service) GetUtilization(ctx context.Context) []resourcemanager.Snapshot triggers := s.triggers.ReadAll() entries := make([]resourcemanager.SnapshotEntry, 0, len(triggers)) for triggerID, trigger := range triggers { - id := s.base.WithWorkflowDonFallback(trigger.workflowDonID) - orgID := orgresolver.ResolveOrEmpty(ctx, s.orgResolver, trigger.workflowOwner, s.lggr) + // Use stored orgID resolved at registration time. entries = append(entries, resourcemanager.SnapshotEntry{ - Identity: id, + Identity: s.base, Utilizations: []*meteringpb.Utilization{ - resourcemanager.NewUtilizationInt(1, s.utilizationFields(triggerID, orgID)), + resourcemanager.NewUtilizationInt(1, s.utilizationFields(triggerID, trigger.orgID)), }, }) } diff --git a/http_trigger/main.go b/http_trigger/main.go index e2bbe7392..939114d5a 100644 --- a/http_trigger/main.go +++ b/http_trigger/main.go @@ -5,16 +5,23 @@ import ( "github.com/smartcontractkit/capabilities/libs/loopserver" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http/server" + "github.com/smartcontractkit/chainlink-common/pkg/durableemitter" "github.com/smartcontractkit/chainlink-common/pkg/loop" "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" ) func main() { loopserver.ServeNew(trigger.ServiceName, func(s *loop.Server) loop.StandardCapabilities { - // ConfigFromEnv is the single, canonical loop-env -> metering mapping - // (enable flags, beholder emitter, snapshot interval, deployment - // identity); no per-main copy of that mapping. - meteringCfg := resourcemanager.ConfigFromEnv(&s.EnvConfig) + // EnvConfig.MeteringConfig is the single, canonical loop-env -> metering + // mapping (enable flags, snapshot interval, deployment identity); no + // per-main copy of that mapping. The durable emitter is resolved here + // and injected, since resourcemanager itself must not reach for the + // process-global emitter. + var emitter resourcemanager.Emitter + if de := durableemitter.GetGlobalEmitter(); de != nil { + emitter = de + } + meteringCfg := s.EnvConfig.MeteringConfig(emitter) svc := trigger.NewService(s.Logger, s.LimitsFactory, meteringCfg) return server.NewHTTPServer(svc) }) diff --git a/http_trigger/trigger/connector_handler.go b/http_trigger/trigger/connector_handler.go index 5ffb96471..ffa94bf09 100644 --- a/http_trigger/trigger/connector_handler.go +++ b/http_trigger/trigger/connector_handler.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "strings" "sync" "time" @@ -59,10 +58,6 @@ type connectorHandler struct { // unregisterMeterable removes this handler from the ResourceManager's // snapshot registry; set on Start, called on Close. unregisterMeterable func() - // orgResolverSvc is the CachingOrgResolver's service handle when one was - // injected; its Start/Close are wired into the handler lifecycle. Nil when - // no resolver was injected. - orgResolverSvc services.Service } func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config ServiceConfig, @@ -84,11 +79,6 @@ func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config Se resourceManager: resourceManager, baseIdentity: baseIdentity, } - // Own the caching resolver's lifecycle when one was injected, so its - // background refresh runs while the handler is up. - if caching, ok := orgResolver.(*orgresolver.CachingOrgResolver); ok { - h.orgResolverSvc = caching - } return h, nil } @@ -97,13 +87,6 @@ func (h *connectorHandler) Start(ctx context.Context) error { h.wg.Add(1) go h.startRequestCacheCleanup(ctx) return h.StartOnce(HandlerName, func() error { - // Start the caching org resolver (if any) so its background refresh - // runs and the snapshot path is served from memory. Fail-open. - if h.orgResolverSvc != nil { - if err := h.orgResolverSvc.Start(ctx); err != nil { - h.lggr.Errorw("failed to start caching org resolver; org attribution may be empty", "err", err) - } - } // Start the ResourceManager as a sub-service (it owns the snapshot // tick) and register this handler as the snapshotted Meterable. The RM // is fail-open: a start failure logs and continues (uniform with the @@ -151,16 +134,11 @@ func (h *connectorHandler) Close() error { // nothing, and billing releases each still-active workflow by its // absence from the next snapshot. Deregister the Meterable from the // ResourceManager FIRST so no snapshot tick can run after teardown, - // then close the caching org resolver and the ResourceManager. + // then close the ResourceManager. if h.unregisterMeterable != nil { h.unregisterMeterable() h.unregisterMeterable = nil } - if h.orgResolverSvc != nil { - if err := h.orgResolverSvc.Close(); err != nil { - h.lggr.Errorw("failed to close caching org resolver", "err", err) - } - } return h.resourceManager.Close() }) } @@ -248,8 +226,15 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR // prior version. The unregister path passes the same workflowID so its -1 hashes // symmetrically with the register +1 it reverses. func (h *connectorHandler) emitMeterRecord(ctx context.Context, delta int64, namespace, workflowID string, workflowDONID uint32, owner string) { - identity := h.baseIdentity.WithWorkflowDonFallback(workflowDONID) - orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, owner, h.lggr) + identity := h.baseIdentity + var orgID string + if h.orgResolver != nil && owner != "" { + if resolved, err := h.orgResolver.Get(ctx, owner); err != nil { + logger.Sugared(h.lggr).Warnw("failed to resolve org ID for metering", "owner", owner, "err", err) + } else { + orgID = resolved + } + } eventID := resourcemanager.EventID(namespace, workflowID) h.resourceManager.EmitDelta(ctx, identity, eventID, delta, resourcemanager.UtilizationFields{ ResourceType: meterResourceType, @@ -258,18 +243,9 @@ func (h *connectorHandler) emitMeterRecord(ctx context.Context, delta int64, nam }) } -// donID returns the DON identifier for an emission. It prefers the -// host-injected capability DON captured in the base identity (capabilities#619) -// and falls back to the per-registration workflow DON when the host did not -// populate one (CapabilityDonID == 0 at Initialise). +// donID returns the DON identifier from the base metering identity. func (h *connectorHandler) donID(workflowDONID uint32) string { - if h.baseIdentity.DonID() != "" { - return h.baseIdentity.DonID() - } - if workflowDONID != 0 { - return strconv.FormatUint(uint64(workflowDONID), 10) - } - return "" + return h.baseIdentity.DonID() } // ResourceIdentity returns the HTTP trigger's base metering identity (six @@ -289,12 +265,14 @@ func (h *connectorHandler) GetUtilization(ctx context.Context) []resourcemanager entries := make([]resourcemanager.SnapshotEntry, 0, len(workflows)) for _, w := range workflows { workflowID := w.workflowSelector.WorkflowID - // Org is resolved from the stored owner via the shared caching resolver - // (a cache hit populated at registration), keeping GetUtilization - // no-network as the snapshot contract requires. - orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, w.workflowSelector.WorkflowOwner, h.lggr) + var orgID string + if h.orgResolver != nil && w.workflowSelector.WorkflowOwner != "" { + if resolved, err := h.orgResolver.Get(ctx, w.workflowSelector.WorkflowOwner); err == nil { + orgID = resolved + } + } entries = append(entries, resourcemanager.SnapshotEntry{ - Identity: h.baseIdentity.WithWorkflowDonFallback(w.metadata.WorkflowDONID), + Identity: h.baseIdentity, Utilizations: []*meteringpb.Utilization{ resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ ResourceType: meterResourceType, diff --git a/http_trigger/trigger/trigger.go b/http_trigger/trigger/trigger.go index f04050872..a85078402 100644 --- a/http_trigger/trigger/trigger.go +++ b/http_trigger/trigger/trigger.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strconv" "strings" "time" @@ -33,11 +34,6 @@ const ( meterResourceType = "operations" ) -// orgResolverRefreshInterval bounds owner->org cache staleness in the snapshot -// path. It matches the snapshot cadence so a re-linked org is reflected within -// one snapshot interval. -const orgResolverRefreshInterval = resourcemanager.DefaultSnapshotInterval - var _ server.HTTPCapability = &service{} type WorkflowRegistrationInput struct { @@ -72,7 +68,7 @@ type service struct { orgResolver orgresolver.OrgResolver // metering is the resolved metering Config (ResourceManagerConfig + // DeploymentIdentity) produced from loop.EnvConfig by - // resourcemanager.ConfigFromEnv in main. + // EnvConfig.MeteringConfig in main. metering resourcemanager.Config } @@ -99,9 +95,7 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa s.lggr.Warn("OrgResolver is nil, HTTP trigger capability will not be able to fetch organization ID") s.orgResolver = nil } else { - // Wrap in a CachingOrgResolver so the snapshot path (no-network) is - // served from memory. The handler owns its Start/Close lifecycle. - s.orgResolver = orgresolver.NewCaching(dependencies.OrgResolver, orgResolverRefreshInterval) + s.orgResolver = dependencies.OrgResolver } workflowStore := newWorkflowStore(s.lggr) var err error @@ -112,10 +106,10 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa metadataPublisher := NewGatewayMetadataPublisher(s.lggr, dependencies.GatewayConnector, workflowStore, s.cfg, s.metrics) requestCache := newRequestCache(s.lggr, dependencies.Store, time.Duration(s.cfg.RequestCacheTTL)*time.Second) resourceManager := resourcemanager.NewResourceManager(s.lggr, s.metering.ResourceManagerConfig) - // The authoritative DON ID is the host-injected CapabilityDonID; when 0, - // NewBaseIdentity leaves don_id empty and each emission falls back to the - // consumer workflow's DON via WithWorkflowDonFallback. - baseIdentity := resourcemanager.NewBaseIdentity(s.metering.DeploymentIdentity, dependencies.CapabilityDonID, meterService, meterResource) + baseIdentity := resourcemanager.NewBaseIdentity(s.metering.DeploymentIdentity, meterService, meterResource) + if dependencies.CapabilityDonID != 0 { + baseIdentity = baseIdentity.WithDonID(strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10)) + } s.connectorHandler, err = NewConnectorHandler(s.lggr, dependencies.GatewayConnector, s.cfg, workflowStore, metadataPublisher, requestCache, s.metrics, s.orgResolver, resourceManager, baseIdentity) if err != nil { return err