diff --git a/chain_capabilities/evm/go.mod b/chain_capabilities/evm/go.mod index 753e68deb..3b67611a1 100644 --- a/chain_capabilities/evm/go.mod +++ b/chain_capabilities/evm/go.mod @@ -2,17 +2,26 @@ module github.com/smartcontractkit/capabilities/chain_capabilities/evm go 1.26.2 +// Unpublished local stack for SHARED-2709; drop once chainlink-common and +// chainlink-protos/metering/go are tagged. +replace ( + github.com/smartcontractkit/chainlink-common => ../../../chainlink-common + github.com/smartcontractkit/chainlink-protos/metering/go => ../../../chainlink-protos/metering/go +) + require ( github.com/ethereum/go-ethereum v1.17.0 github.com/google/go-cmp v0.7.0 + github.com/jonboulle/clockwork v0.5.0 github.com/smartcontractkit/capabilities/chain_capabilities/common v0.0.0-20260615195421-fb87220e503f github.com/smartcontractkit/capabilities/libs v0.0.0-20260609124022-2749e4a32bfb github.com/smartcontractkit/chain-selectors v1.0.104 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260410162948-2dca02f24e98 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260410144512-ca02ad6ed16a - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.uber.org/zap v1.27.1 @@ -69,7 +78,6 @@ require ( github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jonboulle/clockwork v0.5.0 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -90,11 +98,11 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/smartcontractkit/chainlink-common/keystore v1.1.1-0.20260529092756-a94bc8ce96d6 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260326122810-b657beadfb57 // indirect github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260401162955-be2bc6b5264b // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe // indirect diff --git a/chain_capabilities/evm/go.sum b/chain_capabilities/evm/go.sum index 02eed31f7..2ba3254c2 100644 --- a/chain_capabilities/evm/go.sum +++ b/chain_capabilities/evm/go.sum @@ -473,12 +473,12 @@ github.com/smartcontractkit/capabilities/libs v0.0.0-20260609124022-2749e4a32bfb github.com/smartcontractkit/capabilities/libs v0.0.0-20260609124022-2749e4a32bfb/go.mod h1:LS7F8U2YZNc0Vt8f6SVWUUigGLxdxZMpyC7VCcUTagg= github.com/smartcontractkit/chain-selectors v1.0.104 h1:/n9pPGM5W/+r1eHoWZv4VwX9LNS1af4+ICyhM8zKRNM= github.com/smartcontractkit/chain-selectors v1.0.104/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37 h1:ZZlU2e+hVC1Y8VAczVNGZBM3rU3HSqkOCn2KZHHV7gc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37/go.mod h1:paOB/6dy57owHtOGzhgaRBWRDT5BEWfnJF5M7sgkcro= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.1.1-0.20260529092756-a94bc8ce96d6 h1:fWsYxxj35fp1/6YZngoTsOTMLqDie4N5X0osAOdhUTE= github.com/smartcontractkit/chainlink-common/keystore v1.1.1-0.20260529092756-a94bc8ce96d6/go.mod h1:6JexOOhPhknQ0QMuppFIlOpm6wCp54yZMxai+tWugwY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0 h1:NExKM/D0HneOq/N5LGTbkV4VOa0UHCvfTNEb4GqYpto= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260410162948-2dca02f24e98 h1:h/L6wrXYLQalI/vHm6qg/KBv6d7kMb3geMHV5hCM1t4= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260410162948-2dca02f24e98/go.mod h1:6vCMfxz7cMW0wWseNKtct+b1JJbbRVJJhh/t6pQWN3M= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251022073203-7d8ae8cf67c1 h1:NTODgwAil7BLoijS7y6KnEuNbQ9v60VUhIR9FcAzIhg= @@ -491,12 +491,16 @@ github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260410144512- github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260410144512-ca02ad6ed16a/go.mod h1:7ketk4ischPQW/JQgmyHz6zdzLUJv1VC29SiSgosydQ= github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 h1:GCzrxDWn3b7jFfEA+WiYRi8CKoegsayiDoJBCjYkneE= github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4/go.mod h1:HHGeDUpAsPa0pmOx7wrByCitjQ0mbUxf0R9v+g67uCA= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 h1:iRFmfMFQtcnhGDjCuARQG4MPbcmbbJDDw7MUH3GcGy8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b h1:VDgJWDipihV9f7M5+d21d1RzSsg5rEv+iI12oN1VQbo= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e h1:Hv9Mww35LrufCdM9wtS9yVi/rEWGI1UnjHbcKKU0nVY= diff --git a/chain_capabilities/evm/main.go b/chain_capabilities/evm/main.go index 38faa09d5..eff20ef35 100644 --- a/chain_capabilities/evm/main.go +++ b/chain_capabilities/evm/main.go @@ -31,8 +31,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" evmcappb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm" evmcapserver "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm/server" + "github.com/smartcontractkit/chainlink-common/pkg/durableemitter" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/types/core" @@ -46,6 +48,11 @@ type capabilityGRPCService struct { capability lggr logger.Logger limitsFactory limits.Factory + // metering is the resolved metering Config (ResourceManagerConfig + + // DeploymentIdentity) produced from loop.EnvConfig by + // EnvConfig.MeteringConfig at startup. The zero value is valid and + // leaves those dimensions empty/disabled. + metering resourcemanager.Config } type capability struct { @@ -62,7 +69,21 @@ var _ evmcapserver.ClientCapability = &capabilityGRPCService{} func main() { loopserver.ServeNew(CapabilityName, func(s *loop.Server) loop.StandardCapabilities { - return evmcapserver.NewClientServer(&capabilityGRPCService{lggr: s.Logger, limitsFactory: s.LimitsFactory}) + // EnvConfig.MeteringConfig is the single, canonical loop-env -> metering + // mapping (enable flags, snapshot interval, deployment identity); no + // per-main copy of that mapping. The durable emitter is resolved here + // and injected, since resourcemanager itself must not reach for the + // process-global emitter. + var emitter resourcemanager.Emitter + if de := durableemitter.GetGlobalEmitter(); de != nil { + emitter = de + } + meteringCfg := s.EnvConfig.MeteringConfig(emitter) + return evmcapserver.NewClientServer(&capabilityGRPCService{ + lggr: s.Logger, + limitsFactory: s.LimitsFactory, + metering: meteringCfg, + }) }, loop.WithOtelViews(append(consMetrics.MetricViews(), monitoring.MetricViews()...))) } @@ -158,11 +179,20 @@ func (c *capabilityGRPCService) Initialise(ctx context.Context, dependencies cor return fmt.Errorf("failed to init evm relayer for chainID %d from relayer: %w", cfg.ChainID, err) } - // TODO: add org resolver capabilityID := fmt.Sprintf("%s (%d)", c.id, cfg.ChainID) - c.triggerService, err = trigger.NewLogTriggerService(evmRelayer, trigger.NewLogTriggerStore(), c.lggr, capabilityID, capabilityDonID, processor, messageBuilder, + // The ResourceManager owns the snapshot tick; the LogTriggerService starts it + // as a sub-service and registers itself, so it must be configured with a + // snapshot interval here. Identity/snapshots are gated by the same metering + // env flag as MeterRecords. + resourceManager := resourcemanager.NewResourceManager(c.lggr, c.metering.ResourceManagerConfig) + baseIdentity := resourcemanager.NewBaseIdentity(c.metering.DeploymentIdentity, trigger.MeteringService, trigger.MeteringResource) + if dependencies.CapabilityDonID != 0 { + baseIdentity = baseIdentity.WithDonID(strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10)) + } + orgResolver := dependencies.OrgResolver + c.triggerService, err = trigger.NewLogTriggerService(evmRelayer, trigger.NewLogTriggerStore(), c.lggr, capabilityID, processor, messageBuilder, cfg.LogTriggerPollInterval, cfg.LogTriggerSendChannelBufferSize, cfg.LogTriggerLimitQueryLogSize, c.limitsFactory, - dependencies.OrgResolver, dependencies.TriggerEventStore) + orgResolver, dependencies.TriggerEventStore, resourceManager, baseIdentity, c.chainSelector) if err != nil { return fmt.Errorf("error when creating trigger: %w", err) } diff --git a/chain_capabilities/evm/trigger/physical_filter_id.go b/chain_capabilities/evm/trigger/physical_filter_id.go new file mode 100644 index 000000000..0d587739d --- /dev/null +++ b/chain_capabilities/evm/trigger/physical_filter_id.go @@ -0,0 +1,63 @@ +package trigger + +import ( + "crypto/sha256" + "encoding/hex" + "sort" + "strings" + + evmtypes "github.com/smartcontractkit/chainlink-common/pkg/types/chains/evm" +) + +// physicalFilterID returns the workflow-independent content identity of an EVM +// log filter: the lowercase hex SHA-256 over a canonical encoding of the +// filter's physical matching criteria (chain selector, addresses, event +// signatures, and positional topic slots). Two filters that match exactly the +// same on-chain logs hash to the same ID regardless of which workflow or +// trigger registered them, or of the order their addresses/sigs/topics were +// supplied. It is used as ResourceIdentity.ResourceID and as the +// RESERVE/RELEASE event identity so identical filters share one billable +// physical resource (R4). +// +// Canonicalization rules (each rule defeats a source of non-determinism): +// - addresses and event sigs are lowercased 0x-prefixed hex and sorted +// ascending: the matching set is order-independent; +// - topic2/topic3/topic4 are POSITIONAL — a value in topic2 is a different +// filter than the same value in topic3 — so each slot is encoded under its +// own positional tag, and within a slot the values are sorted ascending; +// - the chain selector scopes the hash so identical filters on different +// chains stay distinct. +// +// The preimage uses "|" as a top-level separator and "," within a set; the +// per-element hex encodings are fixed-width and contain neither, so the +// encoding is unambiguous. +func physicalFilterID(chainSelector string, addresses []evmtypes.Address, eventSigs, topic2, topic3, topic4 []evmtypes.Hash) string { + sortedAddrs := make([]string, len(addresses)) + for i, a := range addresses { + sortedAddrs[i] = "0x" + hex.EncodeToString(a[:]) + } + sort.Strings(sortedAddrs) + + canonHashes := func(hs []evmtypes.Hash) string { + out := make([]string, len(hs)) + for i, h := range hs { + out[i] = "0x" + hex.EncodeToString(h[:]) + } + sort.Strings(out) + return strings.Join(out, ",") + } + + // Topic slots are encoded positionally so the same value in different slots + // produces a different identity. + preimage := strings.Join([]string{ + "cs=" + chainSelector, + "addrs=" + strings.Join(sortedAddrs, ","), + "sigs=" + canonHashes(eventSigs), + "t2=" + canonHashes(topic2), + "t3=" + canonHashes(topic3), + "t4=" + canonHashes(topic4), + }, "|") + + sum := sha256.Sum256([]byte(preimage)) + return hex.EncodeToString(sum[:]) +} diff --git a/chain_capabilities/evm/trigger/store.go b/chain_capabilities/evm/trigger/store.go index ee781ea63..2a4978953 100644 --- a/chain_capabilities/evm/trigger/store.go +++ b/chain_capabilities/evm/trigger/store.go @@ -12,9 +12,34 @@ import ( ) type filter struct { - filterID string - expressions []query.Expression - confidence primitives.ConfidenceLevel + filterID string + // physicalFilterID is the workflow-independent content hash of the filter's + // physical matching criteria (chain selector + canonicalized addresses, + // event sigs, and positional topics). It is the metering ResourceID and the + // shared-resource refcount key, so the unregister, cleanup, and snapshot + // paths all reuse it from here without the request input. Identical filters + // registered by different triggers share one physicalFilterID and are billed + // once (a +delta on the 0->1 activation, a -delta on the 1->0 release). + physicalFilterID string + // reservedAddressCount is the number of filter addresses this filter bills: + // the +delta emitted on the physical filter's 0->1 activation, and the + // -delta on its 1->0 release, both carry this value. UnregisterLogTrigger + // ignores its request input, so the count is stashed here at registration. + reservedAddressCount int64 + // donID is stashed from the registration RequestMetadata so the + // unregister/cleanup/snapshot paths reproduce the same identity as the + // activation delta without the original request. It is the resolved metering + // DON ID string (capability DON, or the consumer WorkflowDonID fallback when + // the host did not inject a capability DON); empty when neither is known. + donID string + // workflowOwner is stored for attribution. + workflowOwner string + // orgID is the organization ID resolved from workflowOwner at registration + // time and stored alongside so that emit and snapshot paths can use it + // without a network call. + orgID string + expressions []query.Expression + confidence primitives.ConfidenceLevel } type logTriggerState struct { @@ -38,6 +63,8 @@ type LogTriggerStore interface { Read(triggerID string) (value logTriggerState, ok bool) ReadAll() (values map[string]logTriggerState) Write(triggerID string, value logTriggerState) + WriteAndIsFirstForPhysical(triggerID string, value logTriggerState) (firstForPhysical bool) + DeleteAndIsLastForPhysical(triggerID, physicalFilterID string) (found, lastForPhysical bool) Update(triggerID string, lastBlock *big.Int, unfinalizedSentEventIDs map[string]*big.Int) error Delete(triggerID string) } @@ -71,6 +98,50 @@ func (cs *logTriggerStore) Write(triggerID string, value logTriggerState) { cs.triggers[triggerID] = value } +// WriteAndIsFirstForPhysical writes value under triggerID and reports whether, +// at the instant of the write, no OTHER trigger already held +// value.physicalFilterID. A true result is the 0->1 activation of that shared +// physical filter — the only transition that bills a +delta. Deriving the +// transition from owned state at operation time keeps the emitter stateless +// (no ledger). The scan and write are atomic under the store lock so two +// concurrent registrations of the same physical filter can never both observe +// zero and double-bill. +func (cs *logTriggerStore) WriteAndIsFirstForPhysical(triggerID string, value logTriggerState) (firstForPhysical bool) { + cs.mu.Lock() + defer cs.mu.Unlock() + firstForPhysical = true + for id, existing := range cs.triggers { + if id == triggerID { + continue + } + if existing.physicalFilterID == value.physicalFilterID { + firstForPhysical = false + break + } + } + cs.triggers[triggerID] = value + return firstForPhysical +} + +// DeleteAndIsLastForPhysical deletes triggerID and reports whether any remaining +// trigger still holds physicalFilterID. lastForPhysical is true when none +// remain — the 1->0 deactivation that bills a -delta. found reports whether the +// trigger existed. Scan and delete are atomic under the store lock. +func (cs *logTriggerStore) DeleteAndIsLastForPhysical(triggerID, physicalFilterID string) (found, lastForPhysical bool) { + cs.mu.Lock() + defer cs.mu.Unlock() + _, found = cs.triggers[triggerID] + delete(cs.triggers, triggerID) + lastForPhysical = true + for _, existing := range cs.triggers { + if existing.physicalFilterID == physicalFilterID { + lastForPhysical = false + break + } + } + return found, lastForPhysical +} + func (cs *logTriggerStore) Update(triggerID string, lastBlock *big.Int, unfinalizedSentEventIDs map[string]*big.Int) error { cs.mu.Lock() defer cs.mu.Unlock() diff --git a/chain_capabilities/evm/trigger/trigger.go b/chain_capabilities/evm/trigger/trigger.go index 5f11cfcef..fe0a2dd42 100644 --- a/chain_capabilities/evm/trigger/trigger.go +++ b/chain_capabilities/evm/trigger/trigger.go @@ -7,13 +7,12 @@ import ( "math/big" "strconv" "strings" + "sync" "time" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - capcommon "github.com/smartcontractkit/capabilities/chain_capabilities/common" - commoncfg "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink-common/pkg/beholder" @@ -23,6 +22,7 @@ import ( evmservice "github.com/smartcontractkit/chainlink-common/pkg/chains/evm" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" @@ -34,7 +34,9 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives/evm" "github.com/smartcontractkit/chainlink-common/pkg/workflows" "github.com/smartcontractkit/chainlink-common/pkg/workflows/events" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" + capcommon "github.com/smartcontractkit/capabilities/chain_capabilities/common" "github.com/smartcontractkit/capabilities/chain_capabilities/evm/monitoring" ) @@ -44,6 +46,30 @@ const ( defaultLimitQueryLogSize = 1000 ) +// cleanupInterval is how often stale log-poller filters are swept AND the +// minimum age a filter must reach before it is eligible for that sweep. The +// min-age guard closes a race the cleanup previously assumed away: a filter can +// be live at the log poller for a brief window before its store entry is +// observable, and cleaning it in that window would bill-then-kill a live +// filter. +const cleanupInterval = 30 * time.Second + +// Metering identity constants for the EVM log trigger (SHARED-2711). These are +// the service-level dimensions of the base ResourceIdentity: Service is the +// stable service constant (it must not encode deployment environment or zone, +// which ride on the structured identity's coarse dimensions). Resource pool +// lives on ResourceIdentity; billing unit lives on Utilization.resource_type. +const ( + MeteringService = "evm-log-trigger" + MeteringResource = "log_filters" + MeteringResourceType = "log_filter_addresses" +) + +// LogTriggerService is a resourcemanager.Meterable: it is registered with the +// ResourceManager at start so the manager's snapshot tick polls its active +// filters. +var _ resourcemanager.Meterable = (*LogTriggerService)(nil) + type LogTriggerService struct { services.Service @@ -55,12 +81,18 @@ type LogTriggerService struct { lggr logger.Logger beholderProcessor beholder.ProtoProcessor messageBuilder *monitoring.MessageBuilder - - // capabilityDonID is the on-chain DON ID of this capability DON. - // Used to label emitted events with the sending DON ID, distinct from the - // consumer workflow's DON ID carried in RequestMetadata.WorkflowDonID. Zero - // means unknown; the labeler then falls back to WorkflowDonID. - capabilityDonID uint32 + resourceManager *resourcemanager.ResourceManager + // baseIdentity is the producer's base metering identity: the six coarse + // dimensions plus service/resource_pool, built once at Initialise. + // Per-resource billing fields are set on Utilization. + // When the host did not inject a capability DON ID, baseIdentity has an + // empty DON identifier and is filled per-emit from the consumer's + // WorkflowDonID. + baseIdentity resourcemanager.ResourceIdentity + chainSelector string // decimal chain selector, the chain label on meter records + // rmUnregister removes this service from the ResourceManager's snapshot + // registry; it is set when the RM is started in start and invoked in close. + rmUnregister func() triggers LogTriggerStore logTriggerPollInterval time.Duration @@ -71,18 +103,25 @@ type LogTriggerService struct { filterTopicsPerSlotLimiter limits.BoundLimiter[int] eventRateLimit limits.RateLimiter eventPayloadSizeLimiter limits.BoundLimiter[commoncfg.Size] - orgResolver orgresolver.OrgResolver // Optional org resolver for fetching organization IDs + orgResolver orgresolver.OrgResolver // Optional org resolver for fetching organization IDs + // filterRegisteredAt records, per log-poller filter name, the time it was + // registered at the log poller. cleanUpStaleFilters uses it to skip filters + // younger than one cleanup interval, closing the register-time window where + // a filter is live at the poller before its store entry is observable. + filterRegisteredAt sync.Map // filterID (string) -> time.Time } // NewLogTriggerService creates a new instance of logTriggerService. func NewLogTriggerService(evmService types.EVMService, store LogTriggerStore, lggr logger.Logger, capabilityID string, - capabilityDonID uint32, beholderProcessor beholder.ProtoProcessor, messageBuilder *monitoring.MessageBuilder, logTriggerPollInterval time.Duration, logTriggerSendChannelBufferSize uint64, logTriggerLimitQueryLogSize uint64, limitsFactory limits.Factory, orgResolver orgresolver.OrgResolver, - triggerEventStore capabilities.EventStore) (*LogTriggerService, error) { + triggerEventStore capabilities.EventStore, + resourceManager *resourcemanager.ResourceManager, + baseIdentity resourcemanager.ResourceIdentity, + chainSelector uint64) (*LogTriggerService, error) { if capabilityID == "" { return nil, fmt.Errorf("capabilityID must be non-empty") } @@ -112,7 +151,9 @@ func NewLogTriggerService(evmService types.EVMService, store LogTriggerStore, lg lggr: lggr, beholderProcessor: beholderProcessor, messageBuilder: messageBuilder, - capabilityDonID: capabilityDonID, + resourceManager: resourceManager, + baseIdentity: baseIdentity, + chainSelector: strconv.FormatUint(chainSelector, 10), triggers: store, logTriggerPollInterval: logTriggerPollInterval, logTriggerSendChannelBufferSize: currentSendChannelBufferSize, @@ -122,6 +163,9 @@ func NewLogTriggerService(evmService types.EVMService, store LogTriggerStore, lg if lts.orgResolver == nil { lts.lggr.Warn("OrgResolver is nil, EVM log trigger capability will not be able to fetch organization ID") } + if lts.resourceManager == nil { + lts.lggr.Warn("ResourceManager is nil, EVM log trigger capability will not emit meter records") + } if err := lts.initLimiters(limitsFactory); err != nil { return nil, err } @@ -169,15 +213,39 @@ func (lts *LogTriggerService) start(ctx context.Context) error { if err != nil { return err } - duration := 30 * time.Second - ticker := services.NewTicker(duration) - lts.lggr.Infof("Starting clean up of failed log poller filters every %s seconds", duration) + ticker := services.NewTicker(cleanupInterval) + lts.lggr.Infof("Starting clean up of failed log poller filters every %s", cleanupInterval) lts.srvcEng.GoTick(ticker, lts.cleanUpStaleFilters) + + // The ResourceManager owns the snapshot tick: start it as a sub-service of + // this service and Register ourselves so its tick polls GetUtilization. We + // never run our own snapshot loop. The RM is fail-open and starting it must + // not gate the trigger service, so a start error is logged, not returned. + if lts.resourceManager != nil { + if err := lts.resourceManager.Start(ctx); err != nil { + lts.lggr.Errorw("failed to start metering ResourceManager; snapshots disabled", "err", err) + } else { + lts.rmUnregister = lts.resourceManager.Register(lts) + } + } return nil } +// close performs an orderly shutdown. There are NO process-lifecycle metering +// emissions: a graceful stop emits nothing, and billing releases each +// still-active filter by its absence from the next snapshot. The Meterable is +// deregistered from the ResourceManager FIRST so no snapshot tick can run +// against a half-torn-down service, then the base trigger, caching resolver, +// and ResourceManager are closed. func (lts *LogTriggerService) close() error { + if lts.rmUnregister != nil { + lts.rmUnregister() + lts.rmUnregister = nil + } lts.baseTrigger.Stop() + if lts.resourceManager != nil { + return lts.resourceManager.Close() + } return nil } @@ -211,11 +279,31 @@ func (lts *LogTriggerService) cleanUpStaleFilters(ctx context.Context) { lts.lggr.Debugf("Found %d filters to clean up that are not live", len(toCleanUp)) for filterID := range toCleanUp { + // Min-age guard: a filter that was registered at the log poller less + // than one cleanup interval ago may simply not have its store entry + // observable yet (the register path writes the store right after the + // RegisterLogTracking RPC returns). Skip it this round so we never + // bill-then-kill a filter that is actually live. Filters with no + // recorded registration time (e.g. orphaned from a previous process) + // are always eligible. + if registeredAt, ok := lts.filterRegisteredAt.Load(filterID); ok { + if time.Since(registeredAt.(time.Time)) < cleanupInterval { + lts.lggr.Debugf("Skipping filter %s: younger than one cleanup interval", filterID) + continue + } + } lts.lggr.Debugf("Cleaning up filter %s", filterID) if err := lts.EVMService.UnregisterLogTracking(ctx, filterID); err != nil { summary := fmt.Sprintf("failed to unregister log-tracking from the clean up thread: '%v' source triggerID: %s", err, filterID) monitoring.LogAndEmitError(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerCleanUpError(telemetryContext, summary, err.Error())) + continue } + lts.filterRegisteredAt.Delete(filterID) + // This is log-poller filter hygiene only; it emits no MeterRecord. An + // orphaned filter has no trigger state, so it is already absent from + // GetUtilization and therefore from subsequent Snapshots. Billing + // reconciles the lost level by that absence (the snapshot liveness + // mechanism), not by a synthetic cleanup emission. } } @@ -295,6 +383,32 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID Topic4: t4, } + expressions, confidence := lts.createLogRequest(ctx, addresses, sigs, t2, t3, t4, input.GetConfidence()) + + // Build the filter's metering identity once from the already-converted + // inputs: a workflow-independent content hash and the resolved DON ID. It is + // stashed on the trigger state so every later path (unregister, cleanup, + // snapshot) reproduces the same identity without the request input. The orgID + // is resolved at registration and stored so emit/snapshot paths avoid network. + var orgID string + if lts.orgResolver != nil && meta.WorkflowOwner != "" { + if resolved, err := lts.orgResolver.Get(ctx, meta.WorkflowOwner); err != nil { + lts.lggr.Warnw("failed to resolve org ID for metering", "owner", meta.WorkflowOwner, "err", err) + } else { + orgID = resolved + } + } + loggedFilter := filter{ + filterID: filterID, + physicalFilterID: physicalFilterID(lts.chainSelector, addresses, sigs, t2, t3, t4), + reservedAddressCount: int64(len(addresses)), + donID: lts.resolveDONID(meta.WorkflowDonID), + workflowOwner: meta.WorkflowOwner, + orgID: orgID, + expressions: expressions, + confidence: confidence, + } + if err = lts.EVMService.RegisterLogTracking(ctx, filterQuery); err != nil { registerError := fmt.Errorf("failed to register log-tracking: '%w' for triggerID: %s, addresses: %v, eventSig: %v, topic2: %v, topic3: %v, topic4: %v", err, triggerID, filterQuery.Addresses, filterQuery.EventSigs, filterQuery.Topic2, filterQuery.Topic3, filterQuery.Topic4) @@ -309,7 +423,30 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID monitoring.LogAndEmitError(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerError(telemetryContext, triggerID, summary, err.Error())) return nil, caperrors.NewPublicSystemError(registerError, caperrors.Unavailable) } - expressions, confidence := lts.createLogRequest(ctx, addresses, sigs, t2, t3, t4, input.GetConfidence()) + // The filter is now live at the log poller. Record when so the stale-filter + // cleanup skips it until it is at least one interval old (see + // cleanUpStaleFilters). + lts.filterRegisteredAt.Store(filterID, time.Now()) + + // Create the polling context up front (cancelled on unregister or service + // stop) so the store write is synchronous and carries a working cancelFunc. + pollCtx, cancel := lts.srvcEng.NewCtx() + + // Write the trigger state SYNCHRONOUSLY, before emitting any delta, and let + // the store report whether this is the physical filter's 0->1 activation. + // Writing first means the orphan-cleanup thread can never observe the live + // log-poller filter without its store entry and bill-then-kill it; deriving + // the transition under the store lock keeps identical filters billed once. + firstForPhysical := lts.triggers.WriteAndIsFirstForPhysical(triggerID, logTriggerState{ + cancelFunc: cancel, + lastBlock: fromBlock, + unfinalizedSentEventIDs: make(map[string]*big.Int), + filter: loggedFilter, + }) + if firstForPhysical { + // 0->1 activation of a shared physical filter: bill +addressCount once. + lts.emitDelta(ctx, loggedFilter.reservedAddressCount, "evm-activate", meta.WorkflowID, triggerID, loggedFilter) + } monitoring.EmitInitiated(ctx, lts.lggr, lts.beholderProcessor, lts.messageBuilder.BuildLogTriggerInitiated(telemetryContext, input)) @@ -317,18 +454,7 @@ func (lts *LogTriggerService) RegisterLogTrigger(ctx context.Context, triggerID lts.baseTrigger.RegisterTrigger(triggerID, logCh) - lts.srvcEng.Go(func(ctx context.Context) { - ctx, cancel := context.WithCancel(ctx) - lts.triggers.Write(triggerID, logTriggerState{ - cancelFunc: cancel, - lastBlock: fromBlock, - unfinalizedSentEventIDs: make(map[string]*big.Int), - filter: filter{ - filterID: filterID, - expressions: expressions, - confidence: confidence, - }, - }) + lts.srvcEng.GoCtx(pollCtx, func(ctx context.Context) { ctx = meta.ContextWithCRE(ctx) lts.startPolling(ctx, telemetryContext, triggerID, input, logCh) }) @@ -378,6 +504,110 @@ func (lts *LogTriggerService) generateFilterID(triggerID string) string { return triggerID + SuffixLogTriggerFilterID } +// resolveDONID returns the base identity's DON ID. The DON ID is stamped on +// the identity at construction (via WithDonID) and is the single source of +// truth for metering and event labels. +func (lts *LogTriggerService) resolveDONID(workflowDonID uint32) string { + return lts.baseIdentity.DonID() +} + +// identity returns the base metering identity with DON ID resolved for one +// resource. +func (lts *LogTriggerService) identity(donID string) resourcemanager.ResourceIdentity { + id := lts.baseIdentity + if donID == "" { + return id + } + id.Don = &resourcemanager.DonIdentity{ + DonID: donID, + NodeID: id.NodeID(), + } + return id +} + +// emitDelta emits a signed delta MeterRecord (METER_ACTION_UPDATE) for a shared +// physical log filter: +addressCount on the physical filter's 0->1 activation, +// -addressCount on its 1->0 release. The physical filter content hash is the +// ResourceID, so all triggers sharing it bill against one resource. The org is +// resolved fresh from the stored owner at emit time. Emission is fail-open and +// must never gate the path that calls it. +// +// event_id is derived from the DON-aggregated request that drove the transition +// (workflowID + triggerID of the RegisterLogTrigger / UnregisterLogTrigger call), +// namespaced per action. The remote trigger publisher invokes those methods with +// the mode-aggregated request, byte-identical on every capability node, so the +// parts are DON-consistent. physicalFilterID is intentionally NOT the event_id +// (it stays the resource_id): it would collide across activate/release cycles. +func (lts *LogTriggerService) emitDelta(ctx context.Context, delta int64, namespace, workflowID, triggerID string, f filter) { + if lts.resourceManager == nil { + return + } + identity := lts.identity(f.donID) + eventID := resourcemanager.EventID(namespace, workflowID, triggerID) + lts.resourceManager.EmitDelta(ctx, identity, eventID, delta, resourcemanager.UtilizationFields{ + ResourceType: MeteringResourceType, + ResourceID: f.physicalFilterID, + OrgID: f.orgID, + }) +} + +// ResourceIdentity implements resourcemanager.Meterable: it returns the +// producer's base identity (the six coarse dimensions plus +// service/resource_pool). The per-resource DON ID and billing fields are +// populated per active filter by GetUtilization. +func (lts *LogTriggerService) ResourceIdentity() resourcemanager.ResourceIdentity { + return lts.baseIdentity +} + +// GetUtilization implements resourcemanager.Meterable: it returns one snapshot +// entry per distinct physical filter (NOT one per trigger registration), since +// identical filters registered by many triggers share one billable physical +// resource. The value is the shared filter's address count. Org attribution for +// a shared filter uses the deterministic "lowest triggerID's owner" rule so all +// nodes agree on the same org without coordination. It is a cheap in-memory +// read — triggers.ReadAll already returns a copy — with no I/O and no lock held +// across the loop, and org is served from the caching resolver's memory, as the +// snapshot contract requires (R6). +func (lts *LogTriggerService) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { + triggers := lts.triggers.ReadAll() + + // Dedup by physicalFilterID, keeping the filter owned by the lowest + // triggerID for deterministic org attribution of a shared filter. + type physicalAgg struct { + f filter + lowestTriggerID string + } + byPhysical := make(map[string]*physicalAgg, len(triggers)) + for triggerID, state := range triggers { + agg, ok := byPhysical[state.physicalFilterID] + if !ok { + byPhysical[state.physicalFilterID] = &physicalAgg{f: state.filter, lowestTriggerID: triggerID} + continue + } + if triggerID < agg.lowestTriggerID { + agg.lowestTriggerID = triggerID + agg.f = state.filter + } + } + + entries := make([]resourcemanager.SnapshotEntry, 0, len(byPhysical)) + for _, agg := range byPhysical { + f := agg.f + orgID := f.orgID + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: lts.identity(f.donID), + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(f.reservedAddressCount, resourcemanager.UtilizationFields{ + ResourceType: MeteringResourceType, + ResourceID: f.physicalFilterID, + OrgID: orgID, + }), + }, + }) + } + return entries +} + func (lts *LogTriggerService) startPolling(ctx context.Context, telemetryContext monitoring.TelemetryContext, triggerID string, input *evmcappb.FilterLogTriggerRequest, logCh chan capabilities.TriggerAndId[*evmcappb.Log]) { lts.lggr.Infof("Starting polling for triggerID: %s, interval: %d", triggerID, lts.logTriggerPollInterval) ticker := defaultTickerFactory.NewTicker(lts.logTriggerPollInterval) @@ -496,19 +726,14 @@ func (lts *LogTriggerService) sendLogsToWorkflows(ctx context.Context, telemetry events.KeyWorkflowName, displayWorkflowName, ) - // Emit the *sending* capability DON ID. The trigger plugin runs on a capability - // DON (e.g. chain_capabilities_zone-a), separate from the consumer workflow's - // DON carried in RequestMetadata.WorkflowDonID. The workflow service needs the - // sender's DON to resolve on-chain quorum params (N, F). See CRE-4409. - // capabilityDonID is 0 when the host could not resolve it authoritatively - // (a multi-DON job-spec node, or a core node that pre-dates CRE-4409); in - // that case we fall back to WorkflowDonID. This fallback is permanent, not - // transitional, since the job-spec boot path is still supported. - switch { - case lts.capabilityDonID != 0: - labeler = labeler.With(events.KeyDonID, strconv.Itoa(int(lts.capabilityDonID))) - case telemetryContext.WorkflowDonID != 0: - labeler = labeler.With(events.KeyDonID, strconv.Itoa(int(telemetryContext.WorkflowDonID))) + // Emit the *sending* capability DON ID (CRE-4409). resolveDONID applies + // contract rule 8: the authoritative host-injected CapDONID (carried on + // baseIdentity) wins, and the consumer workflow's WorkflowDonID is used + // only when CapDONID is 0. This is the SAME resolver the metering + // identity uses (filter.donID is set from resolveDONID at registration), + // so the event label and the meter record cannot diverge. + if donID := lts.resolveDONID(telemetryContext.WorkflowDonID); donID != "" { + labeler = labeler.With(events.KeyDonID, donID) } if telemetryContext.WorkflowDonConfigVersion != 0 { labeler = labeler.With(events.KeyDonVersion, strconv.Itoa(int(telemetryContext.WorkflowDonConfigVersion))) @@ -741,8 +966,18 @@ func (lts *LogTriggerService) UnregisterLogTrigger(ctx context.Context, triggerI } lts.lggr.Infof("UnregisterLogTrigger triggerID: %s", triggerID) trigger.cancelFunc() - lts.triggers.Delete(triggerID) + // Delete under the store lock and learn whether this was the physical + // filter's 1->0 release (no other trigger still holds it). Only that + // transition bills a -delta, mirroring the +delta billed on 0->1 activation. + _, lastForPhysical := lts.triggers.DeleteAndIsLastForPhysical(triggerID, trigger.physicalFilterID) + lts.filterRegisteredAt.Delete(lts.generateFilterID(triggerID)) lts.baseTrigger.UnregisterTrigger(triggerID) + if lastForPhysical { + // 1->0 release of the shared physical filter: bill -addressCount once. + // The value and identity are reused from the stashed filter so this + // -delta reverses the exact +delta the activation billed. + lts.emitDelta(ctx, -trigger.reservedAddressCount, "evm-release", meta.WorkflowID, triggerID, trigger.filter) + } err := lts.EVMService.UnregisterLogTracking(ctx, lts.generateFilterID(triggerID)) if err != nil { diff --git a/chain_capabilities/evm/trigger/trigger_metering_test.go b/chain_capabilities/evm/trigger/trigger_metering_test.go new file mode 100644 index 000000000..d4cffc54f --- /dev/null +++ b/chain_capabilities/evm/trigger/trigger_metering_test.go @@ -0,0 +1,566 @@ +package trigger + +import ( + "bytes" + "context" + "errors" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + evmcappb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm" + evmservice "github.com/smartcontractkit/chainlink-common/pkg/chains/evm" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + evmtypes "github.com/smartcontractkit/chainlink-common/pkg/types/chains/evm" + evmmock "github.com/smartcontractkit/chainlink-common/pkg/types/mocks" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" +) + +const testChainSelector = "5009297550715157269" + +// testBaseIdentity is the producer base identity the metering tests build their +// LogTriggerService with. It carries every coarse dimension so the tests can +// assert each one is populated on the emitted records (the host-injected +// identity contract). DONID is the capability DON; an empty DONID exercises the +// WorkflowDonID fallback. +func testBaseIdentity() resourcemanager.ResourceIdentity { + return resourcemanager.ResourceIdentity{ + Product: "cre", + Tenant: "mainline", + NumericTenantID: "42", + Environment: "staging", + Zone: "wf-zone-a", + Don: &resourcemanager.DonIdentity{DonID: "42", NodeID: "csa-pubkey-hex"}, + Service: MeteringService, + ResourcePool: MeteringResource, + } +} + +// fakeMeterEmitter captures MeterRecords and MeterSnapshots emitted through the +// ResourceManager. The two message types are distinguished by the entity +// attribute the emitter is called with. The manager now emits one MeterSnapshot +// per active resource (no single Snapshot envelope), so snapshots accumulates +// one message per resource. +type fakeMeterEmitter struct { + err error + emitCalls int + records []*meteringpb.MeterRecord + recordDomains []string + snapshots []*meteringpb.MeterSnapshot +} + +func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) error { + f.emitCalls++ + if f.err != nil { + return f.err + } + if isSnapshotEmit(attrKVs) { + var snapshot meteringpb.MeterSnapshot + if err := proto.Unmarshal(body, &snapshot); err != nil { + return err + } + f.snapshots = append(f.snapshots, &snapshot) + return nil + } + var record meteringpb.MeterRecord + if err := proto.Unmarshal(body, &record); err != nil { + return err + } + f.records = append(f.records, &record) + f.recordDomains = append(f.recordDomains, attrString(attrKVs, beholder.AttrKeyDomain)) + return nil +} + +// attrString returns the string value for key in the alternating key/value +// attrs the ResourceManager passes to Emit, or "" if absent. +func attrString(attrKVs []any, key string) string { + for i := 0; i+1 < len(attrKVs); i += 2 { + if attrKVs[i] == key { + if v, ok := attrKVs[i+1].(string); ok { + return v + } + } + } + return "" +} + +// isSnapshotEmit reports whether the emitter attributes name the MeterSnapshot +// entity, so the fake can demux the two message types off the same Emit method. +// The key is beholder.AttrKeyEntity ("beholder_entity") and the value is the +// snapshot entity constant the ResourceManager emits with. +func isSnapshotEmit(attrKVs []any) bool { + for i := 0; i+1 < len(attrKVs); i += 2 { + if attrKVs[i] == beholder.AttrKeyEntity && attrKVs[i+1] == "metering.v1.MeterSnapshot" { + return true + } + } + return false +} + +// newMeteredTriggerObject builds a LogTriggerService whose ResourceManager is +// enabled and wired to a fake emitter. The poll interval is stretched so the +// polling goroutine stays quiet; metering happens on the register, unregister, +// cleanup, snapshot, and close paths only. +func newMeteredTriggerObject(t *testing.T, mockEVM *evmmock.EVMService, store LogTriggerStore) (*LogTriggerService, *fakeMeterEmitter, *clockwork.FakeClock) { + t.Helper() + lts := createTriggerObject(t, mockEVM, store) + lts.logTriggerPollInterval = time.Hour + emitter := &fakeMeterEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + lts.resourceManager = resourcemanager.NewResourceManager(logger.Test(t), + resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + lts.baseIdentity = testBaseIdentity() + lts.chainSelector = testChainSelector + return lts, emitter, clock +} + +// meteringTestInput is a registration request with two filter addresses, so +// tests can tell an address count apart from a hardcoded 1. +func meteringTestInput() *evmcappb.FilterLogTriggerRequest { + return &evmcappb.FilterLogTriggerRequest{ + Addresses: [][]byte{expectedAddress, bytes.Repeat([]byte{0x42}, evmtypes.AddressLength)}, + Topics: topicsWithEventSig0, + } +} + +// assertBaseIdentity checks the six coarse dimensions + service/resource_pool on the +// emitted record identity, proving the host-injected identity is carried. +func assertBaseIdentity(t *testing.T, id *meteringpb.ResourceIdentity) { + t.Helper() + require.NotNil(t, id) + require.Equal(t, "cre", id.GetProduct()) + require.Equal(t, "mainline", id.GetTenant()) + require.Equal(t, "42", id.GetNumericTenantId()) + require.Equal(t, "staging", id.GetEnvironment()) + require.Equal(t, "wf-zone-a", id.GetZone()) + require.Equal(t, "42", id.GetDon().GetDonId()) + require.Equal(t, "csa-pubkey-hex", id.GetDon().GetNodeId()) + require.Equal(t, MeteringService, id.GetService()) + require.Equal(t, MeteringResource, id.GetResourcePool()) +} + +// expectedPhysicalFilterID recomputes the physical filter id for the metering +// test input via the production helper, so the tests assert against the real +// canonicalization rather than a frozen literal. +func expectedPhysicalFilterID(t *testing.T, input *evmcappb.FilterLogTriggerRequest) string { + t.Helper() + svc := &LogTriggerService{} + eventSigs, t2, t3, t4 := svc.getTopics(input) + addrs, err := evmservice.ConvertAddressesFromProto(input.GetAddresses()) + require.NoError(t, err) + sigs, err := evmservice.ConvertHashesFromProto(eventSigs) + require.NoError(t, err) + h2, err := evmservice.ConvertHashesFromProto(t2) + require.NoError(t, err) + h3, err := evmservice.ConvertHashesFromProto(t3) + require.NoError(t, err) + h4, err := evmservice.ConvertHashesFromProto(t4) + require.NoError(t, err) + return physicalFilterID(testChainSelector, addrs, sigs, h2, h3, h4) +} + +func TestLogTriggerMetering_RegisterEmitsPositiveDelta(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + meta := capabilities.RequestMetadata{WorkflowID: "wf-id", WorkflowOwner: "0xOwner"} + _, err := service.RegisterLogTrigger(t.Context(), triggerID, meta, meteringTestInput()) + require.NoError(t, err) + + require.Len(t, emitter.records, 1) + record := emitter.records[0] + assertBaseIdentity(t, record.GetIdentity()) + physID := expectedPhysicalFilterID(t, meteringTestInput()) + require.Equal(t, physID, record.GetUtilizations()[0].GetResourceId(), "resource_id must be the physical filter content hash") + // Producers emit only signed-delta UPDATE records; a fresh registration is + // the physical filter's 0->1 activation and bills +addressCount. + require.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, record.GetAction()) + require.Len(t, record.GetUtilizations(), 1) + require.Equal(t, "2", record.GetUtilizations()[0].GetValue(), "activation delta must equal the filter address count") + require.Equal(t, MeteringResourceType, record.GetUtilizations()[0].GetResourceType()) + require.NotEmpty(t, record.GetUtilizations()[0].GetEventId(), "event_id is stamped per emission") + // The record carries the cll-meter billing domain. + require.Equal(t, "cll-meter", emitter.recordDomains[0]) + // The metering identity DON and the events.KeyDonID label derive from the + // same resolveDONID, so they cannot diverge. + require.Equal(t, service.resolveDONID(meta.WorkflowDonID), record.GetIdentity().GetDon().GetDonId()) +} + +func TestLogTriggerMetering_DonIDFallbackToWorkflowDon(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + // Host did not inject a capability DON; the consumer's WorkflowDonID is the + // documented fallback resolved at emit time. + service.baseIdentity.Don = &resourcemanager.DonIdentity{NodeID: "csa-pubkey-hex"} + + meta := capabilities.RequestMetadata{WorkflowID: "wf-id", WorkflowOwner: "0xOwner", WorkflowDonID: 7} + _, err := service.RegisterLogTrigger(t.Context(), triggerID, meta, meteringTestInput()) + require.NoError(t, err) + + require.Len(t, emitter.records, 1) + require.Equal(t, "7", emitter.records[0].GetIdentity().GetDon().GetDonId(), "empty capability DON must fall back to WorkflowDonID") +} + +func TestLogTriggerMetering_NoReserveOnRegisterFailure(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(errors.New("mocked register failure")).Once() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + _, err := service.RegisterLogTrigger(t.Context(), triggerID, capabilities.RequestMetadata{WorkflowID: "wf-id"}, meteringTestInput()) + require.Error(t, err) + require.Zero(t, emitter.emitCalls, "no RESERVE may be emitted for a failed registration") +} + +func TestLogTriggerMetering_ReleaseOnUnregister(t *testing.T) { + meta := capabilities.RequestMetadata{WorkflowID: "wf-id", WorkflowOwner: "0xOwner"} + + registerTrigger := func(t *testing.T, service *LogTriggerService) { + t.Helper() + _, err := service.RegisterLogTrigger(t.Context(), triggerID, meta, meteringTestInput()) + require.NoError(t, err) + // The trigger state (holding the reserved address count) is written by + // the polling goroutine; wait for it before unregistering. + require.Eventually(t, func() bool { + _, ok := service.triggers.Read(triggerID) + return ok + }, time.Second, time.Millisecond) + } + + assertRelease := func(t *testing.T, service *LogTriggerService, record *meteringpb.MeterRecord) { + t.Helper() + assertBaseIdentity(t, record.GetIdentity()) + require.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, record.GetAction()) + require.Len(t, record.GetUtilizations(), 1) + require.Equal(t, "-2", record.GetUtilizations()[0].GetValue(), "the 1->0 release delta negates the activation value") + physID := expectedPhysicalFilterID(t, meteringTestInput()) + require.Equal(t, physID, record.GetUtilizations()[0].GetResourceId()) + } + + t.Run("release negates the activation", func(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + evmService.On("UnregisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + registerTrigger(t, service) + require.NoError(t, service.UnregisterLogTrigger(t.Context(), triggerID, meta, &evmcappb.FilterLogTriggerRequest{})) + + require.Len(t, emitter.records, 2) + require.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, emitter.records[0].GetAction()) + require.Equal(t, "2", emitter.records[0].GetUtilizations()[0].GetValue()) + assertRelease(t, service, emitter.records[1]) + require.Equal(t, emitter.records[0].GetUtilizations()[0].GetResourceId(), emitter.records[1].GetUtilizations()[0].GetResourceId(), + "activation and release must share one physical resource_id") + require.NotEqual(t, emitter.records[0].GetUtilizations()[0].GetEventId(), emitter.records[1].GetUtilizations()[0].GetEventId(), + "each emission gets a distinct event_id") + }) + + t.Run("release emitted even when UnregisterLogTracking fails", func(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + evmService.On("UnregisterLogTracking", mock.Anything, mock.Anything).Return(errors.New("mocked unregister failure")).Once() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + registerTrigger(t, service) + // The -delta is emitted here (from the stashed count) before the + // UnregisterLogTracking RPC. If the RPC fails the filter is orphaned at + // the log poller; the cleanup thread unregisters it silently, emitting + // no further metering record. + require.Error(t, service.UnregisterLogTrigger(t.Context(), triggerID, meta, &evmcappb.FilterLogTriggerRequest{})) + + require.Len(t, emitter.records, 2) + assertRelease(t, service, emitter.records[1]) + }) +} + +func TestLogTriggerMetering_OrphanCleanupEmitsNothing(t *testing.T) { + // Orphan cleanup is log-poller filter hygiene, never a metering event. A + // lost reservation is reconciled by the resource's absence from subsequent + // Snapshots (the liveness mechanism), not by a synthetic cleanup RELEASE. + t.Run("stale filter cleanup emits no meter record", func(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + store := NewLogTriggerStore() + service, emitter, _ := newMeteredTriggerObject(t, mockEVM, store) + + liveFilterID := service.generateFilterID("live-trigger") + staleFilterID := service.generateFilterID("stale-trigger") + mockEVM.On("GetFiltersNames", mock.Anything).Return([]string{liveFilterID, staleFilterID}, nil).Once() + mockEVM.On("UnregisterLogTracking", mock.Anything, staleFilterID).Return(nil).Once() + // mimicking there's a live trigger with the filter registered to log poller + store.Write("live-trigger", logTriggerState{filter: filter{filterID: liveFilterID}}) + + service.cleanUpStaleFilters(t.Context()) + + require.Zero(t, emitter.emitCalls, "orphan cleanup must not emit any MeterRecord") + }) + + t.Run("emits nothing when cleanup unregister fails", func(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + service, emitter, _ := newMeteredTriggerObject(t, mockEVM, NewLogTriggerStore()) + + staleFilterID := service.generateFilterID("stale-trigger") + mockEVM.On("GetFiltersNames", mock.Anything).Return([]string{staleFilterID}, nil).Once() + mockEVM.On("UnregisterLogTracking", mock.Anything, staleFilterID).Return(errors.New("mocked cleanup failure")).Once() + + service.cleanUpStaleFilters(t.Context()) + require.Zero(t, emitter.emitCalls, "orphan cleanup never emits a meter record") + }) +} + +func TestLogTriggerMetering_FailOpen(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + emitter.err = errors.New("mocked emitter failure") + + _, err := service.RegisterLogTrigger(t.Context(), triggerID, capabilities.RequestMetadata{WorkflowID: "wf-id"}, meteringTestInput()) + require.NoError(t, err, "a metering emit failure must never fail registration") + require.Equal(t, 1, emitter.emitCalls, "the emit was attempted and its failure swallowed") +} + +// TestPhysicalFilterID_Canonicalization proves the content hash is independent +// of the order addresses / event sigs / per-slot topic values are supplied, and +// independent of which workflow or trigger registered the filter, while staying +// sensitive to the positional topic slot. +func TestPhysicalFilterID_Canonicalization(t *testing.T) { + addrA := evmtypes.Address(expectedAddress) + addrB := evmtypes.Address(bytes.Repeat([]byte{0x42}, evmtypes.AddressLength)) + sig1 := evmtypes.Hash(eventSig0Example) + sig2 := evmtypes.Hash(bytes.Repeat([]byte{0x11}, evmtypes.HashLength)) + none := []evmtypes.Hash{} + + t.Run("address order does not change the id", func(t *testing.T) { + id1 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA, addrB}, []evmtypes.Hash{sig1}, none, none, none) + id2 := physicalFilterID(testChainSelector, []evmtypes.Address{addrB, addrA}, []evmtypes.Hash{sig1}, none, none, none) + require.Equal(t, id1, id2) + }) + + t.Run("event sig order does not change the id", func(t *testing.T) { + id1 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1, sig2}, none, none, none) + id2 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig2, sig1}, none, none, none) + require.Equal(t, id1, id2) + }) + + t.Run("topic values within a slot are order-independent", func(t *testing.T) { + id1 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, []evmtypes.Hash{sig1, sig2}, none, none) + id2 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, []evmtypes.Hash{sig2, sig1}, none, none) + require.Equal(t, id1, id2) + }) + + t.Run("topic slots are positional", func(t *testing.T) { + inSlot2 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, []evmtypes.Hash{sig2}, none, none) + inSlot3 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, none, []evmtypes.Hash{sig2}, none) + require.NotEqual(t, inSlot2, inSlot3, "the same value in topic2 vs topic3 is a different filter") + }) + + t.Run("different chain selector changes the id", func(t *testing.T) { + id1 := physicalFilterID(testChainSelector, []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, none, none, none) + id2 := physicalFilterID("999", []evmtypes.Address{addrA}, []evmtypes.Hash{sig1}, none, none, none) + require.NotEqual(t, id1, id2) + }) + + t.Run("identical filters from different workflows/triggers share one billed resource", func(t *testing.T) { + // physicalFilterID takes only physical criteria; workflow/trigger are not + // inputs. Two registrations with identical criteria collide by + // construction, so only the first (the 0->1 activation) bills a delta; + // the second shares the already-active physical filter and emits nothing. + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Twice() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Twice() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + _, err := service.RegisterLogTrigger(t.Context(), "trigger-A", + capabilities.RequestMetadata{WorkflowID: "wf-1", WorkflowOwner: "0xOwner"}, meteringTestInput()) + require.NoError(t, err) + _, err = service.RegisterLogTrigger(t.Context(), "trigger-B", + capabilities.RequestMetadata{WorkflowID: "wf-2", WorkflowOwner: "0xOther"}, meteringTestInput()) + require.NoError(t, err) + + require.Len(t, emitter.records, 1, "the shared physical filter is billed once (only the 0->1 activation)") + require.Equal(t, expectedPhysicalFilterID(t, meteringTestInput()), emitter.records[0].GetUtilizations()[0].GetResourceId()) + }) +} + +// TestLogTriggerMetering_SharedFilterRefcount asserts the derived 0<->1 refcount +// billing for a physical filter shared by two triggers: the first register bills +// +addressCount (0->1), the second register bills nothing (1->2), the first +// unregister bills nothing (2->1), and the last unregister bills -addressCount +// (1->0). +func TestLogTriggerMetering_SharedFilterRefcount(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Twice() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Twice() + evmService.On("UnregisterLogTracking", mock.Anything, mock.Anything).Return(nil) + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + + physID := expectedPhysicalFilterID(t, meteringTestInput()) + + // trigger-A: 0->1 activation bills +2. + _, err := service.RegisterLogTrigger(t.Context(), "trigger-A", + capabilities.RequestMetadata{WorkflowID: "wf-1", WorkflowOwner: "0xOwner"}, meteringTestInput()) + require.NoError(t, err) + require.Len(t, emitter.records, 1) + require.Equal(t, "2", emitter.records[0].GetUtilizations()[0].GetValue()) + + // trigger-B shares the same physical filter: 1->2, bills nothing. + _, err = service.RegisterLogTrigger(t.Context(), "trigger-B", + capabilities.RequestMetadata{WorkflowID: "wf-2", WorkflowOwner: "0xOther"}, meteringTestInput()) + require.NoError(t, err) + require.Len(t, emitter.records, 1, "a second holder of the same physical filter bills nothing") + + // Unregister trigger-A: 2->1, still held by trigger-B, bills nothing. + require.NoError(t, service.UnregisterLogTrigger(t.Context(), "trigger-A", capabilities.RequestMetadata{}, &evmcappb.FilterLogTriggerRequest{})) + require.Len(t, emitter.records, 1, "releasing one of two holders bills nothing") + + // Unregister trigger-B: 1->0, bills -2. + require.NoError(t, service.UnregisterLogTrigger(t.Context(), "trigger-B", capabilities.RequestMetadata{}, &evmcappb.FilterLogTriggerRequest{})) + require.Len(t, emitter.records, 2) + require.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, emitter.records[1].GetAction()) + require.Equal(t, "-2", emitter.records[1].GetUtilizations()[0].GetValue()) + require.Equal(t, physID, emitter.records[1].GetUtilizations()[0].GetResourceId()) + + // All emitted event_ids are distinct. + require.NotEqual(t, emitter.records[0].GetUtilizations()[0].GetEventId(), emitter.records[1].GetUtilizations()[0].GetEventId()) +} + +// TestLogTriggerMetering_Snapshot drives one snapshot tick and asserts one +// MeterSnapshot per active filter, each fully identified by its +// ResourceIdentity (physical resource_id) with the right value. The manager +// emits one MeterSnapshot message per resource; there is no label metadata, so +// snapshots are keyed by their physical resource_id. +func TestLogTriggerMetering_Snapshot(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + store := NewLogTriggerStore() + service, emitter, clock := newMeteredTriggerObject(t, mockEVM, store) + + physA := expectedPhysicalFilterID(t, meteringTestInput()) + store.Write("trigger-A", logTriggerState{filter: filter{ + filterID: service.generateFilterID("trigger-A"), + physicalFilterID: physA, + reservedAddressCount: 2, + donID: "42", + }}) + store.Write("trigger-B", logTriggerState{filter: filter{ + filterID: service.generateFilterID("trigger-B"), + physicalFilterID: "physB", + reservedAddressCount: 5, + donID: "42", + }}) + + unregister := service.resourceManager.Register(service) + t.Cleanup(unregister) + servicetest.Run(t, service.resourceManager) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(emitter.snapshots) == 2 + }, time.Second, time.Millisecond) + + require.Len(t, emitter.snapshots, 2, "one MeterSnapshot per active filter") + + byResourceID := map[string]*meteringpb.MeterSnapshot{} + for _, s := range emitter.snapshots { + assertBaseIdentity(t, s.GetIdentity()) + byResourceID[s.GetUtilization()[0].GetResourceId()] = s + } + + a := byResourceID[physA] + require.NotNil(t, a) + require.Equal(t, "2", a.GetUtilization()[0].GetValue()) + require.Equal(t, MeteringResourceType, a.GetUtilization()[0].GetResourceType()) + + b := byResourceID["physB"] + require.NotNil(t, b) + require.Equal(t, "5", b.GetUtilization()[0].GetValue()) +} + +// TestLogTriggerMetering_Snapshot_NothingActive asserts an empty store emits no +// snapshots: billing zeroes a resource out by its absence from later snapshots. +func TestLogTriggerMetering_Snapshot_NothingActive(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + service, emitter, clock := newMeteredTriggerObject(t, mockEVM, NewLogTriggerStore()) + + unregister := service.resourceManager.Register(service) + t.Cleanup(unregister) + servicetest.Run(t, service.resourceManager) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Empty(t, emitter.snapshots, "an empty store emits no MeterSnapshot") +} + +// TestLogTriggerMetering_NoShutdownEmissions asserts that a graceful Close emits +// NO meter records. Process-lifecycle emissions are deleted by design: an active +// filter is released by its absence from the next snapshot, not by a close-time +// drain. +func TestLogTriggerMetering_NoShutdownEmissions(t *testing.T) { + evmService := initMocks(t) + evmService.EXPECT().GetLatestLPBlock(mock.Anything).Return(&finalizedExpBlock, nil).Once() + evmService.On("RegisterLogTracking", mock.Anything, mock.Anything).Return(nil).Once() + evmService.EXPECT().GetFiltersNames(mock.Anything).Return([]string{}, nil).Maybe() + service, emitter, _ := newMeteredTriggerObject(t, evmService, NewLogTriggerStore()) + require.NoError(t, service.Start(t.Context())) + + _, err := service.RegisterLogTrigger(t.Context(), triggerID, + capabilities.RequestMetadata{WorkflowID: "wf", WorkflowOwner: "0xOwner"}, meteringTestInput()) + require.NoError(t, err) + require.Len(t, emitter.records, 1, "registration bills a +delta") + + recordsBefore := len(emitter.records) + require.NoError(t, service.Close()) + require.Len(t, emitter.records, recordsBefore, "graceful close must emit no meter records") +} + +// TestLogTriggerMetering_SnapshotDedup asserts GetUtilization emits one entry +// per DISTINCT physical filter (not per trigger registration): two triggers +// sharing one physicalFilterID snapshot as a single resource. +func TestLogTriggerMetering_SnapshotDedup(t *testing.T) { + mockEVM := evmmock.NewEVMService(t) + store := NewLogTriggerStore() + service, emitter, clock := newMeteredTriggerObject(t, mockEVM, store) + + physShared := expectedPhysicalFilterID(t, meteringTestInput()) + // Two triggers share one physical filter. + store.Write("trigger-A", logTriggerState{filter: filter{ + filterID: service.generateFilterID("trigger-A"), physicalFilterID: physShared, reservedAddressCount: 2, donID: "42", + }}) + store.Write("trigger-B", logTriggerState{filter: filter{ + filterID: service.generateFilterID("trigger-B"), physicalFilterID: physShared, reservedAddressCount: 2, donID: "42", + }}) + + unregister := service.resourceManager.Register(service) + t.Cleanup(unregister) + servicetest.Run(t, service.resourceManager) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(emitter.snapshots) == 1 + }, time.Second, time.Millisecond) + require.Len(t, emitter.snapshots, 1, "two triggers sharing one physical filter snapshot once") + require.Equal(t, physShared, emitter.snapshots[0].GetUtilization()[0].GetResourceId()) + require.Equal(t, "2", emitter.snapshots[0].GetUtilization()[0].GetValue()) +} diff --git a/chain_capabilities/evm/trigger/trigger_test.go b/chain_capabilities/evm/trigger/trigger_test.go index 9cccae9a0..f91b28d79 100644 --- a/chain_capabilities/evm/trigger/trigger_test.go +++ b/chain_capabilities/evm/trigger/trigger_test.go @@ -30,6 +30,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" evmcappb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/chain-capabilities/evm" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" evmtypes "github.com/smartcontractkit/chainlink-common/pkg/types/chains/evm" evmmock "github.com/smartcontractkit/chainlink-common/pkg/types/mocks" @@ -1261,33 +1262,33 @@ func TestNewLogTriggerService(t *testing.T) { t.Run("empty capability id", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, "", 0, beholderProcessor, messageBuilder, time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, capabilities.NewMemEventStore()) + _, err := NewLogTriggerService(evmService, store, lggr, "", beholderProcessor, messageBuilder, time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "capabilityID must be non-empty") }) t.Run("ok initialize interval", func(t *testing.T) { - trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, 10*time.Second, 0, 0, testLimitsFactory(t), nil, capabilities.NewMemEventStore()) + trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, beholderProcessor, messageBuilder, 10*time.Second, 0, 0, testLimitsFactory(t), nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.NoError(t, err) require.Equal(t, 10*time.Second, trigger.logTriggerPollInterval) require.Equal(t, uint64(1000), trigger.logTriggerSendChannelBufferSize) require.Equal(t, uint64(1000), trigger.limitAndSort.Limit.Count) }) t.Run("ok initialize all params", func(t *testing.T) { - trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, 10*time.Second, 100, 50, testLimitsFactory(t), nil, capabilities.NewMemEventStore()) + trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, beholderProcessor, messageBuilder, 10*time.Second, 100, 50, testLimitsFactory(t), nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.NoError(t, err) require.Equal(t, 10*time.Second, trigger.logTriggerPollInterval) require.Equal(t, uint64(100), trigger.logTriggerSendChannelBufferSize) require.Equal(t, uint64(50), trigger.limitAndSort.Limit.Count) }) t.Run("ok initialize buffer only", func(t *testing.T) { - trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, 10*time.Second, 10000, 0, testLimitsFactory(t), nil, capabilities.NewMemEventStore()) + trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, beholderProcessor, messageBuilder, 10*time.Second, 10000, 0, testLimitsFactory(t), nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.NoError(t, err) require.Equal(t, 10*time.Second, trigger.logTriggerPollInterval) require.Equal(t, uint64(10000), trigger.logTriggerSendChannelBufferSize) require.Equal(t, uint64(defaultLimitQueryLogSize), trigger.limitAndSort.Limit.Count) //default value for limit as 0 was provided }) t.Run("ok initialize query limit only", func(t *testing.T) { - trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, 10*time.Second, 0, 100, testLimitsFactory(t), nil, capabilities.NewMemEventStore()) + trigger, err := NewLogTriggerService(evmService, store, logger.Test(t), testLogTriggerCapabilityID, beholderProcessor, messageBuilder, 10*time.Second, 0, 100, testLimitsFactory(t), nil, capabilities.NewMemEventStore(), nil, resourcemanager.ResourceIdentity{}, 0) require.NoError(t, err) require.Equal(t, 10*time.Second, trigger.logTriggerPollInterval) require.Equal(t, uint64(defaultSendChannelBufferSize), trigger.logTriggerSendChannelBufferSize) //default value for buffer size as 0 was provided @@ -1296,25 +1297,25 @@ func TestNewLogTriggerService(t *testing.T) { // negative tests t.Run("negative poll interval", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, -1*time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, nil) + _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, beholderProcessor, messageBuilder, -1*time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, nil, nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "logTriggerPollInterval must be positive, got: -1s") }) t.Run("limit query log size >= send channel buffer size", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, time.Second, 5, 10, limits.Factory{Logger: lggr}, nil, nil) + _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, beholderProcessor, messageBuilder, time.Second, 5, 10, limits.Factory{Logger: lggr}, nil, nil, nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "logTriggerLimitQueryLogSize (10) must be less than logTriggerSendChannelBufferSize (5)") }) t.Run("limit query log size >= default send channel buffer size", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, time.Second, 0, defaultSendChannelBufferSize+1, limits.Factory{Logger: lggr}, nil, nil) + _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, beholderProcessor, messageBuilder, time.Second, 0, defaultSendChannelBufferSize+1, limits.Factory{Logger: lggr}, nil, nil, nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "logTriggerLimitQueryLogSize (1001) must be less than logTriggerSendChannelBufferSize (1000)") }) t.Run("nil trigger event store", func(t *testing.T) { lggr := logger.Test(t) - _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, 0, beholderProcessor, messageBuilder, time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, nil) + _, err := NewLogTriggerService(evmService, store, lggr, testLogTriggerCapabilityID, beholderProcessor, messageBuilder, time.Second, 0, 0, limits.Factory{Logger: lggr}, nil, nil, nil, resourcemanager.ResourceIdentity{}, 0) require.Error(t, err) require.Contains(t, err.Error(), "no trigger event store provided") }) diff --git a/cron/go.mod b/cron/go.mod index d76d4aae6..9a1eeae18 100644 --- a/cron/go.mod +++ b/cron/go.mod @@ -2,13 +2,21 @@ module github.com/smartcontractkit/capabilities/cron go 1.26.2 +// Unpublished local stack for SHARED-2709; drop once chainlink-common and +// chainlink-protos/metering/go are tagged. +replace ( + github.com/smartcontractkit/chainlink-common => ../../chainlink-common + github.com/smartcontractkit/chainlink-protos/metering/go => ../../chainlink-protos/metering/go +) + require ( github.com/go-co-op/gocron/v2 v2.18.0 github.com/google/uuid v1.6.0 github.com/jonboulle/clockwork v0.5.0 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260529092756-a94bc8ce96d6 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260526195338-adcf8013a1b7 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 @@ -82,10 +90,10 @@ require ( github.com/scylladb/go-reflectx v1.0.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/smartcontractkit/chain-selectors v1.0.100 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect - github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260323124644-faea187e6997 // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect + github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect github.com/smartcontractkit/libocr v0.0.0-20250912173940-f3ab0246e23d // indirect diff --git a/cron/go.sum b/cron/go.sum index f6f3aad2a..6b0f0b36a 100644 --- a/cron/go.sum +++ b/cron/go.sum @@ -213,18 +213,18 @@ github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924/go.mod h1:v0O0Au8RE00Z89QxBE6I2q9bR9r3+RO1gLD3oaO2WB0= github.com/smartcontractkit/chain-selectors v1.0.100 h1:wpiSpmI/eFjY+wx/nPr5VuNF4hki0prIBMKEaQWn3g4= github.com/smartcontractkit/chain-selectors v1.0.100/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260529092756-a94bc8ce96d6 h1:hms02zQQ0BPcp9CBwh/xda5KwJWdU0IIA/yjtwyRoA4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260529092756-a94bc8ce96d6/go.mod h1:jueIfDkkRexwGgLbVB7vGCZlNtd383zuwi4uHHwcbqc= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 h1:ucHu2bPDT/58AzSgnPDyp4IjnjVbrVWYD3bG5jCbXMY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260526195338-adcf8013a1b7 h1:iljEJss3WOwcsMkWy72Yn2zvjw7Gyxc+RXL7r8YKM6g= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260526195338-adcf8013a1b7/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b h1:VDgJWDipihV9f7M5+d21d1RzSsg5rEv+iI12oN1VQbo= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260323124644-faea187e6997 h1:W0HKHO8eE8BckTRnhSdqjHKbJcnk068nEWYnWRu6tJY= -github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260323124644-faea187e6997/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= +github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e h1:Hv9Mww35LrufCdM9wtS9yVi/rEWGI1UnjHbcKKU0nVY= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e/go.mod h1:T4zH9R8R8lVWKfU7tUvYz2o2jMv1OpGCdpY2j2QZXzU= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= diff --git a/cron/main.go b/cron/main.go index a085c0391..e61f006a5 100644 --- a/cron/main.go +++ b/cron/main.go @@ -4,16 +4,52 @@ import ( "github.com/smartcontractkit/capabilities/cron/trigger" "github.com/smartcontractkit/capabilities/libs/loopserver" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/cron/server" "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" ) +type meteringConfig struct { + meterRecordsEnabled bool + meterSnapshotsEnabled bool + deployment resourcemanager.DeploymentIdentity +} + +func newMeteringConfig(env loop.EnvConfig) meteringConfig { + return meteringConfig{ + meterRecordsEnabled: env.MeterRecordsEnabled, + meterSnapshotsEnabled: env.MeterSnapshotsEnabled, + deployment: resourcemanager.DeploymentIdentity{ + Product: env.MeterProduct, + Tenant: env.MeterTenant, + NumericTenantID: env.MeterNumericTenantID, + Environment: env.MeterEnvironment, + Zone: env.MeterZone, + NodeID: env.MeterNodeID, + }, + } +} + +func (m meteringConfig) resourceManagerConfig() resourcemanager.ResourceManagerConfig { + return resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: m.meterRecordsEnabled, + MeterSnapshotsEnabled: m.meterSnapshotsEnabled, + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + } +} + func main() { loopserver.ServeNew(trigger.ServiceName, func(s *loop.Server) loop.StandardCapabilities { - triggerService, err := trigger.NewTriggerService(s.Logger, nil, s.LimitsFactory) + meteringCfg := newMeteringConfig(s.EnvConfig) + meters := resourcemanager.NewResourceManager(s.Logger, meteringCfg.resourceManagerConfig()) + + triggerService, err := trigger.NewTriggerService(s.Logger, nil, s.LimitsFactory, meters) if err != nil { s.Logger.Fatalw("Failed to create cron trigger service", "error", err) } + triggerService.Deployment = meteringCfg.deployment return server.NewCronServer(triggerService) }, loop.WithOtelViews(trigger.MetricViews())) diff --git a/cron/trigger/metering_test.go b/cron/trigger/metering_test.go new file mode 100644 index 000000000..c4df305bd --- /dev/null +++ b/cron/trigger/metering_test.go @@ -0,0 +1,404 @@ +package trigger + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + crontypedapi "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/cron" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" +) + +// fakeMeterEmitter captures metering records delivered through a real +// ResourceManager, so tests assert on exactly the bytes production would emit. +// It demultiplexes MeterRecord and MeterSnapshot messages by their beholder +// entity attribute. A non-nil err simulates delivery failure: nothing is +// recorded. +type fakeMeterEmitter struct { + mu sync.Mutex + err error + records []*meteringpb.MeterRecord + recordDomains []string + snapshots []*meteringpb.MeterSnapshot +} + +func (f *fakeMeterEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.err != nil { + return f.err + } + if attrValue(attrKVs, "beholder_entity") == "metering.v1.MeterSnapshot" { + snapshot := &meteringpb.MeterSnapshot{} + if err := proto.Unmarshal(body, snapshot); err != nil { + return err + } + f.snapshots = append(f.snapshots, snapshot) + return nil + } + record := &meteringpb.MeterRecord{} + if err := proto.Unmarshal(body, record); err != nil { + return err + } + f.records = append(f.records, record) + f.recordDomains = append(f.recordDomains, attrValue(attrKVs, "beholder_domain")) + return nil +} + +// attrValue extracts a beholder attribute value by key from the variadic +// key/value attrs the ResourceManager passes to Emit. +func attrValue(attrKVs []any, key string) string { + for i := 0; i+1 < len(attrKVs); i += 2 { + if k, ok := attrKVs[i].(string); ok && k == key { + if v, ok := attrKVs[i+1].(string); ok { + return v + } + } + } + return "" +} + +func (f *fakeMeterEmitter) Records() []*meteringpb.MeterRecord { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*meteringpb.MeterRecord(nil), f.records...) +} + +func (f *fakeMeterEmitter) Snapshots() []*meteringpb.MeterSnapshot { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*meteringpb.MeterSnapshot(nil), f.snapshots...) +} + +// meteredTestDeps are the host-injected dependencies used by metering tests. +// The DON dimension still arrives via the Initialise channel; the +// deployment/node dimensions now arrive via loop.EnvConfig (meteredTestDeployment). +var meteredTestDeps = core.StandardCapabilitiesDependencies{ + CapabilityDonID: 7, +} + +// meteredTestDeployment is the deployment/node identity that main would source +// from loop.EnvConfig and set on the service before Initialise. +var meteredTestDeployment = resourcemanager.DeploymentIdentity{ + Product: "cre-mainline", + Tenant: "mainline", + NumericTenantID: "42", + Environment: "staging", + Zone: "wf-zone-a", + NodeID: "clp-cre-wf-zone-a-1", +} + +// expectedBaseIdentity is the base identity the Service builds from +// meteredTestDeps (resource_id left empty; set per trigger). +var expectedBaseIdentity = resourcemanager.ResourceIdentity{ + Product: "cre-mainline", + Tenant: "mainline", + NumericTenantID: "42", + Environment: "staging", + Zone: "wf-zone-a", + Don: &resourcemanager.DonIdentity{DonID: "7", NodeID: "clp-cre-wf-zone-a-1"}, + Service: "cron-trigger", + ResourcePool: "trigger_registrations", +} + +// newMeteredTriggerService builds an initialised trigger service whose +// ResourceManager is enabled and wired to emitter, with identity sourced from +// meteredTestDeps. Snapshots use a fake clock so tests advance the tick +// deterministically. +func newMeteredTriggerService(t *testing.T, clock clockwork.Clock, emitter resourcemanager.Emitter) (*Service, *resourcemanager.ResourceManager, *clockwork.FakeClock) { + t.Helper() + + fakeClock, ok := clock.(*clockwork.FakeClock) + if !ok { + fakeClock = clockwork.NewFakeClockAt(clock.Now()) + clock = fakeClock + } + + meters := resourcemanager.NewResourceManager(logger.Nop(), resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + ts, err := NewTriggerService(logger.Nop(), clock, limits.Factory{}, meters) + require.NoError(t, err) + ts.Deployment = meteredTestDeployment + + config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) + require.NoError(t, err) + + deps := meteredTestDeps + deps.Config = string(config) + require.NoError(t, ts.Initialise(t.Context(), deps)) + + return ts, meters, fakeClock +} + +func TestCronTrigger_Metering_RegisterUnregisterDeltas(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + emitter := &fakeMeterEmitter{} + ts, _, _ := newMeteredTriggerService(t, fakeClock, emitter) + + metadata := capabilities.RequestMetadata{ + WorkflowID: workflowID1, + WorkflowOwner: "0xOwner-1", + } + ch, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + records := emitter.Records() + require.Len(t, records, 1, "expected exactly one +1 UPDATE on successful registration") + register := records[0] + // Producers emit only signed-delta UPDATE records; register is a +1 delta. + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, register.GetAction()) + // The record carries the cll-meter billing domain. + assert.Equal(t, "cll-meter", emitter.recordDomains[0]) + + // Identity is populated from host-injected deps and points at the cron + // resource pool. Per-trigger fields are carried on utilization. + id := register.GetIdentity() + require.NotNil(t, id) + assert.Equal(t, "cre-mainline", id.GetProduct()) + assert.Equal(t, "mainline", id.GetTenant()) + assert.Equal(t, "42", id.GetNumericTenantId()) + assert.Equal(t, "staging", id.GetEnvironment()) + assert.Equal(t, "wf-zone-a", id.GetZone()) + assert.Equal(t, "7", id.GetDon().GetDonId()) + assert.Equal(t, "clp-cre-wf-zone-a-1", id.GetDon().GetNodeId()) + assert.Equal(t, "cron-trigger", id.GetService()) + assert.Equal(t, "trigger_registrations", id.GetResourcePool()) + // The metering identity DON and the events.KeyDonID label derive from the + // same resolver, so they cannot diverge. + assert.Equal(t, ts.donID(), id.GetDon().GetDonId()) + + require.Len(t, register.GetUtilizations(), 1) + assert.Equal(t, "1", register.GetUtilizations()[0].GetValue()) + assert.Equal(t, "operations", register.GetUtilizations()[0].GetResourceType()) + assert.Equal(t, triggerID1, register.GetUtilizations()[0].GetResourceId()) + // event_id is the deterministic, cross-node-identical id derived from the + // DON-aggregated request identity (workflowID + triggerID). + wantRegisterID := resourcemanager.EventID("cron-register", workflowID1, triggerID1) + assert.Equal(t, wantRegisterID, register.GetUtilizations()[0].GetEventId()) + + // Cross-node determinism: a second node fielding the identical register + // request derives the identical event_id. + emitter2 := &fakeMeterEmitter{} + ts2, _, _ := newMeteredTriggerService(t, clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), emitter2) + _, capErr2 := ts2.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr2) + require.Len(t, emitter2.Records(), 1) + assert.Equal(t, register.GetUtilizations()[0].GetEventId(), emitter2.Records()[0].GetUtilizations()[0].GetEventId(), + "two nodes fielding the same register must emit the identical event_id") + require.NoError(t, ts2.Close()) + + // Each cron tick re-Writes the trigger to reschedule it; the Write + // happens before the channel send, so after receiving the event the + // callback path has fully run. It must not emit. + for range 3 { + fakeClock.Advance(time.Second) + <-ch + } + require.Len(t, emitter.Records(), 1, "cron tick callbacks must not emit meter records") + + require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) + records = emitter.Records() + require.Len(t, records, 2, "expected exactly one -1 UPDATE on unregistration") + unregister := records[1] + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, unregister.GetAction()) + assert.Equal(t, register.GetUtilizations()[0].GetResourceId(), unregister.GetUtilizations()[0].GetResourceId()) + require.Len(t, unregister.GetUtilizations(), 1) + assert.Equal(t, "-1", unregister.GetUtilizations()[0].GetValue(), "unregister is a signed -1 delta") + + // event_id: unregister is namespaced distinctly from register (so the paired + // +1/-1 deltas never dedup into each other) but is otherwise the same + // deterministic derivation over the DON-consistent workflowID+triggerID. + wantUnregisterID := resourcemanager.EventID("cron-unregister", workflowID1, triggerID1) + assert.Equal(t, wantUnregisterID, unregister.GetUtilizations()[0].GetEventId()) + assert.NotEqual(t, register.GetUtilizations()[0].GetEventId(), unregister.GetUtilizations()[0].GetEventId(), + "register and unregister must have distinct event_ids") + + require.NoError(t, ts.Close()) +} + +func TestCronTrigger_Metering_NoEmitOnFailedPaths(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClock() + emitter := &fakeMeterEmitter{} + ts, _, _ := newMeteredTriggerService(t, fakeClock, emitter) + + metadata := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "owner-1"} + + // Invalid schedule: registration fails before allocation, nothing emitted. + _, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: "not-a-schedule"}) + require.NotNil(t, capErr) + require.Empty(t, emitter.Records()) + + // Unregistering a trigger that was never registered emits no delta. + require.Nil(t, ts.UnregisterTrigger(t.Context(), "missing", metadata, &crontypedapi.Config{Schedule: everySecond})) + require.Empty(t, emitter.Records()) + + // Duplicate registration fails and must not emit a second +1 delta. + _, capErr = ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + _, capErr = ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.NotNil(t, capErr) + require.Len(t, emitter.Records(), 1) + + require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) + require.NoError(t, ts.Close()) +} + +func TestCronTrigger_Metering_FailOpen(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClock() + emitter := &fakeMeterEmitter{err: errors.New("collector unavailable")} + ts, _, _ := newMeteredTriggerService(t, fakeClock, emitter) + + metadata := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "owner-1"} + + // Registration and unregistration succeed even though every emission fails. + ch, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + fakeClock.Advance(time.Second) + <-ch // trigger still fires + + require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) + require.Empty(t, emitter.Records()) + + require.NoError(t, ts.Close()) +} + +// TestCronTrigger_Metering_Snapshot asserts the Service implements Meterable +// such that a forced snapshot emits one MeterSnapshot per active trigger, each +// carrying the full base identity and per-trigger utilization. +func TestCronTrigger_Metering_Snapshot(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + emitter := &fakeMeterEmitter{} + ts, rm, clock := newMeteredTriggerService(t, fakeClock, emitter) + _ = rm + + metadata1 := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "0xOwner-1"} + _, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata1, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + metadata2 := capabilities.RequestMetadata{WorkflowID: "workflow-id-2", WorkflowOwner: "owner-2"} + const triggerID2 = "test-id-2" + _, capErr = ts.RegisterTrigger(t.Context(), triggerID2, metadata2, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + // One MeterSnapshot per active trigger, value 1, full per-resource identity. + require.Eventually(t, func() bool { + return len(emitter.Snapshots()) == 2 + }, time.Second, time.Millisecond) + snapshots := emitter.Snapshots() + require.Len(t, snapshots, 2, "one MeterSnapshot per active trigger per tick") + + byTrigger := map[string]*meteringpb.MeterSnapshot{} + for _, s := range snapshots { + byTrigger[s.GetUtilization()[0].GetResourceId()] = s + } + + s1 := byTrigger[triggerID1] + require.NotNil(t, s1) + assert.Equal(t, "1", s1.GetUtilization()[0].GetValue()) + assert.Equal(t, "cron-trigger", s1.GetIdentity().GetService()) + assert.Equal(t, "trigger_registrations", s1.GetIdentity().GetResourcePool()) + assert.Equal(t, "operations", s1.GetUtilization()[0].GetResourceType()) + + s2 := byTrigger[triggerID2] + require.NotNil(t, s2) + assert.Equal(t, "1", s2.GetUtilization()[0].GetValue()) + assert.Equal(t, triggerID2, s2.GetUtilization()[0].GetResourceId()) + + require.NoError(t, ts.Close()) +} + +// TestCronTrigger_Metering_NoShutdownEmissions asserts that a graceful Close +// emits NO meter records. Process-lifecycle emissions are deleted by design: +// billing releases each still-active registration by its absence from the next +// snapshot, not by a shutdown drain. +func TestCronTrigger_Metering_NoShutdownEmissions(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + emitter := &fakeMeterEmitter{} + ts, _, _ := newMeteredTriggerService(t, fakeClock, emitter) + + metadata1 := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "0xOwner-1"} + _, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata1, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + metadata2 := capabilities.RequestMetadata{WorkflowID: "workflow-id-2", WorkflowOwner: "owner-2"} + const triggerID2 = "test-id-2" + _, capErr = ts.RegisterTrigger(t.Context(), triggerID2, metadata2, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + // Two +1 registration deltas so far. + require.Len(t, emitter.Records(), 2) + + require.NoError(t, ts.Close()) + + // Close emitted nothing: no shutdown drain. + require.Len(t, emitter.Records(), 2, "graceful close must emit no meter records") +} + +// TestCronTrigger_Metering_DonIDFallback asserts the DON ID falls back to the +// consumer workflow's DON when the host has not injected a capability DON ID. +func TestCronTrigger_Metering_DonIDFallback(t *testing.T) { + t.Parallel() + + fakeClock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + emitter := &fakeMeterEmitter{} + + meters := resourcemanager.NewResourceManager(logger.Nop(), resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: true, + Emitter: emitter, + }) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, meters) + require.NoError(t, err) + + config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) + require.NoError(t, err) + + // No CapabilityDonID injected (zero) → fall back to WorkflowDonID at emit. + require.NoError(t, ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{Config: string(config)})) + + metadata := capabilities.RequestMetadata{WorkflowID: workflowID1, WorkflowOwner: "owner-1", WorkflowDonID: 42} + _, capErr := ts.RegisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond}) + require.Nil(t, capErr) + + records := emitter.Records() + require.Len(t, records, 1) + assert.Equal(t, "42", records[0].GetIdentity().GetDon().GetDonId(), "DON ID falls back to WorkflowDonID") + // Metering identity DON and events.KeyDonID label share the same resolver. + assert.Equal(t, ts.donID(), records[0].GetIdentity().GetDon().GetDonId()) + // Product falls back to the cron constant when the host injects none. + assert.Equal(t, "cre", records[0].GetIdentity().GetProduct()) + + require.Nil(t, ts.UnregisterTrigger(t.Context(), triggerID1, metadata, &crontypedapi.Config{Schedule: everySecond})) + require.NoError(t, ts.Close()) +} diff --git a/cron/trigger/store.go b/cron/trigger/store.go index 9484c2970..ee9ed1a90 100644 --- a/cron/trigger/store.go +++ b/cron/trigger/store.go @@ -14,6 +14,7 @@ type CronStore interface { Read(triggerID string) (value cronTrigger, ok bool) ReadAll() (values map[string]cronTrigger) Write(triggerID string, value cronTrigger) + WriteIfPresent(triggerID string, value cronTrigger) (written bool) Delete(triggerID string) } @@ -46,6 +47,23 @@ func (cs *cronStore) Write(triggerID string, value cronTrigger) { cs.triggers[triggerID] = value } +// WriteIfPresent updates triggerID only when it currently exists, performing +// the existence check and the write atomically under the store lock. It returns +// false without writing when the trigger has already been deleted (e.g. by a +// concurrent UnregisterTrigger). The cron task callback uses this so a tick that +// began before an unregister cannot re-insert ("resurrect") a trigger that was +// just removed: snapshot absence is the release signal, so a resurrected entry +// would keep the resource billed after the caller stopped it. +func (cs *cronStore) WriteIfPresent(triggerID string, value cronTrigger) (written bool) { + cs.mu.Lock() + defer cs.mu.Unlock() + if _, ok := cs.triggers[triggerID]; !ok { + return false + } + cs.triggers[triggerID] = value + return true +} + func (cs *cronStore) Delete(triggerID string) { cs.mu.Lock() defer cs.mu.Unlock() diff --git a/cron/trigger/trigger.go b/cron/trigger/trigger.go index 3afd24602..ae9d63ea8 100644 --- a/cron/trigger/trigger.go +++ b/cron/trigger/trigger.go @@ -21,6 +21,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/cron/server" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" @@ -28,6 +29,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types/core" "github.com/smartcontractkit/chainlink-common/pkg/workflows" "github.com/smartcontractkit/chainlink-common/pkg/workflows/events" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" ) const ServiceName = "CronCapabilities" @@ -40,6 +42,18 @@ var cronTriggerInfo = capabilities.MustNewCapabilityInfo( "A trigger that uses a cron schedule to run periodically at fixed times, dates, or intervals.", ) +const ( + // meteringService is the stable service constant for cron trigger + // registrations on emitted MeterRecords and Snapshots. It must not encode + // deployment environment or zone: those are discrete identity dimensions + // delivered via loop.EnvConfig (see Service.Deployment). + meteringService = "cron-trigger" + // meteringResource is the resource pool cron records apply to. + meteringResource = "trigger_registrations" + // meteringResourceType is the billing unit for cron registrations. + meteringResourceType = "operations" +) + type Config struct { FastestScheduleIntervalSeconds int `json:"fastestScheduleIntervalSeconds"` } @@ -50,13 +64,19 @@ type Response struct { } type cronTrigger struct { - job gocron.Job - nextRun time.Time - workflowID string - close func() + job gocron.Job + nextRun time.Time + workflowID string + workflowDonID uint32 + workflowOwner string + orgID string + close func() } type Service struct { + services.Service + srvcEng *services.Engine + capabilities.CapabilityInfo limitsFactory limits.Factory fastestScheduleInterval limits.TimeLimiter @@ -66,7 +86,20 @@ type Service struct { triggers *cronStore labeler custmsg.MessageEmitter metrics *Metrics - orgResolver orgresolver.OrgResolver + meters *resourcemanager.ResourceManager + // unregisterMeterable removes this Service from the ResourceManager's + // snapshot registry; set at start, called at close. Nil until started. + unregisterMeterable func() + // base is the resourcemanager identity for cron registrations, built from + // the deployment/node dimensions (Deployment) and DON dimension at Initialise. + base resourcemanager.ResourceIdentity + // Deployment carries the static deployment/node identity dimensions + // delivered to the plugin process via loop.EnvConfig. It is set once at + // startup (by main, before Initialise) and read when building the base + // metering identity. The zero value is valid and leaves those dimensions + // empty. + Deployment resourcemanager.DeploymentIdentity + orgResolver orgresolver.OrgResolver } func (s *Service) RegisterLegacyTrigger(ctx context.Context, triggerID string, metadata capabilities.RequestMetadata, input *crontypedapi.Config) (<-chan capabilities.TriggerAndId[*crontypedapi.LegacyPayload], caperrors.Error) { //nolint:staticcheck @@ -101,13 +134,21 @@ func (s *Service) UnregisterLegacyTrigger(ctx context.Context, triggerID string, return s.UnregisterTrigger(ctx, triggerID, metadata, input) } -var _ services.Service = &Service{} +var ( + _ services.Service = &Service{} + _ resourcemanager.Meterable = &Service{} +) // NewTriggerService creates a new trigger service. Optionally, a clock can be passed in for testing, if nil // the system clock will be used. The orgResolver is optional and can be nil, but should be set in live environments. -func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFactory limits.Factory) (*Service, error) { +// meters reports trigger registrations for billing; if nil, a disabled no-op manager is used. +func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFactory limits.Factory, meters *resourcemanager.ResourceManager) (*Service, error) { lggr := logger.Named(parentLggr, "CRONTrigger") + if meters == nil { + meters = resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{}) + } + metrics, err := NewMetrics() if err != nil { return nil, fmt.Errorf("error creating metrics: %w", err) @@ -131,7 +172,7 @@ func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFa return nil, fmt.Errorf("error creating scheduler: %w", err) } - return &Service{ + s := &Service{ lggr: lggr, CapabilityInfo: cronTriggerInfo, limitsFactory: limitsFactory, @@ -144,7 +185,48 @@ func NewTriggerService(parentLggr logger.Logger, clock clockwork.Clock, limitsFa "capabilityName", cronTriggerInfo.ID, ), metrics: metrics, - }, nil + meters: meters, + } + + // Adopt services.Engine so the trigger can host the ResourceManager as a + // sub-service (the RM owns the snapshot tick) and shut down cleanly. The + // scheduler is started/stopped in s.start / s.close. + s.Service, s.srvcEng = services.Config{ + Name: "CronTrigger", + NewSubServices: func(logger.Logger) []services.Service { return []services.Service{meters} }, + Start: s.start, + Close: s.close, + }.NewServiceEngine(lggr) + + return s, nil +} + +// donID returns the DON identifier stamped on metering identity and event labels. +func (s *Service) donID() string { + return s.base.DonID() +} + +// utilizationFields builds the per-trigger billing fields shared by the record +// and snapshot paths. resource_id is the workflow-scoped trigger_id; org_id is +// resolved at registration time and stored on the cronTrigger. event_id is +// intentionally absent from the fields: for records the caller passes it +// explicitly (a deterministic cross-node id), and for snapshots the +// ResourceManager derives it from the bucket/resource/node key. +func (s *Service) utilizationFields(triggerID, orgID string) resourcemanager.UtilizationFields { + return resourcemanager.UtilizationFields{ + ResourceType: meteringResourceType, + ResourceID: triggerID, + OrgID: orgID, + } +} + +// emitMeterRecord emits a signed delta MeterRecord (METER_ACTION_UPDATE) for a +// change to the durable cron-registration level: register bills +1, unregister +// bills -1. orgID is resolved by the caller before invoking this method. +// Emission is fail-open and never affects the registration itself. +func (s *Service) emitMeterRecord(ctx context.Context, delta int64, namespace, workflowID, triggerID, orgID string) { + eventID := resourcemanager.EventID(namespace, workflowID, triggerID) + s.meters.EmitDelta(ctx, s.base, eventID, delta, s.utilizationFields(triggerID, orgID)) } func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapabilitiesDependencies) error { @@ -168,9 +250,17 @@ func (s *Service) Initialise(ctx context.Context, dependencies core.StandardCapa } s.fastestScheduleInterval = limiter - s.orgResolver = dependencies.OrgResolver - if s.orgResolver == nil { + if dependencies.OrgResolver == nil { s.lggr.Warn("OrgResolver is nil, cron capability will not be able to fetch organization ID") + } else { + s.orgResolver = dependencies.OrgResolver + } + + // Build the base metering identity. The deployment/node dimensions come from + // s.Deployment (delivered via loop.EnvConfig, set by main before Initialise). + s.base = resourcemanager.NewBaseIdentity(s.Deployment, meteringService, meteringResource) + if dependencies.CapabilityDonID != 0 { + s.base = s.base.WithDonID(strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10)) } err = s.Start(ctx) @@ -279,7 +369,7 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat events.KeyWorkflowExecutionID, workflowExecutionID, events.KeyWorkflowOwner, metadata.WorkflowOwner, events.KeyWorkflowName, displayWorkflowName, - events.KeyDonID, strconv.Itoa(int(metadata.WorkflowDonID)), + events.KeyDonID, s.donID(), events.KeyDonVersion, strconv.Itoa(int(metadata.WorkflowDonConfigVersion)), events.KeyOrganizationID, orgID, events.KeyWorkflowRegistryChainSelector, metadata.WorkflowRegistryChainSelector, @@ -306,12 +396,22 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat if callbackCh == nil { return // unregistered already } - s.triggers.Write(triggerID, cronTrigger{ - job: job, - nextRun: nextExecutionTime, - workflowID: metadata.WorkflowID, - close: closeCh, - }) + // Re-check existence atomically with the write: an unregister that + // ran during this callback (after the Read above) deletes the + // trigger, and resurrecting it here would keep the resource billed + // via snapshots after the caller stopped it. WriteIfPresent skips + // the write when the trigger is already gone. + if written := s.triggers.WriteIfPresent(triggerID, cronTrigger{ + job: job, + nextRun: nextExecutionTime, + workflowID: metadata.WorkflowID, + workflowDonID: metadata.WorkflowDonID, + workflowOwner: metadata.WorkflowOwner, + orgID: trigger.orgID, + close: closeCh, + }); !written { + return // unregistered concurrently; do not resurrect or send + } select { case callbackCh <- response: @@ -349,13 +449,28 @@ func (s *Service) RegisterTrigger(ctx context.Context, triggerID string, metadat return nil, caperrors.NewPublicSystemError(fmt.Errorf("RegisterTrigger failed to remove job: %s", err), caperrors.Internal) } + var orgID string + if s.orgResolver != nil && metadata.WorkflowOwner != "" { + if resolved, err := s.orgResolver.Get(ctx, metadata.WorkflowOwner); err != nil { + logger.Sugared(s.lggr).Warnw("failed to resolve org ID for metering", "owner", metadata.WorkflowOwner, "err", err) + } else { + orgID = resolved + } + } + s.triggers.Write(triggerID, cronTrigger{ - job: job, - nextRun: firstRunTime, - workflowID: metadata.WorkflowID, - close: closeCh, + job: job, + nextRun: firstRunTime, + workflowID: metadata.WorkflowID, + workflowDonID: metadata.WorkflowDonID, + workflowOwner: metadata.WorkflowOwner, + orgID: orgID, + close: closeCh, }) + // Register bills a +1 delta to the durable trigger-registration level. + s.emitMeterRecord(ctx, 1, "cron-register", metadata.WorkflowID, triggerID, orgID) + s.lggr.Debugw("Trigger registered", "workflowId", metadata.WorkflowID, "triggerId", triggerID, "jobId", job.ID()) s.metrics.IncActiveTriggersGauge(ctx) return callbackCh, nil @@ -407,45 +522,69 @@ func (s *Service) UnregisterTrigger(ctx context.Context, triggerID string, metad // Remove from triggers context s.triggers.Delete(triggerID) + // Unregister bills a -1 delta. workflowID and orgID come from the STORED + // trigger (not the unregister request's metadata) so the delta reverses + // the exact identity the register +1 billed. + s.emitMeterRecord(ctx, -1, "cron-unregister", trigger.workflowID, triggerID, trigger.orgID) + s.lggr.Debugw("UnregisterTrigger", "triggerId", triggerID, "jobId", jobID) s.metrics.DecActiveTriggersGauge(ctx) return nil } -// Start the service. -func (s *Service) Start(ctx context.Context) error { +// start is the services.Engine start hook. The ResourceManager sub-service has +// already been started by the engine, so start registers this Service as a +// Meterable (the RM polls it once per snapshot tick) and starts the scheduler, +// refreshing next-run times for any registrations that survived a restart. +func (s *Service) start(ctx context.Context) error { if s.scheduler == nil { return errors.New("service has shutdown, it must be built again to restart") } + // Register for snapshots. The RM owns the tick; we only supply state via + // the Meterable interface. unregisterMeterable is called in close. + s.unregisterMeterable = s.meters.Register(s) + s.scheduler.Start() for triggerID, trigger := range s.triggers.ReadAll() { nextExecutionTime, err := trigger.job.NextRun() s.triggers.Write(triggerID, cronTrigger{ - job: trigger.job, - nextRun: nextExecutionTime, - workflowID: trigger.workflowID, - close: trigger.close, + job: trigger.job, + nextRun: nextExecutionTime, + workflowID: trigger.workflowID, + workflowDonID: trigger.workflowDonID, + workflowOwner: trigger.workflowOwner, + orgID: trigger.orgID, + close: trigger.close, }) if err != nil { s.lggr.Errorw("Unable to get next run time", "err", err, "triggerID", triggerID) } } - s.lggr.Info(s.Name() + " started") - return nil } -// Close stops the Service. -// After this call the Service cannot be started again, -// The service will need to be re-built to start scheduling again. -func (s *Service) Close() error { +// close is the services.Engine close hook. After this the Service cannot be +// started again; it must be re-built to schedule again. There are NO +// process-lifecycle metering emissions: a graceful shutdown emits nothing, and +// billing releases each still-active registration by its absence from the next +// snapshot. close deregisters the Meterable from the ResourceManager FIRST (so +// no snapshot can run after the store is torn down), then shuts the scheduler +// down. The ResourceManager sub-service is closed by the engine afterwards. +func (s *Service) close() error { if s.scheduler == nil { return errors.New("service has shutdown, it must be built again to restart") } + // Deregister from the snapshot registry before anything else so no snapshot + // tick can observe a half-torn-down service. + if s.unregisterMeterable != nil { + s.unregisterMeterable() + s.unregisterMeterable = nil + } + err := s.scheduler.Shutdown() if err != nil { return fmt.Errorf("scheduler shutdown encountered a problem: %s", err) @@ -455,23 +594,39 @@ func (s *Service) Close() error { // but calling .Start() on it will not error. Set to nil to mark closed. s.scheduler = nil - s.lggr.Info(s.Name() + " closed") - - return nil -} - -func (s *Service) Ready() error { return nil } -func (s *Service) HealthReport() map[string]error { - return map[string]error{s.Name(): nil} +func (s *Service) Description() string { + return "Cron Trigger Capability" } -func (s *Service) Name() string { - return s.lggr.Name() +// ResourceIdentity implements resourcemanager.Meterable: it returns the base +// six-dimension identity (per-resource billing fields are set per active +// trigger in +// GetUtilization). +func (s *Service) ResourceIdentity() resourcemanager.ResourceIdentity { + return s.base } -func (s *Service) Description() string { - return "Cron Trigger Capability" +// GetUtilization implements resourcemanager.Meterable: it returns the absolute +// state of every currently active cron registration, one SnapshotEntry per +// trigger, each at value 1 (a registration is a single reserved unit). It is a +// cheap in-memory read of the store snapshot and tolerates ctx cancellation. +func (s *Service) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { + if ctx.Err() != nil { + return nil + } + triggers := s.triggers.ReadAll() + entries := make([]resourcemanager.SnapshotEntry, 0, len(triggers)) + for triggerID, trigger := range triggers { + // Use stored orgID resolved at registration time. + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: s.base, + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, s.utilizationFields(triggerID, trigger.orgID)), + }, + }) + } + return entries } diff --git a/cron/trigger/trigger_test.go b/cron/trigger/trigger_test.go index 3eca57abd..bba0a4b38 100644 --- a/cron/trigger/trigger_test.go +++ b/cron/trigger/trigger_test.go @@ -252,7 +252,7 @@ func successWithStandardCronIntervals(t *testing.T, useTypedAPI bool) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -333,7 +333,7 @@ func TestCronTrigger_Load(t *testing.T) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -480,7 +480,7 @@ func testCronTriggerRegisterTriggerBeforeStart(t *testing.T, useTypedAPI bool) { fakeClock := clockwork.NewRealClock() config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -553,7 +553,7 @@ func testCronTriggerTimeWindows(t *testing.T, useTypedAPI bool) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -629,7 +629,7 @@ func testCronTriggerMultipleDifferentSchedules(t *testing.T, useTypedAPI bool) { } config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -752,7 +752,7 @@ func testCronTriggerTimeZone(t *testing.T, useTypedAPI bool) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -865,7 +865,7 @@ func testCronTriggerRegisterTrigger(t *testing.T, useTypedAPI bool) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { fakeClock := clockwork.NewRealClock() - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{}) require.NoError(t, err) @@ -904,7 +904,7 @@ func TestCronTrigger_RegisterTriggerDuplicateError(t *testing.T) { triggerConfig, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) fakeClock := clockwork.NewRealClock() - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(triggerConfig), @@ -939,7 +939,7 @@ func TestCronTrigger_UnregisterTriggerError(t *testing.T) { triggerConfig, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) fakeClock := clockwork.NewRealClock() - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(triggerConfig), @@ -1018,7 +1018,7 @@ func TestCronTrigger_UnregisterTriggerError(t *testing.T) { }) t.Run("NOK fails to unregister if closed", func(t *testing.T) { - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(triggerConfig), @@ -1056,7 +1056,7 @@ func TestCronTrigger_UnregisterTriggerError(t *testing.T) { func TestCronTrigger_CloseStartErrors(t *testing.T) { fakeClock := clockwork.NewRealClock() - ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger.Nop(), fakeClock, limits.Factory{}, nil) require.NoError(t, err) ctx := t.Context() @@ -1082,7 +1082,7 @@ func TestGocronNewTaskPanic(t *testing.T) { config, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) logger, observedLogs := logger.TestObserved(t, zap.ErrorLevel) - ts, err := NewTriggerService(logger, fakeClock, limits.Factory{}) + ts, err := NewTriggerService(logger, fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ Config: string(config), @@ -1176,7 +1176,7 @@ func TestCronTrigger_ExecutionIDWithTriggerIndex(t *testing.T) { triggerConfig, err := json.Marshal(Config{FastestScheduleIntervalSeconds: 1}) require.NoError(t, err) - ts, err := NewTriggerService(lggr, fakeClock, limits.Factory{}) + ts, err := NewTriggerService(lggr, fakeClock, limits.Factory{}, nil) require.NoError(t, err) err = ts.Initialise(t.Context(), core.StandardCapabilitiesDependencies{Config: string(triggerConfig)}) require.NoError(t, err) diff --git a/http_trigger/go.mod b/http_trigger/go.mod index 069b1f7e8..f23e5212a 100644 --- a/http_trigger/go.mod +++ b/http_trigger/go.mod @@ -2,9 +2,17 @@ module github.com/smartcontractkit/capabilities/http_trigger go 1.26.2 +// Unpublished local stack for SHARED-2709; drop once chainlink-common and +// chainlink-protos/metering/go are tagged. +replace ( + github.com/smartcontractkit/chainlink-common => ../../chainlink-common + github.com/smartcontractkit/chainlink-protos/metering/go => ../../chainlink-protos/metering/go +) + require ( + github.com/jonboulle/clockwork v0.5.0 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260603162703-43f4fa4bd88c + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 @@ -75,10 +83,11 @@ require ( github.com/scylladb/go-reflectx v1.0.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/smartcontractkit/chain-selectors v1.0.100 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 // indirect - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260602131523-5168ac1ba014 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 // indirect github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect @@ -119,7 +128,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/grpc v1.80.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/http_trigger/go.sum b/http_trigger/go.sum index 719469218..e3e97ee06 100644 --- a/http_trigger/go.sum +++ b/http_trigger/go.sum @@ -130,6 +130,8 @@ github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5Xum github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -207,16 +209,20 @@ github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924 github.com/smartcontractkit/capabilities/libs v0.0.0-20260210010829-97eb42ca2924/go.mod h1:v0O0Au8RE00Z89QxBE6I2q9bR9r3+RO1gLD3oaO2WB0= github.com/smartcontractkit/chain-selectors v1.0.100 h1:wpiSpmI/eFjY+wx/nPr5VuNF4hki0prIBMKEaQWn3g4= github.com/smartcontractkit/chain-selectors v1.0.100/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260603162703-43f4fa4bd88c h1:Khlxg3kMFJ5DNbZlREoVD5KZ77DKYMOz5qPj9MdrTD4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260603162703-43f4fa4bd88c/go.mod h1:zC5csAXmmn2FZbZ78Rrfc4AvmEKJzKQLMEY/w2SRwDo= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6 h1:ucHu2bPDT/58AzSgnPDyp4IjnjVbrVWYD3bG5jCbXMY= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260529092756-a94bc8ce96d6/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260602131523-5168ac1ba014 h1:4rxcbbe1qe1yR+HcclvOi/e0CFLcBLfx2fgiWxBMMZ4= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260602131523-5168ac1ba014/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b h1:VDgJWDipihV9f7M5+d21d1RzSsg5rEv+iI12oN1VQbo= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9/go.mod h1:GTpDgyK0OObf7jpch6p8N281KxN92wbB8serZhU9yRc= github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e h1:Hv9Mww35LrufCdM9wtS9yVi/rEWGI1UnjHbcKKU0nVY= diff --git a/http_trigger/main.go b/http_trigger/main.go index 337e478c3..939114d5a 100644 --- a/http_trigger/main.go +++ b/http_trigger/main.go @@ -5,11 +5,24 @@ import ( "github.com/smartcontractkit/capabilities/libs/loopserver" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http/server" + "github.com/smartcontractkit/chainlink-common/pkg/durableemitter" "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" ) func main() { loopserver.ServeNew(trigger.ServiceName, func(s *loop.Server) loop.StandardCapabilities { - return server.NewHTTPServer(trigger.NewService(s.Logger, s.LimitsFactory)) + // EnvConfig.MeteringConfig is the single, canonical loop-env -> metering + // mapping (enable flags, snapshot interval, deployment identity); no + // per-main copy of that mapping. The durable emitter is resolved here + // and injected, since resourcemanager itself must not reach for the + // process-global emitter. + var emitter resourcemanager.Emitter + if de := durableemitter.GetGlobalEmitter(); de != nil { + emitter = de + } + meteringCfg := s.EnvConfig.MeteringConfig(emitter) + svc := trigger.NewService(s.Logger, s.LimitsFactory, meteringCfg) + return server.NewHTTPServer(svc) }) } diff --git a/http_trigger/trigger/connector_handler.go b/http_trigger/trigger/connector_handler.go index 48c680c0d..ffa94bf09 100644 --- a/http_trigger/trigger/connector_handler.go +++ b/http_trigger/trigger/connector_handler.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "strings" "sync" "time" @@ -16,13 +15,14 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/custmsg" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" - "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types/core" gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" "github.com/smartcontractkit/chainlink-common/pkg/workflows" "github.com/smartcontractkit/chainlink-common/pkg/workflows/events" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" ) const ( @@ -32,12 +32,17 @@ const ( var _ core.GatewayConnectorHandler = &connectorHandler{} +// connectorHandler implements resourcemanager.Meterable: it owns the +// workflowStore and the base metering identity, so it both emits lifecycle +// edges inline and reports the absolute state of active registrations on the +// ResourceManager's snapshot tick. +var _ resourcemanager.Meterable = &connectorHandler{} + type connectorHandler struct { services.StateMachine lggr logger.Logger gatewayConnector core.GatewayConnector config ServiceConfig - capabilityDonID uint32 // authoritative sending DON ID; 0 = unknown, falls back to WorkflowDONID requestCache *requestCache workflowStore *workflowStore gatewayMetadataPublisher GatewayMetadataPublisher @@ -45,23 +50,36 @@ type connectorHandler struct { wg sync.WaitGroup stopChan services.StopChan orgResolver orgresolver.OrgResolver // Optional org resolver for fetching organization IDs + resourceManager *resourcemanager.ResourceManager + // baseIdentity is the six-dimension + resource_pool metering identity for + // this trigger LOOP, built once at Initialise. Per-workflow billing fields + // are carried by Utilization. + baseIdentity resourcemanager.ResourceIdentity + // unregisterMeterable removes this handler from the ResourceManager's + // snapshot registry; set on Start, called on Close. + unregisterMeterable func() } -func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config ServiceConfig, capabilityDonID uint32, - workflowStore *workflowStore, gatewayMetadataPublisher GatewayMetadataPublisher, requestCache *requestCache, metrics *Metrics, - orgResolver orgresolver.OrgResolver, limitsFactory limits.Factory) (*connectorHandler, error) { - return &connectorHandler{ +func NewConnectorHandler(lggr logger.Logger, gc core.GatewayConnector, config ServiceConfig, + workflowStore *workflowStore, gatewayMetadataPublisher GatewayMetadataPublisher, requestCache *requestCache, metrics *Metrics, orgResolver orgresolver.OrgResolver, + resourceManager *resourcemanager.ResourceManager, baseIdentity resourcemanager.ResourceIdentity) (*connectorHandler, error) { + if resourceManager == nil { + resourceManager = resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{}) + } + h := &connectorHandler{ lggr: logger.Named(lggr, HandlerName), gatewayConnector: gc, config: config, - capabilityDonID: capabilityDonID, workflowStore: workflowStore, gatewayMetadataPublisher: gatewayMetadataPublisher, requestCache: requestCache, metrics: metrics, stopChan: make(chan struct{}), orgResolver: orgResolver, - }, nil + resourceManager: resourceManager, + baseIdentity: baseIdentity, + } + return h, nil } func (h *connectorHandler) Start(ctx context.Context) error { @@ -69,6 +87,15 @@ func (h *connectorHandler) Start(ctx context.Context) error { h.wg.Add(1) go h.startRequestCacheCleanup(ctx) return h.StartOnce(HandlerName, func() error { + // Start the ResourceManager as a sub-service (it owns the snapshot + // tick) and register this handler as the snapshotted Meterable. The RM + // is fail-open: a start failure logs and continues (uniform with the + // other trigger producers) rather than gating the handler. + if err := h.resourceManager.Start(ctx); err != nil { + h.lggr.Errorw("failed to start metering ResourceManager; snapshots disabled", "err", err) + } else { + h.unregisterMeterable = h.resourceManager.Register(h) + } return h.gatewayConnector.AddHandler(ctx, []string{ gateway_common.MethodWorkflowExecute, gateway_common.MethodPullWorkflowMetadata, @@ -103,7 +130,16 @@ func (h *connectorHandler) Close() error { return h.StopOnce(HandlerName, func() error { close(h.stopChan) h.wg.Wait() - return nil + // No process-lifecycle metering emissions: a graceful shutdown emits + // nothing, and billing releases each still-active workflow by its + // absence from the next snapshot. Deregister the Meterable from the + // ResourceManager FIRST so no snapshot tick can run after teardown, + // then close the ResourceManager. + if h.unregisterMeterable != nil { + h.unregisterMeterable() + h.unregisterMeterable = nil + } + return h.resourceManager.Close() }) } @@ -146,14 +182,109 @@ func (h *connectorHandler) RegisterWorkflow(ctx context.Context, input WorkflowR h.metrics.RecordBroadcastMetadataLatency(ctx, latencyMs, h.lggr) workflow := newWorkflowWithMetadata(input.WorkflowSelector, authorizedKeys, sendCh, input.Metadata) - if err := h.workflowStore.upsertWorkflow(workflow); err != nil { + // upsertWorkflow returns the evicted workflow (if any) atomically under the + // store lock, so we never need a separate pre-read (which would be a TOCTOU + // race against a concurrent register/unregister). + evicted, err := h.workflowStore.upsertWorkflow(workflow) + if err != nil { return fmt.Errorf("failed to register workflow (ID: %s, Owner: %s, Name: %s): %w", input.WorkflowSelector.WorkflowID, input.WorkflowSelector.WorkflowOwner, input.WorkflowSelector.WorkflowName, err) } + newWorkflowID := input.WorkflowSelector.WorkflowID + workflowDONID := input.Metadata.WorkflowDONID + owner := input.WorkflowSelector.WorkflowOwner + switch { + case evicted == nil: + // Brand-new registration: bill +1 for the new durable resource. + h.emitMeterRecord(ctx, 1, "http-register", newWorkflowID, workflowDONID, owner) + case evicted.workflowSelector.WorkflowID == newWorkflowID: + // Same-ID re-register: the durable resource is unchanged, so there is + // no level delta to bill. Emit nothing. + default: + // Version update: the same owner/name/tag reference now resolves to a + // new workflow ID. Bill -1 against the evicted workflow's resource_id + // and +1 for the new, both derived from the atomically returned + // eviction so the old reservation cannot leak. + h.emitMeterRecord(ctx, -1, "http-unregister", evicted.workflowSelector.WorkflowID, evicted.metadata.WorkflowDONID, evicted.workflowSelector.WorkflowOwner) + h.emitMeterRecord(ctx, 1, "http-register", newWorkflowID, workflowDONID, owner) + } h.lggr.Debugw("Registered workflow", "workflowID", input.WorkflowSelector.WorkflowID, "workflowOwner", input.WorkflowSelector.WorkflowOwner, "workflowName", input.WorkflowSelector.WorkflowName, "workflowTag", input.WorkflowSelector.WorkflowTag) return nil } +// emitMeterRecord emits a signed delta MeterRecord (METER_ACTION_UPDATE) for a +// change to the durable HTTP-workflow-registration level: register bills +1, +// unregister/version-eviction bills -1. resource_id is the workflow ID (HTTP +// registrations are workflow-scoped, so there is no shared physical resource). +// The org is resolved fresh from owner at emit time. Emission is fail-open and +// never affects the registration outcome. +// +// event_id is derived from the action namespace + the workflow ID, which is +// DON-consistent: the (un)register requests are delivered to every capability +// node as the mode-aggregated request (see the remote trigger publisher), and a +// version update changes the workflow ID so its +1/-1 pair is distinct from the +// prior version. The unregister path passes the same workflowID so its -1 hashes +// symmetrically with the register +1 it reverses. +func (h *connectorHandler) emitMeterRecord(ctx context.Context, delta int64, namespace, workflowID string, workflowDONID uint32, owner string) { + identity := h.baseIdentity + var orgID string + if h.orgResolver != nil && owner != "" { + if resolved, err := h.orgResolver.Get(ctx, owner); err != nil { + logger.Sugared(h.lggr).Warnw("failed to resolve org ID for metering", "owner", owner, "err", err) + } else { + orgID = resolved + } + } + eventID := resourcemanager.EventID(namespace, workflowID) + h.resourceManager.EmitDelta(ctx, identity, eventID, delta, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + OrgID: orgID, + }) +} + +// donID returns the DON identifier from the base metering identity. +func (h *connectorHandler) donID(workflowDONID uint32) string { + return h.baseIdentity.DonID() +} + +// ResourceIdentity returns the HTTP trigger's base metering identity (six +// dimensions + resource_pool). Per-workflow billing fields are populated on +// Utilization in GetUtilization. It implements resourcemanager.Meterable. +func (h *connectorHandler) ResourceIdentity() resourcemanager.ResourceIdentity { + return h.baseIdentity +} + +// GetUtilization returns the absolute state of currently active HTTP workflow +// registrations, one SnapshotEntry per workflow, for the ResourceManager's +// snapshot tick. It is a cheap read-snapshot of in-memory state (a read-locked +// copy from the workflow store) and holds no lock across I/O. It implements +// resourcemanager.Meterable. +func (h *connectorHandler) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { + workflows := h.workflowStore.getWorkflows() + entries := make([]resourcemanager.SnapshotEntry, 0, len(workflows)) + for _, w := range workflows { + workflowID := w.workflowSelector.WorkflowID + var orgID string + if h.orgResolver != nil && w.workflowSelector.WorkflowOwner != "" { + if resolved, err := h.orgResolver.Get(ctx, w.workflowSelector.WorkflowOwner); err == nil { + orgID = resolved + } + } + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: h.baseIdentity, + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + OrgID: orgID, + }), + }, + }) + } + return entries +} + func (h *connectorHandler) validateAuthorizedKeys(inputKeys []*http.AuthorizedKey) ([]gateway_common.AuthorizedKey, error) { if len(inputKeys) == 0 { return nil, fmt.Errorf("HTTP trigger requires at least one authorized key to sign JSON-RPC requests. Add AuthorizedKeys to your http.Trigger configuration with ECDSA EVM public keys (0x-prefixed hex strings)") @@ -181,10 +312,22 @@ func (h *connectorHandler) validateAuthorizedKeys(inputKeys []*http.AuthorizedKe } func (h *connectorHandler) UnregisterWorkflow(ctx context.Context, workflowID string) error { + // Snapshot the workflow DON and owner before removal; the DON feeds the + // meter record's identity fallback and the owner resolves org at emit time. + var workflowDONID uint32 + var owner string + if w, ok := h.workflowStore.getWorkflowByID(workflowID); ok { + workflowDONID = w.metadata.WorkflowDONID + owner = w.workflowSelector.WorkflowOwner + } err := h.workflowStore.removeWorkflow(workflowID) if err != nil { return fmt.Errorf("failed to unregister workflow %s: %w", workflowID, err) } + // Unregister bills a -1 delta against the workflow's resource_id. It hashes + // symmetrically with the register +1 (same workflowID, "http-unregister" + // namespace) so the consumer pairs them by workflowID. + h.emitMeterRecord(ctx, -1, "http-unregister", workflowID, workflowDONID, owner) h.lggr.Debugw("Unregistered workflow", "workflowID", workflowID) return nil } @@ -294,17 +437,13 @@ func (h *connectorHandler) processTrigger(ctx context.Context, gatewayID string, displayWorkflowName = workflowMetadata.WorkflowName } - // Emit the *sending* capability DON ID. The HTTP trigger plugin runs on a - // capability DON, separate from the consumer workflow's DON. The workflow - // service needs the sender's DON to resolve on-chain quorum params (N, F). - // See CRE-4409. capabilityDonID is 0 when the host could not resolve it - // authoritatively (a multi-DON job-spec node, or a core node that pre-dates - // CRE-4409); in that case we fall back to WorkflowDONID. This fallback is - // permanent, not transitional, since the job-spec boot path is still supported. - donIDForEvent := h.capabilityDonID - if donIDForEvent == 0 { - donIDForEvent = workflowMetadata.WorkflowDONID - } + // Resolve the *sending* capability DON ID once (CRE-4409). h.donID applies + // contract rule 8: the authoritative host-injected CapDONID (carried on + // baseIdentity) wins, and the consumer workflow's WorkflowDONID is used only + // when CapDONID is 0. This is the SAME resolver the metering identity uses + // (identityForWorkflow -> donID), so the event label and the meter record + // cannot diverge. + donIDForEvent := h.donID(workflowMetadata.WorkflowDONID) labeler := custmsg.NewLabeler().With( events.KeyTriggerID, req.ID, @@ -315,7 +454,7 @@ func (h *connectorHandler) processTrigger(ctx context.Context, gatewayID string, events.KeyWorkflowRegistryChainSelector, workflowMetadata.WorkflowRegistryChainSelector, events.KeyWorkflowRegistryAddress, workflowMetadata.WorkflowRegistryAddress, events.KeyEngineVersion, workflowMetadata.EngineVersion, - events.KeyDonID, strconv.Itoa(int(donIDForEvent)), + events.KeyDonID, donIDForEvent, ) // Try to fetch organization ID if org resolver is available diff --git a/http_trigger/trigger/connector_handler_test.go b/http_trigger/trigger/connector_handler_test.go index c5c964f0a..4d3003787 100644 --- a/http_trigger/trigger/connector_handler_test.go +++ b/http_trigger/trigger/connector_handler_test.go @@ -4,21 +4,27 @@ import ( "context" "database/sql" "encoding/json" + "errors" "strings" "sync" "testing" "time" + "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/types/core" gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" "github.com/smartcontractkit/chainlink-common/pkg/workflows" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" ) const ( @@ -190,13 +196,13 @@ func setupWithTriggerChannelBuffer(t *testing.T, lggr logger.Logger, triggerChBu lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) sdkCfg := &http.Config{ @@ -596,13 +602,13 @@ func TestRegisterWorkflow_TooManyAuthorizedKeys(t *testing.T) { lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) @@ -715,13 +721,13 @@ func TestConnectorHandler_Start_HealthReport_Ready_Name_Close(t *testing.T) { lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) @@ -876,13 +882,13 @@ func TestHandleGatewayMessage_PullAuthMetadata_EmptyWorkflows(t *testing.T) { lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) @@ -1054,13 +1060,13 @@ func TestConnectorHandler_StartRequestCacheCleanup(t *testing.T) { lggr, mockConnector, cfg, - 0, store, metadataPublisher, requestCache, newMetrics(t), nil, - limits.Factory{}, + nil, + resourcemanager.ResourceIdentity{}, ) require.NoError(t, err) @@ -1122,6 +1128,380 @@ func TestHandleGatewayMessage_NilRequest(t *testing.T) { require.Contains(t, err.Error(), "request cannot be nil") } +// fakeMeterEmitter decodes and records the MeterRecords and MeterSnapshots +// passed to Emit and can be configured to fail, for asserting fail-open +// behavior. It dispatches on the beholder entity attribute so MeterRecord and +// MeterSnapshot bodies are decoded with the correct type. Each MeterSnapshot +// covers exactly one resource. +type fakeMeterEmitter struct { + err error + records []*meteringpb.MeterRecord + recordDomains []string + snapshots []*meteringpb.MeterSnapshot +} + +func (f *fakeMeterEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { + if f.attr(attrKVs, beholder.AttrKeyEntity) == "metering.v1.MeterSnapshot" { + var snapshot meteringpb.MeterSnapshot + if err := proto.Unmarshal(body, &snapshot); err != nil { + return err + } + f.snapshots = append(f.snapshots, &snapshot) + return f.err + } + var record meteringpb.MeterRecord + if err := proto.Unmarshal(body, &record); err != nil { + return err + } + f.records = append(f.records, &record) + f.recordDomains = append(f.recordDomains, f.attr(attrKVs, beholder.AttrKeyDomain)) + return f.err +} + +// attr returns the value of a beholder attribute by key from the alternating +// key/value attrKVs slice, or "" if absent. +func (f *fakeMeterEmitter) attr(attrKVs []any, key string) string { + for i := 0; i+1 < len(attrKVs); i += 2 { + if k, ok := attrKVs[i].(string); ok && k == key { + if v, ok := attrKVs[i+1].(string); ok { + return v + } + } + } + return "" +} + +func (f *fakeMeterEmitter) actions() []meteringpb.MeterAction { + actions := make([]meteringpb.MeterAction, len(f.records)) + for i, r := range f.records { + actions[i] = r.GetAction() + } + return actions +} + +// testBaseIdentity is the base metering identity used by metering tests. It +// carries the six coarse dimensions plus the service-level resource_pool. +var testBaseIdentity = resourcemanager.ResourceIdentity{ + Product: "cre-test", + Tenant: "mainline", + NumericTenantID: "42", + Environment: "staging", + Zone: "wf-zone-a", + Don: &resourcemanager.DonIdentity{DonID: "7", NodeID: "node-csa-pubkey"}, + Service: meterService, + ResourcePool: meterResource, +} + +// setupWithMeterEmitter builds a handler with metering enabled and a fake +// emitter capturing emitted MeterRecords. The ResourceManager is started (so +// the snapshot tick is wired) and the handler is registered as the snapshotted +// Meterable; both are torn down on test cleanup. No workflows are registered. +func setupWithMeterEmitter(t *testing.T, lggr logger.Logger, emitErr error) (*connectorHandler, *fakeMeterEmitter) { + t.Helper() + emitter := &fakeMeterEmitter{err: emitErr} + cfg := ServiceConfig{ + MetadataBatchSize: 10, + MaxAuthorizedKeysPerWorkflow: 3, + } + store := newWorkflowStore(lggr) + metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) + requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) + resourceManager := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + }) + handler, err := NewConnectorHandler( + lggr, + &mockGatewayConnector{}, + cfg, + store, + metadataPublisher, + requestCache, + newMetrics(t), + nil, + resourceManager, + testBaseIdentity, + ) + require.NoError(t, err) + require.NoError(t, handler.Start(t.Context())) + t.Cleanup(func() { require.NoError(t, handler.Close()) }) + return handler, emitter +} + +func meterTestRegistrationInput() WorkflowRegistrationInput { + return WorkflowRegistrationInput{ + WorkflowSelector: gateway_common.WorkflowSelector{ + WorkflowID: testWorkflowID, + WorkflowOwner: testWorkflowOwner, + WorkflowName: testWorkflowName, + WorkflowTag: testWorkflowTag, + }, + Config: &http.Config{ + AuthorizedKeys: []*http.AuthorizedKey{ + { + PublicKey: publicKey, + Type: http.KeyType_KEY_TYPE_ECDSA_EVM, + }, + }, + }, + Metadata: WorkflowRegistrationMetadata{}, + } +} + +func TestRegisterWorkflow_RegisterThenSameIDReRegister(t *testing.T) { + lggr := logger.Test(t) + handler, emitter := setupWithMeterEmitter(t, lggr, nil) + input := meterTestRegistrationInput() + + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + err := handler.RegisterWorkflow(t.Context(), input, sendCh) + require.NoError(t, err) + + // First registration bills a single +1 UPDATE delta with the full + // structured identity populated on the record. + require.Equal(t, []meteringpb.MeterAction{meteringpb.MeterAction_METER_ACTION_UPDATE}, emitter.actions()) + record := emitter.records[0] + require.Equal(t, "cll-meter", emitter.recordDomains[0]) + id := record.GetIdentity() + require.Equal(t, testBaseIdentity.Product, id.GetProduct()) + require.Equal(t, testBaseIdentity.Tenant, id.GetTenant()) + require.Equal(t, testBaseIdentity.NumericTenantID, id.GetNumericTenantId()) + require.Equal(t, testBaseIdentity.Environment, id.GetEnvironment()) + require.Equal(t, testBaseIdentity.Zone, id.GetZone()) + require.Equal(t, testBaseIdentity.DonID(), id.GetDon().GetDonId()) + require.Equal(t, testBaseIdentity.NodeID(), id.GetDon().GetNodeId()) + require.Equal(t, meterService, id.GetService()) + require.Equal(t, meterResource, id.GetResourcePool()) + // The metering identity DON and events.KeyDonID label derive from the same + // donID resolver, so they cannot diverge. + require.Equal(t, handler.donID(input.Metadata.WorkflowDONID), id.GetDon().GetDonId()) + require.Equal(t, meterResourceType, record.GetUtilizations()[0].GetResourceType()) + // resource_id is the workflow ID (HTTP registrations are workflow-scoped). + require.Equal(t, testWorkflowID, record.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "1", record.GetUtilizations()[0].GetValue()) + require.NotEmpty(t, record.GetUtilizations()[0].GetEventId(), "event_id is stamped per emission") + + // Re-registering the SAME workflow ID is not a level change and emits + // nothing (no RESERVE/UPDATE): the durable resource is unchanged. + sendCh2 := make(chan capabilities.TriggerAndId[*http.Payload], 1) + err = handler.RegisterWorkflow(t.Context(), input, sendCh2) + require.NoError(t, err) + require.Equal(t, []meteringpb.MeterAction{meteringpb.MeterAction_METER_ACTION_UPDATE}, emitter.actions(), + "same-ID re-register emits no additional delta") +} + +func TestRegisterWorkflow_VersionUpdate_MetersNegativeThenPositiveDelta(t *testing.T) { + lggr := logger.Test(t) + handler, emitter := setupWithMeterEmitter(t, lggr, nil) + + inputA := meterTestRegistrationInput() + inputA.WorkflowSelector.WorkflowID = testWorkflowID1 + sendChA := make(chan capabilities.TriggerAndId[*http.Payload], 1) + require.NoError(t, handler.RegisterWorkflow(t.Context(), inputA, sendChA)) + + // Re-registering the same owner/name/tag reference with a NEW workflow ID + // is a version update: the previous workflow's resource is billed -1 before + // the new one is billed +1, so the old resource cannot leak. + inputB := meterTestRegistrationInput() + inputB.WorkflowSelector.WorkflowID = testWorkflowID2 + sendChB := make(chan capabilities.TriggerAndId[*http.Payload], 1) + require.NoError(t, handler.RegisterWorkflow(t.Context(), inputB, sendChB)) + + require.Equal(t, []meteringpb.MeterAction{ + meteringpb.MeterAction_METER_ACTION_UPDATE, + meteringpb.MeterAction_METER_ACTION_UPDATE, + meteringpb.MeterAction_METER_ACTION_UPDATE, + }, emitter.actions()) + + // +1 for the old workflow ID (via utilization.resource_id). + registerA := emitter.records[0] + require.Equal(t, testWorkflowID1, registerA.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "1", registerA.GetUtilizations()[0].GetValue()) + + // -1 targets the PREVIOUS (evicted) workflow ID. + release := emitter.records[1] + require.Equal(t, testWorkflowID1, release.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "-1", release.GetUtilizations()[0].GetValue()) + + // +1 for the new workflow ID. + registerB := emitter.records[2] + require.Equal(t, testWorkflowID2, registerB.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "1", registerB.GetUtilizations()[0].GetValue()) + + // event_ids are unique across all three emissions. + ids := map[string]struct{}{} + for _, r := range emitter.records { + ids[r.GetUtilizations()[0].GetEventId()] = struct{}{} + } + require.Len(t, ids, 3, "each emission gets a distinct event_id") +} + +func TestUnregisterWorkflow_MetersNegativeDelta(t *testing.T) { + lggr := logger.Test(t) + handler, emitter := setupWithMeterEmitter(t, lggr, nil) + + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + err := handler.RegisterWorkflow(t.Context(), meterTestRegistrationInput(), sendCh) + require.NoError(t, err) + + err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) + require.NoError(t, err) + require.Equal(t, []meteringpb.MeterAction{ + meteringpb.MeterAction_METER_ACTION_UPDATE, + meteringpb.MeterAction_METER_ACTION_UPDATE, + }, emitter.actions()) + release := emitter.records[1] + require.Equal(t, testWorkflowID, release.GetUtilizations()[0].GetResourceId()) + require.Equal(t, "-1", release.GetUtilizations()[0].GetValue()) + require.NotEqual(t, emitter.records[0].GetUtilizations()[0].GetEventId(), release.GetUtilizations()[0].GetEventId()) + + // Unregistering an absent workflow fails and must not emit a delta. + err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) + require.Error(t, err) + require.Len(t, emitter.records, 2) +} + +func TestRegisterWorkflow_MeteringFailOpen(t *testing.T) { + lggr := logger.Test(t) + handler, emitter := setupWithMeterEmitter(t, lggr, errors.New("emit failed")) + + // Registration and unregistration succeed even though every emit fails. + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + err := handler.RegisterWorkflow(t.Context(), meterTestRegistrationInput(), sendCh) + require.NoError(t, err) + err = handler.UnregisterWorkflow(t.Context(), testWorkflowID) + require.NoError(t, err) + require.Len(t, emitter.records, 2) +} + +// registerMeterWorkflow registers a workflow with the given ID/owner under the +// metering test config (each registration uses a distinct reference so they +// coexist). +func registerMeterWorkflow(t *testing.T, handler *connectorHandler, workflowID, owner string) { + t.Helper() + input := meterTestRegistrationInput() + input.WorkflowSelector.WorkflowID = workflowID + input.WorkflowSelector.WorkflowOwner = owner + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + require.NoError(t, handler.RegisterWorkflow(t.Context(), input, sendCh)) +} + +// TestSnapshot_EmitsOneEntryPerActiveWorkflow starts the ResourceManager tick +// and asserts one MeterSnapshot per active workflow, each carrying the full +// per-workflow identity. +func TestSnapshot_EmitsOneEntryPerActiveWorkflow(t *testing.T) { + lggr := logger.Test(t) + emitter := &fakeMeterEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + cfg := ServiceConfig{MetadataBatchSize: 10, MaxAuthorizedKeysPerWorkflow: 3} + store := newWorkflowStore(lggr) + metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) + requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) + rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, testBaseIdentity) + require.NoError(t, err) + unregister := rm.Register(handler) + t.Cleanup(unregister) + + registerMeterWorkflow(t, handler, testWorkflowID1, testWorkflowOwner1) + registerMeterWorkflow(t, handler, testWorkflowID2, testWorkflowOwner2) + + // Drop the lifecycle RESERVE records; we assert only on the snapshot tick. + emitter.records = nil + servicetest.Run(t, rm) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(emitter.snapshots) == 2 + }, time.Second, time.Millisecond) + + // One MeterSnapshot per active workflow, keyed by utilization.resource_id. + require.Len(t, emitter.snapshots, 2) + byWorkflowID := map[string]*meteringpb.MeterSnapshot{} + for _, s := range emitter.snapshots { + byWorkflowID[s.GetUtilization()[0].GetResourceId()] = s + } + require.Len(t, byWorkflowID, 2) + + r1 := byWorkflowID[testWorkflowID1] + require.NotNil(t, r1) + require.Equal(t, testBaseIdentity.Product, r1.GetIdentity().GetProduct()) + require.Equal(t, meterResource, r1.GetIdentity().GetResourcePool()) + require.Equal(t, meterResourceType, r1.GetUtilization()[0].GetResourceType()) + require.Equal(t, "1", r1.GetUtilization()[0].GetValue()) + + r2 := byWorkflowID[testWorkflowID2] + require.NotNil(t, r2) + require.Equal(t, "1", r2.GetUtilization()[0].GetValue()) +} + +// TestClose_EmitsNoShutdownRecords asserts a graceful close emits NO meter +// records. Process-lifecycle emissions are deleted by design: an active +// workflow is released by its absence from the next snapshot, not by a +// close-time drain. +func TestClose_EmitsNoShutdownRecords(t *testing.T) { + lggr := logger.Test(t) + emitter := &fakeMeterEmitter{} + cfg := ServiceConfig{MetadataBatchSize: 10, MaxAuthorizedKeysPerWorkflow: 3} + store := newWorkflowStore(lggr) + metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) + requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) + rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + }) + handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, testBaseIdentity) + require.NoError(t, err) + require.NoError(t, handler.Start(t.Context())) + + registerMeterWorkflow(t, handler, testWorkflowID1, testWorkflowOwner1) + registerMeterWorkflow(t, handler, testWorkflowID2, testWorkflowOwner2) + + // Drop the register deltas; assert the close path emits nothing. + emitter.records = nil + require.NoError(t, handler.Close()) + require.Empty(t, emitter.records, "graceful close must emit no meter records") +} + +// TestDONIDFallback_UsesWorkflowDON asserts that when the host did not inject a +// capability DON (base DONID empty), records fall back to the per-registration +// workflow DON. +func TestDONIDFallback_UsesWorkflowDON(t *testing.T) { + lggr := logger.Test(t) + emitter := &fakeMeterEmitter{} + cfg := ServiceConfig{MetadataBatchSize: 10, MaxAuthorizedKeysPerWorkflow: 3} + store := newWorkflowStore(lggr) + metadataPublisher := NewGatewayMetadataPublisher(lggr, &mockGatewayConnector{}, store, cfg, newMetrics(t)) + requestCache := newRequestCache(logger.Sugared(lggr), newTestKVStore(), time.Hour) + rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{MeterRecordsEnabled: true, Emitter: emitter}) + // Base identity WITHOUT a capability DON (host did not inject one). + base := testBaseIdentity + base.Don = &resourcemanager.DonIdentity{NodeID: "node-csa-pubkey"} + handler, err := NewConnectorHandler(lggr, &mockGatewayConnector{}, cfg, store, metadataPublisher, requestCache, newMetrics(t), nil, rm, base) + require.NoError(t, err) + + input := meterTestRegistrationInput() + input.Metadata.WorkflowDONID = 99 + sendCh := make(chan capabilities.TriggerAndId[*http.Payload], 1) + require.NoError(t, handler.RegisterWorkflow(t.Context(), input, sendCh)) + + require.Len(t, emitter.records, 1) + require.Equal(t, "99", emitter.records[0].GetIdentity().GetDon().GetDonId()) + // Metering identity DON and events.KeyDonID label share the same resolver. + require.Equal(t, handler.donID(input.Metadata.WorkflowDONID), emitter.records[0].GetIdentity().GetDon().GetDonId()) +} + // TestResolveWorkflowMetadata_PreservesStoredWorkflowOwner tests that the workflowOwner // from the stored workflow is used, even if the incoming request has zeros or missing values. // This is a regression test for the bug where workflowOwner was being set to zeros. diff --git a/http_trigger/trigger/gateway_metadata_publisher_test.go b/http_trigger/trigger/gateway_metadata_publisher_test.go index bedf22bdf..2c6a8c10c 100644 --- a/http_trigger/trigger/gateway_metadata_publisher_test.go +++ b/http_trigger/trigger/gateway_metadata_publisher_test.go @@ -221,9 +221,9 @@ func TestSendWorkflows_Success(t *testing.T) { wf1 := newWorkflow(selector1, authorizedKeys1, sendCh1) wf2 := newWorkflow(selector2, authorizedKeys2, sendCh2) - err := workflowStore.upsertWorkflow(wf1) + _, err := workflowStore.upsertWorkflow(wf1) require.NoError(t, err) - err = workflowStore.upsertWorkflow(wf2) + _, err = workflowStore.upsertWorkflow(wf2) require.NoError(t, err) gatewayID := "gateway1" diff --git a/http_trigger/trigger/trigger.go b/http_trigger/trigger/trigger.go index abb164be9..a85078402 100644 --- a/http_trigger/trigger/trigger.go +++ b/http_trigger/trigger/trigger.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strconv" "strings" "time" @@ -12,6 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http/server" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -21,6 +23,17 @@ import ( const ServiceName = "HTTPTriggerCapability" +// Metering identity constants for the HTTP trigger. Service is the stable +// service constant (it must not encode environment or zone); resource pool and +// utilization resource_type identify the HTTP workflow-registration pool and +// its billing +// unit. +const ( + meterService = "http-trigger" + meterResource = "http_workflows" + meterResourceType = "operations" +) + var _ server.HTTPCapability = &service{} type WorkflowRegistrationInput struct { @@ -53,12 +66,17 @@ type service struct { metrics *Metrics limitsFactory limits.Factory orgResolver orgresolver.OrgResolver + // metering is the resolved metering Config (ResourceManagerConfig + + // DeploymentIdentity) produced from loop.EnvConfig by + // EnvConfig.MeteringConfig in main. + metering resourcemanager.Config } -func NewService(lggr logger.Logger, limitsFactory limits.Factory) *service { +func NewService(lggr logger.Logger, limitsFactory limits.Factory, metering resourcemanager.Config) *service { return &service{ lggr: logger.Sugared(logger.Named(lggr, ServiceName)), limitsFactory: limitsFactory, + metering: metering, } } @@ -73,9 +91,11 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa } } s.cfg = applyDefaults(serviceConfig) - s.orgResolver = dependencies.OrgResolver - if s.orgResolver == nil { + if dependencies.OrgResolver == nil { s.lggr.Warn("OrgResolver is nil, HTTP trigger capability will not be able to fetch organization ID") + s.orgResolver = nil + } else { + s.orgResolver = dependencies.OrgResolver } workflowStore := newWorkflowStore(s.lggr) var err error @@ -85,12 +105,12 @@ func (s *service) Initialise(ctx context.Context, dependencies core.StandardCapa } metadataPublisher := NewGatewayMetadataPublisher(s.lggr, dependencies.GatewayConnector, workflowStore, s.cfg, s.metrics) requestCache := newRequestCache(s.lggr, dependencies.Store, time.Duration(s.cfg.RequestCacheTTL)*time.Second) - // dependencies.CapabilityDonID is the on-chain DON ID this plugin process - // serves, used to label emitted events with the *sending* DON. Zero means the - // host could not resolve it authoritatively (a multi-DON job-spec node, or a - // core node that pre-dates CRE-4409); the handler then falls back to - // RequestMetadata.WorkflowDONID. See CRE-4409. - s.connectorHandler, err = NewConnectorHandler(s.lggr, dependencies.GatewayConnector, s.cfg, dependencies.CapabilityDonID, workflowStore, metadataPublisher, requestCache, s.metrics, s.orgResolver, s.limitsFactory) + resourceManager := resourcemanager.NewResourceManager(s.lggr, s.metering.ResourceManagerConfig) + baseIdentity := resourcemanager.NewBaseIdentity(s.metering.DeploymentIdentity, meterService, meterResource) + if dependencies.CapabilityDonID != 0 { + baseIdentity = baseIdentity.WithDonID(strconv.FormatUint(uint64(dependencies.CapabilityDonID), 10)) + } + s.connectorHandler, err = NewConnectorHandler(s.lggr, dependencies.GatewayConnector, s.cfg, workflowStore, metadataPublisher, requestCache, s.metrics, s.orgResolver, resourceManager, baseIdentity) if err != nil { return err } diff --git a/http_trigger/trigger/trigger_test.go b/http_trigger/trigger/trigger_test.go index 6d0413547..5ab96ac4f 100644 --- a/http_trigger/trigger/trigger_test.go +++ b/http_trigger/trigger/trigger_test.go @@ -13,6 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/triggers/http" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types/core" gcmocks "github.com/smartcontractkit/chainlink-common/pkg/types/core/mocks" @@ -56,7 +57,7 @@ func TestService_RegisterTrigger(t *testing.T) { mockHandler := &mockConnectorHandler{ registerErr: tc.registerErr, } - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, resourcemanager.Config{}) cfgStr := fmt.Sprintf(`{"sendChannelBufferSize": %d}`, tc.sendChannelBufSize) gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ @@ -118,7 +119,7 @@ func TestService_UnregisterTrigger(t *testing.T) { mockHandler := &mockConnectorHandler{ unregisterErr: tt.handlerErr, } - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, resourcemanager.Config{}) cfg := "{}" gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ @@ -141,7 +142,7 @@ func TestService_UnregisterTrigger(t *testing.T) { } func TestService_Initialise_EmptyConfig(t *testing.T) { - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, resourcemanager.Config{}) gc := mockedGatewayConnector(t) err := svc.Initialise(context.Background(), core.StandardCapabilitiesDependencies{ @@ -157,7 +158,7 @@ func TestService_Initialise_EmptyConfig(t *testing.T) { func TestService_Start_HealthReport_Ready_Close(t *testing.T) { mockHandler := &mockConnectorHandler{} - svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}) + svc := NewService(logger.Test(t), limits.Factory{Logger: logger.Test(t)}, resourcemanager.Config{}) cfg := "{}" gc := mockedGatewayConnector(t) err := svc.Initialise(t.Context(), core.StandardCapabilitiesDependencies{ diff --git a/http_trigger/trigger/workflow.go b/http_trigger/trigger/workflow.go index 67daefa28..0e3b13d99 100644 --- a/http_trigger/trigger/workflow.go +++ b/http_trigger/trigger/workflow.go @@ -52,34 +52,38 @@ func newWorkflowStore(lggr logger.Logger) *workflowStore { // workflow reference (owner/name/tag combination) with new workflow instance. // upsertWorkflow should be invoked in the order of workflow registration, so that // the latest workflow instance is always used for the given reference. -func (s *workflowStore) upsertWorkflow(w *workflow) error { +// +// It returns the evicted workflow: the registration that previously held this +// owner/name/tag reference (nil when the reference was new). The eviction is +// determined atomically under the store lock so the caller can meter the +// replaced resource without a separate, racy pre-read. The evicted workflow's +// WorkflowID may equal the new one (a same-ID re-register, no level change) or +// differ (a version update whose old resource_id must be released). +func (s *workflowStore) upsertWorkflow(w *workflow) (evicted *workflow, err error) { // Validate workflow fields if err := validateWorkflowSelector(w.workflowSelector); err != nil { - return fmt.Errorf("invalid workflow selector: %w", err) + return nil, fmt.Errorf("invalid workflow selector: %w", err) } s.mu.Lock() defer s.mu.Unlock() - workflowID, exists := s.workflowReferenceToID[workflowReference{ + ref := workflowReference{ workflowOwner: w.workflowSelector.WorkflowOwner, workflowName: w.workflowSelector.WorkflowName, workflowTag: w.workflowSelector.WorkflowTag, - }] - if exists { + } + if prevWorkflowID, replaced := s.workflowReferenceToID[ref]; replaced { reference := fmt.Sprintf("%s/%s/%s", w.workflowSelector.WorkflowOwner, w.workflowSelector.WorkflowName, w.workflowSelector.WorkflowTag) - s.lggr.Debugw("Updating existing workflow reference and removing previous workflow", "reference", reference, "prevWorkflowID", workflowID) - if oldW, ok := s.workflows[workflowID]; ok { + s.lggr.Debugw("Updating existing workflow reference and removing previous workflow", "reference", reference, "prevWorkflowID", prevWorkflowID) + if oldW, ok := s.workflows[prevWorkflowID]; ok { + evicted = oldW oldW.close() } - delete(s.workflows, workflowID) + delete(s.workflows, prevWorkflowID) } s.workflows[w.workflowSelector.WorkflowID] = w - s.workflowReferenceToID[workflowReference{ - workflowOwner: w.workflowSelector.WorkflowOwner, - workflowName: w.workflowSelector.WorkflowName, - workflowTag: w.workflowSelector.WorkflowTag, - }] = w.workflowSelector.WorkflowID - return nil + s.workflowReferenceToID[ref] = w.workflowSelector.WorkflowID + return evicted, nil } // validateWorkflowSelector validates the workflow selector fields diff --git a/http_trigger/trigger/workflow_test.go b/http_trigger/trigger/workflow_test.go index 189ed0d39..e45a16f75 100644 --- a/http_trigger/trigger/workflow_test.go +++ b/http_trigger/trigger/workflow_test.go @@ -134,8 +134,9 @@ func TestWorkflowStore_upsertWorkflow(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - err := store.upsertWorkflow(wf) + evicted, err := store.upsertWorkflow(wf) require.NoError(t, err) + require.Nil(t, evicted, "a new reference evicts nothing") w, exists := store.getWorkflowByID(wf.workflowSelector.WorkflowID) require.True(t, exists) @@ -257,7 +258,7 @@ func TestWorkflowStore_upsertWorkflow_ValidationErrors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { wf := newWorkflow(tt.selector, authorizedKeys, sendCh) - err := store.upsertWorkflow(wf) + _, err := store.upsertWorkflow(wf) require.Error(t, err) require.Contains(t, err.Error(), tt.wantErr) }) @@ -273,12 +274,15 @@ func TestWorkflowStore_upsertWorkflow_Duplicate(t *testing.T) { w2, _ := testWorkflow() // Add first workflow - err := store.upsertWorkflow(w1) + evicted, err := store.upsertWorkflow(w1) require.NoError(t, err) + require.Nil(t, evicted) // Add second workflow with same ID (this should replace the first) - err = store.upsertWorkflow(w2) + evicted, err = store.upsertWorkflow(w2) require.NoError(t, err) + require.NotNil(t, evicted, "same-reference re-register evicts the previous workflow") + require.Equal(t, w1.workflowSelector.WorkflowID, evicted.workflowSelector.WorkflowID) // Verify the workflow was replaced - since both have same ID/reference, // the second one should be present @@ -294,7 +298,7 @@ func TestWorkflowStore_removeWorkflow_Success(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) w, _ := testWorkflow() - err := store.upsertWorkflow(w) + _, err := store.upsertWorkflow(w) require.NoError(t, err) wf, exists := store.getWorkflowByID(w.workflowSelector.WorkflowID) @@ -393,11 +397,11 @@ func TestWorkflowStore_GetWorkflows_Multiple(t *testing.T) { wf2 := newWorkflow(wfSelector2, authorizedKeys, sendCh2) wf3 := newWorkflow(wfSelector3, authorizedKeys, sendCh3) - err := store.upsertWorkflow(wf1) + _, err := store.upsertWorkflow(wf1) require.NoError(t, err) - err = store.upsertWorkflow(wf2) + _, err = store.upsertWorkflow(wf2) require.NoError(t, err) - err = store.upsertWorkflow(wf3) + _, err = store.upsertWorkflow(wf3) require.NoError(t, err) // Get all workflows @@ -421,7 +425,7 @@ func TestWorkflowStore_getWorkflowIDByReference_Success(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - err := store.upsertWorkflow(wf) + _, err := store.upsertWorkflow(wf) require.NoError(t, err) workflowID, exists := store.getWorkflowIDByReference( @@ -538,8 +542,9 @@ func TestWorkflowStore_upsertWorkflow_ReplaceWithSameReference(t *testing.T) { wf2 := newWorkflow(selector2, authorizedKeys, sendCh2) // Add first workflow - err := store.upsertWorkflow(wf1) + evicted, err := store.upsertWorkflow(wf1) require.NoError(t, err) + require.Nil(t, evicted) // Verify first workflow is there workflow, exists := store.getWorkflowByID(testWorkflowID1) @@ -551,9 +556,12 @@ func TestWorkflowStore_upsertWorkflow_ReplaceWithSameReference(t *testing.T) { require.True(t, exists) require.Equal(t, testWorkflowID1, workflowID) - // Add second workflow with same reference - err = store.upsertWorkflow(wf2) + // Add second workflow with same reference; the evicted workflow is + // surfaced so callers can release its resource. + evicted, err = store.upsertWorkflow(wf2) require.NoError(t, err) + require.NotNil(t, evicted) + require.Equal(t, testWorkflowID1, evicted.workflowSelector.WorkflowID) // First workflow should be removed _, exists = store.getWorkflowByID(testWorkflowID1) @@ -576,7 +584,7 @@ func TestWorkflowStore_removeWorkflow_RemovesReference(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - err := store.upsertWorkflow(wf) + _, err := store.upsertWorkflow(wf) require.NoError(t, err) // Verify workflow and reference exist @@ -735,7 +743,7 @@ func TestWorkflowStore_getWorkflowIDByReference_PartialMatch(t *testing.T) { lggr := logger.Test(t) store := newWorkflowStore(lggr) wf, _ := testWorkflow() - err := store.upsertWorkflow(wf) + _, err := store.upsertWorkflow(wf) require.NoError(t, err) // Test with wrong owner diff --git a/integration_tests/go.mod b/integration_tests/go.mod index ecd6d51c2..70f1fcb86 100644 --- a/integration_tests/go.mod +++ b/integration_tests/go.mod @@ -2,6 +2,13 @@ module github.com/smartcontractkit/capabilities/integration_tests go 1.26.4 +// Unpublished local stack for SHARED-2709; drop once chainlink-common and +// chainlink-protos/metering/go are tagged. +replace ( + github.com/smartcontractkit/chainlink-common => ../../chainlink-common + github.com/smartcontractkit/chainlink-protos/metering/go => ../../chainlink-protos/metering/go +) + replace github.com/smartcontractkit/capabilities/loadtestwritetarget => ../loadtestwritetarget replace github.com/smartcontractkit/capabilities/http_action => ../http_action @@ -26,10 +33,10 @@ require ( github.com/smartcontractkit/capabilities/http_trigger v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/capabilities/loadtestwritetarget v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chain-selectors v1.0.104 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260609161557-8ceae53b8ab1 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260512150409-b4068bf735e6 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528221400-84746b70eeeb github.com/smartcontractkit/chainlink/v2 v2.29.1-cre-beta.0.0.20260609174137-e2407e0bdd98 github.com/smartcontractkit/cre-sdk-go v1.5.0 @@ -314,7 +321,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc // indirect github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260428133800-3b1484e8b1fd // indirect github.com/smartcontractkit/chainlink-common/keystore v1.2.0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect @@ -329,6 +336,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.10.1-0.20260528221400-84746b70eeeb // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/integration_tests/go.sum b/integration_tests/go.sum index ce890a667..493a16f69 100644 --- a/integration_tests/go.sum +++ b/integration_tests/go.sum @@ -1159,12 +1159,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260428133800-3b1484e8b1fd h1:IMopuENFVS63AerRELdfWo6o60UNUidcldJOxJLmk24= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260428133800-3b1484e8b1fd/go.mod h1:SBN8Urnh5sQvrQRbSo1Nr8coWatHg8LZoPw3R/42sho= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37 h1:ZZlU2e+hVC1Y8VAczVNGZBM3rU3HSqkOCn2KZHHV7gc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260619153749-934b00c44d37/go.mod h1:paOB/6dy57owHtOGzhgaRBWRDT5BEWfnJF5M7sgkcro= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0 h1:NExKM/D0HneOq/N5LGTbkV4VOa0UHCvfTNEb4GqYpto= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260601211238-9f526774fef0/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a h1:8bIqv4r7SgDWkXL2Qz/Ijw+YjZY1uroIte3E2v2keVk= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260609161557-8ceae53b8ab1 h1:VdJBtNmasHzISQQF0k0LHFh44WDKO7S00VyaT7qykuc= @@ -1193,12 +1193,14 @@ github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0. github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:ATjAPIVJibHRcIfiG47rEQkUIOoYa6KDvWj3zwCAw6g= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d h1:AJy55QJ/pBhXkZjc7N+ATnWfxrcjq9BI9DmdtdjwDUQ= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:5JdppgngCOUS76p61zCinSCgOhPeYQ+OcDUuome5THQ= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7 h1:iRFmfMFQtcnhGDjCuARQG4MPbcmbbJDDw7MUH3GcGy8= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260618082634-432eb85805e7/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b h1:VDgJWDipihV9f7M5+d21d1RzSsg5rEv+iI12oN1VQbo= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 h1:SG+wAsNyAcA6Kk19ljuxi3HK9Ll2lpHik8OKoY4x7A0= github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/orchestrator v0.10.1-0.20260528221400-84746b70eeeb h1:G8X3SR21VYAHWkDkNGZCjsrWrLJoVmXMpYBa2KKK3GU=