From 1e0b8315c1caa72e2f7ca45fc44e4be64aae09fe Mon Sep 17 00:00:00 2001 From: envestcc Date: Wed, 29 Oct 2025 15:44:14 +0800 Subject: [PATCH 1/8] state candidate list storage optimaze --- action/protocol/staking/candidate.go | 42 +++++++---- state/candidate.go | 80 ++++++++++++++++---- state/candidate_linklist.go | 107 +++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 29 deletions(-) create mode 100644 state/candidate_linklist.go diff --git a/action/protocol/staking/candidate.go b/action/protocol/staking/candidate.go index 19d600f2f4..5f4ce3d9a6 100644 --- a/action/protocol/staking/candidate.go +++ b/action/protocol/staking/candidate.go @@ -18,7 +18,6 @@ import ( "github.com/iotexproject/iotex-core/v2/action" "github.com/iotexproject/iotex-core/v2/action/protocol/staking/stakingpb" - "github.com/iotexproject/iotex-core/v2/pkg/util/byteutil" "github.com/iotexproject/iotex-core/v2/state" "github.com/iotexproject/iotex-core/v2/systemcontracts" ) @@ -429,7 +428,11 @@ func (l *CandidateList) Encode() ([][]byte, []systemcontracts.GenericValue, erro if err != nil { return nil, nil, errors.Wrap(err, "failed to encode candidate") } - gv.AuxiliaryData = byteutil.Uint64ToBytes(uint64(idx)) + var nextAddr []byte + if idx < len(*l)-1 { + nextAddr = (*l)[idx+1].GetIdentifier().Bytes() + } + gv.AuxiliaryData = nextAddr keys = append(keys, key) values = append(values, gv) } @@ -438,20 +441,31 @@ func (l *CandidateList) Encode() ([][]byte, []systemcontracts.GenericValue, erro // Decode decodes candidate list from generic value func (l *CandidateList) Decode(keys [][]byte, gvs []systemcontracts.GenericValue) error { - // reconstruct candidate list - // the order of keys and gvs are guaranteed to be the same - candidateMap := make(map[uint64]*Candidate) - for _, gv := range gvs { + if len(keys) != len(gvs) { + return errors.New("mismatched keys and generic values length") + } + if len(keys) == 0 { + *l = CandidateList{} + return nil + } + + candidates, err := state.DecodeOrderedKvList(keys, gvs, func(k []byte, v systemcontracts.GenericValue) (*Candidate, string, string, error) { c := &Candidate{} - if err := c.Decode(gv); err != nil { - return errors.Wrap(err, "failed to decode candidate") + if err := c.Decode(v); err != nil { + return nil, "", "", errors.Wrap(err, "failed to decode candidate") } - idx := byteutil.BytesToUint64(gv.AuxiliaryData) - candidateMap[idx] = c - } - candidates := make(CandidateList, 0, len(candidateMap)) - for i := 0; i < len(candidateMap); i++ { - candidates = append(candidates, candidateMap[uint64(i)]) + var next string + if len(v.AuxiliaryData) > 0 { + nextAddr, err := address.FromBytes(v.AuxiliaryData) + if err != nil { + return nil, "", "", errors.Wrap(err, "failed to get next candidate address") + } + next = nextAddr.String() + } + return c, c.GetIdentifier().String(), next, nil + }) + if err != nil { + return errors.Wrap(err, "failed to decode candidate list") } *l = candidates return nil diff --git a/state/candidate.go b/state/candidate.go index 8ba4fa1da8..21b54ed877 100644 --- a/state/candidate.go +++ b/state/candidate.go @@ -15,7 +15,6 @@ import ( "github.com/pkg/errors" "google.golang.org/protobuf/proto" - "github.com/iotexproject/iotex-core/v2/pkg/util/byteutil" "github.com/iotexproject/iotex-core/v2/systemcontracts" "github.com/iotexproject/iotex-address/address" @@ -160,7 +159,16 @@ func (l *CandidateList) Encode() ([][]byte, []systemcontracts.GenericValue, erro values []systemcontracts.GenericValue ) for idx, cand := range *l { - data, err := cand.Serialize() + pbCand := candidateToPb(cand) + dataVotes, err := proto.Marshal(&iotextypes.Candidate{ + Votes: pbCand.Votes, + }) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to serialize candidate votes") + } + pbCand.Address = "" // address is stored in the suffix + pbCand.Votes = nil // votes is stored in the secondary data + data, err := proto.Marshal(pbCand) if err != nil { return nil, nil, errors.Wrap(err, "failed to serialize candidate") } @@ -168,29 +176,71 @@ func (l *CandidateList) Encode() ([][]byte, []systemcontracts.GenericValue, erro if err != nil { return nil, nil, errors.Wrapf(err, "failed to get the hash of the address %s", cand.Address) } + // Use linked list format: AuxiliaryData stores the next node's address + // For the last element, AuxiliaryData is nil + var nextAddr []byte + if idx < len(*l)-1 { + nextAddrObj, err := address.FromString((*l)[idx+1].Address) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get the address of the next candidate %s", (*l)[idx+1].Address) + } + nextAddr = nextAddrObj.Bytes() + } suffix = append(suffix, addr.Bytes()) - values = append(values, systemcontracts.GenericValue{PrimaryData: data, SecondaryData: byteutil.Uint64ToBytes(uint64(idx))}) + values = append(values, systemcontracts.GenericValue{ + PrimaryData: data, + SecondaryData: dataVotes, + AuxiliaryData: nextAddr}) } return suffix, values, nil } // Decode decodes a GenericValue into CandidateList func (l *CandidateList) Decode(suffixs [][]byte, values []systemcontracts.GenericValue) error { - // reconstruct candidate list from values - // the order of candidates in the list is determined by the SecondaryData of GenericValue - candidateMap := make(map[uint64]*Candidate) - for _, gv := range values { - cand := &Candidate{} - if err := cand.Deserialize(gv.PrimaryData); err != nil { - return errors.Wrap(err, "failed to deserialize candidate") + if len(suffixs) != len(values) { + return errors.New("suffix and values length mismatch") + } + if len(suffixs) == 0 { + *l = CandidateList{} + return nil + } + + decoder := func(k []byte, v systemcontracts.GenericValue) (*Candidate, string, string, error) { + pb := &iotextypes.Candidate{} + if err := proto.Unmarshal(v.PrimaryData, pb); err != nil { + return nil, "", "", errors.Wrap(err, "failed to unmarshal candidate") + } + addr, err := address.FromBytes(k) + if err != nil { + return nil, "", "", errors.Wrapf(err, "failed to get the string of the address from bytes %x", k) + } + pb.Address = addr.String() + // Load votes from SecondaryData + pbVotes := &iotextypes.Candidate{} + if err := proto.Unmarshal(v.SecondaryData, pbVotes); err != nil { + return nil, "", "", errors.Wrap(err, "failed to unmarshal candidate votes") + } + pb.Votes = pbVotes.Votes + // Convert pb to candidate + cand, err := pbToCandidate(pb) + if err != nil { + return nil, "", "", errors.Wrap(err, "failed to convert protobuf's candidate message to candidate") } - index := byteutil.BytesToUint64(gv.SecondaryData) - candidateMap[index] = cand + var next string + if len(v.AuxiliaryData) > 0 { + nextAddr, err := address.FromBytes(v.AuxiliaryData) + if err != nil { + return nil, "", "", errors.Wrapf(err, "failed to get the string of the next address from bytes %x", v.AuxiliaryData) + } + next = nextAddr.String() + } + return cand, addr.String(), next, nil } - candidates := make(CandidateList, 0, len(candidateMap)) - for i := 0; i < len(candidateMap); i++ { - candidates = append(candidates, candidateMap[uint64(i)]) + candidates, err := DecodeOrderedKvList(suffixs, values, decoder) + if err != nil { + return err } + *l = candidates return nil } diff --git a/state/candidate_linklist.go b/state/candidate_linklist.go new file mode 100644 index 0000000000..aec672fb2b --- /dev/null +++ b/state/candidate_linklist.go @@ -0,0 +1,107 @@ +package state + +import ( + "github.com/pkg/errors" + + "github.com/iotexproject/iotex-core/v2/systemcontracts" +) + +type linkedListNode[T any] struct { + data T + nextAddr string +} + +// DecodeOrderedKvList decodes a list of key-value pairs representing a linked list into an ordered slice +func DecodeOrderedKvList[T any](keys [][]byte, values []systemcontracts.GenericValue, decoder func(k []byte, v systemcontracts.GenericValue) (T, string, string, error)) ([]T, error) { + nodeMap, err := buildCandidateMap(keys, values, decoder) + if err != nil { + return nil, errors.Wrap(err, "failed to build candidate map") + } + + headAddr, err := findLinkedListHead(nodeMap) + if err != nil { + return nil, errors.Wrap(err, "failed to find head of candidate list") + } + + result, err := traverseLinkedList(headAddr, nodeMap) + if err != nil { + return nil, errors.Wrap(err, "failed to traverse candidate linked list") + } + + return result, nil +} + +// buildCandidateMap builds a generic map from address bytes to candidate nodes +func buildCandidateMap[T any]( + keys [][]byte, + values []systemcontracts.GenericValue, + decoder func(k []byte, v systemcontracts.GenericValue) (T, string, string, error), +) (map[string]*linkedListNode[T], error) { + nodeMap := make(map[string]*linkedListNode[T], len(keys)) + + for kid, gv := range values { + data, id, next, err := decoder(keys[kid], gv) + if err != nil { + return nil, errors.Wrap(err, "failed to decode candidate data") + } + nodeMap[id] = &linkedListNode[T]{ + data: data, + nextAddr: next, + } + } + + return nodeMap, nil +} + +// findLinkedListHead finds the head of the linked list (not pointed to by any node) +func findLinkedListHead[T any](nodeMap map[string]*linkedListNode[T]) (string, error) { + // Mark all addresses that are pointed to + pointedTo := make(map[string]bool, len(nodeMap)) + for _, node := range nodeMap { + if len(node.nextAddr) > 0 { + pointedTo[node.nextAddr] = true + } + } + + // Find the address that is not pointed to by any node + for addrKey := range nodeMap { + if !pointedTo[addrKey] { + return addrKey, nil + } + } + + return "", errors.New("failed to find head of candidate list") +} + +// traverseLinkedList traverses the linked list and returns an ordered list +func traverseLinkedList[T any](headAddr string, nodeMap map[string]*linkedListNode[T]) ([]T, error) { + result := make([]T, 0, len(nodeMap)) + visited := make(map[string]bool, len(nodeMap)) + currentAddr := headAddr + + for currentAddr != "" { + addrKey := currentAddr + + // Check for circular reference + if visited[addrKey] { + return nil, errors.New("circular reference detected in candidate list") + } + visited[addrKey] = true + + // Get current node + node, exists := nodeMap[addrKey] + if !exists { + return nil, errors.Errorf("missing candidate for address %x in linked list", currentAddr) + } + + result = append(result, node.data) + currentAddr = node.nextAddr + } + + // Verify all nodes were traversed + if len(result) != len(nodeMap) { + return nil, errors.Errorf("incomplete traversal: %d/%d candidates", len(result), len(nodeMap)) + } + + return result, nil +} From b9dc3702ad726780567833cd2bec7db5cd17c1af Mon Sep 17 00:00:00 2001 From: envestcc Date: Thu, 30 Oct 2025 14:46:12 +0800 Subject: [PATCH 2/8] misc changes --- .../execution/evm/contract_adapter.go | 2 +- .../protocol/execution/evm/contract_erigon.go | 20 +- .../execution/evm/evmstatedbadapter_erigon.go | 2 +- action/protocol/rewarding/fund.go | 43 +++- action/protocol/rewarding/protocol.go | 36 +++- action/protocol/rewarding/reward.go | 21 ++ action/protocol/staking/candidate.go | 4 +- state/candidate.go | 22 ++- state/factory/erigonstore/accountstorage.go | 15 +- state/factory/erigonstore/iterator.go | 2 +- state/factory/erigonstore/keysplitstorage.go | 44 +++-- state/factory/erigonstore/kvliststorage.go | 30 +-- state/factory/erigonstore/registry.go | 69 +++++-- .../erigonstore/rewardhistorystorage.go | 56 ++++++ .../erigonstore/workingsetstore_erigon.go | 47 ++++- .../workingsetstore_erigon_test.go | 185 ++++++++++++++++++ state/iterator.go | 2 +- .../stakingindex/candidate_votes.go | 12 +- systemcontractindex/stakingindex/voteview.go | 3 +- 19 files changed, 522 insertions(+), 93 deletions(-) create mode 100644 state/factory/erigonstore/rewardhistorystorage.go create mode 100644 state/factory/erigonstore/workingsetstore_erigon_test.go diff --git a/action/protocol/execution/evm/contract_adapter.go b/action/protocol/execution/evm/contract_adapter.go index a57bb6fb59..e7c5938dab 100644 --- a/action/protocol/execution/evm/contract_adapter.go +++ b/action/protocol/execution/evm/contract_adapter.go @@ -20,7 +20,7 @@ func newContractAdapter(addr hash.Hash160, account *state.Account, sm protocol.S if err != nil { return nil, errors.Wrap(err, "failed to create contract") } - v2, err := newContractErigon(addr, account, intra) + v2, err := newContractErigon(addr, account.Clone(), intra, sm) if err != nil { return nil, errors.Wrap(err, "failed to create contractV2") } diff --git a/action/protocol/execution/evm/contract_erigon.go b/action/protocol/execution/evm/contract_erigon.go index 25b1e2cd22..a9dd6c44bc 100644 --- a/action/protocol/execution/evm/contract_erigon.go +++ b/action/protocol/execution/evm/contract_erigon.go @@ -9,21 +9,25 @@ import ( "github.com/iotexproject/go-pkgs/hash" "github.com/pkg/errors" + "github.com/iotexproject/iotex-core/v2/action/protocol" "github.com/iotexproject/iotex-core/v2/db/trie" + "github.com/iotexproject/iotex-core/v2/pkg/log" "github.com/iotexproject/iotex-core/v2/state" ) type contractErigon struct { *state.Account intra *erigonstate.IntraBlockState + sr protocol.StateReader addr hash.Hash160 } -func newContractErigon(addr hash.Hash160, account *state.Account, intra *erigonstate.IntraBlockState) (Contract, error) { +func newContractErigon(addr hash.Hash160, account *state.Account, intra *erigonstate.IntraBlockState, sr protocol.StateReader) (Contract, error) { c := &contractErigon{ Account: account, intra: intra, addr: addr, + sr: sr, } return c, nil } @@ -32,14 +36,16 @@ func (c *contractErigon) GetCommittedState(key hash.Hash256) ([]byte, error) { k := libcommon.Hash(key) v := uint256.NewInt(0) c.intra.GetCommittedState(libcommon.Address(c.addr), &k, v) - return v.Bytes(), nil + h := hash.BytesToHash256(v.Bytes()) + return h[:], nil } func (c *contractErigon) GetState(key hash.Hash256) ([]byte, error) { k := libcommon.Hash(key) v := uint256.NewInt(0) c.intra.GetState(libcommon.Address(c.addr), &k, v) - return v.Bytes(), nil + h := hash.BytesToHash256(v.Bytes()) + return h[:], nil } func (c *contractErigon) SetState(key hash.Hash256, value []byte) error { @@ -58,10 +64,10 @@ func (c *contractErigon) SetCode(hash hash.Hash256, code []byte) { func (c *contractErigon) SelfState() *state.Account { acc := &state.Account{} - acc.SetPendingNonce(c.intra.GetNonce(libcommon.Address(c.addr))) - acc.AddBalance(c.intra.GetBalance(libcommon.Address(c.addr)).ToBig()) - codeHash := c.intra.GetCodeHash(libcommon.Address(c.addr)) - acc.CodeHash = codeHash[:] + _, err := c.sr.State(acc, protocol.LegacyKeyOption(c.addr), protocol.ErigonStoreOnlyOption()) + if err != nil { + log.S().Panicf("failed to load account %x: %v", c.addr, err) + } return acc } diff --git a/action/protocol/execution/evm/evmstatedbadapter_erigon.go b/action/protocol/execution/evm/evmstatedbadapter_erigon.go index 2a5b48e994..abe82495de 100644 --- a/action/protocol/execution/evm/evmstatedbadapter_erigon.go +++ b/action/protocol/execution/evm/evmstatedbadapter_erigon.go @@ -53,7 +53,7 @@ func NewErigonStateDBAdapterDryrun(adapter *StateDBAdapter, ) *ErigonStateDBAdapterDryrun { a := NewErigonStateDBAdapter(adapter, intra) adapter.newContract = func(addr hash.Hash160, account *state.Account) (Contract, error) { - return newContractErigon(addr, account, intra) + return newContractErigon(addr, account, intra, adapter.sm) } return &ErigonStateDBAdapterDryrun{ ErigonStateDBAdapter: a, diff --git a/action/protocol/rewarding/fund.go b/action/protocol/rewarding/fund.go index 2131d447fe..5e319b33c0 100644 --- a/action/protocol/rewarding/fund.go +++ b/action/protocol/rewarding/fund.go @@ -23,6 +23,8 @@ import ( "github.com/iotexproject/iotex-core/v2/systemcontracts" ) +type Fund = fund + // fund stores the balance of the rewarding fund. The difference between total and available balance should be // equal to the unclaimed balance in all reward accounts type fund struct { @@ -59,17 +61,52 @@ func (f *fund) Deserialize(data []byte) error { } func (f *fund) Encode() (systemcontracts.GenericValue, error) { - data, err := f.Serialize() + d1, err := proto.Marshal(&rewardingpb.Fund{ + TotalBalance: f.totalBalance.String(), + }) + if err != nil { + return systemcontracts.GenericValue{}, err + } + d2, err := proto.Marshal(&rewardingpb.Fund{ + UnclaimedBalance: f.unclaimedBalance.String(), + }) if err != nil { return systemcontracts.GenericValue{}, err } return systemcontracts.GenericValue{ - AuxiliaryData: data, + PrimaryData: d1, + SecondaryData: d2, }, nil } func (f *fund) Decode(v systemcontracts.GenericValue) error { - return f.Deserialize(v.AuxiliaryData) + gen1 := rewardingpb.Fund{} + if err := proto.Unmarshal(v.PrimaryData, &gen1); err != nil { + return err + } + var totalBalance = big.NewInt(0) + if len(gen1.TotalBalance) > 0 { + b, ok := new(big.Int).SetString(gen1.TotalBalance, 10) + if !ok { + return errors.Errorf("failed to set total balance from string: %s", gen1.TotalBalance) + } + totalBalance = b + } + gen2 := rewardingpb.Fund{} + if err := proto.Unmarshal(v.SecondaryData, &gen2); err != nil { + return err + } + var unclaimedBalance = big.NewInt(0) + if len(gen2.UnclaimedBalance) > 0 { + b, ok := new(big.Int).SetString(gen2.UnclaimedBalance, 10) + if !ok { + return errors.Errorf("failed to set unclaimed balance from string: %s", gen2.UnclaimedBalance) + } + unclaimedBalance = b + } + f.totalBalance = totalBalance + f.unclaimedBalance = unclaimedBalance + return nil } // Deposit deposits token into the rewarding fund diff --git a/action/protocol/rewarding/protocol.go b/action/protocol/rewarding/protocol.go index 1c7cf02055..813c84b53d 100644 --- a/action/protocol/rewarding/protocol.go +++ b/action/protocol/rewarding/protocol.go @@ -21,6 +21,7 @@ import ( accountutil "github.com/iotexproject/iotex-core/v2/action/protocol/account/util" "github.com/iotexproject/iotex-core/v2/action/protocol/rolldpos" "github.com/iotexproject/iotex-core/v2/blockchain/genesis" + "github.com/iotexproject/iotex-core/v2/pkg/enc" "github.com/iotexproject/iotex-core/v2/pkg/log" "github.com/iotexproject/iotex-core/v2/state" ) @@ -102,6 +103,13 @@ func FindProtocol(registry *protocol.Registry) *Protocol { func (p *Protocol) CreatePreStates(ctx context.Context, sm protocol.StateManager) error { g := genesis.MustExtractGenesisContext(ctx) blkCtx := protocol.MustGetBlockCtx(ctx) + // set current block reward not granted for erigon db + var indexBytes [8]byte + enc.MachineEndian.PutUint64(indexBytes[:], blkCtx.BlockHeight) + err := p.deleteState(ctx, sm, append(_blockRewardHistoryKeyPrefix, indexBytes[:]...), &rewardHistory{}, protocol.ErigonStoreOnlyOption()) + if err != nil && !errors.Is(err, state.ErrErigonStoreNotSupported) { + return err + } switch blkCtx.BlockHeight { case g.AleutianBlockHeight: return p.SetReward(ctx, sm, g.AleutianEpochReward(), false) @@ -348,10 +356,11 @@ func (p *Protocol) putState(ctx context.Context, sm protocol.StateManager, key [ return p.putStateV1(sm, key, value) } -func (p *Protocol) putStateV1(sm protocol.StateManager, key []byte, value interface{}) error { +func (p *Protocol) putStateV1(sm protocol.StateManager, key []byte, value interface{}, opts ...protocol.StateOption) error { orgKey := append(p.keyPrefix, key...) keyHash := hash.Hash160b(orgKey) - _, err := sm.PutState(value, protocol.LegacyKeyOption(keyHash), protocol.ErigonStoreKeyOption(orgKey)) + opts = append(opts, protocol.LegacyKeyOption(keyHash), protocol.ErigonStoreKeyOption(orgKey)) + _, err := sm.PutState(value, opts...) return err } @@ -361,10 +370,29 @@ func (p *Protocol) putStateV2(sm protocol.StateManager, key []byte, value interf return err } -func (p *Protocol) deleteStateV1(sm protocol.StateManager, key []byte, obj any) error { +func (p *Protocol) deleteState(ctx context.Context, sm protocol.StateManager, key []byte, obj any, opts ...protocol.StateOption) error { + if useV2Storage(ctx) { + return p.deleteStateV2(sm, key, obj, opts...) + } + return p.deleteStateV1(sm, key, obj, opts...) +} + +func (p *Protocol) deleteStateV1(sm protocol.StateManager, key []byte, obj any, opts ...protocol.StateOption) error { orgKey := append(p.keyPrefix, key...) keyHash := hash.Hash160b(orgKey) - _, err := sm.DelState(protocol.LegacyKeyOption(keyHash), protocol.ObjectOption(obj), protocol.ErigonStoreKeyOption(orgKey)) + opt := append(opts, protocol.LegacyKeyOption(keyHash), protocol.ObjectOption(obj), protocol.ErigonStoreKeyOption(orgKey)) + _, err := sm.DelState(opt...) + if errors.Cause(err) == state.ErrStateNotExist { + // don't care if not exist + return nil + } + return err +} + +func (p *Protocol) deleteStateV2(sm protocol.StateManager, key []byte, value any, opts ...protocol.StateOption) error { + k := append(p.keyPrefix, key...) + opt := append(opts, protocol.KeyOption(k), protocol.ObjectOption(value), protocol.NamespaceOption(_v2RewardingNamespace)) + _, err := sm.DelState(opt...) if errors.Cause(err) == state.ErrStateNotExist { // don't care if not exist return nil diff --git a/action/protocol/rewarding/reward.go b/action/protocol/rewarding/reward.go index 574bee10b7..fdb3c53073 100644 --- a/action/protocol/rewarding/reward.go +++ b/action/protocol/rewarding/reward.go @@ -10,6 +10,7 @@ import ( "math/big" "github.com/pkg/errors" + "go.uber.org/zap" "google.golang.org/protobuf/proto" "github.com/iotexproject/go-pkgs/hash" @@ -29,6 +30,8 @@ import ( "github.com/iotexproject/iotex-core/v2/systemcontracts" ) +type RewardHistory = rewardHistory + // rewardHistory is the dummy struct to record a reward. Only key matters. type rewardHistory struct{} @@ -114,6 +117,24 @@ func (p *Protocol) GrantBlockReward( ) (*action.Log, error) { actionCtx := protocol.MustGetActionCtx(ctx) blkCtx := protocol.MustGetBlockCtx(ctx) + fCtx := protocol.MustGetFeatureCtx(ctx) + + if fCtx.UseV2Storage { + var indexBytes [8]byte + enc.MachineEndian.PutUint64(indexBytes[:], blkCtx.BlockHeight) + key := append(_blockRewardHistoryKeyPrefix, indexBytes[:]...) + err := p.deleteStateV1(sm, key, &rewardHistory{}, protocol.ErigonStoreOnlyOption()) + if err != nil && !errors.Is(err, state.ErrErigonStoreNotSupported) { + return nil, err + } + defer func() { + err = p.putStateV1(sm, key, &rewardHistory{}, protocol.ErigonStoreOnlyOption()) + if err != nil && !errors.Is(err, state.ErrErigonStoreNotSupported) { + log.L().Panic("failed to put block reward history in Erigon store", zap.Error(err)) + } + }() + } + if err := p.assertNoRewardYet(ctx, sm, _blockRewardHistoryKeyPrefix, blkCtx.BlockHeight); err != nil { return nil, err } diff --git a/action/protocol/staking/candidate.go b/action/protocol/staking/candidate.go index 5f4ce3d9a6..f8a10879d0 100644 --- a/action/protocol/staking/candidate.go +++ b/action/protocol/staking/candidate.go @@ -417,7 +417,7 @@ func (l *CandidateList) Deserialize(buf []byte) error { } // Encode encodes candidate list into generic value -func (l *CandidateList) Encode() ([][]byte, []systemcontracts.GenericValue, error) { +func (l *CandidateList) Encodes() ([][]byte, []systemcontracts.GenericValue, error) { var ( keys [][]byte values []systemcontracts.GenericValue @@ -440,7 +440,7 @@ func (l *CandidateList) Encode() ([][]byte, []systemcontracts.GenericValue, erro } // Decode decodes candidate list from generic value -func (l *CandidateList) Decode(keys [][]byte, gvs []systemcontracts.GenericValue) error { +func (l *CandidateList) Decodes(keys [][]byte, gvs []systemcontracts.GenericValue) error { if len(keys) != len(gvs) { return errors.New("mismatched keys and generic values length") } diff --git a/state/candidate.go b/state/candidate.go index 21b54ed877..d54029c401 100644 --- a/state/candidate.go +++ b/state/candidate.go @@ -153,7 +153,23 @@ func (l *CandidateList) LoadProto(candList *iotextypes.CandidateList) error { } // Encode encodes a CandidateList into a GenericValue -func (l *CandidateList) Encode() ([][]byte, []systemcontracts.GenericValue, error) { +func (l *CandidateList) Encode() (systemcontracts.GenericValue, error) { + data, err := l.Serialize() + if err != nil { + return systemcontracts.GenericValue{}, errors.Wrap(err, "failed to serialize candidate list") + } + return systemcontracts.GenericValue{ + PrimaryData: data, + }, nil +} + +// Decode decodes a GenericValue into CandidateList +func (l *CandidateList) Decode(gv systemcontracts.GenericValue) error { + return l.Deserialize(gv.PrimaryData) +} + +// Encodes encodes a CandidateList into a GenericValue +func (l *CandidateList) Encodes() ([][]byte, []systemcontracts.GenericValue, error) { var ( suffix [][]byte values []systemcontracts.GenericValue @@ -195,8 +211,8 @@ func (l *CandidateList) Encode() ([][]byte, []systemcontracts.GenericValue, erro return suffix, values, nil } -// Decode decodes a GenericValue into CandidateList -func (l *CandidateList) Decode(suffixs [][]byte, values []systemcontracts.GenericValue) error { +// Decodes decodes a GenericValue into CandidateList +func (l *CandidateList) Decodes(suffixs [][]byte, values []systemcontracts.GenericValue) error { if len(suffixs) != len(values) { return errors.New("suffix and values length mismatch") } diff --git a/state/factory/erigonstore/accountstorage.go b/state/factory/erigonstore/accountstorage.go index 594b210208..8a41b19873 100644 --- a/state/factory/erigonstore/accountstorage.go +++ b/state/factory/erigonstore/accountstorage.go @@ -2,7 +2,6 @@ package erigonstore import ( erigonComm "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" "github.com/pkg/errors" @@ -81,14 +80,18 @@ func (as *accountStorage) Load(key []byte, obj any) error { case accountpb.AccountType_ZERO_NONCE: pbAcc.Nonce = nonce case accountpb.AccountType_DEFAULT: - pbAcc.Nonce = nonce - 1 + if nonce == 0 { + pbAcc.Nonce = nonce + } else { + pbAcc.Nonce = nonce - 1 + } default: return errors.Errorf("unknown account type %v for address %x", pbAcc.Type, addr.Bytes()) } - - if ch := as.backend.intraBlockState.GetCodeHash(addr); !accounts.IsEmptyCodeHash(ch) { - pbAcc.CodeHash = ch.Bytes() - } + // if ch := as.backend.intraBlockState.GetCodeHash(addr); !accounts.IsEmptyCodeHash(ch) { + // pbAcc.CodeHash = ch.Bytes() + // } + pbAcc.CodeHash = as.backend.intraBlockState.GetCodeHash(addr).Bytes() acct.FromProto(pbAcc) return nil } diff --git a/state/factory/erigonstore/iterator.go b/state/factory/erigonstore/iterator.go index c89498e554..ad83b7ac07 100644 --- a/state/factory/erigonstore/iterator.go +++ b/state/factory/erigonstore/iterator.go @@ -35,11 +35,11 @@ func (gvoi *GenericValueObjectIterator) Next(o interface{}) ([]byte, error) { } value := gvoi.values[gvoi.cur] key := gvoi.keys[gvoi.cur] - gvoi.cur++ if gvoi.exists != nil && !gvoi.exists[gvoi.cur] { gvoi.cur++ return key, state.ErrNilValue } + gvoi.cur++ if err := systemcontracts.DecodeGenericValue(o, value); err != nil { return nil, err } diff --git a/state/factory/erigonstore/keysplitstorage.go b/state/factory/erigonstore/keysplitstorage.go index b093eeed61..7cc0bca4c4 100644 --- a/state/factory/erigonstore/keysplitstorage.go +++ b/state/factory/erigonstore/keysplitstorage.go @@ -1,9 +1,10 @@ package erigonstore import ( + "github.com/pkg/errors" + "github.com/iotexproject/iotex-core/v2/state" "github.com/iotexproject/iotex-core/v2/systemcontracts" - "github.com/pkg/errors" ) type KeySplitter func(key []byte) (part1 []byte, part2 []byte) @@ -14,27 +15,30 @@ type keySplitContainer interface { } type keySplitContractStorage struct { - contract systemcontracts.StorageContract - keySplit KeySplitter - fallback ObjectStorage + contract systemcontracts.StorageContract + keySplit KeySplitter + keyPrefixStorage map[string]ObjectStorage + fallback ObjectStorage } -func newKeySplitContractStorageWithfallback(contract systemcontracts.StorageContract, split KeySplitter, fallback ObjectStorage) *keySplitContractStorage { +func newKeySplitContractStorageWithfallback(contract systemcontracts.StorageContract, split KeySplitter, fallback ObjectStorage, keyPrefixStorage map[string]ObjectStorage) *keySplitContractStorage { return &keySplitContractStorage{ - contract: contract, - keySplit: split, - fallback: fallback, + contract: contract, + keySplit: split, + fallback: fallback, + keyPrefixStorage: keyPrefixStorage, } } func (rhs *keySplitContractStorage) Store(key []byte, obj any) error { pf, sf := rhs.keySplit(key) + fallback := rhs.matchStorage(key) if len(sf) == 0 { - return rhs.fallback.Store(key, obj) + return fallback.Store(key, obj) } gvc, ok := obj.(keySplitContainer) if !ok { - return rhs.fallback.Store(pf, obj) + return fallback.Store(pf, obj) } value, err := gvc.Encode(sf) if err != nil { @@ -45,12 +49,13 @@ func (rhs *keySplitContractStorage) Store(key []byte, obj any) error { func (rhs *keySplitContractStorage) Load(key []byte, obj any) error { pf, sf := rhs.keySplit(key) + fallback := rhs.matchStorage(key) if len(sf) == 0 { - return rhs.fallback.Load(key, obj) + return fallback.Load(key, obj) } gvc, ok := obj.(keySplitContainer) if !ok { - return rhs.fallback.Load(pf, obj) + return fallback.Load(pf, obj) } value, err := rhs.contract.Get(pf) if err != nil { @@ -64,10 +69,11 @@ func (rhs *keySplitContractStorage) Load(key []byte, obj any) error { func (rhs *keySplitContractStorage) Delete(key []byte) error { pf, sf := rhs.keySplit(key) + fallback := rhs.matchStorage(key) if len(sf) == 0 { - return rhs.fallback.Delete(key) + return fallback.Delete(key) } - return rhs.fallback.Delete(pf) + return fallback.Delete(pf) } func (rhs *keySplitContractStorage) List() (state.Iterator, error) { @@ -77,3 +83,13 @@ func (rhs *keySplitContractStorage) List() (state.Iterator, error) { func (rhs *keySplitContractStorage) Batch(keys [][]byte) (state.Iterator, error) { return nil, errors.New("not implemented") } + +func (rhs *keySplitContractStorage) matchStorage(key []byte) ObjectStorage { + for prefix, os := range rhs.keyPrefixStorage { + sk := string(key) + if len(sk) >= len(prefix) && sk[:len(prefix)] == prefix { + return os + } + } + return rhs.fallback +} diff --git a/state/factory/erigonstore/kvliststorage.go b/state/factory/erigonstore/kvliststorage.go index 358a6e62e6..3830a6a667 100644 --- a/state/factory/erigonstore/kvliststorage.go +++ b/state/factory/erigonstore/kvliststorage.go @@ -3,14 +3,15 @@ package erigonstore import ( "bytes" + "github.com/pkg/errors" + "github.com/iotexproject/iotex-core/v2/state" "github.com/iotexproject/iotex-core/v2/systemcontracts" - "github.com/pkg/errors" ) type kvListContainer interface { - Encode() ([][]byte, []systemcontracts.GenericValue, error) - Decode(keys [][]byte, values []systemcontracts.GenericValue) error + Encodes() ([][]byte, []systemcontracts.GenericValue, error) + Decodes(keys [][]byte, values []systemcontracts.GenericValue) error } type kvListStorage struct { @@ -28,7 +29,15 @@ func (cos *kvListStorage) Store(prefix []byte, obj any) error { if !ok { return errors.Errorf("object of type %T does not supported", obj) } - keys, values, err := ct.Encode() + cnt, err := cos.contract.Count() + if err != nil { + return err + } + retval, err := cos.contract.ListKeys(0, cnt.Uint64()) + if err != nil { + return err + } + keys, values, err := ct.Encodes() if err != nil { return err } @@ -41,14 +50,6 @@ func (cos *kvListStorage) Store(prefix []byte, obj any) error { newKeys[string(nk)] = struct{}{} } // remove keys not in the new list - cnt, err := cos.contract.Count() - if err != nil { - return err - } - retval, err := cos.contract.ListKeys(0, cnt.Uint64()) - if err != nil { - return err - } for _, k := range retval.KeyList { if len(k) >= len(prefix) && bytes.Equal(k[:len(prefix)], prefix) { if _, exists := newKeys[string(k)]; !exists { @@ -85,7 +86,10 @@ func (cos *kvListStorage) Load(prefix []byte, obj any) error { values = append(values, retval.Values[i]) } } - return ct.Decode(keys, values) + if len(keys) == 0 { + return errors.Wrapf(state.ErrStateNotExist, "prefix: %x", prefix) + } + return ct.Decodes(keys, values) } func (cos *kvListStorage) Delete(prefix []byte) error { diff --git a/state/factory/erigonstore/registry.go b/state/factory/erigonstore/registry.go index e66b27cff1..bde277ff41 100644 --- a/state/factory/erigonstore/registry.go +++ b/state/factory/erigonstore/registry.go @@ -29,11 +29,17 @@ var ( // ObjectStorageRegistry is a registry for object storage type ObjectStorageRegistry struct { - contracts map[string]map[reflect.Type]int - ns map[string]int - nsPrefix map[string]int - spliter map[int]KeySplitter - kvList map[int]struct{} + contracts map[string]map[reflect.Type]int + ns map[string]int + nsPrefix map[string]int + spliter map[int]KeySplitter + kvList map[int]struct{} + rewardingHistoryPrefix map[int]rewardingHistoryConfig +} + +type rewardingHistoryConfig struct { + Prefixs [][]byte + KeySplit KeySplitter } type RegisterOption func(int, *ObjectStorageRegistry) @@ -50,14 +56,23 @@ func WithKVListOption() RegisterOption { } } +func WithRewardingHistoryPrefixOption(split KeySplitter, prefix ...[]byte) RegisterOption { + return func(index int, osr *ObjectStorageRegistry) { + osr.rewardingHistoryPrefix[index] = rewardingHistoryConfig{ + Prefixs: prefix, + KeySplit: split, + } + } +} + func init() { rewardHistoryPrefixs := [][]byte{ append(state.RewardingKeyPrefix[:], state.BlockRewardHistoryKeyPrefix...), append(state.RewardingKeyPrefix[:], state.EpochRewardHistoryKeyPrefix...), } - pollPrefix := [][]byte{ - []byte(state.PollCandidatesPrefix), - } + // pollPrefix := [][]byte{ + // []byte(state.PollCandidatesPrefix), + // } genKeySplit := func(prefixs [][]byte) KeySplitter { return func(key []byte) (part1 []byte, part2 []byte) { for _, p := range prefixs { @@ -69,11 +84,12 @@ func init() { return key, nil } } - rewardKeySplit := genKeySplit(rewardHistoryPrefixs) - pollKeySplit := genKeySplit(pollPrefix) + epochRewardKeySplit := genKeySplit(rewardHistoryPrefixs[1:]) + blockRewardKeySplit := genKeySplit(rewardHistoryPrefixs[:1]) + // pollKeySplit := genKeySplit(pollPrefix) - assertions.MustNoError(storageRegistry.RegisterNamespace(state.AccountKVNamespace, RewardingContractV1Index, WithKeySplitOption(rewardKeySplit))) - assertions.MustNoError(storageRegistry.RegisterNamespace(state.RewardingNamespace, RewardingContractV2Index, WithKeySplitOption(rewardKeySplit))) + assertions.MustNoError(storageRegistry.RegisterNamespace(state.AccountKVNamespace, RewardingContractV1Index, WithKeySplitOption(epochRewardKeySplit), WithRewardingHistoryPrefixOption(blockRewardKeySplit, rewardHistoryPrefixs[0]))) + assertions.MustNoError(storageRegistry.RegisterNamespace(state.RewardingNamespace, RewardingContractV2Index, WithKeySplitOption(epochRewardKeySplit), WithRewardingHistoryPrefixOption(blockRewardKeySplit, rewardHistoryPrefixs[0]))) assertions.MustNoError(storageRegistry.RegisterNamespace(state.CandidateNamespace, CandidatesContractIndex)) assertions.MustNoError(storageRegistry.RegisterNamespace(state.CandsMapNamespace, CandidateMapContractIndex, WithKVListOption())) assertions.MustNoError(storageRegistry.RegisterNamespace(state.StakingNamespace, BucketPoolContractIndex)) @@ -83,7 +99,7 @@ func init() { assertions.MustNoError(storageRegistry.RegisterNamespacePrefix(state.ContractStakingBucketTypeNamespacePrefix, ContractStakingBucketContractIndex)) assertions.MustNoError(storageRegistry.RegisterObjectStorage(state.AccountKVNamespace, &state.Account{}, AccountIndex)) - assertions.MustNoError(storageRegistry.RegisterObjectStorage(state.AccountKVNamespace, &state.CandidateList{}, PollLegacyCandidateListContractIndex, WithKeySplitOption(pollKeySplit), WithKVListOption())) + assertions.MustNoError(storageRegistry.RegisterObjectStorage(state.AccountKVNamespace, &state.CandidateList{}, PollLegacyCandidateListContractIndex)) assertions.MustNoError(storageRegistry.RegisterObjectStorage(state.SystemNamespace, &state.CandidateList{}, PollCandidateListContractIndex, WithKVListOption())) assertions.MustNoError(storageRegistry.RegisterObjectStorage(state.SystemNamespace, &vote.UnproductiveDelegate{}, PollUnproductiveDelegateContractIndex)) assertions.MustNoError(storageRegistry.RegisterObjectStorage(state.SystemNamespace, &vote.ProbationList{}, PollProbationListContractIndex)) @@ -100,11 +116,12 @@ func GetObjectStorageRegistry() *ObjectStorageRegistry { func newObjectStorageRegistry() *ObjectStorageRegistry { return &ObjectStorageRegistry{ - contracts: make(map[string]map[reflect.Type]int), - ns: make(map[string]int), - nsPrefix: make(map[string]int), - spliter: make(map[int]KeySplitter), - kvList: make(map[int]struct{}), + contracts: make(map[string]map[reflect.Type]int), + ns: make(map[string]int), + nsPrefix: make(map[string]int), + spliter: make(map[int]KeySplitter), + kvList: make(map[int]struct{}), + rewardingHistoryPrefix: make(map[int]rewardingHistoryConfig), } } @@ -135,7 +152,13 @@ func (osr *ObjectStorageRegistry) ObjectStorage(ns string, obj any, backend *con os = newKVListStorage(contract) } if split != nil { - os = newKeySplitContractStorageWithfallback(contract, split, os) + prefixFallbacks := map[string]ObjectStorage{} + if config, ok := osr.rewardingHistoryPrefix[contractIndex]; ok { + for _, prefix := range config.Prefixs { + prefixFallbacks[string(prefix)] = newRewardHistoryStorage(contract, backend.height, config.KeySplit) + } + } + os = newKeySplitContractStorageWithfallback(contract, split, os, prefixFallbacks) } return os, nil default: @@ -152,7 +175,13 @@ func (osr *ObjectStorageRegistry) ObjectStorage(ns string, obj any, backend *con os = newKVListStorage(contract) } if split != nil { - os = newKeySplitContractStorageWithfallback(contract, split, os) + prefixFallbacks := map[string]ObjectStorage{} + if config, ok := osr.rewardingHistoryPrefix[contractIndex]; ok { + for _, prefix := range config.Prefixs { + prefixFallbacks[string(prefix)] = newRewardHistoryStorage(contract, backend.height, config.KeySplit) + } + } + os = newKeySplitContractStorageWithfallback(contract, split, os, prefixFallbacks) } return os, nil } diff --git a/state/factory/erigonstore/rewardhistorystorage.go b/state/factory/erigonstore/rewardhistorystorage.go new file mode 100644 index 0000000000..c583cd6f81 --- /dev/null +++ b/state/factory/erigonstore/rewardhistorystorage.go @@ -0,0 +1,56 @@ +package erigonstore + +import ( + "github.com/pkg/errors" + + "github.com/iotexproject/iotex-core/v2/pkg/enc" + "github.com/iotexproject/iotex-core/v2/state" + "github.com/iotexproject/iotex-core/v2/systemcontracts" +) + +type rewardHistoryStorage struct { + contract systemcontracts.StorageContract + height uint64 + keySplit KeySplitter +} + +func newRewardHistoryStorage(contract systemcontracts.StorageContract, height uint64, keySplit KeySplitter) *rewardHistoryStorage { + return &rewardHistoryStorage{ + contract: contract, + height: height, + keySplit: keySplit, + } +} + +func (ds *rewardHistoryStorage) Store(key []byte, obj any) error { + _, err := ds.contract.Remove(key) + return err +} + +func (ds *rewardHistoryStorage) Delete(key []byte) error { + return ds.contract.Put(key, systemcontracts.GenericValue{}) +} + +func (ds *rewardHistoryStorage) Load(key []byte, obj any) error { + _, heightKey := ds.keySplit(key) + height := enc.MachineEndian.Uint64(heightKey) + if height > ds.height { + return errors.Wrapf(state.ErrStateNotExist, "key: %x at height %d", key, height) + } + result, err := ds.contract.Get(key) + if err != nil { + return errors.Wrapf(err, "failed to get data for key %x", key) + } + if result.KeyExists { + return errors.Wrapf(state.ErrStateNotExist, "key: %x", key) + } + return nil +} + +func (ds *rewardHistoryStorage) Batch(keys [][]byte) (state.Iterator, error) { + return nil, errors.New("not implemented") +} + +func (ds *rewardHistoryStorage) List() (state.Iterator, error) { + return nil, errors.New("not implemented") +} diff --git a/state/factory/erigonstore/workingsetstore_erigon.go b/state/factory/erigonstore/workingsetstore_erigon.go index b852267195..2a6e0b0582 100644 --- a/state/factory/erigonstore/workingsetstore_erigon.go +++ b/state/factory/erigonstore/workingsetstore_erigon.go @@ -47,9 +47,11 @@ type ErigonDB struct { // ErigonWorkingSetStore implements the Erigon working set store type ErigonWorkingSetStore struct { - db *ErigonDB - backend *contractBackend - tx kv.Tx + db *ErigonDB + backend *contractBackend + tx kv.Tx + clean *contractBackend + statesReadBuffer bool } // NewErigonDB creates a new ErigonDB @@ -88,10 +90,13 @@ func (db *ErigonDB) NewErigonStore(ctx context.Context, height uint64) (*ErigonW } r := erigonstate.NewPlainStateReader(tx) intraBlockState := erigonstate.New(r) + g := genesis.MustExtractGenesisContext(ctx) return &ErigonWorkingSetStore{ - db: db, - tx: tx, - backend: newContractBackend(ctx, intraBlockState, r), + db: db, + tx: tx, + backend: newContractBackend(ctx, intraBlockState, r), + clean: newContractBackend(ctx, erigonstate.New(r), r), + statesReadBuffer: g.IsNewfoundland(height), }, nil } @@ -103,10 +108,13 @@ func (db *ErigonDB) NewErigonStoreDryrun(ctx context.Context, height uint64) (*E } tsw := erigonstate.NewPlainState(tx, height, nil) intraBlockState := erigonstate.New(tsw) + g := genesis.MustExtractGenesisContext(ctx) return &ErigonWorkingSetStore{ - db: db, - tx: tx, - backend: newContractBackend(ctx, intraBlockState, tsw), + db: db, + tx: tx, + backend: newContractBackend(ctx, intraBlockState, tsw), + clean: newContractBackend(ctx, erigonstate.New(tsw), tsw), + statesReadBuffer: g.IsNewfoundland(height), }, nil } @@ -337,7 +345,13 @@ func (store *ErigonWorkingSetStore) DeleteObject(ns string, key []byte, obj any) // States gets multiple objects from the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) States(ns string, obj any, keys [][]byte) (state.Iterator, error) { - storage, err := store.NewObjectStorage(ns, obj) + var storage ObjectStorage + var err error + if store.statesReadBuffer { + storage, err = store.NewObjectStorage(ns, obj) + } else { + storage, err = store.NewObjectStorageClean(ns, obj) + } if err != nil { return nil, err } @@ -441,6 +455,19 @@ func (store *ErigonWorkingSetStore) NewObjectStorage(ns string, obj any) (Object } } +func (store *ErigonWorkingSetStore) NewObjectStorageClean(ns string, obj any) (ObjectStorage, error) { + cs, err := storageRegistry.ObjectStorage(ns, obj, store.clean) + switch errors.Cause(err) { + case nil: + return cs, nil + case ErrObjectStorageNotRegistered: + // TODO: fail unknown namespace + return nil, nil + default: + return nil, err + } +} + func (store *ErigonWorkingSetStore) ErigonStore() (any, error) { return store, nil } diff --git a/state/factory/erigonstore/workingsetstore_erigon_test.go b/state/factory/erigonstore/workingsetstore_erigon_test.go new file mode 100644 index 0000000000..34aa2a65b0 --- /dev/null +++ b/state/factory/erigonstore/workingsetstore_erigon_test.go @@ -0,0 +1,185 @@ +package erigonstore + +import ( + "context" + "fmt" + "math/big" + "os" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/iotexproject/iotex-core/v2/action/protocol" + "github.com/iotexproject/iotex-core/v2/action/protocol/rewarding" + "github.com/iotexproject/iotex-core/v2/action/protocol/rewarding/rewardingpb" + "github.com/iotexproject/iotex-core/v2/action/protocol/staking" + "github.com/iotexproject/iotex-core/v2/blockchain/genesis" + "github.com/iotexproject/iotex-core/v2/pkg/enc" + "github.com/iotexproject/iotex-core/v2/state" + "github.com/iotexproject/iotex-core/v2/test/identityset" +) + +func TestErigonStoreNativeState(t *testing.T) { + r := require.New(t) + dbpath := "./data/erigonstore_testdb" + r.NoError(os.RemoveAll(dbpath)) + edb := NewErigonDB(dbpath) + r.NoError(edb.Start(context.Background())) + defer func() { + edb.Stop(context.Background()) + }() + g := genesis.TestDefault() + + fmt.Printf("block: %d -----------------------\n", 0) + ctx := context.Background() + ctx = genesis.WithGenesisContext(ctx, g) + ctx = protocol.WithBlockchainCtx(ctx, protocol.BlockchainCtx{}) + ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{BlockHeight: 0}) + ctx = protocol.WithFeatureCtx(ctx) + store, err := edb.NewErigonStore(ctx, 0) + r.NoError(err) + r.NoError(store.CreateGenesisStates(ctx)) + r.NoError(store.FinalizeTx(ctx)) + r.NoError(store.Commit(ctx, 0)) + + height := uint64(1) + t.Run("state.CandidateList", func(t *testing.T) { + fmt.Printf("block: %d -----------------------\n", height) + ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{BlockHeight: height}) + ctx = protocol.WithFeatureCtx(ctx) + store, err = edb.NewErigonStore(ctx, 1) + r.NoError(err) + candlist := &state.CandidateList{ + &state.Candidate{Address: identityset.Address(1).String(), Votes: big.NewInt(1_000_000_000), RewardAddress: identityset.Address(2).String(), BLSPubKey: []byte("cpc")}, + &state.Candidate{Address: identityset.Address(5).String(), Votes: big.NewInt(2_000_000_000), RewardAddress: identityset.Address(6).String(), BLSPubKey: []byte("iotex")}, + &state.Candidate{Address: identityset.Address(9).String(), Votes: big.NewInt(2_000_000_000), RewardAddress: identityset.Address(10).String(), BLSPubKey: []byte("dev")}, + &state.Candidate{Address: identityset.Address(13).String(), Votes: big.NewInt(2_000_000_000), RewardAddress: identityset.Address(14).String(), BLSPubKey: []byte("test")}, + &state.Candidate{Address: identityset.Address(17).String(), Votes: big.NewInt(2_000_000_000), RewardAddress: identityset.Address(18).String(), BLSPubKey: []byte("prod")}, + } + ns := state.SystemNamespace + key := []byte("key1") + r.NoError(store.PutObject(ns, key, candlist)) + r.NoError(store.FinalizeTx(ctx)) + r.NoError(store.Commit(ctx, 0)) + + height = 2 + fmt.Printf("block: %d -----------------------\n", height) + ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{BlockHeight: height}) + ctx = protocol.WithFeatureCtx(ctx) + store, err = edb.NewErigonStore(ctx, height) + r.NoError(err) + defer store.Close() + got := &state.CandidateList{} + r.NoError(store.GetObject(ns, key, got)) + r.Equal(candlist, got) + }) + + t.Run("staking.Candidate", func(t *testing.T) { + height++ + fmt.Printf("block: %d -----------------------\n", height) + ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{BlockHeight: height}) + ctx = protocol.WithFeatureCtx(ctx) + store, err = edb.NewErigonStore(ctx, 1) + r.NoError(err) + candlist := &staking.CandidateList{ + &staking.Candidate{Owner: identityset.Address(1), Operator: identityset.Address(2), Reward: identityset.Address(3), Identifier: nil, BLSPubKey: []byte("pubkey"), Name: "dev", Votes: big.NewInt(1_000_000_000), SelfStake: big.NewInt(100_000_000), SelfStakeBucketIdx: 1}, + &staking.Candidate{Owner: identityset.Address(5), Operator: identityset.Address(6), Reward: identityset.Address(7), Identifier: nil, BLSPubKey: []byte("pubkey2"), Name: "iotex", Votes: big.NewInt(2_000_000_000), SelfStake: big.NewInt(200_000_000), SelfStakeBucketIdx: 2}, + &staking.Candidate{Owner: identityset.Address(9), Operator: identityset.Address(10), Reward: identityset.Address(11), Identifier: nil, BLSPubKey: []byte("pubkey3"), Name: "test", Votes: big.NewInt(3_000_000_000), SelfStake: big.NewInt(300_000_000), SelfStakeBucketIdx: 3}, + &staking.Candidate{Owner: identityset.Address(13), Operator: identityset.Address(14), Reward: identityset.Address(15), Identifier: nil, BLSPubKey: []byte("pubkey4"), Name: "prod", Votes: big.NewInt(4_000_000_000), SelfStake: big.NewInt(400_000_000), SelfStakeBucketIdx: 4}, + &staking.Candidate{Owner: identityset.Address(17), Operator: identityset.Address(18), Reward: identityset.Address(19), Identifier: nil, BLSPubKey: []byte("pubkey5"), Name: "cpc", Votes: big.NewInt(5_000_000_000), SelfStake: big.NewInt(500_000_000), SelfStakeBucketIdx: 5}, + } + ns := state.CandsMapNamespace + key := []byte("key1") + r.NoError(store.PutObject(ns, key, candlist)) + r.NoError(store.FinalizeTx(ctx)) + r.NoError(store.Commit(ctx, 0)) + + height++ + fmt.Printf("block: %d -----------------------\n", height) + ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{BlockHeight: height}) + ctx = protocol.WithFeatureCtx(ctx) + store, err = edb.NewErigonStore(ctx, height) + r.NoError(err) + defer store.Close() + got := &staking.CandidateList{} + r.NoError(store.GetObject(ns, key, got)) + r.Equal(candlist, got) + }) + + t.Run("rewarding.RewardHistory", func(t *testing.T) { + height++ + fmt.Printf("block: %d -----------------------\n", height) + ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{BlockHeight: height}) + ctx = protocol.WithFeatureCtx(ctx) + store, err = edb.NewErigonStore(ctx, height) + r.NoError(err) + defer store.Close() + rh := &rewarding.RewardHistory{} + ns := state.AccountKVNamespace + var indexBytes [8]byte + enc.MachineEndian.PutUint64(indexBytes[:], height) + key := append(state.RewardingKeyPrefix[:], state.BlockRewardHistoryKeyPrefix...) + key = append(key, indexBytes[:]...) + nextKey := append(state.RewardingKeyPrefix[:], state.BlockRewardHistoryKeyPrefix...) + enc.MachineEndian.PutUint64(indexBytes[:], height+1) + nextKey = append(nextKey, indexBytes[:]...) + // test GetObject/DeleteObject/PutObject + r.NoError(store.GetObject(ns, key, rh)) + r.ErrorIs(store.GetObject(ns, nextKey, rh), state.ErrStateNotExist) + r.NoError(store.DeleteObject(ns, key, rh)) + r.ErrorIs(store.GetObject(ns, key, rh), state.ErrStateNotExist) + r.NoError(store.PutObject(ns, key, rh)) + r.NoError(store.GetObject(ns, key, rh)) + r.NoError(store.FinalizeTx(ctx)) + r.NoError(store.Commit(ctx, 0)) + }) + + t.Run("rewarding.Fund", func(t *testing.T) { + height++ + fmt.Printf("block: %d -----------------------\n", height) + ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{BlockHeight: height}) + ctx = protocol.WithFeatureCtx(ctx) + store, err = edb.NewErigonStore(ctx, height) + r.NoError(err) + defer store.Close() + rh := &rewarding.Fund{} + rhpb := &rewardingpb.Fund{ + TotalBalance: "1000000000000000000", + UnclaimedBalance: "500000000000000000", + } + data, err := proto.Marshal(rhpb) + r.NoError(err) + r.NoError(rh.Deserialize(data)) + ns := state.AccountKVNamespace + _fundKey := []byte("fnd") + key := append(state.RewardingKeyPrefix[:], _fundKey...) + r.NoError(store.PutObject(ns, key, rh)) + got := &rewarding.Fund{} + r.NoError(store.GetObject(ns, key, got)) + r.EqualValues(rh, got) + r.NoError(store.FinalizeTx(ctx)) + r.NoError(store.Commit(ctx, 0)) + + height++ + fmt.Printf("block: %d -----------------------\n", height) + ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{BlockHeight: height}) + ctx = protocol.WithFeatureCtx(ctx) + store, err = edb.NewErigonStore(ctx, height) + r.NoError(err) + defer store.Close() + rhpb = &rewardingpb.Fund{ + TotalBalance: "1000000000000000000", + UnclaimedBalance: "400000000000000000", + } + data, err = proto.Marshal(rhpb) + r.NoError(err) + r.NoError(rh.Deserialize(data)) + r.NoError(store.PutObject(ns, key, rh)) + got = &rewarding.Fund{} + r.NoError(store.GetObject(ns, key, got)) + r.EqualValues(rh, got) + r.NoError(store.FinalizeTx(ctx)) + r.NoError(store.Commit(ctx, 0)) + }) +} diff --git a/state/iterator.go b/state/iterator.go index 60d5a407e0..2f59597256 100644 --- a/state/iterator.go +++ b/state/iterator.go @@ -51,7 +51,7 @@ func (it *iterator) Next(s interface{}) ([]byte, error) { } it.index = i + 1 if it.states[i] == nil { - return nil, ErrNilValue + return it.keys[i], ErrNilValue } return it.keys[i], Deserialize(s, it.states[i]) } diff --git a/systemcontractindex/stakingindex/candidate_votes.go b/systemcontractindex/stakingindex/candidate_votes.go index 921ff1a40e..054d876e07 100644 --- a/systemcontractindex/stakingindex/candidate_votes.go +++ b/systemcontractindex/stakingindex/candidate_votes.go @@ -138,7 +138,7 @@ func (cv *candidateVotes) Deserialize(data []byte) error { return nil } -func (cv *candidateVotes) Encode() ([][]byte, []systemcontracts.GenericValue, error) { +func (cv *candidateVotes) Encodes() ([][]byte, []systemcontracts.GenericValue, error) { var ( keys [][]byte values []systemcontracts.GenericValue @@ -161,7 +161,7 @@ func (cv *candidateVotes) Encode() ([][]byte, []systemcontracts.GenericValue, er return keys, values, nil } -func (cv *candidateVotes) Decode(keys [][]byte, values []systemcontracts.GenericValue) error { +func (cv *candidateVotes) Decodes(keys [][]byte, values []systemcontracts.GenericValue) error { ncv := newCandidateVotes() for i, key := range keys { cand := string(key) @@ -329,14 +329,14 @@ func (cv *candidateVotesWithBuffer) Base() CandidateVotes { return newCandidateVotesWithBuffer(cv.base) } -func (cv *candidateVotesWithBuffer) Encode() ([][]byte, []systemcontracts.GenericValue, error) { +func (cv *candidateVotesWithBuffer) Encodes() ([][]byte, []systemcontracts.GenericValue, error) { if cv.IsDirty() { return nil, nil, errors.Wrap(ErrCandidateVotesIsDirty, "cannot encode dirty candidate votes") } - return cv.base.Encode() + return cv.base.Encodes() } -func (cv *candidateVotesWithBuffer) Decode(keys [][]byte, data []systemcontracts.GenericValue) error { +func (cv *candidateVotesWithBuffer) Decodes(keys [][]byte, data []systemcontracts.GenericValue) error { cv.change = newCandidateVotes() - return cv.base.Decode(keys, data) + return cv.base.Decodes(keys, data) } diff --git a/systemcontractindex/stakingindex/voteview.go b/systemcontractindex/stakingindex/voteview.go index d6e852f535..b8548f1b5d 100644 --- a/systemcontractindex/stakingindex/voteview.go +++ b/systemcontractindex/stakingindex/voteview.go @@ -176,8 +176,9 @@ func (s *voteView) AddBlockReceipts(ctx context.Context, receipts []*action.Rece } func (s *voteView) Commit(ctx context.Context, sm protocol.StateManager) error { + isDirty := s.cur.IsDirty() s.cur = s.cur.Commit() - if sm == nil { + if sm == nil || !isDirty { return nil } return s.cvm.Store(ctx, sm, s.cur) From fef47c054fac3cefa3bc50586c46320b44eef813 Mon Sep 17 00:00:00 2001 From: envestcc Date: Thu, 20 Nov 2025 10:12:14 +0800 Subject: [PATCH 3/8] error if query on closed erigonstore --- .../erigonstore/workingsetstore_erigon.go | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/state/factory/erigonstore/workingsetstore_erigon.go b/state/factory/erigonstore/workingsetstore_erigon.go index 2a6e0b0582..d89f7b6a08 100644 --- a/state/factory/erigonstore/workingsetstore_erigon.go +++ b/state/factory/erigonstore/workingsetstore_erigon.go @@ -39,6 +39,11 @@ var ( heightKey = []byte("height") ) +var ( + // ErrErigonStoreClosed is returned when the erigon working set store is closed + ErrErigonStoreClosed = errors.New("erigon working set store is closed") +) + // ErigonDB implements the Erigon database type ErigonDB struct { path string @@ -261,7 +266,7 @@ func (store *ErigonWorkingSetStore) prepareCommit(ctx context.Context, tx kv.RwT // Commit commits the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) Commit(ctx context.Context, retention uint64) error { - defer store.tx.Rollback() + defer store.close() // BeginRw accounting for the context Done signal // statedb has been committed, so we should not use the context tx, err := store.db.rw.BeginRw(context.Background()) @@ -281,7 +286,18 @@ func (store *ErigonWorkingSetStore) Commit(ctx context.Context, retention uint64 // Close closes the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) Close() { - store.tx.Rollback() + store.close() +} + +func (store *ErigonWorkingSetStore) close() { + if store.tx != nil { + store.tx.Rollback() + store.tx = nil + } +} + +func (store *ErigonWorkingSetStore) closed() bool { + return store.tx == nil } // Snapshot creates a snapshot of the ErigonWorkingSetStore @@ -300,6 +316,9 @@ func (store *ErigonWorkingSetStore) ResetSnapshots() {} // PutObject puts an object into the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) PutObject(ns string, key []byte, obj any) (err error) { + if store.closed() { + return errors.Wrapf(ErrErigonStoreClosed, "cannot put object into closed erigon working set store") + } storage, err := store.NewObjectStorage(ns, obj) if err != nil { return err @@ -314,6 +333,9 @@ func (store *ErigonWorkingSetStore) PutObject(ns string, key []byte, obj any) (e // GetObject gets an object from the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) GetObject(ns string, key []byte, obj any) error { + if store.closed() { + return errors.Wrapf(ErrErigonStoreClosed, "cannot get object from closed erigon working set store") + } storage, err := store.NewObjectStorage(ns, obj) if err != nil { return err @@ -331,6 +353,9 @@ func (store *ErigonWorkingSetStore) GetObject(ns string, key []byte, obj any) er // DeleteObject deletes an object from the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) DeleteObject(ns string, key []byte, obj any) error { + if store.closed() { + return errors.Wrapf(ErrErigonStoreClosed, "cannot delete object from closed erigon working set store") + } storage, err := store.NewObjectStorage(ns, obj) if err != nil { return err @@ -345,6 +370,9 @@ func (store *ErigonWorkingSetStore) DeleteObject(ns string, key []byte, obj any) // States gets multiple objects from the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) States(ns string, obj any, keys [][]byte) (state.Iterator, error) { + if store.closed() { + return nil, errors.Wrapf(ErrErigonStoreClosed, "cannot get states from closed erigon working set store") + } var storage ObjectStorage var err error if store.statesReadBuffer { From 9c3f4118c6e16cb5ff8089df67fa98bcf657d1a2 Mon Sep 17 00:00:00 2001 From: envestcc Date: Mon, 24 Nov 2025 18:26:50 +0800 Subject: [PATCH 4/8] allow read on closed erigonstore --- .../erigonstore/workingsetstore_erigon.go | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/state/factory/erigonstore/workingsetstore_erigon.go b/state/factory/erigonstore/workingsetstore_erigon.go index d89f7b6a08..2255c3a80d 100644 --- a/state/factory/erigonstore/workingsetstore_erigon.go +++ b/state/factory/erigonstore/workingsetstore_erigon.go @@ -333,10 +333,16 @@ func (store *ErigonWorkingSetStore) PutObject(ns string, key []byte, obj any) (e // GetObject gets an object from the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) GetObject(ns string, key []byte, obj any) error { - if store.closed() { - return errors.Wrapf(ErrErigonStoreClosed, "cannot get object from closed erigon working set store") + st := store + if st.closed() { + sr, err := st.newDryrun() + if err != nil { + return err + } + st = sr + defer st.Close() } - storage, err := store.NewObjectStorage(ns, obj) + storage, err := st.NewObjectStorage(ns, obj) if err != nil { return err } @@ -370,15 +376,21 @@ func (store *ErigonWorkingSetStore) DeleteObject(ns string, key []byte, obj any) // States gets multiple objects from the ErigonWorkingSetStore func (store *ErigonWorkingSetStore) States(ns string, obj any, keys [][]byte) (state.Iterator, error) { + st := store if store.closed() { - return nil, errors.Wrapf(ErrErigonStoreClosed, "cannot get states from closed erigon working set store") + sr, err := store.newDryrun() + if err != nil { + return nil, err + } + st = sr + defer st.Close() } var storage ObjectStorage var err error - if store.statesReadBuffer { - storage, err = store.NewObjectStorage(ns, obj) + if st.statesReadBuffer { + storage, err = st.NewObjectStorage(ns, obj) } else { - storage, err = store.NewObjectStorageClean(ns, obj) + storage, err = st.NewObjectStorageClean(ns, obj) } if err != nil { return nil, err @@ -499,3 +511,21 @@ func (store *ErigonWorkingSetStore) NewObjectStorageClean(ns string, obj any) (O func (store *ErigonWorkingSetStore) ErigonStore() (any, error) { return store, nil } + +func (store *ErigonWorkingSetStore) newDryrun() (*ErigonWorkingSetStore, error) { + tx, err := store.db.rw.BeginRo(context.Background()) + if err != nil { + return nil, errors.Wrapf(err, "failed to begin erigon working set store dryrun transaction") + } + height := store.backend.height + 1 + tsw := erigonstate.NewPlainState(tx, height, nil) + intraBlockState := erigonstate.New(tsw) + backend := store.backend + return &ErigonWorkingSetStore{ + db: store.db, + tx: tx, + backend: NewContractBackend(intraBlockState, tsw, backend.height, backend.timestamp, backend.producer, backend.g, backend.evmNetworkID, backend.useZeroNonceForFreshAccount), + clean: NewContractBackend(erigonstate.New(tsw), tsw, backend.height, backend.timestamp, backend.producer, backend.g, backend.evmNetworkID, backend.useZeroNonceForFreshAccount), + statesReadBuffer: backend.g.IsNewfoundland(height), + }, nil +} From a7b9e70dcaf8f67d89b1d01cdeaa3bf0e71317f8 Mon Sep 17 00:00:00 2001 From: envestcc Date: Tue, 16 Dec 2025 19:24:48 +0800 Subject: [PATCH 5/8] fix test --- action/protocol/rewarding/protocol.go | 2 +- action/protocol/rewarding/protocol_test.go | 11 +++++++++++ action/protocol/rewarding/reward_test.go | 2 ++ .../erigonstore/data/erigonstore_testdb/mdbx.lck | 0 4 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 state/factory/erigonstore/data/erigonstore_testdb/mdbx.lck diff --git a/action/protocol/rewarding/protocol.go b/action/protocol/rewarding/protocol.go index 813c84b53d..62ee8df661 100644 --- a/action/protocol/rewarding/protocol.go +++ b/action/protocol/rewarding/protocol.go @@ -108,7 +108,7 @@ func (p *Protocol) CreatePreStates(ctx context.Context, sm protocol.StateManager enc.MachineEndian.PutUint64(indexBytes[:], blkCtx.BlockHeight) err := p.deleteState(ctx, sm, append(_blockRewardHistoryKeyPrefix, indexBytes[:]...), &rewardHistory{}, protocol.ErigonStoreOnlyOption()) if err != nil && !errors.Is(err, state.ErrErigonStoreNotSupported) { - return err + return errors.Wrap(err, "failed to delete block reward history for erigon store") } switch blkCtx.BlockHeight { case g.AleutianBlockHeight: diff --git a/action/protocol/rewarding/protocol_test.go b/action/protocol/rewarding/protocol_test.go index 84668f2e21..b919fab419 100644 --- a/action/protocol/rewarding/protocol_test.go +++ b/action/protocol/rewarding/protocol_test.go @@ -310,6 +310,14 @@ func TestProtocol_Handle(t *testing.T) { }).AnyTimes() sm.EXPECT().Snapshot().Return(1).AnyTimes() sm.EXPECT().Revert(gomock.Any()).Return(nil).AnyTimes() + sm.EXPECT().DelState(gomock.Any()).DoAndReturn(func(opts ...protocol.StateOption) (uint64, error) { + cfg, err := protocol.CreateStateConfig(opts...) + if err != nil { + return 0, err + } + cb.Delete("state", cfg.Key, "failed to delete state") + return 0, nil + }).AnyTimes() g.Rewarding.InitBalanceStr = "1000000" g.Rewarding.BlockRewardStr = "10" @@ -656,6 +664,9 @@ func TestMigrateValue(t *testing.T) { blkCtx := protocol.MustGetBlockCtx(ctx) blkCtx.BlockHeight = v.height fCtx = protocol.WithFeatureCtx(protocol.WithBlockCtx(fCtx, blkCtx)) + // init storage bucket + _, err = sm.PutState(&rewardHistory{}, protocol.NamespaceOption(_v2RewardingNamespace), protocol.KeyOption([]byte("test"))) + r.NoError(err) r.NoError(p.CreatePreStates(fCtx, sm)) // verify v1 is deleted diff --git a/action/protocol/rewarding/reward_test.go b/action/protocol/rewarding/reward_test.go index 51a31b5f50..9c91ebf910 100644 --- a/action/protocol/rewarding/reward_test.go +++ b/action/protocol/rewarding/reward_test.go @@ -610,6 +610,8 @@ func TestProtocol_CalculateReward(t *testing.T) { g := genesis.MustExtractGenesisContext(ctx) blkCtx := protocol.MustGetBlockCtx(ctx) blkCtx.AccumulatedTips.Set(tv.accumuTips) + _, err := sm.PutState(&rewardHistory{}, protocol.NamespaceOption(_v2RewardingNamespace), protocol.KeyOption([]byte("test"))) + req.NoError(err) if tv.isWakeBlock { g.WakeBlockRewardStr = wakeBlockReward.String() blkCtx.BlockHeight = g.WakeBlockHeight diff --git a/state/factory/erigonstore/data/erigonstore_testdb/mdbx.lck b/state/factory/erigonstore/data/erigonstore_testdb/mdbx.lck new file mode 100644 index 0000000000..e69de29bb2 From 25146428cab500423a0f0bb05de0f8ac61685ddd Mon Sep 17 00:00:00 2001 From: envestcc Date: Mon, 29 Sep 2025 13:18:03 +0800 Subject: [PATCH 6/8] read history views --- .../protocol/staking/contractstake_indexer.go | 4 +- .../staking/contractstake_indexer_mock.go | 28 ++++ action/protocol/staking/protocol.go | 59 ++++++- action/protocol/staking/stakeview_builder.go | 39 +---- api/coreservice.go | 8 - api/web3server.go | 16 +- blockindex/contractstaking/history.go | 21 +++ blockindex/contractstaking/indexer.go | 22 ++- state/factory/statedb.go | 62 +++---- systemcontractindex/stakingindex/bucket.go | 4 +- systemcontractindex/stakingindex/history.go | 156 ++++++++++++++++++ systemcontractindex/stakingindex/index.go | 11 +- 12 files changed, 332 insertions(+), 98 deletions(-) create mode 100644 blockindex/contractstaking/history.go create mode 100644 systemcontractindex/stakingindex/history.go diff --git a/action/protocol/staking/contractstake_indexer.go b/action/protocol/staking/contractstake_indexer.go index 7497f27a44..caa9ad8d37 100644 --- a/action/protocol/staking/contractstake_indexer.go +++ b/action/protocol/staking/contractstake_indexer.go @@ -67,7 +67,9 @@ type ( CreateEventProcessor(context.Context, EventHandler) EventProcessor // ContractStakingBuckets returns all the contract staking buckets ContractStakingBuckets() (uint64, map[uint64]*contractstaking.Bucket, error) - + // IndexerAt returns the contract staking indexer at a specific height + IndexerAt(protocol.StateReader) ContractStakingIndexer + // BucketReader defines the interface to read buckets BucketReader } // ContractStakingIndexerWithBucketType defines the interface of contract staking reader with bucket type diff --git a/action/protocol/staking/contractstake_indexer_mock.go b/action/protocol/staking/contractstake_indexer_mock.go index e1da4905a9..d796b9999c 100644 --- a/action/protocol/staking/contractstake_indexer_mock.go +++ b/action/protocol/staking/contractstake_indexer_mock.go @@ -288,6 +288,20 @@ func (mr *MockContractStakingIndexerMockRecorder) Height() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Height", reflect.TypeOf((*MockContractStakingIndexer)(nil).Height)) } +// IndexerAt mocks base method. +func (m *MockContractStakingIndexer) IndexerAt(arg0 protocol.StateReader) ContractStakingIndexer { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IndexerAt", arg0) + ret0, _ := ret[0].(ContractStakingIndexer) + return ret0 +} + +// IndexerAt indicates an expected call of IndexerAt. +func (mr *MockContractStakingIndexerMockRecorder) IndexerAt(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexerAt", reflect.TypeOf((*MockContractStakingIndexer)(nil).IndexerAt), arg0) +} + // LoadStakeView mocks base method. func (m *MockContractStakingIndexer) LoadStakeView(arg0 context.Context, arg1 protocol.StateReader) (ContractStakeView, error) { m.ctrl.T.Helper() @@ -532,6 +546,20 @@ func (mr *MockContractStakingIndexerWithBucketTypeMockRecorder) Height() *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Height", reflect.TypeOf((*MockContractStakingIndexerWithBucketType)(nil).Height)) } +// IndexerAt mocks base method. +func (m *MockContractStakingIndexerWithBucketType) IndexerAt(arg0 protocol.StateReader) ContractStakingIndexer { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IndexerAt", arg0) + ret0, _ := ret[0].(ContractStakingIndexer) + return ret0 +} + +// IndexerAt indicates an expected call of IndexerAt. +func (mr *MockContractStakingIndexerWithBucketTypeMockRecorder) IndexerAt(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexerAt", reflect.TypeOf((*MockContractStakingIndexerWithBucketType)(nil).IndexerAt), arg0) +} + // LoadStakeView mocks base method. func (m *MockContractStakingIndexerWithBucketType) LoadStakeView(arg0 context.Context, arg1 protocol.StateReader) (ContractStakeView, error) { m.ctrl.T.Helper() diff --git a/action/protocol/staking/protocol.go b/action/protocol/staking/protocol.go index 53822f8266..10bfa1c9df 100644 --- a/action/protocol/staking/protocol.go +++ b/action/protocol/staking/protocol.go @@ -742,7 +742,11 @@ func (p *Protocol) HandleReceipt(ctx context.Context, elp action.Envelope, sm pr return err } if p.contractStakingIndexer != nil { - processor := p.contractStakingIndexer.CreateEventProcessor(ctx, handler) + index, err := contractStakingIndexerAt(p.contractStakingIndexer, sm, true) + if err != nil { + return err + } + processor := index.CreateEventProcessor(ctx, handler) if err := processor.ProcessReceipts(ctx, receipt); err != nil { if !errors.Is(err, state.ErrErigonStoreNotSupported) { return errors.Wrap(err, "failed to process receipt for contract staking indexer") @@ -751,7 +755,11 @@ func (p *Protocol) HandleReceipt(ctx context.Context, elp action.Envelope, sm pr } } if p.contractStakingIndexerV2 != nil { - processor := p.contractStakingIndexerV2.CreateEventProcessor(ctx, handler) + index, err := contractStakingIndexerAt(p.contractStakingIndexerV2, sm, true) + if err != nil { + return err + } + processor := index.CreateEventProcessor(ctx, handler) if err := processor.ProcessReceipts(ctx, receipt); err != nil { if !errors.Is(err, state.ErrErigonStoreNotSupported) { return errors.Wrap(err, "failed to process receipt for contract staking indexer v2") @@ -760,7 +768,11 @@ func (p *Protocol) HandleReceipt(ctx context.Context, elp action.Envelope, sm pr } } if p.contractStakingIndexerV3 != nil { - processor := p.contractStakingIndexerV3.CreateEventProcessor(ctx, handler) + index, err := contractStakingIndexerAt(p.contractStakingIndexerV3, sm, true) + if err != nil { + return err + } + processor := index.CreateEventProcessor(ctx, handler) if err := processor.ProcessReceipts(ctx, receipt); err != nil { if !errors.Is(err, state.ErrErigonStoreNotSupported) { return errors.Wrap(err, "failed to process receipt for contract staking indexer v3") @@ -884,13 +896,25 @@ func (p *Protocol) ReadState(ctx context.Context, sr protocol.StateReader, metho // stakeSR is the stake state reader including native and contract staking indexers := []ContractStakingIndexer{} if p.contractStakingIndexer != nil { - indexers = append(indexers, NewDelayTolerantIndexerWithBucketType(p.contractStakingIndexer, time.Second)) + index, err := contractStakingIndexerAt(p.contractStakingIndexer, sr, false) + if err != nil { + return nil, 0, err + } + indexers = append(indexers, NewDelayTolerantIndexerWithBucketType(index.(ContractStakingIndexerWithBucketType), time.Second)) } if p.contractStakingIndexerV2 != nil { - indexers = append(indexers, NewDelayTolerantIndexer(p.contractStakingIndexerV2, time.Second)) + index, err := contractStakingIndexerAt(p.contractStakingIndexerV2, sr, false) + if err != nil { + return nil, 0, err + } + indexers = append(indexers, NewDelayTolerantIndexer(index, time.Second)) } if p.contractStakingIndexerV3 != nil { - indexers = append(indexers, NewDelayTolerantIndexer(p.contractStakingIndexerV3, time.Second)) + index, err := contractStakingIndexerAt(p.contractStakingIndexerV3, sr, false) + if err != nil { + return nil, 0, err + } + indexers = append(indexers, NewDelayTolerantIndexer(index, time.Second)) } stakeSR, err := newCompositeStakingStateReader(p.candBucketsIndexer, sr, p.calculateVoteWeight, indexers...) if err != nil { @@ -1144,3 +1168,26 @@ func readCandCenterStateFromStateDB(sr protocol.StateReader) (CandidateList, Can } return name, operator, owner, nil } + +func contractStakingIndexerAt(index ContractStakingIndexer, sr protocol.StateReader, delay bool) (ContractStakingIndexer, error) { + if index == nil { + return nil, nil + } + srHeight, err := sr.Height() + if err != nil { + return nil, err + } + if delay { + srHeight-- + } + indexHeight, err := index.Height() + if err != nil { + return nil, err + } + if index.StartHeight() > srHeight || indexHeight == srHeight { + return index, nil + } else if indexHeight < srHeight { + return nil, errors.Errorf("indexer height %d is too old for state reader height %d", indexHeight, srHeight) + } + return index.IndexerAt(sr), nil +} diff --git a/action/protocol/staking/stakeview_builder.go b/action/protocol/staking/stakeview_builder.go index be750e9a4e..d3d176bace 100644 --- a/action/protocol/staking/stakeview_builder.go +++ b/action/protocol/staking/stakeview_builder.go @@ -37,42 +37,9 @@ func (b *contractStakeViewBuilder) Build(ctx context.Context, sr protocol.StateR BlockHeight: height, }) ctx = protocol.WithFeatureCtx(ctx) - view, err := b.indexer.LoadStakeView(ctx, sr) + index, err := contractStakingIndexerAt(b.indexer, sr, false) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to get contract staking indexer at height") } - if b.indexer.StartHeight() > height { - return view, nil - } - indexerHeight, err := b.indexer.Height() - if err != nil { - return nil, err - } - if indexerHeight == height { - return view, nil - } - if indexerHeight > height { - return nil, errors.Errorf("indexer height %d is greater than requested height %d", indexerHeight, height) - } - if b.blockdao == nil { - return nil, errors.Errorf("blockdao is nil, cannot build view for height %d", height) - } - for h := indexerHeight + 1; h <= height; h++ { - receipts, err := b.blockdao.GetReceipts(h) - if err != nil { - return nil, errors.Wrapf(err, "failed to get receipts at height %d", h) - } - header, err := b.blockdao.HeaderByHeight(h) - if err != nil { - return nil, errors.Wrapf(err, "failed to get header at height %d", h) - } - ctx = protocol.WithBlockCtx(ctx, protocol.BlockCtx{ - BlockHeight: h, - BlockTimeStamp: header.Timestamp(), - }) - if err = view.AddBlockReceipts(ctx, receipts); err != nil { - return nil, errors.Wrapf(err, "failed to build view with block at height %d", h) - } - } - return view, nil + return index.LoadStakeView(ctx, sr) } diff --git a/api/coreservice.go b/api/coreservice.go index 04dc00d44d..f5933a0620 100644 --- a/api/coreservice.go +++ b/api/coreservice.go @@ -1071,14 +1071,6 @@ func (core *coreService) readState(ctx context.Context, p protocol.Protocol, hei if err != nil { return nil, 0, err } - rp := rolldpos.FindProtocol(core.registry) - if rp != nil { - tipEpochNum := rp.GetEpochNum(tipHeight) - inputEpochNum := rp.GetEpochNum(inputHeight) - if inputEpochNum < tipEpochNum { - inputHeight = rp.GetEpochHeight(inputEpochNum) - } - } if inputHeight < tipHeight { // old data, wrap to history state reader historySR, err := core.sf.WorkingSetAtHeight(ctx, inputHeight) diff --git a/api/web3server.go b/api/web3server.go index 805dd71de5..683081731a 100644 --- a/api/web3server.go +++ b/api/web3server.go @@ -473,6 +473,14 @@ func (svr *web3Handler) call(ctx context.Context, in *gjson.Result) (interface{} to = callMsg.To data = callMsg.Data ) + height, archive, err := svr.blockNumberOrHashToHeight(callMsg.BlockNumberOrHash) + if err != nil { + return nil, err + } + heightStr := "" + if height > 0 { + heightStr = strconv.FormatUint(height, 10) + } if to == _metamaskBalanceContractAddr { return nil, nil } @@ -481,7 +489,7 @@ func (svr *web3Handler) call(ctx context.Context, in *gjson.Result) (interface{} if err != nil { return nil, err } - states, err := svr.coreService.ReadState("staking", "", sctx.Parameters().MethodName, sctx.Parameters().Arguments) + states, err := svr.coreService.ReadState("staking", heightStr, sctx.Parameters().MethodName, sctx.Parameters().Arguments) if err != nil { return nil, err } @@ -496,7 +504,7 @@ func (svr *web3Handler) call(ctx context.Context, in *gjson.Result) (interface{} if err != nil { return nil, err } - states, err := svr.coreService.ReadState("rewarding", "", sctx.Parameters().MethodName, sctx.Parameters().Arguments) + states, err := svr.coreService.ReadState("rewarding", heightStr, sctx.Parameters().MethodName, sctx.Parameters().Arguments) if err != nil { return nil, err } @@ -542,10 +550,6 @@ func (svr *web3Handler) call(ctx context.Context, in *gjson.Result) (interface{} ret string receipt *iotextypes.Receipt ) - height, archive, err := svr.blockNumberOrHashToHeight(callMsg.BlockNumberOrHash) - if err != nil { - return nil, err - } if !archive { ret, receipt, err = svr.coreService.ReadContract(context.Background(), callMsg.From, elp) } else { diff --git a/blockindex/contractstaking/history.go b/blockindex/contractstaking/history.go new file mode 100644 index 0000000000..130dd17f05 --- /dev/null +++ b/blockindex/contractstaking/history.go @@ -0,0 +1,21 @@ +package contractstaking + +import ( + "github.com/pkg/errors" + + "github.com/iotexproject/iotex-core/v2/action/protocol/staking" +) + +type historyIndexer struct { + staking.ContractStakingIndexer +} + +func newHistoryIndexer(indexer staking.ContractStakingIndexer) *historyIndexer { + return &historyIndexer{ + ContractStakingIndexer: indexer, + } +} + +func (h *historyIndexer) BucketTypes(height uint64) ([]*staking.ContractStakingBucketType, error) { + return nil, errors.New("not implemented") +} diff --git a/blockindex/contractstaking/indexer.go b/blockindex/contractstaking/indexer.go index b8f8f237a9..899bc3cd01 100644 --- a/blockindex/contractstaking/indexer.go +++ b/blockindex/contractstaking/indexer.go @@ -55,7 +55,7 @@ type ( } calculateVoteWeightFunc func(v *Bucket) *big.Int - blocksDurationFn func(start uint64, end uint64) time.Duration + blocksDurationFn = stakingindex.BlocksDurationFn blocksDurationAtFn func(start uint64, end uint64, viewAt uint64) time.Duration ) @@ -126,17 +126,13 @@ func (s *Indexer) LoadStakeView(ctx context.Context, sr protocol.StateReader) (s for i, id := range ids { buckets[id] = assembleContractBucket(infos[i], typs[i]) } - calculateUnmutedVoteWeightAt := func(b *contractstaking.Bucket, height uint64) *big.Int { - vb := contractBucketToVoteBucket(0, b, s.contractAddr.String(), s.genBlockDurationFn(height)) - return s.config.CalculateVoteWeight(vb) - } cur := stakingindex.AggregateCandidateVotes(buckets, func(b *contractstaking.Bucket) *big.Int { - return calculateUnmutedVoteWeightAt(b, s.height) + return s.calculateUnmutedVoteWeightAt(b, s.height) }) processorBuilder := newEventProcessorBuilder(s.contractAddr) cfg := &stakingindex.VoteViewConfig{ContractAddr: s.contractAddr} mgr := stakingindex.NewCandidateVotesManager(s.ContractAddress()) - return stakingindex.NewVoteView(s, cfg, s.height, cur, processorBuilder, mgr, calculateUnmutedVoteWeightAt), nil + return stakingindex.NewVoteView(s, cfg, s.height, cur, processorBuilder, mgr, s.calculateUnmutedVoteWeightAt), nil } // Stop stops the indexer @@ -403,6 +399,13 @@ func (s *Indexer) PutBlock(ctx context.Context, blk *block.Block) error { return nil } +// IndexerAt returns the contract staking indexer at a specific height +func (s *Indexer) IndexerAt(sr protocol.StateReader) staking.ContractStakingIndexer { + epb := newEventProcessorBuilder(s.contractAddr) + h := stakingindex.NewHistoryIndexer(sr, s.contractAddr, s.config.ContractDeployHeight, epb, s.calculateUnmutedVoteWeightAt, s.genBlockDurationFn) + return newHistoryIndexer(h) +} + func (s *Indexer) commit(ctx context.Context, handler *contractStakingDirty, height uint64) error { batch, delta := handler.Finalize() cache, err := delta.Commit(ctx, s.contractAddr, nil) @@ -464,3 +467,8 @@ func (s *Indexer) validateHeight(height uint64) error { } return nil } + +func (s *Indexer) calculateUnmutedVoteWeightAt(b *contractstaking.Bucket, height uint64) *big.Int { + vb := contractBucketToVoteBucket(0, b, s.contractAddr.String(), s.genBlockDurationFn(height)) + return s.config.CalculateVoteWeight(vb) +} diff --git a/state/factory/statedb.go b/state/factory/statedb.go index 2a1190ba50..4362982432 100644 --- a/state/factory/statedb.go +++ b/state/factory/statedb.go @@ -243,11 +243,39 @@ func (sdb *stateDB) AddDependency(indexer blockdao.BlockIndexer) { } func (sdb *stateDB) newReadOnlyWorkingSet(ctx context.Context, height uint64) (*workingSet, error) { - return sdb.newWorkingSetWithKVStore(ctx, height, &readOnlyKV{sdb.dao.atHeight(height)}) + ws, err := sdb.newWorkingSetWithKVStore(ctx, height, &readOnlyKV{sdb.dao.atHeight(height)}, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to create new read-only working set") + } + if sdb.erigonDB != nil { + if sdb.cfg.Chain.HistoryBlockRetention > 0 { + sdb.mutex.RLock() + tip := sdb.currentChainHeight + sdb.mutex.RUnlock() + if height < tip-sdb.cfg.Chain.HistoryBlockRetention { + return nil, errors.Wrapf( + ErrNotSupported, + "history is pruned, only supported for latest %d blocks, but requested height %d", + sdb.cfg.Chain.HistoryBlockRetention, height, + ) + } + } + e, err := sdb.erigonDB.NewErigonStoreDryrun(ctx, height+1) + if err != nil { + return nil, err + } + ws.store = newErigonWorkingSetStoreForSimulate(e) + } + views, err := sdb.registry.StartAll(ctx, ws) + if err != nil { + return nil, errors.Wrap(err, "failed to create new read-only working set") + } + ws.views = views + return ws, nil } func (sdb *stateDB) newWorkingSet(ctx context.Context, height uint64) (*workingSet, error) { - ws, err := sdb.newWorkingSetWithKVStore(ctx, height, sdb.dao.atHeight(height)) + ws, err := sdb.newWorkingSetWithKVStore(ctx, height, sdb.dao.atHeight(height), sdb.protocolViews.Fork()) if err != nil { return nil, errors.Wrap(err, "failed to create new working set") } @@ -265,7 +293,7 @@ func (sdb *stateDB) newWorkingSet(ctx context.Context, height uint64) (*workingS return ws, nil } -func (sdb *stateDB) newWorkingSetWithKVStore(ctx context.Context, height uint64, kvstore db.KVStore) (*workingSet, error) { +func (sdb *stateDB) newWorkingSetWithKVStore(ctx context.Context, height uint64, kvstore db.KVStore, views *protocol.Views) (*workingSet, error) { store, err := sdb.createWorkingSetStore(ctx, height, kvstore) if err != nil { return nil, err @@ -273,7 +301,7 @@ func (sdb *stateDB) newWorkingSetWithKVStore(ctx context.Context, height uint64, if err := store.Start(ctx); err != nil { return nil, err } - return newWorkingSet(height, sdb.protocolViews.Fork(), store, sdb), nil + return newWorkingSet(height, views, store, sdb), nil } func (sdb *stateDB) CreateWorkingSetStore(ctx context.Context, height uint64, kvstore db.KVStore) (workingSetStore, error) { @@ -394,13 +422,6 @@ func (sdb *stateDB) WorkingSetAtTransaction(ctx context.Context, height uint64, if err != nil { return nil, err } - if sdb.erigonDB != nil { - e, err := sdb.erigonDB.NewErigonStoreDryrun(ctx, height) - if err != nil { - return nil, err - } - ws.store = newErigonWorkingSetStoreForSimulate(e) - } // handle panic to ensure workingset is closed defer func() { if r := recover(); r != nil { @@ -422,25 +443,6 @@ func (sdb *stateDB) WorkingSetAtHeight(ctx context.Context, height uint64) (prot if err != nil { return nil, err } - if sdb.erigonDB != nil { - if sdb.cfg.Chain.HistoryBlockRetention > 0 { - sdb.mutex.RLock() - tip := sdb.currentChainHeight - sdb.mutex.RUnlock() - if height < tip-sdb.cfg.Chain.HistoryBlockRetention { - return nil, errors.Wrapf( - ErrNotSupported, - "history is pruned, only supported for latest %d blocks, but requested height %d", - sdb.cfg.Chain.HistoryBlockRetention, height, - ) - } - } - e, err := sdb.erigonDB.NewErigonStoreDryrun(ctx, height+1) - if err != nil { - return nil, err - } - ws.store = newErigonWorkingSetStoreForSimulate(e) - } return ws, nil } diff --git a/systemcontractindex/stakingindex/bucket.go b/systemcontractindex/stakingindex/bucket.go index 8a292912a0..b0bd25513b 100644 --- a/systemcontractindex/stakingindex/bucket.go +++ b/systemcontractindex/stakingindex/bucket.go @@ -103,7 +103,7 @@ func (b *Bucket) Clone() *Bucket { } */ -func assembleVoteBucket(token uint64, bkt *Bucket, contractAddr string, blocksToDurationFn blocksDurationFn) *VoteBucket { +func assembleVoteBucket(token uint64, bkt *Bucket, contractAddr string, blocksToDurationFn BlocksDurationFn) *VoteBucket { vb := VoteBucket{ Index: token, StakedAmount: bkt.StakedAmount, @@ -141,7 +141,7 @@ func assembleVoteBucket(token uint64, bkt *Bucket, contractAddr string, blocksTo return &vb } -func batchAssembleVoteBucket(idxs []uint64, bkts []*Bucket, contractAddr string, blocksToDurationFn blocksDurationFn) []*VoteBucket { +func batchAssembleVoteBucket(idxs []uint64, bkts []*Bucket, contractAddr string, blocksToDurationFn BlocksDurationFn) []*VoteBucket { vbs := make([]*VoteBucket, 0, len(bkts)) for i := range bkts { if bkts[i] == nil { diff --git a/systemcontractindex/stakingindex/history.go b/systemcontractindex/stakingindex/history.go new file mode 100644 index 0000000000..ff8e5111fb --- /dev/null +++ b/systemcontractindex/stakingindex/history.go @@ -0,0 +1,156 @@ +package stakingindex + +import ( + "context" + "errors" + + "github.com/iotexproject/iotex-address/address" + + "github.com/iotexproject/iotex-core/v2/action/protocol" + "github.com/iotexproject/iotex-core/v2/action/protocol/staking" + "github.com/iotexproject/iotex-core/v2/action/protocol/staking/contractstaking" + "github.com/iotexproject/iotex-core/v2/blockchain/block" +) + +type genBlockDurationFn func(view uint64) BlocksDurationFn + +// historyIndexer implements historical staking indexer +type historyIndexer struct { + sr protocol.StateReader + startHeight uint64 + contractAddr address.Address + epb EventProcessorBuilder + cuvwFn CalculateUnmutedVoteWeightAtFn + gbdFn genBlockDurationFn +} + +// NewHistoryIndexer creates a new instance of historyIndexer +func NewHistoryIndexer(sr protocol.StateReader, contract address.Address, startHeight uint64, epb EventProcessorBuilder, cuvwFn CalculateUnmutedVoteWeightAtFn, gbdFn genBlockDurationFn) staking.ContractStakingIndexer { + return &historyIndexer{ + sr: sr, + contractAddr: contract, + startHeight: startHeight, + epb: epb, + cuvwFn: cuvwFn, + gbdFn: gbdFn, + } +} + +func (h *historyIndexer) Start(ctx context.Context) error { + return nil +} + +func (h *historyIndexer) Stop(ctx context.Context) error { + return nil +} + +func (h *historyIndexer) PutBlock(ctx context.Context, blk *block.Block) error { + return errors.New("not implemented") +} + +// StartHeight returns the start height of the indexer +func (h *historyIndexer) StartHeight() uint64 { + return h.startHeight +} + +// Height returns the latest indexed height +func (h *historyIndexer) Height() (uint64, error) { + return h.sr.Height() +} + +func (h *historyIndexer) Buckets(height uint64) ([]*VoteBucket, error) { + cssr := contractstaking.NewStateReader(h.sr) + idxs, btks, err := cssr.Buckets(h.contractAddr) + if err != nil { + return nil, err + } + return batchAssembleVoteBucket(idxs, btks, h.contractAddr.String(), h.gbdFn(height)), nil +} + +// BucketsByIndices returns active buckets by indices +func (h *historyIndexer) BucketsByIndices(idxs []uint64, height uint64) ([]*VoteBucket, error) { + cssr := contractstaking.NewStateReader(h.sr) + var btks []*contractstaking.Bucket + for _, idx := range idxs { + bkt, err := cssr.Bucket(h.contractAddr, idx) + if err != nil { + return nil, err + } + btks = append(btks, bkt) + } + return batchAssembleVoteBucket(idxs, btks, h.contractAddr.String(), h.gbdFn(height)), nil +} + +// BucketsByCandidate returns active buckets by candidate +func (h *historyIndexer) BucketsByCandidate(ownerAddr address.Address, height uint64) ([]*VoteBucket, error) { + cssr := contractstaking.NewStateReader(h.sr) + idxs, btks, err := cssr.Buckets(h.contractAddr) + if err != nil { + return nil, err + } + var filteredIdxs []uint64 + var filteredBtks []*contractstaking.Bucket + for i, bkt := range btks { + if bkt.Candidate.String() == ownerAddr.String() { + filteredIdxs = append(filteredIdxs, idxs[i]) + filteredBtks = append(filteredBtks, bkt) + } + } + return batchAssembleVoteBucket(filteredIdxs, filteredBtks, h.contractAddr.String(), h.gbdFn(height)), nil +} + +func (h *historyIndexer) TotalBucketCount(height uint64) (uint64, error) { + cssr := contractstaking.NewStateReader(h.sr) + ssb, err := cssr.NumOfBuckets(h.contractAddr) + if err != nil { + return 0, err + } + return ssb, nil +} + +func (h *historyIndexer) ContractAddress() address.Address { + return h.contractAddr +} + +func (h *historyIndexer) LoadStakeView(ctx context.Context, sr protocol.StateReader) (staking.ContractStakeView, error) { + cvm := NewCandidateVotesManager(h.contractAddr) + cur, err := cvm.Load(ctx, sr) + if err != nil { + return nil, err + } + height, err := sr.Height() + if err != nil { + return nil, err + } + return NewVoteView(h, &VoteViewConfig{ContractAddr: h.contractAddr}, height, cur, h.epb, cvm, h.cuvwFn), nil +} + +func (h *historyIndexer) CreateEventProcessor(ctx context.Context, handler staking.EventHandler) staking.EventProcessor { + return h.epb.Build(ctx, handler) +} + +func (h *historyIndexer) ContractStakingBuckets() (uint64, map[uint64]*contractstaking.Bucket, error) { + cssr := contractstaking.NewStateReader(h.sr) + idxs, btks, err := cssr.Buckets(h.contractAddr) + if err != nil { + return 0, nil, err + } + buckets := make(map[uint64]*contractstaking.Bucket) + for i, id := range idxs { + buckets[id] = btks[i] + } + height, err := h.sr.Height() + if err != nil { + return 0, nil, err + } + return height, buckets, nil +} + +func (h *historyIndexer) DeductBucket(addr address.Address, id uint64) (*contractstaking.Bucket, error) { + cssr := contractstaking.NewStateReader(h.sr) + return cssr.Bucket(addr, id) +} + +func (h *historyIndexer) IndexerAt(sr protocol.StateReader) staking.ContractStakingIndexer { + return NewHistoryIndexer(sr, h.contractAddr, h.startHeight, h.epb, h.cuvwFn, h.gbdFn) +} diff --git a/systemcontractindex/stakingindex/index.go b/systemcontractindex/stakingindex/index.go index 1853c462f7..f291b14e41 100644 --- a/systemcontractindex/stakingindex/index.go +++ b/systemcontractindex/stakingindex/index.go @@ -47,6 +47,7 @@ type ( CreateEventProcessor(context.Context, staking.EventHandler) staking.EventProcessor ContractStakingBuckets() (uint64, map[uint64]*Bucket, error) staking.BucketReader + IndexerAt(protocol.StateReader) staking.ContractStakingIndexer } // Indexer is the staking indexer Indexer struct { @@ -63,7 +64,7 @@ type ( // IndexerOption is the option to create an indexer IndexerOption func(*Indexer) - blocksDurationFn func(start uint64, end uint64) time.Duration + BlocksDurationFn func(start uint64, end uint64) time.Duration blocksDurationAtFn func(start uint64, end uint64, viewAt uint64) time.Duration CalculateVoteWeightFunc func(v *VoteBucket) *big.Int ) @@ -343,6 +344,12 @@ func (s *Indexer) PutBlock(ctx context.Context, blk *block.Block) error { return s.commit(ctx, handler, blk.Height()) } +// IndexerAt returns the staking indexer at the given state reader +func (s *Indexer) IndexerAt(sr protocol.StateReader) staking.ContractStakingIndexer { + epb := newEventProcessorBuilder(s.common.ContractAddress(), s.timestamped, s.muteHeight) + return NewHistoryIndexer(sr, s.common.ContractAddress(), s.common.StartHeight(), epb, s.calculateContractVoteWeight, s.genBlockDurationFn) +} + func (s *Indexer) commit(ctx context.Context, handler *eventHandler, height uint64) error { delta, dirty := handler.Finalize() // update db @@ -377,7 +384,7 @@ func (s *Indexer) checkHeight(height uint64) (unstart bool, err error) { return false, nil } -func (s *Indexer) genBlockDurationFn(view uint64) blocksDurationFn { +func (s *Indexer) genBlockDurationFn(view uint64) BlocksDurationFn { return func(start uint64, end uint64) time.Duration { return s.blocksToDuration(start, end, view) } From fca239361e619bae5fba1c530e8e3e752516c0c5 Mon Sep 17 00:00:00 2001 From: envestcc Date: Fri, 5 Dec 2025 09:03:32 +0800 Subject: [PATCH 7/8] try fix mdbx panic --- action/protocol/staking/protocol.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/action/protocol/staking/protocol.go b/action/protocol/staking/protocol.go index 10bfa1c9df..3a60542291 100644 --- a/action/protocol/staking/protocol.go +++ b/action/protocol/staking/protocol.go @@ -310,19 +310,17 @@ func (p *Protocol) Start(ctx context.Context, sr protocol.StateReader) (protocol return } wg.Add(1) - go func() { - defer wg.Done() - if err := checkIndex(indexer); err != nil { - errChan <- errors.Wrap(err, "failed to check contract staking indexer") - return - } - view, err := NewContractStakeViewBuilder(indexer, p.blockStore).Build(ctx, sr, height) - if err != nil { - errChan <- errors.Wrapf(err, "failed to create stake view for contract %s", indexer.ContractAddress()) - return - } - callback(view) - }() + defer wg.Done() + if err := checkIndex(indexer); err != nil { + errChan <- errors.Wrap(err, "failed to check contract staking indexer") + return + } + view, err := NewContractStakeViewBuilder(indexer, p.blockStore).Build(ctx, sr, height) + if err != nil { + errChan <- errors.Wrapf(err, "failed to create stake view for contract %s", indexer.ContractAddress()) + return + } + callback(view) } buildView(p.contractStakingIndexer, func(view ContractStakeView) { c.contractsStake.v1 = view From 9de48417d321da6556be8f66c5abde80d752a2f3 Mon Sep 17 00:00:00 2001 From: envestcc Date: Tue, 16 Dec 2025 20:47:38 +0800 Subject: [PATCH 8/8] fix test --- e2etest/staking_test.go | 1 + systemcontractindex/stakingindex/candidate_votes_manager.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/e2etest/staking_test.go b/e2etest/staking_test.go index 23e83ed8d3..4dda86aff5 100644 --- a/e2etest/staking_test.go +++ b/e2etest/staking_test.go @@ -157,6 +157,7 @@ func TestStakingContract(t *testing.T) { BlockHeight: genesis.TestDefault().OkhotskBlockHeight, })) bcCtx := protocol.MustGetBlockchainCtx(ctx) + ctx = protocol.WithFeatureWithHeightCtx(ctx) _, err = ns.Votes(ctx, bcCtx.Tip.Timestamp, false) require.Equal(poll.ErrNoData, err) tally, err := ns.Votes(ctx, bcCtx.Tip.Timestamp, true) diff --git a/systemcontractindex/stakingindex/candidate_votes_manager.go b/systemcontractindex/stakingindex/candidate_votes_manager.go index 0abe85364a..d38410abe7 100644 --- a/systemcontractindex/stakingindex/candidate_votes_manager.go +++ b/systemcontractindex/stakingindex/candidate_votes_manager.go @@ -49,7 +49,7 @@ func (s *candidateVotesManager) Load(ctx context.Context, sr protocol.StateReade protocol.KeyOption(s.key()), protocol.NamespaceOption(voteViewNS), ) - if err != nil { + if err != nil && !errors.Is(err, state.ErrStateNotExist) { return nil, errors.Wrap(err, "failed to get candidate votes state") } return newCandidateVotesWithBuffer(cur), nil