feat: consider all proposals for chunk validators (near#11252)

Solve near#11202 and continue simplifying proposals processing logic.

I want to make epoch info generation as straightforward as possible, by
moving code for old protocol versions to local submodule so it won't be
distracting. Now it should be more clear that epoch info generation
consists of couple independent steps.

Couple notes:
* FixStakingThreshold isn't stabilised so we can use whatever protocol
version. Version of epoch being generated makes much more sense.
* I expected chunk validators set to be the biggest, but there is subtle
case when we can't select chunk validators due to small stake ratio but
can select chunk producers instead. It adds a bit of complexity.
* There is subtle change in validator indexing and I believe the new
indexing - basically, sorting by descending stake in majority of cases -
makes more sense.

---------

Co-authored-by: Longarithm <the.aleksandr.logunov@gmail.com>

chain/chain/src/test_utils/kv_runtime.rs

@@ -530,9 +530,6 @@ impl EpochManagerAdapter for MockEpochManager {
             validator_to_index,
             bp_settlement,
             cp_settlement,
-            vec![],
-            vec![],
-            HashMap::new(),
             BTreeMap::new(),
             HashMap::new(),
             HashMap::new(),

chain/epoch-manager/src/proposals.rs

@@ -57,7 +57,6 @@ pub fn proposals_to_epoch_info(
             validator_kickout,
             validator_reward,
             minted_amount,
-            current_version,
             next_version,
         );
     } else {
@@ -237,12 +236,6 @@ mod old_validator_selection {
             chunk_producers_settlement.push(shard_settlement);
         }
 
-        let fishermen_to_index = fishermen
-            .iter()
-            .enumerate()
-            .map(|(index, s)| (s.account_id().clone(), index as ValidatorId))
-            .collect::<HashMap<_, _>>();
-
         let validator_to_index = final_proposals
             .iter()
             .enumerate()
@@ -258,9 +251,6 @@ mod old_validator_selection {
             validator_to_index,
             block_producers_settlement,
             chunk_producers_settlement,
-            vec![],
-            fishermen,
-            fishermen_to_index,
             stake_change,
             validator_reward,
             validator_kickout,

chain/epoch-manager/src/shard_assignment.rs

@@ -2,6 +2,11 @@ use near_primitives::types::validator_stake::ValidatorStake;
 use near_primitives::types::{Balance, NumShards, ShardId};
 use near_primitives::utils::min_heap::{MinHeap, PeekMut};
 
+/// Marker struct to communicate the error where you try to assign validators to shards
+/// and there are not enough to even meet the minimum per shard.
+#[derive(Debug)]
+pub struct NotEnoughValidators;
+
 /// Assign chunk producers (a.k.a. validators) to shards.  The i-th element
 /// of the output corresponds to the validators assigned to the i-th shard.
 ///
@@ -14,13 +19,20 @@ use near_primitives::utils::min_heap::{MinHeap, PeekMut};
 /// producer will be assigned to a single shard.  If there are fewer producers,
 /// some of them will be assigned to multiple shards.
 ///
-/// Panics if chunk_producers vector is not sorted in descending order by
-/// producer’s stake.
+/// Returns error if `chunk_producers.len() < min_validators_per_shard`.
+/// Panics if chunk_producers vector is not sorted in descending order by stake.
 pub fn assign_shards<T: HasStake + Eq + Clone>(
     chunk_producers: Vec<T>,
     num_shards: NumShards,
     min_validators_per_shard: usize,
 ) -> Result<Vec<Vec<T>>, NotEnoughValidators> {
+    // If there's not enough chunk producers to fill up a single shard there’s
+    // nothing we can do. Return with an error.
+    let num_chunk_producers = chunk_producers.len();
+    if num_chunk_producers < min_validators_per_shard {
+        return Err(NotEnoughValidators);
+    }
+
     for (idx, pair) in chunk_producers.windows(2).enumerate() {
         assert!(
             pair[0].get_stake() >= pair[1].get_stake(),
@@ -29,13 +41,6 @@ pub fn assign_shards<T: HasStake + Eq + Clone>(
         );
     }
 
-    // If there’s not enough chunk producers to fill up a single shard there’s
-    // nothing we can do.  Return with an error.
-    let num_chunk_producers = chunk_producers.len();
-    if num_chunk_producers < min_validators_per_shard {
-        return Err(NotEnoughValidators);
-    }
-
     let mut result: Vec<Vec<T>> = (0..num_shards).map(|_| Vec::new()).collect();
 
     // Initially, sort by number of validators first so we fill shards up.
@@ -135,11 +140,6 @@ fn assign_with_possible_repeats<T: HasStake + Eq, I: Iterator<Item = (usize, T)>
     }
 }
 
-/// Marker struct to communicate the error where you try to assign validators to shards
-/// and there are not enough to even meet the minimum per shard.
-#[derive(Debug)]
-pub struct NotEnoughValidators;
-
 pub trait HasStake {
     fn get_stake(&self) -> Balance;
 }
@@ -152,6 +152,7 @@ impl HasStake for ValidatorStake {
 
 #[cfg(test)]
 mod tests {
+    use crate::shard_assignment::NotEnoughValidators;
     use near_primitives::types::{Balance, NumShards};
     use std::collections::HashSet;
 
@@ -226,7 +227,7 @@ mod tests {
         stakes: &[Balance],
         num_shards: NumShards,
         min_validators_per_shard: usize,
-    ) -> Result<Vec<(usize, Balance)>, super::NotEnoughValidators> {
+    ) -> Result<Vec<(usize, Balance)>, NotEnoughValidators> {
         let chunk_producers = stakes.iter().copied().enumerate().collect();
         let assignments =
             super::assign_shards(chunk_producers, num_shards, min_validators_per_shard)?;

chain/epoch-manager/src/test_utils.rs

@@ -76,8 +76,8 @@ pub fn epoch_info_with_num_seats(
     mut accounts: Vec<(AccountId, Balance)>,
     block_producers_settlement: Vec<ValidatorId>,
     chunk_producers_settlement: Vec<Vec<ValidatorId>>,
-    hidden_validators_settlement: Vec<ValidatorWeight>,
-    fishermen: Vec<(AccountId, Balance)>,
+    _hidden_validators_settlement: Vec<ValidatorWeight>,
+    _fishermen: Vec<(AccountId, Balance)>,
     stake_change: BTreeMap<AccountId, Balance>,
     validator_kickout: Vec<(AccountId, ValidatorKickoutReason)>,
     validator_reward: HashMap<AccountId, Balance>,
@@ -91,8 +91,6 @@ pub fn epoch_info_with_num_seats(
         acc.insert(x.0.clone(), i as u64);
         acc
     });
-    let fishermen_to_index =
-        fishermen.iter().enumerate().map(|(i, (s, _))| (s.clone(), i as ValidatorId)).collect();
     let account_to_validators = |accounts: Vec<(AccountId, Balance)>| -> Vec<ValidatorStake> {
         accounts
             .into_iter()
@@ -121,9 +119,6 @@ pub fn epoch_info_with_num_seats(
         validator_to_index,
         block_producers_settlement,
         chunk_producers_settlement,
-        hidden_validators_settlement,
-        account_to_validators(fishermen),
-        fishermen_to_index,
         stake_change,
         validator_reward,
         validator_kickout.into_iter().collect(),