SyncAggregate: 2% of invalid Signatures on block proposals where we are most of the validators.

[rodrigo-o]

#1288 introduced sync aggregates. On setups where we are almos every validator in the main spec, we loose near 2% of blocks due to invalid signatures, here is a debug example, apparently aggregation bits are correct so this may be a deeper issue.

# Subnet 0 the only one with less than 128 validators
"For subnet: 0 validators = [16, 54, 18, 19, 42, 27, 49, 62, 56, 35, 24, 53, 50, 36, 21, 52, 22, 17, 55, 58, 60, 14, 15, 20, 41, 40, 59, 13, 38, 51, 28, 34, 25, 32, 37, 31, 46, 63, 26, 57, 48, 45, 39, 43, 30, 61, 47, 29, 12, 23, 44, 19]"
validators added to the list: [34, 51, -1, 35, 57, -1, 47, 21, -1, -1, 31, 48, 39, 62, 60, -1, 
61, -1, 13, 28, 26, 44, 22, 18, 41, 43, -1, 32, 63, 12, 54, 38, -1, 45, 24, 58, -1, 56, 15, 53,
 37, 17, 27, 40, 19, -1, 55, -1, 46, -1, 59, 52, 14, 20, 30, 29, 
 36, 16, 23, 49, 42, 25, 50, -1, 34, 51, -1, 35, 57, -1, 47, 21, -1, -1, 31, 48, 39, 62, 60, -1,
 61, -1, 13, 28, 26, 44, 22, 18, 41, 43, -1, 32, 63, 12, 54, 38,
 -1, 45, 24, 58, -1, 56, 15, 53, 37, 17, 27, 40, 19, -1, 55, -1, 46, -1, 59, 52, 14, 20, 30, 29,
 36, 16, 23, 49, 42, 25, 50, -1]
aggregation bits: "01111111 11111101 01011111 11101110 11111011 11111101 01111100 11011011 01111111 11111101 01011111 11101110 11111011 11111101 01111100 11011011"

# Sync aggregate bits
sync_committee_bits: "
  11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111
  11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111
  11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111
  01111111 11111101 01011111 11101110 11111011 11111101 01111100 11011011 01111111 11111101 01011111 11101110 11111011 11111101 01111100 11011011"

# Validators from which this siganture was validated given the aggreegation bits
validated_indexes: [34, 51, -11, 35, 57, -9, 47, 21, -4, 0, 31, 48, 39, 62, 60, -1, 61, -8, 13, 28,
  26, 44, 22, 18, 41, 43, -5, 32, 63, 12, 54, 38, -3, 45, 24, 58, -7, 56, 15, 53,
  37, 17, 27, 40, 19, -2, 55, -6, 46, -33, 59, 52, 14, 20, 30, 29, 36, 16, 23,
  49, 42, 25, 50, -10, 34, 51, -11, 35, 57, -9, 47, 21, -4, 0, 31, 48, 39, 62,
  60, -1, 61, -8, 13, 28, 26, 44, 22, 18, 41, 43, -5, 32, 63, 12, 54, 38, -3, 45,
  24, 58, -7, 56, 15, 53, 37, 17, 27, 40, 19, -2, 55, -6, 46, -33, 59, 52, 14,
  20, 30, 29, 36, 16, 23, 49, 42, 25, 50, -10, 34, 51, 11, 35, 57, 9, 47, 21, 4,
  0, 31, 48, 39, 62, 60, 1, 61, 8, 13, 28, 26, 44, 22, 18, 41, 43, 5, 32, 63, 12,
  54, 38, 3, 45, 24, 58, 7, 56, 15, 53, 37, 17, 27, 40, 19, 2, 55, 6, 46, 33, 59,
  52, 14, 20, 30, 29, 36, 16, 23, 49, 42, 25, 50, 10, 34, 51, 11, 35, 57, 9, 47,
  21, 4, 0, 31, 48, 39, 62, 60, 1, 61, 8, 13, 28, 26, 44, 22, 18, 41, 43, 5, 32,
  63, 12, 54, 38, 3, 45, 24, 58, 7, 56, 15, 53, 37, 17, 27, 40, 19, 2, 55, 6, 46,
  33, 59, 52, 14, 20, 30, 29, 36, 16, 23, 49, 42, 25, 50, 10, 34, 51, 11, 35, 57,
  9, 47, 21, 4, 0, 31, 48, 39, 62, 60, 1, 61, 8, 13, 28, 26, 44, 22, 18, 41, 43,
  5, 32, 63, 12, 54, 38, 3, 45, 24, 58, 7, 56, 15, 53, 37, 17, 27, 40, 19, 2, 55,
  6, 46, 33, 59, 52, 14, 20, 30, 29, 36, 16, 23, 49, 42, 25, 50, 10, 34, 51, 11,
  35, 57, 9, 47, 21, 4, 0, 31, 48, 39, 62, 60, 1, 61, 8, 13, 28, 26, 44, 22, 18,
  41, 43, 5, 32, 63, 12, 54, 38, 3, 45, 24, 58, 7, 56, 15, 53, 37, 17, 27, 40,
  19, 2, 55, 6, 46, 33, 59, 52, 14, 20, 30, 29, 36, 16, 23, 49, 42, 25, 50, 10,
  34, 51, 11, 35, 57, 9, 47, 21, 4, 0, 31, 48, 39, 62, 60, 1, 61, 8, 13, 28, 26,
  44, 22, 18, 41, 43, 5, 32, 63, 12, 54, 38, 3, 45, 24, 58, 7, 56, 15, 53, 37,
  17, 27, 40, 19, 2, 55, 6, 46, 33, 59, 52, 14, 20, 30, 29, 36, 16, 23, 49, 42,
  25, 50, 10, 34, 51, 11, 35, 57, 9, 47, 21, 4, 0, 31, 48, 39, 62, 60, 1, 61, 8,
  13, 28, 26, 44, 22, 18, 41, 43, 5, 32, 63, 12, 54, 38, 3, 45, 24, 58, 7, 56,
  15, 53, 37, 17, 27, 40, 19, 2, 55, 6, 46, 33, 59, 52, 14, 20, 30, 29, 36, 16,
  23, 49, 42, 25, 50, 10]

# Comparision where everything looks correct....

agg:                     11011011,                               00111110                           10111111                            11011111        <-  11111011 11111101 01111100 11011011
subnet_check:            34, 51, -1, 35, 57, -1, 47, 21          -1, -1, 31, 48, 39, 62, 60, -1     61, -1, 13, 28, 26, 44, 22, 18      41, 43, -1, 32, 63, 12, 54, 38
validated_participants:  34, 51, -11, 35, 57, -9, 47, 21,        -4, -0, 31, 48, 39, 62, 60, -1,    61, -8, 13, 28, 26, 44, 22, 18,     41, 43, -5, 32, 63, 12, 54, 38,

agg:                     01110111                                11111010                           10111111                            11111110                <-  01111111 11111101 01011111 11101110
subnet_check:            -1, 45, 24, 58, -1, 56, 15, 53          37, 17, 27, 40, 19, -1, 55, -1     46, -1, 59, 52, 14, 20, 30, 29      36, 16, 23, 49, 42, 25, 50, -1
validated_participants:  -3, 45, 24, 58, -7, 56, 15, 53,         37, 17, 27, 40, 19, -2, 55, -6,    46, -33, 59, 52, 14, 20, 30, 29,    36, 16, 23, 49, 42, 25, 50, -10, 

# The following 8 bytes are the same as before
# 34, 51, -11, 35, 57, -9, 47, 21,        -4, -0, 31, 48, 39, 62, 60, -1,     61, -8, 13, 28, 26, 44, 22, 18,     41, 43, -5, 32, 63, 12, 54, 38, 
# -3, 45, 24, 58, -7, 56, 15, 53,         37, 17, 27, 40, 19, -2, 55, -6,     46, -33, 59, 52, 14, 20, 30, 29,    36, 16, 23, 49, 42, 25, 50, -10, 

[rodrigo-o]

Additional Information: This happened at End of epoch + 1 even when all the contributions are from other nodes.