rust compiler crash

[ranwez]

I added the '.txt' extension to the BLOSUM62 file to be able to upload it. I've just started trying to use Rust, and this is a work in progress, but the compiler crash is holding me back since I have no idea how to resolve this issue and continue working on the code.

Code

use include_dir::{include_dir, Dir};
use nalgebra as na;
use std::collections::HashSet;
use std::fmt;
use std::io::{BufRead, BufReader, Read};

static PROJECT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR");

const AA: [char; 21] = [
    'A', 'R', 'N', 'D', 'C', 'Q', 'E', 'G', 'H', 'I', 'L', 'K', 'M', 'F', 'P', 'S', 'T', 'W', 'Y',
    'V', 'X',
];
// short long gap '-' internal and terminal FS and stop '!' and '*' : ['-', '-', '!', '!', '*', '*'];
const ALL_RESIDUES: [char; 27] = [
    'A', 'R', 'N', 'D', 'C', 'Q', 'E', 'G', 'H', 'I', 'L', 'K', 'M', 'F', 'P', 'S', 'T', 'W', 'Y',
    'V', 'X', '-', '-', '!', '!', '*', '*',
];

const DIM: usize = ALL_RESIDUES.len();

struct SiteInfo {
    nb_char: Vec<i32>,
}

//add debug trait
#[derive(Debug)]
pub struct ScoreMat {
    pub residues: Vec<char>,
    pub matrix: Vec<i32>,
    pub matrix_na: na::SMatrix<i32, DIM, DIM>,
}

fn have_same_unique_elements<T: Eq + std::hash::Hash, I1, I2>(iterable1: I1, iterable2: I2) -> bool
where
    I1: ExactSizeIterator<Item = T> + Clone,
    I2: ExactSizeIterator<Item = T> + Clone,
{
    let set1: HashSet<_> = iterable1.clone().collect();
    let set2: HashSet<_> = iterable2.clone().collect();

    if iterable1.len() != set1.len() || iterable2.len() != set2.len() {
        return false;
    }
    set1 == set2
}

fn have_subset_of_unique_elements<T: Eq + std::hash::Hash, I1, I2>(
    iterable1: &I1, iterable2: &I2,
) -> bool
where
    I1: ExactSizeIterator<Item = T> + Clone,
    I2: ExactSizeIterator<Item = T> + Clone,
{
    let set1: HashSet<_> = iterable1.clone().collect();
    let set2: HashSet<_> = iterable2.clone().collect();

    if iterable1.len() != set1.len() || iterable2.len() != set2.len() {
        return false;
    }
    set1.is_subset(&set2)
}

fn indices_it1_to_it2<T: Eq + std::hash::Hash, I1, I2>(
    it1: I1, it2: I2,
) -> Result<Vec<usize>, String>
where
    I1: ExactSizeIterator<Item = T> + Clone,
    I2: ExactSizeIterator<Item = T> + Clone,
{
    match have_subset_of_unique_elements(&it1, &it2) {
        false => Err("Some elements are missing".into()),
        true => {
            let mut reorder = vec![0; it1.len()];

            for (i, item1) in it1.enumerate() {
                if let Some(index) = it2.clone().position(|item2| item1 == item2) {
                    reorder[i] = index;
                }
            }
            Ok(reorder)
        }
    }
}

fn indices_it2_to_it1<T: Eq + std::hash::Hash, I1, I2>(
    it1: I1, it2: I2,
) -> Result<Vec<Option<usize>>, String>
where
    I1: ExactSizeIterator<Item = T> + Clone,
    I2: ExactSizeIterator<Item = T> + Clone,
{
    match have_subset_of_unique_elements(&it1, &it2) {
        false => Err("Some elements are missing".into()),
        true => {
            let mut reorder = vec![None; it2.len()];

            for (i2, item2) in it2.enumerate() {
                if let Some(index) = it1.clone().position(|item1| item1 == item2) {
                    reorder[i2] = Some(index);
                }
            }
            Ok(reorder)
        }
    }
}
enum MatrixType {
    BitRounded,
    RawOdd,
}

impl fmt::Display for ScoreMat {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        for c in ALL_RESIDUES {
            write!(f, "{:10} ", c)?;
        }
        writeln!(f)?;
        for i in 0..DIM {
            for j in 0..DIM {
                write!(f, "{:<10} ", self.matrix_na[(i, j)])?;
            }
            writeln!(f)?;
        }
        write!(f, "")
    }
}

impl ScoreMat {
    fn blosum62() -> ScoreMat {
        let file = PROJECT_DIR.get_file("src/data/matrices/BLOSUM62").unwrap();
        let reader = std::io::BufReader::new(file.contents());
        ScoreMat::new(reader, MatrixType::BitRounded)
    }
    fn blosum62_float() -> ScoreMat {
        let file = PROJECT_DIR.get_file("src/data/matrices/BLOSUM62_ODDS").unwrap();
        let reader = std::io::BufReader::new(file.contents());
        ScoreMat::new(reader, MatrixType::RawOdd)
    }

    fn new<R: Read>(reader: BufReader<R>, m_type: MatrixType) -> ScoreMat {
        let mut lines_iter = reader.lines();

        let br_residue_order = get_residue_names(skip_comment(&mut lines_iter));
        let save_br_residue_order = br_residue_order.clone();
        let br_id_to_map_id = indices_it2_to_it1( AA.into_iter(),br_residue_order.into_iter(),)
            .unwrap_or_else(|err| {
                panic!(
                    "Problem with the residue list of your score matrix file ({:?}) some AA ({:?}) are missing {:}",save_br_residue_order, AA.iter().clone(),
                    err
                );
            });

        let mut br_i = 0;
        let mut matrix: Vec<i32> = vec![0; DIM * DIM];

        while let Some(Ok(line)) = lines_iter.next() {
            let Some(i) = br_id_to_map_id[br_i] else {continue;};

            for (br_j, score) in line.split_whitespace().enumerate() {
                let Some(j) = br_id_to_map_id[br_j] else {continue; };
                //println!("{} {:?}", score, m_type);
                let score_i_j = match m_type {
                    MatrixType::RawOdd => {
                        10 * (score.parse::<f32>().unwrap_or_else(|err| {
                            panic!("Problem with the score matrix file: {}", err);
                        }) * 2.0)
                            .round() as i32
                    }
                    MatrixType::BitRounded => {
                        10 * score.parse::<i32>().unwrap_or_else(|err| {
                            panic!("Problem with the score matrix file: {}", err);
                        })
                    }
                };
                matrix[i * DIM + j] = score_i_j;
                println!("{}=>{} {}=>{} {}", i, br_i, j, br_j, score);
            }
            br_i += 1;
        }
        add_extra_cost(&mut matrix);
        ScoreMat {
            residues: ALL_RESIDUES.to_vec(),
            matrix_na: na::SMatrix::from_vec(matrix.clone()), // TODO check if transposition is needed
            matrix,
        }
    }
}

fn get_residue_names(line: String) -> Vec<char> {
    line.split_whitespace()
        .map(|s| {
            if s.len() != 1 {
                panic!("Invalid character {} in the score matrix file", s);
            } else {
                s.chars()
                    .next()
                    .unwrap_or_else(|| panic!("Invalid character {} in the score matrix file", s))
            }
        })
        .collect::<_>()
}

fn skip_comment<R: Read>(lines_iter: &mut std::io::Lines<BufReader<R>>) -> String {
    let mut gline: String = String::new();
    while let Some(Ok(line)) = lines_iter.next() {
        if line.starts_with('#') {
            continue;
        } else {
            gline = line;
            break;
        }
    }
    gline
}

fn add_extra_cost(matrix: &mut [i32]) {
    matrix[24 * DIM + 24] = -1;
    //todo!()
}
impl SiteInfo {
    fn new() -> SiteInfo {
        SiteInfo { nb_char: vec![0; DIM] }
    }
    fn new_with_params(nb_char: Vec<i32>) -> SiteInfo {
        if nb_char.len() != DIM {
            panic!("The number of characters in the site info is not equal to the number of characters in the alphabet")
        }
        SiteInfo { nb_char: nb_char.to_vec() }
    }

    /*
    fn score_in_front_of(&self, other: &SiteInfo, matrix: [[isize; 20]; 20]) -> isize {
        let mut score = 0;

        for i in 0..20 {
            for j in 0..20 {
                score += self.aa[i] as isize * other.aa[j] as isize * matrix[i][j];
            }
        }

        score
    }*/
}

fn main() {}

// add tests
#[cfg(test)]

mod tests {
    use crate::profile::site_info::ScoreMat;

    //test score_in_front method
    #[test]
    fn test_score_mat() {
        let score_mat = ScoreMat::blosum62();
        println!("to String");
        println!("{}", score_mat.to_string());
        println!("{:?}", score_mat.matrix_na[(0, 2)]);
    }
}

Meta

rustc --version --verbose:

rustc 1.71.0 (8ede3aae2 2023-07-12)
binary: rustc
commit-hash: 8ede3aae28fe6e4d52b38157d7bfe0d3bceef225
commit-date: 2023-07-12
host: aarch64-apple-darwin
release: 1.71.0
LLVM version: 16.0.5

Error output

thread 'rustc' panicked at 'called `Result::unwrap()` on an `Err` value: TryFromIntError(())', compiler/rustc_middle/src/query/on_disk_cache.rs:304:70
stack backtrace:
   0:        0x100c8ace0 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h3fbf720fbb1cb41b
   1:        0x100cd9a84 - core::fmt::write::h8ab6230d6f2a4204
   2:        0x100c80dbc - std::io::Write::write_fmt::h15ecdca1916a179e
   3:        0x100c8ab34 - std::sys_common::backtrace::print::hb6e94bab886a9ccf
   4:        0x100c8d598 - std::panicking::default_hook::{{closure}}::h4a0e30e90cb1c7bb
   5:        0x100c8d3a0 - std::panicking::default_hook::h0bb5bbf65ac7c64d
   6:        0x10914699c - rustc_driver_impl[732cfd3dd2e325d3]::install_ice_hook::{closure#0}
   7:        0x100c8dc4c - std::panicking::rust_panic_with_hook::h17facd9c53870157
   8:        0x100c8da2c - std::panicking::begin_panic_handler::{{closure}}::h9eab8195c369d860
   9:        0x100c8b0c0 - std::sys_common::backtrace::__rust_end_short_backtrace::hce5f67454da3493d
  10:        0x100c8d7a0 - _rust_begin_unwind
  11:        0x100d04a00 - core::panicking::panic_fmt::hc7e96873bfc1c7ba
  12:        0x100d04dc0 - core::result::unwrap_failed::h364bae7050cb2efa
  13:        0x10cfa2670 - <rustc_middle[7e3caebc53abe2aa]::query::on_disk_cache::OnDiskCache>::serialize
  14:        0x10ce921e4 - <rustc_middle[7e3caebc53abe2aa]::ty::context::TyCtxt>::serialize_query_result_cache
  15:        0x10c6320e4 - <rustc_session[1d7b5ca99579105d]::session::Session>::time::<core[d635094314abd71]::result::Result<usize, std[efadba9af302a35a]::io::error::Error>, rustc_incremental[788bec5d68261d83]::persist::save::encode_query_cache::{closure#0}>
  16:        0x10c62ae50 - rustc_incremental[788bec5d68261d83]::persist::file_format::save_in::<rustc_incremental[788bec5d68261d83]::persist::save::save_dep_graph::{closure#0}::{closure#2}::{closure#0}::{closure#0}>
  17:        0x10c6322a8 - <rustc_session[1d7b5ca99579105d]::session::Session>::time::<(), rustc_incremental[788bec5d68261d83]::persist::save::save_dep_graph::{closure#0}::{closure#2}::{closure#0}>
  18:        0x10c60a4c8 - rustc_data_structures[a3df03631e225155]::sync::join::<rustc_incremental[788bec5d68261d83]::persist::save::save_dep_graph::{closure#0}::{closure#2}, rustc_incremental[788bec5d68261d83]::persist::save::save_dep_graph::{closure#0}::{closure#3}, (), ()>
  19:        0x10c61c668 - rustc_incremental[788bec5d68261d83]::persist::save::save_dep_graph
  20:        0x1091055a4 - <rustc_session[1d7b5ca99579105d]::session::Session>::time::<(), <rustc_interface[91fb003f3a234889]::interface::Compiler>::enter<rustc_driver_impl[732cfd3dd2e325d3]::run_compiler::{closure#1}::{closure#2}, core[d635094314abd71]::result::Result<core[d635094314abd71]::option::Option<rustc_interface[91fb003f3a234889]::queries::Linker>, rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>>::{closure#0}>
  21:        0x109148144 - <rustc_interface[91fb003f3a234889]::interface::Compiler>::enter::<rustc_driver_impl[732cfd3dd2e325d3]::run_compiler::{closure#1}::{closure#2}, core[d635094314abd71]::result::Result<core[d635094314abd71]::option::Option<rustc_interface[91fb003f3a234889]::queries::Linker>, rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>>
  22:        0x109100af8 - rustc_span[c72d93d8a9f8a382]::set_source_map::<core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>, rustc_interface[91fb003f3a234889]::interface::run_compiler<core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>, rustc_driver_impl[732cfd3dd2e325d3]::run_compiler::{closure#1}>::{closure#0}::{closure#0}>
  23:        0x1090f7130 - <scoped_tls[6a70c94d6f3f123d]::ScopedKey<rustc_span[c72d93d8a9f8a382]::SessionGlobals>>::set::<rustc_interface[91fb003f3a234889]::interface::run_compiler<core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>, rustc_driver_impl[732cfd3dd2e325d3]::run_compiler::{closure#1}>::{closure#0}, core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>>
  24:        0x109103cac - std[efadba9af302a35a]::sys_common::backtrace::__rust_begin_short_backtrace::<rustc_interface[91fb003f3a234889]::util::run_in_thread_pool_with_globals<rustc_interface[91fb003f3a234889]::interface::run_compiler<core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>, rustc_driver_impl[732cfd3dd2e325d3]::run_compiler::{closure#1}>::{closure#0}, core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>>::{closure#0}::{closure#0}, core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>>
  25:        0x1090f4d20 - <<std[efadba9af302a35a]::thread::Builder>::spawn_unchecked_<rustc_interface[91fb003f3a234889]::util::run_in_thread_pool_with_globals<rustc_interface[91fb003f3a234889]::interface::run_compiler<core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>, rustc_driver_impl[732cfd3dd2e325d3]::run_compiler::{closure#1}>::{closure#0}, core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>>::{closure#0}::{closure#0}, core[d635094314abd71]::result::Result<(), rustc_span[c72d93d8a9f8a382]::ErrorGuaranteed>>::{closure#1} as core[d635094314abd71]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0}
  26:        0x100c96660 - std::sys::unix::thread::Thread::new::thread_start::h999abeaf027c3bba
  27:        0x1a810ffa8 - __pthread_joiner_wake

error: the compiler unexpectedly panicked. this is a bug.

note: we would appreciate a bug report: https://github.com/rust-lang/rust/issues/new?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md

note: rustc 1.71.0 (8ede3aae2 2023-07-12) running on aarch64-apple-darwin

note: compiler flags: -C embed-bitcode=no -C split-debuginfo=unpacked -C debuginfo=2 -C incremental=[REDACTED]

note: some of the compiler flags provided by cargo are hidden

query stack during panic:
end of query stack

Backtrace

<backtrace>

[BLOSUM62_ODDS.csv](https://github.com/rust-lang/rust/files/13060405/BLOSUM62_ODDS.csv) [BLOSUM62.txt](https://github.com/rust-lang/rust/files/13060409/BLOSUM62.txt)

[eopb]

Do you still get this crash after running cargo clean?

[ranwez]

Cargo clean didn't fix the issue. During my last compilation attempt I got a pop up window from macOS stating that applications used all available memory and asking to 'force quit' some of them, this was not sufficient and I got to restart my mac ... So  testing code modification to get rid of this issue is not easy ... the compilation attempt is very very long and now result in crashing my computer

[ranwez]

I identify the problematic lines (searching among my latest code modification). It appears that the issue is in the display function, the compiler crash if I try to lef align the number and work fine if I don't :

write!(f, "{:4} ", self.matrix_na[(i, j)])?;

write!(f, "{:<4} ", self.matrix_na[(i, j)])?;

in the function

impl fmt::Display for ScoreMat {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        for c in ALL_RESIDUES {
            write!(f, "{} ", c)?;
        }
        writeln!(f)?;
        for i in 0..DIM {
            for j in 0..DIM {
                write!(f, "{:4} ", self.matrix_na[(i, j)])?;
            }
            writeln!(f)?;
        }
        write!(f, "")
    }
}

So I can move forward... I hope this extra bit of information will help you fix the compiler bug.

[ranwez]

I was too optimistic ... I try adding a few lines of codes (not modifying the display function) and the compiler crashed again ...

So sometime the compiler crash everytime, even after cargo clean, and sometimes (when I comment/change a problematic line of code and I think I got it fixed), I have to cargo clean before every build to get it working. I'm on a MacBook Air M2 so this is maybe partly related to this issue shekohex/flutterust#21

[saethlin]

I fixed this ICE in #113562. If you update to the latest stable, probably with rustup update stable, you shouldn't see this crash anymore.

[ranwez]

Following your advice I have updated to the latest stable version.

rustc --version --verbose

rustc 1.73.0 (cc66ad468 2023-10-03)
binary: rustc
commit-hash: cc66ad468955717ab92600c770da8c1601a4ff33
commit-date: 2023-10-03
host: aarch64-apple-darwin
release: 1.73.0
LLVM version: 17.0.2

It no longer crash but it still fail to compile without cargo clean. I now got " error: linking with cc failed: exit status: 1 ..." if I try to compile without first using cargo clean. The exact same code compile and work fine if I use cargo clean first, so it is a kind of workaround but cleaning and recompiling every thing for each compilation/test is really slow and quite annoying.

[saethlin]

The problem now is that you are trying to include_dir! your entire project, including all its build artifacts. So each new build approximately doubles the size of the executable.

I think it's technically possible for ELF to support this, but by default relocation offsets or something are 32 bits. And in any case, you probably don't mean to do be doing what you're doing.

[ranwez]

Thanks for your rapid answers, and for fixing my code ;)
Changing

include_dir!("$CARGO_MANIFEST_DIR");

to

 include_dir!("$CARGO_MANIFEST_DIR/src/data");

fixed all my problems.

I'm new to rust and was previously using Java where one can pack data files within the distributed jar file and access them, I kind of struggled to find information on how to do this in Rust. I erroneously assumed the include_dir macro was just a way to get the correct path toward the data files packed within the executable version of the project and I didn't realized that it was an instruction to copy the folder, which in my case is already in the project folder...
Thank you very much for your rapid and efficient help!