From 00fbcdb453a4b261b109d77bdd557b73d4bf9d63 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Fri, 13 Dec 2024 19:04:05 -0500 Subject: [PATCH] Updates ion-rust to rc10 (#185) * Updates ion-rust to rc10 * Version bump to v0.10.0 --- Cargo.lock | 68 ++++++----- Cargo.toml | 8 +- src/bin/ion/commands/generate/generator.rs | 2 +- src/bin/ion/commands/inspect.rs | 128 +++++++++++++++++---- src/bin/ion/commands/schema/mod.rs | 3 + src/bin/ion/hex_reader.rs | 99 ++++++++-------- 6 files changed, 205 insertions(+), 103 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26cb2e3f..2c070b19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler2" @@ -212,6 +212,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "castaway" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.1.30" @@ -346,6 +355,20 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "compact_str" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "convert_case" version = "0.6.0" @@ -914,7 +937,7 @@ dependencies = [ [[package]] name = "ion-cli" -version = "0.9.1" +version = "0.10.0" dependencies = [ "anyhow", "assert_cmd", @@ -923,14 +946,12 @@ dependencies = [ "colored", "convert_case", "derive_builder", - "digest 0.9.0", "flate2", "infer", "ion-rs", "ion-schema", "itertools 0.13.0", "lowcharts", - "matches", "pager", "rstest", "serde", @@ -946,14 +967,15 @@ dependencies = [ [[package]] name = "ion-rs" -version = "1.0.0-rc.9" +version = "1.0.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3870f0125bfc40c9d8a738ac265dbc291d39c0b11882b238e4045d671f40f9c" +checksum = "7a3220cf0ff4ed8d316f359437847f8ce19d38ffdb2e588747efe4704aa00a97" dependencies = [ "arrayvec", "base64 0.12.3", "bumpalo", "chrono", + "compact_str", "delegate", "digest 0.9.0", "ice_code", @@ -970,13 +992,12 @@ dependencies = [ [[package]] name = "ion-schema" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa5961f23d04dc16ed65e67762527344ac02c808c6d1e7ea5af4f2fc5929c434" +checksum = "73d3f6caeca9720ae78eb68516353f6a1e2f8f589170284a03bdc355fbf5309a" dependencies = [ "half", "ion-rs", - "num-bigint", "num-traits", "regex", "thiserror", @@ -1092,12 +1113,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "memchr" version = "2.7.4" @@ -1129,17 +1144,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num-bigint" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-conv" version = "0.1.0" @@ -1533,6 +1537,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + [[package]] name = "ryu" version = "1.0.18" @@ -1701,6 +1711,12 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index 3c68ab65..c1b293af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ion-cli" -version = "0.9.1" +version = "0.10.0" authors = ["The Ion Team "] edition = "2021" description = "Command line tool for working with the Ion data format." @@ -15,23 +15,21 @@ keywords = ["format", "parse", "encode"] anyhow = "1.0" clap = { version = "4.5.8", features = ["cargo", "env", "wrap_help"] } colored = "2.0.0" -digest = "0.9" sha2 = "0.9" sha3 = "0.9" flate2 = "1.0" infer = "0.15.0" # ion-rs version must be pinned because we are using experimental features # See https://github.com/amazon-ion/ion-cli/issues/155 -ion-rs = { version = "=1.0.0-rc.9", features = ["experimental", "experimental-ion-hash"] } +ion-rs = { version = "1.0.0-rc.10", features = ["experimental", "experimental-ion-hash"] } tempfile = "3.2.0" -ion-schema = "0.14.1" +ion-schema = "0.15.0" lowcharts = "0.5.8" serde = { version = "1.0.163", features = ["derive"] } serde_json = { version = "1.0.81", features = ["arbitrary_precision", "preserve_order"] } base64 = "0.21.1" tera = { version = "1.18.1" } convert_case = { version = "0.6.0" } -matches = "0.1.10" thiserror = "1.0.50" zstd = "0.13.0" termcolor = "1.4.1" diff --git a/src/bin/ion/commands/generate/generator.rs b/src/bin/ion/commands/generate/generator.rs index 73d9ee1f..bac52dd3 100644 --- a/src/bin/ion/commands/generate/generator.rs +++ b/src/bin/ion/commands/generate/generator.rs @@ -39,7 +39,7 @@ pub(crate) struct CodeGenerator<'a, L: Language> { impl<'a> CodeGenerator<'a, RustLanguage> { #[allow(dead_code)] - pub fn new(output: &'a Path) -> CodeGenerator { + pub fn new(output: &'a Path) -> CodeGenerator<'a, RustLanguage> { let mut tera = Tera::default(); // Add all templates using `rust_templates` module constants // This allows packaging binary without the need of template resources. diff --git a/src/bin/ion/commands/inspect.rs b/src/bin/ion/commands/inspect.rs index 22b2a0f2..e7bfb33a 100644 --- a/src/bin/ion/commands/inspect.rs +++ b/src/bin/ion/commands/inspect.rs @@ -1,20 +1,19 @@ use std::fmt::Display; -use std::io::{Cursor, ErrorKind, Read, Write}; +use std::io::{Cursor, Write}; use std::str::FromStr; use crate::commands::{CommandIo, IonCliCommand, WithIonCliArgument}; +// The `inspect` command uses the `termcolor` crate to colorize its text when STDOUT is a TTY. +use crate::hex_reader::HexReader; +// When writing to a named file instead of STDOUT, `inspect` will use a `FileWriter` instead. +// `FileWriter` ignores all requests to emit TTY color escape codes. +use crate::output::CommandOutput; use anyhow::{bail, Context, Result}; use clap::builder::ValueParser; use clap::{Arg, ArgAction, ArgMatches, Command}; use ion_rs::v1_0::{EncodedBinaryValue, RawValueRef}; use ion_rs::*; - -// The `inspect` command uses the `termcolor` crate to colorize its text when STDOUT is a TTY. -use crate::hex_reader::HexReader; use termcolor::{Color, ColorSpec, WriteColor}; -// When writing to a named file instead of STDOUT, `inspect` will use a `FileWriter` instead. -// `FileWriter` ignores all requests to emit TTY color escape codes. -use crate::output::CommandOutput; pub struct InspectCommand; @@ -456,12 +455,25 @@ impl<'a, 'b> IonInspector<'a, 'b> { } fn inspect_eexp(&mut self, depth: usize, eexp: EExpression) -> Result<()> { + let LazyRawAnyEExpressionKind::Binary_1_1(raw_eexp) = eexp.raw_invocation().kind() else { + unreachable!("text e-expression") + }; + let mut formatter = BytesFormatter::new( BYTES_PER_ROW, vec![ - // TODO: Add methods to EExpression that allow nuanced access to its encoded spans - // TODO: Length-prefixed and multibyte e-expression addresses - IonBytes::new(BytesKind::MacroId, &eexp.span().bytes()[0..1]), + IonBytes::new( + BytesKind::MacroOpcodeAndAddress, + raw_eexp.opcode_and_address_span().bytes(), + ), + IonBytes::new( + BytesKind::TrailingLength, + raw_eexp.length_prefix_span().bytes(), + ), + IonBytes::new( + BytesKind::ArgumentEncodingBitmap, + raw_eexp.bitmap_span().bytes(), + ), ], ); self.newline()?; @@ -472,7 +484,12 @@ impl<'a, 'b> IonInspector<'a, 'b> { &mut formatter, )?; self.with_style(eexp_style(), |out| { - write!(out, "(:{}", eexp.invoked_macro().id_text())?; + if let Some(macro_name) = eexp.invoked_macro().name() { + write!(out, "(:{macro_name}")?; + } else { + write!(out, "(:{}", eexp.invoked_macro().id())?; + } + Ok(()) })?; for (param, arg_result) in eexp @@ -494,9 +511,66 @@ impl<'a, 'b> IonInspector<'a, 'b> { use MacroExprKind::*; match invocation.kind() { EExp(eexp_arg) => self.inspect_eexp(depth + 1, eexp_arg)?, - EExpArgGroup(_) => todo!("e-exp arg groups"), + EExpArgGroup(arg_group) => { + self.inspect_eexp_arg_group(depth + 1, arg_group)? + } TemplateMacro(_) | TemplateArgGroup(_) => { - unreachable!("e-exp args by definition cannot be template invocations") + unreachable!("e-exp args by definition cannot be TDL macro invocations") + } + } + } + } + } + self.write_text_only_line(depth, eexp_style(), ")")?; + Ok(()) + } + + fn inspect_eexp_arg_group( + &mut self, + depth: usize, + arg_group: EExpArgGroup, + ) -> Result<()> { + self.newline()?; + + let AnyEExpArgGroupKind::Binary_1_1(raw_arg_group) = arg_group.raw_arg_group().kind() + else { + unreachable!("text e-expression arg group") + }; + + let mut formatter = BytesFormatter::new( + BYTES_PER_ROW, + vec![IonBytes::new( + BytesKind::TrailingLength, + raw_arg_group.header_span().bytes(), + )], + ); + + self.write_offset_length_and_bytes( + depth, + raw_arg_group.range().start, + raw_arg_group.span().len(), + &mut formatter, + )?; + + self.write_with_style(eexp_style(), "(::")?; + self.write_with_style(comment_style(), " // arg group")?; + + // TODO: This impl will not evaluate nested e-expressions. + let nested_exprs = MacroExprArgsIterator::from_eexp_arg_group(arg_group.expressions()); + for expr in nested_exprs { + match expr? { + ValueExpr::ValueLiteral(value) => { + self.inspect_value(depth + 1, "", LazyValue::from(value), no_comment())? + } + ValueExpr::MacroInvocation(invocation) => { + use MacroExprKind::*; + match invocation.kind() { + EExp(eexp_arg) => self.inspect_eexp(depth + 1, eexp_arg)?, + EExpArgGroup(_arg_group) => { + unreachable!("e-exp arg groups cannot contain e-exp arg groups") + } + TemplateMacro(_) | TemplateArgGroup(_) => { + unreachable!("e-exp args by definition cannot be TDL macro invocations") } } } @@ -509,7 +583,6 @@ impl<'a, 'b> IonInspector<'a, 'b> { fn write_text_only_line(&mut self, depth: usize, style: ColorSpec, text: &str) -> Result<()> { self.newline()?; self.write_blank_offset_length_and_bytes(depth)?; - self.write_indentation(depth)?; self.write_with_style(style, text) } @@ -591,9 +664,6 @@ impl<'a, 'b> IonInspector<'a, 'b> { Template(_, _element) => { self.inspect_ephemeral_sequence(depth, "(", "", ")", delimiter, sexp, no_comment()) } - Constructed(_, _) => { - todo!() - } } } @@ -655,8 +725,9 @@ impl<'a, 'b> IonInspector<'a, 'b> { delimiter: &str, struct_: LazyStruct<'_, AnyEncoding>, ) -> Result<()> { + use ExpandedStructSource::*; match struct_.expanded().source() { - ExpandedStructSource::ValueLiteral(raw_struct) => { + ValueLiteral(raw_struct) => { use LazyRawValueKind::*; match raw_struct.as_value().kind() { Binary_1_0(v) => self.inspect_literal_struct(depth, delimiter, struct_, v), @@ -664,7 +735,7 @@ impl<'a, 'b> IonInspector<'a, 'b> { Text_1_0(_) | Text_1_1(_) => unreachable!("text value"), } } - ExpandedStructSource::Template(_, _, _) => { + Template(..) | MakeStruct(..) | MakeField(..) => { self.inspect_ephemeral_struct(depth, delimiter, struct_) } } @@ -676,9 +747,10 @@ impl<'a, 'b> IonInspector<'a, 'b> { self.newline()?; self.inspect_annotations(0, value)?; } + use ExpandedStructSource::*; let raw_struct = match struct_.expanded().source() { - ExpandedStructSource::ValueLiteral(raw_struct) => raw_struct, - ExpandedStructSource::Template(_, _, _) => todo!("Ion 1.1 template symbol table"), + ValueLiteral(raw_struct) => raw_struct, + Template(..) | MakeStruct(..) | MakeField(..) => todo!("Ion 1.1 template symbol table"), }; use LazyRawValueKind::*; @@ -906,9 +978,9 @@ impl<'a, 'b> IonInspector<'a, 'b> { let mut has_printed_skip_message = false; for value_res in nested_values { let nested_value = value_res?; + // If this value is a literal in the stream, see if it is in the bounds of byte // ranges we care about. - match self.select_action( depth, &mut has_printed_skip_message, @@ -1261,6 +1333,7 @@ impl<'a, 'b> IonInspector<'a, 'b> { self.write_offset_length_and_bytes_comment(depth, "", "", variable.name())?; ephemeral_value_style().set_underline(true).clone() } else { + self.write_offset_length_and_bytes_comment(depth, "", "", "")?; ephemeral_value_style().clone() }; @@ -1491,7 +1564,8 @@ fn ephemeral_annotations_style() -> ColorSpec { /// Kinds of encoding primitives found in a binary Ion stream. #[derive(Copy, Clone, Debug)] enum BytesKind { - MacroId, + MacroOpcodeAndAddress, + ArgumentEncodingBitmap, FieldId, Opcode, TrailingLength, @@ -1513,12 +1587,18 @@ impl BytesKind { .set_bold(true) .set_fg(Some(Color::Rgb(0, 0, 0))) .set_bg(Some(Color::Rgb(255, 255, 255))), - MacroId => color + MacroOpcodeAndAddress => color .set_bold(true) .set_fg(Some(Color::Rgb(0, 0, 0))) .set_bg(Some(Color::Green)) .set_bold(true) .set_intense(true), + ArgumentEncodingBitmap => color + .set_bold(true) + .set_bg(Some(Color::Rgb(0, 0, 0))) + .set_fg(Some(Color::Green)) + .set_bold(true) + .set_intense(true), TrailingLength => color .set_bold(true) .set_underline(true) diff --git a/src/bin/ion/commands/schema/mod.rs b/src/bin/ion/commands/schema/mod.rs index e51e2532..76a604ea 100644 --- a/src/bin/ion/commands/schema/mod.rs +++ b/src/bin/ion/commands/schema/mod.rs @@ -131,6 +131,9 @@ impl IonSchemaCommandInput { }) } + // If this ever gets used, the `expect` will cause a compiler error so the developer will + // know to come remove this. + #[expect(dead_code)] fn get_schema_system(&self) -> &SchemaSystem { &self.schema_system } diff --git a/src/bin/ion/hex_reader.rs b/src/bin/ion/hex_reader.rs index 1a6cf96f..1e98cdce 100644 --- a/src/bin/ion/hex_reader.rs +++ b/src/bin/ion/hex_reader.rs @@ -1,6 +1,5 @@ -use crate::hex_reader::DigitState::ZeroX; use ion_rs::{IonInput, IonStream}; -use std::io::{Bytes, Cursor, Error, ErrorKind, Read}; +use std::io::{Bytes, Error, ErrorKind, Read}; /// Wraps an existing reader in order to reinterpret the content of that reader as a /// hexadecimal-encoded byte stream. @@ -120,54 +119,60 @@ impl Read for HexReader { } } -#[test] -fn test_read_hex_digits() { - let hex = "00010203"; - let reader = HexReader::from(Cursor::new(hex)); - let translated_bytes: std::io::Result> = reader.bytes().collect(); - let expected = vec![0u8, 1, 2, 3]; - assert_eq!(expected, translated_bytes.unwrap()) -} +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn test_read_hex_digits() { + let hex = "00010203"; + let reader = HexReader::from(Cursor::new(hex)); + let translated_bytes: std::io::Result> = reader.bytes().collect(); + let expected = vec![0u8, 1, 2, 3]; + assert_eq!(expected, translated_bytes.unwrap()) + } -#[test] -fn test_read_hex_digits_with_whitespace() { - let hex = "00 01\n 02 \t \t\t 03 \r\n04"; - let reader = HexReader::from(Cursor::new(hex)); - let translated_bytes: std::io::Result> = reader.bytes().collect(); - let expected = vec![0u8, 1, 2, 3, 4]; - assert_eq!(expected, translated_bytes.unwrap()) -} + #[test] + fn test_read_hex_digits_with_whitespace() { + let hex = "00 01\n 02 \t \t\t 03 \r\n04"; + let reader = HexReader::from(Cursor::new(hex)); + let translated_bytes: std::io::Result> = reader.bytes().collect(); + let expected = vec![0u8, 1, 2, 3, 4]; + assert_eq!(expected, translated_bytes.unwrap()) + } -#[test] -fn test_read_hex_digits_with_leading_0x() { - let hex = "0x00 0x01 0x02 0x03 0x04"; - let reader = HexReader::from(Cursor::new(hex)); - let translated_bytes: std::io::Result> = reader.bytes().collect(); - let expected = vec![0u8, 1, 2, 3, 4]; - assert_eq!(expected, translated_bytes.unwrap()) -} + #[test] + fn test_read_hex_digits_with_leading_0x() { + let hex = "0x00 0x01 0x02 0x03 0x04"; + let reader = HexReader::from(Cursor::new(hex)); + let translated_bytes: std::io::Result> = reader.bytes().collect(); + let expected = vec![0u8, 1, 2, 3, 4]; + assert_eq!(expected, translated_bytes.unwrap()) + } -#[test] -fn test_read_hex_digits_with_commas() { - let hex = "00,01,02,03,04"; - let reader = HexReader::from(Cursor::new(hex)); - let translated_bytes: std::io::Result> = reader.bytes().collect(); - let expected = vec![0u8, 1, 2, 3, 4]; - assert_eq!(expected, translated_bytes.unwrap()) -} + #[test] + fn test_read_hex_digits_with_commas() { + let hex = "00,01,02,03,04"; + let reader = HexReader::from(Cursor::new(hex)); + let translated_bytes: std::io::Result> = reader.bytes().collect(); + let expected = vec![0u8, 1, 2, 3, 4]; + assert_eq!(expected, translated_bytes.unwrap()) + } -#[test] -fn test_read_odd_number_of_hex_digits() { - let hex = "000102030"; - let reader = HexReader::from(Cursor::new(hex)); - let translated_bytes: std::io::Result> = reader.bytes().collect(); - assert!(translated_bytes.is_err()) -} + #[test] + fn test_read_odd_number_of_hex_digits() { + let hex = "000102030"; + let reader = HexReader::from(Cursor::new(hex)); + let translated_bytes: std::io::Result> = reader.bytes().collect(); + assert!(translated_bytes.is_err()) + } -#[test] -fn test_read_hex_digits_with_invalid_char() { - let hex = "000102030Q"; - let reader = HexReader::from(Cursor::new(hex)); - let translated_bytes: std::io::Result> = reader.bytes().collect(); - assert!(translated_bytes.is_err()) + #[test] + fn test_read_hex_digits_with_invalid_char() { + let hex = "000102030Q"; + let reader = HexReader::from(Cursor::new(hex)); + let translated_bytes: std::io::Result> = reader.bytes().collect(); + assert!(translated_bytes.is_err()) + } }