From 0eb7f74c524962f5dfec4816f539d0af72d2bb1b Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Thu, 12 Jan 2023 11:54:15 +0800 Subject: [PATCH] attempt to add primitive types Signed-off-by: Eval EXEC --- Makefile | 6 +- bindings/rust/src/lib.rs | 2 +- std/rust/.gitignore | 3 + std/rust/Cargo.toml | 28 ++++ std/rust/build.rs | 19 +++ std/rust/primitive_types.mol | 14 ++ std/rust/src/lib.rs | 37 +++++ std/rust/src/primitives_pack.rs | 154 ++++++++++++++++++ tools/codegen/Cargo.toml | 1 + tools/codegen/primitive_types.mol | 14 ++ tools/codegen/src/ast/raw/mod.rs | 4 +- tools/codegen/src/ast/verified/complete.rs | 14 ++ tools/codegen/src/ast/verified/mod.rs | 40 ++++- tools/codegen/src/ast/verified/recover.rs | 14 ++ .../src/generator/languages/rust/mod.rs | 10 ++ 15 files changed, 350 insertions(+), 10 deletions(-) create mode 100644 std/rust/.gitignore create mode 100644 std/rust/Cargo.toml create mode 100644 std/rust/build.rs create mode 100644 std/rust/primitive_types.mol create mode 100644 std/rust/src/lib.rs create mode 100644 std/rust/src/primitives_pack.rs create mode 100644 tools/codegen/primitive_types.mol diff --git a/Makefile b/Makefile index c9375ba..373cff4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ ci: @set -eu; \ export RUSTFLAGS='-D warnings'; \ - make fmt clippy; \ + make fmt clippy check; \ make ci-examples ci-crates; \ echo "Success!" @@ -23,6 +23,10 @@ clean: cd - > /dev/null; \ done +check: + @set -eu; \ + diff std/rust/primitive_types.mol tools/codegen/primitive_types.mol + fmt: @set -eu; \ for dir in ${RUST_PROJS}; do \ diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index 528c57b..a7084d3 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -21,7 +21,7 @@ cfg_if::cfg_if! { pub mod error; pub mod prelude; -mod primitive; +pub mod primitive; // Little Endian pub type Number = u32; diff --git a/std/rust/.gitignore b/std/rust/.gitignore new file mode 100644 index 0000000..66ba3f1 --- /dev/null +++ b/std/rust/.gitignore @@ -0,0 +1,3 @@ +target/ +Cargo.lock + diff --git a/std/rust/Cargo.toml b/std/rust/Cargo.toml new file mode 100644 index 0000000..c12eff4 --- /dev/null +++ b/std/rust/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "molecule-std" +version = "0.0.1" +authors = ["Nervos Core Dev "] +edition = "2018" +description = "std primitive types for molecule." +homepage = "https://github.com/nervosnetwork/molecule" +repository = "https://github.com/nervosnetwork/molecule" +keywords = ["molecule", "code-generation", "serialization"] +categories = [ + "parser-implementations", + "development-tools::build-utils", + "encoding", + "data-structures" +] +license = "MIT" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +molecule = { path = "../../bindings/rust" } + +[features] +default = [] +with-primitive-types = [] + +[build-dependencies] +codegen = { package ="molecule-codegen", path = "../../tools/codegen" , features = ["gen-primitive-types"]} \ No newline at end of file diff --git a/std/rust/build.rs b/std/rust/build.rs new file mode 100644 index 0000000..36f2d8c --- /dev/null +++ b/std/rust/build.rs @@ -0,0 +1,19 @@ +use codegen::{Compiler, Language}; + +fn compile_schema(schema: &str) { + let mut compiler = Compiler::new(); + compiler + .input_schema_file(schema) + .generate_code(Language::Rust) + .output_dir_set_default() + .run() + .unwrap(); + println!("cargo:rerun-if-changed={}", schema); +} + +fn main() { + println!("cargo:rerun-if-changed=primitive_types.mol"); + compile_schema("primitive_types.mol"); + let out_dir = ::std::env::var("OUT_DIR").unwrap(); + println!("{}", out_dir); +} diff --git a/std/rust/primitive_types.mol b/std/rust/primitive_types.mol new file mode 100644 index 0000000..8ddbaf8 --- /dev/null +++ b/std/rust/primitive_types.mol @@ -0,0 +1,14 @@ +// Unsigned integers +array uint64 [byte; 8]; +array uint32 [byte; 4]; +array uint16 [byte; 2]; +array uint8 [byte; 1]; + +// Signed integers +array int64 [byte; 8]; +array int32 [byte; 4]; +array int16 [byte; 2]; +array int8 [byte; 1]; + +// Bool +array bool [byte; 1]; diff --git a/std/rust/src/lib.rs b/std/rust/src/lib.rs new file mode 100644 index 0000000..8e0f0d2 --- /dev/null +++ b/std/rust/src/lib.rs @@ -0,0 +1,37 @@ +pub use crate as molecule_std; + +pub use molecule::*; + +pub mod prelude { + pub use crate::primitive::{Byte, ByteReader}; + pub use molecule::prelude::*; + + #[cfg(feature = "with-primitive-types")] + pub use crate::primitives::*; +} + +#[cfg(feature = "with-primitive-types")] +mod primitives_pack; + +pub mod pack { + use crate::prelude::Entity; + pub trait Unpack { + /// Unpack binary data into rust types. + fn unpack(&self) -> T; + } + + /// A syntactic sugar to convert a rust type into binary data. + pub trait Pack { + /// Packs a rust type into binary data. + fn pack(&self) -> T; + } + #[cfg(feature = "with-primitive-types")] + pub use crate::primitives_pack::*; + + #[cfg(feature = "with-primitive-types")] + pub use crate::primitives::*; +} + +pub mod primitives { + include!(concat!(env!("OUT_DIR"), "/primitive_types.rs")); +} diff --git a/std/rust/src/primitives_pack.rs b/std/rust/src/primitives_pack.rs new file mode 100644 index 0000000..23011a3 --- /dev/null +++ b/std/rust/src/primitives_pack.rs @@ -0,0 +1,154 @@ +use crate::bytes::*; +use crate::pack::*; +use crate::prelude::*; + +macro_rules! impl_conversion_for_entity_unpack { + ($original:ty, $entity:ident) => { + impl Unpack<$original> for $entity { + fn unpack(&self) -> $original { + self.as_reader().unpack() + } + } + }; +} + +impl Pack for u64 { + fn pack(&self) -> Uint64 { + Uint64::new_unchecked(Bytes::from(self.to_le_bytes().to_vec())) + } +} + +impl<'r> Unpack for Uint64Reader<'r> { + fn unpack(&self) -> u64 { + let mut b = [0u8; 8]; + b.copy_from_slice(self.as_slice()); + u64::from_le_bytes(b) + } +} +impl_conversion_for_entity_unpack!(u64, Uint64); + +impl Pack for u32 { + fn pack(&self) -> Uint32 { + Uint32::new_unchecked(Bytes::from(self.to_le_bytes().to_vec())) + } +} + +impl<'r> Unpack for Uint32Reader<'r> { + fn unpack(&self) -> u32 { + let mut b = [0u8; 4]; + b.copy_from_slice(self.as_slice()); + u32::from_le_bytes(b) + } +} +impl_conversion_for_entity_unpack!(u32, Uint32); + +impl Pack for u16 { + fn pack(&self) -> Uint16 { + Uint16::new_unchecked(Bytes::from(self.to_le_bytes().to_vec())) + } +} + +impl<'r> Unpack for Uint16Reader<'r> { + fn unpack(&self) -> u16 { + let mut b = [0u8; 2]; + b.copy_from_slice(self.as_slice()); + u16::from_le_bytes(b) + } +} +impl_conversion_for_entity_unpack!(u16, Uint16); + +impl Pack for u8 { + fn pack(&self) -> Uint8 { + Uint8::new_unchecked(Bytes::from(self.to_le_bytes().to_vec())) + } +} + +impl<'r> Unpack for Uint8Reader<'r> { + fn unpack(&self) -> u8 { + let mut b = [0u8; 1]; + b.copy_from_slice(self.as_slice()); + u8::from_le_bytes(b) + } +} +impl_conversion_for_entity_unpack!(u8, Uint8); + +// Signed integers + +impl Pack for i64 { + fn pack(&self) -> Int64 { + Int64::new_unchecked(Bytes::from(self.to_le_bytes().to_vec())) + } +} + +impl<'r> Unpack for Int64Reader<'r> { + fn unpack(&self) -> i64 { + let mut b = [0u8; 8]; + b.copy_from_slice(self.as_slice()); + i64::from_le_bytes(b) + } +} +impl_conversion_for_entity_unpack!(i64, Int64); + +impl Pack for i32 { + fn pack(&self) -> Int32 { + Int32::new_unchecked(Bytes::from(self.to_le_bytes().to_vec())) + } +} + +impl<'r> Unpack for Int32Reader<'r> { + fn unpack(&self) -> i32 { + let mut b = [0u8; 4]; + b.copy_from_slice(self.as_slice()); + i32::from_le_bytes(b) + } +} +impl_conversion_for_entity_unpack!(i32, Int32); + +impl Pack for i16 { + fn pack(&self) -> Int16 { + Int16::new_unchecked(Bytes::from(self.to_le_bytes().to_vec())) + } +} + +impl<'r> Unpack for Int16Reader<'r> { + fn unpack(&self) -> i16 { + let mut b = [0u8; 2]; + b.copy_from_slice(self.as_slice()); + i16::from_le_bytes(b) + } +} +impl_conversion_for_entity_unpack!(i16, Int16); + +impl Pack for i8 { + fn pack(&self) -> Int8 { + Int8::new_unchecked(Bytes::from(self.to_le_bytes().to_vec())) + } +} + +impl<'r> Unpack for Int8Reader<'r> { + fn unpack(&self) -> i8 { + let mut b = [0u8; 1]; + b.copy_from_slice(self.as_slice()); + i8::from_le_bytes(b) + } +} +impl_conversion_for_entity_unpack!(i8, Int8); + +// Bool +impl Pack for bool { + fn pack(&self) -> Bool { + let b = u8::from(*self); + Bool::new_unchecked(Bytes::from(vec![b])) + } +} + +impl<'r> Unpack for BoolReader<'r> { + fn unpack(&self) -> bool { + match self.as_slice()[0] { + 0 => false, + 1 => true, + _ => unreachable!(), + } + } +} +impl_conversion_for_entity_unpack!(bool, Bool); diff --git a/tools/codegen/Cargo.toml b/tools/codegen/Cargo.toml index 4c1202c..6806e33 100644 --- a/tools/codegen/Cargo.toml +++ b/tools/codegen/Cargo.toml @@ -32,6 +32,7 @@ serde_yaml = { version = "0.8.15", optional = true } [features] default = [] compiler-plugin = ["serde", "serde_json", "serde_yaml"] +gen-primitive-types = [] [badges] maintenance = { status = "experimental" } diff --git a/tools/codegen/primitive_types.mol b/tools/codegen/primitive_types.mol new file mode 100644 index 0000000..8ddbaf8 --- /dev/null +++ b/tools/codegen/primitive_types.mol @@ -0,0 +1,14 @@ +// Unsigned integers +array uint64 [byte; 8]; +array uint32 [byte; 4]; +array uint16 [byte; 2]; +array uint8 [byte; 1]; + +// Signed integers +array int64 [byte; 8]; +array int32 [byte; 4]; +array int16 [byte; 2]; +array int8 [byte; 1]; + +// Bool +array bool [byte; 1]; diff --git a/tools/codegen/src/ast/raw/mod.rs b/tools/codegen/src/ast/raw/mod.rs index 6b97274..6765ced 100644 --- a/tools/codegen/src/ast/raw/mod.rs +++ b/tools/codegen/src/ast/raw/mod.rs @@ -44,7 +44,7 @@ pub(crate) struct UnionDecl { imported_depth: usize, } -#[derive(Debug, Property)] +#[derive(Debug, Property, Clone)] pub(crate) struct ArrayDecl { name: String, item: ItemDecl, @@ -73,7 +73,7 @@ pub(crate) struct TableDecl { imported_depth: usize, } -#[derive(Debug, Property)] +#[derive(Debug, Property, Clone)] pub(crate) struct ItemDecl { typ: String, } diff --git a/tools/codegen/src/ast/verified/complete.rs b/tools/codegen/src/ast/verified/complete.rs index 70f6ac5..01ad234 100644 --- a/tools/codegen/src/ast/verified/complete.rs +++ b/tools/codegen/src/ast/verified/complete.rs @@ -3,6 +3,9 @@ use std::{ rc::Rc, }; +#[cfg(not(feature = "gen-primitive-types"))] +use super::must_get_primitive_types; + use super::super::raw; trait CompleteRawDecl { @@ -179,6 +182,17 @@ impl super::Ast { "byte", Rc::new(super::TopDecl::new_primitive("byte").unwrap()), ); + + #[cfg(not(feature = "gen-primitive-types"))] + let bindings = must_get_primitive_types(); + #[cfg(not(feature = "gen-primitive-types"))] + bindings.iter().for_each(|primitive_type| { + decls_result.insert( + primitive_type.name(), + Rc::new(super::TopDecl::new_primitive(primitive_type.name()).unwrap()), + ); + }); + loop { if decls_keys.is_empty() { break; diff --git a/tools/codegen/src/ast/verified/mod.rs b/tools/codegen/src/ast/verified/mod.rs index 3f49e40..5869c42 100644 --- a/tools/codegen/src/ast/verified/mod.rs +++ b/tools/codegen/src/ast/verified/mod.rs @@ -14,6 +14,26 @@ pub use has_name::HasName; type Deps<'a> = HashMap<&'a str, Rc>; +#[cfg(not(feature = "gen-primitive-types"))] +pub fn must_get_primitive_types() -> Vec { + use crate::utils::ParserUtils; + crate::parser::Parser::preprocess(&std::path::PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/primitive_types.mol" + ))) + .unwrap() + .decls() + .iter() + .map(|decl| match decl { + crate::ast::raw::TopDecl::Array(array) => Primitive { + name: array.name().to_string(), + size: array.item_count(), + }, + _ => panic!("primitive types is not array"), + }) + .collect::>() +} + #[derive(Debug, Property)] #[property(get(public))] pub struct Ast { @@ -42,7 +62,7 @@ pub enum TopDecl { Table(Table), } -#[derive(Debug, Property)] +#[derive(Debug, Property, Clone)] #[property(get(public))] pub struct Primitive { name: String, @@ -134,14 +154,22 @@ impl Ast { impl TopDecl { fn new_primitive(name: &str) -> Option { - match name { - "byte" => Some(Primitive { + if name.eq("byte") { + return Some(Primitive { name: name.to_owned(), size: 1, - }), - _ => None, + }) + .map(Self::Primitive); } - .map(Self::Primitive) + + #[cfg(feature = "gen-primitive-types")] + return None; + + #[cfg(not(feature = "gen-primitive-types"))] + must_get_primitive_types() + .iter() + .find(|primitive_decl| primitive_decl.name().eq(name)) + .map(|v| Self::Primitive(v.clone())) } pub fn is_byte(&self) -> bool { diff --git a/tools/codegen/src/ast/verified/recover.rs b/tools/codegen/src/ast/verified/recover.rs index 84b6096..a7c6419 100644 --- a/tools/codegen/src/ast/verified/recover.rs +++ b/tools/codegen/src/ast/verified/recover.rs @@ -5,6 +5,9 @@ use std::{ use crate::ir; +#[cfg(not(feature = "gen-primitive-types"))] +use super::must_get_primitive_types; + trait RecoverFromIr { fn recover(&self, deps: &super::Deps) -> Option; } @@ -186,6 +189,17 @@ impl super::Ast { "byte", Rc::new(super::TopDecl::new_primitive("byte").unwrap()), ); + + #[cfg(not(feature = "gen-primitive-types"))] + let primitive_types = must_get_primitive_types(); + #[cfg(not(feature = "gen-primitive-types"))] + primitive_types.iter().for_each(|primitive_type| { + decls_result.insert( + primitive_type.name(), + Rc::new(super::TopDecl::new_primitive(primitive_type.name()).unwrap()), + ); + }); + loop { if decls_keys.is_empty() { break; diff --git a/tools/codegen/src/generator/languages/rust/mod.rs b/tools/codegen/src/generator/languages/rust/mod.rs index 86bebb3..2e7ba6c 100644 --- a/tools/codegen/src/generator/languages/rust/mod.rs +++ b/tools/codegen/src/generator/languages/rust/mod.rs @@ -40,9 +40,19 @@ impl super::LanguageGenerator for Generator { fn generate(writer: &mut W, ast: &ast::Ast) -> io::Result<()> { writeln!(writer, "// Generated by Molecule {}", VERSION)?; writeln!(writer)?; + + #[cfg(feature = "gen-primitive-types")] let code = quote!( use molecule::prelude::*; ); + + #[cfg(not(feature = "gen-primitive-types"))] + let code = quote!( + use molecule_std as molecule; + + use molecule::prelude::*; + ); + write!(writer, "{}", code)?; let imports = ast.imports(); if !imports.is_empty() {