From 80959ad4910d64f7feb74b8c5df6fee6a74212a4 Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sat, 31 Dec 2016 08:33:13 -0500 Subject: [PATCH 1/5] Implement Postgres JSON data type --- diesel/Cargo.toml | 1 + diesel/src/pg/types/json.rs | 57 +++++++++++++++++++++++++++++++++++++ diesel/src/pg/types/mod.rs | 17 +++++++++++ 3 files changed, 75 insertions(+) create mode 100644 diesel/src/pg/types/json.rs diff --git a/diesel/Cargo.toml b/diesel/Cargo.toml index 8b768753b221..798a6ba3a598 100644 --- a/diesel/Cargo.toml +++ b/diesel/Cargo.toml @@ -17,6 +17,7 @@ byteorder = "0.3.*" quickcheck = { version = "0.3.1", optional = true } chrono = { version = "^0.2.17", optional = true } uuid = { version = ">=0.2.0, <0.4.0", optional = true, features = ["use_std"] } +serde_json = { version = "0.8", optional = true } [dev-dependencies] quickcheck = "0.3.1" diff --git a/diesel/src/pg/types/json.rs b/diesel/src/pg/types/json.rs new file mode 100644 index 000000000000..d65be81191a8 --- /dev/null +++ b/diesel/src/pg/types/json.rs @@ -0,0 +1,57 @@ +//! Support for JSON and `jsonb` values under PostgreSQL. + +extern crate serde_json; + +use std::io::prelude::*; +use std::error::Error; + +use pg::Pg; +use types::{self, ToSql, IsNull, FromSql}; + +primitive_impls!(Json -> (serde_json::Value, pg: (114, 199))); + +impl FromSql for serde_json::Value { + fn from_sql(bytes: Option<&[u8]>) -> Result> { + let bytes = not_none!(bytes); + serde_json::from_slice(&bytes) + .map_err(|e| Box::new(e) as Box) + } +} + +impl ToSql for serde_json::Value { + fn to_sql(&self, out: &mut W) -> Result> { + serde_json::to_writer(out, self) + .map(|_| IsNull::No) + .map_err(|e| Box::new(e) as Box) + } +} + +#[test] +fn json_to_sql() { + let mut bytes = vec![]; + let test_json = serde_json::Value::Bool(true); + ToSql::::to_sql(&test_json, &mut bytes).unwrap(); + assert_eq!(bytes, b"true"); +} + +#[test] +fn some_json_from_sql() { + let input_json = b"true"; + let output_json: serde_json::Value = + FromSql::::from_sql(Some(input_json)).unwrap(); + assert_eq!(output_json, serde_json::Value::Bool(true)); +} + +#[test] +fn bad_json_from_sql() { + let uuid: Result> = + FromSql::::from_sql(Some(b"boom")); + assert_eq!(uuid.unwrap_err().description(), "syntax error"); +} + +#[test] +fn no_json_from_sql() { + let uuid: Result> = + FromSql::::from_sql(None); + assert_eq!(uuid.unwrap_err().description(), "Unexpected null for non-null column"); +} diff --git a/diesel/src/pg/types/mod.rs b/diesel/src/pg/types/mod.rs index 61b1e4652e50..682a6d751a97 100644 --- a/diesel/src/pg/types/mod.rs +++ b/diesel/src/pg/types/mod.rs @@ -5,6 +5,8 @@ mod integers; mod primitives; #[cfg(feature = "uuid")] mod uuid; +#[cfg(feature = "serde_json")] +mod json; /// PostgreSQL specific SQL types /// @@ -88,4 +90,19 @@ pub mod sql_types { #[doc(hidden)] pub type Bpchar = ::types::VarChar; + + #[cfg(feature = "serde_json")] + /// The JSON SQL type. This type can only be used with `feature = + /// "serde_json"` + /// + /// ### [`ToSql`](/diesel/types/trait.ToSql.html) impls + /// + /// - [`serde_json::Value`][Value] + /// + /// ### [`FromSql`](/diesel/types/trait.FromSql.html) impls + /// + /// - [`serde_json`][Value] + /// + /// [Value]: https://docs.serde.rs/serde_json/value/enum.Value.html + #[derive(Debug, Clone, Copy, Default)] pub struct Json; } From 0cd97c689423d9ab2e8d97333606ab59b362f1dc Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sat, 31 Dec 2016 08:47:34 -0500 Subject: [PATCH 2/5] Implement Postgres jsonb (encoding version 1) datatype This is inspired by the following rust-postgres code: https://github.com/sfackler/rust-postgres/blob/de46ba2176d438ddc0b5d0752a6084f241b22242/postgres-shared/src/types/serde_json.rs Note that we only support encoding version 1, which is all that rust-postgres supports. But this seems to work fine in practice with our database. We can add other encodings as necessary. --- diesel/src/pg/types/json.rs | 58 +++++++++++++++++++++++++++++++++++++ diesel/src/pg/types/mod.rs | 15 ++++++++++ 2 files changed, 73 insertions(+) diff --git a/diesel/src/pg/types/json.rs b/diesel/src/pg/types/json.rs index d65be81191a8..2b921ca34a68 100644 --- a/diesel/src/pg/types/json.rs +++ b/diesel/src/pg/types/json.rs @@ -9,6 +9,7 @@ use pg::Pg; use types::{self, ToSql, IsNull, FromSql}; primitive_impls!(Json -> (serde_json::Value, pg: (114, 199))); +primitive_impls!(Jsonb -> (serde_json::Value, pg: (3802, 3807))); impl FromSql for serde_json::Value { fn from_sql(bytes: Option<&[u8]>) -> Result> { @@ -26,6 +27,26 @@ impl ToSql for serde_json::Value { } } +impl FromSql for serde_json::Value { + fn from_sql(bytes: Option<&[u8]>) -> Result> { + let bytes = not_none!(bytes); + if bytes[0] != 1 { + return Err("Unsupported JSONB encoding version".into()); + } + serde_json::from_slice(&bytes[1..]) + .map_err(|e| Box::new(e) as Box) + } +} + +impl ToSql for serde_json::Value { + fn to_sql(&self, out: &mut W) -> Result> { + try!(out.write_all(&[1])); + serde_json::to_writer(out, self) + .map(|_| IsNull::No) + .map_err(|e| Box::new(e) as Box) + } +} + #[test] fn json_to_sql() { let mut bytes = vec![]; @@ -55,3 +76,40 @@ fn no_json_from_sql() { FromSql::::from_sql(None); assert_eq!(uuid.unwrap_err().description(), "Unexpected null for non-null column"); } + +#[test] +fn jsonb_to_sql() { + let mut bytes = vec![]; + let test_json = serde_json::Value::Bool(true); + ToSql::::to_sql(&test_json, &mut bytes).unwrap(); + assert_eq!(bytes, b"\x01true"); +} + +#[test] +fn some_jsonb_from_sql() { + let input_json = b"\x01true"; + let output_json: serde_json::Value = + FromSql::::from_sql(Some(input_json)).unwrap(); + assert_eq!(output_json, serde_json::Value::Bool(true)); +} + +#[test] +fn bad_jsonb_from_sql() { + let uuid: Result> = + FromSql::::from_sql(Some(b"\x01boom")); + assert_eq!(uuid.unwrap_err().description(), "syntax error"); +} + +#[test] +fn bad_jsonb_version_from_sql() { + let uuid: Result> = + FromSql::::from_sql(Some(b"\x02true")); + assert_eq!(uuid.unwrap_err().description(), "Unsupported JSONB encoding version"); +} + +#[test] +fn no_jsonb_from_sql() { + let uuid: Result> = + FromSql::::from_sql(None); + assert_eq!(uuid.unwrap_err().description(), "Unexpected null for non-null column"); +} diff --git a/diesel/src/pg/types/mod.rs b/diesel/src/pg/types/mod.rs index 682a6d751a97..f98ec2d58dbe 100644 --- a/diesel/src/pg/types/mod.rs +++ b/diesel/src/pg/types/mod.rs @@ -105,4 +105,19 @@ pub mod sql_types { /// /// [Value]: https://docs.serde.rs/serde_json/value/enum.Value.html #[derive(Debug, Clone, Copy, Default)] pub struct Json; + + #[cfg(feature = "serde_json")] + /// The JSON SQL type. This type can only be used with `feature = + /// "serde_json"` + /// + /// ### [`ToSql`](/diesel/types/trait.ToSql.html) impls + /// + /// - [`serde_json::Value`][Value] + /// + /// ### [`FromSql`](/diesel/types/trait.FromSql.html) impls + /// + /// - [`serde_json`][Value] + /// + /// [Value]: https://docs.serde.rs/serde_json/value/enum.Value.html + #[derive(Debug, Clone, Copy, Default)] pub struct Jsonb; } From 3eeee825a8dc03cb033522f68e649843e7160d52 Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sat, 31 Dec 2016 10:57:54 -0500 Subject: [PATCH 3/5] Test JSON and jsonb in diesel_tests This allows us to verify that PostgreSQL actually understands the data that we're generating, and vice versa. --- diesel_tests/Cargo.toml | 3 ++- diesel_tests/tests/types.rs | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/diesel_tests/Cargo.toml b/diesel_tests/Cargo.toml index 1197c9c6900c..5ac807a002c7 100644 --- a/diesel_tests/Cargo.toml +++ b/diesel_tests/Cargo.toml @@ -13,11 +13,12 @@ dotenv = "0.8.0" [dependencies] assert_matches = "1.0.1" chrono = { version = "^0.2.17" } -diesel = { path = "../diesel", default-features = false, features = ["quickcheck", "chrono", "uuid"] } +diesel = { path = "../diesel", default-features = false, features = ["quickcheck", "chrono", "uuid", "serde_json"] } diesel_codegen = { path = "../diesel_codegen", optional = true } dotenv = "0.8.0" quickcheck = { version = "0.3.1", features = ["unstable"] } uuid = { version = ">=0.2.0, <0.4.0" } +serde_json = "0.8" [features] default = ["with-syntex"] diff --git a/diesel_tests/tests/types.rs b/diesel_tests/tests/types.rs index dfd1e2b35f20..5d8c299d983d 100644 --- a/diesel_tests/tests/types.rs +++ b/diesel_tests/tests/types.rs @@ -385,6 +385,41 @@ fn pg_uuid_to_sql_uuid() { assert!(!query_to_sql_equality::(expected_non_equal_value, value)); } +#[test] +#[cfg(feature = "postgres")] +fn pg_json_from_sql() { + extern crate serde_json; + + let query = "'true'::json"; + let expected_value = serde_json::Value::Bool(true); + assert_eq!(expected_value, query_single_value::(query)); +} + +// See http://stackoverflow.com/q/32843213/12089 for why we don't have a +// pg_json_to_sql_json test. There's no `'true':json = 'true':json` +// because JSON string representations are ambiguous. We _do_ have this +// test for `jsonb` values. + +#[test] +#[cfg(feature = "postgres")] +fn pg_jsonb_from_sql() { + extern crate serde_json; + + let query = "'true'::jsonb"; + let expected_value = serde_json::Value::Bool(true); + assert_eq!(expected_value, query_single_value::(query)); +} + +#[test] +#[cfg(feature = "postgres")] +fn pg_jsonb_to_sql_jsonb() { + extern crate serde_json; + + let expected_value = "'false'::jsonb"; + let value = serde_json::Value::Bool(false); + assert!(query_to_sql_equality::(expected_value, value)); +} + #[test] #[cfg(feature = "postgres")] fn text_array_can_be_assigned_to_varchar_array_column() { From e249061820a72b47658282612455e520edec09bc Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sat, 31 Dec 2016 12:57:07 -0500 Subject: [PATCH 4/5] Address various code review comments from @killercup --- diesel/src/pg/types/json.rs | 8 +++++++- diesel/src/pg/types/mod.rs | 23 ++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/diesel/src/pg/types/json.rs b/diesel/src/pg/types/json.rs index 2b921ca34a68..a7cbe8a3df91 100644 --- a/diesel/src/pg/types/json.rs +++ b/diesel/src/pg/types/json.rs @@ -8,13 +8,19 @@ use std::error::Error; use pg::Pg; use types::{self, ToSql, IsNull, FromSql}; +// The OIDs used to identify `json` and `jsonb` are not documented anywhere +// obvious, but they are discussed on various PostgreSQL mailing lists, +// including: +// +// https://www.postgresql.org/message-id/CA+mi_8Yv2SVOdhAtx-4CbpzoDtaJGkf8QvnushdF8bMgySAbYg@mail.gmail.com +// https://www.postgresql.org/message-id/CA+mi_8bd_g-MDPMwa88w0HXfjysaLFcrCza90+KL9zpRGbxKWg@mail.gmail.com primitive_impls!(Json -> (serde_json::Value, pg: (114, 199))); primitive_impls!(Jsonb -> (serde_json::Value, pg: (3802, 3807))); impl FromSql for serde_json::Value { fn from_sql(bytes: Option<&[u8]>) -> Result> { let bytes = not_none!(bytes); - serde_json::from_slice(&bytes) + serde_json::from_slice(bytes) .map_err(|e| Box::new(e) as Box) } } diff --git a/diesel/src/pg/types/mod.rs b/diesel/src/pg/types/mod.rs index f98ec2d58dbe..c50365e2607e 100644 --- a/diesel/src/pg/types/mod.rs +++ b/diesel/src/pg/types/mod.rs @@ -95,6 +95,9 @@ pub mod sql_types { /// The JSON SQL type. This type can only be used with `feature = /// "serde_json"` /// + /// Normally you should prefer `Jsonb` instead, for the reasons + /// discussed there. + /// /// ### [`ToSql`](/diesel/types/trait.ToSql.html) impls /// /// - [`serde_json::Value`][Value] @@ -107,9 +110,27 @@ pub mod sql_types { #[derive(Debug, Clone, Copy, Default)] pub struct Json; #[cfg(feature = "serde_json")] - /// The JSON SQL type. This type can only be used with `feature = + /// The `jsonb` SQL type. This type can only be used with `feature = /// "serde_json"` /// + /// `jsonb` offers [several advantages][adv] over regular JSON: + /// + /// > There are two JSON data types: `json` and `jsonb`. They accept almost + /// > identical sets of values as input. The major practical difference + /// > is one of efficiency. The `json` data type stores an exact copy of + /// > the input text, which processing functions must reparse on each + /// > execution; while `jsonb` data is stored in a decomposed binary format + /// > that makes it slightly slower to input due to added conversion + /// > overhead, but significantly faster to process, since no reparsing + /// > is needed. `jsonb` also supports indexing, which can be a significant + /// > advantage. + /// > + /// > ...In general, most applications should prefer to store JSON data as + /// > `jsonb`, unless there are quite specialized needs, such as legacy + /// > assumptions about ordering of object keys. + /// + /// [adv]: https://www.postgresql.org/docs/9.6/static/datatype-json.html + /// /// ### [`ToSql`](/diesel/types/trait.ToSql.html) impls /// /// - [`serde_json::Value`][Value] From ed1b4900dfe579405b083aab9effbc580407733c Mon Sep 17 00:00:00 2001 From: Pascal Hertleif Date: Fri, 13 Jan 2017 13:58:33 +0100 Subject: [PATCH 5/5] Add doc test for Jsonb type --- diesel/src/pg/types/mod.rs | 75 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/diesel/src/pg/types/mod.rs b/diesel/src/pg/types/mod.rs index c50365e2607e..3d8f62d05a16 100644 --- a/diesel/src/pg/types/mod.rs +++ b/diesel/src/pg/types/mod.rs @@ -140,5 +140,80 @@ pub mod sql_types { /// - [`serde_json`][Value] /// /// [Value]: https://docs.serde.rs/serde_json/value/enum.Value.html + /// + /// # Examples + /// + /// ```rust + /// # #![allow(dead_code)] + /// extern crate serde_json; + /// # #[macro_use] extern crate diesel; + /// # include!("src/doctest_setup.rs"); + /// # + /// # table! { + /// # users { + /// # id -> Serial, + /// # name -> VarChar, + /// # } + /// # } + /// # + /// struct Contact { + /// id: i32, + /// name: String, + /// address: serde_json::Value, + /// } + /// + /// impl_Queryable! { + /// struct Contact { + /// id: i32, + /// name: String, + /// address: serde_json::Value, + /// } + /// } + /// + /// struct NewContact { + /// name: String, + /// address: serde_json::Value, + /// } + /// + /// impl_Insertable! { + /// (contacts) + /// struct NewContact { + /// name: String, + /// address: serde_json::Value, + /// } + /// } + /// + /// table! { + /// contacts { + /// id -> Integer, + /// name -> VarChar, + /// address -> Jsonb, + /// } + /// } + /// + /// # fn main() { + /// # use self::diesel::insert; + /// # use self::contacts::dsl::*; + /// # let connection = connection_no_data(); + /// # connection.execute("CREATE TABLE contacts ( + /// # id SERIAL PRIMARY KEY, + /// # name VARCHAR NOT NULL, + /// # address JSONB NOT NULL + /// # )").unwrap(); + /// let santas_address: serde_json::Value = serde_json::from_str(r#"{ + /// "street": "Article Circle Expressway 1", + /// "city": "North Pole", + /// "postcode": "99705", + /// "state": "Alaska" + /// }"#).unwrap(); + /// let new_contact = NewContact { + /// name: "Claus".into(), + /// address: santas_address.clone() + /// }; + /// let inserted_contact = insert(&new_contact).into(contacts) + /// .get_result::(&connection).unwrap(); + /// assert_eq!(santas_address, inserted_contact.address); + /// # } + /// ``` #[derive(Debug, Clone, Copy, Default)] pub struct Jsonb; }