From 3fb13537ebedb23f38e93a9f3557cdf7d3ddeab3 Mon Sep 17 00:00:00 2001
From: Nicholas Gates <gatesn@users.noreply.github.com>
Date: Wed, 12 Jun 2024 14:58:22 +0100
Subject: [PATCH] DataFusion expr conversion (#349)

---
 Cargo.lock                                    | 711 +++++++++++++-----
 Cargo.toml                                    |   2 +
 vortex-array/benches/compare.rs               |   8 +-
 .../src/array/bool/compute/compare.rs         |  24 +-
 .../src/array/primitive/compute/compare.rs    |  12 +-
 vortex-dtype/src/field_paths.rs               |   4 +
 vortex-expr/Cargo.toml                        |   6 +-
 vortex-expr/src/datafusion.rs                 |  63 ++
 vortex-expr/src/display.rs                    |  12 +-
 vortex-expr/src/expressions.rs                |  14 +-
 vortex-expr/src/field_paths.rs                |  12 +-
 vortex-expr/src/lib.rs                        |   1 +
 vortex-expr/src/operators.rs                  |  48 +-
 vortex-scalar/Cargo.toml                      |   2 +
 vortex-scalar/src/datafusion.rs               |  68 ++
 vortex-scalar/src/lib.rs                      |   1 +
 16 files changed, 714 insertions(+), 274 deletions(-)
 create mode 100644 vortex-expr/src/datafusion.rs
 create mode 100644 vortex-scalar/src/datafusion.rs

diff --git a/Cargo.lock b/Cargo.lock
index d0d5358626..59eefc6eec 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -118,32 +118,68 @@ version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658"
 dependencies = [
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-csv",
- "arrow-data",
- "arrow-ipc",
- "arrow-json",
- "arrow-ord",
- "arrow-row",
- "arrow-schema",
- "arrow-select",
- "arrow-string",
+ "arrow-arith 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-csv 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-ipc 51.0.0",
+ "arrow-json 51.0.0",
+ "arrow-ord 51.0.0",
+ "arrow-row 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
+ "arrow-string 51.0.0",
  "pyo3",
 ]
 
+[[package]]
+name = "arrow"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38"
+dependencies = [
+ "arrow-arith 52.0.0",
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-cast 52.0.0",
+ "arrow-csv 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-ipc 52.0.0",
+ "arrow-json 52.0.0",
+ "arrow-ord 52.0.0",
+ "arrow-row 52.0.0",
+ "arrow-schema 52.0.0",
+ "arrow-select 52.0.0",
+ "arrow-string 52.0.0",
+]
+
 [[package]]
 name = "arrow-arith"
 version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "chrono",
+ "half",
+ "num",
+]
+
+[[package]]
+name = "arrow-arith"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043"
+dependencies = [
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
  "chrono",
  "half",
  "num",
@@ -156,11 +192,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea"
 dependencies = [
  "ahash",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
+ "arrow-buffer 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
  "chrono",
- "chrono-tz",
+ "chrono-tz 0.8.6",
+ "half",
+ "hashbrown",
+ "num",
+]
+
+[[package]]
+name = "arrow-array"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97"
+dependencies = [
+ "ahash",
+ "arrow-buffer 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
+ "chrono",
+ "chrono-tz 0.9.0",
  "half",
  "hashbrown",
  "num",
@@ -177,17 +230,49 @@ dependencies = [
  "num",
 ]
 
+[[package]]
+name = "arrow-buffer"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1"
+dependencies = [
+ "bytes",
+ "half",
+ "num",
+]
+
 [[package]]
 name = "arrow-cast"
 version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
+ "atoi",
+ "base64 0.22.1",
+ "chrono",
+ "comfy-table",
+ "half",
+ "lexical-core",
+ "num",
+ "ryu",
+]
+
+[[package]]
+name = "arrow-cast"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab"
+dependencies = [
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
+ "arrow-select 52.0.0",
  "atoi",
  "base64 0.22.1",
  "chrono",
@@ -204,11 +289,30 @@ version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-schema",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "chrono",
+ "csv",
+ "csv-core",
+ "lazy_static",
+ "lexical-core",
+ "regex",
+]
+
+[[package]]
+name = "arrow-csv"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc"
+dependencies = [
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-cast 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
  "chrono",
  "csv",
  "csv-core",
@@ -223,8 +327,20 @@ version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5"
 dependencies = [
- "arrow-buffer",
- "arrow-schema",
+ "arrow-buffer 51.0.0",
+ "arrow-schema 51.0.0",
+ "half",
+ "num",
+]
+
+[[package]]
+name = "arrow-data"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10"
+dependencies = [
+ "arrow-buffer 52.0.0",
+ "arrow-schema 52.0.0",
  "half",
  "num",
 ]
@@ -235,27 +351,61 @@ version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-schema",
- "flatbuffers",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "flatbuffers 23.5.26",
  "lz4_flex",
  "zstd",
 ]
 
+[[package]]
+name = "arrow-ipc"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8"
+dependencies = [
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-cast 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
+ "flatbuffers 24.3.25",
+]
+
 [[package]]
 name = "arrow-json"
 version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-schema",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "chrono",
+ "half",
+ "indexmap",
+ "lexical-core",
+ "num",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "arrow-json"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1"
+dependencies = [
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-cast 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
  "chrono",
  "half",
  "indexmap",
@@ -271,11 +421,26 @@ version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
+ "half",
+ "num",
+]
+
+[[package]]
+name = "arrow-ord"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413"
+dependencies = [
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
+ "arrow-select 52.0.0",
  "half",
  "num",
 ]
@@ -287,10 +452,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228"
 dependencies = [
  "ahash",
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "half",
+ "hashbrown",
+]
+
+[[package]]
+name = "arrow-row"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e"
+dependencies = [
+ "ahash",
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
  "half",
  "hashbrown",
 ]
@@ -304,6 +484,12 @@ dependencies = [
  "bitflags 2.5.0",
 ]
 
+[[package]]
+name = "arrow-schema"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393"
+
 [[package]]
 name = "arrow-select"
 version = "51.0.0"
@@ -311,10 +497,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830"
 dependencies = [
  "ahash",
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "num",
+]
+
+[[package]]
+name = "arrow-select"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102"
+dependencies = [
+ "ahash",
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
  "num",
 ]
 
@@ -324,11 +524,28 @@ version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
+ "memchr",
+ "num",
+ "regex",
+ "regex-syntax",
+]
+
+[[package]]
+name = "arrow-string"
+version = "52.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597"
+dependencies = [
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-data 52.0.0",
+ "arrow-schema 52.0.0",
+ "arrow-select 52.0.0",
  "memchr",
  "num",
  "regex",
@@ -748,8 +965,8 @@ dependencies = [
 name = "bench-vortex"
 version = "0.1.0"
 dependencies = [
- "arrow-array",
- "arrow-select",
+ "arrow-array 51.0.0",
+ "arrow-select 51.0.0",
  "bzip2",
  "criterion",
  "csv",
@@ -1036,7 +1253,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e"
 dependencies = [
  "chrono",
- "chrono-tz-build",
+ "chrono-tz-build 0.2.1",
+ "phf",
+]
+
+[[package]]
+name = "chrono-tz"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb"
+dependencies = [
+ "chrono",
+ "chrono-tz-build 0.3.0",
  "phf",
 ]
 
@@ -1051,6 +1279,17 @@ dependencies = [
  "phf_codegen",
 ]
 
+[[package]]
+name = "chrono-tz-build"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1"
+dependencies = [
+ "parse-zoneinfo",
+ "phf",
+ "phf_codegen",
+]
+
 [[package]]
 name = "ciborium"
 version = "0.2.2"
@@ -1383,18 +1622,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85069782056753459dc47e386219aa1fdac5b731f26c28abb8c0ffd4b7c5ab11"
 dependencies = [
  "ahash",
- "arrow",
- "arrow-array",
- "arrow-ipc",
- "arrow-schema",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-ipc 51.0.0",
+ "arrow-schema 51.0.0",
  "async-trait",
  "bytes",
  "chrono",
  "dashmap",
- "datafusion-common",
+ "datafusion-common 37.1.0",
  "datafusion-common-runtime",
  "datafusion-execution",
- "datafusion-expr",
+ "datafusion-expr 37.1.0",
  "datafusion-functions",
  "datafusion-functions-array",
  "datafusion-optimizer",
@@ -1413,7 +1652,7 @@ dependencies = [
  "parking_lot",
  "pin-project-lite",
  "rand",
- "sqlparser",
+ "sqlparser 0.44.0",
  "tempfile",
  "tokio",
  "url",
@@ -1427,17 +1666,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "309d9040751f6dc9e33c85dce6abb55a46ef7ea3644577dd014611c379447ef3"
 dependencies = [
  "ahash",
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-schema",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-schema 51.0.0",
  "chrono",
  "half",
  "instant",
  "libc",
  "num_cpus",
  "object_store",
- "sqlparser",
+ "sqlparser 0.44.0",
+]
+
+[[package]]
+name = "datafusion-common"
+version = "39.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971"
+dependencies = [
+ "ahash",
+ "arrow 52.0.0",
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
+ "arrow-schema 52.0.0",
+ "chrono",
+ "half",
+ "hashbrown",
+ "instant",
+ "libc",
+ "num_cpus",
+ "sqlparser 0.47.0",
 ]
 
 [[package]]
@@ -1455,11 +1714,11 @@ version = "37.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06a3a29ae36bcde07d179cc33b45656a8e7e4d023623e320e48dcf1200eeee95"
 dependencies = [
- "arrow",
+ "arrow 51.0.0",
  "chrono",
  "dashmap",
- "datafusion-common",
- "datafusion-expr",
+ "datafusion-common 37.1.0",
+ "datafusion-expr 37.1.0",
  "futures",
  "hashbrown",
  "log",
@@ -1477,12 +1736,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a3542aa322029c2121a671ce08000d4b274171070df13f697b14169ccf4f628"
 dependencies = [
  "ahash",
- "arrow",
- "arrow-array",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "chrono",
+ "datafusion-common 37.1.0",
+ "paste",
+ "sqlparser 0.44.0",
+ "strum",
+ "strum_macros",
+]
+
+[[package]]
+name = "datafusion-expr"
+version = "39.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3"
+dependencies = [
+ "ahash",
+ "arrow 52.0.0",
+ "arrow-array 52.0.0",
+ "arrow-buffer 52.0.0",
  "chrono",
- "datafusion-common",
+ "datafusion-common 39.0.0",
  "paste",
- "sqlparser",
+ "serde_json",
+ "sqlparser 0.47.0",
  "strum",
  "strum_macros",
 ]
@@ -1493,14 +1771,14 @@ version = "37.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd221792c666eac174ecc09e606312844772acc12cbec61a420c2fca1ee70959"
 dependencies = [
- "arrow",
+ "arrow 51.0.0",
  "base64 0.22.1",
  "blake2",
  "blake3",
  "chrono",
- "datafusion-common",
+ "datafusion-common 37.1.0",
  "datafusion-execution",
- "datafusion-expr",
+ "datafusion-expr 37.1.0",
  "datafusion-physical-expr",
  "hex",
  "itertools 0.12.1",
@@ -1518,14 +1796,14 @@ version = "37.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e501801e84d9c6ef54caaebcda1b18a6196a24176c12fb70e969bc0572e03c55"
 dependencies = [
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-ord",
- "arrow-schema",
- "datafusion-common",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-ord 51.0.0",
+ "arrow-schema 51.0.0",
+ "datafusion-common 37.1.0",
  "datafusion-execution",
- "datafusion-expr",
+ "datafusion-expr 37.1.0",
  "datafusion-functions",
  "itertools 0.12.1",
  "log",
@@ -1538,11 +1816,11 @@ version = "37.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "76bd7f5087817deb961764e8c973d243b54f8572db414a8f0a8f33a48f991e0a"
 dependencies = [
- "arrow",
+ "arrow 51.0.0",
  "async-trait",
  "chrono",
- "datafusion-common",
- "datafusion-expr",
+ "datafusion-common 37.1.0",
+ "datafusion-expr 37.1.0",
  "datafusion-physical-expr",
  "hashbrown",
  "itertools 0.12.1",
@@ -1557,19 +1835,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5cabc0d9aaa0f5eb1b472112f16223c9ffd2fb04e58cbf65c0a331ee6e993f96"
 dependencies = [
  "ahash",
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-ord",
- "arrow-schema",
- "arrow-string",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-ord 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-string 51.0.0",
  "base64 0.22.1",
  "blake2",
  "blake3",
  "chrono",
- "datafusion-common",
+ "datafusion-common 37.1.0",
  "datafusion-execution",
- "datafusion-expr",
+ "datafusion-expr 37.1.0",
  "half",
  "hashbrown",
  "hex",
@@ -1592,16 +1870,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17c0523e9c8880f2492a88bbd857dde02bed1ed23f3e9211a89d3d7ec3b44af9"
 dependencies = [
  "ahash",
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-schema",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-schema 51.0.0",
  "async-trait",
  "chrono",
- "datafusion-common",
+ "datafusion-common 37.1.0",
  "datafusion-common-runtime",
  "datafusion-execution",
- "datafusion-expr",
+ "datafusion-expr 37.1.0",
  "datafusion-physical-expr",
  "futures",
  "half",
@@ -1622,13 +1900,13 @@ version = "37.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49eb54b42227136f6287573f2434b1de249fe1b8e6cd6cc73a634e4a3ec29356"
 dependencies = [
- "arrow",
- "arrow-array",
- "arrow-schema",
- "datafusion-common",
- "datafusion-expr",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-schema 51.0.0",
+ "datafusion-common 37.1.0",
+ "datafusion-expr 37.1.0",
  "log",
- "sqlparser",
+ "sqlparser 0.44.0",
  "strum",
 ]
 
@@ -1837,6 +2115,16 @@ dependencies = [
  "rustc_version",
 ]
 
+[[package]]
+name = "flatbuffers"
+version = "24.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f"
+dependencies = [
+ "bitflags 1.3.2",
+ "rustc_version",
+]
+
 [[package]]
 name = "flatc"
 version = "0.2.2+23.5.26"
@@ -2466,14 +2754,14 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "362b480df322cd9a1d0ae9336197001d29140843d3716a31a8f55ced9c000a54"
 dependencies = [
- "arrow",
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
- "arrow-ord",
- "arrow-row",
- "arrow-schema",
- "arrow-select",
+ "arrow 51.0.0",
+ "arrow-arith 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-ord 51.0.0",
+ "arrow-row 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
  "async-recursion",
  "async-trait",
  "async_cell",
@@ -2525,12 +2813,12 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a227e1c408cce3db3270a41e80b353f4abb90d0891de7bc53aa9ee7683f77d8f"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
  "getrandom",
  "half",
  "num-traits",
@@ -2544,14 +2832,14 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a88115f28618a2ca888fe41aeecca49bb90d4c61ba4c25f457c5a86f29fce5f9"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-schema",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-schema 51.0.0",
  "async-trait",
  "byteorder",
  "bytes",
  "chrono",
- "datafusion-common",
+ "datafusion-common 37.1.0",
  "datafusion-sql",
  "futures",
  "lance-arrow",
@@ -2580,13 +2868,13 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1d43fb160792ce395d76ed864b7e1c6ef3a9fe46e5fe54f5339409aaf89c037"
 dependencies = [
- "arrow",
- "arrow-array",
- "arrow-ord",
- "arrow-schema",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-ord 51.0.0",
+ "arrow-schema 51.0.0",
  "async-trait",
  "datafusion",
- "datafusion-common",
+ "datafusion-common 37.1.0",
  "datafusion-functions",
  "datafusion-physical-expr",
  "futures",
@@ -2604,10 +2892,10 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7abd3dc7a5420ba82abb8dc3f826ff0d2b08c8e61ec20e5c9b087695ff583aa8"
 dependencies = [
- "arrow",
- "arrow-array",
- "arrow-cast",
- "arrow-schema",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-schema 51.0.0",
  "chrono",
  "futures",
  "hex",
@@ -2621,12 +2909,12 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "694a0d2ce6617d83ccbccbe03b092343e03d0d3de389773143a6994575bb6c40"
 dependencies = [
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-schema",
- "arrow-select",
+ "arrow-arith 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
  "bytes",
  "futures",
  "lance-arrow",
@@ -2648,17 +2936,17 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fa0d5fa512333bf420475d34d631262ba4de6d604c93eaa7fb3d7fa2760c1758"
 dependencies = [
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow-arith 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
  "async-recursion",
  "async-trait",
  "byteorder",
  "bytes",
- "datafusion-common",
+ "datafusion-common 37.1.0",
  "futures",
  "lance-arrow",
  "lance-core",
@@ -2684,16 +2972,16 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2f1892475738d35b46adb4f474934999545f46d23cdbb38181b45d8cdcbf471"
 dependencies = [
- "arrow",
- "arrow-array",
- "arrow-ord",
- "arrow-schema",
- "arrow-select",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-ord 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
  "async-recursion",
  "async-trait",
  "datafusion",
- "datafusion-common",
- "datafusion-expr",
+ "datafusion-common 37.1.0",
+ "datafusion-expr 37.1.0",
  "datafusion-physical-expr",
  "datafusion-sql",
  "futures",
@@ -2730,14 +3018,14 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e7143cf3e0175ed720be9515a188d7c37f748f2ffe32b08af65d45aed5a0af00"
 dependencies = [
- "arrow",
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow 51.0.0",
+ "arrow-arith 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
  "async-recursion",
  "async-trait",
  "aws-config",
@@ -2770,9 +3058,9 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6d737aabcaa7c34c1bd70282d667b45df7549b5cf80e44ed2a3481fbdebd260"
 dependencies = [
- "arrow-array",
- "arrow-ord",
- "arrow-schema",
+ "arrow-array 51.0.0",
+ "arrow-ord 51.0.0",
+ "arrow-schema 51.0.0",
  "cc",
  "futures",
  "half",
@@ -2793,11 +3081,11 @@ version = "0.10.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9017c6d5aa45b233a835ac156a48621f61e552bebb8fdbb11952e96581c6aca"
 dependencies = [
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-ipc",
- "arrow-schema",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-ipc 51.0.0",
+ "arrow-schema 51.0.0",
  "async-trait",
  "aws-credential-types",
  "byteorder",
@@ -3465,13 +3753,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32"
 dependencies = [
  "ahash",
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-ipc",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-cast 51.0.0",
+ "arrow-data 51.0.0",
+ "arrow-ipc 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
  "base64 0.22.1",
  "brotli",
  "bytes",
@@ -3835,7 +4123,7 @@ dependencies = [
 name = "pyvortex"
 version = "0.1.0"
 dependencies = [
- "arrow",
+ "arrow 51.0.0",
  "log",
  "paste",
  "pyo3",
@@ -4538,6 +4826,16 @@ dependencies = [
  "sqlparser_derive",
 ]
 
+[[package]]
+name = "sqlparser"
+version = "0.47.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25"
+dependencies = [
+ "log",
+ "sqlparser_derive",
+]
+
 [[package]]
 name = "sqlparser_derive"
 version = "0.2.2"
@@ -5104,13 +5402,13 @@ dependencies = [
 name = "vortex-array"
 version = "0.1.0"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-schema",
+ "arrow-array 51.0.0",
+ "arrow-buffer 51.0.0",
+ "arrow-schema 51.0.0",
  "build-vortex",
  "criterion",
  "enum-iterator",
- "flatbuffers",
+ "flatbuffers 23.5.26",
  "flexbuffers",
  "futures-util",
  "getrandom",
@@ -5137,7 +5435,7 @@ dependencies = [
 name = "vortex-buffer"
 version = "0.1.0"
 dependencies = [
- "arrow-buffer",
+ "arrow-buffer 51.0.0",
  "bytes",
  "flexbuffers",
 ]
@@ -5177,7 +5475,7 @@ name = "vortex-dtype"
 version = "0.1.0"
 dependencies = [
  "build-vortex",
- "flatbuffers",
+ "flatbuffers 23.5.26",
  "half",
  "itertools 0.12.1",
  "num-traits",
@@ -5193,8 +5491,8 @@ dependencies = [
 name = "vortex-error"
 version = "0.1.0"
 dependencies = [
- "arrow-schema",
- "flatbuffers",
+ "arrow-schema 51.0.0",
+ "flatbuffers 23.5.26",
  "flexbuffers",
  "parquet",
  "thiserror",
@@ -5205,6 +5503,8 @@ dependencies = [
 name = "vortex-expr"
 version = "0.1.0"
 dependencies = [
+ "datafusion-common 39.0.0",
+ "datafusion-expr 39.0.0",
  "serde",
  "vortex-dtype",
  "vortex-error",
@@ -5233,23 +5533,23 @@ dependencies = [
 name = "vortex-flatbuffers"
 version = "0.1.0"
 dependencies = [
- "flatbuffers",
+ "flatbuffers 23.5.26",
 ]
 
 [[package]]
 name = "vortex-ipc"
 version = "0.1.0"
 dependencies = [
- "arrow",
- "arrow-array",
- "arrow-ipc",
- "arrow-schema",
- "arrow-select",
+ "arrow 51.0.0",
+ "arrow-array 51.0.0",
+ "arrow-ipc 51.0.0",
+ "arrow-schema 51.0.0",
+ "arrow-select 51.0.0",
  "build-vortex",
  "bytes",
  "criterion",
  "derive_builder",
- "flatbuffers",
+ "flatbuffers 23.5.26",
  "futures-executor",
  "futures-util",
  "itertools 0.12.1",
@@ -5285,7 +5585,7 @@ dependencies = [
 name = "vortex-roaring"
 version = "0.1.0"
 dependencies = [
- "arrow-buffer",
+ "arrow-buffer 51.0.0",
  "croaring",
  "log",
  "num-traits",
@@ -5302,7 +5602,8 @@ name = "vortex-scalar"
 version = "0.1.0"
 dependencies = [
  "build-vortex",
- "flatbuffers",
+ "datafusion-common 39.0.0",
+ "flatbuffers 23.5.26",
  "flexbuffers",
  "itertools 0.12.1",
  "num-traits",
diff --git a/Cargo.toml b/Cargo.toml
index 080a89a81a..30be15cffb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -56,6 +56,8 @@ cargo_metadata = "0.18.1"
 criterion = { version = "0.5.1", features = ["html_reports"] }
 croaring = "1.0.1"
 csv = "1.3.0"
+datafusion-common = "39.0.0"
+datafusion-expr = "39.0.0"
 derive_builder = "0.20.0"
 divan = "0.1.14"
 duckdb = { version = "0.10.1", features = ["bundled"] }
diff --git a/vortex-array/benches/compare.rs b/vortex-array/benches/compare.rs
index 2a00411d9e..fe2135a2e7 100644
--- a/vortex-array/benches/compare.rs
+++ b/vortex-array/benches/compare.rs
@@ -27,9 +27,7 @@ fn filter_bool_indices(c: &mut Criterion) {
 
     group.bench_function("compare_bool", |b| {
         b.iter(|| {
-            let indices =
-                vortex::compute::compare::compare(&arr, &arr2, Operator::GreaterThanOrEqualTo)
-                    .unwrap();
+            let indices = vortex::compute::compare::compare(&arr, &arr2, Operator::Gte).unwrap();
             black_box(indices);
             Ok::<(), VortexError>(())
         });
@@ -53,9 +51,7 @@ fn filter_indices(c: &mut Criterion) {
 
     group.bench_function("compare_int", |b| {
         b.iter(|| {
-            let indices =
-                vortex::compute::compare::compare(&arr, &arr2, Operator::GreaterThanOrEqualTo)
-                    .unwrap();
+            let indices = vortex::compute::compare::compare(&arr, &arr2, Operator::Gte).unwrap();
             black_box(indices);
             Ok::<(), VortexError>(())
         });
diff --git a/vortex-array/src/array/bool/compute/compare.rs b/vortex-array/src/array/bool/compute/compare.rs
index cc9969b1d4..fc26d3d8c7 100644
--- a/vortex-array/src/array/bool/compute/compare.rs
+++ b/vortex-array/src/array/bool/compute/compare.rs
@@ -13,13 +13,13 @@ impl CompareFn for BoolArray {
         let lhs = self.boolean_buffer();
         let rhs = flattened.boolean_buffer();
         let result_buf = match op {
-            Operator::EqualTo => lhs.bitxor(&rhs).not(),
-            Operator::NotEqualTo => lhs.bitxor(&rhs),
+            Operator::Eq => lhs.bitxor(&rhs).not(),
+            Operator::NotEq => lhs.bitxor(&rhs),
 
-            Operator::GreaterThan => lhs.bitand(&rhs.not()),
-            Operator::GreaterThanOrEqualTo => lhs.bitor(&rhs.not()),
-            Operator::LessThan => lhs.not().bitand(&rhs),
-            Operator::LessThanOrEqualTo => lhs.not().bitor(&rhs),
+            Operator::Gt => lhs.bitand(&rhs.not()),
+            Operator::Gte => lhs.bitor(&rhs.not()),
+            Operator::Lt => lhs.not().bitand(&rhs),
+            Operator::Lte => lhs.not().bitor(&rhs),
         };
         Ok(BoolArray::from(
             self.validity()
@@ -58,10 +58,10 @@ mod test {
         )
         .into_array();
 
-        let matches = compare(&arr, &arr, Operator::EqualTo)?.flatten_bool()?;
+        let matches = compare(&arr, &arr, Operator::Eq)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [1u64, 2, 3, 4]);
 
-        let matches = compare(&arr, &arr, Operator::NotEqualTo)?.flatten_bool()?;
+        let matches = compare(&arr, &arr, Operator::NotEq)?.flatten_bool()?;
         let empty: [u64; 0] = [];
         assert_eq!(to_int_indices(matches), empty);
 
@@ -71,16 +71,16 @@ mod test {
         )
         .into_array();
 
-        let matches = compare(&arr, &other, Operator::LessThanOrEqualTo)?.flatten_bool()?;
+        let matches = compare(&arr, &other, Operator::Lte)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [2u64, 3, 4]);
 
-        let matches = compare(&arr, &other, Operator::LessThan)?.flatten_bool()?;
+        let matches = compare(&arr, &other, Operator::Lt)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [4u64]);
 
-        let matches = compare(&other, &arr, Operator::GreaterThanOrEqualTo)?.flatten_bool()?;
+        let matches = compare(&other, &arr, Operator::Gte)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [2u64, 3, 4]);
 
-        let matches = compare(&other, &arr, Operator::GreaterThan)?.flatten_bool()?;
+        let matches = compare(&other, &arr, Operator::Gt)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [4u64]);
         Ok(())
     }
diff --git a/vortex-array/src/array/primitive/compute/compare.rs b/vortex-array/src/array/primitive/compute/compare.rs
index 4d4ce18238..53b0a70740 100644
--- a/vortex-array/src/array/primitive/compute/compare.rs
+++ b/vortex-array/src/array/primitive/compute/compare.rs
@@ -78,10 +78,10 @@ mod test {
         ])
         .into_array();
 
-        let matches = compare(&arr, &arr, Operator::EqualTo)?.flatten_bool()?;
+        let matches = compare(&arr, &arr, Operator::Eq)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [0u64, 1, 2, 3, 5, 6, 7, 8, 10]);
 
-        let matches = compare(&arr, &arr, Operator::NotEqualTo)?.flatten_bool()?;
+        let matches = compare(&arr, &arr, Operator::NotEq)?.flatten_bool()?;
         let empty: [u64; 0] = [];
         assert_eq!(to_int_indices(matches), empty);
 
@@ -101,16 +101,16 @@ mod test {
         ])
         .into_array();
 
-        let matches = compare(&arr, &other, Operator::LessThanOrEqualTo)?.flatten_bool()?;
+        let matches = compare(&arr, &other, Operator::Lte)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [0u64, 1, 2, 3, 5, 6, 7, 8, 10]);
 
-        let matches = compare(&arr, &other, Operator::LessThan)?.flatten_bool()?;
+        let matches = compare(&arr, &other, Operator::Lt)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [5u64, 6, 7, 8, 10]);
 
-        let matches = compare(&other, &arr, Operator::GreaterThanOrEqualTo)?.flatten_bool()?;
+        let matches = compare(&other, &arr, Operator::Gte)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [0u64, 1, 2, 3, 5, 6, 7, 8, 10]);
 
-        let matches = compare(&other, &arr, Operator::GreaterThan)?.flatten_bool()?;
+        let matches = compare(&other, &arr, Operator::Gt)?.flatten_bool()?;
         assert_eq!(to_int_indices(matches), [5u64, 6, 7, 8, 10]);
         Ok(())
     }
diff --git a/vortex-dtype/src/field_paths.rs b/vortex-dtype/src/field_paths.rs
index 00f2052fe7..07368a0952 100644
--- a/vortex-dtype/src/field_paths.rs
+++ b/vortex-dtype/src/field_paths.rs
@@ -24,6 +24,10 @@ impl FieldPath {
             Some(Self::builder().join_all(new_field_names).build())
         }
     }
+
+    pub fn parts(&self) -> &[FieldIdentifier] {
+        &self.field_names
+    }
 }
 
 #[derive(Clone, Debug, PartialEq)]
diff --git a/vortex-expr/Cargo.toml b/vortex-expr/Cargo.toml
index 874b0cb563..b59f87119a 100644
--- a/vortex-expr/Cargo.toml
+++ b/vortex-expr/Cargo.toml
@@ -15,14 +15,16 @@ rust-version = { workspace = true }
 workspace = true
 
 [dependencies]
+datafusion-common = { workspace = true, optional = true }
+datafusion-expr = { workspace = true, optional = true }
 vortex-dtype = { path = "../vortex-dtype" }
 vortex-error = { path = "../vortex-error" }
 vortex-scalar = { path = "../vortex-scalar" }
 serde = { workspace = true, optional = true, features = ["derive"] }
 
-
 [dev-dependencies]
 
-
 [features]
+default = []
+datafusion = ["dep:datafusion-common", "dep:datafusion-expr", "vortex-scalar/datafusion"]
 serde = ["dep:serde", "vortex-dtype/serde", "vortex-scalar/serde"]
\ No newline at end of file
diff --git a/vortex-expr/src/datafusion.rs b/vortex-expr/src/datafusion.rs
new file mode 100644
index 0000000000..2d3776665a
--- /dev/null
+++ b/vortex-expr/src/datafusion.rs
@@ -0,0 +1,63 @@
+#![cfg(feature = "datafusion")]
+use datafusion_common::Column;
+use datafusion_expr::{BinaryExpr, Expr};
+use vortex_dtype::field_paths::{FieldIdentifier, FieldPath};
+use vortex_scalar::Scalar;
+
+use crate::expressions::{Predicate, Value};
+use crate::operators::Operator;
+
+impl From<Predicate> for Expr {
+    fn from(value: Predicate) -> Self {
+        Expr::BinaryExpr(BinaryExpr::new(
+            Box::new(FieldPathWrapper(value.left).into()),
+            value.op.into(),
+            Box::new(value.right.into()),
+        ))
+    }
+}
+
+impl From<Operator> for datafusion_expr::Operator {
+    fn from(value: Operator) -> Self {
+        match value {
+            Operator::Eq => datafusion_expr::Operator::Eq,
+            Operator::NotEq => datafusion_expr::Operator::NotEq,
+            Operator::Gt => datafusion_expr::Operator::Gt,
+            Operator::Gte => datafusion_expr::Operator::GtEq,
+            Operator::Lt => datafusion_expr::Operator::Lt,
+            Operator::Lte => datafusion_expr::Operator::LtEq,
+        }
+    }
+}
+
+impl From<Value> for Expr {
+    fn from(value: Value) -> Self {
+        match value {
+            Value::Field(field_path) => FieldPathWrapper(field_path).into(),
+            Value::Literal(literal) => ScalarWrapper(literal).into(),
+        }
+    }
+}
+
+struct FieldPathWrapper(FieldPath);
+impl From<FieldPathWrapper> for Expr {
+    fn from(value: FieldPathWrapper) -> Self {
+        let mut field = String::new();
+        for part in value.0.parts() {
+            match part {
+                // TODO(ngates): escape quotes?
+                FieldIdentifier::Name(identifier) => field.push_str(&format!("\"{}\"", identifier)),
+                FieldIdentifier::ListIndex(idx) => field.push_str(&format!("[{}]", idx)),
+            }
+        }
+
+        Expr::Column(Column::from(field))
+    }
+}
+
+struct ScalarWrapper(Scalar);
+impl From<ScalarWrapper> for Expr {
+    fn from(value: ScalarWrapper) -> Self {
+        Expr::Literal(value.0.into())
+    }
+}
diff --git a/vortex-expr/src/display.rs b/vortex-expr/src/display.rs
index cc37916207..a3d61323ea 100644
--- a/vortex-expr/src/display.rs
+++ b/vortex-expr/src/display.rs
@@ -42,12 +42,12 @@ impl Display for Value {
 impl Display for Operator {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
         let display = match &self {
-            Operator::EqualTo => "=",
-            Operator::NotEqualTo => "!=",
-            Operator::GreaterThan => ">",
-            Operator::GreaterThanOrEqualTo => ">=",
-            Operator::LessThan => "<",
-            Operator::LessThanOrEqualTo => "<=",
+            Operator::Eq => "=",
+            Operator::NotEq => "!=",
+            Operator::Gt => ">",
+            Operator::Gte => ">=",
+            Operator::Lt => "<",
+            Operator::Lte => "<=",
         };
         write!(f, "{display}")
     }
diff --git a/vortex-expr/src/expressions.rs b/vortex-expr/src/expressions.rs
index a03da6f5f2..c6eac21ceb 100644
--- a/vortex-expr/src/expressions.rs
+++ b/vortex-expr/src/expressions.rs
@@ -50,7 +50,7 @@ impl Value {
     pub fn eq(self, field: impl Into<FieldPath>) -> Predicate {
         Predicate {
             left: field.into(),
-            op: Operator::EqualTo,
+            op: Operator::Eq,
             right: self,
         }
     }
@@ -58,7 +58,7 @@ impl Value {
     pub fn not_eq(self, field: impl Into<FieldPath>) -> Predicate {
         Predicate {
             left: field.into(),
-            op: Operator::NotEqualTo.inverse(),
+            op: Operator::NotEq.inverse(),
             right: self,
         }
     }
@@ -66,7 +66,7 @@ impl Value {
     pub fn gt(self, field: impl Into<FieldPath>) -> Predicate {
         Predicate {
             left: field.into(),
-            op: Operator::GreaterThan.inverse(),
+            op: Operator::Gt.inverse(),
             right: self,
         }
     }
@@ -74,7 +74,7 @@ impl Value {
     pub fn gte(self, field: impl Into<FieldPath>) -> Predicate {
         Predicate {
             left: field.into(),
-            op: Operator::GreaterThanOrEqualTo.inverse(),
+            op: Operator::Gte.inverse(),
             right: self,
         }
     }
@@ -82,7 +82,7 @@ impl Value {
     pub fn lt(self, field: impl Into<FieldPath>) -> Predicate {
         Predicate {
             left: field.into(),
-            op: Operator::LessThan.inverse(),
+            op: Operator::Lt.inverse(),
             right: self,
         }
     }
@@ -90,7 +90,7 @@ impl Value {
     pub fn lte(self, field: impl Into<FieldPath>) -> Predicate {
         Predicate {
             left: field.into(),
-            op: Operator::LessThanOrEqualTo.inverse(),
+            op: Operator::Lte.inverse(),
             right: self,
         }
     }
@@ -109,7 +109,7 @@ mod test {
         let field = field("id");
         let expr = Predicate {
             left: field,
-            op: Operator::EqualTo,
+            op: Operator::Eq,
             right: value,
         };
         assert_eq!(format!("{}", expr), "($id = 1)");
diff --git a/vortex-expr/src/field_paths.rs b/vortex-expr/src/field_paths.rs
index 208abb92d4..b1e0a9c7aa 100644
--- a/vortex-expr/src/field_paths.rs
+++ b/vortex-expr/src/field_paths.rs
@@ -17,7 +17,7 @@ impl FieldPathOperations for FieldPath {
     fn eq(self, other: Value) -> Predicate {
         Predicate {
             left: self,
-            op: Operator::EqualTo,
+            op: Operator::Eq,
             right: other,
         }
     }
@@ -25,7 +25,7 @@ impl FieldPathOperations for FieldPath {
     fn not_eq(self, other: Value) -> Predicate {
         Predicate {
             left: self,
-            op: Operator::NotEqualTo,
+            op: Operator::NotEq,
             right: other,
         }
     }
@@ -33,7 +33,7 @@ impl FieldPathOperations for FieldPath {
     fn gt(self, other: Value) -> Predicate {
         Predicate {
             left: self,
-            op: Operator::GreaterThan,
+            op: Operator::Gt,
             right: other,
         }
     }
@@ -41,7 +41,7 @@ impl FieldPathOperations for FieldPath {
     fn gte(self, other: Value) -> Predicate {
         Predicate {
             left: self,
-            op: Operator::GreaterThanOrEqualTo,
+            op: Operator::Gte,
             right: other,
         }
     }
@@ -49,7 +49,7 @@ impl FieldPathOperations for FieldPath {
     fn lt(self, other: Value) -> Predicate {
         Predicate {
             left: self,
-            op: Operator::LessThan,
+            op: Operator::Lt,
             right: other,
         }
     }
@@ -57,7 +57,7 @@ impl FieldPathOperations for FieldPath {
     fn lte(self, other: Value) -> Predicate {
         Predicate {
             left: self,
-            op: Operator::LessThanOrEqualTo,
+            op: Operator::Lte,
             right: other,
         }
     }
diff --git a/vortex-expr/src/lib.rs b/vortex-expr/src/lib.rs
index 886afbb297..d512ed175b 100644
--- a/vortex-expr/src/lib.rs
+++ b/vortex-expr/src/lib.rs
@@ -1,6 +1,7 @@
 #![feature(iter_intersperse)]
 extern crate core;
 
+mod datafusion;
 mod display;
 pub mod expressions;
 pub mod field_paths;
diff --git a/vortex-expr/src/operators.rs b/vortex-expr/src/operators.rs
index b6c6c289af..12b9166c4c 100644
--- a/vortex-expr/src/operators.rs
+++ b/vortex-expr/src/operators.rs
@@ -8,12 +8,12 @@ use crate::expressions::Predicate;
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub enum Operator {
     // comparison
-    EqualTo,
-    NotEqualTo,
-    GreaterThan,
-    GreaterThanOrEqualTo,
-    LessThan,
-    LessThanOrEqualTo,
+    Eq,
+    NotEq,
+    Gt,
+    Gte,
+    Lt,
+    Lte,
 }
 
 impl ops::Not for Predicate {
@@ -21,12 +21,12 @@ impl ops::Not for Predicate {
 
     fn not(self) -> Self::Output {
         let inverse_op = match self.op {
-            Operator::EqualTo => Operator::NotEqualTo,
-            Operator::NotEqualTo => Operator::EqualTo,
-            Operator::GreaterThan => Operator::LessThanOrEqualTo,
-            Operator::GreaterThanOrEqualTo => Operator::LessThan,
-            Operator::LessThan => Operator::GreaterThanOrEqualTo,
-            Operator::LessThanOrEqualTo => Operator::GreaterThan,
+            Operator::Eq => Operator::NotEq,
+            Operator::NotEq => Operator::Eq,
+            Operator::Gt => Operator::Lte,
+            Operator::Gte => Operator::Lt,
+            Operator::Lt => Operator::Gte,
+            Operator::Lte => Operator::Gt,
         };
         Predicate {
             left: self.left,
@@ -39,23 +39,23 @@ impl ops::Not for Predicate {
 impl Operator {
     pub fn inverse(self) -> Self {
         match self {
-            Operator::EqualTo => Operator::NotEqualTo,
-            Operator::NotEqualTo => Operator::EqualTo,
-            Operator::GreaterThan => Operator::LessThanOrEqualTo,
-            Operator::GreaterThanOrEqualTo => Operator::LessThan,
-            Operator::LessThan => Operator::GreaterThanOrEqualTo,
-            Operator::LessThanOrEqualTo => Operator::GreaterThan,
+            Operator::Eq => Operator::NotEq,
+            Operator::NotEq => Operator::Eq,
+            Operator::Gt => Operator::Lte,
+            Operator::Gte => Operator::Lt,
+            Operator::Lt => Operator::Gte,
+            Operator::Lte => Operator::Gt,
         }
     }
 
     pub fn to_predicate<T: NativePType>(&self) -> fn(&T, &T) -> bool {
         match self {
-            Operator::EqualTo => PartialEq::eq,
-            Operator::NotEqualTo => PartialEq::ne,
-            Operator::GreaterThan => PartialOrd::gt,
-            Operator::GreaterThanOrEqualTo => PartialOrd::ge,
-            Operator::LessThan => PartialOrd::lt,
-            Operator::LessThanOrEqualTo => PartialOrd::le,
+            Operator::Eq => PartialEq::eq,
+            Operator::NotEq => PartialEq::ne,
+            Operator::Gt => PartialOrd::gt,
+            Operator::Gte => PartialOrd::ge,
+            Operator::Lt => PartialOrd::lt,
+            Operator::Lte => PartialOrd::le,
         }
     }
 }
diff --git a/vortex-scalar/Cargo.toml b/vortex-scalar/Cargo.toml
index e5f8f06172..b9902dd0b6 100644
--- a/vortex-scalar/Cargo.toml
+++ b/vortex-scalar/Cargo.toml
@@ -12,6 +12,7 @@ edition = { workspace = true }
 rust-version = { workspace = true }
 
 [dependencies]
+datafusion-common = { workspace = true, optional = true }
 flatbuffers = { workspace = true, optional = true }
 flexbuffers = { workspace = true, optional = true }
 itertools = { workspace = true }
@@ -34,6 +35,7 @@ workspace = true
 [features]
 # Uncomment for improved IntelliJ support
 # default = ["flatbuffers", "proto", "serde"]
+datafusion = ["dep:datafusion-common"]
 flatbuffers = [
     "dep:flatbuffers",
     "dep:flexbuffers",
diff --git a/vortex-scalar/src/datafusion.rs b/vortex-scalar/src/datafusion.rs
new file mode 100644
index 0000000000..594a6c3c3d
--- /dev/null
+++ b/vortex-scalar/src/datafusion.rs
@@ -0,0 +1,68 @@
+#![cfg(feature = "datafusion")]
+use datafusion_common::ScalarValue;
+use vortex_dtype::{DType, PType};
+
+use crate::{PValue, Scalar};
+
+impl From<Scalar> for ScalarValue {
+    fn from(value: Scalar) -> Self {
+        match value.dtype {
+            DType::Null => ScalarValue::Null,
+            DType::Bool(_) => ScalarValue::Boolean(value.value.as_bool().expect("should be bool")),
+            DType::Primitive(ptype, _) => {
+                let pvalue = value.value.as_pvalue().expect("should be pvalue");
+                match pvalue {
+                    None => match ptype {
+                        PType::U8 => ScalarValue::UInt8(None),
+                        PType::U16 => ScalarValue::UInt16(None),
+                        PType::U32 => ScalarValue::UInt32(None),
+                        PType::U64 => ScalarValue::UInt64(None),
+                        PType::I8 => ScalarValue::Int8(None),
+                        PType::I16 => ScalarValue::Int16(None),
+                        PType::I32 => ScalarValue::Int32(None),
+                        PType::I64 => ScalarValue::Int64(None),
+                        PType::F16 => ScalarValue::Float16(None),
+                        PType::F32 => ScalarValue::Float32(None),
+                        PType::F64 => ScalarValue::Float64(None),
+                    },
+                    Some(pvalue) => match pvalue {
+                        PValue::U8(v) => ScalarValue::UInt8(Some(v)),
+                        PValue::U16(v) => ScalarValue::UInt16(Some(v)),
+                        PValue::U32(v) => ScalarValue::UInt32(Some(v)),
+                        PValue::U64(v) => ScalarValue::UInt64(Some(v)),
+                        PValue::I8(v) => ScalarValue::Int8(Some(v)),
+                        PValue::I16(v) => ScalarValue::Int16(Some(v)),
+                        PValue::I32(v) => ScalarValue::Int32(Some(v)),
+                        PValue::I64(v) => ScalarValue::Int64(Some(v)),
+                        PValue::F16(v) => ScalarValue::Float16(Some(v)),
+                        PValue::F32(v) => ScalarValue::Float32(Some(v)),
+                        PValue::F64(v) => ScalarValue::Float64(Some(v)),
+                    },
+                }
+            }
+            DType::Utf8(_) => ScalarValue::Utf8(
+                value
+                    .value
+                    .as_buffer_string()
+                    .expect("should be buffer string")
+                    .map(|b| b.as_str().to_string()),
+            ),
+            DType::Binary(_) => ScalarValue::Binary(
+                value
+                    .value
+                    .as_buffer()
+                    .expect("should be buffer")
+                    .map(|b| b.as_slice().to_vec()),
+            ),
+            DType::Struct(..) => {
+                todo!("struct scalar conversion")
+            }
+            DType::List(..) => {
+                todo!("list scalar conversion")
+            }
+            DType::Extension(..) => {
+                todo!("extension scalar conversion")
+            }
+        }
+    }
+}
diff --git a/vortex-scalar/src/lib.rs b/vortex-scalar/src/lib.rs
index 77b5446a47..e628c098de 100644
--- a/vortex-scalar/src/lib.rs
+++ b/vortex-scalar/src/lib.rs
@@ -4,6 +4,7 @@ use vortex_dtype::DType;
 
 mod binary;
 mod bool;
+mod datafusion;
 mod display;
 mod extension;
 mod list;