Skip to content

Commit

Permalink
Address issues in decoding data from the database (#211)
Browse files Browse the repository at this point in the history 
* Retool PostgresDataDecoder not to violate Coding usage invariants, to throw more consistent errors, and to forward its decoding logic consistently.

* Fix tests to clean up connections properly on error, to respect environment overrides for test config, to handle errors sensibly, and to not open lots of pointless extra connections.

* Heavily updates the CI, including fixing some brokenness caused by a GH Actions bug.
  • Loading branch information
@gwynne
gwynne authored Sep 30, 2021
1 parent 6f35f30 commit a050d33
Show file tree
Hide file tree
Showing 4 changed files with 233 additions and 229 deletions.
163 changes: 108 additions & 55 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
@@ -1,63 +1,116 @@
name: test
on:
- pull_request
pull_request:
push: { branches: [ main ] }

jobs:
postgres-kit_xenial:
container:
image: vapor/swift:5.2-xenial
services:
psql:
image: postgres
ports:
- 5432:5432
env:
POSTGRES_USER: vapor_username
POSTGRES_DB: vapor_database
POSTGRES_PASSWORD: vapor_password
linux-plus-dependents:
strategy:
fail-fast: false
matrix:
dbimage:
- postgres:13
- postgres:12
- postgres:11
dbauth:
- trust
- md5
- scram-sha-256
swiftver:
- 'swift:5.2'
- 'swift:5.5'
- 'swiftlang/swift:nightly-main'
swiftos:
- focal
- amazonlinux2
dependent:
- fluent-postgres-driver
container: ${{ format('{0}-{1}', matrix.swiftver, matrix.swiftos) }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- run: swift test --enable-test-discovery --sanitize=thread
postgres-kit_bionic:
container:
image: vapor/swift:5.2-bionic
env:
LOG_LEVEL: debug
POSTGRES_HOSTNAME: 'psql-a'
POSTGRES_HOSTNAME_A: 'psql-a'
POSTGRES_HOSTNAME_B: 'psql-b'
services:
psql:
image: postgres
ports:
- 5432:5432
psql-a:
image: ${{ matrix.dbimage }}
env:
POSTGRES_USER: vapor_username
POSTGRES_DB: vapor_database
POSTGRES_PASSWORD: vapor_password
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- run: swift test --enable-test-discovery --sanitize=thread
fluent-postgres-driver:
container:
image: vapor/swift:5.2
services:
postgres-a:
image: postgres
env:
POSTGRES_USER: vapor_username
POSTGRES_DB: vapor_database
POSTGRES_PASSWORD: vapor_password
postgres-b:
image: postgres
POSTGRES_USER: 'vapor_username'
POSTGRES_DB: 'vapor_database'
POSTGRES_PASSWORD: 'vapor_password'
POSTGRES_HOST_AUTH_METHOD: ${{ matrix.dbauth }}
POSTGRES_INITDB_ARGS: --auth-host=${{ matrix.dbauth }}
psql-b:
image: ${{ matrix.dbimage }}
env:
POSTGRES_USER: vapor_username
POSTGRES_DB: vapor_database
POSTGRES_PASSWORD: vapor_password
runs-on: ubuntu-latest
POSTGRES_USER: 'vapor_username'
POSTGRES_DB: 'vapor_database'
POSTGRES_PASSWORD: 'vapor_password'
POSTGRES_HOST_AUTH_METHOD: ${{ matrix.dbauth }}
POSTGRES_INITDB_ARGS: --auth-host=${{ matrix.dbauth }}
steps:
- name: Workaround SPM incompatibility with old Git on CentOS 7
if: ${{ contains(matrix.swiftos, 'centos7') }}
run: |
yum install -y make libcurl-devel
git clone https://github.com/git/git -bv2.28.0 --depth 1 && cd git
make prefix=/usr -j all install NO_OPENSSL=1 NO_EXPAT=1 NO_TCLTK=1 NO_GETTEXT=1 NO_PERL=1
- name: Check out package
uses: actions/checkout@v2
with:
path: package
- name: Check out dependent
uses: actions/checkout@v2
with:
repository: vapor/${{ matrix.dependent }}
path: dependent
- name: Use local package
run: swift package edit postgres-kit --path ../package
working-directory: dependent
- name: Run local tests with Thread Sanitizer
run: swift test --enable-test-discovery --sanitize=thread
working-directory: package
- name: Run dependent tests with Thread Sanitizer
run: swift test --enable-test-discovery --sanitize=thread
working-directory: dependent

macos-plus-dependents:
strategy:
fail-fast: false
matrix:
xcode:
- latest-stable
- latest
dbauth:
- trust
- md5
- scram-sha-256
formula:
- postgresql@11
- postgresql@12
- postgresql@13
dependent:
- fluent-postgres-driver
runs-on: macos-latest
env:
LOG_LEVEL: debug
POSTGRES_HOSTNAME: 127.0.0.1
POSTGRES_DB: postgres
POSTGRES_HOST_AUTH_METHOD: ${{ matrix.dbauth }}
steps:
- run: git clone -b main https://github.com/vapor/fluent-postgres-driver.git
working-directory: ./
- run: swift package edit postgres-kit --revision ${{ github.sha }}
working-directory: ./fluent-postgres-driver
- run: swift test --enable-test-discovery --sanitize=thread
working-directory: ./fluent-postgres-driver
env:
POSTGRES_HOSTNAME_A: postgres-a
POSTGRES_HOSTNAME_B: postgres-b
- name: Select latest available Xcode
uses: maxim-lobanov/setup-xcode@v1
with:
xcode-version: ${{ matrix.xcode }}
- name: Install Postgres, setup DB and auth, and wait for server start
run: |
export PATH="/usr/local/opt/${{ matrix.formula }}/bin:$PATH" PGDATA=/tmp/vapor-postgres-test
brew install ${{ matrix.formula }}
initdb --locale=C --auth-host ${{ matrix.dbauth }} -U vapor_username --pwfile=<(echo vapor_password)
pg_ctl start --wait
timeout-minutes: 2
- name: Checkout code
uses: actions/checkout@v2
- name: Run local tests with Thread Sanitizer
run: swift test --enable-test-discovery --sanitize=thread
177 changes: 71 additions & 106 deletions Sources/PostgresKit/PostgresDataDecoder.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,29 @@ public final class PostgresDataDecoder {
public func decode<T>(_ type: T.Type, from data: PostgresData) throws -> T
where T: Decodable
{
// If `T` can be converted directly, just do so.
if let convertible = T.self as? PostgresDataConvertible.Type {
guard let value = convertible.init(postgresData: data) else {
throw DecodingError.typeMismatch(T.self, DecodingError.Context.init(
throw DecodingError.typeMismatch(T.self, .init(
codingPath: [],
debugDescription: "Could not convert to \(T.self): \(data)"
debugDescription: "Could not convert PostgreSQL data to \(T.self): \(data)"
))
}
return value as! T
} else {
return try T.init(from: _Decoder(data: data, json: self.json))
// Probably a Postgres array, JSON array/object, or enum type not using @Enum. See if it can be "unwrapped"
// as a single-value decoding container, since this is much faster than attempting a JSON decode, or as an
// array in the Postgres-native sense; this will handle "box" types such as `RawRepresentable` enums while
// still allowing falling back to JSON.
do {
return try T.init(from: GiftBoxUnwrapDecoder(decoder: self, data: data))
} catch DecodingError.dataCorrupted {
// Couldn't unwrap it either. Fall back to attempting a JSON decode.
guard let jsonData = data.jsonb ?? data.json else {
throw Error.unexpectedDataType(data.type, expected: "jsonb/json")
}
return try self.json.decode(T.self, from: jsonData)
}
}
}

Expand All @@ -39,125 +52,77 @@ public final class PostgresDataDecoder {
}
}

final class _Decoder: Decoder {
var codingPath: [CodingKey] {
return []
}

var userInfo: [CodingUserInfoKey : Any] {
return [:]
}
private final class GiftBoxUnwrapDecoder: Decoder, SingleValueDecodingContainer {
var codingPath: [CodingKey] { [] }
var userInfo: [CodingUserInfoKey : Any] { [:] }

let dataDecoder: PostgresDataDecoder
let data: PostgresData
let json: PostgresJSONDecoder

init(data: PostgresData, json: PostgresJSONDecoder) {
init(decoder: PostgresDataDecoder, data: PostgresData) {
self.dataDecoder = decoder
self.data = data
self.json = json
}

func container<Key>(keyedBy type: Key.Type) throws -> KeyedDecodingContainer<Key> where Key: CodingKey {
throw DecodingError.dataCorruptedError(in: self, debugDescription: "Dictionary containers must be JSON-encoded")
}

func unkeyedContainer() throws -> UnkeyedDecodingContainer {
guard let data = self.data.array else {
throw Error.unexpectedDataType(self.data.type, expected: "array")
guard let array = self.data.array else {
throw DecodingError.dataCorruptedError(in: self, debugDescription: "Non-natively typed arrays must be JSON-encoded")
}
return _UnkeyedDecoder(data: data, json: self.json)
}

func container<Key>(
keyedBy type: Key.Type
) throws -> KeyedDecodingContainer<Key> where Key : CodingKey {
let data: Data
if let jsonb = self.data.jsonb {
data = jsonb
} else if let json = self.data.json {
data = json
} else {
throw Error.unexpectedDataType(self.data.type, expected: "json")
return ArrayContainer(data: array, dataDecoder: self.dataDecoder)
}

struct ArrayContainer: UnkeyedDecodingContainer {
let data: [PostgresData]
let dataDecoder: PostgresDataDecoder
var codingPath: [CodingKey] { [] }
var count: Int? { self.data.count }
var isAtEnd: Bool { self.currentIndex >= self.data.count }
var currentIndex: Int = 0

mutating func decodeNil() throws -> Bool {
// Do _not_ shorten this using `defer`, otherwise `currentIndex` is incorrectly incremented.
if self.data[self.currentIndex].value == nil {
self.currentIndex += 1
return true
}
return false
}

mutating func decode<T>(_ type: T.Type) throws -> T where T : Decodable {
// Do _not_ shorten this using `defer`, otherwise `currentIndex` is incorrectly incremented.
let result = try self.dataDecoder.decode(T.self, from: self.data[self.currentIndex])
self.currentIndex += 1
return result
}

mutating func nestedContainer<NewKey: CodingKey>(keyedBy _: NewKey.Type) throws -> KeyedDecodingContainer<NewKey> {
throw DecodingError.dataCorruptedError(in: self, debugDescription: "Data nesting is not supported")
}

mutating func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer {
throw DecodingError.dataCorruptedError(in: self, debugDescription: "Data nesting is not supported")
}

mutating func superDecoder() throws -> Decoder {
throw DecodingError.dataCorruptedError(in: self, debugDescription: "Data nesting is not supported")
}
return try self.json
.decode(DecoderUnwrapper.self, from: data)
.decoder.container(keyedBy: Key.self)
}

func singleValueContainer() throws -> SingleValueDecodingContainer {
_ValueDecoder(data: self.data, json: self.json)
}
}

struct _UnkeyedDecoder: UnkeyedDecodingContainer {
var count: Int? {
self.data.count
}

var isAtEnd: Bool {
self.currentIndex == self.data.count
}
var currentIndex: Int = 0

let data: [PostgresData]
let json: PostgresJSONDecoder
var codingPath: [CodingKey] {
[]
return self
}

mutating func decodeNil() throws -> Bool {
defer { self.currentIndex += 1 }
return self.data[self.currentIndex].value == nil
}

mutating func decode<T>(_ type: T.Type) throws -> T where T : Decodable {
defer { self.currentIndex += 1 }
let data = self.data[self.currentIndex]
return try PostgresDataDecoder(json: self.json).decode(T.self, from: data)
}

mutating func nestedContainer<NestedKey>(
keyedBy type: NestedKey.Type
) throws -> KeyedDecodingContainer<NestedKey>
where NestedKey : CodingKey
{
throw Error.nestingNotSupported
}

mutating func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer {
throw Error.nestingNotSupported
}

mutating func superDecoder() throws -> Decoder {
throw Error.nestingNotSupported
}
}

struct _ValueDecoder: SingleValueDecodingContainer {
let data: PostgresData
let json: PostgresJSONDecoder
var codingPath: [CodingKey] {
[]
}


func decodeNil() -> Bool {
return self.data.value == nil
self.data.value == nil
}

func decode<T>(_ type: T.Type) throws -> T where T : Decodable {
if let convertible = T.self as? PostgresDataConvertible.Type {
guard let value = convertible.init(postgresData: data) else {
throw DecodingError.typeMismatch(T.self, DecodingError.Context.init(
codingPath: [],
debugDescription: "Could not convert to \(T.self): \(data)"
))
}
return value as! T
} else {
return try T.init(from: _Decoder(data: self.data, json: self.json))
}
// Recurse back into the data decoder, don't repeat its logic here.
return try self.dataDecoder.decode(T.self, from: self.data)
}
}
}

struct DecoderUnwrapper: Decodable {
let decoder: Decoder
init(from decoder: Decoder) {
self.decoder = decoder
}
}
Loading

0 comments on commit a050d33

Please sign in to comment.