diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce03e0e..23145c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,63 +1,116 @@ name: test on: -- pull_request + pull_request: + push: { branches: [ main ] } + jobs: - postgres-kit_xenial: - container: - image: vapor/swift:5.2-xenial - services: - psql: - image: postgres - ports: - - 5432:5432 - env: - POSTGRES_USER: vapor_username - POSTGRES_DB: vapor_database - POSTGRES_PASSWORD: vapor_password + linux-plus-dependents: + strategy: + fail-fast: false + matrix: + dbimage: + - postgres:13 + - postgres:12 + - postgres:11 + dbauth: + - trust + - md5 + - scram-sha-256 + swiftver: + - 'swift:5.2' + - 'swift:5.5' + - 'swiftlang/swift:nightly-main' + swiftos: + - focal + - amazonlinux2 + dependent: + - fluent-postgres-driver + container: ${{ format('{0}-{1}', matrix.swiftver, matrix.swiftos) }} runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - run: swift test --enable-test-discovery --sanitize=thread - postgres-kit_bionic: - container: - image: vapor/swift:5.2-bionic + env: + LOG_LEVEL: debug + POSTGRES_HOSTNAME: 'psql-a' + POSTGRES_HOSTNAME_A: 'psql-a' + POSTGRES_HOSTNAME_B: 'psql-b' services: - psql: - image: postgres - ports: - - 5432:5432 + psql-a: + image: ${{ matrix.dbimage }} env: - POSTGRES_USER: vapor_username - POSTGRES_DB: vapor_database - POSTGRES_PASSWORD: vapor_password - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - run: swift test --enable-test-discovery --sanitize=thread - fluent-postgres-driver: - container: - image: vapor/swift:5.2 - services: - postgres-a: - image: postgres - env: - POSTGRES_USER: vapor_username - POSTGRES_DB: vapor_database - POSTGRES_PASSWORD: vapor_password - postgres-b: - image: postgres + POSTGRES_USER: 'vapor_username' + POSTGRES_DB: 'vapor_database' + POSTGRES_PASSWORD: 'vapor_password' + POSTGRES_HOST_AUTH_METHOD: ${{ matrix.dbauth }} + POSTGRES_INITDB_ARGS: --auth-host=${{ matrix.dbauth }} + psql-b: + image: ${{ matrix.dbimage }} env: - POSTGRES_USER: vapor_username - POSTGRES_DB: vapor_database - POSTGRES_PASSWORD: vapor_password - runs-on: ubuntu-latest + POSTGRES_USER: 'vapor_username' + POSTGRES_DB: 'vapor_database' + POSTGRES_PASSWORD: 'vapor_password' + POSTGRES_HOST_AUTH_METHOD: ${{ matrix.dbauth }} + POSTGRES_INITDB_ARGS: --auth-host=${{ matrix.dbauth }} + steps: + - name: Workaround SPM incompatibility with old Git on CentOS 7 + if: ${{ contains(matrix.swiftos, 'centos7') }} + run: | + yum install -y make libcurl-devel + git clone https://github.com/git/git -bv2.28.0 --depth 1 && cd git + make prefix=/usr -j all install NO_OPENSSL=1 NO_EXPAT=1 NO_TCLTK=1 NO_GETTEXT=1 NO_PERL=1 + - name: Check out package + uses: actions/checkout@v2 + with: + path: package + - name: Check out dependent + uses: actions/checkout@v2 + with: + repository: vapor/${{ matrix.dependent }} + path: dependent + - name: Use local package + run: swift package edit postgres-kit --path ../package + working-directory: dependent + - name: Run local tests with Thread Sanitizer + run: swift test --enable-test-discovery --sanitize=thread + working-directory: package + - name: Run dependent tests with Thread Sanitizer + run: swift test --enable-test-discovery --sanitize=thread + working-directory: dependent + + macos-plus-dependents: + strategy: + fail-fast: false + matrix: + xcode: + - latest-stable + - latest + dbauth: + - trust + - md5 + - scram-sha-256 + formula: + - postgresql@11 + - postgresql@12 + - postgresql@13 + dependent: + - fluent-postgres-driver + runs-on: macos-latest + env: + LOG_LEVEL: debug + POSTGRES_HOSTNAME: 127.0.0.1 + POSTGRES_DB: postgres + POSTGRES_HOST_AUTH_METHOD: ${{ matrix.dbauth }} steps: - - run: git clone -b main https://github.com/vapor/fluent-postgres-driver.git - working-directory: ./ - - run: swift package edit postgres-kit --revision ${{ github.sha }} - working-directory: ./fluent-postgres-driver - - run: swift test --enable-test-discovery --sanitize=thread - working-directory: ./fluent-postgres-driver - env: - POSTGRES_HOSTNAME_A: postgres-a - POSTGRES_HOSTNAME_B: postgres-b + - name: Select latest available Xcode + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: ${{ matrix.xcode }} + - name: Install Postgres, setup DB and auth, and wait for server start + run: | + export PATH="/usr/local/opt/${{ matrix.formula }}/bin:$PATH" PGDATA=/tmp/vapor-postgres-test + brew install ${{ matrix.formula }} + initdb --locale=C --auth-host ${{ matrix.dbauth }} -U vapor_username --pwfile=<(echo vapor_password) + pg_ctl start --wait + timeout-minutes: 2 + - name: Checkout code + uses: actions/checkout@v2 + - name: Run local tests with Thread Sanitizer + run: swift test --enable-test-discovery --sanitize=thread diff --git a/Sources/PostgresKit/PostgresDataDecoder.swift b/Sources/PostgresKit/PostgresDataDecoder.swift index 7c736e7..a4b238d 100644 --- a/Sources/PostgresKit/PostgresDataDecoder.swift +++ b/Sources/PostgresKit/PostgresDataDecoder.swift @@ -12,16 +12,29 @@ public final class PostgresDataDecoder { public func decode(_ type: T.Type, from data: PostgresData) throws -> T where T: Decodable { + // If `T` can be converted directly, just do so. if let convertible = T.self as? PostgresDataConvertible.Type { guard let value = convertible.init(postgresData: data) else { - throw DecodingError.typeMismatch(T.self, DecodingError.Context.init( + throw DecodingError.typeMismatch(T.self, .init( codingPath: [], - debugDescription: "Could not convert to \(T.self): \(data)" + debugDescription: "Could not convert PostgreSQL data to \(T.self): \(data)" )) } return value as! T } else { - return try T.init(from: _Decoder(data: data, json: self.json)) + // Probably a Postgres array, JSON array/object, or enum type not using @Enum. See if it can be "unwrapped" + // as a single-value decoding container, since this is much faster than attempting a JSON decode, or as an + // array in the Postgres-native sense; this will handle "box" types such as `RawRepresentable` enums while + // still allowing falling back to JSON. + do { + return try T.init(from: GiftBoxUnwrapDecoder(decoder: self, data: data)) + } catch DecodingError.dataCorrupted { + // Couldn't unwrap it either. Fall back to attempting a JSON decode. + guard let jsonData = data.jsonb ?? data.json else { + throw Error.unexpectedDataType(data.type, expected: "jsonb/json") + } + return try self.json.decode(T.self, from: jsonData) + } } } @@ -39,125 +52,77 @@ public final class PostgresDataDecoder { } } - final class _Decoder: Decoder { - var codingPath: [CodingKey] { - return [] - } - - var userInfo: [CodingUserInfoKey : Any] { - return [:] - } + private final class GiftBoxUnwrapDecoder: Decoder, SingleValueDecodingContainer { + var codingPath: [CodingKey] { [] } + var userInfo: [CodingUserInfoKey : Any] { [:] } + let dataDecoder: PostgresDataDecoder let data: PostgresData - let json: PostgresJSONDecoder - init(data: PostgresData, json: PostgresJSONDecoder) { + init(decoder: PostgresDataDecoder, data: PostgresData) { + self.dataDecoder = decoder self.data = data - self.json = json + } + + func container(keyedBy type: Key.Type) throws -> KeyedDecodingContainer where Key: CodingKey { + throw DecodingError.dataCorruptedError(in: self, debugDescription: "Dictionary containers must be JSON-encoded") } func unkeyedContainer() throws -> UnkeyedDecodingContainer { - guard let data = self.data.array else { - throw Error.unexpectedDataType(self.data.type, expected: "array") + guard let array = self.data.array else { + throw DecodingError.dataCorruptedError(in: self, debugDescription: "Non-natively typed arrays must be JSON-encoded") } - return _UnkeyedDecoder(data: data, json: self.json) - } - - func container( - keyedBy type: Key.Type - ) throws -> KeyedDecodingContainer where Key : CodingKey { - let data: Data - if let jsonb = self.data.jsonb { - data = jsonb - } else if let json = self.data.json { - data = json - } else { - throw Error.unexpectedDataType(self.data.type, expected: "json") + return ArrayContainer(data: array, dataDecoder: self.dataDecoder) + } + + struct ArrayContainer: UnkeyedDecodingContainer { + let data: [PostgresData] + let dataDecoder: PostgresDataDecoder + var codingPath: [CodingKey] { [] } + var count: Int? { self.data.count } + var isAtEnd: Bool { self.currentIndex >= self.data.count } + var currentIndex: Int = 0 + + mutating func decodeNil() throws -> Bool { + // Do _not_ shorten this using `defer`, otherwise `currentIndex` is incorrectly incremented. + if self.data[self.currentIndex].value == nil { + self.currentIndex += 1 + return true + } + return false + } + + mutating func decode(_ type: T.Type) throws -> T where T : Decodable { + // Do _not_ shorten this using `defer`, otherwise `currentIndex` is incorrectly incremented. + let result = try self.dataDecoder.decode(T.self, from: self.data[self.currentIndex]) + self.currentIndex += 1 + return result + } + + mutating func nestedContainer(keyedBy _: NewKey.Type) throws -> KeyedDecodingContainer { + throw DecodingError.dataCorruptedError(in: self, debugDescription: "Data nesting is not supported") + } + + mutating func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer { + throw DecodingError.dataCorruptedError(in: self, debugDescription: "Data nesting is not supported") + } + + mutating func superDecoder() throws -> Decoder { + throw DecodingError.dataCorruptedError(in: self, debugDescription: "Data nesting is not supported") } - return try self.json - .decode(DecoderUnwrapper.self, from: data) - .decoder.container(keyedBy: Key.self) } - + func singleValueContainer() throws -> SingleValueDecodingContainer { - _ValueDecoder(data: self.data, json: self.json) - } - } - - struct _UnkeyedDecoder: UnkeyedDecodingContainer { - var count: Int? { - self.data.count - } - - var isAtEnd: Bool { - self.currentIndex == self.data.count - } - var currentIndex: Int = 0 - - let data: [PostgresData] - let json: PostgresJSONDecoder - var codingPath: [CodingKey] { - [] + return self } - - mutating func decodeNil() throws -> Bool { - defer { self.currentIndex += 1 } - return self.data[self.currentIndex].value == nil - } - - mutating func decode(_ type: T.Type) throws -> T where T : Decodable { - defer { self.currentIndex += 1 } - let data = self.data[self.currentIndex] - return try PostgresDataDecoder(json: self.json).decode(T.self, from: data) - } - - mutating func nestedContainer( - keyedBy type: NestedKey.Type - ) throws -> KeyedDecodingContainer - where NestedKey : CodingKey - { - throw Error.nestingNotSupported - } - - mutating func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer { - throw Error.nestingNotSupported - } - - mutating func superDecoder() throws -> Decoder { - throw Error.nestingNotSupported - } - } - - struct _ValueDecoder: SingleValueDecodingContainer { - let data: PostgresData - let json: PostgresJSONDecoder - var codingPath: [CodingKey] { - [] - } - + func decodeNil() -> Bool { - return self.data.value == nil + self.data.value == nil } func decode(_ type: T.Type) throws -> T where T : Decodable { - if let convertible = T.self as? PostgresDataConvertible.Type { - guard let value = convertible.init(postgresData: data) else { - throw DecodingError.typeMismatch(T.self, DecodingError.Context.init( - codingPath: [], - debugDescription: "Could not convert to \(T.self): \(data)" - )) - } - return value as! T - } else { - return try T.init(from: _Decoder(data: self.data, json: self.json)) - } + // Recurse back into the data decoder, don't repeat its logic here. + return try self.dataDecoder.decode(T.self, from: self.data) } } } - -struct DecoderUnwrapper: Decodable { - let decoder: Decoder - init(from decoder: Decoder) { - self.decoder = decoder - } -} diff --git a/Tests/PostgresKitTests/PostgresKitTests.swift b/Tests/PostgresKitTests/PostgresKitTests.swift index c6c2cc2..52b608b 100644 --- a/Tests/PostgresKitTests/PostgresKitTests.swift +++ b/Tests/PostgresKitTests/PostgresKitTests.swift @@ -13,14 +13,7 @@ class PostgresKitTests: XCTestCase { } func testPerformance() throws { - let db = PostgresConnectionSource( - configuration: .init( - hostname: hostname, - username: "vapor_username", - password: "vapor_password", - database: "vapor_database" - ) - ) + let db = PostgresConnectionSource(configuration: .test) let pool = EventLoopGroupConnectionPool( source: db, maxConnectionsPerEventLoop: 2, @@ -130,12 +123,7 @@ class PostgresKitTests: XCTestCase { } func testEventLoopGroupSQL() throws { - var configuration = PostgresConfiguration( - hostname: hostname, - username: "vapor_username", - password: "vapor_password", - database: "vapor_database" - ) + var configuration = PostgresConfiguration.test configuration.searchPath = ["foo", "bar", "baz"] let source = PostgresConnectionSource(configuration: configuration) let pool = EventLoopGroupConnectionPool(source: source, on: self.eventLoopGroup) @@ -147,59 +135,47 @@ class PostgresKitTests: XCTestCase { } func testArrayEncoding_json() throws { - _ = try self.connection.query("DROP TABLE IF EXISTS foo").wait() - _ = try self.connection.query("CREATE TABLE foo (bar integer[] not null)").wait() + let connection = try PostgresConnection.test(on: self.eventLoop).wait() + defer { try! connection.close().wait() } + _ = try connection.query("DROP TABLE IF EXISTS foo").wait() + _ = try connection.query("CREATE TABLE foo (bar integer[] not null)").wait() defer { - _ = try! self.connection.query("DROP TABLE foo").wait() + _ = try! connection.query("DROP TABLE foo").wait() } - _ = try self.connection.query("INSERT INTO foo (bar) VALUES ($1)", [ + _ = try connection.query("INSERT INTO foo (bar) VALUES ($1)", [ PostgresDataEncoder().encode([Bar]()) ]).wait() - let rows = try self.connection.query("SELECT * FROM foo").wait() + let rows = try connection.query("SELECT * FROM foo").wait() print(rows) } func testEnum() throws { - try self.benchmark.testEnum() - } - - var db: SQLDatabase { - self.connection.sql() - } - var benchmark: SQLBenchmarker { - .init(on: self.db) - } - var eventLoop: EventLoop { - self.eventLoopGroup.next() + let connection = try PostgresConnection.test(on: self.eventLoop).wait() + defer { try! connection.close().wait() } + try SQLBenchmarker(on: connection.sql()).testEnum() } + var eventLoop: EventLoop { self.eventLoopGroup.next() } var eventLoopGroup: EventLoopGroup! - var connection: PostgresConnection! - override func setUp() { + override func setUpWithError() throws { + try super.setUpWithError() XCTAssertTrue(isLoggingConfigured) self.eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2) - self.connection = try! PostgresConnection.test( - on: self.eventLoopGroup.next() - ).wait() } - override func tearDown() { - try! self.connection.close().wait() - self.connection = nil - try! self.eventLoopGroup.syncShutdownGracefully() + override func tearDownWithError() throws { + try self.eventLoopGroup.syncShutdownGracefully() self.eventLoopGroup = nil + try super.tearDownWithError() } } enum Bar: Int, Codable { case one, two } -extension Bar: PostgresDataConvertible { } -func env(_ name: String) -> String? { - getenv(name).flatMap { String(cString: $0) } -} +extension Bar: PostgresDataConvertible { } let isLoggingConfigured: Bool = { LoggingSystem.bootstrap { label in diff --git a/Tests/PostgresKitTests/Utilities.swift b/Tests/PostgresKitTests/Utilities.swift index 463d42a..132e5ab 100644 --- a/Tests/PostgresKitTests/Utilities.swift +++ b/Tests/PostgresKitTests/Utilities.swift @@ -1,19 +1,36 @@ +import XCTest import PostgresKit +import NIOCore +import Logging +#if canImport(Darwin) +import Darwin.C +#else +import Glibc +#endif extension PostgresConnection { static func test(on eventLoop: EventLoop) -> EventLoopFuture { - do { - let address: SocketAddress - address = try .makeAddressResolvingHost(hostname, port: PostgresConfiguration.ianaPortNumber) - return connect(to: address, on: eventLoop).flatMap { conn in - return conn.authenticate( - username: "vapor_username", - database: "vapor_database", - password: "vapor_password" - ).map { conn } + let config = PostgresConfiguration.test + + return eventLoop.flatSubmit { () -> EventLoopFuture in + do { + let address = try config.address() + return self.connect(to: address, on: eventLoop) + } catch { + return eventLoop.makeFailedFuture(error) + } + }.flatMap { conn in + return conn.authenticate( + username: config.username, + database: config.database, + password: config.password + ) + .map { conn } + .flatMapError { error in + conn.close().flatMapThrowing { + throw error + } } - } catch { - return eventLoop.makeFailedFuture(error) } } } @@ -21,23 +38,16 @@ extension PostgresConnection { extension PostgresConfiguration { static var test: Self { .init( - hostname: hostname, + hostname: env("POSTGRES_HOSTNAME") ?? "localhost", port: Self.ianaPortNumber, - username: "vapor_username", - password: "vapor_password", - database: "vapor_database" + username: env("POSTGRES_USER") ?? "vapor_username", + password: env("POSTGRES_PASSWORD") ?? "vapor_password", + database: env("POSTGRES_DB") ?? "vapor_database", + tlsConfiguration: nil ) } } -var hostname: String { - if let hostname = env("POSTGRES_HOSTNAME") { - return hostname - } else { - #if os(Linux) - return "psql" - #else - return "127.0.0.1" - #endif - } +func env(_ name: String) -> String? { + getenv(name).flatMap { String(cString: $0) } }