Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Add tools-specific Issue kind that can be used by third-party test libraries. #513

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,49 @@ extension ABIv0 {
/// The location in source where this issue occurred, if available.
var sourceLocation: SourceLocation?

/// Any tool-specific context about the issue including the name of the tool
/// that recorded it.
///
/// When decoding using `JSONDecoder`, the value of this property is set to
/// `nil`. Tools that need access to their context values should not use
/// ``ABIv0/EncodedIssue`` to decode issues.
var toolContext: (any Issue.Kind.ToolContext)?

init(encoding issue: borrowing Issue) {
isKnown = issue.isKnown
sourceLocation = issue.sourceLocation
if case let .recordedByTool(toolContext) = issue.kind {
self.toolContext = toolContext
}
}
}
}

// MARK: - Codable

extension ABIv0.EncodedIssue: Codable {}
extension ABIv0.EncodedIssue: Codable {
private enum CodingKeys: String, CodingKey {
case isKnown
case sourceLocation
case toolContext
}

func encode(to encoder: any Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(isKnown, forKey: .isKnown)
try container.encode(sourceLocation, forKey: .sourceLocation)
if let toolContext {
func encodeToolContext(_ toolContext: some Issue.Kind.ToolContext) throws {
try container.encode(toolContext, forKey: .toolContext)
}
try encodeToolContext(toolContext)
}
}

init(from decoder: any Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
isKnown = try container.decode(Bool.self, forKey: .isKnown)
sourceLocation = try container.decode(SourceLocation.self, forKey: .sourceLocation)
toolContext = nil // not decoded
}
}
26 changes: 26 additions & 0 deletions Sources/Testing/Issues/Issue+Recording.swift
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,32 @@ extension Issue {
let issue = Issue(kind: .unconditional, comments: Array(comment), sourceContext: sourceContext)
return issue.record()
}

/// Record an issue on behalf of a tool or library.
///
/// - Parameters:
/// - comment: A comment describing the expectation.
/// - toolContext: Any tool-specific context about the issue including the
/// name of the tool that recorded it.
/// - sourceLocation: The source location to which the issue should be
/// attributed.
///
/// - Returns: The issue that was recorded.
///
/// Test authors do not generally need to use this function. Rather, a tool
/// or library based on the testing library can use it to record a
/// domain-specific issue and to propagatre additional information about that
/// issue to other layers of the testing library's infrastructure.
@_spi(Experimental)
@discardableResult public static func record(
_ comment: Comment? = nil,
context toolContext: some Issue.Kind.ToolContext,
sourceLocation: SourceLocation = #_sourceLocation
) -> Self {
let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation)
let issue = Issue(kind: .recordedByTool(toolContext), comments: Array(comment), sourceContext: sourceContext)
return issue.record()
}
}

// MARK: - Recording issues for errors
Expand Down
37 changes: 36 additions & 1 deletion Sources/Testing/Issues/Issue.swift
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,33 @@ public struct Issue: Sendable {
/// An issue due to a failure in the underlying system, not due to a failure
/// within the tests being run.
case system

/// A protocol describing additional context provided by an external tool or
/// library that recorded an issue of kind
/// ``Issue/Kind/recordedByTool(_:)``.
///
/// Test authors do not generally need to use this protocol. Rather, a tool
/// or library based on the testing library can use it to propagate
/// additional information about an issue to other layers of the testing
/// library's infrastructure.
///
/// A tool or library may conform as many types as it needs to this
/// protocol. Instances of types conforming to this protocol must be
/// encodable as JSON so that they can be included in event streams produced
/// by the testing library.
public protocol ToolContext: Sendable, Encodable {
/// The human-readable name of the tool that recorded the issue.
var toolName: String { get }
}

/// An issue recorded by an external tool or library that uses the testing
/// library.
///
/// - Parameters:
/// - toolContext: Any tool-specific context about the issue including the
/// name of the tool that recorded it.
@_spi(Experimental)
indirect case recordedByTool(_ toolContext: any ToolContext)
}

/// The kind of issue this value represents.
Expand Down Expand Up @@ -135,7 +162,11 @@ extension Issue: CustomStringConvertible, CustomDebugStringConvertible {
let joinedComments = comments.lazy
.map(\.rawValue)
.joined(separator: "\n")
return "\(kind): \(joinedComments)"
if case let .recordedByTool(toolContext) = kind {
return "\(joinedComments) (from '\(toolContext.toolName)')"
} else {
return "\(kind): \(joinedComments)"
}
}

public var debugDescription: String {
Expand Down Expand Up @@ -172,6 +203,8 @@ extension Issue.Kind: CustomStringConvertible {
"An API was misused"
case .system:
"A system failure occurred"
case let .recordedByTool(toolContext):
"'\(toolContext.toolName)' recorded an issue"
}
}
}
Expand Down Expand Up @@ -310,6 +343,8 @@ extension Issue.Kind {
.apiMisused
case .system:
.system
case .recordedByTool:
.unconditional // TBD
}
}

Expand Down
35 changes: 35 additions & 0 deletions Tests/TestingTests/IssueTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -994,6 +994,41 @@ final class IssueTests: XCTestCase {
}.run(configuration: configuration)
}

func testFailBecauseOfToolSpecificIssue() async throws {
struct ToolContext: Issue.Kind.ToolContext {
var value: Int
var toolName: String {
"Swift Testing Itself"
}
}

var configuration = Configuration()
configuration.eventHandler = { event, _ in
guard case let .issueRecorded(issue) = event.kind else {
return
}
XCTAssertFalse(issue.isKnown)
guard case let .recordedByTool(toolContext) = issue.kind else {
XCTFail("Unexpected issue kind \(issue.kind)")
return
}
guard let toolContext = toolContext as? ToolContext else {
XCTFail("Unexpected tool context \(toolContext)")
return
}
XCTAssertEqual(toolContext.toolName, "Swift Testing Itself")
XCTAssertEqual(toolContext.value, 12345)

XCTAssertEqual(String(describingForTest: issue), "Something went wrong (from 'Swift Testing Itself')")
XCTAssertEqual(String(describingForTest: issue.kind), "'Swift Testing Itself' recorded an issue")
}

await Test {
let toolContext = ToolContext(value: 12345)
Issue.record("Something went wrong", context: toolContext)
}.run(configuration: configuration)
}

func testErrorPropertyValidForThrownErrors() async throws {
var configuration = Configuration()
configuration.eventHandler = { event, _ in
Expand Down