From 42f03418c5d9cc073712ab0837ed5c65553e1a7c Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 15 Dec 2017 21:48:18 +0100 Subject: [PATCH] Properly adjust the orientation for full-screen shots (#16) * Properly adjust the orientation for full-screen shots * Only fix the orientation if needed * Revert "Only fix the orientation if needed" This reverts commit fae7914d5df7752ef131af2845e4bfc2131021ed. * Tune error messages * Fix taking screenshots for elements with 'correct' frames in landscape mode * Apply some small refactoring * Use appFrame everywhere --- .../Categories/XCUIDevice+FBHelpers.m | 14 ++++---- .../Categories/XCUIElement+FBUtilities.m | 33 ++++++++----------- .../Utilities/FBBaseActionsSynthesizer.m | 2 +- WebDriverAgentLib/Utilities/FBMathUtils.h | 3 ++ WebDriverAgentLib/Utilities/FBMathUtils.m | 24 ++++++++++++++ .../Utilities/FBW3CActionsSynthesizer.m | 2 +- 6 files changed, 49 insertions(+), 29 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m index 1528fa8b..4d1e212e 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m @@ -56,7 +56,8 @@ - (NSData *)fb_screenshotWithError:(NSError*__autoreleasing*)error } id mainScreen = [xcScreen valueForKey:@"mainScreen"]; - CGSize screenSize = FBAdjustDimensionsForApplication(FBApplication.fb_activeApplication.frame.size, (UIInterfaceOrientation)[self.class sharedDevice].orientation); + FBApplication *activeApplication = FBApplication.fb_activeApplication; + UIInterfaceOrientation orientation = activeApplication.interfaceOrientation; SEL mSelector = NSSelectorFromString(@"screenshotDataForQuality:rect:error:"); NSMethodSignature *mSignature = [mainScreen methodSignatureForSelector:mSelector]; NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:mSignature]; @@ -67,18 +68,17 @@ - (NSData *)fb_screenshotWithError:(NSError*__autoreleasing*)error // and the resulting screenshot does not fit the memory buffer preallocated for it by the operating system NSUInteger quality = 1; [invocation setArgument:&quality atIndex:2]; + CGSize screenSize = FBAdjustDimensionsForApplication(activeApplication.frame.size, orientation); CGRect screenRect = CGRectMake(0, 0, screenSize.width, screenSize.height); [invocation setArgument:&screenRect atIndex:3]; [invocation setArgument:&error atIndex:4]; [invocation invoke]; - NSData __unsafe_unretained *result; - [invocation getReturnValue:&result]; - if (nil == result) { + NSData __unsafe_unretained *imageData; + [invocation getReturnValue:&imageData]; + if (nil == imageData) { return nil; } - // The resulting data is a JPEG image, so we need to convert it to PNG representation - UIImage *image = [UIImage imageWithData:result]; - return (NSData *)UIImagePNGRepresentation(image); + return FBAdjustScreenshotOrientationForApplication(imageData, orientation); } - (BOOL)fb_fingerTouchShouldMatch:(BOOL)shouldMatch diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBUtilities.m b/WebDriverAgentLib/Categories/XCUIElement+FBUtilities.m index 9fde69dd..12e16381 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBUtilities.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBUtilities.m @@ -153,6 +153,18 @@ - (NSData *)fb_screenshotWithError:(NSError **)error NSUInteger quality = 1; [invocation setArgument:&quality atIndex:2]; CGRect elementRect = self.frame; + UIInterfaceOrientation orientation = self.application.interfaceOrientation; + if (orientation == UIInterfaceOrientationLandscapeLeft || orientation == UIInterfaceOrientationLandscapeRight) { + // Workaround XCTest bug when element frame is returned as in portrait mode even if the screenshot is rotated + XCElementSnapshot *parentWindow = [self.fb_lastSnapshot fb_parentMatchingType:XCUIElementTypeWindow]; + CGRect appFrame = self.application.frame; + if (CGRectEqualToRect(appFrame, nil == parentWindow ? elementRect : parentWindow.frame)) { + CGPoint fixedOrigin = orientation == UIInterfaceOrientationLandscapeLeft ? + CGPointMake(appFrame.size.height - elementRect.origin.y - elementRect.size.height, elementRect.origin.x) : + CGPointMake(elementRect.origin.y, appFrame.size.width - elementRect.origin.x - elementRect.size.width); + elementRect = CGRectMake(fixedOrigin.x, fixedOrigin.y, elementRect.size.height, elementRect.size.width); + } + } [invocation setArgument:&elementRect atIndex:3]; [invocation setArgument:&error atIndex:4]; [invocation invoke]; @@ -161,26 +173,7 @@ - (NSData *)fb_screenshotWithError:(NSError **)error if (nil == imageData) { return nil; } - - UIImage *image = [UIImage imageWithData:imageData]; - UIInterfaceOrientation orientation = self.application.interfaceOrientation; - UIImageOrientation imageOrientation = UIImageOrientationUp; - // The received element screenshot will be rotated, if the current interface orientation differs from portrait, so we need to fix that first - if (orientation == UIInterfaceOrientationLandscapeRight) { - imageOrientation = UIImageOrientationLeft; - } else if (orientation == UIInterfaceOrientationLandscapeLeft) { - imageOrientation = UIImageOrientationRight; - } else if (orientation == UIInterfaceOrientationPortraitUpsideDown) { - imageOrientation = UIImageOrientationDown; - } - CGSize size = image.size; - UIGraphicsBeginImageContext(CGSizeMake(size.width, size.height)); - [[UIImage imageWithCGImage:(CGImageRef)[image CGImage] scale:1.0 orientation:imageOrientation] drawInRect:CGRectMake(0, 0, size.width, size.height)]; - UIImage *fixedImage = UIGraphicsGetImageFromCurrentImageContext(); - UIGraphicsEndImageContext(); - - // The resulting data is a JPEG image, so we need to convert it to PNG representation - return (NSData *)UIImagePNGRepresentation(fixedImage); + return FBAdjustScreenshotOrientationForApplication(imageData, orientation); } @end diff --git a/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m index fd10ba2f..9bb5b4d6 100644 --- a/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m @@ -52,7 +52,7 @@ - (nullable NSValue *)hitpointWithElement:(nullable XCUIElement *)element positi XCElementSnapshot *snapshot = element.fb_lastSnapshot; CGRect frameInWindow = snapshot.fb_frameInWindow; if (CGRectIsEmpty(frameInWindow)) { - NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen", element.debugDescription]; + NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen and thus is not interactable", element.description]; if (error) { *error = [[FBErrorBuilder.builder withDescription:description] build]; } diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.h b/WebDriverAgentLib/Utilities/FBMathUtils.h index e27e606d..10bdd00c 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.h +++ b/WebDriverAgentLib/Utilities/FBMathUtils.h @@ -36,3 +36,6 @@ CGPoint FBInvertOffsetForOrientation(CGPoint offset, UIInterfaceOrientation orie /*! Inverts size if necessary to match current screen orientation */ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation); + +/*! Fixes the screenshot orientation if necessary to match current screen orientation */ +NSData *FBAdjustScreenshotOrientationForApplication(NSData *screenshotData, UIInterfaceOrientation orientation); diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.m b/WebDriverAgentLib/Utilities/FBMathUtils.m index c28ed92b..9f47dd38 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.m +++ b/WebDriverAgentLib/Utilities/FBMathUtils.m @@ -82,3 +82,27 @@ This verification is just to make sure the bug is still there (since height is n } return actualSize; } + +NSData *FBAdjustScreenshotOrientationForApplication(NSData *screenshotData, UIInterfaceOrientation orientation) +{ + UIImage *image = [UIImage imageWithData:screenshotData]; + UIImageOrientation imageOrientation; + if (orientation == UIInterfaceOrientationLandscapeRight) { + imageOrientation = UIImageOrientationLeft; + } else if (orientation == UIInterfaceOrientationLandscapeLeft) { + imageOrientation = UIImageOrientationRight; + } else if (orientation == UIInterfaceOrientationPortraitUpsideDown) { + imageOrientation = UIImageOrientationDown; + } else { + return (NSData *)UIImagePNGRepresentation(image); + } + + UIGraphicsBeginImageContext(CGSizeMake(image.size.width, image.size.height)); + [[UIImage imageWithCGImage:(CGImageRef)[image CGImage] scale:1.0 orientation:imageOrientation] + drawInRect:CGRectMake(0, 0, image.size.width, image.size.height)]; + UIImage *fixedImage = UIGraphicsGetImageFromCurrentImageContext(); + UIGraphicsEndImageContext(); + + // The resulting data should be a PNG image + return (NSData *)UIImagePNGRepresentation(fixedImage); +} diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 0c5cda4f..3b7c7d39 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -131,7 +131,7 @@ - (nullable NSValue *)hitpointWithElement:(nullable XCUIElement *)element positi // An offset relative to the element is defined XCElementSnapshot *snapshot = element.fb_lastSnapshot; if (CGRectIsEmpty(snapshot.fb_frameInWindow)) { - NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen", element.debugDescription]; + NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen and thus is not interactable", element.description]; if (error) { *error = [[FBErrorBuilder.builder withDescription:description] build]; }