Merge pull request #304 from oouyang/2.2_stingray_ssl

V2.2 stingray - add security:ssl test

certsuite/cert.py

@@ -496,9 +496,9 @@ def parse_permissions_results(expected_results_path, results, prefix, logger, re
     return not unexpected_results
 
 
-def run_marionette_script(script, chrome=False, async=False):
+def run_marionette_script(script, chrome=False, async=False, host='localhost', port=2828):
     """Create a Marionette instance and run the provided script"""
-    m = marionette.Marionette()
+    m = marionette.Marionette(host, port)
     m.start_session()
     if chrome:
         m.set_context(marionette.Marionette.CONTEXT_CHROME)

certsuite/harness.py

@@ -49,7 +49,7 @@
 import report
 
 DeviceBackup = adb_b2g.DeviceBackup
-_adbflag = False
+_hasadb = True
 _host = 'localhost'
 _port = 2828
 logger = None
@@ -82,14 +82,19 @@ def load_config(path):
     return config
 
 
-def iter_test_lists(suites_config):
+def iter_test_lists(suites_config, mode='phone'):
     '''
     Query each subharness for the list of test groups it can run and
     yield a tuple of (subharness, test group) for each one.
     '''
     for name, opts in suites_config.iteritems():
         try:
             cmd = [opts["cmd"], '--list-test-groups'] + opts.get("common_args", [])
+
+            if mode == 'stingray':
+                cmd.append('--mode')
+                cmd.append('stingray')
+
             for group in subprocess.check_output(cmd).splitlines():
                 yield name, group
         except (subprocess.CalledProcessError, OSError) as e:
@@ -109,11 +114,11 @@ def log_metadata():
 
 # Consider upstreaming this to marionette-client:
 class MarionetteSession(object):
-    def __init__(self, device):
+    def __init__(self, device, host='localhost', port=2828):
         global _host
         global _port
         self.device = device
-        self.marionette = marionette.Marionette(host=_host, port=_port)
+        self.marionette = marionette.Marionette(host=host, port=port)
 
     def __enter__(self):
         self.device.forward("tcp:2828", "tcp:2828")
@@ -162,8 +167,8 @@ def iter_suites(self):
             default_tests = self.config["suites"].keys()
 
             # stingray only test 'webapi' part
-            if args.mode == 'stingray':
-                default_tests = [default_tests['webapi']]
+            if self.args.mode == 'stingray':
+                default_tests = ['webapi', 'security']
 
             tests = default_tests
         else:
@@ -200,6 +205,7 @@ def get_test_name(self, test, test_data):
     def generate_retry(self):
         test_map = {'certsuite' : 'cert',
                     'webapi' : 'webapi',
+                    'securitysuite' : 'security',                    
                     'web-platform-tests':'web-platform-tests'}
 
         failed = []
@@ -309,7 +315,8 @@ def build_command(self, suite, groups, temp_dir):
         output_files = [log_name]
         output_files += [item % subn for item in suite_opts.get("extra_files", [])]
 
-        cmd.extend([u'--host=%s' % _host, u'--port=%s' % _port])
+        if self.args.mode == 'stingray' and (suite == 'webapi' or suite == 'security'):
+            cmd.extend([u'--host=%s' % _host, u'--port=%s' % _port, u'--mode=stingray'])
 
         return cmd, output_files, log_name
 
@@ -349,6 +356,8 @@ def __enter__(self):
         return self
     def __exit__(self, *args, **kwargs):
         pass
+    def restore(self):
+        pass
 
 class NoADB():
     def reboot(self):
@@ -367,7 +376,7 @@ def restart(self):
 def check_adb():
     try:
         logger.info("Testing ADB connection")
-        if _adbflag:
+        if not _hasadb:
             logger.debug('Dummy ADB, please remember install Marionette and Cert Test App to device ')
             return NoADB()
         return adb_b2g.ADBB2G()
@@ -395,7 +404,7 @@ def check_root(device):
 
 
 def install_marionette(device, version):
-    if _adbflag:
+    if not _hasadb:
         logger.debug('The marionette should be installed manually by user.')
         return True
     try:
@@ -434,7 +443,7 @@ def ensure_settings(device):
                      "screen.brightness": 1.0,
                      "screen.timeout": 0.0}
     logger.info("Setting up device for testing")
-    with MarionetteSession(device) as marionette:
+    with MarionetteSession(device, _host, _port) as marionette:
         settings = gaiautils.Settings(marionette)
         for k, v in test_settings.iteritems():
             settings.set(k, v)
@@ -455,7 +464,7 @@ def wait_for_homescreen(marionette, timeout):
 def check_server(device):
     logger.info("Checking access to host machine")
 
-    if _adbflag:
+    if not _hasadb:
         return True
 
     routes = [("GET", "/", test_handler)]
@@ -488,7 +497,15 @@ def check_server(device):
 
 
 def list_tests(args, config):
-    for test, group in iter_test_lists(config["suites"]):
+    suites = OrderedDict.copy(config["suites"])
+
+    if args.mode == 'stingray':
+        stingray_suite_keys = ['webapi', 'security']
+        for key in suites.keys():
+            if key not in stingray_suite_keys:
+                del suites[key]
+
+    for test, group in iter_test_lists(suites, args.mode):
         print "%s:%s" % (test, group)
     return True
 
@@ -674,13 +691,13 @@ def main():
 
     global _host
     global _port
-    global _adbflag
+    global _hasadb
     global DeviceBackup
     _host = args.host
     _port = int(args.port)
 
     if args.mode == 'stingray':
-        _adbflag = True
+        _hasadb = False
         DeviceBackup = NoADBDeviceBackup
 
     if args.list_tests:

securitysuite/filesystem.py

@@ -171,7 +171,7 @@ def whitelist_check(cls, name, flag='ok', whitelist=None):
         return r.match(name) is not None
 
     @classmethod
-    def run(cls, version=None):
+    def run(cls, version=None, host='localhost', port=2828, hasadb=True):
         logger = get_default_logger()
 
         try:
@@ -231,7 +231,7 @@ def whitelist_check(cls, name, flag='ok', whitelist=None):
         return r.match(name) is not None
 
     @classmethod
-    def run(cls, version=None):
+    def run(cls, version=None, host='localhost', port=2828, hasadb=True):
         logger = get_default_logger()
 
         try:

securitysuite/kernel.py

@@ -117,7 +117,7 @@ class seccomp(ExtraTest):
     module = sys.modules[__name__]
 
     @classmethod
-    def run(cls, version=None):
+    def run(cls, version=None, host='localhost', port=2828, hasadb=True):
         logger = get_default_logger()
 
         try:

securitysuite/ssl.py

@@ -75,26 +75,31 @@ def js_certdump():
         '''
 
 
-    def __init__(self):
+    def __init__(self, hasadb=True):
+        self.hasadb = hasadb
         self.logger = get_default_logger()
         try:
-            self.dm = mozdevice.DeviceManagerADB(runAdbAsRoot=True)
+            if hasadb:
+                self.dm = mozdevice.DeviceManagerADB(runAdbAsRoot=True)
         except mozdevice.DMError as e:
             self.logger.error("Error connecting to device via adb (error: %s). Please be " \
                               "sure device is connected and 'remote debugging' is enabled." % \
                               e.msg)
             raise
         self.logger.debug("Attempting to set up port forwarding for marionette")
 
+    def get_via_marionette(self, host='localhost', port=2828):
+        if self.hasadb:
+            self.dm.forward("tcp:2828", "tcp:2828")
+        return run_marionette_script(certdump.js_certdump(),
+                                     chrome=True, host=host, port=port)
 
-    def get_via_marionette(self):
-        self.dm.forward("tcp:2828", "tcp:2828")
-        return run_marionette_script(certdump.js_certdump(), chrome=True)
 
-
-    def nssversion_via_marionette(self):
-        self.dm.forward("tcp:2828", "tcp:2828")
-        return run_marionette_script(certdump.js_nssversions(), chrome=True)
+    def nssversion_via_marionette(self, host='localhost', port=2828):
+        if self.hasadb:
+            self.dm.forward("tcp:2828", "tcp:2828")
+        return run_marionette_script(certdump.js_nssversions(),
+                                     chrome=True, host=host, port=port)
 
 
 #######################################################################################################################
@@ -117,12 +122,12 @@ class certdb_info(ExtraTest):
     module = sys.modules[__name__]
 
     @classmethod
-    def run(cls, version=None):
+    def run(cls, version=None, host='localhost', port=2828, hasadb=True):
         logger = get_default_logger()
 
         try:
-            dumper = certdump()
-            certs = dumper.get_via_marionette()
+            dumper = certdump(hasadb)
+            certs = dumper.get_via_marionette(host, port)
         except:
             cls.log_status('FAIL', 'Failed to gather information from the device via Marionette.')
             return False
@@ -145,12 +150,12 @@ class nssversion_info(ExtraTest):
     module = sys.modules[__name__]
 
     @classmethod
-    def run(cls, version=None):
+    def run(cls, version=None, host='localhost', port=2828, hasadb=True):
         logger = get_default_logger()
 
         try:
-            dumper = certdump()
-            versions = dumper.nssversion_via_marionette()
+            dumper = certdump(hasadb)
+            versions = dumper.nssversion_via_marionette(host, port)
         except:
             cls.log_status('FAIL', 'Failed to gather information from the device via Marionette.')
             return False

securitysuite/suite.py

@@ -34,21 +34,25 @@ def groupname(cls):
             return 'unknown'
 
     @staticmethod
-    def group_list():
+    def group_list(mode='phone'):
         """
         Returns a list of all groups in the test suite.
         """
+        if mode == 'stingray':
+            return ['ssl']
         groups = []
         for t in ExtraTest.__subclasses__():
             if t.groupname() not in groups:
                 groups.append(t.groupname())
         return groups
 
     @staticmethod
-    def test_list(group=None):
+    def test_list(group=None, mode='phone'):
         """
         Returns a list of all tests, optionally filtered by group.
         """
+        if mode == 'stingray' and group is not None:
+            return 'ssl'
         if group is None:
             return ExtraTest.__subclasses__()
         else:
@@ -59,19 +63,20 @@ def test_list(group=None):
             return tests
 
     @staticmethod
-    def run_groups(groups=[], version=None):
+    def run_groups(groups=[], version=None, host='localhost', port=2828, mode='phone'):
+        hasadb = mode == 'phone'
         logger = get_default_logger()
         if groups is None or len(groups) == 0:  # run all groups
-            logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list()))
-            groups = ExtraTest.group_list()
+            logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list(mode=mode)))
+            groups = ExtraTest.group_list(mode=mode)
         else:
             logger.debug('running securitysuite tests for groups %s' % str(groups))
         logger.suite_start(tests=groups)
         for g in groups:
             logger.debug("running securitysuite test group %s" % g)
             logger.test_start(g)
             try:
-                ExtraTest.run(g, version=version)
+                ExtraTest.run(g, version=version, host=host, port=port, hasadb=hasadb)
                 logger.test_end(g, 'OK')
             except:
                 logger.critical(traceback.format_exc())
@@ -80,12 +85,12 @@ def run_groups(groups=[], version=None):
         logger.suite_end()
 
     @classmethod
-    def run(cls, group=None, version=None):
+    def run(cls, group=None, version=None, host='localhost', port=2828, hasadb=True):
         """
         Runs all the tests, optionally just within the specified group.
         """
         for t in cls.test_list(group):
-            t.run(version=version)
+            t.run(version=version, host=host, port=port, hasadb=hasadb)
 
     @classmethod
     def log_status(cls, status, msg):
@@ -141,6 +146,15 @@ def securitycli():
                         help="B2G version")
     parser.add_argument("--ipython", dest="ipython", action="store_true",
                         help="drop to ipython session")
+    parser.add_argument('-H', '--host',
+                        help='Hostname or ip for target device',
+                        action='store', default='localhost')
+    parser.add_argument('-P', '--port',
+                        help='Port for target device',
+                        action='store', default=2828)
+    parser.add_argument('-m', '--mode',
+                        help='Test mode (stingray, phone) default (phone)',
+                        action='store', default='phone')
     parser.add_argument("-v", dest="verbose", action="store_true",
                         help="Verbose output")
 
@@ -154,15 +168,20 @@ def securitycli():
     try:
         logger.debug("security cli runnng with args %s" % args)
         if args.list_test_groups:
-            for group in ExtraTest.group_list():
+            for group in ExtraTest.group_list(args.mode):
                 print group
         elif args.list_all_tests:
-            for test in ExtraTest.test_list():
+            for test in ExtraTest.test_list(args.mode):
                 print "%s.%s" % (test.group, test.__name__)
         elif args.ipython:
             from IPython import embed
 
             embed()
+        elif args.mode == 'stingray':
+            ExtraTest.run_groups(args.include,
+                                 version=args.version,
+                                 host=args.host, port=int(args.port),
+                                 mode=args.mode)
         else:
             wait_for_adb_device()
             if not adb_has_root():