From 4b038b2ad0f0372ecfa764c6d8141251208bbfeb Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 8 May 2023 13:41:34 -0500 Subject: [PATCH 1/7] Changes to collection handling, notes/variable changes Signed-off-by: Tomas --- redfish_service_validator/catalog.py | 541 +++++++++--------- redfish_service_validator/helper.py | 1 + redfish_service_validator/metadata.py | 2 + redfish_service_validator/schema.py | 1 + redfish_service_validator/validateRedfish.py | 47 +- redfish_service_validator/validateResource.py | 2 +- 6 files changed, 311 insertions(+), 283 deletions(-) diff --git a/redfish_service_validator/catalog.py b/redfish_service_validator/catalog.py index 29aab0b..0f358a0 100644 --- a/redfish_service_validator/catalog.py +++ b/redfish_service_validator/catalog.py @@ -241,6 +241,10 @@ def getTypeInSchemaDoc(self, currentType, tagType=["EntityType", "ComplexType"]) """ if isinstance(currentType, RedfishType): currentType = currentType.fulltype + + if 'Collection(' in currentType: + currentType = currentType.replace('Collection(', "").replace(')', "") + pnamespace, ptype = getNamespace(currentType), getType(currentType) pnamespace = self.catalog.alias.get(pnamespace, pnamespace) pbase = getNamespaceUnversioned(pnamespace) @@ -354,7 +358,7 @@ def __init__(self, soup, owner: SchemaClass): if self.tag_type in ['NavigationProperty', 'Property', 'Term']: self.IsPropertyType = True self.IsNav = self.tag_type in ['NavigationProperty'] - self.fulltype, _ = self.parent_type + self.fulltype = self.parent_type else: self.IsPropertyType = False self.IsNav = False @@ -481,7 +485,6 @@ def DynamicProperties(self): return None - def getUris(self): """ Return Redfish.Uris annotation values @@ -503,43 +506,38 @@ def getUris(self): my_logger.warning('Could not gather info from Redfish.Uris annotation') expectedUris = [] return expectedUris - - @property + + @property def parent_type(self): """ - Returns string of the parent type, and if that type is a collection + Returns string of the parent type Returns: - string, boolean - None, False + string of type """ soup = self.type_soup parent_type = ( soup["UnderlyingType"] if self.tag_type == "TypeDefinition" else soup.get("BaseType", soup.get("Type", None)) ) - if parent_type is not None: - IsCollection = re.match('Collection\(.*\)', parent_type) is not None - return parent_type.replace('Collection(', "").replace(')', ""), IsCollection - else: - return None, False + return parent_type - def getTypeTree(self, tree=None): + def getTypeTree(self): """ Returns tree of RedfishType/string of parent types """ - if not tree: tree = [self] - my_type, collection = self.parent_type - if my_type: + tree = [self] + my_type = self.parent_type + while my_type: if 'Edm.' not in my_type: - my_real_type = my_type - type_obj = self.owner.parent_doc.catalog.getSchemaDocByClass(my_real_type).getTypeInSchemaDoc(my_real_type) - return type_obj.getTypeTree(tree + [type_obj]) + type_obj = self.owner.parent_doc.catalog.getSchemaDocByClass(my_type).getTypeInSchemaDoc(my_type) + tree.append(type_obj) + my_type = type_obj.parent_type else: return tree + [my_type] return tree - def getBaseType(self, is_collection=False): + def getBaseType(self): """ Returns string representing our tag type, and if that type is a collection @@ -551,19 +549,31 @@ def getBaseType(self, is_collection=False): None, False """ if self.tag_type == "EnumType": - return 'enum', is_collection + return 'enum' if self.tag_type == "ComplexType": - return 'complex', is_collection + return 'complex' if self.tag_type == "EntityType": - return 'entity', is_collection + return 'entity' + + my_type = self.parent_type + if 'Edm.' in my_type: + return my_type if self.IsPropertyType: - my_type, parent_collection = self.parent_type - is_collection=parent_collection or is_collection - if 'Edm.' in my_type: - return my_type, is_collection type_obj = self.owner.parent_doc.catalog.getSchemaDocByClass(my_type).getTypeInSchemaDoc(my_type) - return type_obj.getBaseType(is_collection) - return 'none', is_collection + return type_obj.getBaseType() + return 'none' + + def isCollection(self): + tree = [self.fulltype] + my_type = self.parent_type + while my_type: + if isinstance(my_type, RedfishType): + tree.append(my_type.fulltype) + my_type = self.parent_type + else: + tree.append(my_type) + break + return any([re.match(r'Collection(.*)', typ) for typ in tree]) def getProperties(self): """ @@ -580,12 +590,12 @@ def validate(self, val, added_pattern=None): """ my_logger.debug((self, val, self.fulltype, self.tag_type, self.parent_type)) if val == REDFISH_ABSENT: - if self.type_soup.find("Annotation", attrs={"Term": "Redfish.Required"}): + if self.IsMandatory: raise ValueError("Should not be absent") else: return True - if val is None: - if self.type_soup.get("Nullable") in ["false", "False", False]: + if val is None: + if self.IsNullable: raise ValueError("Should not be null") else: return True @@ -600,7 +610,7 @@ def validate(self, val, added_pattern=None): my_complex = self.createObject().populate(val) if self.tag_type == "EntityType": return True - my_type, collection = self.parent_type + my_type = self.parent_type if my_type: if 'Edm.' not in my_type: type_obj = self.owner.parent_doc.catalog.getSchemaDocByClass(my_type).getTypeInSchemaDoc(my_type) @@ -624,7 +634,7 @@ def validate(self, val, added_pattern=None): return True def as_json(self): - return self.createObj().as_json() + return self.createObject().as_json() def createObject(self): return RedfishObject(self) @@ -646,7 +656,7 @@ def __init__(self, my_type, name="Property", parent=None): self.HasSchema = self.Type != REDFISH_ABSENT self.Populated = False self.Value = None - self.IsValid = False # Needs consistency, should be @property + self.IsValid = False self.InAnnotation = False self.SchemaExists = False self.Exists = REDFISH_ABSENT @@ -657,23 +667,22 @@ def populate(self, val, check=False): eval_prop = copy.copy(self) eval_prop.Populated = True eval_prop.Value = val - eval_prop.IsValid = True # Needs consistency, should be @property - eval_prop.InAnnotation = False + eval_prop.InAnnotation = False eval_prop.SchemaExists = True eval_prop.Exists = val != REDFISH_ABSENT + + eval_prop.IsValid = True if isinstance(eval_prop.Type, str) and 'Edm.' in eval_prop.Type and check: try: - eval_prop.IsValid = eval_prop.Exists and RedfishProperty.validate_basic( - val, eval_prop.Type - ) + eval_prop.IsValid = eval_prop.Exists and RedfishProperty.validate_basic(eval_prop.Value, eval_prop.Type) except ValueError as e: - my_logger.error('{}: {}'.format(self.Name, e)) # log this + my_logger.error('{}: {}'.format(eval_prop.Name, e)) # log this eval_prop.IsValid = False elif isinstance(eval_prop.Type, RedfishType) and check: try: - eval_prop.IsValid = eval_prop.Type.validate(val, self.added_pattern) + eval_prop.IsValid = eval_prop.Type.validate(eval_prop.Value, eval_prop.added_pattern) except ValueError as e: - my_logger.error('{}: {}'.format(self.Name, e)) # log this + my_logger.error('{}: {}'.format(eval_prop.Name, e)) # log this eval_prop.IsValid = False return eval_prop @@ -683,7 +692,6 @@ def as_json(self): my_dict['IsRequired'] = self.Type.IsMandatory my_dict['IsNullable'] = self.Type.IsNullable my_dict['HasAdditional'] = self.Type.HasAdditional - pass return my_dict def getLinks(self): @@ -784,7 +792,6 @@ def validate_basic(val, my_type, validPattern=None, min=None, max=None): else: return False - class RedfishObject(RedfishProperty): """Represents Redfish as they are represented as Resource/ComplexTypes @@ -805,14 +812,13 @@ def __contains__(self, item): def __init__(self, redfish_type: RedfishType, name="Object", parent=None): super().__init__(redfish_type, name, parent) self.payload = None - self.Collection = None self.IsValid = False self.HasValidUri = False self.HasValidUriStrict = False self.properties = {} for prop, typ in redfish_type.getProperties().items(): try: - base, collection = typ.getBaseType() + base = typ.getBaseType() if base == 'complex': self.properties[prop] = RedfishObject(typ, prop, self) else: @@ -822,223 +828,192 @@ def __init__(self, redfish_type: RedfishType, name="Object", parent=None): my_logger.warning('Schema not found for {}'.format(typ)) def populate(self, payload, check=False, casted=False): - eval_obj = super().populate(payload) - eval_obj.payload = payload + if isinstance(payload, list): + return RedfishObjectCollection(self.Type, "Collection", self.parent).populate(payload) + populated_object = super().populate(payload) + populated_object.payload = payload # todo: redesign objects to have consistent variables, not only when populated # if populated, should probably just use another python class? # remember that populated RedfishObjects may have more objects embedded in them # i.e. OEM or complex or arrays if payload == REDFISH_ABSENT or payload is None: - eval_obj.Collection = [] - if payload is None: - sub_obj = copy.copy(eval_obj) - eval_obj.Collection = [sub_obj] - eval_obj.IsValid = eval_obj.Type.IsNullable - eval_obj.HasValidUri = True - eval_obj.HasValidUriStrict = False - eval_obj.properties = {x:y.populate(REDFISH_ABSENT) for x, y in eval_obj.properties.items()} - return eval_obj - - # Representation for Complexes as Collection, unless it is not a list - # If an list object changes when populated, it will be represented properly here - evals = [] - payloads = [payload] - if isinstance(payload, list): - payloads = payload - for sub_payload in payloads: - if sub_payload is None: - # If the object is null, treat it as an empty object for the cataloging - sub_payload = {} - sub_obj = copy.copy(eval_obj) - - # Only valid if we are a dictionary... - # todo: see above None/REDFISH_ABSENT block - sub_obj.IsValid = isinstance(sub_payload, dict) - if not sub_obj.IsValid: - my_logger.error("This complex object {} should be a dictionary or None, but it's of type {}...".format(sub_obj.Name, str(type(sub_payload)))) - sub_obj.Collection = [] - sub_obj.HasValidUri = True - sub_obj.HasValidUriStrict = False - sub_obj.properties = {x:y.populate(REDFISH_ABSENT) for x, y in sub_obj.properties.items()} - evals.append(sub_obj) + populated_object.HasValidUri = True + populated_object.HasValidUriStrict = False + populated_object.properties = {x: y.populate(REDFISH_ABSENT) for x, y in populated_object.properties.items()} + return populated_object + + populated_object.IsValid = isinstance(payload, dict) + + # Only valid if we are a dictionary... + # todo: see above None/REDFISH_ABSENT block + if not populated_object.IsValid: + my_logger.error("This complex object {} should be a dictionary or None, but it's of type {}...".format(populated_object.Name, str(type(payload)))) + populated_object.HasValidUri = True + populated_object.HasValidUriStrict = False + populated_object.properties = {x: y.populate(REDFISH_ABSENT) for x, y in populated_object.properties.items()} + + # Cast types if they're below their parent or are OemObjects + # Don't cast objects with odata.type that matches their current object type + already_typed = False + sub_payload = payload + my_odata_type = sub_payload.get('@odata.type') + if my_odata_type is not None and str(populated_object.Type) == my_odata_type.strip('#'): + already_typed = True + if populated_object.Type.IsNav: + already_typed = True + + # If our item is an OemObject type and hasn't been casted to a type, then cast it + if 'Resource.OemObject' in populated_object.Type.getTypeTree() and not casted: + my_logger.verbose1(('Morphing OemObject', my_odata_type, populated_object.Type)) + if my_odata_type: + my_odata_type = my_odata_type.strip('#') + try: + type_obj = populated_object.Type.catalog.getSchemaDocByClass(my_odata_type).getTypeInSchemaDoc(my_odata_type) + populated_object = RedfishObject(type_obj, populated_object.Name, populated_object.parent).populate(sub_payload, check=check, casted=True) + except MissingSchemaError: + my_logger.warning("Couldn't get schema for object, skipping OemObject {}".format(populated_object.Name)) + except Exception as e: + my_logger.warning("Couldn't get schema for object (?), skipping OemObject {} : {}".format(populated_object.Name, e)) + return populated_object + # Otherwise, if we're not casted, or we don't have an odata type, then cast it + elif not casted and not already_typed: + my_ns, my_ns_unversioned = populated_object.Type.Namespace, getNamespaceUnversioned(populated_object.Type.Namespace) + # if we have an odata type, use it as our upper limit + if my_odata_type: + my_limit = getNamespace(my_odata_type).strip('#') + else: + my_limit = 'v9_9_9' + # If our item is not a Resource.Resource type, determine its parent's version limit for later... + # NOTE: Resource items always seem to be cast to highest type, not determined by its parent's type + # not sure where this is backed up in documentation + if populated_object.parent and my_ns_unversioned not in ['Resource']: + parent = populated_object + while parent.parent and parent.parent.Type.Namespace.startswith(my_ns_unversioned + '.'): + parent = parent.parent + my_limit = parent.Type.Namespace + my_type = populated_object.Type.Type + top_ns = my_ns + # get type order from bottom up of SchemaDoc + for top_ns, schema in reversed(list(populated_object.Type.catalog.getSchemaDocByClass(my_ns).classes.items())): + # if our object type is in schema... check for limit + if my_type in schema.my_types: + if splitVersionString(top_ns) <= splitVersionString(my_limit): + my_ns = top_ns + break + # ISSUE: We can't cast under v1_0_0, get the next best Type + if my_ns == my_ns_unversioned: + my_ns = top_ns + if my_ns not in populated_object.Type.Namespace: + my_logger.verbose1(('Morphing Complex', my_ns, my_type, my_limit)) + new_type_obj = populated_object.Type.catalog.getSchemaDocByClass(my_ns).getTypeInSchemaDoc('.'.join([my_ns, my_type])) + populated_object = RedfishObject(new_type_obj, populated_object.Name, populated_object.parent).populate(sub_payload, check=check, casted=True) + return populated_object + + # Validate our Uri + populated_object.HasValidUri = True + populated_object.HasValidUriStrict = True + allowable_uris = populated_object.Type.getUris() + # If we have expected URIs and @odata.id + # And we AREN'T a navigation property + if not populated_object.Type.catalog.flags['ignore_uri_checks'] and len(allowable_uris) and '@odata.id' in sub_payload: + # Strip our URI and warn if that's the case + my_odata_id = sub_payload['@odata.id'] + if my_odata_id != '/redfish/v1/' and my_odata_id.endswith('/'): + if check: my_logger.warning('Stripping end of URI... {}'.format(my_odata_id)) + my_odata_id = my_odata_id.rstrip('/') + + # Initial check if our URI matches our format at all + # Setup REGEX... + my_uri_regex = "^{}$".format("|".join(allowable_uris)) + my_uri_regex = re.sub(URI_ID_REGEX, VALID_ID_REGEX, my_uri_regex) + populated_object.HasValidUri = re.fullmatch(my_uri_regex, my_odata_id) is not None + populated_object.HasValidUriStrict = populated_object.HasValidUri + + if 'Resource.Resource' in populated_object.Type.getTypeTree(): + if '#' in my_odata_id: + my_logger.warning('Found uri with fragment, which Resource.Resource types do not use {}'.format(my_odata_id)) + elif 'Resource.ReferenceableMember' in populated_object.Type.getTypeTree(): + if '#' not in my_odata_id: + my_logger.warning('No fragment in URI, but ReferenceableMembers require it {}'.format(my_odata_id)) + + # check that our ID is matching + # this won't check NavigationProperties but the Resources will + if populated_object.HasValidUri and not populated_object.Type.IsNav: + # pair our type, Id value, and odata.id value + my_odata_split = my_odata_id.split('/') + my_type, my_id, my_uri_id = populated_object.Type.Type, sub_payload.get('Id'), my_odata_split[-1] + + for schema_uri in allowable_uris: + # regex URI check to confirm which URI + my_uri_regex = re.sub(URI_ID_REGEX, VALID_ID_REGEX, "^{}$".format(schema_uri)) + if re.fullmatch(my_uri_regex, my_odata_id): + # pair our uri with the current resource + schema_uri_end = schema_uri.rsplit('/')[-1] + # if our Uri is expecting an Id, then check if they match, otherwise we are already passing + if re.match(URI_ID_REGEX, schema_uri_end): + if my_id is not None: + populated_object.HasValidUriStrict = my_id == my_uri_id + break + + # TODO: Oem support is able, but it is tempermental for Actions and Additional properties + #if 'Resource.OemObject' in sub_obj.Type.getTypeTree(): + # evals.append(sub_obj) + # continue + + # populate properties + if populated_object.Name == 'Actions': + populated_object.properties = {x: y.populate(sub_payload.get(x, REDFISH_ABSENT)) for x, y in populated_object.properties.items() if x != 'Oem'} + else: + populated_object.properties = {x: y.populate(sub_payload.get(x, REDFISH_ABSENT)) for x, y in populated_object.properties.items()} + + # additional_props + if populated_object.Type.DynamicProperties: + my_dynamic = populated_object.Type.DynamicProperties + my_odata_type = my_dynamic.get('Type', 'Resource.OemObject') + prop_pattern = my_dynamic.get('Pattern', '.*') + allow_property_generation = populated_object.Name != 'Actions' + else: + my_odata_type = 'Resource.OemObject' + prop_pattern = '.*' + allow_property_generation = populated_object.Type.HasAdditional and populated_object.Name != 'Actions' + + if allow_property_generation: + my_property_names = [x for x in sub_payload if x not in populated_object.properties if re.match(prop_pattern, x) and '@' not in x] + for add_name in my_property_names: + if 'Edm.' in my_odata_type: + my_new_term = ' '.format(add_name, my_odata_type) # Make a pseudo tag because RedfishType requires it... + new_soup = BeautifulSoup(my_new_term, "xml").find('Term') + type_obj = RedfishType(new_soup, populated_object.Type.owner) + else: + type_obj = populated_object.Type.catalog.getSchemaDocByClass(my_odata_type).getTypeInSchemaDoc(my_odata_type) + if type_obj.getBaseType() == 'complex': + object = RedfishObject(type_obj, name=add_name, parent=populated_object) + else: + object = RedfishProperty(type_obj, name=add_name, parent=populated_object) + my_logger.debug('Populated {} with {}'.format(my_property_names, object.as_json())) + my_logger.verbose1(('Adding Additional', add_name, my_odata_type, populated_object.Type)) + populated_object.properties[add_name] = object.populate(sub_payload.get(add_name, REDFISH_ABSENT)) + + my_annotations = [x for x in sub_payload if x not in populated_object.properties if '@' in x and '@odata' not in x] + for key in my_annotations: + splitKey = key.split('@', 1) + fullItem = splitKey[1] + if getNamespace(fullItem) not in allowed_annotations: + my_logger.warning("getAnnotations: {} is not an allowed annotation namespace, please check spelling/capitalization.".format(fullItem)) continue - - # Cast types if they're below their parent or are OemObjects - # Don't cast objects with odata.type that matches their current object type - already_typed = False - my_odata_type = sub_payload.get('@odata.type') - if my_odata_type is not None and str(sub_obj.Type) == my_odata_type.strip('#'): - already_typed = True - if sub_obj.Type.IsNav: - already_typed = True - - # If our item is an OemObject type and hasn't been casted to a type, then cast it - if 'Resource.OemObject' in sub_obj.Type.getTypeTree() and not casted: - my_logger.verbose1(('Morphing OemObject', my_odata_type, sub_obj.Type)) - if my_odata_type: - my_odata_type = my_odata_type.strip('#') - try: - type_obj = sub_obj.Type.catalog.getSchemaDocByClass(my_odata_type).getTypeInSchemaDoc(my_odata_type) - sub_obj = RedfishObject(type_obj, sub_obj.Name, sub_obj.parent).populate(sub_payload, check=check, casted=True) - except MissingSchemaError: - my_logger.warning("Couldn't get schema for object, skipping OemObject {}".format(sub_obj.Name)) - except Exception as e: - my_logger.warning("Couldn't get schema for object (?), skipping OemObject {} : {}".format(sub_obj.Name, e)) - evals.append(sub_obj) - continue - # Otherwise, if we're not casted, or we don't have an odata type, then cast it - elif not casted and not already_typed: - my_ns, my_ns_unversioned = sub_obj.Type.Namespace, getNamespaceUnversioned(sub_obj.Type.Namespace) - # if we have an odata type, use it as our upper limit - if my_odata_type: - my_limit = getNamespace(my_odata_type).strip('#') + try: + type_obj = populated_object.Type.catalog.getSchemaInCatalog(fullItem).terms[getType(fullItem)] + if type_obj.getBaseType() == 'complex': + object = RedfishObject(type_obj, name=key, parent=populated_object) else: - my_limit = 'v9_9_9' - # If our item is not a Resource.Resource type, determine its parent's version limit for later... - # NOTE: Resource items always seem to be cast to highest type, not determined by its parent's type - # not sure where this is backed up in documentation - if sub_obj.parent and my_ns_unversioned not in ['Resource']: - parent = sub_obj - while parent.parent and parent.parent.Type.Namespace.startswith(my_ns_unversioned + '.'): - parent = parent.parent - my_limit = parent.Type.Namespace - my_type = sub_obj.Type.Type - top_ns = my_ns - # get type order from bottom up of SchemaDoc - for top_ns, schema in reversed(list(sub_obj.Type.catalog.getSchemaDocByClass(my_ns).classes.items())): - # if our object type is in schema... check for limit - if my_type in schema.my_types: - if splitVersionString(top_ns) <= splitVersionString(my_limit): - my_ns = top_ns - break - # ISSUE: We can't cast under v1_0_0, get the next best Type - if my_ns == my_ns_unversioned: - my_ns = top_ns - if my_ns not in sub_obj.Type.Namespace: - my_logger.verbose1(('Morphing Complex', my_ns, my_type, my_limit)) - new_type_obj = sub_obj.Type.catalog.getSchemaDocByClass(my_ns).getTypeInSchemaDoc('.'.join([my_ns, my_type])) - sub_obj = RedfishObject(new_type_obj, sub_obj.Name, sub_obj.parent).populate(sub_payload, check=check, casted=True) - evals.append(sub_obj) - continue + object = RedfishProperty(type_obj, name=key, parent=self) + my_logger.verbose1(('Adding Additional', key, my_odata_type, populated_object.Type)) + populated_object.properties[key] = object.populate(sub_payload[key]) + except: + my_logger.error("Unable to locate the definition of the annotation '@{}'.".format(fullItem)) - # Validate our Uri - sub_obj.HasValidUri = True - sub_obj.HasValidUriStrict = True - allowable_uris = sub_obj.Type.getUris() - # If we have expected URIs and @odata.id - # And we AREN'T a navigation property - if not sub_obj.Type.catalog.flags['ignore_uri_checks'] and len(allowable_uris) and '@odata.id' in sub_payload: - # Strip our URI and warn if that's the case - my_odata_id = sub_payload['@odata.id'] - if my_odata_id != '/redfish/v1/' and my_odata_id.endswith('/'): - if check: my_logger.warning('Stripping end of URI... {}'.format(my_odata_id)) - my_odata_id = my_odata_id.rstrip('/') - - # Initial check if our URI matches our format at all - # Setup REGEX... - my_uri_regex = "^{}$".format("|".join(allowable_uris)) - my_uri_regex = re.sub(URI_ID_REGEX, VALID_ID_REGEX, my_uri_regex) - sub_obj.HasValidUri = re.fullmatch(my_uri_regex, my_odata_id) is not None - sub_obj.HasValidUriStrict = sub_obj.HasValidUri - - if 'Resource.Resource' in sub_obj.Type.getTypeTree(): - if '#' in my_odata_id: - my_logger.warning('Found uri with fragment, which Resource.Resource types do not use {}'.format(my_odata_id)) - elif 'Resource.ReferenceableMember' in sub_obj.Type.getTypeTree(): - if '#' not in my_odata_id: - my_logger.warning('No fragment in URI, but ReferenceableMembers require it {}'.format(my_odata_id)) - - # check that our ID is matching - # this won't check NavigationProperties but the Resources will - if sub_obj.HasValidUri and not sub_obj.Type.IsNav: - # pair our type, Id value, and odata.id value - my_odata_split = my_odata_id.split('/') - my_type, my_id, my_uri_id = sub_obj.Type.Type, sub_payload.get('Id'), my_odata_split[-1] - - for schema_uri in allowable_uris: - # regex URI check to confirm which URI - my_uri_regex = re.sub(URI_ID_REGEX, VALID_ID_REGEX, "^{}$".format(schema_uri)) - if re.fullmatch(my_uri_regex, my_odata_id): - # pair our uri with the current resource - schema_uri_end = schema_uri.rsplit('/')[-1] - # if our Uri is expecting an Id, then check if they match, otherwise we are already passing - if re.match(URI_ID_REGEX, schema_uri_end): - if my_id is not None: - sub_obj.HasValidUriStrict = my_id == my_uri_id - break - - # TODO: Oem support is able, but it is tempermental for Actions and Additional properties - #if 'Resource.OemObject' in sub_obj.Type.getTypeTree(): - # evals.append(sub_obj) - # continue - - # populate properties - if sub_obj.Name == 'Actions': - sub_obj.properties = {x:y.populate(sub_payload.get(x, REDFISH_ABSENT)) for x, y in sub_obj.properties.items() if x != 'Oem'} - else: - sub_obj.properties = {x:y.populate(sub_payload.get(x, REDFISH_ABSENT)) for x, y in sub_obj.properties.items()} - - # additional_props - if sub_obj.Type.DynamicProperties: - my_dynamic = sub_obj.Type.DynamicProperties - my_odata_type = my_dynamic.get('Type', 'Resource.OemObject') - prop_pattern = my_dynamic.get('Pattern', '.*') - allow_property_generation = sub_obj.Name != 'Actions' - else: - my_odata_type = 'Resource.OemObject' - prop_pattern = '.*' - allow_property_generation = sub_obj.Type.HasAdditional and sub_obj.Name != 'Actions' - - if allow_property_generation: - my_property_names = [x for x in sub_payload if x not in sub_obj.properties if re.match(prop_pattern, x) and '@' not in x] - for add_name in my_property_names: - if 'Edm.' in my_odata_type: - my_new_term = ' '.format(add_name, my_odata_type) # Make a pseudo tag because RedfishType requires it... - new_soup = BeautifulSoup(my_new_term, "xml").find('Term') - type_obj = RedfishType(new_soup, sub_obj.Type.owner) - else: - type_obj = sub_obj.Type.catalog.getSchemaDocByClass(my_odata_type).getTypeInSchemaDoc(my_odata_type) - if type_obj.getBaseType()[0] == 'complex': - object = RedfishObject(type_obj, name=add_name, parent=self) - else: - object = RedfishProperty(type_obj, name=add_name, parent=self) - my_logger.debug('Populated {} with {}'.format(my_property_names, object.as_json())) - my_logger.verbose1(('Adding Additional', add_name, my_odata_type, sub_obj.Type)) - sub_obj.properties[add_name] = object.populate(sub_payload.get(add_name, REDFISH_ABSENT)) - - my_annotations = [x for x in sub_payload if x not in sub_obj.properties if '@' in x and '@odata' not in x] - for key in my_annotations: - splitKey = key.split('@', 1) - fullItem = splitKey[1] - if getNamespace(fullItem) not in allowed_annotations: - my_logger.warning("getAnnotations: {} is not an allowed annotation namespace, please check spelling/capitalization.".format(fullItem)) - continue - try: - type_obj = sub_obj.Type.catalog.getSchemaInCatalog(fullItem).terms[getType(fullItem)] - if type_obj.getBaseType()[0] == 'complex': - object = RedfishObject(type_obj, name=key, parent=self) - else: - object = RedfishProperty(type_obj, name=key, parent=self) - my_logger.verbose1(('Adding Additional', key, my_odata_type, sub_obj.Type)) - sub_obj.properties[key] = object.populate(sub_payload[key]) - except: - my_logger.error("Unable to locate the definition of the annotation '@{}'.".format(fullItem)) - - evals.append(sub_obj) - if not isinstance(payload, list): - sub_obj.Collection = evals - return sub_obj - else: - for e, v in zip(evals, eval_obj.Value): - e.Value = v - eval_obj.Collection = evals - return eval_obj - - @property - def IsCollection(self): - my_base, is_collection = self.Type.getBaseType() - return is_collection + return populated_object def as_json(self): if self.Populated: @@ -1076,13 +1051,51 @@ def getLinks(self): links.append(new_link) else: links.append(item) - elif item.Type.getBaseType()[0] == 'complex': - for sub in item.Collection: - if sub.Value is None: - continue - InAnnotation = sub.Name in ['@Redfish.Settings', '@Redfish.ActionInfo', '@Redfish.CollectionCapabilities'] - my_links = sub.getLinks() - for item in my_links: - item.InAnnotation = InAnnotation - links.extend(my_links) + elif item.Type.getBaseType() == 'complex': + if item.Value is None: + continue + InAnnotation = item.Name in ['@Redfish.Settings', '@Redfish.ActionInfo', '@Redfish.CollectionCapabilities'] + my_links = item.getLinks() + for sub_item in my_links: + sub_item.InAnnotation = InAnnotation + links.extend(my_links) return links + + +class RedfishObjectCollection(RedfishObject): + """Represents Redfish as they are represented as Resource/ComplexTypes + + Can be indexed with [] + Can be populated with Data + If Populated, can be grabbed for Links + Can get json representation of type properties with as_json + """ + def __getitem__(self, index): + return self.collection[index] + + def __contains__(self, item): + if self.Populated: + return item in self.collection + else: + return item in self.properties + + def __iter__(self): + return self.collection.__iter__ + + def __init__(self, redfish_type: RedfishType, name="Collection", parent=None): + super().__init__(redfish_type, name, parent) + self.collection = [] + + def populate(self, collection): + new_collection = super().populate(collection[0] if len(collection) else None) + new_collection.Value = collection + base_object = RedfishObject(self.Type, "Object", self.parent) + for payload in collection: + new_collection.collection.append(base_object.populate(payload)) + return new_collection + + def getLinks(self): + all_links = [] + for item in self.collection: + all_links.extend(item.getLinks()) + return all_links \ No newline at end of file diff --git a/redfish_service_validator/helper.py b/redfish_service_validator/helper.py index 34cb431..a534358 100644 --- a/redfish_service_validator/helper.py +++ b/redfish_service_validator/helper.py @@ -6,6 +6,7 @@ import logging from types import SimpleNamespace +# TODO: Replace logger with custom logger with custom verbose levels, remove verbose1 and verbose2 my_logger = logging.getLogger() my_logger.setLevel(logging.DEBUG) diff --git a/redfish_service_validator/metadata.py b/redfish_service_validator/metadata.py index d4e681d..a067f56 100644 --- a/redfish_service_validator/metadata.py +++ b/redfish_service_validator/metadata.py @@ -7,6 +7,8 @@ from collections import Counter, OrderedDict, defaultdict import redfish_service_validator.schema as schema +# TODO: Use catalog.py instead of schema.py, remove uses of schema.py + EDM_NAMESPACE = "http://docs.oasis-open.org/odata/ns/edm" EDMX_NAMESPACE = "http://docs.oasis-open.org/odata/ns/edmx" EDM_TAGS = ['Action', 'Annotation', 'Collection', 'ComplexType', 'EntityContainer', 'EntityType', 'EnumType', 'Key', diff --git a/redfish_service_validator/schema.py b/redfish_service_validator/schema.py index 381110b..01fd04c 100644 --- a/redfish_service_validator/schema.py +++ b/redfish_service_validator/schema.py @@ -13,6 +13,7 @@ import logging my_logger = logging.getLogger(__name__) +# TODO: In metadata.py, use catalog.py instead of schema.py. Remove file schema.py. def storeSchemaToLocal(xml_data, origin, service): """storeSchemaToLocal diff --git a/redfish_service_validator/validateRedfish.py b/redfish_service_validator/validateRedfish.py index aa7427c..9c3ec0f 100644 --- a/redfish_service_validator/validateRedfish.py +++ b/redfish_service_validator/validateRedfish.py @@ -1,6 +1,6 @@ from collections import Counter, OrderedDict -from redfish_service_validator.catalog import REDFISH_ABSENT, MissingSchemaError, ExcerptTypes, get_fuzzy_property +from redfish_service_validator.catalog import REDFISH_ABSENT, MissingSchemaError, ExcerptTypes, get_fuzzy_property, RedfishObject, RedfishObjectCollection, RedfishType from redfish_service_validator.helper import getNamespace, getNamespaceUnversioned, getType, checkPayloadConformance @@ -11,7 +11,7 @@ def validateExcerpt(prop, val): # check Navprop if it's NEUTRAL or CONTAINS - base, _ = prop.Type.getBaseType() + base = prop.Type.getBaseType() if base == 'entity': my_excerpt_type, my_excerpt_tags = prop.Type.excerptType, prop.Type.excerptTags @@ -269,7 +269,7 @@ def displayType(propTypeObject, is_collection=False): :param is_collection: For collections: True if these types are for the collection; False if for a member :return: the simplified type to display """ - propRealType, propCollection = propTypeObject.getBaseType() + propRealType, propCollection = propTypeObject.getBaseType(), propTypeObject.isCollection() propType = propTypeObject.fulltype # Edm.* and other explicit types if propRealType == 'Edm.Boolean' or propRealType == 'Boolean': @@ -426,7 +426,7 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ # Note: consider http://docs.oasis-open.org/odata/odata-csdl-xml/v4.01/csprd01/odata-csdl-xml-v4.01-csprd01.html#_Toc472333112 # Note: make sure it checks each one # propCollectionType = PropertyDict.get('isCollection') - propRealType, isCollection = prop.Type.getBaseType() + propRealType, isCollection = prop.Type.getBaseType(), prop.Type.isCollection() excerptPass = True if isCollection and prop.Value is None: @@ -457,12 +457,11 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ elif not isinstance(prop.Value, list): my_logger.error('{}: property is expected to contain an array'.format(prop_name)) counts['failInvalidArray'] += 1 - return {prop_name: ( '-', displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', 'FAIL')}, counts + resultList[prop_name] = ('-', displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', 'FAIL') + propValueList = [prop.Value] else: propValueList = prop.Value - resultList[prop_name] = ('Array (size: {})'.format(len(prop.Value)), - displayType(prop.Type, is_collection=True), - 'Yes' if prop.Exists else 'No', '...') + resultList[prop_name] = ('Array (size: {})'.format(len(prop.Value)), displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', '...') else: # not a collection propValueList = [prop.Value] @@ -473,11 +472,14 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ my_logger.error("{}: Mandatory prop does not exist".format(prop_name)) counts['failMandatoryExist'] += 1 result_str = 'FAIL' - resultList[prop_name] = ( - '[JSON Object]', displayType(prop.Type), - 'Yes' if prop.Exists else 'No', - result_str) - for n, sub_obj in enumerate(prop.Collection): + resultList[prop_name] = ('[JSON Object]', displayType(prop.Type), 'Yes' if prop.Exists else 'No', result_str) + + if isinstance(prop, RedfishObjectCollection): + object_list = prop.collection + else: + object_list = [prop] + + for n, sub_obj in enumerate(object_list): try: if sub_obj.Value is None: if prop.Type.IsNullable or 'EventDestination.v1_0_0.HttpHeaderProperty' == str(prop.Type.fulltype): @@ -488,7 +490,7 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ my_logger.error('{}: Property is null but is not Nullable'.format(prop_name)) counts['failNullable'] += 1 result_str = 'FAIL' - if len(prop.Collection) == 1: + if isinstance(prop, RedfishObject): resultList['{}.[Value]'.format(prop_name)] = ('[null]', displayType(prop.Type), 'Yes' if prop.Exists else 'No', result_str) else: @@ -498,7 +500,7 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ subMsgs, subCounts = validateComplex(service, sub_obj, prop_name, oem_check) if isCollection: subMsgs = {'{}[{}].{}'.format(prop_name,n,x):y for x,y in subMsgs.items()} - elif len(prop.Collection) == 1: + elif isinstance(prop, RedfishObject): subMsgs = {'{}.{}'.format(prop_name,x):y for x,y in subMsgs.items()} else: subMsgs = {'{}.{}#{}'.format(prop_name,x,n):y for x,y in subMsgs.items()} @@ -533,9 +535,18 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ else: propNullablePass = False - prop = prop.populate(val, check=True) - - paramPass = prop.IsValid + if isinstance(prop.Type, str) and 'Edm.' in prop.Type: + try: + paramPass = prop.Exists and prop.validate_basic(val, prop.Type) + except ValueError as e: + my_logger.error('{}: {}'.format(prop.Name, e)) # log this + paramPass = False + elif isinstance(prop.Type, RedfishType): + try: + paramPass = prop.Type.validate(prop.Value, prop.added_pattern) + except ValueError as e: + my_logger.error('{}: {}'.format(prop.Name, e)) # log this + paramPass = False if propRealType == 'entity': paramPass = validateEntity(service, prop, val) diff --git a/redfish_service_validator/validateResource.py b/redfish_service_validator/validateResource.py index e81ab7e..bb0fb96 100644 --- a/redfish_service_validator/validateResource.py +++ b/redfish_service_validator/validateResource.py @@ -230,7 +230,7 @@ def validateSingleURI(service, URI, uriName='', expectedType=None, expectedJson= except Exception as ex: my_logger.verbose1('Exception caught while validating single URI', exc_info=1) my_logger.error('{}: Could not finish check on this property ({})'.format(prop_name, str(ex))) - propMessages[prop_name] = create_entry(prop_name, '', '', prop.Exists, 'exception') + messages[prop_name] = create_entry(prop_name, '', '', prop.Exists, 'exception') counts['exceptionPropCheck'] += 1 SchemaFullType, jsonData = me['fulltype'], me['payload'] From bbe1b65fb6642e69f6fd91a16ce1a0742f0a2baa Mon Sep 17 00:00:00 2001 From: Tomas Date: Sat, 27 May 2023 12:27:19 -0500 Subject: [PATCH 2/7] Change variable name to support collection variable Signed-off-by: Tomas --- redfish_service_validator/validateResource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redfish_service_validator/validateResource.py b/redfish_service_validator/validateResource.py index bb0fb96..1517bc1 100644 --- a/redfish_service_validator/validateResource.py +++ b/redfish_service_validator/validateResource.py @@ -312,7 +312,7 @@ def validateURITree(service, URI, uriName, expectedType=None, expectedJson=None, if validateSuccess and 'MessageRegistryFile.MessageRegistryFile' in thisobj.Type.getTypeTree(): # thisobj['Location'].Collection[0]['Uri'].Exists if 'Location' in thisobj: - for sub_obj in thisobj['Location'].Collection: + for sub_obj in thisobj['Location'].collection: if 'Uri' in sub_obj: links.append(sub_obj) From ab5591c277c553fa68e4b4d7c4c07446928af993 Mon Sep 17 00:00:00 2001 From: Tomas Date: Fri, 30 Jun 2023 08:00:18 -0500 Subject: [PATCH 3/7] Reworked collection logic Signed-off-by: Tomas --- RedfishServiceValidator.py | 9 +- .../RedfishServiceValidator.py | 7 +- redfish_service_validator/catalog.py | 152 ++++++-------- redfish_service_validator/helper.py | 9 + redfish_service_validator/validateRedfish.py | 191 +++++++++--------- redfish_service_validator/validateResource.py | 11 +- 6 files changed, 182 insertions(+), 197 deletions(-) diff --git a/RedfishServiceValidator.py b/RedfishServiceValidator.py index 0343d32..f451c42 100755 --- a/RedfishServiceValidator.py +++ b/RedfishServiceValidator.py @@ -3,9 +3,12 @@ # License: BSD 3-Clause License. For full text see link: # https://github.com/DMTF/Redfish-Service-Validator/blob/master/LICENSE.md -from redfish_service_validator.RedfishServiceValidator import main +from redfish_service_validator.RedfishServiceValidator import main, my_logger import sys if __name__ == '__main__': - status_code, lastResultsPage, exit_string = main() - sys.exit(status_code) + try: + status_code, lastResultsPage, exit_string = main() + sys.exit(status_code) + except Exception as e: + my_logger.exception("Program finished prematurely: %s", e) diff --git a/redfish_service_validator/RedfishServiceValidator.py b/redfish_service_validator/RedfishServiceValidator.py index 1849f83..942be80 100755 --- a/redfish_service_validator/RedfishServiceValidator.py +++ b/redfish_service_validator/RedfishServiceValidator.py @@ -244,5 +244,8 @@ def main(argslist=None, configfile=None): if __name__ == '__main__': - status_code, lastResultsPage, exit_string = main() - sys.exit(status_code) + try: + status_code, lastResultsPage, exit_string = main() + sys.exit(status_code) + except Exception as e: + my_logger.exception("Program finished prematurely: %s", e) diff --git a/redfish_service_validator/catalog.py b/redfish_service_validator/catalog.py index 0f358a0..d906571 100644 --- a/redfish_service_validator/catalog.py +++ b/redfish_service_validator/catalog.py @@ -13,6 +13,7 @@ getType, getVersion, splitVersionString, + stripCollection ) includeTuple = namedtuple("include", ["Namespace", "Uri"]) @@ -124,9 +125,7 @@ def getSchemaDocByClass(self, typename): :return: Schema Document :rtype: SchemaDoc """ - if 'Collection(' in typename: - typename = typename.replace('Collection(', "").replace(')', "") - typename = getNamespaceUnversioned(typename) + typename = getNamespaceUnversioned(stripCollection(typename)) typename = self.alias.get(typename, typename) if typename in self.catalog_by_class: return self.catalog_by_class[typename][0] @@ -242,8 +241,7 @@ def getTypeInSchemaDoc(self, currentType, tagType=["EntityType", "ComplexType"]) if isinstance(currentType, RedfishType): currentType = currentType.fulltype - if 'Collection(' in currentType: - currentType = currentType.replace('Collection(', "").replace(')', "") + currentType = stripCollection(currentType) pnamespace, ptype = getNamespace(currentType), getType(currentType) pnamespace = self.catalog.alias.get(pnamespace, pnamespace) @@ -363,11 +361,16 @@ def __init__(self, soup, owner: SchemaClass): self.IsPropertyType = False self.IsNav = False self.fulltype = self.owner.class_name + '.' + soup['Name'] - self.Namespace, self.Type = getNamespace(self.fulltype), getType(self.fulltype) + + if 'Collection(' in self.fulltype: + my_fulltype = self.fulltype.replace('Collection(', "").replace(')', "") + self.Namespace, self.Type = getNamespace(my_fulltype), getType(my_fulltype) + else: + self.Namespace, self.Type = getNamespace(self.fulltype), getType(self.fulltype) self.tags = {} for tag in self.type_soup.find_all(recursive=False): - if(tag.get('Term')): + if tag.get('Term'): self.tags[tag['Term']] = tag.attrs if (tag.get('Term') == 'Redfish.Revisions'): self.tags[tag['Term']] = tag.find_all('Record') @@ -396,7 +399,6 @@ def __init__(self, soup, owner: SchemaClass): self.property_pattern = None - # get properties prop_tags = self.type_soup.find_all( ["NavigationProperty", "Property"], recursive=False) @@ -595,7 +597,7 @@ def validate(self, val, added_pattern=None): else: return True if val is None: - if self.IsNullable: + if not self.IsNullable: raise ValueError("Should not be null") else: return True @@ -657,9 +659,10 @@ def __init__(self, my_type, name="Property", parent=None): self.Populated = False self.Value = None self.IsValid = False - self.InAnnotation = False + self.IsCollection = False + self.InAnnotation = False self.SchemaExists = False - self.Exists = REDFISH_ABSENT + self.Exists = False self.parent = parent self.added_pattern = None @@ -667,6 +670,7 @@ def populate(self, val, check=False): eval_prop = copy.copy(self) eval_prop.Populated = True eval_prop.Value = val + eval_prop.IsCollection = isinstance(val, list) eval_prop.InAnnotation = False eval_prop.SchemaExists = True eval_prop.Exists = val != REDFISH_ABSENT @@ -743,17 +747,10 @@ def validate_number(val, minVal=None, maxVal=None): @staticmethod def validate_basic(val, my_type, validPattern=None, min=None, max=None): if "Collection(" in my_type: - if not isinstance(val, list): - raise ValueError("Collection is not list") - my_collection_type = my_type.replace("Collection(", "").replace(")", "") - paramPass = True - for cnt, item in enumerate(val): - try: - paramPass = paramPass and RedfishProperty.validate_basic(item, my_collection_type, validPattern, min, max) - except ValueError as e: - paramPass = False - raise ValueError('{} invalid'.format(cnt)) - return paramPass + my_type = my_type.replace("Collection(", "").replace(")", "") + + if isinstance(val, list): + return all([RedfishProperty.validate_basic(sub_val, my_type, validPattern, min, max) for sub_val in val]) elif my_type == "Edm.Boolean": if not isinstance(val, bool): @@ -813,6 +810,7 @@ def __init__(self, redfish_type: RedfishType, name="Object", parent=None): super().__init__(redfish_type, name, parent) self.payload = None self.IsValid = False + self.IsCollection = False self.HasValidUri = False self.HasValidUriStrict = False self.properties = {} @@ -828,11 +826,17 @@ def __init__(self, redfish_type: RedfishType, name="Object", parent=None): my_logger.warning('Schema not found for {}'.format(typ)) def populate(self, payload, check=False, casted=False): - if isinstance(payload, list): - return RedfishObjectCollection(self.Type, "Collection", self.parent).populate(payload) + """ + Return a populated object, or list of objects + """ populated_object = super().populate(payload) populated_object.payload = payload + if isinstance(payload, list): + populated_object.IsCollection = True + populated_object.Value = [self.populate(sub_item, check, casted) for sub_item in payload] + return populated_object + # todo: redesign objects to have consistent variables, not only when populated # if populated, should probably just use another python class? # remember that populated RedfishObjects may have more objects embedded in them @@ -892,8 +896,10 @@ def populate(self, payload, check=False, casted=False): while parent.parent and parent.parent.Type.Namespace.startswith(my_ns_unversioned + '.'): parent = parent.parent my_limit = parent.Type.Namespace + my_type = populated_object.Type.Type top_ns = my_ns + # get type order from bottom up of SchemaDoc for top_ns, schema in reversed(list(populated_object.Type.catalog.getSchemaDocByClass(my_ns).classes.items())): # if our object type is in schema... check for limit @@ -1030,72 +1036,36 @@ def getLinks(self): links = [] # if we're populated... if self.Populated: - for n, item in self.properties.items(): + for n, property in self.properties.items(): # if we don't exist or our type is Basic - if not item.Exists: continue - if not isinstance(item.Type, RedfishType): continue - if n == 'Actions': - new_type = item.Type.catalog.getTypeInCatalog('ActionInfo.ActionInfo') - for act in item.Value.values(): - if isinstance(act, dict): - uri = act.get('@Redfish.ActionInfo') - if isinstance(uri, str): - my_link = RedfishObject(new_type, 'ActionInfo', item).populate({'@odata.id': uri}) - my_link.InAnnotation = True - links.append(my_link) - if item.Type.IsNav: - if isinstance(item.Value, list): - for num, val in enumerate(item.Value): - new_link = item.populate(val) - new_link.Name = new_link.Name + '#{}'.format(num) - links.append(new_link) - else: - links.append(item) - elif item.Type.getBaseType() == 'complex': - if item.Value is None: - continue - InAnnotation = item.Name in ['@Redfish.Settings', '@Redfish.ActionInfo', '@Redfish.CollectionCapabilities'] - my_links = item.getLinks() - for sub_item in my_links: - sub_item.InAnnotation = InAnnotation - links.extend(my_links) + if not isinstance(property, list): + property = [property] + for item in property: + if not item.Exists: continue + if not isinstance(item.Type, RedfishType): continue + if n == 'Actions': + new_type = item.Type.catalog.getTypeInCatalog('ActionInfo.ActionInfo') + for act in item.Value.values(): + if isinstance(act, dict): + uri = act.get('@Redfish.ActionInfo') + if isinstance(uri, str): + my_link = RedfishObject(new_type, 'ActionInfo', item).populate({'@odata.id': uri}) + my_link.InAnnotation = True + links.append(my_link) + if item.Type.IsNav: + if isinstance(item.Value, list): + for num, val in enumerate(item.Value): + new_link = item.populate(val) + new_link.Name = new_link.Name + '#{}'.format(num) + links.append(new_link) + else: + links.append(item) + elif item.Type.getBaseType() == 'complex': + if item.Value is None: + continue + InAnnotation = item.Name in ['@Redfish.Settings', '@Redfish.ActionInfo', '@Redfish.CollectionCapabilities'] + my_links = item.getLinks() + for sub_item in my_links: + sub_item.InAnnotation = InAnnotation + links.extend(my_links) return links - - -class RedfishObjectCollection(RedfishObject): - """Represents Redfish as they are represented as Resource/ComplexTypes - - Can be indexed with [] - Can be populated with Data - If Populated, can be grabbed for Links - Can get json representation of type properties with as_json - """ - def __getitem__(self, index): - return self.collection[index] - - def __contains__(self, item): - if self.Populated: - return item in self.collection - else: - return item in self.properties - - def __iter__(self): - return self.collection.__iter__ - - def __init__(self, redfish_type: RedfishType, name="Collection", parent=None): - super().__init__(redfish_type, name, parent) - self.collection = [] - - def populate(self, collection): - new_collection = super().populate(collection[0] if len(collection) else None) - new_collection.Value = collection - base_object = RedfishObject(self.Type, "Object", self.parent) - for payload in collection: - new_collection.collection.append(base_object.populate(payload)) - return new_collection - - def getLinks(self): - all_links = [] - for item in self.collection: - all_links.extend(item.getLinks()) - return all_links \ No newline at end of file diff --git a/redfish_service_validator/helper.py b/redfish_service_validator/helper.py index a534358..67dbd18 100644 --- a/redfish_service_validator/helper.py +++ b/redfish_service_validator/helper.py @@ -48,6 +48,15 @@ def splitVersionString(v_string): return tuple([int(v) for v in payload_split]) +def stripCollection(typename): + """ + Remove "Collection()" from a type string + """ + if 'Collection(' in typename: + typename = typename.replace('Collection(', "").replace(')', "") + return typename + + def navigateJsonFragment(decoded, URILink): if '#' in URILink: URIfragless, frag = tuple(URILink.rsplit('#', 1)) diff --git a/redfish_service_validator/validateRedfish.py b/redfish_service_validator/validateRedfish.py index 9c3ec0f..646f2b4 100644 --- a/redfish_service_validator/validateRedfish.py +++ b/redfish_service_validator/validateRedfish.py @@ -1,6 +1,6 @@ from collections import Counter, OrderedDict -from redfish_service_validator.catalog import REDFISH_ABSENT, MissingSchemaError, ExcerptTypes, get_fuzzy_property, RedfishObject, RedfishObjectCollection, RedfishType +from redfish_service_validator.catalog import REDFISH_ABSENT, MissingSchemaError, ExcerptTypes, get_fuzzy_property, RedfishObject, RedfishType from redfish_service_validator.helper import getNamespace, getNamespaceUnversioned, getType, checkPayloadConformance @@ -369,6 +369,7 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ my_logger.verbose1(prop_name) my_logger.verbose1("\tvalue: {} {}".format(prop.Value, type(prop.Value))) + # Basic Validation of all properties prop_name = '.'.join([x for x in (parent_name, prop_name) if x]) propNullable = prop.Type.IsNullable @@ -435,7 +436,6 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ # HttpHeaders in EventDestination has non-conformant details in the long description we need to allow to not break existing implementations my_logger.info('Value HttpHeaders can be Null') propNullable = True - propValueList = [] resultList[prop_name] = ('Array (size: null)', displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', '...') else: my_logger.error('{}: Value of Collection property is null but Collections cannot be null, only their entries'.format(prop_name)) @@ -448,7 +448,6 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ # rs-assumption: check @odata.link property my_logger.verbose1("\tis Collection") if prop.Value == 'n/a': - propValueList = [] resultList[prop_name] = ('Array (absent) {}'.format(len(prop.Value)), displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', 'PASS' if propMandatoryPass else 'FAIL') @@ -458,27 +457,24 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ my_logger.error('{}: property is expected to contain an array'.format(prop_name)) counts['failInvalidArray'] += 1 resultList[prop_name] = ('-', displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', 'FAIL') - propValueList = [prop.Value] else: - propValueList = prop.Value resultList[prop_name] = ('Array (size: {})'.format(len(prop.Value)), displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', '...') - else: - # not a collection - propValueList = [prop.Value] + # If we're validating a complex object if propRealType == 'complex': result_str = 'complex' if prop.Type.IsMandatory and not prop.Exists: my_logger.error("{}: Mandatory prop does not exist".format(prop_name)) counts['failMandatoryExist'] += 1 result_str = 'FAIL' - resultList[prop_name] = ('[JSON Object]', displayType(prop.Type), 'Yes' if prop.Exists else 'No', result_str) - if isinstance(prop, RedfishObjectCollection): - object_list = prop.collection + if prop.IsCollection: + resultList[prop_name] = ('Array (size: {})'.format(len(prop.Value)), displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', result_str) + object_list = prop.Value else: + resultList[prop_name] = ('[JSON Object]', displayType(prop.Type), 'Yes' if prop.Exists else 'No', result_str) object_list = [prop] - + for n, sub_obj in enumerate(object_list): try: if sub_obj.Value is None: @@ -492,18 +488,17 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ result_str = 'FAIL' if isinstance(prop, RedfishObject): resultList['{}.[Value]'.format(prop_name)] = ('[null]', displayType(prop.Type), - 'Yes' if prop.Exists else 'No', result_str) + 'Yes' if prop.Exists else 'No', result_str) else: - resultList['{}.[Value]#{}'.format(prop_name,n)] = ('[null]', displayType(prop.Type), - 'Yes' if prop.Exists else 'No', result_str) + resultList['{}.[Value]#{}'.format(prop_name, n)] = ('[null]', displayType(prop.Type), 'Yes' if prop.Exists else 'No', result_str) else: subMsgs, subCounts = validateComplex(service, sub_obj, prop_name, oem_check) if isCollection: - subMsgs = {'{}[{}].{}'.format(prop_name,n,x):y for x,y in subMsgs.items()} + subMsgs = {'{}[{}].{}'.format(prop_name, n, x): y for x, y in subMsgs.items()} elif isinstance(prop, RedfishObject): - subMsgs = {'{}.{}'.format(prop_name,x):y for x,y in subMsgs.items()} + subMsgs = {'{}.{}'.format(prop_name, x): y for x, y in subMsgs.items()} else: - subMsgs = {'{}.{}#{}'.format(prop_name,x,n):y for x,y in subMsgs.items()} + subMsgs = {'{}.{}#{}'.format(prop_name, x, n): y for x, y in subMsgs.items()} resultList.update(subMsgs) counts.update(subCounts) except Exception as ex: @@ -511,83 +506,85 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ my_logger.error('{}: Could not finish check on this property ({})'.format(prop_name, str(ex))) counts['exceptionPropCheck'] += 1 return resultList, counts - - # all other types... - for cnt, val in enumerate(propValueList): - appendStr = (('[' + str(cnt) + ']') if isCollection else '') - sub_item = prop_name + appendStr - - excerptPass = validateExcerpt(prop, val) - - if isinstance(val, str): - if val == '' and prop.Type.Permissions == 'OData.Permission/Read': - my_logger.warning('{}: Empty string found - Services should omit properties if not supported'.format(sub_item)) - nullValid = False - if val.lower() == 'null': - my_logger.warning('{}: "null" string found - Did you mean to use an actual null value?'.format(sub_item)) - nullValid = False - - if prop.Exists: - paramPass = propNullablePass = True - if val is None: - if propNullable: - my_logger.debug('Property {} is nullable and is null, so Nullable checking passes'.format(sub_item)) - else: - propNullablePass = False - - if isinstance(prop.Type, str) and 'Edm.' in prop.Type: - try: - paramPass = prop.Exists and prop.validate_basic(val, prop.Type) - except ValueError as e: - my_logger.error('{}: {}'.format(prop.Name, e)) # log this - paramPass = False - elif isinstance(prop.Type, RedfishType): - try: - paramPass = prop.Type.validate(prop.Value, prop.added_pattern) - except ValueError as e: - my_logger.error('{}: {}'.format(prop.Name, e)) # log this - paramPass = False - - if propRealType == 'entity': - paramPass = validateEntity(service, prop, val) - - - # Render our result - my_type = prop.Type.fulltype - - if all([paramPass, propMandatoryPass, propNullablePass, excerptPass]): - my_logger.verbose1("\tSuccess") - counts['pass'] += 1 - result_str = 'PASS' - if deprecatedPassOrSinceVersion is False: - result_str = 'Deprecated' - if isinstance(deprecatedPassOrSinceVersion, str): - result_str = 'Deprecated/{}'.format(deprecatedPassOrSinceVersion) - if not nullValid: - counts['invalidPropertyValue'] += 1 - result_str = 'WARN' - else: - my_logger.verbose1("\tFAIL") - counts['err.' + str(my_type)] += 1 - result_str = 'FAIL' - if not paramPass: - if prop.Type.IsMandatory: - counts['failMandatoryProp'] += 1 - else: - counts['failProp'] += 1 - elif not propMandatoryPass: - my_logger.error("{}: Mandatory prop does not exist".format(sub_item)) - counts['failMandatoryExist'] += 1 - elif not propNullablePass: - my_logger.error('{}: Property is null but is not Nullable'.format(sub_item)) - counts['failNullable'] += 1 - elif not excerptPass: - counts['errorExcerpt'] += 1 - result_str = 'errorExcerpt' - - resultList[sub_item] = ( - displayValue(val, sub_item if prop.Type.AutoExpand else None), displayType(prop.Type), - 'Yes' if prop.Exists else 'No', result_str) - - return resultList, counts + # Everything else... + else: + propValueList = prop.Value if prop.IsCollection else [prop.Value] + for cnt, val in enumerate(propValueList): + appendStr = (('[' + str(cnt) + ']') if prop.IsCollection else '') + sub_item = prop_name + appendStr + + excerptPass = validateExcerpt(prop, val) + + if isinstance(val, str): + if val == '' and prop.Type.Permissions == 'OData.Permission/Read': + my_logger.warning('{}: Empty string found - Services should omit properties if not supported'.format(sub_item)) + nullValid = False + if val.lower() == 'null': + my_logger.warning('{}: "null" string found - Did you mean to use an actual null value?'.format(sub_item)) + nullValid = False + + if prop.Exists: + paramPass = propNullablePass = True + if val is None: + if propNullable: + my_logger.debug('Property {} is nullable and is null, so Nullable checking passes'.format(sub_item)) + else: + propNullablePass = False + + if isinstance(prop.Type, str) and 'Edm.' in prop.Type: + try: + paramPass = prop.Exists and prop.validate_basic(val, prop.Type) + except ValueError as e: + my_logger.error('{}: {}'.format(prop.Name, e)) # log this + paramPass = False + elif isinstance(prop.Type, RedfishType): + try: + paramPass = prop.Type.validate(val, prop.added_pattern) + except ValueError as e: + my_logger.error('{}: {}'.format(prop.Name, e)) # log this + paramPass = False + + if propRealType == 'entity': + paramPass = validateEntity(service, prop, val) + + + + # Render our result + my_type = prop.Type.fulltype + + if all([paramPass, propMandatoryPass, propNullablePass, excerptPass]): + my_logger.verbose1("\tSuccess") + counts['pass'] += 1 + result_str = 'PASS' + if deprecatedPassOrSinceVersion is False: + result_str = 'Deprecated' + if isinstance(deprecatedPassOrSinceVersion, str): + result_str = 'Deprecated/{}'.format(deprecatedPassOrSinceVersion) + if not nullValid: + counts['invalidPropertyValue'] += 1 + result_str = 'WARN' + else: + my_logger.verbose1("\tFAIL") + counts['err.' + str(my_type)] += 1 + result_str = 'FAIL' + if not paramPass: + if prop.Type.IsMandatory: + counts['failMandatoryProp'] += 1 + else: + counts['failProp'] += 1 + elif not propMandatoryPass: + my_logger.error("{}: Mandatory prop does not exist".format(sub_item)) + counts['failMandatoryExist'] += 1 + elif not propNullablePass: + my_logger.error('{}: Property is null but is not Nullable'.format(sub_item)) + counts['failNullable'] += 1 + elif not excerptPass: + counts['errorExcerpt'] += 1 + result_str = 'errorExcerpt' + + resultList[sub_item] = ( + displayValue(val, sub_item if prop.Type.AutoExpand else None), displayType(prop.Type), + 'Yes' if prop.Exists else 'No', result_str) + + return resultList, counts \ No newline at end of file diff --git a/redfish_service_validator/validateResource.py b/redfish_service_validator/validateResource.py index 1517bc1..845d9b1 100644 --- a/redfish_service_validator/validateResource.py +++ b/redfish_service_validator/validateResource.py @@ -103,6 +103,7 @@ def validateSingleURI(service, URI, uriName='', expectedType=None, expectedJson= my_type = my_type.fulltype redfish_schema = service.catalog.getSchemaDocByClass(my_type) redfish_type = redfish_schema.getTypeInSchemaDoc(my_type) + redfish_obj = catalog.RedfishObject(redfish_type, 'Object', parent=parent).populate(me['payload']) if redfish_type else None if redfish_obj: @@ -174,8 +175,6 @@ def validateSingleURI(service, URI, uriName='', expectedType=None, expectedJson= messages['@odata.id'].result = 'FAIL' my_logger.error('URI {} does not match the following required URIs in Schema of {}'.format(odata_id, redfish_obj.Type)) - - if response and response.getheader('Allow'): allowed_responses = [x.strip().upper() for x in response.getheader('Allow').split(',')] if not redfish_obj.Type.CanInsert and 'POST' in allowed_responses: @@ -230,7 +229,7 @@ def validateSingleURI(service, URI, uriName='', expectedType=None, expectedJson= except Exception as ex: my_logger.verbose1('Exception caught while validating single URI', exc_info=1) my_logger.error('{}: Could not finish check on this property ({})'.format(prop_name, str(ex))) - messages[prop_name] = create_entry(prop_name, '', '', prop.Exists, 'exception') + messages[prop_name] = create_entry(prop_name, '', '', '...', 'exception') counts['exceptionPropCheck'] += 1 SchemaFullType, jsonData = me['fulltype'], me['payload'] @@ -312,7 +311,11 @@ def validateURITree(service, URI, uriName, expectedType=None, expectedJson=None, if validateSuccess and 'MessageRegistryFile.MessageRegistryFile' in thisobj.Type.getTypeTree(): # thisobj['Location'].Collection[0]['Uri'].Exists if 'Location' in thisobj: - for sub_obj in thisobj['Location'].collection: + if thisobj['Location'].IsCollection: + val_list = thisobj['Location'].Value + else: + val_list = [thisobj['Location'].Value] + for sub_obj in val_list: if 'Uri' in sub_obj: links.append(sub_obj) From c0c7855f6c17ae426993ee9c74d47aebe32a7c30 Mon Sep 17 00:00:00 2001 From: Tomas Date: Fri, 14 Jul 2023 08:31:16 -0500 Subject: [PATCH 4/7] Return appropriate objects when Collection values are not lists/arrays Signed-off-by: Tomas --- redfish_service_validator/catalog.py | 30 +++++++++++++++---- redfish_service_validator/validateRedfish.py | 16 ++++++---- redfish_service_validator/validateResource.py | 2 +- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/redfish_service_validator/catalog.py b/redfish_service_validator/catalog.py index d906571..596e9a9 100644 --- a/redfish_service_validator/catalog.py +++ b/redfish_service_validator/catalog.py @@ -565,7 +565,7 @@ def getBaseType(self): return type_obj.getBaseType() return 'none' - def isCollection(self): + def IsCollection(self): tree = [self.fulltype] my_type = self.parent_type while my_type: @@ -638,8 +638,8 @@ def validate(self, val, added_pattern=None): def as_json(self): return self.createObject().as_json() - def createObject(self): - return RedfishObject(self) + def createObject(self, name='Object'): + return RedfishObject(self, name) class RedfishProperty(object): @@ -833,9 +833,28 @@ def populate(self, payload, check=False, casted=False): populated_object.payload = payload if isinstance(payload, list): + populated_object.IsValid = populated_object.Type.IsCollection() + if not populated_object.IsValid: + my_logger.error("This object's type {} should be a Collection, but it's of type {}...".format(populated_object.Name, populated_object.Type)) + return populated_object populated_object.IsCollection = True - populated_object.Value = [self.populate(sub_item, check, casted) for sub_item in payload] + + my_new_type = stripCollection(populated_object.Type.fulltype) + + new_type_obj = populated_object.Type.catalog.getSchemaDocByClass(getNamespace(my_new_type)).getTypeInSchemaDoc(my_new_type) + + new_rf_object = RedfishObject(new_type_obj, populated_object.Name, populated_object.parent) + + populated_object.Value = [new_rf_object.populate(sub_item, check, casted) for sub_item in payload] return populated_object + else: + if populated_object.Type.IsCollection(): + if payload in [REDFISH_ABSENT, None]: + populated_object.Value = payload + return populated_object + else: + my_logger.error("This object {} should be a list, but it's of type {}...".format(populated_object.Name, type(payload).__name__)) + return populated_object # todo: redesign objects to have consistent variables, not only when populated # if populated, should probably just use another python class? @@ -852,7 +871,7 @@ def populate(self, payload, check=False, casted=False): # Only valid if we are a dictionary... # todo: see above None/REDFISH_ABSENT block if not populated_object.IsValid: - my_logger.error("This complex object {} should be a dictionary or None, but it's of type {}...".format(populated_object.Name, str(type(payload)))) + my_logger.error("This complex object {} should be a dictionary or None, but it's of type {}...".format(populated_object.Name, type(payload).__name__)) populated_object.HasValidUri = True populated_object.HasValidUriStrict = False populated_object.properties = {x: y.populate(REDFISH_ABSENT) for x, y in populated_object.properties.items()} @@ -911,6 +930,7 @@ def populate(self, payload, check=False, casted=False): if my_ns == my_ns_unversioned: my_ns = top_ns if my_ns not in populated_object.Type.Namespace: + # NOTE: This returns a Type object without IsPropertyType my_logger.verbose1(('Morphing Complex', my_ns, my_type, my_limit)) new_type_obj = populated_object.Type.catalog.getSchemaDocByClass(my_ns).getTypeInSchemaDoc('.'.join([my_ns, my_type])) populated_object = RedfishObject(new_type_obj, populated_object.Name, populated_object.parent).populate(sub_payload, check=check, casted=True) diff --git a/redfish_service_validator/validateRedfish.py b/redfish_service_validator/validateRedfish.py index 646f2b4..61719c0 100644 --- a/redfish_service_validator/validateRedfish.py +++ b/redfish_service_validator/validateRedfish.py @@ -2,7 +2,7 @@ from collections import Counter, OrderedDict from redfish_service_validator.catalog import REDFISH_ABSENT, MissingSchemaError, ExcerptTypes, get_fuzzy_property, RedfishObject, RedfishType -from redfish_service_validator.helper import getNamespace, getNamespaceUnversioned, getType, checkPayloadConformance +from redfish_service_validator.helper import getNamespace, getNamespaceUnversioned, getType, checkPayloadConformance, stripCollection import logging @@ -15,7 +15,12 @@ def validateExcerpt(prop, val): if base == 'entity': my_excerpt_type, my_excerpt_tags = prop.Type.excerptType, prop.Type.excerptTags - my_props = prop.Type.createObject().populate(val).properties + + my_new_type = stripCollection(prop.Type.fulltype) + + new_type_obj = prop.Type.catalog.getSchemaDocByClass(getNamespace(my_new_type)).getTypeInSchemaDoc(my_new_type) + + my_props = new_type_obj.createObject(prop.Name).populate(val).properties for name, innerprop in my_props.items(): if not innerprop.HasSchema: @@ -269,7 +274,7 @@ def displayType(propTypeObject, is_collection=False): :param is_collection: For collections: True if these types are for the collection; False if for a member :return: the simplified type to display """ - propRealType, propCollection = propTypeObject.getBaseType(), propTypeObject.isCollection() + propRealType, propCollection = propTypeObject.getBaseType(), propTypeObject.IsCollection() propType = propTypeObject.fulltype # Edm.* and other explicit types if propRealType == 'Edm.Boolean' or propRealType == 'Boolean': @@ -427,7 +432,7 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ # Note: consider http://docs.oasis-open.org/odata/odata-csdl-xml/v4.01/csprd01/odata-csdl-xml-v4.01-csprd01.html#_Toc472333112 # Note: make sure it checks each one # propCollectionType = PropertyDict.get('isCollection') - propRealType, isCollection = prop.Type.getBaseType(), prop.Type.isCollection() + propRealType, isCollection = prop.Type.getBaseType(), prop.Type.IsCollection() excerptPass = True if isCollection and prop.Value is None: @@ -447,7 +452,7 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ # rs-assumption: check @odata.count property # rs-assumption: check @odata.link property my_logger.verbose1("\tis Collection") - if prop.Value == 'n/a': + if prop.Value == REDFISH_ABSENT: resultList[prop_name] = ('Array (absent) {}'.format(len(prop.Value)), displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', 'PASS' if propMandatoryPass else 'FAIL') @@ -457,6 +462,7 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ my_logger.error('{}: property is expected to contain an array'.format(prop_name)) counts['failInvalidArray'] += 1 resultList[prop_name] = ('-', displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', 'FAIL') + return resultList, counts else: resultList[prop_name] = ('Array (size: {})'.format(len(prop.Value)), displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', '...') diff --git a/redfish_service_validator/validateResource.py b/redfish_service_validator/validateResource.py index 845d9b1..669827e 100644 --- a/redfish_service_validator/validateResource.py +++ b/redfish_service_validator/validateResource.py @@ -75,7 +75,7 @@ def validateSingleURI(service, URI, uriName='', expectedType=None, expectedJson= if expectedJson is None: ret = service.callResourceURI(URI) success, me['payload'], response, me['rtime'] = ret - me['rcode'] = response.status + me['rcode'] = response.status if response else -1 else: success, me['payload'], me['rcode'], me['rtime'] = True, expectedJson, -1, 0 response = None From bf9ee90a1597959d1c2d90f3c473628babe3fd2e Mon Sep 17 00:00:00 2001 From: Tomas Date: Thu, 20 Jul 2023 07:00:09 -0500 Subject: [PATCH 5/7] More errors for incorrect arrays, AutoExpanded logic, removed schema.py Consolidated schema.py into metadata.py Signed-off-by: Tomas --- .../RedfishServiceValidator.py | 6 +- redfish_service_validator/catalog.py | 46 ++- redfish_service_validator/metadata.py | 221 ++++++++++- redfish_service_validator/schema.py | 344 ------------------ redfish_service_validator/validateRedfish.py | 12 +- redfish_service_validator/validateResource.py | 5 +- 6 files changed, 262 insertions(+), 372 deletions(-) delete mode 100644 redfish_service_validator/schema.py diff --git a/redfish_service_validator/RedfishServiceValidator.py b/redfish_service_validator/RedfishServiceValidator.py index 942be80..bc73b72 100755 --- a/redfish_service_validator/RedfishServiceValidator.py +++ b/redfish_service_validator/RedfishServiceValidator.py @@ -9,11 +9,11 @@ import json from datetime import datetime import traceback +from redfish_service_validator.metadata import getSchemaDetails from redfish_service_validator.config import convert_config_to_args, convert_args_to_config from redfish_service_validator.validateResource import validateSingleURI, validateURITree -import redfish_service_validator.schema as schema from redfish_service_validator import tohtml, schema_pack, traverse -from urllib.parse import urlparse, urlunparse +from urllib.parse import urlparse from collections import Counter tool_version = '2.3.1' @@ -231,7 +231,7 @@ def main(argslist=None, configfile=None): my_logger.info("\n".join('{}: {} '.format(x, y) for x, y in sorted(finalCounts.items()))) # dump cache info to debug log - my_logger.debug('getSchemaDetails() -> {}'.format(schema.getSchemaDetails.cache_info())) + my_logger.debug('getSchemaDetails() -> {}'.format(getSchemaDetails.cache_info())) my_logger.debug('callResourceURI() -> {}'.format(currentService.callResourceURI.cache_info())) if not success: diff --git a/redfish_service_validator/catalog.py b/redfish_service_validator/catalog.py index 596e9a9..72dbe6c 100644 --- a/redfish_service_validator/catalog.py +++ b/redfish_service_validator/catalog.py @@ -26,6 +26,7 @@ VALID_ID_REGEX = '([A-Za-z0-9.!#$&-;=?\[\]_~])+' + # Excerpt definitions class ExcerptTypes(Enum): NEUTRAL = auto() @@ -33,14 +34,17 @@ class ExcerptTypes(Enum): ALLOWED = auto() EXCLUSIVE = auto() + excerpt_info_by_type = { 'Redfish.ExcerptCopy': ExcerptTypes.CONTAINS, 'Redfish.Excerpt': ExcerptTypes.ALLOWED, 'Redfish.ExcerptCopyOnly': ExcerptTypes.EXCLUSIVE } + allowed_annotations = ['odata', 'Redfish', 'Privileges', 'Message'] + def get_fuzzy_property(prop_name: str, jsondata: dict, allPropList=[]): """ Get property closest to the discovered property. @@ -81,6 +85,7 @@ class SchemaCatalog: From Catalog, you can get any Schema by it's filename, or its classes """ + # TODO: Generate documents on demand, not all at once def __init__(self, filepath: str, metadata: object = None): """Init @@ -364,9 +369,9 @@ def __init__(self, soup, owner: SchemaClass): if 'Collection(' in self.fulltype: my_fulltype = self.fulltype.replace('Collection(', "").replace(')', "") - self.Namespace, self.Type = getNamespace(my_fulltype), getType(my_fulltype) + self.Namespace, self.TypeName = getNamespace(my_fulltype), getType(my_fulltype) else: - self.Namespace, self.Type = getNamespace(self.fulltype), getType(self.fulltype) + self.Namespace, self.TypeName = getNamespace(self.fulltype), getType(self.fulltype) self.tags = {} for tag in self.type_soup.find_all(recursive=False): @@ -379,7 +384,7 @@ def __init__(self, soup, owner: SchemaClass): self.IsMandatory = self.tags.get('Redfish.Required') is not None self.IsNullable = self.type_soup.get("Nullable", "true") not in ["false", False, "False"] - self.AutoExpand = self.tags.get('OData.AutoExpand', None) is not None or self.tags.get('OData.AutoExpand'.lower(), None) is not None + self.AutoExpand = (self.tags.get('OData.AutoExpand') or self.tags.get('OData.AutoExpand'.lower())) is not None self.Deprecated = self.tags.get('Redfish.Deprecated') self.Revisions = self.tags.get('Redfish.Revisions') self.Excerpt = False @@ -421,7 +426,8 @@ def HasAdditional(self): HasAdditional = ( False if additionalElement is None else ( True if additionalElement.get("Bool", False) in ["True", "true", True] else False)) - if HasAdditional: return True + if HasAdditional: + return True return False @property @@ -576,6 +582,11 @@ def IsCollection(self): tree.append(my_type) break return any([re.match(r'Collection(.*)', typ) for typ in tree]) + + def getCollectionType(self): + my_new_type = stripCollection(self.fulltype) + new_type_obj = self.catalog.getSchemaDocByClass(getNamespace(my_new_type)).getTypeInSchemaDoc(my_new_type) + return new_type_obj def getProperties(self): """ @@ -661,6 +672,7 @@ def __init__(self, my_type, name="Property", parent=None): self.IsValid = False self.IsCollection = False self.InAnnotation = False + self.IsAutoExpanded = False self.SchemaExists = False self.Exists = False self.parent = parent @@ -676,6 +688,20 @@ def populate(self, val, check=False): eval_prop.Exists = val != REDFISH_ABSENT eval_prop.IsValid = True + + if isinstance(eval_prop.Type, str): + is_type_collection = 'Collection(' in eval_prop.Type + elif isinstance(eval_prop.Type, RedfishType): + is_type_collection = eval_prop.Type.IsCollection() + else: + raise ValueError('Type is not String or RedfishType') + if eval_prop.IsCollection and not is_type_collection: + my_logger.error('Property {} should not be a List'.format(self.Name)) + eval_prop.IsValid = False + elif not eval_prop.IsCollection and is_type_collection and val not in [None, REDFISH_ABSENT]: + my_logger.error('Collection Property {} is not a List'.format(self.Name)) + eval_prop.IsValid = False + if isinstance(eval_prop.Type, str) and 'Edm.' in eval_prop.Type and check: try: eval_prop.IsValid = eval_prop.Exists and RedfishProperty.validate_basic(eval_prop.Value, eval_prop.Type) @@ -809,8 +835,6 @@ def __contains__(self, item): def __init__(self, redfish_type: RedfishType, name="Object", parent=None): super().__init__(redfish_type, name, parent) self.payload = None - self.IsValid = False - self.IsCollection = False self.HasValidUri = False self.HasValidUriStrict = False self.properties = {} @@ -916,7 +940,7 @@ def populate(self, payload, check=False, casted=False): parent = parent.parent my_limit = parent.Type.Namespace - my_type = populated_object.Type.Type + my_type = populated_object.Type.TypeName top_ns = my_ns # get type order from bottom up of SchemaDoc @@ -968,7 +992,7 @@ def populate(self, payload, check=False, casted=False): if populated_object.HasValidUri and not populated_object.Type.IsNav: # pair our type, Id value, and odata.id value my_odata_split = my_odata_id.split('/') - my_type, my_id, my_uri_id = populated_object.Type.Type, sub_payload.get('Id'), my_odata_split[-1] + my_type, my_id, my_uri_id = populated_object.Type.TypeName, sub_payload.get('Id'), my_odata_split[-1] for schema_uri in allowable_uris: # regex URI check to confirm which URI @@ -1075,8 +1099,12 @@ def getLinks(self): if item.Type.IsNav: if isinstance(item.Value, list): for num, val in enumerate(item.Value): - new_link = item.populate(val) + # TODO: Along with example Excerpt and RedfishObject, replace following code with hypothetical RedfishType.getCollectionType + new_type_obj = item.Type.getCollectionType() + new_link = RedfishObject(new_type_obj, item.Name, item.parent).populate(val) new_link.Name = new_link.Name + '#{}'.format(num) + if item.Type.AutoExpand: + new_link.IsAutoExpanded = True links.append(new_link) else: links.append(item) diff --git a/redfish_service_validator/metadata.py b/redfish_service_validator/metadata.py index a067f56..bc20d35 100644 --- a/redfish_service_validator/metadata.py +++ b/redfish_service_validator/metadata.py @@ -5,9 +5,16 @@ import os import time from collections import Counter, OrderedDict, defaultdict -import redfish_service_validator.schema as schema +from collections import namedtuple +from bs4 import BeautifulSoup +from functools import lru_cache +import os.path + +from redfish_service_validator.helper import getNamespace, getNamespaceUnversioned + +import logging +my_logger = logging.getLogger(__name__) -# TODO: Use catalog.py instead of schema.py, remove uses of schema.py EDM_NAMESPACE = "http://docs.oasis-open.org/odata/ns/edm" EDMX_NAMESPACE = "http://docs.oasis-open.org/odata/ns/edmx" @@ -16,6 +23,7 @@ 'Schema', 'Singleton', 'Term', 'TypeDefinition'] EDMX_TAGS = ['DataServices', 'Edmx', 'Include', 'Reference'] + def bad_edm_tags(tag): return tag.namespace == EDM_NAMESPACE and tag.name not in EDM_TAGS @@ -92,10 +100,8 @@ def __init__(self, data, service, logger): self.elapsed_secs = time.time() - start self.schema_obj = None if data: - soup = schema.BeautifulSoup(data, "xml") - self.schema_obj = schema.rfSchema(soup, '$metadata', 'service') - self.md_soup = self.schema_obj.soup - self.service_refs = self.schema_obj.refs + self.md_soup = BeautifulSoup(data, "xml") + self.service_refs = getReferenceDetails(self.md_soup) self.success_get = True # set of namespaces included in $metadata self.metadata_namespaces = {k for k in self.service_refs.keys()} @@ -121,11 +127,10 @@ def __init__(self, data, service, logger): logger.debug('Metadata: bad_schema_uris = {}'.format(self.bad_schema_uris)) logger.debug('Metadata: bad_namespace_include = {}'.format(self.bad_namespace_include)) for ref in self.service_refs: + print(ref) name, uri = self.service_refs[ref] - self.schema_store[name] = schema.getSchemaObject(self.service, name, uri) - if self.schema_store[name] is not None: - for ref in self.schema_store[name].refs: - pass + success, soup, origin = getSchemaDetails(service, name, uri) + self.schema_store[name] = soup else: logger.warning('Metadata: getSchemaDetails() did not return success') @@ -192,7 +197,7 @@ def check_namespaces_in_schemas(self): if '#' in schema_uri: schema_uri, frag = k.split('#', 1) schema_type = os.path.basename(os.path.normpath(k)).strip('.xml').strip('_v1') - success, soup, _ = schema.getSchemaDetails(self.service, schema_type, schema_uri) + success, soup, _ = getSchemaDetails(self.service, schema_type, schema_uri) if success: for namespace in self.uri_to_namespaces[k]: if soup.find('Schema', attrs={'Namespace': namespace}) is None: @@ -309,3 +314,197 @@ def __repr__(self): def __reduce__(self): return self.__class__, (OrderedDict(self),) + +def storeSchemaToLocal(xml_data, origin, service): + """storeSchemaToLocal + + Moves data pulled from service/online to local schema storage + + Does NOT do so if preferonline is specified + + :param xml_data: data being transferred + :param origin: origin of xml pulled + """ + config = service.config + SchemaLocation = config['metadatafilepath'] + if not os.path.isdir(SchemaLocation): + os.makedirs(SchemaLocation) + if 'localFile' not in origin and '$metadata' not in origin: + __, xml_name = origin.rsplit('/', 1) + new_file = os.path.join(SchemaLocation, xml_name) + if not os.path.isfile(new_file): + with open(new_file, "w") as filehandle: + filehandle.write(xml_data) + my_logger.info('Writing online XML to file: {}'.format(xml_name)) + else: + my_logger.info('NOT writing online XML to file: {}'.format(xml_name)) + + +@lru_cache(maxsize=64) +def getSchemaDetails(service, SchemaType, SchemaURI): + """ + Find Schema file for given Namespace. + + param SchemaType: Schema Namespace, such as ServiceRoot + param SchemaURI: uri to grab schema, given LocalOnly is False + return: (success boolean, a Soup object, origin) + """ + my_logger.debug('getting Schema of {} {}'.format(SchemaType, SchemaURI)) + + if SchemaType is None: + return False, None, None + + if service is None: + return getSchemaDetailsLocal(SchemaType, SchemaURI, {}) + + elif service.active and getNamespace(SchemaType) in service.metadata.schema_store: + result = service.metadata.schema_store[getNamespace(SchemaType)] + if result is not None: + return True, result.soup, result.origin + + success, soup, origin = getSchemaDetailsLocal(SchemaType, SchemaURI, service.config) + if success: + return success, soup, origin + + xml_suffix = '_v1.xml' + + if (SchemaURI is not None) or (SchemaURI is not None and '/redfish/v1/$metadata' in SchemaURI): + # Get our expected Schema file here + # if success, generate Soup, then check for frags to parse + # start by parsing references, then check for the refLink + if '#' in SchemaURI: + base_schema_uri, frag = tuple(SchemaURI.rsplit('#', 1)) + else: + base_schema_uri, frag = SchemaURI, None + success, data, response, elapsed = service.callResourceURI(base_schema_uri) + if success: + soup = BeautifulSoup(data, "xml") + # if frag, look inside xml for real target as a reference + if frag is not None: + # prefer type over frag, truncated down + # using frag, check references + frag = getNamespace(SchemaType) + frag = frag.split('.', 1)[0] + refType, refLink = getReferenceDetails( + soup, name=base_schema_uri).get(frag, (None, None)) + if refLink is not None: + success, linksoup, newlink = getSchemaDetails(service, refType, refLink) + if success: + return True, linksoup, newlink + else: + my_logger.error( + "SchemaURI couldn't call reference link {} inside {}".format(frag, base_schema_uri)) + else: + my_logger.error( + "SchemaURI missing reference link {} inside {}".format(frag, base_schema_uri)) + # error reported; assume likely schema uri to allow continued validation + uri = 'http://redfish.dmtf.org/schemas/v1/{}{}'.format(frag, xml_suffix) + my_logger.info("Continue assuming schema URI for {} is {}".format(SchemaType, uri)) + return getSchemaDetails(service, SchemaType, uri) + else: + storeSchemaToLocal(data, base_schema_uri, service) + return True, soup, base_schema_uri + else: + my_logger.debug("SchemaURI called unsuccessfully: {}".format(base_schema_uri)) + return getSchemaDetailsLocal(SchemaType, SchemaURI, service.config) + + +def getSchemaDetailsLocal(SchemaType, SchemaURI, config): + """ + Find Schema file for given Namespace, from local directory + + param SchemaType: Schema Namespace, such as ServiceRoot + param SchemaURI: uri to grab schem (generate information from it) + return: (success boolean, a Soup object, origin) + """ + Alias = getNamespaceUnversioned(SchemaType) + SchemaLocation, SchemaSuffix = config['metadatafilepath'], '_v1.xml' + if SchemaURI is not None: + uriparse = SchemaURI.split('/')[-1].split('#') + xml = uriparse[0] + else: + my_logger.warning("SchemaURI was empty, must generate xml name from type {}".format(SchemaType)), + return getSchemaDetailsLocal(SchemaType, Alias + SchemaSuffix, config) + my_logger.debug(('local', SchemaType, SchemaURI, SchemaLocation + '/' + xml)) + filestring = Alias + SchemaSuffix if xml is None else xml + try: + # get file + with open(SchemaLocation + '/' + xml, "r") as filehandle: + data = filehandle.read() + + # get tags + soup = BeautifulSoup(data, "xml") + edmxTag = soup.find('Edmx', recursive=False) + parentTag = edmxTag.find('DataServices', recursive=False) + child = parentTag.find('Schema', recursive=False) + SchemaNamespace = child['Namespace'] + FoundAlias = SchemaNamespace.split(".")[0] + my_logger.debug(FoundAlias) + + if FoundAlias in Alias: + return True, soup, "localFile:" + SchemaLocation + '/' + filestring + + except FileNotFoundError: + # if we're looking for $metadata locally... ditch looking for it, go straight to file + if '/redfish/v1/$metadata' in SchemaURI and Alias != '$metadata': + my_logger.debug("Unable to find a xml of {} at {}, defaulting to {}".format(SchemaURI, SchemaLocation, Alias + SchemaSuffix)) + return getSchemaDetailsLocal(SchemaType, Alias + SchemaSuffix, config) + else: + my_logger.warning("Schema file {} not found in {}".format(filestring, SchemaLocation)) + if Alias == '$metadata': + my_logger.warning("If $metadata cannot be found, Annotations may be unverifiable") + except Exception as ex: + my_logger.error("A problem when getting a local schema has occurred {}".format(SchemaURI)) + my_logger.warning("output: ", exc_info=True) + return False, None, None + + +def check_redfish_extensions_alias(name, namespace, alias): + """ + Check that edmx:Include for Namespace RedfishExtensions has the expected 'Redfish' Alias attribute + :param name: the name of the resource + :param item: the edmx:Include item for RedfishExtensions + :return: bool + """ + if alias is None or alias != 'Redfish': + msg = ("In the resource {}, the {} namespace must have an alias of 'Redfish'. The alias is {}. " + + "This may cause properties of the form [PropertyName]@Redfish.TermName to be unrecognized.") + my_logger.error(msg.format(name, namespace, + 'missing' if alias is None else "'" + str(alias) + "'")) + return False + return True + + +def getReferenceDetails(soup, metadata_dict=None, name='xml'): + """ + Create a reference dictionary from a soup file + + param arg1: soup + param metadata_dict: dictionary of service metadata, compare with + return: dictionary + """ + includeTuple = namedtuple('include', ['Namespace', 'Uri']) + refDict = {} + + maintag = soup.find("Edmx", recursive=False) + reftags = maintag.find_all('Reference', recursive=False) + for ref in reftags: + includes = ref.find_all('Include', recursive=False) + for item in includes: + uri = ref.get('Uri') + ns, alias = (item.get(x) for x in ['Namespace', 'Alias']) + if ns is None or uri is None: + my_logger.error("Reference incorrect for: {}".format(item)) + continue + if alias is None: + alias = ns + refDict[alias] = includeTuple(ns, uri) + # Check for proper Alias for RedfishExtensions + if name == '$metadata' and ns.startswith('RedfishExtensions.'): + check_bool = check_redfish_extensions_alias(name, ns, alias) + + cntref = len(refDict) + if metadata_dict is not None: + refDict.update(metadata_dict) + my_logger.debug("METADATA: References generated from {}: {} out of {}".format(name, cntref, len(refDict))) + return refDict \ No newline at end of file diff --git a/redfish_service_validator/schema.py b/redfish_service_validator/schema.py deleted file mode 100644 index 01fd04c..0000000 --- a/redfish_service_validator/schema.py +++ /dev/null @@ -1,344 +0,0 @@ -# Copyright Notice: -# Copyright 2016-2020 DMTF. All rights reserved. -# License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Service-Validator/blob/master/LICENSE.md - -from collections import namedtuple -from re import split -from bs4 import BeautifulSoup -from functools import lru_cache -import os.path - -from redfish_service_validator.helper import getType, getNamespace, getNamespaceUnversioned, getVersion, splitVersionString - -import logging -my_logger = logging.getLogger(__name__) - -# TODO: In metadata.py, use catalog.py instead of schema.py. Remove file schema.py. - -def storeSchemaToLocal(xml_data, origin, service): - """storeSchemaToLocal - - Moves data pulled from service/online to local schema storage - - Does NOT do so if preferonline is specified - - :param xml_data: data being transferred - :param origin: origin of xml pulled - """ - config = service.config - SchemaLocation = config['metadatafilepath'] - if not os.path.isdir(SchemaLocation): - os.makedirs(SchemaLocation) - if 'localFile' not in origin and '$metadata' not in origin: - __, xml_name = origin.rsplit('/', 1) - new_file = os.path.join(SchemaLocation, xml_name) - if not os.path.isfile(new_file): - with open(new_file, "w") as filehandle: - filehandle.write(xml_data) - my_logger.info('Writing online XML to file: {}'.format(xml_name)) - else: - my_logger.info('NOT writing online XML to file: {}'.format(xml_name)) - - -@lru_cache(maxsize=64) -def getSchemaDetails(service, SchemaType, SchemaURI): - """ - Find Schema file for given Namespace. - - param SchemaType: Schema Namespace, such as ServiceRoot - param SchemaURI: uri to grab schema, given LocalOnly is False - return: (success boolean, a Soup object, origin) - """ - my_logger.debug('getting Schema of {} {}'.format(SchemaType, SchemaURI)) - - if SchemaType is None: - return False, None, None - - if service is None: - return getSchemaDetailsLocal(SchemaType, SchemaURI, {}) - - - elif service.active and getNamespace(SchemaType) in service.metadata.schema_store: - result = service.metadata.schema_store[getNamespace(SchemaType)] - if result is not None: - return True, result.soup, result.origin - - success, soup, origin = getSchemaDetailsLocal(SchemaType, SchemaURI, service.config) - if success: - return success, soup, origin - - xml_suffix = '_v1.xml' - - if (SchemaURI is not None) or (SchemaURI is not None and '/redfish/v1/$metadata' in SchemaURI): - # Get our expected Schema file here - # if success, generate Soup, then check for frags to parse - # start by parsing references, then check for the refLink - if '#' in SchemaURI: - base_schema_uri, frag = tuple(SchemaURI.rsplit('#', 1)) - else: - base_schema_uri, frag = SchemaURI, None - success, data, response, elapsed = service.callResourceURI(base_schema_uri) - if success: - soup = BeautifulSoup(data, "xml") - # if frag, look inside xml for real target as a reference - if frag is not None: - # prefer type over frag, truncated down - # using frag, check references - frag = getNamespace(SchemaType) - frag = frag.split('.', 1)[0] - refType, refLink = getReferenceDetails( - soup, name=base_schema_uri).get(frag, (None, None)) - if refLink is not None: - success, linksoup, newlink = getSchemaDetails(service, refType, refLink) - if success: - return True, linksoup, newlink - else: - my_logger.error( - "SchemaURI couldn't call reference link {} inside {}".format(frag, base_schema_uri)) - else: - my_logger.error( - "SchemaURI missing reference link {} inside {}".format(frag, base_schema_uri)) - # error reported; assume likely schema uri to allow continued validation - uri = 'http://redfish.dmtf.org/schemas/v1/{}{}'.format(frag, xml_suffix) - my_logger.info("Continue assuming schema URI for {} is {}".format(SchemaType, uri)) - return getSchemaDetails(service, SchemaType, uri) - else: - storeSchemaToLocal(data, base_schema_uri, service) - return True, soup, base_schema_uri - else: - my_logger.debug("SchemaURI called unsuccessfully: {}".format(base_schema_uri)) - return getSchemaDetailsLocal(SchemaType, SchemaURI, service.config) - - -def getSchemaDetailsLocal(SchemaType, SchemaURI, config): - """ - Find Schema file for given Namespace, from local directory - - param SchemaType: Schema Namespace, such as ServiceRoot - param SchemaURI: uri to grab schem (generate information from it) - return: (success boolean, a Soup object, origin) - """ - Alias = getNamespaceUnversioned(SchemaType) - SchemaLocation, SchemaSuffix = config['metadatafilepath'], '_v1.xml' - if SchemaURI is not None: - uriparse = SchemaURI.split('/')[-1].split('#') - xml = uriparse[0] - else: - my_logger.warning("SchemaURI was empty, must generate xml name from type {}".format(SchemaType)), - return getSchemaDetailsLocal(SchemaType, Alias + SchemaSuffix, config) - my_logger.debug(('local', SchemaType, SchemaURI, SchemaLocation + '/' + xml)) - filestring = Alias + SchemaSuffix if xml is None else xml - try: - # get file - with open(SchemaLocation + '/' + xml, "r") as filehandle: - data = filehandle.read() - - # get tags - soup = BeautifulSoup(data, "xml") - edmxTag = soup.find('Edmx', recursive=False) - parentTag = edmxTag.find('DataServices', recursive=False) - child = parentTag.find('Schema', recursive=False) - SchemaNamespace = child['Namespace'] - FoundAlias = SchemaNamespace.split(".")[0] - my_logger.debug(FoundAlias) - - if FoundAlias in Alias: - return True, soup, "localFile:" + SchemaLocation + '/' + filestring - - except FileNotFoundError: - # if we're looking for $metadata locally... ditch looking for it, go straight to file - if '/redfish/v1/$metadata' in SchemaURI and Alias != '$metadata': - my_logger.debug("Unable to find a xml of {} at {}, defaulting to {}".format(SchemaURI, SchemaLocation, Alias + SchemaSuffix)) - return getSchemaDetailsLocal(SchemaType, Alias + SchemaSuffix, config) - else: - my_logger.warning("Schema file {} not found in {}".format(filestring, SchemaLocation)) - if Alias == '$metadata': - my_logger.warning("If $metadata cannot be found, Annotations may be unverifiable") - except Exception as ex: - my_logger.error("A problem when getting a local schema has occurred {}".format(SchemaURI)) - my_logger.warning("output: ", exc_info=True) - return False, None, None - - -def check_redfish_extensions_alias(name, namespace, alias): - """ - Check that edmx:Include for Namespace RedfishExtensions has the expected 'Redfish' Alias attribute - :param name: the name of the resource - :param item: the edmx:Include item for RedfishExtensions - :return: bool - """ - if alias is None or alias != 'Redfish': - msg = ("In the resource {}, the {} namespace must have an alias of 'Redfish'. The alias is {}. " + - "This may cause properties of the form [PropertyName]@Redfish.TermName to be unrecognized.") - my_logger.error(msg.format(name, namespace, - 'missing' if alias is None else "'" + str(alias) + "'")) - return False - return True - - -def getReferenceDetails(soup, metadata_dict=None, name='xml'): - """ - Create a reference dictionary from a soup file - - param arg1: soup - param metadata_dict: dictionary of service metadata, compare with - return: dictionary - """ - includeTuple = namedtuple('include', ['Namespace', 'Uri']) - refDict = {} - - maintag = soup.find("Edmx", recursive=False) - reftags = maintag.find_all('Reference', recursive=False) - for ref in reftags: - includes = ref.find_all('Include', recursive=False) - for item in includes: - uri = ref.get('Uri') - ns, alias = (item.get(x) for x in ['Namespace', 'Alias']) - if ns is None or uri is None: - my_logger.error("Reference incorrect for: {}".format(item)) - continue - if alias is None: - alias = ns - refDict[alias] = includeTuple(ns, uri) - # Check for proper Alias for RedfishExtensions - if name == '$metadata' and ns.startswith('RedfishExtensions.'): - check_bool = check_redfish_extensions_alias(name, ns, alias) - - cntref = len(refDict) - if metadata_dict is not None: - refDict.update(metadata_dict) - my_logger.debug("References generated from {}: {} out of {}".format(name, cntref, len(refDict))) - return refDict - - -class rfSchema: - def __init__(self, soup, context, origin, metadata=None, name='xml'): - self.soup = soup - self.refs = getReferenceDetails(soup, metadata, name) - self.context = context - self.origin = origin - self.name = name - - def getSchemaFromReference(self, namespace): - """getSchemaFromReference - - Get SchemaObj from generated references - - :param namespace: Namespace of reference - """ - tup = self.refs.get(namespace) - tupVersionless = self.refs.get(getNamespace(namespace)) - if tup is None: - if tupVersionless is None: - my_logger.warning('No such reference {} in {}'.format(namespace, self.origin)) - return None - else: - tup = tupVersionless - my_logger.warning('No such reference {} in {}, using unversioned'.format(namespace, self.origin)) - typ, uri = tup - newSchemaObj = getSchemaObject(typ, uri) - return newSchemaObj - - def getTypeTagInSchema(self, currentType, tagType=['EntityType', 'ComplexType']): - """getTypeTagInSchema - - Get type tag in schema - - :param currentType: type string - :param tagType: Array or single string containing the xml tag name - """ - pnamespace, ptype = getNamespace(currentType), getType(currentType) - soup = self.soup - - currentSchema = soup.find( - 'Schema', attrs={'Namespace': pnamespace}) - - if currentSchema is None: - return None - - currentEntity = currentSchema.find(tagType, attrs={'Name': ptype}, recursive=False) - - return currentEntity - - def getParentType(self, currentType, tagType=['EntityType', 'ComplexType']): - """getParentType - - Get parent of this Entity/ComplexType - - :param currentType: type string - :param tagType: Array or single string containing the xml tag name - """ - currentType = currentType.replace('#', '') - typetag = self.getTypeTagInSchema(currentType, tagType) - if typetag is not None: - currentType = typetag.get('BaseType') - if currentType is None: - return False, None, None - typetag = self.getTypeTagInSchema(currentType, tagType) - if typetag is not None: - return True, self, currentType - else: - namespace = getNamespace(currentType) - schemaObj = self.getSchemaFromReference(namespace) - if schemaObj is None: - return False, None, None - propSchema = schemaObj.soup.find( - 'Schema', attrs={'Namespace': namespace}) - if propSchema is None: - return False, None, None - return True, schemaObj, currentType - else: - return False, None, None - - def getHighestType(self, acquiredtype: str, limit=None): - """getHighestType - - get Highest possible version for given type - - :param acquiredtype: Type available - :param limit: Version string limit (full namespace or just version 'v1_x_x') - """ - typelist = list() - - if limit is not None: - if getVersion(limit) is None: - if 'Collection' not in limit: - my_logger.warning('Limiting namespace has no version, erasing: {}'.format(limit)) - else: - my_logger.info('Limiting namespace has no version, erasing: {}'.format(limit)) - limit = None - else: - limit = getVersion(limit) - - for schema in self.soup.find_all('Schema'): - newNamespace = schema.get('Namespace') - if limit is not None: - if getVersion(newNamespace) is None: - continue - if splitVersionString(newNamespace) > splitVersionString(limit): - continue - if schema.find(['EntityType', 'ComplexType'], attrs={'Name': getType(acquiredtype)}, recursive=False): - typelist.append(splitVersionString(newNamespace)) - - if len(typelist) > 1: - for ns in reversed(sorted(typelist)): - my_logger.debug( - "{} {}".format(ns, getType(acquiredtype))) - acquiredtype = getNamespaceUnversioned(acquiredtype) + '.v{}_{}_{}'.format(*ns) + '.' + getType(acquiredtype) - return acquiredtype - return acquiredtype - - -@lru_cache(maxsize=64) -def getSchemaObject(service, typename, uri, metadata=None): - """getSchemaObject - - Wrapper for getting an rfSchema object - - :param typename: Type with namespace of schema - :param uri: Context/URI of metadata/schema containing reference to namespace - :param metadata: parent refs of service - """ - success, soup, origin = getSchemaDetails(service, typename, uri) - - return rfSchema(soup, uri, origin, metadata=metadata, name=typename) if success else None diff --git a/redfish_service_validator/validateRedfish.py b/redfish_service_validator/validateRedfish.py index 61719c0..c25ea53 100644 --- a/redfish_service_validator/validateRedfish.py +++ b/redfish_service_validator/validateRedfish.py @@ -99,7 +99,7 @@ def validateEntity(service, prop, val, parentURI=""): """ Validates an entity based on its uri given """ - name, val, autoExpand = prop.Name, val, prop.Type.AutoExpand + name, val, autoExpand = prop.Name, val, prop.IsAutoExpanded excerptType = prop.Type.excerptType if prop.Type.Excerpt else ExcerptTypes.NEUTRAL my_logger.debug('validateEntity: name = {}'.format(name)) @@ -435,6 +435,11 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ propRealType, isCollection = prop.Type.getBaseType(), prop.Type.IsCollection() excerptPass = True + if not isCollection and isinstance(prop.Value, list): + my_logger.error('{}: Value of property is an array but is not a Collection'.format(prop_name)) + counts['failInvalidArray'] += 1 + return {prop_name: ( '-', displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', 'FAIL')}, counts + if isCollection and prop.Value is None: # illegal for a collection to be null if 'EventDestination.v1_0_0.HttpHeaderProperty' == str(prop.Type.fulltype): @@ -474,6 +479,9 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ counts['failMandatoryExist'] += 1 result_str = 'FAIL' + if not prop.Exists: + return resultList, counts + if prop.IsCollection: resultList[prop_name] = ('Array (size: {})'.format(len(prop.Value)), displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', result_str) object_list = prop.Value @@ -590,7 +598,7 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ result_str = 'errorExcerpt' resultList[sub_item] = ( - displayValue(val, sub_item if prop.Type.AutoExpand else None), displayType(prop.Type), + displayValue(val, sub_item if prop.IsAutoExpanded else None), displayType(prop.Type), 'Yes' if prop.Exists else 'No', result_str) return resultList, counts \ No newline at end of file diff --git a/redfish_service_validator/validateResource.py b/redfish_service_validator/validateResource.py index 669827e..fa0c4bf 100644 --- a/redfish_service_validator/validateResource.py +++ b/redfish_service_validator/validateResource.py @@ -341,7 +341,7 @@ def validateURITree(service, URI, uriName, expectedType=None, expectedJson=None, if link.Type.Excerpt: continue - if any(x in str(link.parent.Type) or x in link.Name for x in ['RelatedItem', 'Redundancy', 'Links', 'OriginOfCondition']) and not link.Type.AutoExpand: + if any(x in str(link.parent.Type) or x in link.Name for x in ['RelatedItem', 'Redundancy', 'Links', 'OriginOfCondition']) and not link.IsAutoExpanded: refLinks.append((link, thisobj)) continue if link_destination in allLinks: @@ -364,7 +364,7 @@ def validateURITree(service, URI, uriName, expectedType=None, expectedJson=None, counts['repeat'] += 1 continue - if link.Type is not None and link.Type.AutoExpand: + if link.Type is not None and link.IsAutoExpanded: returnVal = validateURITree(service, link_destination, uriName + ' -> ' + link.Name, link.Type, link.Value, thisobj, allLinks, link.InAnnotation) else: returnVal = validateURITree(service, link_destination, uriName + ' -> ' + link.Name, parent=parent, allLinks=allLinks, inAnnotation=link.InAnnotation) @@ -411,7 +411,6 @@ def validateURITree(service, URI, uriName, expectedType=None, expectedJson=None, else: continue - my_link_type = link.Type.fulltype success, my_data, _, _ = service.callResourceURI(link_destination) # Using None instead of refparent simply because the parent is not where the link comes from From fe943a835d0eb4796da943fb74070966ec70701f Mon Sep 17 00:00:00 2001 From: Tomas Date: Fri, 21 Jul 2023 12:12:20 -0500 Subject: [PATCH 6/7] Removed redundant error count, extra print, raise on exit Signed-off-by: Tomas --- RedfishServiceValidator.py | 1 + redfish_service_validator/RedfishServiceValidator.py | 1 + redfish_service_validator/metadata.py | 1 - redfish_service_validator/validateRedfish.py | 2 -- 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/RedfishServiceValidator.py b/RedfishServiceValidator.py index d901d86..9659f25 100755 --- a/RedfishServiceValidator.py +++ b/RedfishServiceValidator.py @@ -11,3 +11,4 @@ sys.exit(main()) except Exception as e: my_logger.exception("Program finished prematurely: %s", e) + raise diff --git a/redfish_service_validator/RedfishServiceValidator.py b/redfish_service_validator/RedfishServiceValidator.py index 505f58d..f3a2566 100755 --- a/redfish_service_validator/RedfishServiceValidator.py +++ b/redfish_service_validator/RedfishServiceValidator.py @@ -254,3 +254,4 @@ def main(): sys.exit(main()) except Exception as e: my_logger.exception("Program finished prematurely: %s", e) + raise diff --git a/redfish_service_validator/metadata.py b/redfish_service_validator/metadata.py index bc20d35..73b3955 100644 --- a/redfish_service_validator/metadata.py +++ b/redfish_service_validator/metadata.py @@ -127,7 +127,6 @@ def __init__(self, data, service, logger): logger.debug('Metadata: bad_schema_uris = {}'.format(self.bad_schema_uris)) logger.debug('Metadata: bad_namespace_include = {}'.format(self.bad_namespace_include)) for ref in self.service_refs: - print(ref) name, uri = self.service_refs[ref] success, soup, origin = getSchemaDetails(service, name, uri) self.schema_store[name] = soup diff --git a/redfish_service_validator/validateRedfish.py b/redfish_service_validator/validateRedfish.py index c25ea53..0f450f4 100644 --- a/redfish_service_validator/validateRedfish.py +++ b/redfish_service_validator/validateRedfish.py @@ -461,8 +461,6 @@ def checkPropertyConformance(service, prop_name, prop, parent_name=None, parent_ resultList[prop_name] = ('Array (absent) {}'.format(len(prop.Value)), displayType(prop.Type, is_collection=True), 'Yes' if prop.Exists else 'No', 'PASS' if propMandatoryPass else 'FAIL') - my_logger.error("{}: Mandatory prop does not exist".format(prop_name)) - counts['failMandatoryExist'] += 1 elif not isinstance(prop.Value, list): my_logger.error('{}: property is expected to contain an array'.format(prop_name)) counts['failInvalidArray'] += 1 From 6cb66019cd1ca376d46bfe955b948104cd5f999e Mon Sep 17 00:00:00 2001 From: Tomas Date: Thu, 10 Aug 2023 10:05:02 -0500 Subject: [PATCH 7/7] Added proper checks for Excerpts Signed-off-by: Tomas --- redfish_service_validator/catalog.py | 3 +++ redfish_service_validator/validateResource.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/redfish_service_validator/catalog.py b/redfish_service_validator/catalog.py index 72dbe6c..91ed0ee 100644 --- a/redfish_service_validator/catalog.py +++ b/redfish_service_validator/catalog.py @@ -673,6 +673,7 @@ def __init__(self, my_type, name="Property", parent=None): self.IsCollection = False self.InAnnotation = False self.IsAutoExpanded = False + self.IsExcerpt = False self.SchemaExists = False self.Exists = False self.parent = parent @@ -1105,6 +1106,8 @@ def getLinks(self): new_link.Name = new_link.Name + '#{}'.format(num) if item.Type.AutoExpand: new_link.IsAutoExpanded = True + if item.Type.Excerpt: + new_link.IsExcerpt = True links.append(new_link) else: links.append(item) diff --git a/redfish_service_validator/validateResource.py b/redfish_service_validator/validateResource.py index fa0c4bf..9374b61 100644 --- a/redfish_service_validator/validateResource.py +++ b/redfish_service_validator/validateResource.py @@ -339,7 +339,7 @@ def validateURITree(service, URI, uriName, expectedType=None, expectedJson=None, continue link_destination = link.Value.get('@odata.id', link.Value.get('Uri')) - if link.Type.Excerpt: + if link.IsExcerpt or link.Type.Excerpt: continue if any(x in str(link.parent.Type) or x in link.Name for x in ['RelatedItem', 'Redundancy', 'Links', 'OriginOfCondition']) and not link.IsAutoExpanded: refLinks.append((link, thisobj)) @@ -386,7 +386,7 @@ def validateURITree(service, URI, uriName, expectedType=None, expectedJson=None, my_logger.warning('Link is None, does it exist?') continue link_destination = link.Value.get('@odata.id', link.Value.get('Uri')) - if link.Type.Excerpt: + if link.IsExcerpt or link.Type.Excerpt: continue elif link_destination is None: errmsg = 'Referenced URI for NavigationProperty is missing {} {} {}'.format(link_destination, link.Name, link.parent)