diff --git a/README.md b/README.md index 5e41424..665aa4b 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ The Tool has an option to ignore SSL certificate check if certificate is not ins UseSSL = \ +OemCheck = \ Specify if we want to check OEM properties + CertificateCheck = \ CertificateBundle = ca_bundle Specify a bundle (file or directory) with certificates of trusted CAs. See [SelfSignedCerts.md](https://github.com/DMTF/Redfish-Service-Validator/blob/master/SelfSignedCerts.md) for tips on creating the bundle. diff --git a/RedfishServiceValidator.py b/RedfishServiceValidator.py index 6539a39..8ad4048 100644 --- a/RedfishServiceValidator.py +++ b/RedfishServiceValidator.py @@ -3,21 +3,22 @@ # License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/Redfish-Service-Validator/blob/master/LICENSE.md import argparse -import configparser -import io import os import sys import re -from datetime import datetime -from collections import Counter, OrderedDict import logging import json +from io import StringIO +from datetime import datetime +from collections import Counter, OrderedDict + +import traverseService as rst + from simpletypes import * from traverseService import AuthenticationError from tohtml import renderHtml, writeHtml -import traverseService as rst from metadata import setup_schema_pack tool_version = '1.1.1' @@ -31,29 +32,25 @@ def verboseout(self, message, *args, **kws): self._log(VERBO_NUM, message, args, **kws) logging.Logger.verboseout = verboseout -attributeRegistries = dict() +def validateActions(name: str, val: dict, propTypeObj: rst.PropType, payloadType: str): + """validateActions -def validateActions(name, val, propTypeObj, payloadType): - # checks for all Schema parents and gets their Action tags, does validation - # info: what tags are we getting, treat them as if they were properties in a complex - # error: broken/missing action, largely the only problem that's validateable - # warn: action is missing something, but is fine or not mandatory - """ - Validates an action dict (val) + Validates actions dict + + :param name: Identity of the property + :type name: str + :param val: Dictionary of the Actions + :type val: dict + :param propTypeObj: TypeObject of the Actions + :type propTypeObj: rst.PropType + :param payloadType: Payload type of the owner of Actions + :type payloadType: str """ actionMessages, actionCounts = OrderedDict(), Counter() - # traverse through all parent types to discover Action tags - success, baseSoup, baseRefs, baseType = True, propTypeObj.soup, propTypeObj.refs, payloadType - actionsDict = dict() - while success: - SchemaNamespace = rst.getNamespace(baseType) - innerschema = baseSoup.find('Schema', attrs={'Namespace': SchemaNamespace}) - actions = innerschema.find_all('Action') - for act in actions: - keyname = '#{}.{}'.format(SchemaNamespace, act['Name']) - actionsDict[keyname] = act - success, baseSoup, baseRefs, baseType = rst.getParentType(baseSoup, baseRefs, baseType, 'EntityType') + + parentTypeObj = rst.PropType(payloadType, propTypeObj.schemaObj) + actionsDict = {act.name: act.actTag for act in parentTypeObj.getActions()} # For each action found, check action dictionary for existence and conformance # No action is required unless specified, target is not required unless specified @@ -70,13 +67,13 @@ def validateActions(name, val, propTypeObj, payloadType): elif not isinstance(target, str): actPass = False rsvLogger.error('{}: target for action is malformed; expected string, got {}' - .format(name + '.' + k, str(type(target)).strip('<>'))) - # check for unexpected properties + .format(name + '.' + k, str(type(target)).strip('<>'))) + # check for unexpected properties for prop in actionDecoded: if prop not in ['target', 'title', '@Redfish.ActionInfo'] and '@Redfish.AllowableValues' not in prop: actPass = False rsvLogger.error('{}: Property "{}" is not allowed in actions property. Allowed properties are "{}", "{}", "{}" and "{}"' - .format(name + '.' + k, prop, 'target', 'title', '@Redfish.ActionInfo', '*@Redfish.AllowableValues')) + .format(name + '.' + k, prop, 'target', 'title', '@Redfish.ActionInfo', '*@Redfish.AllowableValues')) else: # if actionsDict[k].find('annotation', {'term': 'Redfish.Required'}): @@ -85,9 +82,9 @@ def validateActions(name, val, propTypeObj, payloadType): else: rsvLogger.debug('{}: action not found, is not mandatory'.format(name + '.' + k)) actionMessages[name + '.' + k] = ( - 'Action', '-', - 'Yes' if actionDecoded != 'n/a' else 'No', - 'PASS' if actPass else 'FAIL') + 'Action', '-', + 'Yes' if actionDecoded != 'n/a' else 'No', + 'PASS' if actPass else 'FAIL') if actPass: actionCounts['pass'] += 1 else: @@ -95,14 +92,12 @@ def validateActions(name, val, propTypeObj, payloadType): return actionMessages, actionCounts -def validateEntity(name, val, propType, propCollectionType, soup, refs, autoExpand, parentURI=""): - # info: what are we looking for (the type), what are we getting (the uri), does the uri make sense based on type (does not do this yet) - # error: this type is bad, could not get resource, could not find the type, no reference, cannot construct type (doesn't do this yet) - # debug: what types do we have, what reference did we get back +def validateEntity(name: str, val: dict, propType: str, propCollectionType: str, schemaObj, autoExpand, parentURI=""): """ Validates an entity based on its uri given """ rsvLogger.debug('validateEntity: name = {}'.format(name)) + # check for required @odata.id if '@odata.id' not in val: if autoExpand: @@ -125,20 +120,22 @@ def validateEntity(name, val, propType, propCollectionType, soup, refs, autoExpa rsvLogger.debug('(success, uri, status, delay) = {}, (propType, propCollectionType) = {}, data = {}' .format((success, uri, status, delay), (propType, propCollectionType), data)) # if the reference is a Resource, save us some trouble as most/all basetypes are Resource - if propCollectionType == 'Resource.Item' or propType in ['Resource.ResourceCollection', 'Resource.Item'] and success: - paramPass = success - elif success: + generics = ['Resource.ItemOrCollection', 'Resource.ResourceCollection', 'Resource.Item'] + if propCollectionType in generics or propType in generics and success: + return True + if success: # Attempt to grab an appropriate type to test against and its schema # Default lineup: payload type, collection type, property type currentType = data.get('@odata.type', propCollectionType) if currentType is None: currentType = propType + soup, refs = schemaObj.soup, schemaObj.refs baseLink = refs.get(rst.getNamespace(propCollectionType if propCollectionType is not None else propType)) if soup.find('Schema', attrs={'Namespace': rst.getNamespace(currentType)}) is not None: - - success, baseSoup = True, soup + success, baseObj = True, schemaObj elif baseLink is not None: - success, baseSoup, uri = rst.getSchemaDetails(*baseLink) + baseObj = schemaObj.getSchemaFromReference(rst.getNamespaceUnversioned(currentType)) + success = schemaObj is not None else: success = False rsvLogger.debug('success = {}, currentType = {}, baseLink = {}'.format(success, currentType, baseLink)) @@ -147,11 +144,10 @@ def validateEntity(name, val, propType, propCollectionType, soup, refs, autoExpa if currentType is not None and success: currentType = currentType.replace('#', '') - baseRefs = rst.getReferenceDetails(baseSoup, refs, uri) allTypes = [] while currentType not in allTypes and success: allTypes.append(currentType) - success, baseSoup, baseRefs, currentType = rst.getParentType(baseSoup, baseRefs, currentType, 'EntityType') + success, schemaObj, currentType = baseObj.getParentType(currentType, 'EntityType') rsvLogger.debug('success = {}, currentType = {}'.format(success, currentType)) rsvLogger.debug('propType = {}, propCollectionType = {}, allTypes = {}' @@ -169,48 +165,29 @@ def validateEntity(name, val, propType, propCollectionType, soup, refs, autoExpa return paramPass -def validateComplex(name, val, propTypeObj, payloadType, attrRegistryId): - # one of the more complex validation methods, but is similar to validateSingleURI - # info: treat this like an individual payload, where is it, what is going on, perhaps reuse same code by moving it to helper - # warn: lacks an odata type, defaulted to highest type (this would happen during type gen) - # error: this isn't a dict, these properties aren't good/missing/etc, just like a payload - # debug: what are the properties we are looking for, what vals, etc (this is genned during checkPropertyConformance) - +def validateComplex(name, val, propComplexObj, payloadType, attrRegistryId): """ Validate a complex property """ rsvLogger.verboseout('\t***going into Complex') if not isinstance(val, dict): rsvLogger.error(name + ': Complex item not a dictionary') # Printout FORMAT - return False, None, None + return False, None, None, None # Check inside of complexType, treat it like an Entity complexMessages = OrderedDict() complexCounts = Counter() propList = list() - serviceRefs = rst.currentService.metadata.get_service_refs() - serviceSchemaSoup = rst.currentService.metadata.get_soup() - if serviceSchemaSoup is not None: - successService, additionalProps = rst.getAnnotations(serviceSchemaSoup, serviceRefs, val) - propSoup, propRefs = serviceSchemaSoup, serviceRefs - for prop in additionalProps: - propMessages, propCounts = checkPropertyConformance(propSoup, prop.name, prop.propDict, val, propRefs, - ParentItem=name) - complexMessages.update(propMessages) - complexCounts.update(propCounts) - + if 'OemObject' in propComplexObj.typeobj.fulltype: + rsvLogger.error('{}: OemObjects are required to be typecast with @odata.type'.format(str(name))) + return False, complexCounts, complexMessages + for prop in propComplexObj.getResourceProperties(): + propMessages, propCounts = checkPropertyConformance(propComplexObj.schemaObj, prop.name, prop.propDict, val, ParentItem=name) + complexMessages.update(propMessages) + complexCounts.update(propCounts) - node = propTypeObj - while node is not None: - propList, propSoup, propRefs = node.propList, node.soup, node.refs - for prop in propList: - propMessages, propCounts = checkPropertyConformance(propSoup, prop.name, prop.propDict, val, propRefs, - ParentItem=name) - complexMessages.update(propMessages) - complexCounts.update(propCounts) - node = node.parent successPayload, odataMessages = checkPayloadConformance('', val, ParentItem=name) complexMessages.update(odataMessages) if not successPayload: @@ -218,55 +195,25 @@ def validateComplex(name, val, propTypeObj, payloadType, attrRegistryId): rsvLogger.error('{}: complex payload error, @odata property non-conformant'.format(str(name))) rsvLogger.verboseout('\t***out of Complex') rsvLogger.verboseout('complex {}'.format(str(complexCounts))) + + propTypeObj = propComplexObj.typeobj + if name == 'Actions': aMsgs, aCounts = validateActions(name, val, propTypeObj, payloadType) complexMessages.update(aMsgs) complexCounts.update(aCounts) # validate the Redfish.DynamicPropertyPatterns if present - if propTypeObj.propPattern is not None and len(propTypeObj.propPattern) > 0: + # current issue, missing refs where they are appropriate, may cause issues + if propTypeObj.additional or propTypeObj.propPattern: patternMessages, patternCounts = validateDynamicPropertyPatterns(name, val, propTypeObj, - payloadType, attrRegistryId) + payloadType, attrRegistryId, ParentItem=name) complexMessages.update(patternMessages) complexCounts.update(patternCounts) return True, complexCounts, complexMessages - -def validateDynamicPropertyType(name, key, value, prop_type): - """ - Check the type of the property value - :param name: the name of the dictionary of properties being validated - :param key: the key of the individual property being validated - :param value: the value of the individual property being validated - :param prop_type: the expected type of the value - :return: True if the type check passes, False otherwise - """ - type_pass = True - if value is None: - # null value is OK - type_pass = True - elif prop_type == 'Edm.Primitive' or prop_type == 'Edm.PrimitiveType': - type_pass = isinstance(value, (int, float, str, bool)) - elif prop_type == 'Edm.String': - type_pass = isinstance(value, str) - elif prop_type == 'Edm.Boolean': - type_pass = isinstance(value, bool) - elif prop_type == 'Edm.DateTimeOffset': - type_pass = validateDatetime(key, value) - elif prop_type == 'Edm.Int' or prop_type == 'Edm.Int16' or prop_type == 'Edm.Int32' or prop_type == 'Edm.Int64': - type_pass = isinstance(value, int) - elif prop_type == 'Edm.Decimal' or prop_type == 'Edm.Double': - type_pass = isinstance(value, (int, float)) - elif prop_type == 'Edm.Guid': - type_pass = validateGuid(key, value) - else: - rsvLogger.debug('{}: Do not know how to validate type {}' - .format(name + '.' + key, prop_type)) - if not type_pass: - rsvLogger.error('{} with value {} is not of type {}'.format(name + '.' + key, value, prop_type)) - return type_pass - +attributeRegistries = dict() def validateAttributeRegistry(name, key, value, attr_reg): """ @@ -401,7 +348,7 @@ def validateAttributeRegistry(name, key, value, attr_reg): return reg_pass, type_prop -def validateDynamicPropertyPatterns(name, val, propTypeObj, payloadType, attrRegistryId): +def validateDynamicPropertyPatterns(name, val, propTypeObj, payloadType, attrRegistryId, ParentItem=None): """ Checks the value type and key pattern of the properties specified via Redfish.DynamicPropertyPatterns annotation :param name: the name of the dictionary of properties being validated @@ -471,12 +418,6 @@ def validateDynamicPropertyPatterns(name, val, propTypeObj, payloadType, attrReg else: counts['failDynamicPropertyPatterns'] += 1 # validate the value type against the Type - type_pass = validateDynamicPropertyType(name, key, value, prop_type) - if type_pass: - counts['pass'] += 1 - else: - counts['failDynamicPropertyPatterns'] += 1 - # validate against the attribute registry if present reg_pass = True attr_reg_type = None if attr_reg is not None: @@ -485,9 +426,6 @@ def validateDynamicPropertyPatterns(name, val, propTypeObj, payloadType, attrReg counts['pass'] += 1 else: counts['failAttributeRegistry'] += 1 - messages[name + '.' + key] = ( - displayValue(value), displayType('', prop_type if attr_reg_type is None else attr_reg_type), - 'Yes', 'PASS' if type_pass and pattern_pass and reg_pass else 'FAIL') return messages, counts @@ -639,16 +577,22 @@ def loadAttributeRegDict(odata_type, json_data): rsvLogger.debug('{}: "{}" AttributeRegistry dict has zero entries; not adding'.format(fn, reg_id)) -def checkPropertyConformance(soup, PropertyName, PropertyItem, decoded, refs, ParentItem=None, parentURI=""): - # The biggest piece of code, but also mostly collabs info for other functions - # this part of the program should maybe do ALL setup for functions above, do not let them do requests? - # info: what about this property is important (read/write, name, val, nullability, mandatory), - # warn: No compiled info but that's ok, it's not implemented - # error: no pass, mandatory/null fail, no compiled info but is present/mandatory - """ +def checkPropertyConformance(schemaObj, PropertyName, PropertyItem, decoded, ParentItem=None, parentURI=""): + """checkPropertyConformance + Given a dictionary of properties, check the validitiy of each item, and return a list of counted properties + :param soup: + :param PropertyName: + :param PropertyItem: + :param decoded: + :param refs: + :param ParentItem: + :param parentURI: + """ + """ + param arg1: property name param arg2: property item dictionary param arg3: json payload @@ -656,6 +600,7 @@ def checkPropertyConformance(soup, PropertyName, PropertyItem, decoded, refs, Pa """ resultList = OrderedDict() counts = Counter() + soup, refs = schemaObj.soup, schemaObj.refs rsvLogger.verboseout(PropertyName) item = PropertyName.split(':')[-1] @@ -688,11 +633,11 @@ def checkPropertyConformance(soup, PropertyName, PropertyItem, decoded, refs, Pa # why not actually check oem # rs-assertion: 7.4.7.2 - if 'Oem' in PropertyName: + if 'Oem' in PropertyName and not rst.currentService.config.get('oemcheck', False): rsvLogger.verboseout('\tOem is skipped') counts['skipOem'] += 1 - return {item: ('-', '-', - 'Yes' if propExists else 'No', 'OEM')}, counts + return {item: ('-', '-', 'Yes' if propExists else 'No', 'OEM')}, counts + pass propMandatory = False propMandatoryPass = True @@ -808,10 +753,20 @@ def checkPropertyConformance(soup, PropertyName, PropertyItem, decoded, refs, Pa else: if propRealType == 'complex': - innerPropType = PropertyItem['typeprops'] - success, complexCounts, complexMessages = validateComplex(sub_item, val, innerPropType, - decoded.get('@odata.type'), - decoded.get('AttributeRegistry')) + if PropertyItem['typeprops'] is not None: + if isCollection: + innerComplex = PropertyItem['typeprops'][cnt] + innerPropType = PropertyItem['typeprops'][cnt].typeobj + else: + innerComplex = PropertyItem['typeprops'] + innerPropType = PropertyItem['typeprops'].typeobj + + success, complexCounts, complexMessages = validateComplex(sub_item, val, innerComplex, + decoded.get('@odata.type'), + decoded.get('AttributeRegistry')) + else: + success = False + if not success: counts['failComplex'] += 1 resultList[sub_item] = ( @@ -826,14 +781,16 @@ def checkPropertyConformance(soup, PropertyName, PropertyItem, decoded, refs, Pa counts.update(complexCounts) resultList.update(complexMessages) - additionalComplex = innerPropType.additional - for key in val: - if sub_item + '.' + key not in complexMessages and not additionalComplex: + allowAdditional = innerPropType.additional + for key in innerComplex.unknownProperties: + if sub_item + '.' + key not in complexMessages and not allowAdditional: rsvLogger.error('{} not defined in schema {} (check version, spelling and casing)' .format(sub_item + '.' + key, innerPropType.snamespace)) counts['failComplexAdditional'] += 1 resultList[sub_item + '.' + key] = (displayValue(val[key]), '-', '-', 'FAIL') elif sub_item + '.' + key not in complexMessages: + rsvLogger.warn('{} not defined in schema {} (check version, spelling and casing)' + .format(sub_item + '.' + key, innerPropType.snamespace)) counts['unverifiedComplexAdditional'] += 1 resultList[sub_item + '.' + key] = (displayValue(val[key]), '-', '-', 'Additional') continue @@ -845,7 +802,7 @@ def checkPropertyConformance(soup, PropertyName, PropertyItem, decoded, refs, Pa paramPass = validateDeprecatedEnum(sub_item, val, PropertyItem['typeprops']) elif propRealType == 'entity': - paramPass = validateEntity(sub_item, val, propType, propCollectionType, soup, refs, autoExpand, parentURI) + paramPass = validateEntity(sub_item, val, propType, propCollectionType, schemaObj, autoExpand, parentURI) else: rsvLogger.error("%s: This type is invalid %s" % (sub_item, propRealType)) # Printout FORMAT paramPass = False @@ -933,8 +890,8 @@ class WarnFilter(logging.Filter): def filter(self, rec): return rec.levelno == logging.WARN - errorMessages = io.StringIO() - warnMessages = io.StringIO() + errorMessages = StringIO() + warnMessages = StringIO() fmt = logging.Formatter('%(levelname)s - %(message)s') errh = logging.StreamHandler(errorMessages) errh.setLevel(logging.ERROR) @@ -987,9 +944,9 @@ def filter(self, rec): # Generate dictionary of property info try: - propResourceObj = rst.ResourceObj( - uriName, URI, expectedType, expectedSchema, expectedJson, parent) - if not propResourceObj.initiated: + propResourceObj = rst.createResourceObject( + uriName, URI, expectedJson, expectedType, expectedSchema, parent) + if not propResourceObj: counts['problemResource'] += 1 rsvLogger.removeHandler(errh) # Printout FORMAT rsvLogger.removeHandler(warnh) # Printout FORMAT @@ -1025,26 +982,9 @@ def filter(self, rec): if namespace == '#AttributeRegistry' and type_name == 'AttributeRegistry': loadAttributeRegDict(odata_type, propResourceObj.jsondata) - node = propResourceObj.typeobj - while node is not None: - for prop in node.propList: - try: - propMessages, propCounts = checkPropertyConformance(node.soup, prop.name, prop.propDict, propResourceObj.jsondata, node.refs, parentURI=URI) - messages.update(propMessages) - counts.update(propCounts) - except AuthenticationError as e: - raise # re-raise exception - except Exception as ex: - rsvLogger.exception("Something went wrong") # Printout FORMAT - rsvLogger.error('%s: Could not finish check on this property' % (prop.name)) # Printout FORMAT - counts['exceptionPropCheck'] += 1 - node = node.parent - - serviceRefs = rst.currentService.metadata.get_service_refs() - serviceSchemaSoup = rst.currentService.metadata.get_soup() - if serviceSchemaSoup is not None: - for prop in propResourceObj.additionalList: - propMessages, propCounts = checkPropertyConformance(serviceSchemaSoup, prop.name, prop.propDict, propResourceObj.jsondata, serviceRefs) + for prop in propResourceObj.getResourceProperties(): + try: + propMessages, propCounts = checkPropertyConformance(propResourceObj.schemaObj, prop.name, prop.propDict, propResourceObj.jsondata, parentURI=URI) if '@Redfish.Copyright' in propMessages and 'MessageRegistry' not in propResourceObj.typeobj.fulltype: modified_entry = list(propMessages['@Redfish.Copyright']) modified_entry[-1] = 'FAIL' @@ -1052,6 +992,13 @@ def filter(self, rec): rsvLogger.error('@Redfish.Copyright is only allowed for mockups, and should not be allowed in official implementations') messages.update(propMessages) counts.update(propCounts) + except AuthenticationError as e: + raise # re-raise exception + except Exception as ex: + rsvLogger.exception("Something went wrong") # Printout FORMAT + rsvLogger.error('%s: Could not finish check on this property' % (prop.name)) # Printout FORMAT + counts['exceptionPropCheck'] += 1 + uriName, SchemaFullType, jsonData = propResourceObj.name, propResourceObj.typeobj.fulltype, propResourceObj.jsondata SchemaNamespace, SchemaType = rst.getNamespace(SchemaFullType), rst.getType(SchemaFullType) @@ -1065,20 +1012,24 @@ def filter(self, rec): item = jsonData[key] rsvLogger.verboseout(fmt % ( # Printout FORMAT key, messages[key][3] if key in messages else 'Exists, no schema check')) - if key not in messages: - # note: extra messages for "unchecked" properties - if not propResourceObj.typeobj.additional: - rsvLogger.error('{} not defined in schema {} (check version, spelling and casing)' - .format(key, SchemaNamespace)) - counts['failAdditional'] += 1 - messages[key] = (displayValue(item), '-', - '-', - 'FAIL') - else: - counts['unverifiedAdditional'] += 1 - messages[key] = (displayValue(item), '-', - '-', - 'Additional') + + allowAdditional = propResourceObj.typeobj.additional + for key in [k for k in jsonData if k not in messages and k not in propResourceObj.unknownProperties] + propResourceObj.unknownProperties: + # note: extra messages for "unchecked" properties + if not allowAdditional: + rsvLogger.error('{} not defined in schema {} (check version, spelling and casing)' + .format(key, SchemaNamespace)) + counts['failAdditional'] += 1 + messages[key] = (displayValue(item), '-', + '-', + 'FAIL') + else: + rsvLogger.warn('{} not defined in schema {} (check version, spelling and casing)' + .format(sub_item + '.' + key, innerPropType.snamespace)) + counts['unverifiedAdditional'] += 1 + messages[key] = (displayValue(item), '-', + '-', + 'Additional') for key in messages: if key not in jsonData: @@ -1178,7 +1129,7 @@ def main(argv=None, direct_parser=None): # tool argget.add_argument('--schemadir', type=str, default='./SchemaFiles/metadata', help='directory for local schema files') - argget.add_argument('--schema_pack', type=str, help='Deploy DMTF schema from zip distribution, for use with --localonly (Specify url or type "latest", overwrites current schema)') + argget.add_argument('--schema_pack', type=str, default='', help='Deploy DMTF schema from zip distribution, for use with --localonly (Specify url or type "latest", overwrites current schema)') argget.add_argument('--desc', type=str, default='No desc', help='sysdescription for identifying logs') argget.add_argument('--logdir', type=str, default='./logs', help='directory for log files') argget.add_argument('--payload', type=str, help='mode to validate payloads [Tree, Single, SingleFile, TreeFile] followed by resource/filepath', nargs=2) @@ -1189,6 +1140,7 @@ def main(argv=None, direct_parser=None): help='Output debug statements to text log, otherwise it only uses INFO') argget.add_argument('--verbose_checks', action="store_const", const=VERBO_NUM, default=logging.INFO, help='Show all checks in logging') + argget.add_argument('--nooemcheck', action='store_true', help='Don\'t check OEM items') # service argget.add_argument('-i', '--ip', type=str, help='ip to test on [host:port]') diff --git a/commonRedfish.py b/commonRedfish.py new file mode 100644 index 0000000..41dd750 --- /dev/null +++ b/commonRedfish.py @@ -0,0 +1,35 @@ +# ex: #Power.1.1.1.Power , #Power.v1_0_0.Power + +versionpattern = 'v[0-9]_[0-9]_[0-9]' +def getNamespace(string): + if '#' in string: + string = string.rsplit('#', 1)[1] + return string.rsplit('.', 1)[0] + +def getVersion(string): + return re.search(versionpattern, item).group() + + +def getNamespaceUnversioned(string): + if '#' in string: + string = string.rsplit('#', 1)[1] + return string.split('.', 1)[0] + + # alt version concatenation, unused + version = getVersion(string) + if version not in [None, '']: + return getNamespace(string) + return '{}.{}'.format(getNamespace(string), version) + + +def getType(string): + if '#' in string: + string = string.rsplit('#', 1)[1] + return string.rsplit('.', 1)[-1] + + +def createContext(typestring): + ns_name = getNamespaceUnversioned(typestring) + type_name = getType(typestring) + context = '/redfish/v1/$metadata' + '#' + ns_name + '.' + type_name + return context diff --git a/config.ini b/config.ini index be99dc3..1bb551c 100644 --- a/config.ini +++ b/config.ini @@ -13,6 +13,7 @@ CertificateBundle = [Options] MetadataFilePath = ./SchemaFiles/metadata Schema_Pack = +OemCheck = On CacheMode = Off CacheFilePath = SchemaSuffix = _v1.xml diff --git a/metadata.py b/metadata.py index 71b5d86..720acd6 100644 --- a/metadata.py +++ b/metadata.py @@ -19,7 +19,7 @@ EDMX_TAGS = ['DataServices', 'Edmx', 'Include', 'Reference'] -live_zip_uri = 'http://redfish.dmtf.org/schemas/DSP8010_2017.3.zip' +live_zip_uri = 'http://redfish.dmtf.org/schemas/DSP8010_2018.1.zip' def setup_schema_pack(uri, local_dir, proxies, timeout): rst.traverseLogger.info('Unpacking schema pack...') @@ -104,8 +104,6 @@ class Metadata(object): def __init__(self, logger): logger.info('Constructing metadata...') self.success_get = False - self.md_soup = None - self.service_refs = dict() self.uri_to_namespaces = defaultdict(list) self.elapsed_secs = 0 self.metadata_namespaces = set() @@ -122,10 +120,13 @@ def __init__(self, logger): self.redfish_extensions_alias_ok = False start = time.time() - self.success_get, self.md_soup, uri = rst.getSchemaDetails(Metadata.schema_type, Metadata.metadata_uri) + self.schema_obj = rst.getSchemaObject(Metadata.schema_type, Metadata.metadata_uri) + uri = Metadata.metadata_uri + self.elapsed_secs = time.time() - start - if self.success_get: - self.service_refs = rst.getReferenceDetails(self.md_soup, name=Metadata.schema_type) + if self.schema_obj: + self.md_soup = self.schema_obj.soup + self.service_refs = self.schema_obj.refs # set of namespaces included in $metadata self.metadata_namespaces = {k for k in self.service_refs.keys()} # create map of schema URIs to namespaces from $metadata @@ -151,12 +152,13 @@ def __init__(self, logger): logger.debug('Metadata: bad_namespace_include = {}'.format(self.bad_namespace_include)) for schema in self.service_refs: name, uri = self.service_refs[schema] - result = rst.getSchemaDetails(name, uri) - self.schema_store[name] = result - + self.schema_store[name] = rst.getSchemaObject(name, uri) else: logger.warning('Metadata: getSchemaDetails() did not return success') + def get_schema_obj(self): + return self.schema_obj + def get_soup(self): return self.md_soup @@ -225,7 +227,7 @@ def check_namespaces_in_schemas(self): self.logger.debug('Metadata: {}'.format(msg)) self.bad_namespace_include.add(msg) else: - self.logger.debug('Metadata: failure opening schema {} of type {}'.format(schema_uri, schema_type)) + self.logger.error('Metadata: failure opening schema {} of type {}'.format(schema_uri, schema_type)) self.bad_schema_uris.add(schema_uri) def get_counter(self): diff --git a/traverseService.py b/traverseService.py index 7480b4e..38118bb 100644 --- a/traverseService.py +++ b/traverseService.py @@ -20,6 +20,7 @@ import configparser import metadata as md +from commonRedfish import * traverseLogger = logging.getLogger(__name__) @@ -55,18 +56,18 @@ def getLogger(): 'http_proxy': 'httpproxy', 'localonly': 'localonlymode', 'https_proxy': 'httpsproxy', 'passwd': 'password', 'ip': 'targetip', 'logdir': 'logpath', 'desc': 'systeminfo', 'authtype': 'authtype', 'payload': 'payloadmode+payloadfilepath', 'cache': 'cachemode+cachefilepath', 'token': 'token', - 'linklimit': 'linklimit', 'sample': 'sample' + 'linklimit': 'linklimit', 'sample': 'sample', 'nooemcheck': '!oemcheck' } configset = { "targetip": str, "username": str, "password": str, "authtype": str, "usessl": bool, "certificatecheck": bool, "certificatebundle": str, "metadatafilepath": str, "cachemode": (bool, str), "cachefilepath": str, "schemasuffix": str, "timeout": int, "httpproxy": str, "httpsproxy": str, "systeminfo": str, "localonlymode": bool, "servicemode": bool, "token": str, 'linklimit': dict, 'sample': int, 'extrajsonheaders': dict, 'extraxmlheaders': dict, "schema_pack": str, - "forceauth": bool + "forceauth": bool, "oemcheck": bool } defaultconfig = { - 'authtype': 'basic', 'username': "", 'password': "", 'token': '', + 'authtype': 'basic', 'username': "", 'password': "", 'token': '', 'oemcheck': True, 'certificatecheck': True, 'certificatebundle': "", 'metadatafilepath': './SchemaFiles/metadata', 'cachemode': 'Off', 'cachefilepath': './cache', 'schemasuffix': '_v1.xml', 'httpproxy': "", 'httpsproxy': "", 'localonlymode': False, 'servicemode': False, 'linklimit': {'LogEntry':20}, 'sample': 0, 'schema_pack': None, 'forceauth': False @@ -282,11 +283,7 @@ def callResourceURI(URILink): traverseLogger.debug("This URI is empty!") return False, None, -1, 0 nonService = isNonService(URILink) - payload = None - statusCode = '' - elapsed = 0 - auth = None - noauthchk = True + payload, statusCode, elapsed, auth, noauthchk = None, '', 0, None, True isXML = False if "$metadata" in URILink or ".xml" in URILink: @@ -323,7 +320,7 @@ def callResourceURI(URILink): payload = json.loads(f.read()) payload = navigateJsonFragment(payload, URILink) if nonService and config['servicemode']: - traverseLogger.debug('Disallowed out of service URI') + traverseLogger.warn('Disallowed out of service URI') return False, None, -1, 0 # rs-assertion: do not send auth over http @@ -331,7 +328,6 @@ def callResourceURI(URILink): if (not UseSSL and not config['forceauth']) or nonService or AuthType != 'Basic': auth = None - # only send token when we're required to chkauth, during a Session, and on Service and Secure if UseSSL and not nonService and AuthType == 'Session' and not noauthchk: headers = {"X-Auth-Token": currentSession.getSessionKey()} @@ -413,18 +409,6 @@ def callResourceURI(URILink): return False, None, statusCode, elapsed -# note: Use some sort of re expression to parse SchemaType -# ex: #Power.1.1.1.Power , #Power.v1_0_0.Power -def getNamespace(string): - if '#' in string: - string = string.rsplit('#', 1)[1] - return string.rsplit('.', 1)[0] - - -def getType(string): - if '#' in string: - string = string.rsplit('#', 1)[1] - return string.rsplit('.', 1)[-1] @lru_cache(maxsize=64) @@ -442,7 +426,9 @@ def getSchemaDetails(SchemaType, SchemaURI): return False, None, None if currentService.active and getNamespace(SchemaType) in currentService.metadata.schema_store: - return currentService.metadata.schema_store[getNamespace(SchemaType)] + result = currentService.metadata.schema_store[getNamespace(SchemaType)] + if result is not None: + return True, result.soup, result.origin config = currentService.config LocalOnly, SchemaLocation, ServiceOnly = config['localonlymode'], config['metadatafilepath'], config['servicemode'] @@ -581,8 +567,6 @@ def getReferenceDetails(soup, metadata_dict=None, name='xml'): return: dictionary """ refDict = {} - config = currentService.config - ServiceOnly = config['servicemode'] maintag = soup.find("edmx:Edmx", recursive=False) refs = maintag.find_all('edmx:Reference', recursive=False) @@ -607,68 +591,51 @@ def getReferenceDetails(soup, metadata_dict=None, name='xml'): return refDict -def getParentType(soup, refs, currentType, tagType='EntityType'): - # overhauling needed: deprecated function that should be realigned with the current type function - # debug: what are we working towards? did we get it? it's fine if we didn't - # error: none, should lend that to whatever calls it - """ - Get parent type of given type. - - param arg1: soup - param arg2: refs - param arg3: current type - param tagType: the type of tag for inheritance, default 'EntityType' - return: success, associated soup, associated ref, new type - """ - pnamespace, ptype = getNamespace(currentType), getType(currentType) - - currentSchema = soup.find( # BS4 line - 'Schema', attrs={'Namespace': pnamespace}) - - if currentSchema is None: - return False, None, None, None - - currentEntity = currentSchema.find(tagType, attrs={'Name': ptype}, recursive=False) # BS4 line - - if currentEntity is None: - return False, None, None, None - currentType = currentEntity.get('BaseType') - if currentType is None: - return False, None, None, None - - currentType = currentType.replace('#', '') - SchemaNamespace = getNamespace( - currentType) - parentSchema = soup.find('Schema', attrs={'Namespace': SchemaNamespace}) # BS4 line - - if parentSchema is None: - success, innerSoup, uri = getSchemaDetails( - *refs.get(SchemaNamespace, (None, None))) +def createResourceObject(name, uri, jsondata=None, typename=None, context=None, parent=None, isComplex=False): + """ + Factory for resource object, move certain work here + """ + traverseLogger.debug( + 'Creating ResourceObject {} {} {}'.format(name, uri, typename)) + + # Create json from service or from given + if jsondata is None and not isComplex: + success, jsondata, status, rtime = callResourceURI(uri) + traverseLogger.debug('{}, {}, {}'.format(success, jsondata, status)) if not success: - return False, None, None, None - innerRefs = getReferenceDetails(innerSoup, refs, uri) - propSchema = innerSoup.find( - 'Schema', attrs={'Namespace': SchemaNamespace}) - if propSchema is None: - return False, None, None, None + traverseLogger.error( + '{}: URI could not be acquired: {}'.format(uri, status)) + return None else: - innerSoup = soup - innerRefs = refs + jsondata, rtime = jsondata, 0 + + if not isinstance(jsondata, dict): + if not isComplex: + traverseLogger.error("Resource no longer a dictionary...") + else: + traverseLogger.debug("ComplexType does not have val") + return None - return True, innerSoup, innerRefs, currentType + newResource = ResourceObj(name, uri, jsondata, typename, context, parent, isComplex) + newResource.rtime = rtime + return newResource + class ResourceObj: robjcache = {} - - def __init__(self, name, uri, expectedType=None, expectedSchema=None, expectedJson=None, parent=None): + def __init__(self, name: str, uri: str, jsondata: dict, typename: str, context: str, parent=None, isComplex=False): self.initiated = False self.parent = parent self.uri, self.name = uri, name self.rtime = 0 self.isRegistry = False + self.errorindex = { + "badtype": 0 + + } # Check if this is a Registry resource parent_type = parent.typeobj.stype if parent is not None and parent.typeobj is not None else None @@ -676,122 +643,231 @@ def __init__(self, name, uri, expectedType=None, expectedSchema=None, expectedJs traverseLogger.debug('{} is a Registry resource'.format(self.uri)) self.isRegistry = True - # Check if we provide a json - if expectedJson is None: - success, self.jsondata, status, self.rtime = callResourceURI(self.uri) - traverseLogger.debug('{}, {}, {}'.format(success, self.jsondata, status)) - if not success: - traverseLogger.error( - '{}: URI could not be acquired: {}'.format(self.uri, status)) - return - else: - self.jsondata = expectedJson + # Check if we provide a valid json + self.jsondata = jsondata traverseLogger.debug("payload: {}".format(json.dumps(self.jsondata, indent=4, sort_keys=True))) + if not isinstance(self.jsondata, dict): traverseLogger.error("Resource no longer a dictionary...") - return + raise ValueError - # Check if we provide a type besides json's - if expectedType is None: - fullType = self.jsondata.get('@odata.type') - if fullType is None: - traverseLogger.error( - '{}: Json does not contain @odata.type'.format(self.uri)) - return - else: - fullType = self.jsondata.get('@odata.type', expectedType) + # Check if this is a Registry resource + parent_type = parent.typeobj.stype if parent is not None and parent.typeobj is not None else None + + if parent_type == 'MessageRegistryFile': + traverseLogger.debug('{} is a Registry resource'.format(uri)) + isRegistry = True - # Check for @odata.id + # Check for @odata.id (todo: regex) odata_id = self.jsondata.get('@odata.id') - if odata_id is None: + if odata_id is None and not isComplex: if self.isRegistry: traverseLogger.debug('{}: @odata.id missing, but not required for Registry resource' .format(self.uri)) else: traverseLogger.error('{}: Json does not contain @odata.id'.format(self.uri)) - # Provide a context for this - if expectedSchema is None: - self.context = self.jsondata.get('@odata.context') - if self.context is None: + # Get our real type (check for version) + acquiredtype = jsondata.get('@odata.type', typename) + if acquiredtype is None: + traverseLogger.error( + '{}: Json does not contain @odata.type or NavType'.format(uri)) + raise ValueError + if acquiredtype is not typename and isComplex: + context = None + + # Provide a context for this (todo: regex) + if context is None: + context = self.jsondata.get('@odata.context') + if context is None and not isComplex: + context = createContext(acquiredtype) if self.isRegistry: # If this is a Registry resource, @odata.context is not required; do our best to construct one - ns_name = getNamespace(fullType).split('.')[0] - type_name = getType(fullType) - self.context = '/redfish/v1/$metadata' + '#' + ns_name + '.' + type_name traverseLogger.debug('{}: @odata.context missing from Registry resource; constructed context {}' - .format(fullType, self.context)) + .format(acquiredtype, context)) else: - traverseLogger.error('{}: Json does not contain @odata.context'.format(self.uri)) - expectedSchema = self.context - else: - self.context = expectedSchema + traverseLogger.error('{}: Json does not contain @odata.context'.format(uri)) + if isComplex: + context = createContext(acquiredtype) - success, typesoup, self.context = getSchemaDetails( - fullType, SchemaURI=self.context) + self.context = context - if not success: - traverseLogger.error("validateURI: No schema XML for {}".format(fullType)) - return + # Get Schema object + schemaObj = getSchemaObject(acquiredtype, self.context) + self.schemaObj = schemaObj + + if schemaObj is None: + traverseLogger.error("validateURI: No schema XML for {} {} {}".format(typename, acquiredtype, self.context)) # Use string comprehension to get highest type - if fullType is expectedType: - typelist = list() - schlist = list() - for schema in typesoup.find_all('Schema'): - newNamespace = schema.get('Namespace') - typelist.append(newNamespace) - schlist.append(schema) - for item, schema in reversed(sorted(zip(typelist, schlist))): - traverseLogger.debug( - "{} {}".format(item, getType(fullType))) - if schema.find('EntityType', attrs={'Name': getType(fullType)}, recursive=False): - fullType = item + '.' + getType(fullType) - break - traverseLogger.warn( - 'No @odata.type present, assuming highest type {}'.format(fullType)) + if acquiredtype is typename: + acquiredtype = schemaObj.getHighestType(typename) + if not isComplex: + traverseLogger.warn( + 'No @odata.type present, assuming highest type {}'.format(typename)) + + # Check if we provide a valid type (todo: regex) + self.typename = acquiredtype + typename = self.typename - self.additionalList = [] self.initiated = True - idtag = (fullType, self.context) + # get our metadata metadata = currentService.metadata serviceRefs = metadata.get_service_refs() serviceSchemaSoup = metadata.get_soup() - if serviceSchemaSoup is not None: - successService, additionalProps = getAnnotations( - serviceSchemaSoup, serviceRefs, self.jsondata) - for prop in additionalProps: - self.additionalList.append(prop) - # if we've generated this type, use it, else generate type + idtag = (typename, context) if idtag in ResourceObj.robjcache: self.typeobj = ResourceObj.robjcache[idtag] else: - typerefs = getReferenceDetails(typesoup, serviceRefs, self.context) self.typeobj = PropType( - fullType, typesoup, typerefs, 'EntityType', topVersion=getNamespace(fullType)) - ResourceObj.robjcache[idtag] = self.typeobj + typename, schemaObj, topVersion=getNamespace(typename)) + + self.propertyList = self.typeobj.getProperties(self.jsondata) + propertyList = [prop.name.split(':')[-1] for prop in self.propertyList] + + + # get additional + self.additionalList = [] + propTypeObj = self.typeobj + if propTypeObj.propPattern is not None and len(propTypeObj.propPattern) > 0: + prop_pattern = propTypeObj.propPattern.get('Pattern', '.*') + prop_type = propTypeObj.propPattern.get('Type','Resource.OemObject') + + regex = re.compile(prop_pattern) + for key in [k for k in self.jsondata if k not in propertyList]: + val = self.jsondata.get(key) + value_obj = PropItem(propTypeObj.schemaObj, propTypeObj.fulltype, key, val, customType=prop_type) + self.additionalList.append(value_obj) + + if propTypeObj.additional: + for key in [k for k in self.jsondata if k not in propertyList + + [prop.name.split(':')[-1] for prop in self.additionalList]]: + val = self.jsondata.get(key) + value_obj = PropItem(propTypeObj.schemaObj, propTypeObj.fulltype, key, val, customType=prop_type) + self.additionalList.append(value_obj) + + + # get annotation + if serviceSchemaSoup is not None: + successService, annotationProps = getAnnotations( + metadata.get_schema_obj(), self.jsondata) + self.additionalList.extend(annotationProps) + + + # list illegitimate properties together + self.unknownProperties = [k for k in self.jsondata if k not in propertyList + + [prop.name for prop in self.additionalList] and '@odata' not in k] self.links = OrderedDict() node = self.typeobj - while node is not None: - self.links.update(getAllLinks( - self.jsondata, node.propList, node.refs, context=expectedSchema, linklimits=currentService.config['linklimit'], - sample_size=currentService.config['sample'])) - node = node.parent + self.links.update(self.typeobj.getLinksFromType(self.jsondata, self.context, self.propertyList)) + + self.links.update(getAllLinks( + self.jsondata, self.additionalList, schemaObj, context=context, linklimits=currentService.config.get('linklimits',{}), + sample_size=currentService.config['sample'])) + def getResourceProperties(self): + allprops = self.propertyList + self.additionalList[:min(len(self.additionalList), 100)] + return allprops + + +class rfSchema: + def __init__(self, soup, context, origin, metadata=None, name='xml'): + self.soup = soup + self.refs = getReferenceDetails(soup, metadata, name) + self.context = context + self.origin = origin + self.name = name + + def getSchemaFromReference(self, namespace): + tup = self.refs.get(namespace) + tupVersionless = self.refs.get(getNamespace(namespace)) + if tup is None: + if tupVersionless is None: + traverseLogger.warn('No such reference {} in {}'.format(namespace, self.origin)) + return None + else: + tup = tupVersionless + traverseLogger.warn('No such reference {} in {}, using unversioned'.format(namespace, self.origin)) + typ, uri = tup + newSchemaObj = getSchemaObject(typ, uri) + return newSchemaObj + + def getTypeTagInSchema(self, currentType, tagType): + pnamespace, ptype = getNamespace(currentType), getType(currentType) + soup = self.soup + + currentSchema = soup.find( # BS4 line + 'Schema', attrs={'Namespace': pnamespace}) + + if currentSchema is None: + return None + + currentEntity = currentSchema.find(tagType, attrs={'Name': ptype}, recursive=False) # BS4 line + + return currentEntity + + def getParentType(self, currentType, tagType=['EntityType', 'ComplexType']): + currentType = currentType.replace('#', '') + typetag = self.getTypeTagInSchema(currentType, tagType) + if typetag is not None: + currentType = typetag.get('BaseType') + if currentType is None: + return False, None, None + typetag = self.getTypeTagInSchema(currentType, tagType) + if typetag is not None: + return True, self, currentType + else: + namespace = getNamespace(currentType) + schemaObj = self.getSchemaFromReference(namespace) + if schemaObj is None: + return False, None, None + propSchema = schemaObj.soup.find( + 'Schema', attrs={'Namespace': namespace}) + if propSchema is None: + return False, None, None + return True, schemaObj, currentType + else: + return False, None, None + + def getHighestType(self, acquiredtype): + typesoup = self.soup + typelist = list() + for schema in typesoup.find_all('Schema'): + newNamespace = schema.get('Namespace') + typelist.append((newNamespace, schema)) + for ns, schema in reversed(sorted(typelist)): + traverseLogger.debug( + "{} {}".format(ns, getType(acquiredtype))) + if schema.find(['EntityType', 'ComplexType'], attrs={'Name': getType(acquiredtype)}, recursive=False): + acquiredtype = ns + '.' + getType(acquiredtype) + break + return acquiredtype + + +def getSchemaObject(typename, uri, metadata=None): + + result = getSchemaDetails(typename, uri) + success, soup = result[0], result[1] + origin = result[2] + + + if success is False: + return None + return rfSchema(soup, uri, origin, metadata=metadata, name=typename) class PropItem: - def __init__(self, soup, refs, propOwner, propChild, tagType, topVersion): + def __init__(self, schemaObj, propOwner, propChild, val, topVersion=None, customType=None): try: self.name = propOwner + ':' + propChild self.propOwner, self.propChild = propOwner, propChild self.propDict = getPropertyDetails( - soup, refs, propOwner, propChild, tagType, topVersion) + schemaObj, propOwner, propChild, val, topVersion, customType) self.attr = self.propDict['attrs'] except Exception as ex: traverseLogger.exception("Something went wrong") @@ -801,44 +877,101 @@ def __init__(self, soup, refs, propOwner, propChild, tagType, topVersion): return pass +class PropAction: + def __init__(self, propOwner, propChild, act): + try: + self.name = '#{}.{}'.format(propOwner, propChild) + self.propOwner, self.propChild = propOwner, propChild + self.actTag = act + except Exception as ex: + traverseLogger.exception("Something went wrong") + traverseLogger.error( + '{}:{} : Could not get details on this action'.format(str(propOwner),str(propChild))) + self.actTag = None + class PropType: - def __init__(self, fulltype, soup, refs, tagType, topVersion=None): + def __init__(self, typename, schemaObj, topVersion=None): + # if we've generated this type, use it, else generate type self.initiated = False - self.fulltype = fulltype - self.soup, self.refs = soup, refs + self.fulltype = typename + self.schemaObj = schemaObj self.snamespace, self.stype = getNamespace( self.fulltype), getType(self.fulltype) self.additional = False - self.tagType = tagType self.isNav = False self.propList = [] + self.actionList = [] self.parent = None self.propPattern = None - propertyList = self.propList - success, baseSoup, baseRefs, baseType = True, self.soup, self.refs, self.fulltype + # get all properties and actions in Type chain + success, currentSchemaObj, baseType = True, self.schemaObj, self.fulltype try: - self.additional, newList, self.propPattern = getTypeDetails( - baseSoup, baseRefs, baseType, self.tagType, topVersion) - propertyList.extend(newList) - success, baseSoup, baseRefs, baseType = getParentType( - baseSoup, baseRefs, baseType, self.tagType) + newPropList, newActionList, self.additional, self.propPattern = getTypeDetails( + currentSchemaObj, baseType, topVersion) + + self.propList.extend(newPropList) + self.actionList.extend(newActionList) + + success, currentSchemaObj, baseType = currentSchemaObj.getParentType(baseType) if success: self.parent = PropType( - baseType, baseSoup, baseRefs, self.tagType, topVersion=topVersion) + baseType, currentSchemaObj, topVersion=topVersion) if not self.additional: self.additional = self.parent.additional - self.initiated = True except Exception as ex: traverseLogger.exception("Something went wrong") traverseLogger.error( '{}: Getting type failed for {}'.format(str(self.fulltype), str(baseType))) return + self.initiated = True + + def getTypeChain(self): + if self.fulltype is None: + raise StopIteration + else: + node = self + tlist = [] + while node is not None: + tlist.append(node.fulltype) + yield node.fulltype + node = node.parent + raise StopIteration + + def getLinksFromType(self, jsondata, context, propList=None): + node = self + links = OrderedDict() + while node is not None: + links.update(getAllLinks( + jsondata, node.getProperties(jsondata) if propList is None else propList, node.schemaObj, context=context, linklimits=currentService.config.get('linklimits',{}), + sample_size=currentService.config['sample'])) + node = node.parent + return links + + def getProperties(self, jsondata): + node = self + props = [] + while node is not None: + for prop in node.propList: + schemaObj, newPropOwner, newProp, topVersion = prop + val = jsondata.get(newProp) + props.append(PropItem(schemaObj, newPropOwner, newProp, val, topVersion=topVersion)) + node = node.parent + return props + + def getActions(self): + node = self + while node is not None: + for prop in node.actionList: + yield prop + node = node.parent + raise StopIteration + -def getTypeDetails(soup, refs, SchemaAlias, tagType, topVersion=None): +def getTypeDetails(schemaObj, SchemaAlias, topVersion=None): # spits out information on the type we have, prone to issues if references/soup is ungettable, this shouldn't be ran without it # has been prone to a lot of confusing errors: rehaul information that user expects to know before this point is reached # info: works undercover, but maybe can point out what type was generated and how many properties were found, if additional props allowed... @@ -850,13 +983,16 @@ def getTypeDetails(soup, refs, SchemaAlias, tagType, topVersion=None): """ metadata = currentService.metadata PropertyList = list() + ActionList = list() PropertyPattern = None additional = False + + soup, refs = schemaObj.soup, schemaObj.refs SchemaNamespace, SchemaType = getNamespace( SchemaAlias), getType(SchemaAlias) - traverseLogger.debug("Generating type: {} of tagType {}".format(SchemaAlias, tagType)) + traverseLogger.debug("Generating type: {}".format(SchemaAlias)) traverseLogger.debug("Schema is {}, {}".format( SchemaType, SchemaNamespace)) @@ -871,15 +1007,14 @@ def getTypeDetails(soup, refs, SchemaAlias, tagType, topVersion=None): uri = metadata.get_schema_uri(SchemaType) traverseLogger.error('Schema namespace {} not found in schema file {}. Will not be able to gather type details.' .format(SchemaNamespace, uri if uri is not None else SchemaType)) - return False, PropertyList, PropertyPattern + return PropertyList, ActionList, False, PropertyPattern - element = innerschema.find(tagType, attrs={'Name': SchemaType}, recursive=False) + element = innerschema.find(['EntityType', 'ComplexType'], attrs={'Name': SchemaType}, recursive=False) traverseLogger.debug("___") - traverseLogger.debug(element['Name']) + traverseLogger.debug(element.get('Name')) traverseLogger.debug(element.attrs) traverseLogger.debug(element.get('BaseType')) - usableProperties = element.find_all(['NavigationProperty', 'Property'], recursive=False) additionalElement = element.find( 'Annotation', attrs={'Term': 'OData.AdditionalProperties'}) additionalElementOther = element.find( @@ -910,6 +1045,9 @@ def getTypeDetails(soup, refs, SchemaAlias, tagType, topVersion=None): PropertyPattern['Type'] = prop_type additional = True + # get properties + usableProperties = element.find_all(['NavigationProperty', 'Property'], recursive=False) + for innerelement in usableProperties: traverseLogger.debug(innerelement['Name']) traverseLogger.debug(innerelement.get('Type')) @@ -917,22 +1055,23 @@ def getTypeDetails(soup, refs, SchemaAlias, tagType, topVersion=None): newPropOwner = SchemaAlias if SchemaAlias is not None else 'SomeSchema' newProp = innerelement['Name'] traverseLogger.debug("ADDING :::: {}:{}".format(newPropOwner, newProp)) - if newProp not in PropertyList: - PropertyList.append( - PropItem(soup, refs, newPropOwner, newProp, tagType=tagType, topVersion=topVersion)) - - return additional, PropertyList, PropertyPattern - - -def getPropertyDetails(soup, refs, propertyOwner, propertyName, ownerTagType='EntityType', topVersion=None): - # gets an individual property's details, can be prone to problems if info does not exist in soup or is bad - # HOWEVER, this will rarely be the case: a property that does not exist in soup would never be expected to generate - # info: under the hood, too much info to be worth showing - # debug: however, individual property concerns can go here - # error: much like above function, what if we can't find the type we need? should not happen... - # if this happens, is it necessarily an error? could be an outbound referenced type that isn't needed or stored - # example-- if we have a type for StorageXxx but don't have it stored on our system, why bother? we don't use it - # the above is not technically error, pass it on? + PropertyList.append( + (schemaObj, newPropOwner, newProp, topVersion)) + + # get actions + usableActions = innerschema.find_all(['Action'], recursive=False) + + for act in usableActions: + newPropOwner = getNamespace(SchemaAlias) if SchemaAlias is not None else 'SomeSchema' + newProp = act['Name'] + traverseLogger.debug("ADDING ACTION :::: {}:{}".format(newPropOwner, newProp)) + ActionList.append( + PropAction(newPropOwner, newProp, act)) + + return PropertyList, ActionList, additional, PropertyPattern + + +def getPropertyDetails(schemaObj, propertyOwner, propertyName, val, topVersion=None, customType=None): """ Get dictionary of tag attributes for properties given, including basetypes. @@ -942,46 +1081,62 @@ def getPropertyDetails(soup, refs, propertyOwner, propertyName, ownerTagType='En """ propEntry = dict() + propEntry['val'] = val OwnerNamespace, OwnerType = getNamespace(propertyOwner), getType(propertyOwner) traverseLogger.debug('___') - traverseLogger.debug('{}, {}:{}, {}'.format(OwnerNamespace, propertyOwner, propertyName, ownerTagType)) + traverseLogger.debug('{}, {}:{}'.format(OwnerNamespace, propertyOwner, propertyName)) - # Get Schema of the Owner that owns this prop - ownerSchema = soup.find('Schema', attrs={'Namespace': OwnerNamespace}) + soup, refs = schemaObj.soup, schemaObj.refs - if ownerSchema is None: - traverseLogger.warn( - "getPropertyDetails: Schema could not be acquired, {}".format(OwnerNamespace)) - return None + if customType is None: + # Get Schema of the Owner that owns this prop + ownerSchema = soup.find('Schema', attrs={'Namespace': OwnerNamespace}) + + if ownerSchema is None: + traverseLogger.warn( + "getPropertyDetails: Schema could not be acquired, {}".format(OwnerNamespace)) + return None - # Get Entity of Owner, then the property of the Property we're targeting - ownerEntity = ownerSchema.find( - ownerTagType, attrs={'Name': OwnerType}, recursive=False) # BS4 line + # Get Entity of Owner, then the property of the Property we're targeting + ownerEntity = ownerSchema.find( + ['EntityType', 'ComplexType'], attrs={'Name': OwnerType}, recursive=False) # BS4 line - propertyTag = ownerEntity.find( - ['NavigationProperty', 'Property'], attrs={'Name': propertyName}, recursive=False) # BS4 line + # check if this property is a nav property + # Checks if this prop is an annotation + success, propertySoup, propertyRefs, propertyFullType = True, soup, refs, OwnerType - # check if this property is a nav property - # Checks if this prop is an annotation - success, propertySoup, propertyRefs, propertyFullType = True, soup, refs, OwnerType + if '@' not in propertyName: + propEntry['isTerm'] = False # not an @ annotation + propertyTag = ownerEntity.find( + ['NavigationProperty', 'Property'], attrs={'Name': propertyName}, recursive=False) # BS4 line - if '@' not in propertyName: - propEntry['isTerm'] = False # not an @ annotation - # start adding attrs and props together - propertyInnerTags = propertyTag.find_all() # BS4 line - for tag in propertyInnerTags: - propEntry[tag['Term']] = tag.attrs - propertyFullType = propertyTag.get('Type') - else: - propEntry['isTerm'] = True - propertyTag = ownerEntity - propertyFullType = propertyTag.get('Type', propertyOwner) + # start adding attrs and props together + propertyInnerTags = propertyTag.find_all() # BS4 line + for tag in propertyInnerTags: + propEntry[tag['Term']] = tag.attrs + propertyFullType = propertyTag.get('Type') + else: + propEntry['isTerm'] = True + ownerEntity = ownerSchema.find( + ['Term'], attrs={'Name': OwnerType}, recursive=False) # BS4 line + propertyTag = ownerEntity + propertyFullType = propertyTag.get('Type', propertyOwner) - propEntry['isNav'] = propertyTag.name == 'NavigationProperty' - propEntry['attrs'] = propertyTag.attrs - traverseLogger.debug(propEntry) + propEntry['isNav'] = propertyTag.name == 'NavigationProperty' + propEntry['attrs'] = propertyTag.attrs + traverseLogger.debug(propEntry) - propEntry['realtype'] = 'none' + propEntry['realtype'] = 'none' + + else: + propertyFullType = customType + propEntry['realtype'] = 'none' + propEntry['attrs'] = dict() + propEntry['attrs']['Type'] = customType + metadata = currentService.metadata + serviceRefs = currentService.metadata.get_service_refs() + serviceSchemaSoup = currentService.metadata.get_soup() + success, propertySoup, propertyRefs, propertyFullType = True, serviceSchemaSoup, serviceRefs, customType # find the real type of this, by inheritance while propertyFullType is not None: @@ -1004,10 +1159,11 @@ def getPropertyDetails(soup, refs, propertyOwner, propertyName, ownerTagType='En # get proper soup, check if this Namespace is the same as its Owner, otherwise find its SchemaXml if PropertyNamespace.split('.')[0] != OwnerNamespace.split('.')[0]: - success, propertySoup, uri = getSchemaDetails( - *refs.get(PropertyNamespace, (None, None))) + schemaObj = schemaObj.getSchemaFromReference(PropertyNamespace) + success = schemaObj is not None if success: - propertyRefs = getReferenceDetails(propertySoup, refs, name=uri) + propertySoup = schemaObj.soup + propertyRefs = schemaObj.refs else: success, propertySoup, uri = True, soup, 'of parent' @@ -1056,7 +1212,7 @@ def getPropertyDetails(soup, refs, propertyOwner, propertyName, ownerTagType='En success, baseSoup, baseRefs, baseType = True, propertySoup, propertyRefs, propertyFullType # If we're outside of our normal Soup, then do something different, otherwise elif - if PropertyNamespace.split('.')[0] != OwnerNamespace.split('.')[0]: + if PropertyNamespace.split('.')[0] != OwnerNamespace.split('.')[0] and not customType: typelist = [] schlist = [] for schema in baseSoup.find_all('Schema'): @@ -1088,15 +1244,18 @@ def getPropertyDetails(soup, refs, propertyOwner, propertyName, ownerTagType='En break else: nextEntity = currentSchema.find( # BS4 line - 'EntityType', attrs={'Name': OwnerType}) + ['EntityType', 'ComplexType'], attrs={'Name': OwnerType}) nextType = nextEntity.get('BaseType') currentVersion = getNamespace(nextType) currentSchema = baseSoup.find( # BS4 line 'Schema', attrs={'Namespace': currentVersion}) continue propEntry['realtype'] = 'complex' - propEntry['typeprops'] = PropType( - baseType, baseSoup, baseRefs, 'ComplexType') + if propEntry.get('isCollection') is None: + propEntry['typeprops'] = createResourceObject(propertyName, 'complex', val, context=schemaObj.context, typename=baseType, isComplex=True) + else: + val = val if val is not None else {} + propEntry['typeprops'] = [createResourceObject(propertyName, 'complex', item, context=schemaObj.context, typename=baseType, isComplex=True) for item in val] break elif nameOfTag == 'EnumType': @@ -1162,12 +1321,7 @@ def enumerate_collection(items, cTypeName, linklimits, sample_size): yield from enumerate(items) -def getAllLinks(jsonData, propList, refDict, prefix='', context='', linklimits=None, sample_size=0): - # gets all links, this can miss something if it is not designated navigatable or properly autoextended, collections, etc - # info: works underneath, can maybe report how many links it has gotten or leave that to whatever calls it? - # debug: should be reported by what calls it? not much debug is neede besides what is already generated earlier, - # error: it really depends on what type generation has done: if done correctly, this should have no problem, if propList is empty, it does nothing - # cannot think of errors that would be neccesary to know +def getAllLinks(jsonData, propList, schemaObj, prefix='', context='', linklimits=None, sample_size=0): """ Function that returns all links provided in a given JSON response. This result will include a link to itself. @@ -1181,7 +1335,7 @@ def getAllLinks(jsonData, propList, refDict, prefix='', context='', linklimits=N """ linkList = OrderedDict() if linklimits is None: - linklimits = {} + linklimits = {} # check keys in propertyDictionary # if it is a Nav property, check that it exists # if it is not a Nav Collection, add it to list @@ -1190,28 +1344,33 @@ def getAllLinks(jsonData, propList, refDict, prefix='', context='', linklimits=N # if it is, recurse on collection or individual item if not isinstance(jsonData, dict): traverseLogger.error("Generating links requires a dict") + refDict = schemaObj.refs try: for propx in propList: propDict = propx.propDict + if propDict is None: + continue + + isNav = propDict.get('isNav', False) key = propx.name item = getType(key).split(':')[-1] + + insideItem = jsonData.get(item) + autoExpand = propDict.get('OData.AutoExpand', None) is not None or\ + propDict.get('OData.AutoExpand'.lower(), None) is not None + cType = propDict.get('isCollection') ownerNS = propx.propOwner.split('.')[0] ownerType = propx.propOwner.split('.')[-1] - if propDict is None: - continue - elif propDict['isNav']: - insideItem = jsonData.get(item) + + if isNav: if insideItem is not None: - cType = propDict.get('isCollection') - autoExpand = propDict.get('OData.AutoExpand', None) is not None or\ - propDict.get('OData.AutoExpand'.lower(), None) is not None if cType is not None: cTypeName = getType(cType) cSchema = refDict.get(getNamespace(cType), (None, None))[1] if cSchema is None: cSchema = context for cnt, listItem in enumerate_collection(insideItem, cTypeName, linklimits, sample_size): - linkList[prefix + str(item) + '.' + getType(propDict['isCollection']) + + linkList[prefix + str(item) + '.' + cTypeName + '#' + str(cnt)] = (listItem.get('@odata.id'), autoExpand, cType, cSchema, listItem) else: cType = propDict['attrs'].get('Type') @@ -1222,12 +1381,9 @@ def getAllLinks(jsonData, propList, refDict, prefix='', context='', linklimits=N insideItem.get('@odata.id'), autoExpand, cType, cSchema, insideItem) elif item == 'Uri' and ownerNS == 'MessageRegistryFile' and ownerType == 'Location': # special handling for MessageRegistryFile Location Uri - insideItem = jsonData.get(item) if insideItem is not None and isinstance(insideItem, str) and len(insideItem) > 0: uriItem = {'@odata.id': insideItem} cType = ownerNS + '.' + ownerNS - autoExpand = propDict.get('OData.AutoExpand', None) is not None or \ - propDict.get('OData.AutoExpand'.lower(), None) is not None cSchema = refDict.get(getNamespace(cType), (None, None))[1] if cSchema is None: cSchema = context @@ -1237,11 +1393,8 @@ def getAllLinks(jsonData, propList, refDict, prefix='', context='', linklimits=N uriItem.get('@odata.id'), autoExpand, cType, cSchema, uriItem) elif item == 'Actions': # special handling for @Redfish.ActionInfo payload annotations - insideItem = jsonData.get(item) if isinstance(insideItem, dict): cType = 'ActionInfo.ActionInfo' - autoExpand = propDict.get('OData.AutoExpand', None) is not None or \ - propDict.get('OData.AutoExpand'.lower(), None) is not None cSchema = refDict.get(getNamespace(cType), (None, None))[1] for k, v in insideItem.items(): if not isinstance(v, dict): @@ -1255,23 +1408,23 @@ def getAllLinks(jsonData, propList, refDict, prefix='', context='', linklimits=N for propx in propList: propDict = propx.propDict + if propDict is None: + continue + propDict = propx.propDict key = propx.name item = getType(key).split(':')[-1] + cType = propDict.get('isCollection') if propDict is None: continue elif propDict['realtype'] == 'complex': + tp = propDict['typeprops'] if jsonData.get(item) is not None: - cType = propDict.get('isCollection') if cType is not None: cTypeName = getType(cType) - for cnt, listItem in enumerate_collection(jsonData[item], cTypeName, linklimits, sample_size): - linkList.update(getAllLinks( - listItem, propDict['typeprops'].propList, refDict, prefix + item + '.', context, - linklimits=linklimits, sample_size=sample_size)) + for item in tp: + linkList.update(item.links) else: - linkList.update(getAllLinks( - jsonData[item], propDict['typeprops'].propList, refDict, prefix + item + '.', context, - linklimits=linklimits, sample_size=sample_size)) + linkList.update(tp.links) traverseLogger.debug(str(linkList)) except Exception as ex: traverseLogger.exception("Something went wrong") @@ -1282,7 +1435,7 @@ def getAllLinks(jsonData, propList, refDict, prefix='', context='', linklimits=N return linkList -def getAnnotations(soup, refs, decoded, prefix=''): +def getAnnotations(schemaObj, decoded, prefix=''): """ Function to gather @ additional props in a payload """ @@ -1303,22 +1456,22 @@ def getAnnotations(soup, refs, decoded, prefix=''): else: # add the namespace to the set of namespaces referenced by this service metadata.add_service_namespace(getNamespace(fullItem)) - realType, refLink = refs.get(getNamespace(fullItem), (None, None)) - success, annotationSoup, uri = getSchemaDetails(realType, refLink) - traverseLogger.debug('{}, {}, {}, {}, {}'.format( - str(success), key, splitKey, decoded[key], realType)) - if success: - annotationRefs = getReferenceDetails(annotationSoup, refs, uri) + annotationSchemaObj = schemaObj.getSchemaFromReference(getNamespace(fullItem)) + realType = annotationSchemaObj.name + traverseLogger.debug('{}, {}, {}'.format(key, splitKey, decoded[key])) + if annotationSchemaObj is not None: if isinstance(decoded[key], dict) and decoded[key].get('@odata.type') is not None: - payloadType = decoded[key].get('@odata.type').replace('#', '') - realType, refLink = annotationRefs.get(getNamespace(payloadType).split('.')[0], (None, None)) - success, annotationSoup, uri = getSchemaDetails(realType, refLink) + payloadType = decoded[key].get('@odata.type','').replace('#', '') + annotationSchemaObj = annotationSchemaObj.getSchemaFromReference(getNamespace(payloadType)) + if annotationSchemaObj is not None: + realType = annotationSchemaObj.name + else: + traverseLogger.warn("getAnnotations: {} cannot be acquired from metadata -> {}".format(payloadType, fullItem)) realItem = payloadType tagtype = 'ComplexType' else: realItem = realType + '.' + fullItem.split('.', 1)[1] - tagtype = 'Term' additionalProps.append( - PropItem(annotationSoup, annotationRefs, realItem, key, tagtype, None)) + PropItem(annotationSchemaObj, realItem, key, decoded[key])) traverseLogger.debug("Annotations generated: {} out of {}".format(len(additionalProps), annotationsFound)) return True, additionalProps