import collections import re from DataVaultGenerator.Components import DataVaultEntity, MappingSource, DBEntity, ErrorCollection class ViewAttribute(): def __init__(self, entity, definition): self.entity = entity self.definition = definition self.name = definition.get('name') self.attribute = entity.get_attribute(self.name) # DataVaultEntityAttribute self.components = definition.get('components', []) self.reference = definition.get('reference', '') # entity.attribute self.referencetype = definition.get('referencetype', '') # 1:n, m:n, .. self.order = definition.get('order') def get_components(self): # returns the component attribute instances return [self.entity.get_query_entity_by_alias(c.split('.')[0]).get_attribute(c.split('.')[1]) for c in self.components] def get_referenced_attribute(self): if self.reference: ref = self.reference.split('.') # reference: entityname.attributename return self.entity.model.get_entity(ref[0]).get_attribute(ref[1]) else: return None class View(DataVaultEntity, MappingSource): # name: customer_d # type: view # subtype: dimension, fact # layer: mart # attributes: # - {name: customer_id, type: 'char(40)', components: [h.customer_h_hk]} # - {name: cust_no, type: 'varchar(32)', components: [h.cust_no]} # materialize: true #default: false # materialization: # mode: merge # merge|full # target: customer_d_mat # default: name+'_mat' # layer: mart # default: same as view # mergekeys: # - customer_id # - cust_no def __init__(self, model, filename, definition: dict = None): DataVaultEntity.__init__(self, model, filename, definition) MappingSource.__init__(self, model, self) self._viewattributes = collections.OrderedDict() self.materialize = definition.get('materialize', False) self.materialization = definition.get('materialization', {}) for attrdef in definition['attributes']: self._viewattributes[attrdef.get('name')] = ViewAttribute(self, attrdef) def get_viewattributes(self, roles: list = 'all', exclude: list = ()): """returns a list of attributes for one or more given roles. You can exclude certain attribute-roles""" if 'all' in roles: return [va for va in self._viewattributes.values() if va.attribute.role not in exclude] else: return [va for va in self._viewattributes.values() if va.attribute.role in roles and va.attribute.role not in exclude] def get_viewattribute(self, name): return self._viewattributes.get(name) def safe_list_get(self, l, idx, default=None): try: return l[idx] except IndexError: return default @property def query(self): parsed_result = self.rawquery for alias, entity in self.get_query_entities().items(): if entity: include_db = False if self.dbentity.database == entity.dbentity.database else True replacement = self.model.basetemplates.get('query_entity_alias').render(entity=entity, includeDB=include_db, alias=str(alias)) parsed_result = parsed_result.replace('{' + str(entity.name) + ':' + str(alias) + '}', replacement) return parsed_result @property def rawquery(self): return self._definition.get('query', '') def get_query_entities(self): """ Parses Querystrings like: Select * from {entityname1:alias1} join {entityname2:alias2} and returns a list of entity instances. """ regex = r"\{(.*?):(.*?)?\}" entities = {} matches = re.finditer(regex, self.rawquery, re.MULTILINE) for matchNum, match in enumerate(matches): for groupNum in range(0, len(match.groups())): entities[match.group(2)] = self.model.get_entity(match.group(1)) return entities def get_referenced_entities(self): ref_entities = [] for vattr in self._viewattributes.values(): if vattr.reference: e = vattr.get_referenced_attribute().entity if e not in ref_entities: ref_entities.append(e) return ref_entities def get_query_entity_by_alias(self, alias): return self.get_query_entities().get(alias) def get_component_entities(self): return [{'entity': self, 'component': c, 'type': c.type} for c in self.get_query_entities().values()] def get_component_attributes(self, attributename): components = [] viewattribute = self.get_viewattribute(attributename) return [{'attribute': viewattribute.attribute, 'sourceentity': cattr.entity, 'sourceattribute': cattr} for cattr in viewattribute.get_components() ] @property def materialization_dbentity(self): return DBEntity(self.materialization.get('target'), self, self.model.config.layer.get(self.materialization.get('layer', self._layername )).get( 'defaultdatabaseobject'), None) @property def materialization_rawquery(self): return self.materialization.get('query', '') @property def materialization_query(self): return self.model.get_parsed_query(self, self.materialization_rawquery) def get_materialization_query_entities(self): return self.model.get_query_entities(self.materialization_rawquery) def validate(self): errors = ErrorCollection() for attr in self.attributes.values(): spec = self.layer.sys_specification errors.append(attr.validate(spec)) # Validating entity references: if self._definition.get('query'): for alias, entity in self.get_query_entities().items(): if entity is None: errors.add("VALIDATION ERROR", (self.filename, "View", "<" + self.name + ">"), f'Viewentity for alias <{alias}> not found.') #Skip next validations because of errors above: if errors.count > 0: return errors # Validating component references: viewentities = self.get_query_entities() for vattrname, vattr in self._viewattributes.items(): for comp in vattr.components: c = comp.split('.') if c[0] not in viewentities.keys(): errors.add("VALIDATION ERROR", (self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"), f'components: Viewentity for alias <{c[0]}> not found.') elif self.get_query_entity_by_alias(c[0]).get_attribute(c[1]) is None: errors.add("VALIDATION ERROR", (self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"), f'components: Attribute <{c[1]}> for alias <{c[0]}> not found.') # Validating attribute references: for vattrname, vattr in self._viewattributes.items(): if vattr.reference: ref = vattr.reference.split('.') entity = self.model.get_entity(ref[0]) if entity is None: errors.add("VALIDATION ERROR", (self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"), f'reference: Entity <{ref[0]}> not found.') elif entity.get_attribute(ref[1]) is None: errors.add("VALIDATION ERROR", (self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"), f'reference: Attribute <{ref[1]}> for entity <{ref[0]}> not found.') return errors