200 lines
8.1 KiB
Python
200 lines
8.1 KiB
Python
import collections
|
|
import re
|
|
from DataVaultGenerator.Components import DataVaultEntity, MappingSource, DBEntity, ErrorCollection
|
|
|
|
|
|
class ViewAttribute():
|
|
def __init__(self, entity, definition):
|
|
self.entity = entity
|
|
self.definition = definition
|
|
self.name = definition.get('name')
|
|
self.attribute = entity.get_attribute(self.name) # DataVaultEntityAttribute
|
|
self.components = definition.get('components', [])
|
|
self.reference = definition.get('reference', '') # entity.attribute
|
|
self.referencetype = definition.get('referencetype', '') # 1:n, m:n, ..
|
|
self.order = definition.get('order')
|
|
|
|
def get_components(self):
|
|
# returns the component attribute instances
|
|
return [self.entity.get_query_entity_by_alias(c.split('.')[0]).get_attribute(c.split('.')[1]) for c in self.components]
|
|
|
|
def get_referenced_attribute(self):
|
|
if self.reference:
|
|
ref = self.reference.split('.') # reference: entityname.attributename
|
|
return self.entity.model.get_entity(ref[0]).get_attribute(ref[1])
|
|
else:
|
|
return None
|
|
|
|
class View(DataVaultEntity, MappingSource):
|
|
|
|
# name: customer_d
|
|
# type: view
|
|
# subtype: dimension, fact
|
|
# layer: mart
|
|
# attributes:
|
|
# - {name: customer_id, type: 'char(40)', components: [h.customer_h_hk]}
|
|
# - {name: cust_no, type: 'varchar(32)', components: [h.cust_no]}
|
|
# materialize: true #default: false
|
|
# materialization:
|
|
# mode: merge # merge|full
|
|
# target: customer_d_mat # default: name+'_mat'
|
|
# layer: mart # default: same as view
|
|
# mergekeys:
|
|
# - customer_id
|
|
# - cust_no
|
|
|
|
def __init__(self, model, filename, definition: dict = None):
|
|
DataVaultEntity.__init__(self, model, filename, definition)
|
|
MappingSource.__init__(self, model, self)
|
|
|
|
self._viewattributes = collections.OrderedDict()
|
|
|
|
self.materialize = definition.get('materialize', False)
|
|
self.materialization = definition.get('materialization', {})
|
|
|
|
for attrdef in definition['attributes']:
|
|
self._viewattributes[attrdef.get('name')] = ViewAttribute(self, attrdef)
|
|
|
|
def get_viewattributes(self, roles: list = 'all', exclude: list = ()):
|
|
"""returns a list of attributes for one or more given roles. You can exclude certain attribute-roles"""
|
|
if 'all' in roles:
|
|
return [va for va in self._viewattributes.values() if va.attribute.role not in exclude]
|
|
else:
|
|
return [va for va in self._viewattributes.values() if va.attribute.role in roles and va.attribute.role not in exclude]
|
|
|
|
def get_viewattribute(self, name):
|
|
return self._viewattributes.get(name)
|
|
|
|
def safe_list_get(self, l, idx, default=None):
|
|
try:
|
|
return l[idx]
|
|
except IndexError:
|
|
return default
|
|
|
|
@property
|
|
def query(self):
|
|
|
|
parsed_result = self.rawquery
|
|
|
|
for alias, entity in self.get_query_entities().items():
|
|
if entity:
|
|
include_db = False if self.dbentity.database == entity.dbentity.database else True
|
|
replacement = self.model.basetemplates.get('query_entity_alias').render(entity=entity, includeDB=include_db, alias=str(alias))
|
|
parsed_result = parsed_result.replace('{' + str(entity.name) + ':' + str(alias) + '}', replacement)
|
|
|
|
return parsed_result
|
|
|
|
@property
|
|
def rawquery(self):
|
|
return self._definition.get('query', '')
|
|
|
|
def get_query_entities(self):
|
|
""" Parses Querystrings like: Select * from {entityname1:alias1} join {entityname2:alias2} and returns a list
|
|
of entity instances. """
|
|
regex = r"\{(.*?):(.*?)?\}"
|
|
|
|
entities = {}
|
|
|
|
matches = re.finditer(regex, self.rawquery, re.MULTILINE)
|
|
for matchNum, match in enumerate(matches):
|
|
for groupNum in range(0, len(match.groups())):
|
|
entities[match.group(2)] = self.model.get_entity(match.group(1))
|
|
|
|
return entities
|
|
|
|
def get_referenced_entities(self):
|
|
ref_entities = []
|
|
for vattr in self._viewattributes.values():
|
|
if vattr.reference:
|
|
e = vattr.get_referenced_attribute().entity
|
|
if e not in ref_entities:
|
|
ref_entities.append(e)
|
|
return ref_entities
|
|
|
|
def get_query_entity_by_alias(self, alias):
|
|
return self.get_query_entities().get(alias)
|
|
|
|
def get_component_entities(self):
|
|
return [{'entity': self, 'component': c, 'type': c.type} for c in self.get_query_entities().values()]
|
|
|
|
def get_component_attributes(self, attributename):
|
|
components = []
|
|
|
|
viewattribute = self.get_viewattribute(attributename)
|
|
|
|
return [{'attribute': viewattribute.attribute,
|
|
'sourceentity': cattr.entity,
|
|
'sourceattribute': cattr} for cattr in viewattribute.get_components() ]
|
|
|
|
|
|
@property
|
|
def materialization_dbentity(self):
|
|
return DBEntity(self.materialization.get('target'),
|
|
self,
|
|
self.model.config.layer.get(self.materialization.get('layer', self._layername )).get(
|
|
'defaultdatabaseobject'),
|
|
None)
|
|
|
|
@property
|
|
def materialization_rawquery(self):
|
|
return self.materialization.get('query', '')
|
|
|
|
@property
|
|
def materialization_query(self):
|
|
return self.model.get_parsed_query(self, self.materialization_rawquery)
|
|
|
|
def get_materialization_query_entities(self):
|
|
return self.model.get_query_entities(self.materialization_rawquery)
|
|
|
|
|
|
def validate(self):
|
|
|
|
errors = ErrorCollection()
|
|
|
|
for attr in self.attributes.values():
|
|
spec = self.layer.sys_specification
|
|
errors.append(attr.validate(spec))
|
|
|
|
# Validating entity references:
|
|
if self._definition.get('query'):
|
|
for alias, entity in self.get_query_entities().items():
|
|
if entity is None:
|
|
errors.add("VALIDATION ERROR",
|
|
(self.filename, "View", "<" + self.name + ">"),
|
|
f'Viewentity for alias <{alias}> not found.')
|
|
|
|
#Skip next validations because of errors above:
|
|
if errors.count > 0:
|
|
return errors
|
|
|
|
# Validating component references:
|
|
viewentities = self.get_query_entities()
|
|
for vattrname, vattr in self._viewattributes.items():
|
|
for comp in vattr.components:
|
|
c = comp.split('.')
|
|
if c[0] not in viewentities.keys():
|
|
errors.add("VALIDATION ERROR",
|
|
(self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"),
|
|
f'components: Viewentity for alias <{c[0]}> not found.')
|
|
|
|
elif self.get_query_entity_by_alias(c[0]).get_attribute(c[1]) is None:
|
|
errors.add("VALIDATION ERROR",
|
|
(self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"),
|
|
f'components: Attribute <{c[1]}> for alias <{c[0]}> not found.')
|
|
|
|
# Validating attribute references:
|
|
for vattrname, vattr in self._viewattributes.items():
|
|
if vattr.reference:
|
|
ref = vattr.reference.split('.')
|
|
entity = self.model.get_entity(ref[0])
|
|
|
|
if entity is None:
|
|
errors.add("VALIDATION ERROR",
|
|
(self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"),
|
|
f'reference: Entity <{ref[0]}> not found.')
|
|
elif entity.get_attribute(ref[1]) is None:
|
|
errors.add("VALIDATION ERROR",
|
|
(self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"),
|
|
f'reference: Attribute <{ref[1]}> for entity <{ref[0]}> not found.')
|
|
|
|
return errors |