snapshot current state before gitea sync

DataVaultGenerator/Entities/Bridge.py

@@ -0,0 +1,82 @@
+import re
+from DataVaultGenerator.Components import DataVaultEntity, DataVaultEntityAttribute, ErrorCollection
+
+
+class Bridge(DataVaultEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+
+        self.updatemode = self._definition.get('updatemode', 'full') 
+
+    @property
+    def snapshotattribute(self):
+        return DataVaultEntityAttribute(self, self._definition['snapshotattribute'])
+
+    @property
+    def snapshotquery(self, include_db=True):
+        parsed_result = self.rawquery
+
+        for alias, entity in self.get_query_entities().items():
+            if entity:
+                replacement = self.model.basetemplates.get('query_entity_alias').render(entity=entity, includeDB=include_db, alias=str(alias))
+                parsed_result = parsed_result.replace('{' + str(entity.name) + ':' + str(alias) + '}', replacement) 
+                parsed_result = parsed_result.replace('{' + str(entity.name) + '}', replacement) 
+
+        return parsed_result
+
+    def get_query_entities(self):
+        """ Parses Querystrings like: Select * from {entityname1:alias1} join {entityname2:alias2} and returns a list
+        of entity instances. """
+        regex = r"\{(.*?):(.*?)?\}"
+
+        entities = {}
+
+        matches = re.finditer(regex, self.rawquery, re.MULTILINE)
+        for matchNum, match in enumerate(matches):
+            for groupNum in range(0, len(match.groups())):
+                entities[match.group(2)] = self.model.get_entity(match.group(1))
+
+        #regex101.com
+        regex = r"\{(.[^:]*?)\}"
+        matches = re.finditer(regex, self.rawquery, re.MULTILINE)
+        for matchNum, match in enumerate(matches):
+            for groupNum in range(0, len(match.groups())):
+                entities[match.group(1)] = self.model.get_entity(match.group(1))
+
+        return entities
+
+    @property
+    def rawquery(self):
+        return self._definition.get('snapshotquery', '')
+
+    def get_linked_entities(self): #FIXME: in docu aufnehmen?, überhaupt relevant?
+        """returns a list of linked entities."""
+        return [self.model.get_entity(le) for le in self._definition['hubs'] + self._definition.get('links',[])]
+
+   
+    def has_attributes(self):
+        return True if self._definition.get('bridgeattributes') else False
+
+    def get_bridgeattributes(self):
+        return [DataVaultEntityAttribute(self, attrdef) for attrdef in self._definition.get('bridgeattributes', [])]
+
+    def validate(self):
+        
+        errors = ErrorCollection()
+
+        for attr in self.attributes.values():
+            spec = self.layer.sys_specification
+            errors.append(attr.validate(spec))
+
+        # Validating entity references:
+        if self._definition.get('snapshotquery'):
+            for name, e in self.get_query_entities().items():
+                if e is None:
+                   errors.add("VALIDATION ERROR",
+                              (self.filename, "Bridge", "<" + self.name + ">"),
+                              f'query-entity <{name}> not found.')
+        return errors
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': c, 'type': c.type} for c in self.get_query_entities().values() if self != c ]
+     

DataVaultGenerator/Entities/Composite.py

@@ -0,0 +1,30 @@
+from DataVaultGenerator.Components import DataVaultEntity, ErrorCollection, MappingSource
+
+
+class Composite(DataVaultEntity, MappingSource):
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+        MappingSource.__init__(self, model, self)
+
+    @property
+    def query(self):
+        return self.model.get_parsed_query(self, self.rawquery)
+
+    def get_query_entities(self):
+        return self.model.get_query_entities(self.rawquery)
+
+    @property
+    def rawquery(self):
+        return self._definition.get('query', '')
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': c, 'type': c.type} for c in self.get_query_entities().values()]
+
+    def validate(self):
+
+        errors = ErrorCollection()
+        for attr in self.attributes.values():
+            spec = self.layer.sys_specification
+            errors.append(attr.validate(spec))
+        
+        return errors

DataVaultGenerator/Entities/Delivery.py

@@ -0,0 +1,89 @@
+from DataVaultGenerator.Components import DataVaultEntity, MappingSource, ErrorCollection
+
+
+class Delivery(DataVaultEntity, MappingSource):
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+        MappingSource.__init__(self, model, self)
+
+        self.properties = definition.get('properties', {})
+
+    @property
+    def delta_attribute(self):
+        return self.get_attribute(self._definition.get('deltaattribute'))
+
+    @property
+    def delta_initialvalue(self):
+        return self._definition.get('deltainitialvalue')
+
+    @property
+    def recordsource(self):
+        return self._definition.get('recordsource', '')
+
+    @property
+    def batchmode(self):
+        return self._definition.get('batchmode', 'single')  # multi, single
+
+    @property
+    def deliverymode(self):
+        return 'delta' if self._definition.get('deltaattribute') else 'full'  # multi, single
+
+    @property
+    def interfaces(self):
+        return [self.model.get_interface(i) for i in self._definition['interfaces']]
+    
+    @property
+    def ldts_source(self):
+        return self.get_attribute(self._definition.get('ldts_source'))
+    
+    @property
+    def overwrite_ldts(self):
+        return True if self._definition.get('ldts_source') else False
+
+    @property
+    def query(self):
+        return self._definition.get('query', '')
+
+    @property
+    def source_system(self):
+        if self._definition.get('sourcesystem'):
+            return self.model.get_source_system(self._definition.get('sourcesystem'))
+        else:
+            return self.interfaces[0].source_system                                         
+    
+    @property
+    def source_type(self):
+        return self._definition.get('sourcetype',self.interfaces[0].source_type)
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': c, 'type': c.type} for c in
+                self.interfaces]
+
+    def validate(self):
+
+        errors = ErrorCollection()
+        
+        for attr in self.attributes.values():
+            spec = self.layer.sys_specification
+            errors.append(attr.validate(spec))
+
+        # Validating entity references:
+        for i in self._definition['interfaces']:
+            if self.model.get_interface(i) is None:
+                errors.add("VALIDATION ERROR",
+                           (self.filename, "Delivery", "<" + self.name + ">"),
+                           f'Interface <{i}> not found')
+
+        if self._definition.get('deltaattribute'):
+            if self.delta_attribute is None:
+                errors.add("VALIDATION ERROR",
+                           (self.filename, "Delivery", "<" + self.name + ">"),
+                           f'Deltaattribute <{self._definition.get("deltaattribute")}> not found in attributes.')
+        
+        if self._definition.get('ldts_source'):
+            if self.ldts_source is None:
+                errors.add("VALIDATION ERROR",
+                           (self.filename, "Delivery", "<" + self.name + ">"),
+                            f'ldts_source <{self._definition.get("ldts_source")}> not found in attributes.')
+
+        return errors

DataVaultGenerator/Entities/GenericTable.py

@@ -0,0 +1,22 @@
+from DataVaultGenerator.Components import DataVaultEntity, ErrorCollection
+
+
+class GenericTable(DataVaultEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+
+    def get_component_entities(self):
+        c = [{'entity': self, 'component': c, 'type': c.type}
+             for c in self.model.get_entities_by_type('generictransformation') if self in c.get_target_entities()
+             ]
+        #FIXME: Um GenericTask erweitern
+        return c
+
+    def validate(self):
+
+        errors = ErrorCollection()
+        for attr in self.attributes.values():
+            spec = self.layer.sys_specification
+            errors.append(attr.validate(spec))
+        
+        return errors

DataVaultGenerator/Entities/GenericTask.py

@@ -0,0 +1,45 @@
+from DataVaultGenerator.Components import ErrorCollection, GeneratorEntity, DBEntity
+
+
+class GenericTask(GeneratorEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        GeneratorEntity.__init__(self, model, filename, definition)
+
+        self._layername = definition.get('layer', self.model.config.entitydefaults[self.type]['layer'])
+       
+   
+    @property
+    def layer(self):
+        """Returns the entity layer."""
+        return self.model.get_layer(self._layername)
+
+    def validate(self):
+
+        errors = ErrorCollection()
+        
+        # Validating entity references:
+
+        for e in self._definition.get('sources'):
+            if self.model.get_entity(e) is None:
+               errors.add("VALIDATION ERROR",
+                           (self.filename, "Generic Task", "<" + self.name + ">"),
+                           f'source <{e}> not found')
+
+        for e in self._definition.get('targets'):
+            if self.model.get_entity(e) is None:
+               errors.add("VALIDATION ERROR",
+                           (self.filename, "Generic Task", "<" + self.name + ">"),
+                           f'target <{e}> not found')
+
+        return errors
+
+    def get_source_entities(self):
+        """returns a list of linked entities."""
+        return [self.model.get_entity(e) for e in self._definition['sources']]
+
+    def get_target_entities(self):
+        """returns a list of linked entities."""
+        return [self.model.get_entity(e) for e in self._definition['targets']]
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': c, 'type': c.type} for c in self.get_source_entities()]

DataVaultGenerator/Entities/GenericTransformation.py

@@ -0,0 +1,77 @@
+from DataVaultGenerator.Components import ErrorCollection, GeneratorEntity, DBEntity
+
+
+class GenericTransformation(GeneratorEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        GeneratorEntity.__init__(self, model, filename, definition)
+
+        self._layername = definition.get('layer', self.model.config.entitydefaults[self.type]['layer'])
+       
+    @property
+    def dbentity(self):
+        return DBEntity(self.name,
+                        self,
+                        self.model.config.layer.get(self._layername).get('defaultdatabaseobject'),
+                        self._definition.get('dbentity'))
+
+    @property
+    def layer(self):
+        """Returns the entity layer."""
+        return self.model.get_layer(self._layername)
+
+    def validate(self):
+
+        errors = ErrorCollection()
+        
+        # Validating entity references:
+
+        for e in self._definition.get('sources'):
+            if self.model.get_entity(e) is None:
+               errors.add("VALIDATION ERROR",
+                           (self.filename, "Generic Transformation", "<" + self.name + ">"),
+                           f'source <{e}> not found')
+
+        for e in self._definition.get('targets'):
+            if self.model.get_entity(e) is None:
+               errors.add("VALIDATION ERROR",
+                           (self.filename, "Generic Transformation", "<" + self.name + ">"),
+                           f'target <{e}> not found')
+
+        for name, e in self.get_query_entities().items():
+            if e is None:
+               errors.add("VALIDATION ERROR",
+                           (self.filename, "Generic Transformation", "<" + self.name + ">"),
+                           f'query-entity <{name}> not found.')
+
+        for name, e in self.get_query_entities().items():
+            if name not in self._definition.get('sources') and name not in self._definition.get('targets'):
+               errors.add("VALIDATION ERROR",
+                           (self.filename, "Generic Transformation", "<" + self.name + ">"),
+                           f'query-entity <{name}> not specified as source or target.')
+
+        return errors
+
+    def get_source_entities(self):
+        """returns a list of linked entities."""
+        return [self.model.get_entity(h) for h in self._definition['sources']]
+
+    def get_target_entities(self):
+        """returns a list of linked entities."""
+        return [self.model.get_entity(h) for h in self._definition['targets']]
+
+    @property
+    def query(self):
+        return self.model.get_parsed_query(self, self.rawquery)
+
+    def get_query_entities(self):
+        return self.model.get_query_entities(self.rawquery)
+
+    @property
+    def rawquery(self):
+        return self._definition.get('query', '')
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': c, 'type': c.type} for c in self.get_source_entities()]
+
+    def get_attributes(self, roles=(), exclude=()):
+        return []

DataVaultGenerator/Entities/Hub.py

@@ -0,0 +1,84 @@
+from DataVaultGenerator.Components import DataVaultEntity, DataVaultEntityAttribute, ErrorCollection
+
+
+class Hub(DataVaultEntity):
+
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+
+        self.isCaseSensitive = definition.get('caseSesitive', False)
+        self.role_of = definition.get('roleof')
+        self.bkcc_attribute = definition.get('bkcc_attribute')
+
+        if self.role_of:
+            self.generate = 0
+
+        key = DataVaultEntityAttribute(self, self.model.config.datavault.keyattribute)
+        key.name = self.key_columnname
+        self.add_attribute(key)
+
+    @property
+    def key_columnname(self):
+        """returns name of the primary Key Attribute. If no name was defined in its definition, a template applies."""
+        return self._definition.get('key', self.model.basetemplates.get('entity_key_name').render(entity=self))
+
+    @property
+    def hash_attribute_trim(self):
+        return self._definition.get('key_treatment'
+                                    , self.model.config.datavault.business_key_treatment)\
+                                .get('trim',
+                                     self.model.config.datavault.business_key_treatment.trim)
+    @property
+    def hash_attribute_case(self):
+        return self._definition.get('key_treatment'
+                                    , self.model.config.datavault.business_key_treatment)\
+                                .get('case',
+                                     self.model.config.datavault.business_key_treatment.case)
+    @property
+    def key_attribute(self):
+        return self.get_role_attribute(self.model.config.datavault.keyattribute.get('role','sk'))
+
+    def get_satellites(self):
+        return [e for e in self.model.get_entities_by_type('satellite') if e.get_parent_entity() == self]
+
+    def get_component_entities(self):
+        c = [{'entity': self, 'component': c, 'type': c.type} for c in
+             self.get_source_entities().values()]  # holt derzeit nur die Deliveries über die Mappings
+        if self.role_of:
+            c.extend(
+                [{'entity': self, 'component': self.model.get_entity(self.role_of), 'type': 'hub'}]
+            )
+        return c
+
+    def validate(self):
+
+        errors = ErrorCollection()
+
+        for attr in self.attributes.values():
+            spec = self.layer.sys_specification
+            errors.append(attr.validate(spec))
+        
+        # Validating entity references:
+
+        # role-of-reference
+
+        if self.role_of and self.model.get_entity(self.role_of) is None:
+           errors.add("VALIDATION ERROR",
+                          (self.filename,"Hub", "<" + self.name + ">"),
+                            f'role-of Hub <{self.role_of}> not found.')
+
+        # constraints: 
+        enforce_bk_type = self.model.config.datavault.constraints.get('enforce_bk_type')
+        if enforce_bk_type:
+            for attr in self.get_attributes('base'):
+                if attr.native_datatype not in enforce_bk_type:
+                   errors.add("VALIDATION ERROR",
+                               (self.filename,"Hub", "<" + self.name + ">"),
+                                 f'Datatype of attribute <{attr.name}> not valid (enforced: {enforce_bk_type})')
+
+        return errors
+
+    def get_roles(self):
+        """returns a list of hubs with this hub as role_of-target""" 
+
+        return [e for e in self.model.get_entities_by_type('hub', generatable_only=False) if e.role_of == self.name ]

DataVaultGenerator/Entities/Interface.py

@@ -0,0 +1,40 @@
+from DataVaultGenerator.Components import DataVaultEntity, DynamicProperties, ErrorCollection
+
+
+class Interface(DataVaultEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+
+        self.properties = definition.get('properties', {})
+
+        self.prop = DynamicProperties.from_kwargs(**definition.get('properties', {}))
+ 
+
+    @property
+    def source_type(self):
+        return self._definition.get('sourcetype')
+
+    @property
+    def source_system(self):
+        return self.model.get_source_system(self._definition.get('sourcesystem'))
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': self.source_system,
+                 'type': self.source_system.type}]
+
+    def validate(self):
+
+        errors = ErrorCollection()
+        
+        # Validating sourcesystem:
+        sourcesystem = self._definition.get('sourcesystem')
+        if self.model.get_source_system(sourcesystem) is None:
+           errors.add("VALIDATION ERROR",
+                       (self.filename, "Interface", "<" + self.name + ">"),
+                       f'Sourcesystem <{sourcesystem}> not found')
+
+        for attr in self.attributes.values():
+            spec = self.source_system.sys_specification
+            errors.append(attr.validate(spec))
+
+        return errors

DataVaultGenerator/Entities/Link.py

@@ -0,0 +1,79 @@
+from DataVaultGenerator.Components import DataVaultEntity, DataVaultEntityAttribute, ErrorCollection
+
+
+class Link(DataVaultEntity):
+
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+
+        self.drivingkeys = self._definition.get('drivingkeys',[])
+        key = DataVaultEntityAttribute(self, self.model.config.datavault.keyattribute)
+        key.name = self.key_columnname
+        self.add_attribute(key)
+
+    def get_drivingkey_entities(self):
+        return [self.model.get_entity(d) for d in self.drivingkeys]
+
+    def get_foreign_attribute(self, name: str):
+        for e in self._definition['hubs']:
+            if self.model.get_entity(e).key_attribute.name == name:
+                return self.model.get_entity(e).key_attribute
+
+    def get_foreign_attributes(self):
+        fa = []
+        for e in self._definition['hubs']:
+            fa.append(DataVaultEntityAttribute(self, definition=dict(name=self.model.get_entity(e).key_attribute.name,
+                                                                     datatype=self.model.get_entity(e).key_attribute.datatype,
+                                                                     role='fk')
+                                               ))
+        return fa
+
+    @property
+    def key_columnname(self):
+        """returns name of the primary Key Attribute. If no name was defined in its definition, a template applies."""
+        return self._definition.get('key', self.model.basetemplates.get('entity_key_name').render(entity=self))
+
+    @property
+    def key_attribute(self):
+        return self.get_role_attribute(self.model.config.datavault.keyattribute.get('role','sk'))
+
+    def get_linked_entities(self):
+        """returns a list of linked entities."""
+        return [self.model.get_entity(le) for le in self._definition['hubs'] + self._definition.get('links',[])]
+
+    def get_satellites(self):
+        return [e for e in self.model.get_entities_by_type('satellite') if e.get_parent_entity() == self]
+
+    def validate(self):
+
+        errors = ErrorCollection()
+
+        for attr in self.attributes.values():
+            spec = self.layer.sys_specification
+            errors.append(attr.validate(spec))
+
+        # Validating entity references:
+        for name in self._definition['hubs']:
+            if self.model.get_entity(name) is None:
+               suggest = self.model.get_entity_name_suggestion('hub', name) 
+               suggest = f'Do you mean <{suggest}>?' if suggest else ''
+               errors.add("VALIDATION ERROR",
+                          (self.filename,"Link", "<" + self.name + ">"),
+                           f'Hub <{name}> not found. ' + suggest)
+               
+                        
+
+        # Validating entity references:
+        for name in self._definition.get('links',[]):
+            if self.model.get_entity(name) is None:
+               suggest = self.model.get_entity_name_suggestion('link', name) 
+               suggest = f'Do you mean <{suggest}>?' if suggest else ''
+               errors.add("VALIDATION ERROR",
+                          (self.filename,"Link", "<" + self.name + ">"),
+                           f'Link <{name}> not found. ' + suggest)
+
+        return errors
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': c, 'type': c.type} for c in
+                self.get_source_entities().values()]  # holt derzeit nur die Deliveries über die Mappings

DataVaultGenerator/Entities/PIT.py

@@ -0,0 +1,88 @@
+from DataVaultGenerator.Components import DataVaultEntity, DataVaultEntityAttribute, ErrorCollection
+
+
+class PIT(DataVaultEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+
+        self.baseentity = self._definition.get('baseentity')
+        self.snapshotmode = self._definition.get('snapshotmode')
+        self.snapshottable = self._definition.get('snapshottable')
+        self.snapshottableattribute = self._definition.get('snapshottableattribute')
+
+
+    def include_ledts(self):
+        return self._definition.get('include_ledts')
+
+    @property
+    def snapshotattribute(self):
+        return DataVaultEntityAttribute(self, self._definition['snapshotattribute'])
+
+    @property
+    def snapshotquery(self):
+        return self.model.get_parsed_query(self, self.rawsnapshotquery)
+
+    def get_snaphotquery_entities(self):
+        return self.model.get_query_entities(self.rawsnapshotquery)
+
+    @property
+    def rawsnapshotquery(self):
+        return self._definition.get('snapshotquery', '')
+
+    @property
+    def query(self):
+        return self.model.get_parsed_query(self, self.query)
+
+    def get_query_entities(self):
+        return self.model.get_query_entities(self.rawquery)
+
+    @property
+    def rawquery(self):
+        return self._definition.get('query', '')
+
+    def get_base_entity(self):
+        return self.model.get_entity(self.baseentity)
+
+    def get_satellites(self):
+        return [self.model.get_entity(sat) for sat in self._definition.get('satellites')]
+
+    def has_attributes(self):
+        return True if self._definition.get('pitattributes') else False
+
+    def get_pitattributes(self):
+        attributes = []
+
+        for attr in self._definition.get('pitattributes', []):
+            attributes.append(self.model.get_entity(attr[0]).get_attribute(attr[1]))
+           # attributes.append({'attribute': self.model.get_entity(attr[1]).get_attribute(attr[2])
+           #                   , 'alias': attr[0]})
+        return attributes
+
+    def validate(self):
+        
+        errors = ErrorCollection()
+
+        # Validating entity references:
+        if self._definition.get('snapshotquery'):
+            for name, e in self.get_snaphotquery_entities().items():
+                if e is None:
+                   errors.add("VALIDATION ERROR",
+                              (self.filename, "PIT", "<" + self.name + ">"),
+                              f'query-entity <{name}> not found.')
+
+        if self._definition.get('query'):
+            for name, e in self.get_query_entities().items():
+                if e is None:
+                   errors.add("VALIDATION ERROR",
+                              (self.filename, "PIT", "<" + self.name + ">"),
+                              f'query-entity <{name}> not found.')
+
+        return errors
+
+    def get_component_entities(self):
+        c = [{'entity': self, 'component': c, 'type': c.type} for c in self.get_snaphotquery_entities().values()]
+        c.extend([{'entity': self, 'component': c, 'type': c.type} for c in self.get_query_entities().values()])
+        c.extend([{'entity': self, 'component': c, 'type': c.type} for c in self.get_satellites()])
+        c.extend([{'entity': self, 'component': self.get_base_entity(), 'type': self.get_base_entity().type}])
+
+        return c

DataVaultGenerator/Entities/Reference.py

@@ -0,0 +1,39 @@
+from DataVaultGenerator.Components import DataVaultEntity
+
+#TODO: um Möglichkeit, einen Satellite dranzuhängen erweitern sowie Laden über Mapping ermöglichen
+class Reference(DataVaultEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+
+    @property
+    def key_attribute(self): #FIXME: nur EIN Key-Attribute könnte bei Referenz-Daten zu wenig sein. 
+        return self.get_role_attribute('key')
+        #FIXME: Analag zu dem keyattribute (hk) müsste es für reference Daten einen globalen typ geben mit NOT NULL - Sonst gibt es Probleme beim PK-Index der NICHT NULL sein darf. Workarround: am key-attribue mandatory: true definieren
+
+    def get_satellites(self):
+        return [e for e in self.model.get_entities_by_type('satellite') if e.get_parent_entity() == self]
+
+    @property
+    def data(self):
+        return self._definition.get('data', {})
+
+    @property
+    def query(self):
+        return self.model.get_parsed_query(self, self.rawquery)
+
+    def get_query_entities(self):
+        return self.model.get_query_entities(self.rawquery)
+
+    @property
+    def rawquery(self):
+        return self._definition.get('query', '')
+
+    def get_component_entities(self):
+        c = [{'entity': self, 'component': c, 'type': c.type} for c in
+             self.get_source_entities().values()]  # holt derzeit nur die Deliveries über die Mappings
+
+        c.extend([{'entity': self, 'component': c, 'type': c.type}
+                for c in self.model.get_entities_by_type('generictransformation') if self in c.get_target_entities()
+                ])
+    #FIXME: um generictask erweitern
+        return c

DataVaultGenerator/Entities/Report.py

@@ -0,0 +1,26 @@
+from DataVaultGenerator.Components import ErrorCollection, GeneratorEntity
+
+
+class Report(GeneratorEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        GeneratorEntity.__init__(self, model, filename, definition)
+
+        self._layername = definition.get('layer', self.model.config.entitydefaults[self.type]['layer'])
+
+    @property
+    def layer(self):
+        """Returns the entity layer."""
+        return self.model.get_layer(self._layername)
+
+    @property
+    def dbentity(self):
+        return None
+
+    def get_component_entities(self):
+        return []
+
+    def get_attributes(self, roles=(), exclude=()):
+        return []
+
+    def validate(self):
+        return ErrorCollection()

DataVaultGenerator/Entities/Satellite.py

@@ -0,0 +1,54 @@
+from DataVaultGenerator.Components import DataVaultEntity, DataVaultEntityAttribute, ErrorCollection
+
+
+class Satellite(DataVaultEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+
+        self.parent = definition.get('parent')
+
+    def get_foreign_attribute(self, name: str) -> DataVaultEntityAttribute:
+        return self.get_parent_key_attribute() if self.get_parent_key_attribute().name == name else None
+
+    def get_parent_entity(self) -> DataVaultEntity:
+        return self.model.get_entity(self.parent)
+
+    def get_parent_key_attribute(self) -> DataVaultEntityAttribute:
+        return self.get_parent_entity().key_attribute
+
+    @property
+    def hashdiff_fk_attribute(self):#FIXME: Durch config/template flexibler gestalten 
+        return self.get_role_attribute('hashdiff').copy(self.name + "_" + self.get_role_attribute('hashdiff').name)
+
+    @property
+    def hash_attribute_trim(self):
+        return self._definition.get('hashdiff_attribute_treatment'
+                                    , self.model.config.datavault.hashdiff_attribute_treatment)\
+                                .get('trim',
+                                     self.model.config.datavault.hashdiff_attribute_treatment.trim)
+    @property
+    def hash_attribute_case(self):
+        return self._definition.get('hashdiff_attribute_treatment'
+                                    , self.model.config.datavault.hashdiff_attribute_treatment)\
+                                .get('case',
+                                     self.model.config.datavault.hashdiff_attribute_treatment.case)
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': c, 'type': c.type} for c in
+                self.get_source_entities().values()]  # holt derzeit nur die Deliveries über die Mappings
+
+    def validate(self):
+
+        errors = ErrorCollection()
+
+        for attr in self.attributes.values():
+            spec = self.layer.sys_specification
+            errors.append(attr.validate(spec))
+
+        # Validating entity references:
+        if self.get_parent_entity() is None:
+           errors.add("VALIDATION ERROR",
+                       (self.filename,"Satellite", "<" + self.name + ">"),
+                       f'Parent <{self.parent}> not found')
+
+        return errors

DataVaultGenerator/Entities/Sourcesystem.py

@@ -0,0 +1,22 @@
+from DataVaultGenerator.Components import GeneratorEntity
+
+
+class SourceSystem(GeneratorEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        GeneratorEntity.__init__(self, model, filename, definition)
+        self.shortname = self._definition.get('shortname', self.name)
+        self.sys_specification = self._definition.get('sys_specification', '')
+
+    def get_interfaces(self):
+        return [i for i in self.model.interfaces.values() if i.source_system == self]
+
+    def get_interface_count(self):
+        return sum(1 for i in self.model.interfaces.values() if i.source_system == self)
+
+    @property
+    def connection_name(self):
+        return self._definition.get('connectionname', '')
+
+    @property
+    def sourcesystem_type(self):
+        return self._definition.get('sourcesystemtype', '')

DataVaultGenerator/Entities/SubDag.py

@@ -0,0 +1,84 @@
+from DataVaultGenerator.Dag import DagNode
+from DataVaultGenerator.Components import ErrorCollection, GeneratorEntity
+
+
+class SubDag(GeneratorEntity):
+    def __init__(self, model, filename, definition: dict = None):
+        GeneratorEntity.__init__(self, model, filename, definition)
+
+        self.entrypoints = definition.get('entrypoints',[])
+        self.key = definition.get('key',definition.get('name'))
+        
+        self.excludes = definition.get('excludes',[]) 
+
+        self.tree = []
+
+    def validate(self):
+
+        errors = ErrorCollection()
+        
+        # Validating entity references:
+
+        for ep in self.entrypoints: 
+            if self.model.get_entity(ep) is None:
+               errors.add("VALIDATION ERROR",
+                           (self.filename,"SubDag", "<" + self.name + ">"),
+                           f'Entrypoint <{ep}> not found')
+                                        
+        for ex in self.excludes: 
+            if self.model.get_entity(ex) is None:
+               errors.add("VALIDATION ERROR",
+                           (self.filename,"SubDag", "<" + self.name + ">"),
+                           f'Exclude <{ex}> not found')
+
+        return errors
+
+    def get_entrypoints_nodes(self):
+        if self.entrypoints:
+            return [self.model.dag.get_node(n) for n in self.entrypoints]
+        else: 
+            return [n for n in self.model.dag.get_roots()]    
+
+    def get_tree(self):
+        return self.get_nodes()
+
+    def get_nodes(self):
+        self.model.dag.reset()
+
+        if self.subtype == 'forward':   
+            r = []
+            for en in self.get_entrypoints_nodes():
+                r.extend(self.model.dag.get_forward_tree(en,excludes=self.excludes))
+            self.tree = self.dedup_tree(r)
+            return self.tree
+
+        if self.subtype == 'backward': 
+            r = []
+            for en in self.get_entrypoints_nodes():
+                r.extend(self.model.dag.get_backward_tree(en))
+            
+            r = self.model.dag.reverse_level(r)
+            self.tree = self.dedup_tree(r)
+            return self.tree
+        
+        return []
+
+    def dedup_tree(self, tree: list):
+        dedup = {}
+        for e in tree: 
+            if e.name not in dedup:
+                dedup[e.name] = e
+            elif dedup[e.name].level < e.level: # Replace if existing elements level is lower than current elements level
+                dedup[e.name] = e
+        return [e for e in dedup.values()]
+
+
+
+    def get_leveldict(self, nodes: list) -> dict:
+        # returns dict. Each key represents one level. Each level contains a list of nodes. 
+        ld = dict()
+        for n in nodes:
+            if n.level not in ld:
+               ld[n.level] = []
+               ld[n.level].append(n)
+        return ld

DataVaultGenerator/Entities/View.py

@@ -0,0 +1,200 @@
+import collections
+import re
+from DataVaultGenerator.Components import DataVaultEntity, MappingSource, DBEntity, ErrorCollection
+
+
+class ViewAttribute():
+    def __init__(self, entity, definition): 
+        self.entity = entity
+        self.definition = definition
+        self.name = definition.get('name')
+        self.attribute = entity.get_attribute(self.name) # DataVaultEntityAttribute
+        self.components = definition.get('components', [])
+        self.reference = definition.get('reference', '') # entity.attribute
+        self.referencetype = definition.get('referencetype', '') # 1:n, m:n, ..
+        self.order = definition.get('order')
+
+    def get_components(self):
+        # returns the component attribute instances
+        return [self.entity.get_query_entity_by_alias(c.split('.')[0]).get_attribute(c.split('.')[1]) for c in self.components]
+
+    def get_referenced_attribute(self):
+        if self.reference:
+            ref = self.reference.split('.') # reference: entityname.attributename
+            return self.entity.model.get_entity(ref[0]).get_attribute(ref[1]) 
+        else: 
+            return None
+
+class View(DataVaultEntity, MappingSource):
+
+    # name: customer_d
+    # type: view
+    # subtype: dimension, fact  
+    # layer: mart
+    # attributes: 
+    # - {name: customer_id, type: 'char(40)', components: [h.customer_h_hk]}
+    # - {name: cust_no, type: 'varchar(32)', components: [h.cust_no]}
+    # materialize: true #default: false
+    # materialization:
+    #   mode: merge # merge|full
+    #   target: customer_d_mat # default: name+'_mat'
+    #   layer: mart # default: same as view
+    #   mergekeys:
+    #     - customer_id
+    #     - cust_no
+
+    def __init__(self, model, filename, definition: dict = None):
+        DataVaultEntity.__init__(self, model, filename, definition)
+        MappingSource.__init__(self, model, self)
+        
+        self._viewattributes = collections.OrderedDict()
+
+        self.materialize = definition.get('materialize', False)
+        self.materialization = definition.get('materialization', {})
+
+        for attrdef in definition['attributes']:
+            self._viewattributes[attrdef.get('name')] = ViewAttribute(self, attrdef)
+
+    def get_viewattributes(self, roles: list = 'all', exclude: list = ()):
+        """returns a list of attributes for one or more given roles. You can exclude certain attribute-roles"""
+        if 'all' in roles:
+            return [va for va in self._viewattributes.values() if va.attribute.role not in exclude]
+        else:
+            return [va for va in self._viewattributes.values() if va.attribute.role in roles and va.attribute.role not in exclude]
+
+    def get_viewattribute(self, name):
+        return self._viewattributes.get(name)
+
+    def safe_list_get(self, l, idx, default=None):
+        try:
+            return l[idx]
+        except IndexError:
+            return default
+
+    @property
+    def query(self):
+
+        parsed_result = self.rawquery
+
+        for alias, entity in self.get_query_entities().items():
+            if entity:
+                include_db = False if self.dbentity.database == entity.dbentity.database else True
+                replacement = self.model.basetemplates.get('query_entity_alias').render(entity=entity, includeDB=include_db, alias=str(alias))
+                parsed_result = parsed_result.replace('{' + str(entity.name) + ':' + str(alias) + '}', replacement) 
+
+        return parsed_result
+
+    @property
+    def rawquery(self):
+        return self._definition.get('query', '')
+
+    def get_query_entities(self):
+        """ Parses Querystrings like: Select * from {entityname1:alias1} join {entityname2:alias2} and returns a list
+        of entity instances. """
+        regex = r"\{(.*?):(.*?)?\}"
+
+        entities = {}
+
+        matches = re.finditer(regex, self.rawquery, re.MULTILINE)
+        for matchNum, match in enumerate(matches):
+            for groupNum in range(0, len(match.groups())):
+                entities[match.group(2)] = self.model.get_entity(match.group(1))
+
+        return entities
+
+    def get_referenced_entities(self):
+        ref_entities = []
+        for vattr in self._viewattributes.values():
+            if vattr.reference:
+                e = vattr.get_referenced_attribute().entity
+                if e not in ref_entities:
+                    ref_entities.append(e)
+        return ref_entities
+
+    def get_query_entity_by_alias(self, alias):
+        return self.get_query_entities().get(alias)
+
+    def get_component_entities(self):
+        return [{'entity': self, 'component': c, 'type': c.type} for c in self.get_query_entities().values()]
+
+    def get_component_attributes(self, attributename):
+        components = []
+
+        viewattribute = self.get_viewattribute(attributename)
+
+        return [{'attribute': viewattribute.attribute,
+                 'sourceentity': cattr.entity,
+                 'sourceattribute': cattr} for cattr in viewattribute.get_components() ]
+
+   
+    @property
+    def materialization_dbentity(self):
+        return DBEntity(self.materialization.get('target'),
+                        self,
+                        self.model.config.layer.get(self.materialization.get('layer', self._layername )).get( 
+                            'defaultdatabaseobject'),
+                        None)
+
+    @property
+    def materialization_rawquery(self):
+        return self.materialization.get('query', '')
+
+    @property
+    def materialization_query(self):
+        return self.model.get_parsed_query(self, self.materialization_rawquery)
+
+    def get_materialization_query_entities(self):
+        return self.model.get_query_entities(self.materialization_rawquery)
+
+
+    def validate(self):
+
+        errors = ErrorCollection()
+
+        for attr in self.attributes.values():
+            spec = self.layer.sys_specification
+            errors.append(attr.validate(spec))
+
+        # Validating entity references:
+        if self._definition.get('query'):
+            for alias, entity in self.get_query_entities().items():
+                if entity is None:
+                   errors.add("VALIDATION ERROR", 
+                               (self.filename, "View", "<" + self.name + ">"),
+                               f'Viewentity for alias <{alias}> not found.')
+                               
+        #Skip next validations because of errors above: 
+        if errors.count > 0:
+           return errors
+
+         # Validating component references:
+        viewentities = self.get_query_entities()
+        for vattrname, vattr in self._viewattributes.items():
+            for comp in vattr.components:
+                c = comp.split('.')
+                if c[0] not in viewentities.keys():
+                   errors.add("VALIDATION ERROR", 
+                              (self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"),
+                               f'components: Viewentity for alias <{c[0]}> not found.')
+                 
+                elif self.get_query_entity_by_alias(c[0]).get_attribute(c[1]) is None:
+                     errors.add("VALIDATION ERROR", 
+                               (self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"),
+                               f'components: Attribute <{c[1]}> for alias <{c[0]}> not found.')
+             
+        # Validating attribute references:
+        for vattrname, vattr in self._viewattributes.items():
+             if vattr.reference: 
+                ref = vattr.reference.split('.')
+                entity = self.model.get_entity(ref[0])
+                
+                if entity is None: 
+                   errors.add("VALIDATION ERROR",
+                                                (self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"),
+                                                 f'reference: Entity <{ref[0]}> not found.')
+                elif entity.get_attribute(ref[1]) is None:
+                     errors.add("VALIDATION ERROR",
+                                                (self.filename, "View", "<" + self.name + ">", "Attribute <" + vattrname + ">"),
+                                              f'reference: Attribute <{ref[1]}> for entity <{ref[0]}> not found.')
+
+        return errors