snapshot current state before gitea sync

DataVaultGenerator/schema/config.yaml

@@ -0,0 +1,286 @@
+type: config
+name: Configuration
+displayname: Configuration
+namespace: model
+schema: 
+# ------------------------------------------------------------------------------------------------
+#   Model 
+# ------------------------------------------------------------------------------------------------
+
+    model: 
+      type: dict
+      schema: 
+        name: 
+          type: string
+        paths: 
+          type: dict
+          schema: 
+            log:
+              type: string
+              required: True
+            entities:
+              type: string
+              required: True
+            mappings:
+              type: string
+              required: True
+            templates:
+              type: string
+              required: True
+            output:
+              type: string
+              required: True
+        ignore_file_prefix:
+          type: string
+
+# ------------------------------------------------------------------------------------------------
+#   Variables 
+# ------------------------------------------------------------------------------------------------
+    vars:
+      type: dict
+
+# ------------------------------------------------------------------------------------------------
+#   Hooks 
+# ------------------------------------------------------------------------------------------------
+    pre_hooks:
+      type: dict
+      valuesrules:
+        type: list
+
+    post_hooks:
+      type: dict     
+      valuesrules:
+        type: list        
+# ------------------------------------------------------------------------------------------------
+#   Key Definition
+# ------------------------------------------------------------------------------------------------
+
+    keyattribute: 
+      type: dict
+      required: True
+      schema: 
+        type:
+          type: string
+          required: True
+        role:
+          type: string
+          required: True
+        mandatory:
+          type: boolean
+        ghost: 
+          type: string
+    
+    zerokey: 
+      type: string
+
+# ------------------------------------------------------------------------------------------------
+#   Modelling Constraints
+# ------------------------------------------------------------------------------------------------
+
+    constraints: 
+      type: dict
+      schema:
+        enforce_bk_type:
+          type: [string, list] 
+
+# ------------------------------------------------------------------------------------------------
+#   HASH Definition
+# ------------------------------------------------------------------------------------------------
+
+    hash_algorithm:
+      type: string
+      required: true
+    hash_separator:
+      type: string
+      required: true
+    hash_case:
+      type: string
+      required: True        
+      allowed: ['upper', 'lower', 'keep']
+      
+# ------------------------------------------------------------------------------------------------
+#   Business Key Treatment
+# ------------------------------------------------------------------------------------------------
+
+    business_key_treatment:
+      type: dict
+      required: True
+      schema:
+        trim:
+          type: string
+          required: True
+          allowed: ['left', 'right', 'both']
+        case:
+          type: string
+          required: True        
+          allowed: ['upper', 'lower', 'keep']
+
+# ------------------------------------------------------------------------------------------------
+#   Hashdiff Attribute Treatment
+# ------------------------------------------------------------------------------------------------
+
+    hashdiff_attribute_treatment:
+      type: dict
+      required: True
+      schema:
+        trim:
+          type: string
+          required: True
+          allowed: ['left', 'right', 'both']
+        case:
+          type: string
+          required: True
+          allowed: ['upper', 'lower', 'keep']
+
+# ------------------------------------------------------------------------------------------------
+#   Common Attributes
+# ------------------------------------------------------------------------------------------------
+
+    commonattributes:
+      type: dict
+      valuesrules: 
+          type: dict
+          schema: 
+            name: 
+              type: string
+              required: True
+            type: 
+              type: string
+              required: True
+            mandatory: 
+              type: boolean
+            ghost:
+              type: string
+
+# ------------------------------------------------------------------------------------------------
+#   Ghost-records
+# ------------------------------------------------------------------------------------------------
+
+    ghostrecord: 
+      type: dict
+      valuesrules:
+        type: string
+
+# ------------------------------------------------------------------------------------------------
+#   Layer
+# ------------------------------------------------------------------------------------------------
+
+    layer:
+      type: dict
+      valuesrules: 
+        type: dict
+        schema: 
+          name: 
+            type: string
+            required: True
+          description: 
+            type: string
+          connectionname: 
+            type: string
+          sys_specification:
+            type: string
+          defaultdatabaseobject: 
+            type: dict
+            schema: 
+              database: 
+                type: string
+              schema:
+                type: string
+              filegroup: 
+                type: string
+              properties:
+                type: dict
+    
+# ------------------------------------------------------------------------------------------------
+#   Entity Defaults
+# ------------------------------------------------------------------------------------------------
+
+    entitydefaults:
+      type: dict
+      valuesrules:
+        type: dict
+        schema: 
+          layer:
+            type: string
+            required: True
+          attributes: 
+            type: list
+            schema:
+              type: string
+          attribute_role:
+            type: string
+          map_implicit_roles:
+            type: list
+            schema:
+              type: string
+          extra:
+            type: dict
+
+# ------------------------------------------------------------------------------------------------
+#   Generator config
+# ------------------------------------------------------------------------------------------------
+    
+    generator: 
+      type: dict
+      valuesrules: 
+        type: dict
+        schema: 
+          templates: 
+            type: list
+            schema: 
+              type: dict
+              schema: 
+                subtype:
+                  type: [string,list]
+                template:
+                  type: string
+                  required: True
+                targetfolder: 
+                  type: string
+                  required: True
+                filename: 
+                  type: string
+                  required: True
+                lang: 
+                  type: string
+                  required: True
+                synchtarget:
+                  type: string
+
+# ------------------------------------------------------------------------------------------------
+#   Base templates
+# ------------------------------------------------------------------------------------------------            
+         
+    templates:
+      type: dict
+      schema:
+        column_ddl:
+          type: string
+          required: True
+        table_qualifier: 
+          type: string
+          required: True
+        attribute_expression: 
+          type: string
+          required: True
+        entity_key_name:         
+          type: string
+          required: True
+        query_entity_alias:
+          type: string
+          required: True
+
+# ------------------------------------------------------------------------------------------------
+#   Template Engine
+# ------------------------------------------------------------------------------------------------            
+         
+    jinja:
+      type: dict
+      
+# ------------------------------------------------------------------------------------------------
+#   sys_specifications
+# ------------------------------------------------------------------------------------------------            
+         
+    sys_specification:
+      type: dict
+      

DataVaultGenerator/schema/entities/bridge.yaml

@@ -0,0 +1,52 @@
+type: bridge
+name: bridge
+displayname: Bridge Table
+namespace: model
+schema: 
+# GeneratorEntity
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+    allowed: [base]
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+  exclude_commonattributes:
+    type: list
+# Specific
+  updatemode:
+    type: string
+    allowed: [full, merge, append, custom]
+  snapshotattribute: 
+    type: dict
+    schema: attribute
+  snapshotquery: 
+    type: string
+  bridgeattributes: attributes
+  hubs: 
+    type: list
+    schema:
+      type: string
+  links: 
+    type: list
+    schema:
+      type: string
+       

DataVaultGenerator/schema/entities/composite.yaml

@@ -0,0 +1,49 @@
+type: composite
+name: composite
+displayname: Composite
+namespace: model
+schema: 
+# Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    type: dict
+    schema: 
+      name: 
+        type: string
+      filegroup:
+        type: string
+      database:
+        type: string
+      schema:
+        type: string
+      properties: 
+        type: dict
+  exclude_commonattributes:
+    type: list
+# Attributes:
+  attributes: attributes
+# Specific:
+  query: 
+    type: string
+    required: True

DataVaultGenerator/schema/entities/delivery.yaml

@@ -0,0 +1,60 @@
+type: delivery
+name: delivery
+displayname: Delivery
+namespace: model
+schema: 
+# Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+  exclude_commonattributes:
+    type: list
+# Specific:
+  recordsource: 
+    type: string
+  batchmode: 
+    type: string
+  deltaattribute:
+    type: string  
+  deltainitialvalue:
+    type: string  
+  query: 
+    type: string
+  interfaces:
+    type: list
+    required: True
+    schema: 
+      type: string  
+  sourcesystem: 
+    type: string
+  sourcetype: 
+    type: string
+  ldts_source: 
+    type: string
+  properties:
+    type: dict
+
+# Attributes:
+  attributes: attributes
+   

DataVaultGenerator/schema/entities/generictable.yaml

@@ -0,0 +1,37 @@
+type: generictable
+name: generictable
+displayname: Generic Table
+namespace: model
+schema: 
+# Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+    
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+  exclude_commonattributes:
+    type: list
+# Specific:
+
+# Attributes:
+  attributes: attributes

DataVaultGenerator/schema/entities/generictask.yaml

@@ -0,0 +1,39 @@
+type: generictask
+name: generictask
+displayname: Generic Task
+namespace: model
+schema: 
+# Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+    required: True
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+
+# Specific:
+  sources: 
+    type: list
+    required: True
+  targets: 
+    type: list
+    required: True

DataVaultGenerator/schema/entities/generictransformation.yaml

@@ -0,0 +1,41 @@
+type: generictransformation
+name: generictransformation
+displayname: Generic Transformation
+namespace: model
+schema: 
+# Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+
+# Specific:
+  query: 
+    type: string
+    required: True
+  sources: 
+    type: list
+    required: True
+  targets: 
+    type: list
+    required: True

DataVaultGenerator/schema/entities/hub.yaml

@@ -0,0 +1,59 @@
+type: hub
+name: Hub
+displayname: Hub
+namespace: model
+schema: 
+# GeneratorEntity
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+
+# Specific
+  key: 
+    type: string
+  key_treatment:
+    type: dict
+    schema:
+      trim:
+         type: string
+         allowed: ['left', 'right', 'both']
+      case:
+         type: string
+         allowed: ['upper', 'lower', 'keep']
+  roleof: 
+    type: string
+  caseSesitive:
+    type: integer
+    allowed: [0,1]   
+
+# Attributes:
+  attributes: attributes
+
+boilerplate: |
+  name: {unique_name}
+  type: hub
+  key: primary_key_name # Hashkey
+  description: 'optional description'
+  attributes:
+    - {name: 'businesskey1', type: 'nvarchar(200)'}

DataVaultGenerator/schema/entities/link.yaml

@@ -0,0 +1,52 @@
+type: link
+name: Link
+displayname: Link
+namespace: model
+schema: 
+# GeneratorEntity
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+  exclude_commonattributes:
+    type: list
+# Specific
+  key: 
+    type: string
+  hubs: 
+    type: list
+    required: True
+    schema:
+      type: string
+  links: 
+    type: list
+    schema:
+      type: string
+  drivingkeys: 
+    type: list
+    schema:
+      type: string
+      
+# Attributes:
+  attributes: attributes
+        

DataVaultGenerator/schema/entities/pit.yaml

@@ -0,0 +1,61 @@
+type: pit
+name: pit
+displayname: Point in Time Table
+namespace: model
+schema: 
+# GeneratorEntity
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+    allowed: [base]
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+  exclude_commonattributes:
+    type: list
+# Specific
+  snapshotmode:
+    type: string
+    required: True
+    allowed: [latest, snapshotquery, full, snapshottable]
+  baseentity: 
+    type: string
+    required: True  
+  satellites:
+    type: list
+    required: True
+    schema:
+      type: string
+  snapshotattribute: 
+    type: dict
+    schema: attribute
+  snapshottable: 
+    type: string
+  snapshottableattribute:
+    type: string
+  snapshotquery: 
+    type: string
+  pitattributes:
+    type: list
+    schema:
+      type: list
+  query:
+    type: string

DataVaultGenerator/schema/entities/reference.yaml

@@ -0,0 +1,41 @@
+type: reference
+name: reference
+displayname: Reference Table
+namespace: model
+schema: 
+# GeneratorEntity
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+  exclude_commonattributes:
+    type: list
+# Specific
+  data:
+    type: list
+    schema:
+      type: list
+  query:
+    type: string
+# Attributes:
+  attributes: attributes

DataVaultGenerator/schema/entities/report.yaml

@@ -0,0 +1,6 @@
+type: report
+name: report
+displayname: Report
+namespace: reporting
+schema: 
+  

DataVaultGenerator/schema/entities/satellite.yaml

@@ -0,0 +1,48 @@
+type: satellite
+name: Satellite
+displayname: Satellite
+namespace: model
+schema: 
+# GeneratorEntity
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+    allowed: [base,drivingkeystatus]
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+  exclude_commonattributes:
+    type: list
+# Specific
+  parent: 
+    type: string
+    required: True
+  hashdiff_attribute_treatment:
+    type: dict
+    schema:
+      trim:
+        type: string
+        allowed: ['left', 'right', 'both']
+      case:
+        type: string
+        allowed: ['upper', 'lower', 'keep']
+# Attributes:
+  attributes: attributes

DataVaultGenerator/schema/entities/source.yaml

@@ -0,0 +1,26 @@
+type: source
+name: source
+displayname: Source interface
+namespace: source
+schema: 
+# Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  description: 
+    type: string
+# Specific:
+  sourcetype: 
+    type: string
+  sourcesystem: 
+    type: string
+  dbentity:
+    schema: dbentity
+  properties:
+    type: dict  
+# Attributes:
+  attributes: attributes
+          

DataVaultGenerator/schema/entities/sourcesystem.yaml

@@ -0,0 +1,23 @@
+type: sourcesystem
+name: sourcesystem
+displayname: Source System
+namespace: sourcesystem
+schema: 
+# Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  description: 
+    type: string
+# Specific:
+  shortname: 
+    type: string
+  connectionname: 
+    type: string
+  sourcesystemtype: 
+    type: string
+  sys_specification:
+    type: string

DataVaultGenerator/schema/entities/subdag.yaml

@@ -0,0 +1,29 @@
+type: subdag
+name: subdag
+displayname: Sub Dag
+namespace: dag
+schema: 
+# Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+# Specific:
+  entrypoints: 
+    type: list
+  excludes: 
+    type: list
+  key: 
+    type: string
+    

DataVaultGenerator/schema/entities/view.yaml

@@ -0,0 +1,59 @@
+type: view
+name: view
+displayname: View
+namespace: model
+schema: 
+  # Generator Entity:
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  subtype: 
+    type: string
+  generate: 
+    type: integer
+    allowed: [0,1]
+  extra: 
+    type: [list, dict, string, integer]  
+  description: 
+    type: string
+  sql_pre_hook:
+    type: string
+  sql_post_hook:
+    type: string
+# DataVaultEntity
+  layer: 
+    type: string
+  dbentity:
+    schema: dbentity
+  exclude_commonattributes:
+    type: list
+# Attributes:
+  attributes: attributes
+# Specific:  
+  query: 
+    type: string
+    required: True
+  materialize:
+    type: integer
+    allowed: [0,1]
+  materialization: 
+    type: dict
+    schema: 
+      mode:
+        type: string
+        allowed: ['merge', 'full']
+      target:
+        type: string
+      layer:
+        type: string
+      mergekeys:
+        type: list
+        schema: 
+          type: string
+      query:
+        type: string
+           
+

DataVaultGenerator/schema/mapping.yaml

@@ -0,0 +1,34 @@
+type: mapping
+name: mapping
+displayname: Mapping
+schema:
+  root:  # Workaround, da cerberus dynamische roots nicht unterstützt
+    type: dict
+    valuesrules: 
+      type: list
+      schema: 
+        type: dict
+        schema: 
+          target:
+            type: string
+            required: True
+          mappingmode: 
+            type: string
+            allowed: ['explicit', 'implicit']
+          type: 
+            type: string
+            allowed: ['mappingonly', 'master']
+            
+          mapping:
+            type: list
+            schema: 
+              type: list
+              schema:
+                type: [string, dict]
+                schema: 
+                    expression:
+                      type: string
+                      required: True 
+                    resulttype:
+                      type: string
+                      

DataVaultGenerator/schema/model.yaml

@@ -0,0 +1,5 @@
+type: model
+name: model
+displayname: Model
+schema: 
+  

DataVaultGenerator/schema/registry/attribute.yaml

@@ -0,0 +1,42 @@
+name: attribute
+type: schema
+schema: 
+  name:
+    type: string
+    required: True
+  type:
+    type: string
+    required: True
+  mandatory:
+    type: boolean
+  default:
+    type: string
+  description: 
+    type: string
+  ghost:
+    type: string
+  role: 
+      type: string
+  precision:
+    type: [integer, string]
+  scale: 
+    type: [integer, string]
+  length: 
+    type: [integer, string]
+  order:
+    type: integer
+  pii: 
+    type: boolean
+  props:
+    type: dict
+  
+  # View related
+  reference: 
+      type: string
+  referencetype:
+      type: string
+
+  components: 
+      type: list
+      schema: 
+        type: string

DataVaultGenerator/schema/registry/attributes.yaml

@@ -0,0 +1,9 @@
+name: attributes
+type: ruleset
+schema:
+  type: list
+  schema: 
+    type: dict
+    schema: attribute
+
+   

DataVaultGenerator/schema/registry/dbentity.yaml

@@ -0,0 +1,13 @@
+name: dbentity
+type: schema
+schema: 
+    name: 
+      type: string
+    filegroup:
+      type: string
+    database:
+      type: string
+    schema:
+      type: string
+    properties: 
+      type: dict

DataVaultGenerator/schema/sys_specification.yaml

@@ -0,0 +1,19 @@
+type: sys_specification
+name: sys_specification
+displayname: System Specification
+namespace: lang
+schema:
+  sys_specification:
+    type: string
+    required: true
+  objectnames:
+    type: string
+  datatypes:
+    type: dict
+    valuesrules:
+      type: dict
+      schema: 
+          pattern:
+            type: string
+            required: True 
+