Merge pull request #161 from george0st/change

New dataset (change in events) & test pipelines
george0st · May 31, 2024 · 9f806c4 · 9f806c4
2 parents c7b74b6 + fb62c03
commit 9f806c4
Show file tree

Hide file tree

Showing 33 changed files with 596 additions and 575 deletions.
diff --git a/02-data/01-size-100/01-basic-party.csv.gz b/02-data/01-size-100/01-basic-party.csv.gz
diff --git a/02-data/01-size-100/01-basic-party.parquet b/02-data/01-size-100/01-basic-party.parquet
diff --git a/02-data/01-size-100/02-basic-contact.csv.gz b/02-data/01-size-100/02-basic-contact.csv.gz
diff --git a/02-data/01-size-100/02-basic-contact.parquet b/02-data/01-size-100/02-basic-contact.parquet
diff --git a/02-data/01-size-100/03-basic-relation.csv.gz b/02-data/01-size-100/03-basic-relation.csv.gz
diff --git a/02-data/01-size-100/03-basic-relation.parquet b/02-data/01-size-100/03-basic-relation.parquet
diff --git a/02-data/01-size-100/04-basic-account.csv.gz b/02-data/01-size-100/04-basic-account.csv.gz
diff --git a/02-data/01-size-100/04-basic-account.parquet b/02-data/01-size-100/04-basic-account.parquet
diff --git a/02-data/01-size-100/05-basic-transaction.csv.gz b/02-data/01-size-100/05-basic-transaction.csv.gz
diff --git a/02-data/01-size-100/05-basic-transaction.parquet b/02-data/01-size-100/05-basic-transaction.parquet
diff --git a/02-data/01-size-100/06-basic-event.csv.gz b/02-data/01-size-100/06-basic-event.csv.gz
diff --git a/02-data/01-size-100/06-basic-event.parquet b/02-data/01-size-100/06-basic-event.parquet
diff --git a/02-data/01-size-100/07-basic-communication.csv.gz b/02-data/01-size-100/07-basic-communication.csv.gz
diff --git a/02-data/01-size-100/07-basic-communication.parquet b/02-data/01-size-100/07-basic-communication.parquet
diff --git a/02-data/02-size-1K/01-basic-party.csv.gz b/02-data/02-size-1K/01-basic-party.csv.gz
diff --git a/02-data/02-size-1K/01-basic-party.parquet b/02-data/02-size-1K/01-basic-party.parquet
diff --git a/02-data/02-size-1K/02-basic-contact.csv.gz b/02-data/02-size-1K/02-basic-contact.csv.gz
diff --git a/02-data/02-size-1K/02-basic-contact.parquet b/02-data/02-size-1K/02-basic-contact.parquet
diff --git a/02-data/02-size-1K/03-basic-relation.csv.gz b/02-data/02-size-1K/03-basic-relation.csv.gz
diff --git a/02-data/02-size-1K/03-basic-relation.parquet b/02-data/02-size-1K/03-basic-relation.parquet
diff --git a/02-data/02-size-1K/04-basic-account.csv.gz b/02-data/02-size-1K/04-basic-account.csv.gz
diff --git a/02-data/02-size-1K/04-basic-account.parquet b/02-data/02-size-1K/04-basic-account.parquet
diff --git a/02-data/02-size-1K/05-basic-transaction.csv.gz b/02-data/02-size-1K/05-basic-transaction.csv.gz
diff --git a/02-data/02-size-1K/05-basic-transaction.parquet b/02-data/02-size-1K/05-basic-transaction.parquet
diff --git a/02-data/02-size-1K/06-basic-event.csv.gz b/02-data/02-size-1K/06-basic-event.csv.gz
diff --git a/02-data/02-size-1K/06-basic-event.parquet b/02-data/02-size-1K/06-basic-event.parquet
diff --git a/02-data/02-size-1K/07-basic-communication.csv.gz b/02-data/02-size-1K/07-basic-communication.csv.gz
diff --git a/02-data/02-size-1K/07-basic-communication.parquet b/02-data/02-size-1K/07-basic-communication.parquet
diff --git a/03-test/01-size-100.json b/03-test/01-size-100.json
diff --git a/03-test/02-size-1k.json b/03-test/02-size-1k.json
diff --git a/03-test/20-pipeline.json b/03-test/20-pipeline.json
@@ -3,33 +3,54 @@
   "description": "Setting for test pipelines for ingestion",
   "kind": "test",
   "spec": {
-        "featuresets": ["basic-party", "basic-contact", "basic-relation", "basic-account"],
+        "featuresets": ["basic-party", "basic-contact", "basic-relation", "basic-account", "basic-transaction",
+        "basic-event", "basic-communication"],
         "tests": {
           "basic-party": {
             "filter": "(event['party-gender'] == 'F')",
-            "extend": "({'ext': 'd'})",
+            "extend": "({'bparty': 'party'})",
             "calc": "",
             "agreg": ""
           },
           "basic-contact": {
-            "filter": "",
-            "extend": "",
+            "filter": "(event['contact-state'] == 'Active')",
+            "extend": "({'bcontact': 'contact'})",
             "calc": "",
             "agreg": ""
           },
           "basic-relation": {
-            "filter": "",
+            "filter": "(event['relation-type'] == 'TTT')",
             "extend": "",
             "calc": "",
             "agreg": ""
           },
           "basic-account": {
-            "filter": "",
-            "extend": "",
+            "filter": "(event['account-state'] == 'active')",
+            "extend": "({'baccount': 'account'})",
+            "calc": "",
+            "agreg": ""
+          },
+          "basic-transaction":
+          {
+            "filter": "(event['transaction-direction'] == 'Incoming' and event['transaction-type'] == 'Standard')",
+            "extend": "({'btransaction': 'transaction'})",
+            "calc": "",
+            "agreg": ""
+          },
+          "basic-event":
+          {
+            "filter": "(event['event-group'] == 'access' and event['event-action'] == 'login')",
+            "extend": "({'bevent': 'event'})",
+            "calc": "",
+            "agreg": ""
+          },
+          "basic-communication":
+          {
+            "filter": "(event['content-type'] == 'text' and event['channel'] == 'xxx')",
+            "extend": "({'bcommunication': 'communication'})",
             "calc": "",
             "agreg": ""
           }
-
         }
   }
 }
diff --git a/generator/basic_event.py b/generator/basic_event.py
@@ -115,8 +115,8 @@ def generate(self, count):
                     if event==0:
                         # add default group, category and action -> login for first event in bundle
                         group="access"
-                        category="login"
-                        action="mobile" if self.rnd_bool() else "web"
+                        category="mobile" if self.rnd_bool() else "web"
+                        action="login"
                     else:
                         # add random group, category, action
                         if party_customer:

diff --git a/generator/version.py b/generator/version.py
@@ -1,3 +1,3 @@
 # Store the version here so:
 
-__version__ = '0.2.3'
+__version__ = '0.2.4'