Merge pull request #38 from splunk-soar-connectors/next

Merging next to main for release 2.15.0

.github/workflows/generate-doc.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - '*.json'
       - 'readme.html'
+      - 'manual_readme_content.md'
     tags-ignore:
       - '**'
     branches-ignore:

README.md

@@ -2,11 +2,11 @@
 # Splunk
 
 Publisher: Splunk  
-Connector Version: 2.14.0  
+Connector Version: 2.15.0  
 Product Vendor: Splunk Inc.  
 Product Name: Splunk Enterprise  
 Product Version Supported (regex): ".\*"  
-Minimum Product Version: 5.5.0  
+Minimum Product Version: 6.0.0  
 
 This app integrates with Splunk to update data on the device, in addition to investigate and ingestion actions
 
@@ -347,6 +347,7 @@ VARIABLE | REQUIRED | TYPE | DESCRIPTION
 **retry_count** |  optional  | numeric | Number of retries
 **remove_empty_cef** |  optional  | boolean | Remove CEF fields having empty values from the artifact
 **sleeptime_in_requests** |  optional  | numeric | The time to wait for next REST call (max 120 seconds)
+**include_cim_fields** |  optional  | boolean | Option to keep original Splunk CIM together with SOAR CEF fields
 
 ### Supported Actions  
 [test connectivity](#action-test-connectivity) - Validate the asset configuration for connectivity. This action logs into the device to check the connection and credentials  
@@ -457,15 +458,15 @@ action_result.parameter.command | string |  |   savedsearch
 action_result.parameter.display | string |  |   _time  index 
 action_result.parameter.end_time | string |  |   -2d  2022-03-18T16:12:09.130+00:00 
 action_result.parameter.parse_only | boolean |  |   True  False 
-action_result.parameter.search_mode | string |  |   smart 
 action_result.parameter.query | string |  `splunk query`  |   "Send to test" 
+action_result.parameter.search_mode | string |  |   smart 
 action_result.parameter.start_time | string |  |   -2d  2022-03-18T16:12:07.130+00:00 
-action_result.data.\*._key | string |  |   1659398400|_audit 
-action_result.data.\*._origtime | string |  |   1659398400 
 action_result.data.\*._bkt | string |  |  
 action_result.data.\*._cd | string |  |  
 action_result.data.\*._indextime | string |  |  
+action_result.data.\*._key | string |  |   1659398400|_audit 
 action_result.data.\*._kv | string |  |  
+action_result.data.\*._origtime | string |  |   1659398400 
 action_result.data.\*._raw | string |  |  
 action_result.data.\*._serial | string |  |  
 action_result.data.\*._si | string |  |  

release_notes/2.15.0.md

@@ -0,0 +1,2 @@
+* Bug fix for removing temp files [PAPP-30430]
+* Added a new feature to include both CEF and original CIM field [PAPP-30037]

requirements.txt

@@ -1,7 +1,6 @@
 beautifulsoup4==4.9.1
 python-dateutil==2.8.1
 pytz==2021.1
-requests==2.25.0
 simplejson==3.17.2
 splunk-sdk==1.7.2
 xmltodict==0.13.0

splunk.json

@@ -24,22 +24,25 @@
         },
         {
             "name": "Philip Royer"
+        },
+        {
+            "name": "Bartosz Debek"
         }
     ],
     "type": "siem",
     "main_module": "splunk_connector.py",
-    "app_version": "2.14.0",
+    "app_version": "2.15.0",
     "utctime_updated": "2022-09-08T08:47:45.000000Z",
     "package_name": "phantom_splunk",
     "product_name": "Splunk Enterprise",
     "product_vendor": "Splunk Inc.",
     "product_version_regex": ".*",
-    "min_phantom_version": "5.5.0",
+    "min_phantom_version": "6.0.0",
     "fips_compliant": true,
     "python_version": "3",
     "latest_tested_versions": [
-        "On-premise, Splunk Enterprise Security v9.0.0, April 3 2023",
-        "Cloud, Splunk Cloud Platform v8.2.2112, April 3 2023"
+        "On-premise, Splunk Enterprise Security v9.0.0, May 8 2023",
+        "Cloud, Splunk Cloud Platform v8.2.2112, May 8 2023"
     ],
     "logo": "logo_splunk.svg",
     "logo_dark": "logo_splunk_dark.svg",
@@ -50,18 +53,6 @@
                 "module": "beautifulsoup4",
                 "input_file": "wheels/py3/beautifulsoup4-4.9.1-py3-none-any.whl"
             },
-            {
-                "module": "certifi",
-                "input_file": "wheels/py3/certifi-2022.12.7-py3-none-any.whl"
-            },
-            {
-                "module": "chardet",
-                "input_file": "wheels/shared/chardet-3.0.4-py2.py3-none-any.whl"
-            },
-            {
-                "module": "idna",
-                "input_file": "wheels/shared/idna-2.10-py2.py3-none-any.whl"
-            },
             {
                 "module": "python_dateutil",
                 "input_file": "wheels/shared/python_dateutil-2.8.1-py2.py3-none-any.whl"
@@ -70,10 +61,6 @@
                 "module": "pytz",
                 "input_file": "wheels/shared/pytz-2021.1-py2.py3-none-any.whl"
             },
-            {
-                "module": "requests",
-                "input_file": "wheels/shared/requests-2.25.0-py2.py3-none-any.whl"
-            },
             {
                 "module": "simplejson",
                 "input_file": "wheels/py36/simplejson-3.17.2-cp36-cp36m-manylinux2010_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
@@ -90,10 +77,6 @@
                 "module": "splunk_sdk",
                 "input_file": "wheels/py3/splunk_sdk-1.7.2-py3-none-any.whl"
             },
-            {
-                "module": "urllib3",
-                "input_file": "wheels/shared/urllib3-1.26.15-py2.py3-none-any.whl"
-            },
             {
                 "module": "xmltodict",
                 "input_file": "wheels/shared/xmltodict-0.13.0-py2.py3-none-any.whl"
@@ -219,6 +202,13 @@
             "data_type": "numeric",
             "order": 19,
             "default": 1
+        },
+        "include_cim_fields": {
+            "description": "Option to keep original Splunk CIM together with SOAR CEF fields",
+            "data_type": "boolean",
+            "order": 20,
+            "default": false,
+            "name": "include_cim_fields"
         }
     },
     "actions": [
@@ -561,13 +551,6 @@
                         false
                     ]
                 },
-                {
-                    "data_path": "action_result.parameter.search_mode",
-                    "data_type": "string",
-                    "example_values": [
-                        "smart"
-                    ]
-                },
                 {
                     "data_path": "action_result.parameter.query",
                     "data_type": "string",
@@ -579,25 +562,18 @@
                     ]
                 },
                 {
-                    "data_path": "action_result.parameter.start_time",
-                    "data_type": "string",
-                    "example_values": [
-                        "-2d",
-                        "2022-03-18T16:12:07.130+00:00"
-                    ]
-                },
-                {
-                    "data_path": "action_result.data.*._key",
+                    "data_path": "action_result.parameter.search_mode",
                     "data_type": "string",
                     "example_values": [
-                        "1659398400|_audit"
+                        "smart"
                     ]
                 },
                 {
-                    "data_path": "action_result.data.*._origtime",
+                    "data_path": "action_result.parameter.start_time",
                     "data_type": "string",
                     "example_values": [
-                        "1659398400"
+                        "-2d",
+                        "2022-03-18T16:12:07.130+00:00"
                     ]
                 },
                 {
@@ -612,11 +588,25 @@
                     "data_path": "action_result.data.*._indextime",
                     "data_type": "string"
                 },
+                {
+                    "data_path": "action_result.data.*._key",
+                    "data_type": "string",
+                    "example_values": [
+                        "1659398400|_audit"
+                    ]
+                },
                 {
                     "data_path": "action_result.data.*._kv",
                     "data_type": "string",
                     "example_value": "1"
                 },
+                {
+                    "data_path": "action_result.data.*._origtime",
+                    "data_type": "string",
+                    "example_values": [
+                        "1659398400"
+                    ]
+                },
                 {
                     "data_path": "action_result.data.*._raw",
                     "column_name": "Raw",
@@ -1226,18 +1216,6 @@
                 "module": "beautifulsoup4",
                 "input_file": "wheels/py3/beautifulsoup4-4.9.1-py3-none-any.whl"
             },
-            {
-                "module": "certifi",
-                "input_file": "wheels/py3/certifi-2022.12.7-py3-none-any.whl"
-            },
-            {
-                "module": "chardet",
-                "input_file": "wheels/shared/chardet-3.0.4-py2.py3-none-any.whl"
-            },
-            {
-                "module": "idna",
-                "input_file": "wheels/shared/idna-2.10-py2.py3-none-any.whl"
-            },
             {
                 "module": "python_dateutil",
                 "input_file": "wheels/shared/python_dateutil-2.8.1-py2.py3-none-any.whl"
@@ -1246,10 +1224,6 @@
                 "module": "pytz",
                 "input_file": "wheels/shared/pytz-2021.1-py2.py3-none-any.whl"
             },
-            {
-                "module": "requests",
-                "input_file": "wheels/shared/requests-2.25.0-py2.py3-none-any.whl"
-            },
             {
                 "module": "simplejson",
                 "input_file": "wheels/py39/simplejson-3.17.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
@@ -1260,16 +1234,12 @@
             },
             {
                 "module": "soupsieve",
-                "input_file": "wheels/py3/soupsieve-2.4-py3-none-any.whl"
+                "input_file": "wheels/py3/soupsieve-2.4.1-py3-none-any.whl"
             },
             {
                 "module": "splunk_sdk",
                 "input_file": "wheels/py3/splunk_sdk-1.7.2-py3-none-any.whl"
             },
-            {
-                "module": "urllib3",
-                "input_file": "wheels/shared/urllib3-1.26.15-py2.py3-none-any.whl"
-            },
             {
                 "module": "xmltodict",
                 "input_file": "wheels/shared/xmltodict-0.13.0-py2.py3-none-any.whl"

splunk_connector.py

@@ -37,6 +37,7 @@
 from urllib.request import urlopen  # noqa
 
 import phantom.app as phantom
+import phantom.rules as soar_vault
 import pytz
 import requests
 import simplejson as json
@@ -121,16 +122,7 @@ def initialize(self):
 
         config = self.get_config()
 
-        # Fetching the Python major version
-        try:
-            self._python_version = int(sys.version_info[0])
-        except Exception:
-            return self.set_status(phantom.APP_ERROR, "Error occurred while getting the Phantom server's Python major version")
-
-        try:
-            self.splunk_server = config[phantom.APP_JSON_DEVICE]
-        except Exception:
-            return phantom.APP_ERROR
+        self.splunk_server = config[phantom.APP_JSON_DEVICE]
 
         self._username = config.get(phantom.APP_JSON_USERNAME)
         self._password = config.get(phantom.APP_JSON_PASSWORD)
@@ -878,7 +870,7 @@ def _get_fips_enabled(self):
             self.debug_print('FIPS is not enabled')
         return fips_enabled
 
-    def _on_poll(self, param):
+    def _on_poll(self, param):  # noqa: C901
 
         action_result = self.add_action_result(phantom.ActionResult(dict(param)))
 
@@ -889,6 +881,7 @@ def _on_poll(self, param):
         search_command = config.get('on_poll_command')
         search_string = config.get('on_poll_query')
         po = config.get('on_poll_parse_only', False)
+        include_cim_fields = config.get('include_cim_fields', False)
 
         if not search_string:
             self.save_progress("Need to specify Query String to use polling")
@@ -950,10 +943,17 @@ def _on_poll(self, param):
                         # Use this to keep the orignal capitalization from splunk
                         name_mappings[k.lower()] = k
                 for h in header_set:
-                    cef[name_mappings.get(consts.CIM_CEF_MAP.get(h, h), h)] = item.get(name_mappings.get(h, h))
+                    cef_name = consts.CIM_CEF_MAP.get(h, h)
+                    cef_key_value = name_mappings.get(h, h)
+                    cef[cef_name] = item.get(cef_key_value)
+                    # Add original CIM fields if option is checked
+                    cef.update({cef_key_value: item.get(cef_key_value)} if include_cim_fields else {})
             else:
                 for k, v in list(item.items()):
                     cef[consts.CIM_CEF_MAP.get(k, k)] = v
+                    # Add original CIM fields if option is checked
+                    cef.update({k: v} if include_cim_fields else {})
+
             input_str = json.dumps(item)
             input_str = UnicodeDammit(input_str).unicode_markup.encode('utf-8')
 
@@ -1340,15 +1340,17 @@ def _run_query(self, search_query, action_result, attach_result=False, kwargs_cr
 
     def add_json_result(self, action_result, data):
 
-        tmp_dir = tempfile.mkdtemp(prefix='splunk_result_attach')
-        file_path = "{}/splunk_run_query_result.json".format(tmp_dir)
+        if hasattr(Vault, 'get_vault_tmp_dir'):
+            tmp = tempfile.NamedTemporaryFile(dir=Vault.get_vault_tmp_dir(), delete=False)
+        else:
+            tmp = tempfile.NamedTemporaryFile(dir='/opt/phantom/vault/tmp/', delete=False)
         vault_attach_dict = {}
 
         vault_attach_dict[phantom.APP_JSON_ACTION_NAME] = self.get_action_name()
         vault_attach_dict[phantom.APP_JSON_APP_RUN_ID] = self.get_app_run_id()
 
         try:
-            with open(file_path, 'w') as f:
+            with open(tmp.name, 'w') as f:
                 json.dump(data, f)
 
         except Exception as e:
@@ -1360,19 +1362,17 @@ def add_json_result(self, action_result, data):
 
         container_id = self.get_container_id()
 
-        vault_ret = {}
-
         try:
-            vault_ret = Vault.add_attachment(file_path, container_id, 'splunk_run_query_result.json', vault_attach_dict)
+            success, message, _ = soar_vault.vault_add(container_id, tmp.name, 'splunk_run_query_result.json', vault_attach_dict)
 
         except Exception as e:
             self._dump_error_log(e)
             err = self._get_error_message_from_exception(e)
             self.debug_print(phantom.APP_ERR_FILE_ADD_TO_VAULT.format(err))
             return action_result.set_status(phantom.APP_ERROR, phantom.APP_ERR_FILE_ADD_TO_VAULT.format(err))
 
-        if (not vault_ret.get('succeeded')):
-            err = "Failed to add file to Vault: {0}".format(json.dumps(vault_ret))
+        if not success:
+            err = "Failed to add file to Vault: {0}".format(message)
             self.debug_print(err)
             return action_result.set_status(phantom.APP_ERROR, err)