diff --git a/.github/workflows/generate-doc.yml b/.github/workflows/generate-doc.yml index 7a3ba90..9284f9d 100644 --- a/.github/workflows/generate-doc.yml +++ b/.github/workflows/generate-doc.yml @@ -5,6 +5,7 @@ on: paths: - '*.json' - 'readme.html' + - 'manual_readme_content.md' tags-ignore: - '**' branches-ignore: diff --git a/README.md b/README.md index 0a2c1a7..3faa3dc 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ # Splunk Publisher: Splunk -Connector Version: 2.14.0 +Connector Version: 2.15.0 Product Vendor: Splunk Inc. Product Name: Splunk Enterprise Product Version Supported (regex): ".\*" -Minimum Product Version: 5.5.0 +Minimum Product Version: 6.0.0 This app integrates with Splunk to update data on the device, in addition to investigate and ingestion actions @@ -347,6 +347,7 @@ VARIABLE | REQUIRED | TYPE | DESCRIPTION **retry_count** | optional | numeric | Number of retries **remove_empty_cef** | optional | boolean | Remove CEF fields having empty values from the artifact **sleeptime_in_requests** | optional | numeric | The time to wait for next REST call (max 120 seconds) +**include_cim_fields** | optional | boolean | Option to keep original Splunk CIM together with SOAR CEF fields ### Supported Actions [test connectivity](#action-test-connectivity) - Validate the asset configuration for connectivity. This action logs into the device to check the connection and credentials @@ -457,15 +458,15 @@ action_result.parameter.command | string | | savedsearch action_result.parameter.display | string | | _time index action_result.parameter.end_time | string | | -2d 2022-03-18T16:12:09.130+00:00 action_result.parameter.parse_only | boolean | | True False -action_result.parameter.search_mode | string | | smart action_result.parameter.query | string | `splunk query` | "Send to test" +action_result.parameter.search_mode | string | | smart action_result.parameter.start_time | string | | -2d 2022-03-18T16:12:07.130+00:00 -action_result.data.\*._key | string | | 1659398400|_audit -action_result.data.\*._origtime | string | | 1659398400 action_result.data.\*._bkt | string | | action_result.data.\*._cd | string | | action_result.data.\*._indextime | string | | +action_result.data.\*._key | string | | 1659398400|_audit action_result.data.\*._kv | string | | +action_result.data.\*._origtime | string | | 1659398400 action_result.data.\*._raw | string | | action_result.data.\*._serial | string | | action_result.data.\*._si | string | | diff --git a/release_notes/2.15.0.md b/release_notes/2.15.0.md new file mode 100644 index 0000000..d45f138 --- /dev/null +++ b/release_notes/2.15.0.md @@ -0,0 +1,2 @@ +* Bug fix for removing temp files [PAPP-30430] +* Added a new feature to include both CEF and original CIM field [PAPP-30037] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cc10458..8bccee1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ beautifulsoup4==4.9.1 python-dateutil==2.8.1 pytz==2021.1 -requests==2.25.0 simplejson==3.17.2 splunk-sdk==1.7.2 xmltodict==0.13.0 diff --git a/splunk.json b/splunk.json index 615e147..b6d8d5f 100644 --- a/splunk.json +++ b/splunk.json @@ -24,22 +24,25 @@ }, { "name": "Philip Royer" + }, + { + "name": "Bartosz Debek" } ], "type": "siem", "main_module": "splunk_connector.py", - "app_version": "2.14.0", + "app_version": "2.15.0", "utctime_updated": "2022-09-08T08:47:45.000000Z", "package_name": "phantom_splunk", "product_name": "Splunk Enterprise", "product_vendor": "Splunk Inc.", "product_version_regex": ".*", - "min_phantom_version": "5.5.0", + "min_phantom_version": "6.0.0", "fips_compliant": true, "python_version": "3", "latest_tested_versions": [ - "On-premise, Splunk Enterprise Security v9.0.0, April 3 2023", - "Cloud, Splunk Cloud Platform v8.2.2112, April 3 2023" + "On-premise, Splunk Enterprise Security v9.0.0, May 8 2023", + "Cloud, Splunk Cloud Platform v8.2.2112, May 8 2023" ], "logo": "logo_splunk.svg", "logo_dark": "logo_splunk_dark.svg", @@ -50,18 +53,6 @@ "module": "beautifulsoup4", "input_file": "wheels/py3/beautifulsoup4-4.9.1-py3-none-any.whl" }, - { - "module": "certifi", - "input_file": "wheels/py3/certifi-2022.12.7-py3-none-any.whl" - }, - { - "module": "chardet", - "input_file": "wheels/shared/chardet-3.0.4-py2.py3-none-any.whl" - }, - { - "module": "idna", - "input_file": "wheels/shared/idna-2.10-py2.py3-none-any.whl" - }, { "module": "python_dateutil", "input_file": "wheels/shared/python_dateutil-2.8.1-py2.py3-none-any.whl" @@ -70,10 +61,6 @@ "module": "pytz", "input_file": "wheels/shared/pytz-2021.1-py2.py3-none-any.whl" }, - { - "module": "requests", - "input_file": "wheels/shared/requests-2.25.0-py2.py3-none-any.whl" - }, { "module": "simplejson", "input_file": "wheels/py36/simplejson-3.17.2-cp36-cp36m-manylinux2010_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" @@ -90,10 +77,6 @@ "module": "splunk_sdk", "input_file": "wheels/py3/splunk_sdk-1.7.2-py3-none-any.whl" }, - { - "module": "urllib3", - "input_file": "wheels/shared/urllib3-1.26.15-py2.py3-none-any.whl" - }, { "module": "xmltodict", "input_file": "wheels/shared/xmltodict-0.13.0-py2.py3-none-any.whl" @@ -219,6 +202,13 @@ "data_type": "numeric", "order": 19, "default": 1 + }, + "include_cim_fields": { + "description": "Option to keep original Splunk CIM together with SOAR CEF fields", + "data_type": "boolean", + "order": 20, + "default": false, + "name": "include_cim_fields" } }, "actions": [ @@ -561,13 +551,6 @@ false ] }, - { - "data_path": "action_result.parameter.search_mode", - "data_type": "string", - "example_values": [ - "smart" - ] - }, { "data_path": "action_result.parameter.query", "data_type": "string", @@ -579,25 +562,18 @@ ] }, { - "data_path": "action_result.parameter.start_time", - "data_type": "string", - "example_values": [ - "-2d", - "2022-03-18T16:12:07.130+00:00" - ] - }, - { - "data_path": "action_result.data.*._key", + "data_path": "action_result.parameter.search_mode", "data_type": "string", "example_values": [ - "1659398400|_audit" + "smart" ] }, { - "data_path": "action_result.data.*._origtime", + "data_path": "action_result.parameter.start_time", "data_type": "string", "example_values": [ - "1659398400" + "-2d", + "2022-03-18T16:12:07.130+00:00" ] }, { @@ -612,11 +588,25 @@ "data_path": "action_result.data.*._indextime", "data_type": "string" }, + { + "data_path": "action_result.data.*._key", + "data_type": "string", + "example_values": [ + "1659398400|_audit" + ] + }, { "data_path": "action_result.data.*._kv", "data_type": "string", "example_value": "1" }, + { + "data_path": "action_result.data.*._origtime", + "data_type": "string", + "example_values": [ + "1659398400" + ] + }, { "data_path": "action_result.data.*._raw", "column_name": "Raw", @@ -1226,18 +1216,6 @@ "module": "beautifulsoup4", "input_file": "wheels/py3/beautifulsoup4-4.9.1-py3-none-any.whl" }, - { - "module": "certifi", - "input_file": "wheels/py3/certifi-2022.12.7-py3-none-any.whl" - }, - { - "module": "chardet", - "input_file": "wheels/shared/chardet-3.0.4-py2.py3-none-any.whl" - }, - { - "module": "idna", - "input_file": "wheels/shared/idna-2.10-py2.py3-none-any.whl" - }, { "module": "python_dateutil", "input_file": "wheels/shared/python_dateutil-2.8.1-py2.py3-none-any.whl" @@ -1246,10 +1224,6 @@ "module": "pytz", "input_file": "wheels/shared/pytz-2021.1-py2.py3-none-any.whl" }, - { - "module": "requests", - "input_file": "wheels/shared/requests-2.25.0-py2.py3-none-any.whl" - }, { "module": "simplejson", "input_file": "wheels/py39/simplejson-3.17.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" @@ -1260,16 +1234,12 @@ }, { "module": "soupsieve", - "input_file": "wheels/py3/soupsieve-2.4-py3-none-any.whl" + "input_file": "wheels/py3/soupsieve-2.4.1-py3-none-any.whl" }, { "module": "splunk_sdk", "input_file": "wheels/py3/splunk_sdk-1.7.2-py3-none-any.whl" }, - { - "module": "urllib3", - "input_file": "wheels/shared/urllib3-1.26.15-py2.py3-none-any.whl" - }, { "module": "xmltodict", "input_file": "wheels/shared/xmltodict-0.13.0-py2.py3-none-any.whl" diff --git a/splunk_connector.py b/splunk_connector.py index 6a1e6d3..97c61f9 100644 --- a/splunk_connector.py +++ b/splunk_connector.py @@ -37,6 +37,7 @@ from urllib.request import urlopen # noqa import phantom.app as phantom +import phantom.rules as soar_vault import pytz import requests import simplejson as json @@ -121,16 +122,7 @@ def initialize(self): config = self.get_config() - # Fetching the Python major version - try: - self._python_version = int(sys.version_info[0]) - except Exception: - return self.set_status(phantom.APP_ERROR, "Error occurred while getting the Phantom server's Python major version") - - try: - self.splunk_server = config[phantom.APP_JSON_DEVICE] - except Exception: - return phantom.APP_ERROR + self.splunk_server = config[phantom.APP_JSON_DEVICE] self._username = config.get(phantom.APP_JSON_USERNAME) self._password = config.get(phantom.APP_JSON_PASSWORD) @@ -878,7 +870,7 @@ def _get_fips_enabled(self): self.debug_print('FIPS is not enabled') return fips_enabled - def _on_poll(self, param): + def _on_poll(self, param): # noqa: C901 action_result = self.add_action_result(phantom.ActionResult(dict(param))) @@ -889,6 +881,7 @@ def _on_poll(self, param): search_command = config.get('on_poll_command') search_string = config.get('on_poll_query') po = config.get('on_poll_parse_only', False) + include_cim_fields = config.get('include_cim_fields', False) if not search_string: self.save_progress("Need to specify Query String to use polling") @@ -950,10 +943,17 @@ def _on_poll(self, param): # Use this to keep the orignal capitalization from splunk name_mappings[k.lower()] = k for h in header_set: - cef[name_mappings.get(consts.CIM_CEF_MAP.get(h, h), h)] = item.get(name_mappings.get(h, h)) + cef_name = consts.CIM_CEF_MAP.get(h, h) + cef_key_value = name_mappings.get(h, h) + cef[cef_name] = item.get(cef_key_value) + # Add original CIM fields if option is checked + cef.update({cef_key_value: item.get(cef_key_value)} if include_cim_fields else {}) else: for k, v in list(item.items()): cef[consts.CIM_CEF_MAP.get(k, k)] = v + # Add original CIM fields if option is checked + cef.update({k: v} if include_cim_fields else {}) + input_str = json.dumps(item) input_str = UnicodeDammit(input_str).unicode_markup.encode('utf-8') @@ -1340,15 +1340,17 @@ def _run_query(self, search_query, action_result, attach_result=False, kwargs_cr def add_json_result(self, action_result, data): - tmp_dir = tempfile.mkdtemp(prefix='splunk_result_attach') - file_path = "{}/splunk_run_query_result.json".format(tmp_dir) + if hasattr(Vault, 'get_vault_tmp_dir'): + tmp = tempfile.NamedTemporaryFile(dir=Vault.get_vault_tmp_dir(), delete=False) + else: + tmp = tempfile.NamedTemporaryFile(dir='/opt/phantom/vault/tmp/', delete=False) vault_attach_dict = {} vault_attach_dict[phantom.APP_JSON_ACTION_NAME] = self.get_action_name() vault_attach_dict[phantom.APP_JSON_APP_RUN_ID] = self.get_app_run_id() try: - with open(file_path, 'w') as f: + with open(tmp.name, 'w') as f: json.dump(data, f) except Exception as e: @@ -1360,10 +1362,8 @@ def add_json_result(self, action_result, data): container_id = self.get_container_id() - vault_ret = {} - try: - vault_ret = Vault.add_attachment(file_path, container_id, 'splunk_run_query_result.json', vault_attach_dict) + success, message, _ = soar_vault.vault_add(container_id, tmp.name, 'splunk_run_query_result.json', vault_attach_dict) except Exception as e: self._dump_error_log(e) @@ -1371,8 +1371,8 @@ def add_json_result(self, action_result, data): self.debug_print(phantom.APP_ERR_FILE_ADD_TO_VAULT.format(err)) return action_result.set_status(phantom.APP_ERROR, phantom.APP_ERR_FILE_ADD_TO_VAULT.format(err)) - if (not vault_ret.get('succeeded')): - err = "Failed to add file to Vault: {0}".format(json.dumps(vault_ret)) + if not success: + err = "Failed to add file to Vault: {0}".format(message) self.debug_print(err) return action_result.set_status(phantom.APP_ERROR, err) diff --git a/wheels/py3/certifi-2022.12.7-py3-none-any.whl b/wheels/py3/certifi-2022.12.7-py3-none-any.whl deleted file mode 100644 index a083056..0000000 Binary files a/wheels/py3/certifi-2022.12.7-py3-none-any.whl and /dev/null differ diff --git a/wheels/py3/soupsieve-2.4-py3-none-any.whl b/wheels/py3/soupsieve-2.4-py3-none-any.whl deleted file mode 100644 index c16c128..0000000 Binary files a/wheels/py3/soupsieve-2.4-py3-none-any.whl and /dev/null differ diff --git a/wheels/py3/soupsieve-2.4.1-py3-none-any.whl b/wheels/py3/soupsieve-2.4.1-py3-none-any.whl new file mode 100644 index 0000000..26a486c Binary files /dev/null and b/wheels/py3/soupsieve-2.4.1-py3-none-any.whl differ diff --git a/wheels/shared/chardet-3.0.4-py2.py3-none-any.whl b/wheels/shared/chardet-3.0.4-py2.py3-none-any.whl deleted file mode 100644 index d276977..0000000 Binary files a/wheels/shared/chardet-3.0.4-py2.py3-none-any.whl and /dev/null differ diff --git a/wheels/shared/idna-2.10-py2.py3-none-any.whl b/wheels/shared/idna-2.10-py2.py3-none-any.whl deleted file mode 100644 index 41225cb..0000000 Binary files a/wheels/shared/idna-2.10-py2.py3-none-any.whl and /dev/null differ diff --git a/wheels/shared/requests-2.25.0-py2.py3-none-any.whl b/wheels/shared/requests-2.25.0-py2.py3-none-any.whl deleted file mode 100644 index c3f28e5..0000000 Binary files a/wheels/shared/requests-2.25.0-py2.py3-none-any.whl and /dev/null differ diff --git a/wheels/shared/urllib3-1.26.15-py2.py3-none-any.whl b/wheels/shared/urllib3-1.26.15-py2.py3-none-any.whl deleted file mode 100644 index ad723d5..0000000 Binary files a/wheels/shared/urllib3-1.26.15-py2.py3-none-any.whl and /dev/null differ