diff --git a/plaso/data/formatters/generic.yaml b/plaso/data/formatters/generic.yaml index 2fea44a300..5295e2a195 100644 --- a/plaso/data/formatters/generic.yaml +++ b/plaso/data/formatters/generic.yaml @@ -352,12 +352,12 @@ type: 'conditional' data_type: 'gcp:log:entry' message: - '{text_payload}' -- 'User {user}' +- 'User {principal_email}' - 'performed {event_subtype}' - 'on {resource_name}' short_message: - '{text_payload}' -- 'User {user}' +- 'User {principal_email}' - 'performed {event_subtype}' - 'on {resource_name}' short_source: 'LOG' diff --git a/plaso/parsers/jsonl_plugins/gcp_log.py b/plaso/parsers/jsonl_plugins/gcp_log.py index 4398516499..c7ca8022af 100644 --- a/plaso/parsers/jsonl_plugins/gcp_log.py +++ b/plaso/parsers/jsonl_plugins/gcp_log.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- """JSON-L parser plugin for Google Cloud (GCP) log files.""" +import re + from dfdatetime import time_elements as dfdatetime_time_elements from plaso.containers import events @@ -12,18 +14,32 @@ class GCPLogEventData(events.EventData): """Google Cloud (GCP) log event data. Attributes: + caller_ip (str): IP address of the client that requested the operation. container (str): TODO + dcsa_emails (list[str]): default compute service account attached to a + Google Compute Engine (GCE) instance. + dcsa_scopes (list[str]): OAuth scopes granted to the default compute service + account. + delegation_chain (str): service account delegation chain. event_subtype (str): JSON event sub type or protocol buffer method. event_type (str): TODO filename (str): TODO firewall_rules (list[str]): firewall rules. firewall_source_ranges (list[str]): firewall source ranges. + gcloud_command_identity (str): unique gcloud command identity. + gcloud_command_partial (str): partial gcloud command. log_name (str): name of the log entry. message (str): TODO + method_name (str): operation performed. + permissions (list[str]): IAM permission used for the operation. policy_deltas (list[str]): TODO + principal_email (str): email address of the requester. + principal_subject (str): subject name of the requester. recorded_time (dfdatetime.DateTimeValues): date and time the log entry was recorded. request_account_identifier (str): GCP account identifier of the request. + request_address (str): IP address assigned to a Google Cloud Engine (GCE) + instance. request_description (str): description of the request. request_direction (str): direction of the request. request_email (str): email address of the request. @@ -33,11 +49,20 @@ class GCPLogEventData(events.EventData): request_target_tags (str): TODO resource_labels (list[str]): resource labels. resource_name (str): name of the resource. + service_account_delegation (list[str]): service accounts delegation in the + authentication. service_account_display_name (str): display name of the service account. - service_name (str): name of the servie. + service_account_key_name (str): service account key name used in + authentication. + service_name (str): name of the service. severity (str): log entry severity. + source_images (list[str]): source images of disks attached to a compute + engine instance. + status_code (str): operation success or failure code. + status_message (str); operation success or failure message. + status_reasons (list[str]): reasons for operation failure. text_payload (str): text payload for logs not using a JSON or proto payload. - user (str): user principal performing the logged action. + user_agent (str): user agent used in the request. """ DATA_TYPE = 'gcp:log:entry' @@ -45,17 +70,28 @@ class GCPLogEventData(events.EventData): def __init__(self): """Initializes event data.""" super(GCPLogEventData, self).__init__(data_type=self.DATA_TYPE) + self.caller_ip = None self.container = None + self.dcsa_emails = None + self.dcsa_scopes = None + self.delegation_chain = None self.event_subtype = None self.event_type = None self.filename = None self.firewall_rules = None self.firewall_source_ranges = None + self.gcloud_command_identity = None + self.gcloud_command_partial = None self.log_name = None + self.method_name = None self.message = None + self.permissions = None self.policy_deltas = None + self.principal_email = None + self.principal_subject = None self.recorded_time = None self.request_account_identifier = None + self.request_address = None self.request_description = None self.request_direction = None self.request_email = None @@ -65,11 +101,16 @@ def __init__(self): self.request_target_tags = None self.resource_labels = None self.resource_name = None + self.service_account_delegation = None self.service_account_display_name = None + self.service_account_key_name = None self.service_name = None self.severity = None + self.source_images = None + self.status_code = None + self.status_message = None self.text_payload = None - self.user = None + self.user_agent = None class GCPLogJSONLPlugin(interface.JSONLPlugin): @@ -78,6 +119,10 @@ class GCPLogJSONLPlugin(interface.JSONLPlugin): NAME = 'gcp_log' DATA_FORMAT = 'Google Cloud (GCP) log' + _USER_AGENT_COMMAND_RE = re.compile(r'command/([^\s]+)') + + _USER_AGENT_INVOCATION_ID_RE = re.compile(r'invocation-id/([^\s]+)') + def _ParseJSONPayload(self, json_dict, event_data): """Extracts information from a jsonPayload value. @@ -99,6 +144,189 @@ def _ParseJSONPayload(self, json_dict, event_data): if actor_json: event_data.user = self._GetJSONValue(actor_json, 'user') + def _ParseAuthenticationInfo(self, proto_payload, event_data): + """Extracts information from `protoPayload.authenticationInfo`. + + Args: + proto_payload (dict): JSON dictionary of the `protoPayload` value. + event_data (GCPLogEventData): event data. + """ + authentication_info = self._GetJSONValue( + proto_payload, 'authenticationInfo') + if not authentication_info: + return + + principal_email = self._GetJSONValue(authentication_info, 'principalEmail') + if principal_email: + event_data.principal_email = principal_email + + principal_subject = self._GetJSONValue( + authentication_info, 'principalSubject') + if principal_subject: + event_data.principal_subject = principal_subject + + service_account_key_name = self._GetJSONValue( + authentication_info, 'serviceAccountKeyName') + if service_account_key_name: + event_data.service_account_key_name = service_account_key_name + + delegations = [] + + delegation_info_list = self._GetJSONValue( + authentication_info, 'serviceAccountDelegationInfo', []) + for delegation_info in delegation_info_list: + first_party_principal = self._GetJSONValue( + delegation_info, 'firstPartyPrincipal', {}) + + first_party_principal_email = self._GetJSONValue(first_party_principal, + 'principalEmail') + if first_party_principal_email: + delegations.append(first_party_principal_email) + else: + first_party_principal_subject = self._GetJSONValue( + first_party_principal, 'principalSubject') + if first_party_principal_subject: + delegations.append(first_party_principal_subject) + + if delegations: + event_data.service_account_delegation = delegations + event_data.delegation_chain = '->'.join(delegations) + + def _ParseAuthorizationInfo(self, proto_payload, event_data): + """Extracts information from `protoPayload.authorizationInfo`. + + Args: + proto_payload (dict): JSON dictionary of the `protoPayload` value. + event_data (GCPLogEventData): event data. + """ + permissions = [] + + authorization_info_list = self._GetJSONValue( + proto_payload, 'authorizationInfo', []) + for authorization_info in authorization_info_list: + permission = self._GetJSONValue(authorization_info, 'permission') + if permission: + permissions.append(permission) + + if permissions: + event_data.permissions = permissions + + def _ParseRequestMetadata(self, proto_payload, event_data): + """Extracts information from `protoPayload.requestMetadata`. + + Args: + proto_payload (dict): JSON dictionary of the `protoPayload` value. + event_data (GCPLogEventData): event data. + """ + request_metadata = self._GetJSONValue(proto_payload, 'requestMetadata') + if not request_metadata: + return + + event_data.caller_ip = self._GetJSONValue(request_metadata, 'callerIp') + event_data.user_agent = self._GetJSONValue( + request_metadata, 'callerSuppliedUserAgent') + + if event_data.user_agent: + if 'command/' in event_data.user_agent: + matches = self._USER_AGENT_COMMAND_RE.search(event_data.user_agent) + if matches: + command_string = matches.group(1).replace('.', ' ') + event_data.gcloud_command_partial = command_string + + if 'invocation-id' in event_data.user_agent: + matches = self._USER_AGENT_INVOCATION_ID_RE.search( + event_data.user_agent) + if matches: + event_data.gcloud_command_identity = matches.group(1) + + def _ParseProtoPayloadStatus(self, proto_payload, event_data): + """Extracts information from `protoPayload.status`. + + Args: + proto_payload (dict): JSON dictionary of the `protoPayload` value. + event_data (GCPLogEventData): event data. + """ + status = self._GetJSONValue(proto_payload, 'status') + if status: + # Non empty `protoPayload.status` field could have empty + # `protoPayload.status.code` field. + # + # Empty `code` and `message` fields indicate the operation was successful. + + event_data.status_code = str(self._GetJSONValue(status, 'code', '')) + event_data.status_message = self._GetJSONValue(status, 'message') + + # `protoPayload.status.details[].reason` contains reason for an operation + # failure. + status_reasons = [] + + for status_detail in self._GetJSONValue(status, 'details', []): + status_reason = self._GetJSONValue(status_detail, 'reason') + if status_reason: + status_reasons.append(status_reason) + + if status_reasons: + event_data.status_reasons = status_reasons + + def _ParseComputeInsertRequest(self, request, event_data): + """Extracts compute.instances.insert information. + + Args: + request (dict): JSON dictionary of the `protoPayload.request` field. + event_data (GCPLogEventData): event data. + """ + # source_images hold Google Cloud source disk path used in creating a GCE + # instance. + source_images = [] + + for disk in self._GetJSONValue(request, 'disks', []): + initialize_params = self._GetJSONValue(disk, 'initializeParams', {}) + + source_image = self._GetJSONValue(initialize_params, 'sourceImage') + if source_image: + source_images.append(source_image) + + if source_images: + event_data.source_images = source_images + + # Default compute service account aka dcsa + dcsa_emails = [] + dcsa_scopes = [] + + service_account_list = self._GetJSONValue(request, 'serviceAccounts', []) + for service_account in service_account_list: + email = self._GetJSONValue(service_account, 'email') + if email: + dcsa_emails.append(email) + + scopes = self._GetJSONValue(service_account, 'scopes') + if scopes: + dcsa_scopes.extend(scopes) + + if dcsa_emails: + event_data.dcsa_emails = dcsa_emails + + if dcsa_scopes: + event_data.dcsa_scopes = dcsa_scopes + + def _ParseComputeProtoPayload(self, proto_payload, event_data): + """Extracts compute.googleapis.com information. + + Args: + proto_payload (dict): JSON dictionary of the `protoPayload` value. + event_data (GCPLogEventData): event data. + """ + request = self._GetJSONValue(proto_payload, 'request') + if not request: + return + + request_type = self._GetJSONValue(request, '@type') + if not request_type: + return + + if request_type == 'type.googleapis.com/compute.instances.insert': + self._ParseComputeInsertRequest(request, event_data) + def _ParseProtoPayload(self, json_dict, event_data): """Extracts information from a protoPayload value. @@ -110,41 +338,38 @@ def _ParseProtoPayload(self, json_dict, event_data): if not proto_payload: return - authentication_info = self._GetJSONValue( - proto_payload, 'authenticationInfo') - if authentication_info and not event_data.user: - event_data.user = self._GetJSONValue( - authentication_info, 'principalEmail') - - request_metadata = self._GetJSONValue( - proto_payload, 'requestMetadata', default_value={}) - event_data.request_metadata = [ - '{0:s}: {1!s}'.format(name, value) - for name, value in request_metadata.items()] - event_data.service_name = self._GetJSONValue(proto_payload, 'serviceName') event_data.resource_name = self._GetJSONValue(proto_payload, 'resourceName') method_name = self._GetJSONValue(proto_payload, 'methodName') if method_name and not event_data.event_subtype: event_data.event_subtype = method_name + event_data.method_name = method_name + self._ParseAuthenticationInfo(proto_payload, event_data) + self._ParseAuthorizationInfo(proto_payload, event_data) + self._ParseRequestMetadata(proto_payload, event_data) + self._ParseProtoPayloadStatus(proto_payload, event_data) self._ParseProtoPayloadRequest(proto_payload, event_data) self._ParseProtoPayloadServiceData(proto_payload, event_data) - def _ParseProtoPayloadRequest(self, json_dict, event_data): + if event_data.service_name == 'compute.googleapis.com': + self._ParseComputeProtoPayload(proto_payload, event_data) + + def _ParseProtoPayloadRequest(self, proto_payload, event_data): """Extracts information from the request field of a protoPayload field. Args: - json_dict (dict): JSON dictionary of the protoPayload value. + proto_payload (dict): JSON dictionary of the `protoPayload` value. event_data (GCPLogEventData): event data. """ - request = self._GetJSONValue(json_dict, 'request') + request = self._GetJSONValue(proto_payload, 'request') if not request: return event_data.request_account_identifier = self._GetJSONValue( request, 'account_id') + event_data.request_address = self._GetJSONValue(request, 'address') event_data.request_description = self._GetJSONValue(request, 'description') event_data.request_direction = self._GetJSONValue(request, 'direction') event_data.request_email = self._GetJSONValue(request, 'email') @@ -182,14 +407,14 @@ def _ParseProtoPayloadRequest(self, json_dict, event_data): event_data.service_account_display_name = self._GetJSONValue( service_account, 'display_name') - def _ParseProtoPayloadServiceData(self, json_dict, event_data): + def _ParseProtoPayloadServiceData(self, proto_payload, event_data): """Extracts information from the serviceData in the protoPayload value. Args: - json_dict (dict): JSON dictionary of the protoPayload value. + proto_payload (dict): JSON dictionary of the `protoPayload` value. event_data (GCPLogEventData): event data. """ - service_data = self._GetJSONValue(json_dict, 'serviceData') + service_data = self._GetJSONValue(proto_payload, 'serviceData') if not service_data: return @@ -202,11 +427,9 @@ def _ParseProtoPayloadServiceData(self, json_dict, event_data): binding_deltas = self._GetJSONValue( policy_delta, 'bindingDeltas', default_value=[]) for binding_delta_value in binding_deltas: - action = self._GetJSONValue( - binding_delta_value, 'action', default_value='') - member = self._GetJSONValue( - binding_delta_value, 'member', default_value='') - role = self._GetJSONValue(binding_delta_value, 'role', default_value='') + action = self._GetJSONValue(binding_delta_value, 'action') or 'N/A' + member = self._GetJSONValue(binding_delta_value, 'member') or 'N/A' + role = self._GetJSONValue(binding_delta_value, 'role') or 'N/A' policy_delta = '{0:s} {1:s} with role {2:s}'.format(action, member, role) policy_deltas.append(policy_delta) diff --git a/test_data/gcp_logging.jsonl b/test_data/gcp_logging.jsonl index 6b408e01c4..cf826f6ad2 100644 --- a/test_data/gcp_logging.jsonl +++ b/test_data/gcp_logging.jsonl @@ -7,3 +7,5 @@ {"insertId": "-g30hzhe5pe18", "logName": "projects/fake-project/logs/cloudaudit.googleapis.com%2Factivity", "operation": {"first": true, "id": "operation-1634611333792-5ceab9be58292-d54d52b1-12d290a9", "producer": "compute.googleapis.com"}, "protoPayload": {"@type": "type.googleapis.com/google.cloud.audit.AuditLog", "authenticationInfo": {"principalEmail": "fakeemailxyz@gmail.com"}, "authorizationInfo": [{"granted": true, "permission": "compute.instances.create", "resourceAttributes": {"name": "projects/fake-project/zones/us-central1-a/instances/instance-1", "service": "compute", "type": "compute.instances"}}, {"granted": true, "permission": "compute.disks.create", "resourceAttributes": {"name": "projects/fake-project/zones/us-central1-a/disks/instance-1", "service": "compute", "type": "compute.disks"}}, {"granted": true, "permission": "compute.subnetworks.use", "resourceAttributes": {"name": "projects/fake-project/regions/us-central1/subnetworks/default", "service": "compute", "type": "compute.subnetworks"}}, {"granted": true, "permission": "compute.subnetworks.useExternalIp", "resourceAttributes": {"name": "projects/fake-project/regions/us-central1/subnetworks/default", "service": "compute", "type": "compute.subnetworks"}}, {"granted": true, "permission": "compute.instances.setServiceAccount", "resourceAttributes": {"name": "projects/fake-project/zones/us-central1-a/instances/instance-1", "service": "compute", "type": "compute.instances"}}], "methodName": "beta.compute.instances.insert", "request": {"@type": "type.googleapis.com/compute.instances.insert", "canIpForward": false, "confidentialInstanceConfig": {"enableConfidentialCompute": false}, "deletionProtection": false, "description": "", "disks": [{"autoDelete": true, "boot": true, "deviceName": "instance-1", "initializeParams": {"diskSizeGb": "10", "diskType": "projects/fake-project/zones/us-central1-a/diskTypes/pd-balanced", "sourceImage": "projects/debian-cloud/global/images/debian-10-buster-v20210916"}, "mode": "READ_WRITE", "type": "PERSISTENT"}], "displayDevice": {"enableDisplay": false}, "machineType": "projects/fake-project/zones/us-central1-a/machineTypes/e2-medium", "name": "instance-1", "networkInterfaces": [{"accessConfigs": [{"name": "External NAT", "networkTier": "PREMIUM"}], "subnetwork": "projects/fake-project/regions/us-central1/subnetworks/default"}], "reservationAffinity": {"consumeReservationType": "ANY_ALLOCATION"}, "scheduling": {"automaticRestart": true, "onHostMaintenance": "MIGRATE", "preemptible": false}, "serviceAccounts": [{"email": "123456123456-compute@developer.gserviceaccount.com", "scopes": ["https://www.googleapis.com/auth/devstorage.read_only", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/monitoring.write", "https://www.googleapis.com/auth/servicecontrol", "https://www.googleapis.com/auth/service.management.readonly", "https://www.googleapis.com/auth/trace.append"]}], "shieldedInstanceConfig": {"enableIntegrityMonitoring": true, "enableSecureBoot": false, "enableVtpm": true}}, "requestMetadata": {"callerIp": "1.1.1.1", "callerSuppliedUserAgent": "UserAgent", "destinationAttributes": {}, "requestAttributes": {"auth": {}, "reason": "8uSywAYQGg5Db2xpc2V1bSBGbG93cw", "time": "2021-10-19T02:42:15.094342Z"}}, "resourceLocation": {"currentLocations": ["us-central1-a"]}, "resourceName": "projects/fake-project/zones/us-central1-a/instances/instance-1", "response": {"@type": "type.googleapis.com/operation", "id": "3135492963006910057", "insertTime": "2021-10-18T19:42:14.870-07:00", "name": "operation-1634611333792-5ceab9be58292-d54d52b1-12d290a9", "operationType": "insert", "progress": "0", "selfLink": "https://www.googleapis.com/compute/beta/projects/fake-project/zones/us-central1-a/operations/operation-1634611333792-5ceab9be58292-d54d52b1-12d290a9", "selfLinkWithId": "https://www.googleapis.com/compute/beta/projects/fake-project/zones/us-central1-a/operations/3135492963006910057", "startTime": "2021-10-18T19:42:14.871-07:00", "status": "RUNNING", "targetId": "866011396029255273", "targetLink": "https://www.googleapis.com/compute/beta/projects/fake-project/zones/us-central1-a/instances/instance-1", "user": "fakeemailxyz@gmail.com", "zone": "https://www.googleapis.com/compute/beta/projects/fake-project/zones/us-central1-a"}, "serviceName": "compute.googleapis.com"}, "receiveTimestamp": "2021-10-19T02:42:15.838342093Z", "resource": {"labels": {"instance_id": "866011396029255273", "project_id": "fake-project", "zone": "us-central1-a"}, "type": "gce_instance"}, "severity": "NOTICE", "timestamp": "2021-10-19T02:42:13.839954Z"} {"insertId": "1k28f3cfv7aknt", "jsonPayload": {"content": "This is a json payload", "event_subtype": "test_subtype", "actor": {"user": "fakeemailxyz@gmail.com"}}, "logName": "projects/fake-project/logs/testlog", "receiveTimestamp": "2021-10-19T02:05:41.496590981Z", "resource": {"labels": {"location": "", "namespace": "", "node_id": "", "project_id": "fake-project", "instance_id": "866011396029255273"}, "type": "generic_node"}, "timestamp": "2021-10-19T02:05:41.496590981Z"} {"insertId": "1io3yo2fursxdi", "logName": "projects/fake-project/logs/testlog", "receiveTimestamp": "2021-10-19T02:04:00.272384509Z", "resource": {"labels": {"location": "", "namespace": "", "node_id": "", "project_id": "fake-project"}, "type": "generic_node"}, "textPayload": "This is a text payload", "timestamp": "2021-10-19T02:04:00.272384509Z"} +{"insertId":"-duywnve29mpi","labels":{"compute.googleapis.com/root_trigger_id":"b0966c41-45b9-4484-b3b9-b436bfdde977"},"logName":"projects/fake-project/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1714162209773-617057d99d773-818c5509-b54204dc","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"fake-account@fake-project.com", "principalSubject":"user:fake-account@fake-project.com","serviceAccountDelegationInfo":[{"firstPartyPrincipal":{"principalEmail":"service-account-one@fake-project.com"}},{"firstPartyPrincipal":{"principalEmail":"service-account-two@fake-project.com"}}]},"authorizationInfo":[{"granted":true,"permission":"compute.instances.create","permissionType":"ADMIN_WRITE","resource":"projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","resourceAttributes":{"name":"projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.disks.create","permissionType":"ADMIN_WRITE","resource":"projects/fake-project/zones/us-central1-b/disks/fake-compute-instance","resourceAttributes":{"name":"projects/fake-project/zones/us-central1-b/disks/fake-compute-instance","service":"compute","type":"compute.disks"}},{"granted":true,"permission":"compute.subnetworks.use","permissionType":"ADMIN_WRITE","resource":"projects/fake-project/regions/us-central1/subnetworks/default","resourceAttributes":{"name":"projects/fake-project/regions/us-central1/subnetworks/default","service":"compute","type":"compute.subnetworks"}},{"granted":true,"permission":"compute.subnetworks.useExternalIp","permissionType":"ADMIN_WRITE","resource":"projects/fake-project/regions/us-central1/subnetworks/default","resourceAttributes":{"name":"projects/fake-project/regions/us-central1/subnetworks/default","service":"compute","type":"compute.subnetworks"}},{"granted":true,"permission":"compute.instances.setMetadata","permissionType":"ADMIN_WRITE","resource":"projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","resourceAttributes":{"name":"projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.instances.setLabels","permissionType":"ADMIN_WRITE","resource":"projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","resourceAttributes":{"name":"projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.instances.setServiceAccount","permissionType":"ADMIN_WRITE","resource":"projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","resourceAttributes":{"name":"projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","service":"compute","type":"compute.instances"}}],"methodName":"beta.compute.instances.insert","request":{"@type":"type.googleapis.com/compute.instances.insert","description":"Fake GCE instance","disks":[{"autoDelete":true,"boot":true,"initializeParams":{"diskSizeGb":"100","diskType":"zones/us-central1-b/diskTypes/pd-ssd","sourceImage":"projects/fake-project/global/images/fake-source-image"}}],"labels":[],"machineType":"zones/us-central1-b/machineTypes/n1-highmem-16","name":"fake-compute-instance","networkInterfaces":[{"accessConfigs":[{"name":"External NAT","type":"ONE_TO_ONE_NAT"}],"network":"global/networks/default"}],"scheduling":{"automaticRestart":true},"serviceAccounts":[{"email":"fake-service-account@fake-project.com","scopes":["https://www.googleapis.com/auth/cloud-platform"]}]},"requestMetadata":{"callerIp":"1.1.1.1","callerSuppliedUserAgent":"fake-user-agent-string command/gcloud.compute.instances.insert invocation-id/a1b2c3d4e5f6 environment/GCE","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2024-04-26T20:10:11.171739Z"}},"resourceLocation":{"currentLocations":["us-central1-b"]},"resourceName":"projects/1234567890/zones/us-central1-b/instances/fake-compute-instance","response":{"@type":"type.googleapis.com/operation","id":"30220247688656077","insertTime":"2024-04-26T13:10:11.055-07:00","name":"operation-1714162209773-617057d99d773-818c5509-b54204dc","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/beta/projects/fake-project/zones/us-central1-b/operations/operation-1714162209773-617057d99d773-818c5509-b54204dc","selfLinkWithId":"https://www.googleapis.com/compute/beta/projects/fake-project/zones/us-central1-b/operations/30220247688656077","startTime":"2024-04-26T13:10:11.056-07:00","status":"RUNNING","targetId":"9876543210","targetLink":"https://www.googleapis.com/compute/beta/projects/fake-project/zones/us-central1-b/instances/fake-compute-instance","user":"fake-service-account@fake-project.com","zone":"https://www.googleapis.com/compute/beta/projects/fake-project/zones/us-central1-b"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2024-04-26T20:10:11.491740716Z","resource":{"labels":{"instance_id":"9876543210","project_id":"fake-project","zone":"us-central1-b"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2024-04-26T20:10:10.024055Z"} +{"insertId":"1awjxggeaxqgz","logName":"projects/ketchup/logs/cloudaudit.googleapis.com%2Factivity","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","status":{"code":7,"message":"Permission \"iam.serviceAccounts.create\" denied on resource (or it may not exist).","details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"IAM_PERMISSION_DENIED","domain":"iam.googleapis.com","metadata":{"permission":"iam.serviceAccounts.create"}}]},"authenticationInfo":{"principalEmail":"dvwa-service-account@ketchup.iam.gserviceaccount.com","serviceAccountDelegationInfo":[{"firstPartyPrincipal":{"principalEmail":"service-1234567890@compute-system.iam.gserviceaccount.com"}}],"principalSubject":"serviceAccount:dvwa-service-account@ketchup.iam.gserviceaccount.com"},"requestMetadata":{"callerIp":"34.72.217.225","callerSuppliedUserAgent":"(gzip),gzip(gfe)","requestAttributes":{"time":"2024-12-03T17:58:45.019694350Z","auth":{}},"destinationAttributes":{}},"serviceName":"iam.googleapis.com","methodName":"google.iam.admin.v1.CreateServiceAccount","authorizationInfo":[{"resource":"projects/ketchup","permission":"iam.serviceAccounts.create","resourceAttributes":{"type":"iam.googleapis.com/ServiceAccount"},"permissionType":"ADMIN_WRITE"}],"resourceName":"projects/ketchup","request":{"service_account":{"display_name":"This is the attacker account"},"account_id":"theattacker","name":"projects/ketchup","@type":"type.googleapis.com/google.iam.admin.v1.CreateServiceAccountRequest"},"response":{"@type":"type.googleapis.com/google.iam.admin.v1.ServiceAccount"}},"receiveTimestamp":"2024-12-03T17:58:45.716564605Z","resource":{"type":"service_account","labels":{"unique_id":"","project_id":"ketchup","email_id":""}},"timestamp":"2024-12-03T17:58:44.882119699Z","severity":"ERROR"} diff --git a/tests/parsers/jsonl_parser.py b/tests/parsers/jsonl_parser.py index 372bd4f5bf..d4de35cfde 100644 --- a/tests/parsers/jsonl_parser.py +++ b/tests/parsers/jsonl_parser.py @@ -37,7 +37,7 @@ def testParse(self): number_of_event_data = storage_writer.GetNumberOfAttributeContainers( 'event_data') - self.assertEqual(number_of_event_data, 9) + self.assertEqual(number_of_event_data, 11) number_of_warnings = storage_writer.GetNumberOfAttributeContainers( 'extraction_warning') diff --git a/tests/parsers/jsonl_plugins/gcp_log.py b/tests/parsers/jsonl_plugins/gcp_log.py index 5ce83b8ada..a95d695216 100644 --- a/tests/parsers/jsonl_plugins/gcp_log.py +++ b/tests/parsers/jsonl_plugins/gcp_log.py @@ -19,7 +19,7 @@ def testProcess(self): number_of_event_data = storage_writer.GetNumberOfAttributeContainers( 'event_data') - self.assertEqual(number_of_event_data, 9) + self.assertEqual(number_of_event_data, 11) number_of_warnings = storage_writer.GetNumberOfAttributeContainers( 'extraction_warning') @@ -30,6 +30,7 @@ def testProcess(self): self.assertEqual(number_of_warnings, 0) expected_event_values = { + 'caller_ip': '1.1.1.1', 'container': None, 'event_subtype': 'beta.compute.networks.insert', 'event_type': None, @@ -46,8 +47,6 @@ def testProcess(self): 'request_direction': None, 'request_email': None, 'request_member': None, - 'request_metadata': [ - 'callerIp: 1.1.1.1', 'callerSuppliedUserAgent: UserAgent'], 'request_name': None, 'request_target_tags': None, 'resource_labels': [ @@ -57,11 +56,142 @@ def testProcess(self): 'service_name': 'compute.googleapis.com', 'severity': 'NOTICE', 'text_payload': None, - 'user': 'fakeemailxyz@gmail.com'} + 'user_agent': 'UserAgent'} event_data = storage_writer.GetAttributeContainerByIndex('event_data', 0) self.CheckEventData(event_data, expected_event_values) + def testComputeInstancesInsert(self): + """Tests for the JSON-L parser plugin parsing request type + compute.instances.insert.""" + plugin = gcp_log.GCPLogJSONLPlugin() + storage_writer = self._ParseJSONLFileWithPlugin( + ['gcp_logging.jsonl'], plugin) + + expected_event_values = { + 'caller_ip': '1.1.1.1', + 'container': None, + 'dcsa_emails': ['fake-service-account@fake-project.com'], + 'dcsa_scopes': ['https://www.googleapis.com/auth/cloud-platform'], + 'delegation_chain': ( + 'service-account-one@fake-project.com->' + 'service-account-two@fake-project.com'), + 'event_subtype': 'beta.compute.instances.insert', + 'event_type': None, + 'filename': None, + 'firewall_rules': None, + 'firewall_source_ranges': None, + 'gcloud_command_identity': 'a1b2c3d4e5f6', + 'gcloud_command_partial': 'gcloud compute instances insert', + 'log_name': ( + 'projects/fake-project/logs/cloudaudit.googleapis.com%2Factivity'), + 'method_name': 'beta.compute.instances.insert', + 'message': None, + 'permissions': [ + 'compute.instances.create', + 'compute.disks.create', + 'compute.subnetworks.use', + 'compute.subnetworks.useExternalIp', + 'compute.instances.setMetadata', + 'compute.instances.setLabels', + 'compute.instances.setServiceAccount'], + 'policy_deltas': None, + 'principal_email': 'fake-account@fake-project.com', + 'principal_subject': 'user:fake-account@fake-project.com', + 'recorded_time': '2024-04-26T20:10:10.024055+00:00', + 'request_account_identifier': None, + 'request_description': 'Fake GCE instance', + 'request_direction': None, + 'request_email': None, + 'request_member': None, + 'request_metadata': None, + 'request_name': 'fake-compute-instance', + 'request_target_tags': None, + 'resource_labels': [ + 'instance_id: 9876543210', + 'project_id: fake-project', + 'zone: us-central1-b'], + 'resource_name': ( + 'projects/1234567890/zones/us-central1-b/instances/' + 'fake-compute-instance'), + 'service_account_delegation': [ + 'service-account-one@fake-project.com', + 'service-account-two@fake-project.com'], + 'service_account_display_name': None, + 'service_account_key_name': None, + 'service_name': 'compute.googleapis.com', + 'severity': 'NOTICE', + 'source_images': [ + 'projects/fake-project/global/images/fake-source-image'], + 'status_code': None, + 'status_message': None, + 'text_payload': None, + 'user_agent': ( + 'fake-user-agent-string command/gcloud.compute.instances.insert' + ' invocation-id/a1b2c3d4e5f6 environment/GCE')} + + event_data = storage_writer.GetAttributeContainerByIndex('event_data', 9) + self.CheckEventData(event_data, expected_event_values) + + def testServiceAccountCreateFailure(self): + """Tests service account creation failure log.""" + plugin = gcp_log.GCPLogJSONLPlugin() + storage_writer = self._ParseJSONLFileWithPlugin( + ['gcp_logging.jsonl'], plugin) + + expected_event_values = { + 'caller_ip': '34.72.217.225', + 'container': None, + 'dcsa_emails': None, + 'dcsa_scopes': None, + 'delegation_chain': ('service-1234567890@compute-system.iam.' + 'gserviceaccount.com'), + 'event_subtype': 'google.iam.admin.v1.CreateServiceAccount', + 'event_type': None, + 'filename': None, + 'firewall_rules': None, + 'firewall_source_ranges': None, + 'gcloud_command_identifier': None, + 'gcloud_command_pattern': None, + 'log_name': ('projects/ketchup/logs/cloudaudit.googleapis.com%2F' + 'activity'), + 'message': None, + 'method_name': 'google.iam.admin.v1.CreateServiceAccount', + 'permissions': ['iam.serviceAccounts.create'], + 'policy_deltas': None, + 'principal_email': ('dvwa-service-account@ketchup.iam.' + 'gserviceaccount.com'), + 'principal_subject': ('serviceAccount:dvwa-service-account@ketchup.' + 'iam.gserviceaccount.com'), + 'recorded_time': '2024-12-03T17:58:44.882119+00:00', + 'request_account_identifier': 'theattacker', + 'request_address': None, + 'request_description': None, + 'request_direction': None, + 'request_email': None, + 'request_member': None, + 'request_metadata': None, + 'request_name': 'projects/ketchup', + 'request_target': None, + 'request_labels': None, + 'resource_name': 'projects/ketchup', + 'service_account_delegation': [ + 'service-1234567890@compute-system.iam.gserviceaccount.com'], + 'service_account_display_name': 'This is the attacker account', + 'service_account_key_name': None, + 'service_name': 'iam.googleapis.com', + 'severity': 'ERROR', + 'source_images': None, + 'status_code': '7', + 'status_message': ('Permission "iam.serviceAccounts.create" denied on' + ' resource (or it may not exist).'), + 'status_reasons': ['IAM_PERMISSION_DENIED'], + 'text_payload': None, + 'user_agent': '(gzip),gzip(gfe)', + } + + event_data = storage_writer.GetAttributeContainerByIndex('event_data', 10) + self.CheckEventData(event_data, expected_event_values) if __name__ == '__main__': unittest.main()