Skip to content

Commit

Permalink
Add column obfuscator for prepared statement bound values
Browse files Browse the repository at this point in the history
The HideBlobsObfuscator can be used to avoid logging blob values
in prepared statements. The obfuscator will also handle collections
containing blobs.

Closes Ericsson#126
  • Loading branch information
etedpet committed Oct 9, 2019
1 parent 82e6d1c commit a43ed4f
Show file tree
Hide file tree
Showing 19 changed files with 659 additions and 19 deletions.
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Changes

## Version 2.3.0
* Avoid logging BLOB values #126
* Configurable format of printed records with eclog tool #102

## Version 2.2.0
Expand Down
21 changes: 21 additions & 0 deletions conf/audit.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,3 +89,24 @@ logger_backend:
# - log_format: "{?client:'${CLIENT_IP}'|?}user:'${USER}'{?|batchId:'${BATCH_ID}'?}|status:'${STATUS}'|operation:'${OPERATION}'"
# time_format:
# time_zone:


# The column obfuscator determines how column values should be logged for prepared statements (i.e. bound values).
# The obfuscator is applied to all prepared statements regardless of which table is used.
#
# - ShowAllObfuscator -> Will not obfuscate. Prints all values for prepared statements.
#
# - HideBlobsObfuscator -> Will obfuscate all blob values. Values will be replaced by a "<blob>"-tag in the log.
# Collection types containing blobs will also be replaced, e.g. "<list<blob>>".
# This obfuscator can be useful since blobs can be very big and perhaps not that useful
# to log.
#
# Note - The obfuscator only impacts to the log message if the OPERATION field is logged (see log_format above).
# If OPERATION_NAKED is selected then no bound values will be logged.
#
# A custom column obfuscator can be plugged in by setting the full name of an obfuscator class implementing the
# ColumnObfuscator interface.
#
# By default the ShowAllObfuscator will be used.
#
column_obfuscator: ShowAllObfuscator
11 changes: 11 additions & 0 deletions doc/slf4j_logger.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,17 @@ Which will generate logs entries like this (assuming Logback pattern does not co
2019-02-28 15:18:14.091-> client=127.0.0.1, user=cassandra, status=ATTEMPT, batch-id=6f3cae9b-f1f1-4a4c-baa2-ed168ee79f9d, operation='INSERT INTO ecks.ectbl (partk, clustk, value) VALUES (?, ?, ?)[2, '2', 'valid']'
```

### Column Obfuscator

A column obfuscator defining how column values should be logged for prepared statements. Using one of the non-default column obfuscators
can be useful to avoid logging sensitive or less useful information, e.g. large blobs.

For more information about the available column obfuscators, see the ```audit.yaml``` reference documentation.
Use the example below will obfuscate/hide blobs values from being logged.
```YAML
column_obfuscator: HideBlobsObfuscator
```
It is also possible to plugin a custom obfuscator.

## Configure Logback

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;

import com.google.common.annotations.VisibleForTesting;

import com.ericsson.bss.cassandra.ecaudit.common.record.SimpleAuditOperation;
import com.ericsson.bss.cassandra.ecaudit.common.record.Status;
import com.ericsson.bss.cassandra.ecaudit.entry.AuditEntry;
import com.ericsson.bss.cassandra.ecaudit.entry.PreparedAuditOperation;
import com.ericsson.bss.cassandra.ecaudit.entry.factory.AuditEntryBuilderFactory;
import com.ericsson.bss.cassandra.ecaudit.entry.obfuscator.ColumnObfuscator;
import com.ericsson.bss.cassandra.ecaudit.facade.Auditor;
import com.ericsson.bss.cassandra.ecaudit.utils.Exceptions;
import org.apache.cassandra.cql3.BatchQueryOptions;
Expand All @@ -49,6 +52,7 @@ public class AuditAdapter

private final Auditor auditor;
private final AuditEntryBuilderFactory entryBuilderFactory;
private ColumnObfuscator columnObfuscator;

private final Map<MD5Digest, String> idQueryCache = new ConcurrentHashMap<>();

Expand All @@ -57,11 +61,13 @@ public class AuditAdapter
*
* @param auditor the auditor to use
* @param entryBuilderFactory the audit entry builder factory to use
* @param columnObfuscator the column obfuscator
*/
AuditAdapter(Auditor auditor, AuditEntryBuilderFactory entryBuilderFactory)
AuditAdapter(Auditor auditor, AuditEntryBuilderFactory entryBuilderFactory, ColumnObfuscator columnObfuscator)
{
this.auditor = auditor;
this.entryBuilderFactory = entryBuilderFactory;
this.columnObfuscator = columnObfuscator;
}

public static AuditAdapter getInstance()
Expand Down Expand Up @@ -122,7 +128,7 @@ public void auditPrepared(MD5Digest id, CQLStatement statement, ClientState stat
.client(state.getRemoteAddress())
.coordinator(FBUtilities.getBroadcastAddress())
.user(state.getUser().getName())
.operation(new PreparedAuditOperation(idQueryCache.get(id), options))
.operation(new PreparedAuditOperation(idQueryCache.get(id), options, columnObfuscator))
.status(status)
.timestamp(timestamp)
.build();
Expand Down Expand Up @@ -234,7 +240,7 @@ private Collection<AuditEntry> getBatchOperations(AuditEntry.Builder builder, Ba
if (queryOrId instanceof MD5Digest)
{
entryBuilderFactory.updateBatchEntryBuilder(builder, batchStatement.getStatements().get(statementIndex));
builder.operation(new PreparedAuditOperation(idQueryCache.get(queryOrId), options.forStatement(statementIndex)));
builder.operation(new PreparedAuditOperation(idQueryCache.get(queryOrId), options.forStatement(statementIndex), columnObfuscator));
batchOperations.add(builder.build());
}
else
Expand All @@ -253,4 +259,10 @@ public Auditor getAuditor()
{
return auditor;
}

@VisibleForTesting
public void setColumnObfuscator(ColumnObfuscator columnObfuscator)
{
this.columnObfuscator = columnObfuscator;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import com.ericsson.bss.cassandra.ecaudit.config.AuditConfig;
import com.ericsson.bss.cassandra.ecaudit.entry.factory.AuditEntryBuilderFactory;
import com.ericsson.bss.cassandra.ecaudit.entry.obfuscator.ColumnObfuscator;
import com.ericsson.bss.cassandra.ecaudit.facade.Auditor;
import com.ericsson.bss.cassandra.ecaudit.facade.DefaultAuditor;
import com.ericsson.bss.cassandra.ecaudit.filter.AuditFilter;
Expand All @@ -36,6 +37,7 @@
import com.ericsson.bss.cassandra.ecaudit.obfuscator.PasswordObfuscator;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.utils.FBUtilities;

/**
* Factory class for creating configured instances of AuditAdapter.
Expand Down Expand Up @@ -77,7 +79,9 @@ static AuditAdapter createAuditAdapter(AuditConfig auditConfig)
Auditor auditor = new DefaultAuditor(logger, filter, obfuscator, logStrategy);
AuditEntryBuilderFactory entryBuilderFactory = new AuditEntryBuilderFactory();

return new AuditAdapter(auditor, entryBuilderFactory);
ColumnObfuscator columnObfuscator = createColumnObfuscator(auditConfig);

return new AuditAdapter(auditor, entryBuilderFactory, columnObfuscator);
}

/**
Expand Down Expand Up @@ -144,4 +148,15 @@ private static LogTimingStrategy getLogTimingStrategy(AuditConfig auditConfig)
? LogTimingStrategy.POST_LOGGING_STRATEGY
: LogTimingStrategy.PRE_LOGGING_STRATEGY;
}

private static ColumnObfuscator createColumnObfuscator(AuditConfig auditConfig)
{
String obfuscatorClassName = auditConfig.getColumnObfuscator();
if (!obfuscatorClassName.contains("."))
{
String packageName = ColumnObfuscator.class.getPackage().getName();
obfuscatorClassName = packageName + "." + obfuscatorClassName;
}
return FBUtilities.construct(obfuscatorClassName, "ColumnObfuscator");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,13 @@ public String getWrappedAuthorizer()
return yamlConfig.getWrappedAuthorizer();
}

public String getColumnObfuscator()
{
loadConfigIfNeeded();

return yamlConfig.getColumnObfuscator();
}

private synchronized void loadConfigIfNeeded()
{
if (yamlConfig == null)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.Map;
import java.util.stream.Collectors;

import com.ericsson.bss.cassandra.ecaudit.entry.obfuscator.ShowAllObfuscator;
import com.ericsson.bss.cassandra.ecaudit.logger.Slf4jAuditLogger;
import org.apache.cassandra.config.ParameterizedClass;

Expand All @@ -31,6 +32,7 @@ public final class AuditYamlConfig
private static final List<String> DEFAULT_WHITELIST = Collections.emptyList();
private static final ParameterizedClass DEFAULT_LOGGER_BACKEND = new ParameterizedClass(Slf4jAuditLogger.class.getCanonicalName(), Collections.emptyMap());
private static final String DEFAULT_WRAPPED_AUTHORIZER = "org.apache.cassandra.auth.CassandraAuthorizer";
private static final String DEFAULT_COLUMN_OBFUSCATOR = ShowAllObfuscator.class.getName();

private boolean fromFile = true;

Expand All @@ -40,6 +42,7 @@ public final class AuditYamlConfig
public ParameterizedClass logger_backend;
public LoggerTiming log_timing_strategy;
public String wrapped_authorizer;
public String column_obfuscator;

static AuditYamlConfig createWithoutFile()
{
Expand Down Expand Up @@ -99,4 +102,9 @@ String getWrappedAuthorizer()
{
return wrapped_authorizer != null ? wrapped_authorizer : DEFAULT_WRAPPED_AUTHORIZER;
}

String getColumnObfuscator()
{
return column_obfuscator != null ? column_obfuscator : DEFAULT_COLUMN_OBFUSCATOR;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.TimestampType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.serializers.TimestampSerializer;
Expand Down Expand Up @@ -54,6 +55,10 @@ public static String toCQLLiteral(ByteBuffer serializedValue, ColumnSpecificatio
{
return DATE_FORMAT.format(TimestampSerializer.instance.deserialize(serializedValue));
}
if (isBlobType(column))
{
return "0x" + column.type.getString(serializedValue);
}

return column.type.getString(serializedValue);
}
Expand All @@ -73,6 +78,11 @@ private static boolean isTimestampType(ColumnSpecification column)
return column.type instanceof TimestampType;
}

private static boolean isBlobType(ColumnSpecification column)
{
return column.type instanceof BytesType;
}

private static DateFormat createDateFormat()
{
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.getDefault(Locale.Category.FORMAT));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.Queue;

import com.ericsson.bss.cassandra.ecaudit.common.record.AuditOperation;
import com.ericsson.bss.cassandra.ecaudit.entry.obfuscator.ColumnObfuscator;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.cql3.QueryOptions;

Expand All @@ -37,6 +38,7 @@ public class PreparedAuditOperation implements AuditOperation
private final String preparedStatement;
private final QueryOptions options;
private String effectiveStatement; // lazy initialization
private final ColumnObfuscator columnObfuscator;

/**
* Construct a new prepared audit operation based on the prepared statement and options.
Expand All @@ -45,11 +47,14 @@ public class PreparedAuditOperation implements AuditOperation
* the prepared statement
* @param options
* the query options of an operation
* @param columnObfuscator
* the column obfuscator to process bound values
*/
public PreparedAuditOperation(String preparedStatement, QueryOptions options)
public PreparedAuditOperation(String preparedStatement, QueryOptions options, ColumnObfuscator columnObfuscator)
{
this.preparedStatement = preparedStatement;
this.options = options;
this.columnObfuscator = columnObfuscator;
}

@Override
Expand Down Expand Up @@ -87,9 +92,10 @@ private String preparedWithValues()
Queue<ByteBuffer> values = new LinkedList<>(options.getValues());
for (ColumnSpecification column : options.getColumnSpecifications())
{
String value = CqlLiteralVersionAdapter.toCQLLiteral(values.remove(), column);

fullStatement.append(value).append(", ");
ByteBuffer value = values.remove();
String valueString = columnObfuscator.obfuscate(column, value)
.orElseGet(() -> CqlLiteralVersionAdapter.toCQLLiteral(value, column));
fullStatement.append(valueString).append(", ");
}

fullStatement.setLength(fullStatement.length() - 1);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* Copyright 2019 Telefonaktiebolaget LM Ericsson
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.ericsson.bss.cassandra.ecaudit.entry.obfuscator;

import java.nio.ByteBuffer;
import java.util.Optional;

import org.apache.cassandra.cql3.ColumnSpecification;

/**
* Column obfuscator used to handle prepared statement bound values.
*/
public interface ColumnObfuscator
{
/**
* Creates an obfuscated string that represent the column value only IF the column should be obfuscated.
*
* @param column the column to check
* @param value the value that may be obfuscated
* @return the obfuscated string representation of the column value, or {@link Optional#empty()} if the value
* should not be obfuscated.
*/
Optional<String> obfuscate(ColumnSpecification column, ByteBuffer value);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright 2019 Telefonaktiebolaget LM Ericsson
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.ericsson.bss.cassandra.ecaudit.entry.obfuscator;

import java.nio.ByteBuffer;
import java.util.Optional;

import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.ColumnSpecification;
import org.apache.cassandra.db.marshal.AbstractType;

public class HideBlobsObfuscator implements ColumnObfuscator
{
@Override
public Optional<String> obfuscate(ColumnSpecification column, ByteBuffer value)
{
return isBlobType(column.type) || isCollectionContainingBlobType(column.type)
? Optional.of("<" + column.type.asCQL3Type() + ">")
: Optional.empty();
}

private static boolean isBlobType(AbstractType<?> type)
{
return type.asCQL3Type() == CQL3Type.Native.BLOB;
}

private static boolean isCollectionContainingBlobType(AbstractType<?> type)
{
return type.isCollection() && type.asCQL3Type().toString().contains("blob");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Copyright 2019 Telefonaktiebolaget LM Ericsson
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.ericsson.bss.cassandra.ecaudit.entry.obfuscator;

import java.nio.ByteBuffer;
import java.util.Optional;

import org.apache.cassandra.cql3.ColumnSpecification;

public class ShowAllObfuscator implements ColumnObfuscator
{
@Override
public Optional<String> obfuscate(ColumnSpecification column, ByteBuffer value)
{
return Optional.empty(); // No values should be obfuscated
}
}
Loading

0 comments on commit a43ed4f

Please sign in to comment.