Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PseudoController now correctly returns a JSON object with correct content-type and encoding #83

Merged
merged 1 commit into from
Feb 13, 2024
Merged

PseudoController now correctly returns a JSON object with correct content-type and encoding #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 29 additions & 29 deletions src/main/java/no/ssb/dlp/pseudo/service/pseudo/PseudoController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,14 @@
import no.ssb.dlp.pseudo.service.security.PseudoServiceRole;
import no.ssb.dlp.pseudo.service.sid.InvalidSidSnapshotDateException;
import no.ssb.dlp.pseudo.service.sid.SidIndexUnavailableException;
import org.reactivestreams.Publisher;

import javax.validation.constraints.Min;
import javax.validation.constraints.NotBlank;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.Principal;
Expand Down Expand Up @@ -73,7 +73,7 @@ public class PseudoController {
@Produces(MediaType.APPLICATION_JSON)
@Post(value = "/pseudonymize/field", consumes = MediaType.APPLICATION_JSON)
@ExecuteOn(TaskExecutors.IO)
public HttpResponse<Publisher<String>> pseudonymizeField(
public HttpResponse<Flowable<byte[]>> pseudonymizeField(
@Header(CORRELATION_ID_HEADER) Optional<String> clientCorrelationId,
@Schema(implementation = PseudoFieldRequest.class) String request) {
try {
Expand All @@ -82,10 +82,10 @@ public HttpResponse<Publisher<String>> pseudonymizeField(
PseudoField pseudoField = new PseudoField(req.getName(), req.getPseudoFunc(), req.getKeyset());

final String correlationId = validateOrCreate(clientCorrelationId);
MutableHttpResponse<Publisher<String>> mutableHttpResponse = HttpResponse.ok(pseudoField.process(pseudoConfigSplitter,
recordProcessorFactory, req.values, PseudoOperation.PSEUDONYMIZE, correlationId));
// TODO: Must hard-code to plain text to avoid that Micronaut converts the JSON to a JSON array
mutableHttpResponse.contentType(MediaType.TEXT_PLAIN_TYPE).characterEncoding("UTF-8");
MutableHttpResponse<Flowable<byte[]>> mutableHttpResponse = HttpResponse.ok(pseudoField.process(pseudoConfigSplitter,
recordProcessorFactory, req.values, PseudoOperation.PSEUDONYMIZE, correlationId)
.map(o -> o.getBytes(StandardCharsets.UTF_8)))
.characterEncoding(StandardCharsets.UTF_8);
mutableHttpResponse.getHeaders().add(CORRELATION_ID_HEADER, correlationId);
return mutableHttpResponse;

Expand All @@ -105,7 +105,7 @@ public HttpResponse<Publisher<String>> pseudonymizeField(
@Secured({PseudoServiceRole.ADMIN})
@Post(value = "/depseudonymize/field", consumes = MediaType.APPLICATION_JSON)
@ExecuteOn(TaskExecutors.IO)
public HttpResponse<Publisher<String>> depseudonymizeField(
public HttpResponse<Flowable<byte[]>> depseudonymizeField(
@Header(CORRELATION_ID_HEADER) Optional<String> clientCorrelationId,
@Schema(implementation = DepseudoFieldRequest.class) String request) {
try {
Expand All @@ -114,10 +114,10 @@ public HttpResponse<Publisher<String>> depseudonymizeField(
PseudoField pseudoField = new PseudoField(req.getName(), req.getPseudoFunc(), req.getKeyset());

final String correlationId = validateOrCreate(clientCorrelationId);
MutableHttpResponse<Publisher<String>> mutableHttpResponse = HttpResponse.ok(pseudoField.process(
pseudoConfigSplitter, recordProcessorFactory,req.values, PseudoOperation.DEPSEUDONYMIZE, correlationId));
// TODO: Must hard-code to plain text to avoid that Micronaut converts the JSON to a JSON array
mutableHttpResponse.contentType(MediaType.TEXT_PLAIN_TYPE).characterEncoding("UTF-8");
MutableHttpResponse<Flowable<byte[]>> mutableHttpResponse = HttpResponse.ok(pseudoField.process(
pseudoConfigSplitter, recordProcessorFactory,req.values, PseudoOperation.DEPSEUDONYMIZE, correlationId)
.map(o -> o.getBytes(StandardCharsets.UTF_8)))
.characterEncoding(StandardCharsets.UTF_8);
mutableHttpResponse.getHeaders().add(CORRELATION_ID_HEADER, correlationId);
return mutableHttpResponse;

Expand All @@ -137,7 +137,7 @@ public HttpResponse<Publisher<String>> depseudonymizeField(
@Secured({PseudoServiceRole.ADMIN})
@Post(value = "/repseudonymize/field", consumes = MediaType.APPLICATION_JSON)
@ExecuteOn(TaskExecutors.IO)
public HttpResponse<Publisher<String>> repseudonymizeField(
public HttpResponse<Flowable<byte[]>> repseudonymizeField(
@Header(CORRELATION_ID_HEADER) Optional<String> clientCorrelationId,
@Schema(implementation = RepseudoFieldRequest.class) String request) {
try {
Expand All @@ -147,10 +147,10 @@ public HttpResponse<Publisher<String>> repseudonymizeField(
PseudoField targetPseudoField = new PseudoField(req.getName(), req.getTargetPseudoFunc(), req.getTargetKeyset());

final String correlationId = validateOrCreate(clientCorrelationId);
MutableHttpResponse<Publisher<String>> mutableHttpResponse = HttpResponse.ok(
sourcePseudoField.process(recordProcessorFactory, req.values, targetPseudoField, correlationId));
// TODO: Must hard-code to plain text to avoid that Micronaut converts the JSON to a JSON array
mutableHttpResponse.contentType(MediaType.TEXT_PLAIN_TYPE).characterEncoding("UTF-8");
MutableHttpResponse<Flowable<byte[]>> mutableHttpResponse = HttpResponse.ok(
sourcePseudoField.process(recordProcessorFactory, req.values, targetPseudoField, correlationId)
.map(o -> o.getBytes(StandardCharsets.UTF_8)))
.characterEncoding(StandardCharsets.UTF_8);
mutableHttpResponse.getHeaders().add(CORRELATION_ID_HEADER, correlationId);
return mutableHttpResponse;

Expand Down Expand Up @@ -186,7 +186,7 @@ Pseudo rules will most times refer to crypto keys. You can provide your own keys
@Produces(value = MediaType.APPLICATION_JSON)
@SingleResult
@ExecuteOn(TaskExecutors.IO)
public HttpResponse<Publisher<String>> pseudonymizeFile(
public HttpResponse<Flowable<byte[]>> pseudonymizeFile(
@Header(CORRELATION_ID_HEADER) Optional<String> clientCorrelationId,
@Schema(implementation = PseudoRequest.class) String request, StreamingFileUpload data) {
log.info(Strings.padEnd(String.format("*** Pseudonymize file: %s", data.getFilename()), 80, '*'));
Expand All @@ -198,9 +198,9 @@ public HttpResponse<Publisher<String>> pseudonymizeFile(
final String correlationId = validateOrCreate(clientCorrelationId);
RecordMapProcessor<PseudoMetadataProcessor> recordProcessor = recordProcessorFactory.newPseudonymizeRecordProcessor(pseudoConfigs, correlationId);
ProcessFileResult res = processFile(data, PseudoOperation.PSEUDONYMIZE, recordProcessor, req.getTargetContentType(), req.getCompression());
// TODO: Must hard-code to plain text to avoid that Micronaut converts the JSON to a JSON array
MutableHttpResponse<Publisher<String>> mutableHttpResponse = HttpResponse.ok(res.getResponse())
.contentType(MediaType.TEXT_PLAIN_TYPE).characterEncoding("UTF-8");
MutableHttpResponse<Flowable<byte[]>> mutableHttpResponse = HttpResponse.ok(res.getResponse()
.map(o -> o.getBytes(StandardCharsets.UTF_8)))
.characterEncoding(StandardCharsets.UTF_8);
mutableHttpResponse.getHeaders().add(CORRELATION_ID_HEADER, correlationId);
return mutableHttpResponse;
} catch (RuntimeException e) {
Expand Down Expand Up @@ -239,7 +239,7 @@ Pseudo rules will most times refer to crypto keys. You can provide your own keys
@Produces(MediaType.APPLICATION_JSON)
@Secured({PseudoServiceRole.ADMIN})
@ExecuteOn(TaskExecutors.IO)
public HttpResponse<Publisher<String>> depseudonymizeFile(
public HttpResponse<Flowable<byte[]>> depseudonymizeFile(
@Header(CORRELATION_ID_HEADER) Optional<String> clientCorrelationId,
@Schema(implementation = PseudoRequest.class) String request, StreamingFileUpload data, Principal principal) {
log.info(Strings.padEnd(String.format("*** Depseudonymize file: %s", data.getFilename()), 80, '*'));
Expand All @@ -252,9 +252,9 @@ public HttpResponse<Publisher<String>> depseudonymizeFile(
final String correlationId = validateOrCreate(clientCorrelationId);
RecordMapProcessor<PseudoMetadataProcessor> recordProcessor = recordProcessorFactory.newDepseudonymizeRecordProcessor(pseudoConfigs, correlationId);
ProcessFileResult res = processFile(data, PseudoOperation.DEPSEUDONYMIZE, recordProcessor, req.getTargetContentType(), req.getCompression());
// TODO: Must hard-code to plain text to avoid that Micronaut converts the JSON to a JSON array
MutableHttpResponse<Publisher<String>> mutableHttpResponse = HttpResponse.ok(res.getResponse())
.contentType(MediaType.TEXT_PLAIN_TYPE).characterEncoding("UTF-8");
MutableHttpResponse<Flowable<byte[]>> mutableHttpResponse = HttpResponse.ok(res.getResponse()
.map(o -> o.getBytes(StandardCharsets.UTF_8)))
.characterEncoding(StandardCharsets.UTF_8);
mutableHttpResponse.getHeaders().add(CORRELATION_ID_HEADER, correlationId);
return mutableHttpResponse;
} catch (Exception e) {
Expand Down Expand Up @@ -294,7 +294,7 @@ Pseudo rules will most times refer to crypto keys. You can provide your own keys
@Produces(MediaType.APPLICATION_JSON)
@Secured({PseudoServiceRole.ADMIN})
@ExecuteOn(TaskExecutors.IO)
public HttpResponse<Publisher<String>> repseudonymizeFile(
public HttpResponse<Flowable<byte[]>> repseudonymizeFile(
@Header(CORRELATION_ID_HEADER) Optional<String> clientCorrelationId,
@Schema(implementation = RepseudoRequest.class) String request, StreamingFileUpload data, Principal principal) {
log.info(Strings.padEnd(String.format("*** Repseudonymize file: %s", data.getFilename()), 80, '*'));
Expand All @@ -305,9 +305,9 @@ public HttpResponse<Publisher<String>> repseudonymizeFile(
final String correlationId = validateOrCreate(clientCorrelationId);
RecordMapProcessor<PseudoMetadataProcessor> recordProcessor = recordProcessorFactory.newRepseudonymizeRecordProcessor(req.getSourcePseudoConfig(), req.getTargetPseudoConfig(), correlationId);
ProcessFileResult res = processFile(data, PseudoOperation.REPSEUDONYMIZE, recordProcessor, req.getTargetContentType(), req.getCompression());
// TODO: Must hard-code to plain text to avoid that Micronaut converts the JSON to a JSON array
MutableHttpResponse<Publisher<String>> mutableHttpResponse = HttpResponse.ok(res.getResponse())
.contentType(MediaType.TEXT_PLAIN_TYPE).characterEncoding("UTF-8");
MutableHttpResponse<Flowable<byte[]>> mutableHttpResponse = HttpResponse.ok(res.getResponse()
.map(o -> o.getBytes(StandardCharsets.UTF_8)))
.characterEncoding(StandardCharsets.UTF_8);
mutableHttpResponse.getHeaders().add(CORRELATION_ID_HEADER, correlationId);
return mutableHttpResponse;
} catch (Exception e) {
Expand Down Expand Up @@ -492,7 +492,7 @@ public static class ProcessFileResult {
@Schema(implementation = String.class)
private final MediaType targetContentType;

private final Publisher<String> response;
private final Flowable<String> response;
}

@Data
Expand Down
9 changes: 6 additions & 3 deletions src/main/java/no/ssb/dlp/pseudo/service/pseudo/PseudoField.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import no.ssb.dlp.pseudo.core.map.RecordMapProcessor;
import no.ssb.dlp.pseudo.core.tink.model.EncryptedKeysetWrapper;
import no.ssb.dlp.pseudo.core.util.Json;
import no.ssb.dlp.pseudo.service.pseudo.metadata.FieldMetric;
import no.ssb.dlp.pseudo.service.pseudo.metadata.PseudoMetadataProcessor;

import java.util.List;
Expand Down Expand Up @@ -88,7 +89,7 @@ public Flowable<String> process(PseudoConfigSplitter pseudoConfigSplitter,
final Flowable<String> metrics = Flowable.fromPublisher(metadataProcessor.getMetrics());

Flowable<String> result = preprocessor.andThen(Flowable.fromIterable(values.stream()
.map(v -> mapOptional(v, recordMapProcessor)).toList()
.map(v -> mapOptional(v, recordMapProcessor, metadataProcessor)).toList()
))
.map(v -> v.map(Json::from).orElse("null"))
.doOnError(throwable -> {
Expand Down Expand Up @@ -127,7 +128,7 @@ public Flowable<String> process(RecordMapProcessorFactory recordProcessorFactory
final Flowable<String> metrics = Flowable.fromPublisher(metadataProcessor.getMetrics());

Flowable<String> result = preprocessor.andThen(Flowable.fromIterable(values.stream()
.map(v -> mapOptional(v, recordMapProcessor)).toList()
.map(v -> mapOptional(v, recordMapProcessor, metadataProcessor)).toList()
))
.map(v -> v.map(Json::from).orElse("null"))
.doOnError(throwable -> {
Expand All @@ -142,8 +143,10 @@ public Flowable<String> process(RecordMapProcessorFactory recordProcessorFactory
return PseudoResponseSerializer.serialize(result, metadata, logs, metrics);
}

private Optional<Object> mapOptional(String v, RecordMapProcessor<PseudoMetadataProcessor> recordMapProcessor) {
private Optional<Object> mapOptional(String v, RecordMapProcessor<PseudoMetadataProcessor> recordMapProcessor,
PseudoMetadataProcessor metadataProcessor) {
if (v == null) {
metadataProcessor.addMetric(FieldMetric.NULL_VALUE);
return Optional.empty();
} else {
return Optional.of(recordMapProcessor.process(Map.of(this.getName(), v)).get(this.getName()));
Expand Down
8 changes: 5 additions & 3 deletions src/main/java/no/ssb/dlp/pseudo/service/pseudo/RecordMapProcessorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,9 @@ private String process(PseudoOperation operation,
PseudoFuncDeclaration funcDeclaration = PseudoFuncDeclaration.fromString(match.getRule().getFunc());
final boolean isSidMapping = funcDeclaration.getFuncName().equals(PseudoFuncNames.MAP_SID);
final String sidSnapshotDate = output.getMetadata().getOrDefault(MapFuncConfig.Param.SNAPSHOT_DATE, null);
if (isSidMapping && sidSnapshotDate == null) {
// Unsuccessful SID-mapping do not have any sidSnapshotDate
final String mappedValue = (String) output.getFirstValue();
if (isSidMapping && varValue.equals(mappedValue)) {
// Unsuccessful SID-mapping
metadataProcessor.addMetric(FieldMetric.MISSING_SID);
} else if (isSidMapping) {
metadataProcessor.addMetric(FieldMetric.MAPPED_SID);
Expand All @@ -143,10 +144,11 @@ private String process(PseudoOperation operation,
.build());
}
output.getWarnings().forEach(metadataProcessor::addLog);
return mappedValue;
} else {
output = match.getFunc().restore(PseudoFuncInput.of(varValue));
return (String) output.getFirstValue();
}
return (String) output.getFirstValue();
} catch (Exception e) {
throw new PseudoException(String.format("pseudonymize error - field='%s', originalValue='%s'",
field.getPath(), varValue), e);
Expand Down
4 changes: 1 addition & 3 deletions src/test/java/no/ssb/dlp/pseudo/service/pseudo/DepseudoFieldTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import no.ssb.dlp.pseudo.core.PseudoOperation;
import no.ssb.dlp.pseudo.core.map.RecordMapProcessor;
import no.ssb.dlp.pseudo.service.pseudo.metadata.FieldMetadata;
import no.ssb.dlp.pseudo.service.pseudo.metadata.FieldMetric;
import no.ssb.dlp.pseudo.service.pseudo.metadata.PseudoMetadataProcessor;
import org.json.JSONException;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -54,7 +53,6 @@ private static PseudoMetadataProcessor createPseudoMetadataProcessor() {
.stableIdentifierType(STABLE_IDENTIFIER_TYPE)
.build());
processor.addLog("Log line");
processor.addMetric(FieldMetric.MISSING_SID);
return processor;
}

Expand Down Expand Up @@ -101,7 +99,7 @@ void processWithNullValues() throws JSONException {
},
"metrics": [
{
"MISSING_SID": 1
"NULL_VALUE": 1
}
],
"logs": [
Expand Down
10 changes: 1 addition & 9 deletions src/test/java/no/ssb/dlp/pseudo/service/pseudo/PseudoFieldTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,6 @@ private static PseudoMetadataProcessor createPseudoMetadataProcessor() {
.stableIdentifierType(STABLE_IDENTIFIER_TYPE)
.build());
processor.addLog("Log line");
processor.addMetric(FieldMetric.NULL_VALUE);
processor.addMetric(FieldMetric.MISSING_SID);
processor.addMetric(FieldMetric.NULL_VALUE);
processor.addMetric(FieldMetric.NULL_VALUE);
processor.addMetric(FieldMetric.MISSING_SID);
return processor;
}

Expand Down Expand Up @@ -135,10 +130,7 @@ void processWithNullValues() throws JSONException {
},
"metrics": [
{
"MISSING_SID": 2
},
{
"NULL_VALUE": 3
"NULL_VALUE": 1
}
],
"logs": [
Expand Down
3 changes: 1 addition & 2 deletions src/test/java/no/ssb/dlp/pseudo/service/pseudo/RepseudoFieldTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ private static PseudoMetadataProcessor createPseudoMetadataProcessor() {
.stableIdentifierType(STABLE_IDENTIFIER_TYPE)
.build());
processor.addLog("Log line");
processor.addMetric(FieldMetric.MISSING_SID);
return processor;
}
@Test
Expand Down Expand Up @@ -100,7 +99,7 @@ void processWithNullValues() throws JSONException {
},
"metrics": [
{
"MISSING_SID": 1
"NULL_VALUE": 1
}
],
"logs": [
Expand Down
Loading