Merge pull request #23 from juanjux/avoid_memorystream

Avoid memorystream
bblfsh · Jan 25, 2019 · 05b5a19 · 05b5a19
2 parents 9e3b934 + 95419f1
commit 05b5a19
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 14 deletions.
diff --git a/driver/normalizer/annotation.go b/driver/normalizer/annotation.go
@@ -18,6 +18,8 @@ var Code = []CodeTransformer{
 	positioner.NewFillLineColFromOffset(),
 }
 
+var PreprocessCode = []CodeTransformer{}
+
 func annotateTypeToken(typ, token string, roles ...role.Role) Mapping {
 	return AnnotateType(typ,
 		FieldRoles{

diff --git a/native/pom.xml b/native/pom.xml
@@ -71,7 +71,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.8.11.1</version>
+            <version>2.8.11.3</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

diff --git a/native/src/main/java/tech/sourced/babelfish/DriverResponse.java b/native/src/main/java/tech/sourced/babelfish/DriverResponse.java
@@ -38,12 +38,16 @@ void parseCode(EclipseCPPParser parser, String source) {
         translationUnit = parser.parseCPP(source);
     }
 
+    // Note: since we're using the System.out output stream with Jackson, output will
+    // start to be written before this call so its not a deterministic "send everything".
+    // The reason to not use a ByteArrayOutputStream and send everything in one go is that
+    // sometimes memory can grow too much with some files.
     void send() throws ResponseSendException {
         // FIXME: this includes the errors in the already started document
         try {
             formatWritter.writeValue(this);
-            ByteArrayOutputStream byteOut = formatWritter.getByteOutputStream();
-            System.out.write(byteOut.toByteArray());
+            OutputStream byteOut = formatWritter.getOutputStream();
+            byteOut.flush();
             System.out.write('\n');
         } catch (IOException e) {
             throw new DriverResponse.ResponseSendException(e);

diff --git a/native/src/main/java/tech/sourced/babelfish/IExchangeFormatWritter.java b/native/src/main/java/tech/sourced/babelfish/IExchangeFormatWritter.java
@@ -1,6 +1,6 @@
 package tech.sourced.babelfish;
 
-import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
 import java.io.IOException;
 
 /**
@@ -10,5 +10,5 @@
 public interface IExchangeFormatWritter
 {
     void writeValue(DriverResponse response) throws IOException;
-    ByteArrayOutputStream getByteOutputStream();
+    OutputStream getOutputStream();
 }
diff --git a/native/src/main/java/tech/sourced/babelfish/Main.java b/native/src/main/java/tech/sourced/babelfish/Main.java
@@ -20,7 +20,7 @@ public static void main(String args[]) {
     private static ProcessCycle trySendError(String msg, Exception e) {
         try {
             final TranslationUnitJSONMapper responseJSONMapper =
-                    new TranslationUnitJSONMapper(false, new ByteArrayOutputStream());
+                    new TranslationUnitJSONMapper(false, System.out);
             DriverResponse response = new DriverResponse(responseJSONMapper);
             response.sendError(e, msg);
             return ProcessCycle.CONTINUE;
@@ -38,7 +38,7 @@ static private ProcessCycle process() {
         try {
             final EclipseCPPParser parser = new EclipseCPPParser();
             final TranslationUnitJSONMapper responseJSONMapper =
-                new TranslationUnitJSONMapper(false, new ByteArrayOutputStream());
+                new TranslationUnitJSONMapper(false, System.out);
             response = new DriverResponse(responseJSONMapper);
 
             BufferedReader in = new BufferedReader(new InputStreamReader(System.in));

diff --git a/native/src/main/java/tech/sourced/babelfish/TranslationUnitJSONMapper.java b/native/src/main/java/tech/sourced/babelfish/TranslationUnitJSONMapper.java
@@ -10,20 +10,21 @@
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
 
-import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
 import java.io.IOException;
 
 class TranslationUnitJSONMapper implements IExchangeFormatWritter {
 
     final JsonGenerator generator;
     final JsonFactory jsonFactory = new JsonFactory();
     final ObjectMapper mapper = new ObjectMapper();
-    private ByteArrayOutputStream byteOutputStream;
+    private OutputStream printStream;
 
-    TranslationUnitJSONMapper(boolean prettyPrint, ByteArrayOutputStream byteOutput) throws IOException {
-        this.byteOutputStream = byteOutput;
+    TranslationUnitJSONMapper(boolean prettyPrint, PrintStream byteOutput) throws IOException {
+        this.printStream = byteOutput;
 
-        generator = jsonFactory.createGenerator(byteOutputStream);
+        generator = jsonFactory.createGenerator(printStream);
         if (prettyPrint) {
             generator.setPrettyPrinter(new DefaultPrettyPrinter());
             mapper.enable(SerializationFeature.INDENT_OUTPUT);
@@ -40,7 +41,7 @@ public void writeValue(DriverResponse response) throws IOException {
         mapper.writeValue(generator, response);
     }
 
-    public ByteArrayOutputStream getByteOutputStream() {
-        return byteOutputStream;
+    public OutputStream getOutputStream() {
+        return printStream;
     }
 }