feat: Properly match file footers (#136)

awslabs · Oct 8, 2018 · f4828b6 · f4828b6
1 parent 21705b1
commit f4828b6
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/setup → setup.sh b/setup → setup.sh
diff --git a/src/com/amazon/kinesis/streaming/agent/tailing/AbstractParser.java b/src/com/amazon/kinesis/streaming/agent/tailing/AbstractParser.java
@@ -469,9 +469,9 @@ private void getNewCurrentBuffer() {
 
     private R buildRecord(int offset, int length) {
         ByteBuffer data = ByteBuffers.getPartialView(currentBuffer, offset, length);
-        
+
         if (fileFooterPattern != null) {
-            final Matcher fileFooterMatcher = fileFooterPattern.matcher(data.asCharBuffer());
+            final Matcher fileFooterMatcher = fileFooterPattern.matcher(ByteBuffers.toString(data, StandardCharsets.UTF_8).trim());
             if(fileFooterMatcher.matches()) {
                 stopParsing("End of file reached, file footer pattern matched");
                 return null;

diff --git a/src/com/amazon/kinesis/streaming/agent/tailing/FileFlow.java b/src/com/amazon/kinesis/streaming/agent/tailing/FileFlow.java
@@ -111,7 +111,7 @@ protected FileFlow(AgentContext context, Configuration config) {
 
         String footerPattern = readString(FILE_FOOTER_PATTERN, null);
 
-        fileFooterPattern = Strings.isNullOrEmpty(footerPattern)? null : Pattern.compile(footerPattern);
+        fileFooterPattern = Strings.isNullOrEmpty(footerPattern)? null : Pattern.compile(footerPattern, Pattern.MULTILINE);
 
         String terminatorConfig = readString("truncatedRecordTerminator", DEFAULT_TRUNCATED_RECORD_TERMINATOR);
         if (terminatorConfig == null || terminatorConfig.getBytes(StandardCharsets.UTF_8).length >= getMaxRecordSizeBytes()) {