Merge branch 'master' into next

.travis.yml

@@ -1,6 +1,6 @@
 language: java
 before_script: ./build.sh pom.xml
-script: mvn test --quiet
+script: export MAVEN_OPTS="-Xmx1024m" && mvn test --quiet
 jdk:
   - oraclejdk7
   - openjdk6

NEWS

@@ -1,6 +1,22 @@
 OpenTSDB - User visible changes.
 
-* Version 2.0.0 (2014-05-5) [???????]
+* Version 2.0.1 (2014-11-09)
+
+Bug Fixes:
+  - Fix tree REST endpoint to allow modification of the strictMatch and storeFailure fields
+  - Fix compacted millisecond columns throwing an error during fsck
+  - Fix tsd startup directory permission checks
+  - Fix thread safety of pending UID assignment when they are complete
+  - Fix compaction queue flushing on tsd shutdown
+  - Fix RTPublisher plugin floating point value detection where it was previously posting to the wrong method
+  - Fix TsdbQuery.toString() to avoid looing up UIDs if an error was thrown as this can cause cause a deadlock
+
+Noteworthy changes:
+  - Add Travis CI build support
+  - Update to Asynchbase 1.6.0 and update additional dependencies
+  - Modify scripts to allow easier compilation under FreeBSD
+
+* Version 2.0.0 (2014-05-5)
 
 API incompatible changes:
   - The `TSDB' class now takes a `Config' object in argument instead of an

build-aux/create-src-dir-overlay.sh

@@ -1,6 +1,6 @@
 # Creates directory structure overlay on top of original source directories so
 # that the overlay matches Java package hierarchy.
-#!/bin/bash
+#!/usr/bin/env bash
 
 if [ ! -d src-main ]; then
   mkdir src-main

build-aux/fetchdep.sh.in

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 # Getting this value out of Autoconf is not a good idea, because it might be
 # set to a relative path to install-sh.  We don't install `install-sh' during

build-aux/gen_build_data.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 # Generates BuildData.java
 # Usage: gen_build_data.sh path/to/BuildData.java my.package.name
 # Author: Benoit Sigoure ([email protected])

build.sh

@@ -1,7 +1,9 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -xe
 test -f configure || ./bootstrap
 test -d build || mkdir build
 cd build
 test -f Makefile || ../configure "$@"
-exec make "$@"
+MAKE=make
+[ `uname -s` = "FreeBSD" ] && MAKE=gmake
+exec ${MAKE} "$@"

src/core/TsdbQuery.java

@@ -35,7 +35,6 @@
 
 import static org.hbase.async.Bytes.ByteMap;
 import net.opentsdb.stats.Histogram;
-import net.opentsdb.uid.NoSuchUniqueId;
 import net.opentsdb.uid.NoSuchUniqueName;
 import net.opentsdb.uid.UniqueId;
 
@@ -146,7 +145,7 @@ public void setStartTime(final long timestamp) {
   }
 
   /**
-   * @returns the start time for the query
+   * @return the start time for the query
    * @throws IllegalStateException if the start time hasn't been set yet
    */
   @Override
@@ -784,55 +783,42 @@ public String toString() {
        .append(getStartTime())
        .append(", end_time=")
        .append(getEndTime());
-   if (tsuids != null && !tsuids.isEmpty()) {
-     buf.append(", tsuids=");
-     for (final String tsuid : tsuids) {
-       buf.append(tsuid).append(",");
-     }
-   } else {
-      buf.append(", metric=").append(Arrays.toString(metric));
-      try {
-        buf.append(" (").append(tsdb.metrics.getName(metric));
-      } catch (NoSuchUniqueId e) {
-        buf.append(" (<").append(e.getMessage()).append('>');
-      }
-      try {
-        buf.append("), tags=").append(Tags.resolveIds(tsdb, tags));
-      } catch (NoSuchUniqueId e) {
-        buf.append("), tags=<").append(e.getMessage()).append('>');
+    if (tsuids != null && !tsuids.isEmpty()) {
+      buf.append(", tsuids=");
+      for (final String tsuid : tsuids) {
+        buf.append(tsuid).append(",");
       }
-    }
-    buf.append(", rate=").append(rate)
-       .append(", aggregator=").append(aggregator)
-       .append(", group_bys=(");
-    if (group_bys != null) {
-      for (final byte[] tag_id : group_bys) {
-        try {
-          buf.append(tsdb.tag_names.getName(tag_id));
-        } catch (NoSuchUniqueId e) {
-          buf.append('<').append(e.getMessage()).append('>');
+    } else {
+      buf.append(", metric=").append(Arrays.toString(metric));
+      buf.append(", tags=[");
+      for (final Iterator<byte[]> it = tags.iterator(); it.hasNext(); ) {
+        buf.append(Arrays.toString(it.next()));
+        if (it.hasNext()) {
+          buf.append(',');
         }
-        buf.append(' ')
-           .append(Arrays.toString(tag_id));
-        if (group_by_values != null) {
-          final byte[][] value_ids = group_by_values.get(tag_id);
-          if (value_ids == null) {
-            continue;
-          }
-          buf.append("={");
-          for (final byte[] value_id : value_ids) {
-            try {
-              buf.append(tsdb.tag_values.getName(value_id));
-            } catch (NoSuchUniqueId e) {
-              buf.append('<').append(e.getMessage()).append('>');
+      }
+      buf.append("], rate=").append(rate)
+        .append(", aggregator=").append(aggregator)
+        .append(", group_bys=(");
+      if (group_bys != null) {
+        for (final byte[] tag_id : group_bys) {
+          buf.append(Arrays.toString(tag_id));
+          if (group_by_values != null) {
+            final byte[][] value_ids = group_by_values.get(tag_id);
+            if (value_ids == null) {
+              continue;
+            }
+            buf.append("={");
+            for (int i = 0; i < value_ids.length; i++) {
+              buf.append(Arrays.toString(value_ids[i]));
+              if (i < value_ids.length - 1) {
+                buf.append(',');
+              }
             }
-            buf.append(' ')
-               .append(Arrays.toString(value_id))
-               .append(", ");
+            buf.append('}');
           }
-          buf.append('}');
+          buf.append(", ");
         }
-        buf.append(", ");
       }
     }
     buf.append("))");

src/meta/Annotation.java

@@ -273,7 +273,7 @@ public Deferred<Annotation> call(final ArrayList<KeyValue> row)
           return Deferred.fromResult(null);
         }
 
-        Annotation note = JSON.parseToObject(row.get(0).value(), 
+        Annotation note = JSON.parseToObject(row.get(0).value(),
             Annotation.class);
         return Deferred.fromResult(note);
       }
@@ -356,7 +356,7 @@ public Deferred<List<Annotation>> call (
           for (KeyValue column : row) {
             if ((column.qualifier().length == 3 || column.qualifier().length == 5) 
                 && column.qualifier()[0] == PREFIX()) {
-              Annotation note = JSON.parseToObject(row.get(0).value(), 
+              Annotation note = JSON.parseToObject(column.value(),
                   Annotation.class);
               if (note.start_time < start_time || note.end_time > end_time) {
                 continue;

src/opentsdb.conf

@@ -55,6 +55,6 @@ tsd.http.cachedir =
 # Path under which the znode for the -ROOT- region is located, default is "/hbase"
 #tsd.storage.hbase.zk_basedir = /hbase
 
-# A space separated list of Zookeeper hosts to connect to, with or without 
+# A comma separated list of Zookeeper hosts to connect to, with or without 
 # port specifiers, default is "localhost"
 #tsd.storage.hbase.zk_quorum = localhost

src/tsd/RTPublisher.java

@@ -97,7 +97,7 @@ public abstract class RTPublisher {
   public final Deferred<Object> sinkDataPoint(final String metric, 
       final long timestamp, final byte[] value, final Map<String, String> tags, 
       final byte[] tsuid, final short flags) {
-    if ((flags & Const.FLAG_FLOAT) == 0x0) {
+    if ((flags & Const.FLAG_FLOAT) != 0x0) {
       return publishDataPoint(metric, timestamp, 
           Internal.extractFloatingPointValue(value, 0, (byte) flags), 
           tags, tsuid);

src/uid/UniqueId.java

@@ -767,7 +767,12 @@ public Object call(final ArrayList<ArrayList<KeyValue>> rows) {
         }
         suggestions.add(name);
         if ((short) suggestions.size() >= max_results) {  // We have enough.
-          return suggestions;
+          return scanner.close().addCallback(new Callback<Object, Object>() {
+            @Override
+            public Object call(Object ignored) throws Exception {
+              return suggestions;
+            }
+          });
         }
         row.clear();  // free()
       }

test/meta/TestAnnotation.java

@@ -179,6 +179,10 @@ public void getGlobalAnnotations() throws Exception {
         1328141000).joinUninterruptibly();
     assertNotNull(notes);
     assertEquals(2, notes.size());
+    Annotation note0 = notes.get(0);
+    Annotation note1 = notes.get(1);
+    assertEquals("Description", note0.getDescription());
+    assertEquals("Global 2", note1.getDescription());
   }
 
   @Test

test/tsd/TestPutRpc.java

@@ -549,13 +549,19 @@ public void ValueNInfiniy() throws Exception {
     assertTrue(response.contains("\"success\":0"));
   }
 
-  @Test (expected = BadRequestException.class)
+  @Test
   public void ValueInfinityUnsigned() throws Exception {
     HttpQuery query = NettyMocks.postQuery(tsdb, "/api/put?details", 
         "{\"metric\":\"sys.cpu.nice\",\"timestamp\":1365465600,\"value\""
         +":Infinity,\"tags\":{\"host\":\"web01\"}}");
     PutDataPointRpc put = new PutDataPointRpc();
     put.execute(tsdb, query);
+    assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus());
+    final String response = 
+      query.response().getContent().toString(Charset.forName("UTF-8"));
+    assertTrue(response.contains("\"error\":\"Unable to parse value to a number\""));
+    assertTrue(response.contains("\"failed\":1"));
+    assertTrue(response.contains("\"success\":0"));
   }
 
   @Test

tools/clean_cache.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 CACHE_DIR=/tmp/tsd
 

tsdb.in

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 me=`basename "$0"`