From 368350b321938b1e9f6746430ff20cf7c276ac2a Mon Sep 17 00:00:00 2001 From: humanfy Date: Wed, 26 May 2021 08:55:09 +0800 Subject: [PATCH 1/8] fc --- .../tsinghua/iginx/rest/MetricsResource.java | 27 ++++- .../iginx/rest/query/QueryParser.java | 114 ++++++++++++++++++ 2 files changed, 137 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java index 1835f688b..a6e671378 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java @@ -63,15 +63,32 @@ public Response OK() @POST @Path("query") - public Response Grafana_query(final InputStream stream) + public Response grafanaQuery(String jsonStr) { + try + { + if (jsonStr == null) + { + throw new Exception("query json must not be null or empty"); + } + QueryParser parser = new QueryParser(); + Query query = parser.parseGrafanaQueryMetric(jsonStr); + QueryExecutor executor = new QueryExecutor(query); + QueryResult result = executor.execute(false); + String entity = parser.parseResultToGrafanaJson(result); + return setHeaders(Response.status(Status.OK).entity(entity + "\n")).build(); - return setHeaders(Response.status(Status.OK)).build(); + } + catch (Exception e) + { + LOGGER.error("Error occurred during execution ", e); + return setHeaders(Response.status(Status.BAD_REQUEST).entity("Error occurred during execution\n")).build(); + } } @POST @Path("{string : .+}") - public Response errorPath(@PathParam("string") String str) + public Response postErrorPath(@PathParam("string") String str) { return setHeaders(Response.status(Status.NOT_FOUND).entity("Wrong path\n")).build(); } @@ -79,7 +96,7 @@ public Response errorPath(@PathParam("string") String str) @GET @Path("{string : .+}") - public Response geterrorPath(@PathParam("string") String str) + public Response getErrorPath(@PathParam("string") String str) { return setHeaders(Response.status(Status.NOT_FOUND).entity("Wrong path\n")).build(); } @@ -220,4 +237,6 @@ void deleteMetric(String metricName) throws Exception restSession.deleteColumns(ins); restSession.closeSession(); } + + } \ No newline at end of file diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java index 795ff2fda..f015658f3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java @@ -4,9 +4,15 @@ import cn.edu.tsinghua.iginx.rest.query.aggregator.*; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.Iterator; public class QueryParser @@ -19,6 +25,23 @@ public QueryParser() } + public Query parseGrafanaQueryMetric(String json) throws Exception + { + Query ret; + try + { + JsonNode node = mapper.readTree(json); + ret = getGrafanaQuery(node); + } + catch (Exception e) + { + LOGGER.error("Error occurred during parsing query ", e); + throw e; + } + return ret; + } + + public Query parseQueryMetric(String json) throws Exception { Query ret; @@ -35,6 +58,67 @@ public Query parseQueryMetric(String json) throws Exception return ret; } + + public static Long dealDateFormat(String oldDateStr) + { + DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + try + { + Date date = df.parse(oldDateStr); + return date.getTime() + 28800000l; + } + catch (ParseException e) + { + e.printStackTrace(); + } + return null; + } + + + private Query getGrafanaQuery(JsonNode node) + { + Query ret = new Query(); + JsonNode timerange = node.get("range"); + if (timerange == null) + return null; + + JsonNode start_absolute = timerange.get("from"); + JsonNode end_absolute = timerange.get("to"); + + + if (start_absolute == null || end_absolute == null) + return null; + + Long start = dealDateFormat(start_absolute.asText()); + Long end = dealDateFormat(end_absolute.asText()); + ret.setStartAbsolute(start); + ret.setEndAbsolute(end); + + JsonNode array = node.get("targets"); + if (!array.isArray()) + return null; + for (JsonNode jsonNode: array) + { + QueryMetric queryMetric = new QueryMetric(); + JsonNode type = jsonNode.get("type"); + if (type == null) + return null; + JsonNode target = jsonNode.get("target"); + if (target == null) + return null; + if (type.asText().equals("table")) + { + queryMetric.setName(target.asText()); + } + else + { + queryMetric.setName(target.asText()); + } + ret.addQueryMetrics(queryMetric); + } + return ret; + } + private Query getQuery(JsonNode node) { Query ret = new Query(); @@ -404,4 +488,34 @@ public String parseResultToJson(QueryResult result, boolean isDelete) ret.append("]}"); return ret.toString(); } + + public String parseResultToGrafanaJson(QueryResult result) + { + StringBuilder ret = new StringBuilder("["); + for (int i=0; i< result.getSiz(); i++) + { + ret.append("{"); + ret.append(String.format("\"target\":\"%s\",", result.getQueryMetrics().get(i).getName())); + ret.append("\"datapoints\":["); + int n = result.getQueryResultDatasets().get(i).getSize(); + for (int j=0;j Date: Tue, 1 Jun 2021 09:18:28 +0800 Subject: [PATCH 2/8] fc --- .../tsinghua/iginx/rest/MetricsResource.java | 39 +++++++++++ .../iginx/rest/insert/DataPointsParser.java | 19 +++++- .../tsinghua/iginx/rest/insert/Metric.java | 11 ++++ .../tsinghua/iginx/rest/query/Annotation.java | 15 +++++ .../iginx/rest/query/QueryExecutor.java | 6 +- .../iginx/rest/query/QueryMetric.java | 13 +++- .../iginx/rest/query/QueryParser.java | 65 +++++++++++++++++++ .../iginx/rest/query/QueryResult.java | 19 ++++++ 8 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java index 5787711d3..f1ed71179 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java @@ -27,6 +27,8 @@ import cn.edu.tsinghua.iginx.rest.query.QueryExecutor; import cn.edu.tsinghua.iginx.rest.query.QueryParser; import cn.edu.tsinghua.iginx.rest.query.QueryResult; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +47,11 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -71,6 +76,8 @@ public MetricsResource() { static Response.ResponseBuilder setHeaders(Response.ResponseBuilder responseBuilder) { responseBuilder.header("Access-Control-Allow-Origin", "*"); + responseBuilder.header("Access-Control-Allow-Methods", "POST"); + responseBuilder.header("Access-Control-Allow-Headers", "accept, content-type"); responseBuilder.header("Pragma", NO_CACHE); responseBuilder.header("Cache-Control", NO_CACHE); responseBuilder.header("Expires", 0); @@ -121,6 +128,19 @@ public Response grafanaQuery(String jsonStr) { } } + @POST + @Path("annotations") + public Response grafanaAnnotation(String jsonStr) { + + try { + return postAnnotationQuery(jsonStr); + } catch (Exception e) { + LOGGER.error("Error occurred during execution ", e); + return setHeaders(Response.status(Status.BAD_REQUEST).entity("Error occurred during execution\n")).build(); + } + } + + @POST @Path("{string : .+}") public Response errorPath(@PathParam("string") String str) { @@ -169,6 +189,25 @@ public Response postQuery(String jsonStr) { } } + + public Response postAnnotationQuery(String jsonStr) { + try { + if (jsonStr == null) { + throw new Exception("query json must not be null or empty"); + } + QueryParser parser = new QueryParser(); + Query query = parser.parseAnnotationQueryMetric(jsonStr); + QueryExecutor executor = new QueryExecutor(query); + QueryResult result = executor.execute(false); + String entity = parser.parseResultToGrafanaAnnotationJson(result); + return setHeaders(Response.status(Status.OK).entity(entity + "\n")).build(); + + } catch (Exception e) { + LOGGER.error("Error occurred during execution ", e); + return setHeaders(Response.status(Status.BAD_REQUEST).entity("Error occurred during execution\n")).build(); + } + } + @POST @Path(DELETE_URL) public Response postDelete(final InputStream stream) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java index b8d2e63ba..61b89fe36 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java @@ -31,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.io.Reader; import java.util.ArrayList; import java.util.Iterator; @@ -41,6 +40,7 @@ import java.util.concurrent.ConcurrentHashMap; public class DataPointsParser { + public static final String ANNOTATION_SPLIT_STRING = "@@annotation"; private static final Logger LOGGER = LoggerFactory.getLogger(DataPointsParser.class); private static Config config = ConfigDescriptor.getInstance().getConfig(); private final IMetaManager metaManager = SortedListAbstractMetaManager.getInstance(); @@ -114,6 +114,12 @@ private Metric getMetricObject(JsonNode node) { } } } + JsonNode anno = node.get("annotation"); + if (anno != null) + { + ret.setAnnotation(anno.toString().replace("\n", "") + .replace("\t", "").replace(" ", "")); + } return ret; } @@ -185,6 +191,17 @@ private void sendMetricsData() throws Exception valuesList[0] = values; try { session.insertColumnRecords(paths, metric.getTimestamps().stream().mapToLong(t -> t.longValue()).toArray(), valuesList, type, null); + if (metric.getAnnotation() != null) + { + for (int i = 0; i < size; i++) { + values[i] = metric.getAnnotation().getBytes(); + } + valuesList[0] = values; + path.append(ANNOTATION_SPLIT_STRING); + paths.set(0, path.toString()); + type.set(0, DataType.BINARY); + session.insertColumnRecords(paths, metric.getTimestamps().stream().mapToLong(t -> t.longValue()).toArray(), valuesList, type, null); + } } catch (ExecutionException e) { LOGGER.error("Error occurred during insert ", e); throw e; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/Metric.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/Metric.java index a5453bbaf..af7d4a5bd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/Metric.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/Metric.java @@ -28,6 +28,7 @@ public class Metric { private Map tags = new TreeMap<>(); private List timestamps = new ArrayList<>(); private List values = new ArrayList<>(); + private String annotation = null; public String getName() { return name; @@ -72,4 +73,14 @@ public void addTimestamp(Long timestamp) { public void addValue(String value) { values.add(value); } + + public String getAnnotation() + { + return annotation; + } + + public void setAnnotation(String annotation) + { + this.annotation = annotation; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java new file mode 100644 index 000000000..7b34cf54b --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java @@ -0,0 +1,15 @@ +package cn.edu.tsinghua.iginx.rest.query; + +import java.util.List; + +public class Annotation +{ + List tags; + String text; + String title; + Long timestamp; + Annotation(String str, Long tim) + { + timestamp = tim; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryExecutor.java index 9083e6d25..bfcd95abd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryExecutor.java @@ -23,6 +23,7 @@ import cn.edu.tsinghua.iginx.metadata.IMetaManager; import cn.edu.tsinghua.iginx.metadata.SortedListAbstractMetaManager; import cn.edu.tsinghua.iginx.rest.RestSession; +import cn.edu.tsinghua.iginx.rest.insert.DataPointsParser; import cn.edu.tsinghua.iginx.rest.query.aggregator.QueryAggregator; import cn.edu.tsinghua.iginx.rest.query.aggregator.QueryAggregatorNone; import org.slf4j.Logger; @@ -109,7 +110,10 @@ void dfsInsert(int depth, List Paths, Map pos2path, Que path.append("*."); now++; } - path.append(queryMetric.getName()); + if (queryMetric.getAnnotation()) + path.append(queryMetric.getName() + DataPointsParser.ANNOTATION_SPLIT_STRING); + else + path.append(queryMetric.getName()); Paths.add(path.toString()); return; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryMetric.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryMetric.java index c99d8b68a..59b75cb92 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryMetric.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryMetric.java @@ -30,7 +30,7 @@ public class QueryMetric { private Long limit; private Map> tags = new TreeMap(); private List aggregators = new ArrayList<>(); - + private Boolean annotation = false; public String getName() { return name; @@ -74,4 +74,15 @@ public void addTag(String key, String value) { public void addAggregator(QueryAggregator qa) { aggregators.add(qa); } + + + public void setAnnotation(Boolean annotation) + { + this.annotation = annotation; + } + + public Boolean getAnnotation() + { + return annotation; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java index 1876fe288..6ec4711d6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java @@ -35,6 +35,7 @@ import cn.edu.tsinghua.iginx.rest.query.aggregator.QueryAggregatorSampler; import cn.edu.tsinghua.iginx.rest.query.aggregator.QueryAggregatorSaveAs; import cn.edu.tsinghua.iginx.rest.query.aggregator.QueryAggregatorSum; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.JsonArray; @@ -84,6 +85,17 @@ public Query parseQueryMetric(String json) throws Exception { return ret; } + public Query parseAnnotationQueryMetric(String json) throws Exception { + Query ret; + try { + JsonNode node = mapper.readTree(json); + ret = getAnnotationQuery(node); + } catch (Exception e) { + LOGGER.error("Error occurred during parsing query ", e); + throw e; + } + return ret; + } public static Long dealDateFormat(String oldDateStr) { DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); @@ -271,6 +283,51 @@ else if (start_absolute != null && end_absolute != null) { return ret; } + private Query getAnnotationQuery(JsonNode node) throws JsonProcessingException + { + Query ret = new Query(); + JsonNode range = node.get("range"); + if (range == null) + return null; + JsonNode start_absolute = range.get("from"); + JsonNode end_absolute = range.get("to"); + if (start_absolute == null || end_absolute == null) + return null; + else + { + Long start = dealDateFormat(start_absolute.asText()); + Long end = dealDateFormat(end_absolute.asText()); + ret.setStartAbsolute(start); + ret.setEndAbsolute(end); + } + + JsonNode metric = node.get("annotation"); + if (metric == null) + return null; + QueryMetric ins = new QueryMetric(); + JsonNode name = metric.get("name"); + if (name != null) + ins.setName(name.asText()); + JsonNode tags = metric.get("query"); + if (tags != null) + { + tags = mapper.readTree(tags.asText()); + tags = tags.get("tags"); + Iterator fieldNames = tags.fieldNames(); + while (fieldNames.hasNext()) + { + String key = fieldNames.next(); + JsonNode valuenode = tags.get(key); + ins.addTag(key, valuenode.asText()); + } + } + ins.setAnnotation(true); + ret.addQueryMetrics(ins); + + return ret; + } + + public void addAggregators(QueryMetric q, JsonNode node) { JsonNode aggregators = node.get("aggregators"); if (aggregators == null || !aggregators.isArray()) @@ -490,6 +547,14 @@ public String parseResultToJson(QueryResult result, boolean isDelete) { return ret.toString(); } + public String parseResultToGrafanaAnnotationJson(QueryResult result) + { + StringBuilder ret = new StringBuilder("["); + ret.append(result.toAnnotationResultString()); + ret.append("]"); + return ret.toString(); + } + public String parseResultToGrafanaJson(QueryResult result) { StringBuilder ret = new StringBuilder("["); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java index c37123653..765ddd058 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java @@ -106,6 +106,25 @@ public String toResultString(int num) { return ret.toString(); } + + + + /* +String entity = "[{\"text\": \"text shown in body\",\"title\": \"Annotation Title\",\"isRegion\": true,\"time\": \"1400000013000\",\"timeEnd\": \"1400000015000\",\"tags\": [\"tag1\"]}]"; +*/ + + public String toAnnotationResultString() { + StringBuilder ret = new StringBuilder("{"); + List values = new ArrayList<>(); + int siz = queryResultDatasets.get(0).getValues().size(); + for (int i = 0 ;i < siz; i++) + values.add(new Annotation(new String((byte[]) queryResultDatasets.get(0).getValues().get(i)), + queryResultDatasets.get(0).getTimestamps().get(i))); + + ret.append("}"); + return ret.toString(); + } + private String nameToString(int num) { if (queryAggregators.get(num).getType() == QueryAggregatorType.SAVE_AS) { return String.format("\"name\": \"%s\"", queryAggregators.get(num).getMetric_name()); From a4429525788f0e67e059c8acef576f24a4528359 Mon Sep 17 00:00:00 2001 From: humanfy Date: Tue, 1 Jun 2021 09:49:17 +0800 Subject: [PATCH 3/8] fix bug --- .../tsinghua/iginx/rest/query/Annotation.java | 62 ++++++++++++++++++- .../iginx/rest/query/QueryResult.java | 36 +++++++++-- 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java index 7b34cf54b..9fcd1b8af 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java @@ -1,15 +1,75 @@ package cn.edu.tsinghua.iginx.rest.query; +import cn.edu.tsinghua.iginx.rest.insert.DataPointsParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.jetbrains.annotations.NotNull; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; import java.util.List; +import java.util.logging.Logger; + +import static java.lang.Thread.sleep; public class Annotation { - List tags; + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(Annotation.class); + private ObjectMapper mapper = new ObjectMapper(); + List tags = new ArrayList<>(); String text; String title; Long timestamp; Annotation(String str, Long tim) { + timestamp = tim; + try + { + JsonNode node = mapper.readTree(str); + if (node == null) + return; + JsonNode text = node.get("text"); + if (text != null) + { + this.text = text.asText(); + } + JsonNode title = node.get("title"); + if (title != null) + { + this.title = title.asText(); + } + JsonNode tags = node.get("tags"); + if (tags != null && tags.isArray()) + { + for (JsonNode tagsnode : tags) + { + this.tags.add(tagsnode.asText()); + } + } + } + catch (Exception e) + { + e.printStackTrace(); + LOGGER.error("Wrong annotation form in database"); + } + + } + + public boolean isEqual(Annotation p) + { + if (p.text.compareTo(text) != 0) + return false; + if (p.title.compareTo(title) != 0) + return false; + if (p.tags.size() != tags.size()) + return false; + for (int i = 0; i < p.tags.size(); i++) + if (p.tags.get(i).compareTo(tags.get(i)) != 0) + { + return false; + } + return true; } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java index 765ddd058..52e4e3af6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java @@ -114,14 +114,42 @@ public String toResultString(int num) { */ public String toAnnotationResultString() { - StringBuilder ret = new StringBuilder("{"); + StringBuilder ret = new StringBuilder(""); List values = new ArrayList<>(); int siz = queryResultDatasets.get(0).getValues().size(); - for (int i = 0 ;i < siz; i++) + for (int i = 0; i < siz; i++) values.add(new Annotation(new String((byte[]) queryResultDatasets.get(0).getValues().get(i)), queryResultDatasets.get(0).getTimestamps().get(i))); - - ret.append("}"); + int now = 0; + for (int i = 1; i < siz; i++) + if (!values.get(i).isEqual(values.get(i-1))) + { + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(i-1).text)); + ret.append(String.format("\"title\": \"%s\",", values.get(i-1).title)); + ret.append("\"isRegion\": true,"); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(i-1).timestamp)); + ret.append("\"tags\": ["); + for (String tag : values.get(i-1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]},"); + now = i; + } + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(siz-1).text)); + ret.append(String.format("\"title\": \"%s\",", values.get(siz-1).title)); + ret.append("\"isRegion\": true,"); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(siz-1).timestamp)); + ret.append("\"tags\": ["); + for (String tag : values.get(siz-1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]}"); return ret.toString(); } From 1ab1e0144390a06e33942898756713bc7dd828e9 Mon Sep 17 00:00:00 2001 From: humanfy Date: Tue, 1 Jun 2021 10:08:28 +0800 Subject: [PATCH 4/8] fix bug --- .../tsinghua/iginx/rest/MetricsResource.java | 8 ++ .../iginx/rest/insert/DataPointsParser.java | 120 +++++++++++++++++- .../rest/insert/InsertAnnotationWorker.java | 56 ++++++++ 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/InsertAnnotationWorker.java diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java index f1ed71179..93feb6ba1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java @@ -22,6 +22,7 @@ import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; import cn.edu.tsinghua.iginx.metadata.IMetaManager; import cn.edu.tsinghua.iginx.metadata.SortedListAbstractMetaManager; +import cn.edu.tsinghua.iginx.rest.insert.InsertAnnotationWorker; import cn.edu.tsinghua.iginx.rest.insert.InsertWorker; import cn.edu.tsinghua.iginx.rest.query.Query; import cn.edu.tsinghua.iginx.rest.query.QueryExecutor; @@ -60,6 +61,7 @@ public class MetricsResource { private final IMetaManager metaManager = SortedListAbstractMetaManager.getInstance(); private static final String INSERT_URL = "api/v1/datapoints"; + private static final String INSERT_ANNOTATION_URL = "api/v1/datapoints/annotation"; private static final String QUERY_URL = "api/v1/datapoints/query"; private static final String DELETE_URL = "api/v1/datapoints/delete"; private static final String DELETE_METRIC_URL = "api/v1/metric/{metricName}"; @@ -128,6 +130,12 @@ public Response grafanaQuery(String jsonStr) { } } + @POST + @Path(INSERT_ANNOTATION_URL) + public void addAnnotation(@Context HttpHeaders httpheaders, final InputStream stream, @Suspended final AsyncResponse asyncResponse) { + threadPool.execute(new InsertAnnotationWorker(asyncResponse, httpheaders, stream)); + } + @POST @Path("annotations") public Response grafanaAnnotation(String jsonStr) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java index 61b89fe36..ff19e797e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java @@ -59,7 +59,6 @@ public DataPointsParser(Reader stream) { public void parse() throws Exception { - try { session.openSession(); } catch (SessionException e) { @@ -90,6 +89,72 @@ public void parse() throws Exception } } + public void parseAnnotation() throws Exception + { + try { + session.openSession(); + } catch (SessionException e) { + LOGGER.error("Error occurred during opening session", e); + throw e; + } + try { + JsonNode node = mapper.readTree(inputStream); + if (node.isArray()) { + for (JsonNode objNode : node) { + metricList.add(getAnnotationMetricObject(objNode)); + } + } else { + metricList.add(getAnnotationMetricObject(node)); + } + + } catch (Exception e) { + LOGGER.error("Error occurred during parsing data ", e); + throw e; + } + try { + sendAnnotationMetricsData(); + } catch (Exception e) { + LOGGER.debug("Exception occur for create and send ", e); + throw e; + } finally { + session.closeSession(); + } + } + + private Metric getAnnotationMetricObject(JsonNode node) { + Metric ret = new Metric(); + ret.setName(node.get("name").asText()); + Iterator fieldNames = node.get("tags").fieldNames(); + Iterator elements = node.get("tags").elements(); + while (elements.hasNext() && fieldNames.hasNext()) { + ret.addTag(fieldNames.next(), elements.next().textValue()); + } + JsonNode tim = node.get("timestamp"), val = node.get("value"); + if (tim != null && val != null) { + ret.addTimestamp(tim.asLong()); + ret.addValue(val.asText()); + } + JsonNode dp = node.get("datapoints"); + if (dp != null) + { + if (dp.isArray()) + { + for (JsonNode dpnode : dp) + { + ret.addTimestamp(dpnode.asLong()); + } + } + } + JsonNode anno = node.get("annotation"); + if (anno != null) + { + ret.setAnnotation(anno.toString().replace("\n", "") + .replace("\t", "").replace(" ", "")); + } + return ret; + } + + private Metric getMetricObject(JsonNode node) { Metric ret = new Metric(); ret.setName(node.get("name").asText()); @@ -209,6 +274,59 @@ private void sendMetricsData() throws Exception } } + private void sendAnnotationMetricsData() throws Exception + { + for (Metric metric: metricList) + { + boolean needUpdate = false; + Map metricschema = metaManager.getSchemaMapping(metric.getName()); + if (metricschema == null) { + needUpdate = true; + metricschema = new ConcurrentHashMap<>(); + } + Iterator iter = metric.getTags().entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = (Map.Entry) iter.next(); + if (metricschema.get(entry.getKey()) == null) { + needUpdate = true; + int pos = metricschema.size() + 1; + metricschema.put((String) entry.getKey(), pos); + } + } + if (needUpdate) + metaManager.addOrUpdateSchemaMapping(metric.getName(), metricschema); + Map pos2path = new TreeMap<>(); + for (Map.Entry entry : metricschema.entrySet()) + pos2path.put(entry.getValue(), entry.getKey()); + StringBuilder path = new StringBuilder(""); + iter = pos2path.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = (Map.Entry) iter.next(); + String ins = metric.getTags().get(entry.getValue()); + if (ins != null) + path.append(ins + "."); + else + path.append("null."); + } + path.append(metric.getName()); + path.append(ANNOTATION_SPLIT_STRING); + List paths = new ArrayList<>(); + paths.add(path.toString()); + List type = new ArrayList<>(); + type.add(DataType.BINARY); + int size = metric.getTimestamps().size(); + Object[] valuesList = new Object[1]; + Object[] values = new Object[size]; + for (int i = 0; i < size; i++) + { + values[i] = metric.getAnnotation().getBytes(); + } + valuesList[0] = values; + session.insertColumnRecords(paths, metric.getTimestamps().stream().mapToLong(t -> t.longValue()).toArray(), valuesList, type, null); + } + } + + Object getType(String str, DataType tp) { switch (tp) { case BINARY: diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/InsertAnnotationWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/InsertAnnotationWorker.java new file mode 100644 index 000000000..1d9f8e29f --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/InsertAnnotationWorker.java @@ -0,0 +1,56 @@ +package cn.edu.tsinghua.iginx.rest.insert; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.container.AsyncResponse; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.Response; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.zip.GZIPInputStream; + +public class InsertAnnotationWorker extends Thread +{ + private static final String NO_CACHE = "no-cache"; + private static final Logger LOGGER = LoggerFactory.getLogger(InsertAnnotationWorker.class); + private HttpHeaders httpheaders; + private InputStream stream; + private AsyncResponse asyncResponse; + + public InsertAnnotationWorker(final AsyncResponse asyncResponse, HttpHeaders httpheaders, + InputStream stream) { + this.asyncResponse = asyncResponse; + this.httpheaders = httpheaders; + this.stream = stream; + } + + static Response.ResponseBuilder setHeaders(Response.ResponseBuilder responseBuilder) { + responseBuilder.header("Access-Control-Allow-Origin", "*"); + responseBuilder.header("Pragma", NO_CACHE); + responseBuilder.header("Cache-Control", NO_CACHE); + responseBuilder.header("Expires", 0); + return (responseBuilder); + } + + @Override + public void run() { + Response response; + try { + if (httpheaders != null) { + List requestHeader = httpheaders.getRequestHeader("Content-Encoding"); + if (requestHeader != null && requestHeader.contains("gzip")) { + stream = new GZIPInputStream(stream); + } + } + DataPointsParser parser = new DataPointsParser(new InputStreamReader(stream, StandardCharsets.UTF_8)); + parser.parseAnnotation(); + response = Response.status(Response.Status.OK).build(); + } catch (Exception e) { + response = setHeaders(Response.status(Response.Status.BAD_REQUEST).entity("Error occurred during execution\n")).build(); + } + asyncResponse.resume(response); + } +} From 3a455ac87735cbe33f4fe2fc1c1daf7ac521437a Mon Sep 17 00:00:00 2001 From: humanfy Date: Tue, 1 Jun 2021 11:01:42 +0800 Subject: [PATCH 5/8] change name --- .../java/cn/edu/tsinghua/iginx/rest/query/Annotation.java | 4 ++-- .../java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java index 9fcd1b8af..5f3230ba2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java @@ -30,7 +30,7 @@ public class Annotation JsonNode node = mapper.readTree(str); if (node == null) return; - JsonNode text = node.get("text"); + JsonNode text = node.get("description"); if (text != null) { this.text = text.asText(); @@ -40,7 +40,7 @@ public class Annotation { this.title = title.asText(); } - JsonNode tags = node.get("tags"); + JsonNode tags = node.get("category"); if (tags != null && tags.isArray()) { for (JsonNode tagsnode : tags) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java index 6ec4711d6..c453074dd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java @@ -311,7 +311,8 @@ private Query getAnnotationQuery(JsonNode node) throws JsonProcessingException JsonNode tags = metric.get("query"); if (tags != null) { - tags = mapper.readTree(tags.asText()); + if (tags.get("tags") == null) + tags = mapper.readTree(tags.asText()); tags = tags.get("tags"); Iterator fieldNames = tags.fieldNames(); while (fieldNames.hasNext()) From 03bb051a2f776182e1b48d0d8cbc8507eeda86dd Mon Sep 17 00:00:00 2001 From: humanfy Date: Tue, 1 Jun 2021 11:04:20 +0800 Subject: [PATCH 6/8] fix bug --- .../main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java index 52e4e3af6..619012afe 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java @@ -121,6 +121,8 @@ public String toAnnotationResultString() { values.add(new Annotation(new String((byte[]) queryResultDatasets.get(0).getValues().get(i)), queryResultDatasets.get(0).getTimestamps().get(i))); int now = 0; + if (siz == 0) + return "{}"; for (int i = 1; i < siz; i++) if (!values.get(i).isEqual(values.get(i-1))) { From 466c2ee388d3094839732f80233fd7505b7f81e5 Mon Sep 17 00:00:00 2001 From: humanfy Date: Thu, 3 Jun 2021 07:28:45 +0800 Subject: [PATCH 7/8] add annotation query --- .../tsinghua/iginx/rest/MetricsResource.java | 23 +- .../tsinghua/iginx/rest/query/Annotation.java | 19 +- .../iginx/rest/query/AnnotationLimit.java | 38 ++++ .../iginx/rest/query/QueryMetric.java | 11 + .../iginx/rest/query/QueryParser.java | 208 ++++++++++++++---- .../iginx/rest/query/QueryResult.java | 92 +++++--- 6 files changed, 312 insertions(+), 79 deletions(-) create mode 100644 core/src/main/java/cn/edu/tsinghua/iginx/rest/query/AnnotationLimit.java diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java index 93feb6ba1..b3830e54d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java @@ -61,8 +61,9 @@ public class MetricsResource { private final IMetaManager metaManager = SortedListAbstractMetaManager.getInstance(); private static final String INSERT_URL = "api/v1/datapoints"; - private static final String INSERT_ANNOTATION_URL = "api/v1/datapoints/annotation"; + private static final String INSERT_ANNOTATION_URL = "api/v1/datapoints/annotations"; private static final String QUERY_URL = "api/v1/datapoints/query"; + private static final String QUERY_ANNOTATION_URL = "api/v1/datapoints/query/annotations"; private static final String DELETE_URL = "api/v1/datapoints/delete"; private static final String DELETE_METRIC_URL = "api/v1/metric/{metricName}"; private static final String NO_CACHE = "no-cache"; @@ -141,7 +142,19 @@ public void addAnnotation(@Context HttpHeaders httpheaders, final InputStream st public Response grafanaAnnotation(String jsonStr) { try { - return postAnnotationQuery(jsonStr); + return postAnnotationQuery(jsonStr, true); + } catch (Exception e) { + LOGGER.error("Error occurred during execution ", e); + return setHeaders(Response.status(Status.BAD_REQUEST).entity("Error occurred during execution\n")).build(); + } + } + + @POST + @Path(QUERY_ANNOTATION_URL) + public Response queryAnnotation(String jsonStr) { + + try { + return postAnnotationQuery(jsonStr, false); } catch (Exception e) { LOGGER.error("Error occurred during execution ", e); return setHeaders(Response.status(Status.BAD_REQUEST).entity("Error occurred during execution\n")).build(); @@ -198,16 +211,16 @@ public Response postQuery(String jsonStr) { } - public Response postAnnotationQuery(String jsonStr) { + public Response postAnnotationQuery(String jsonStr, boolean isGrafana) { try { if (jsonStr == null) { throw new Exception("query json must not be null or empty"); } QueryParser parser = new QueryParser(); - Query query = parser.parseAnnotationQueryMetric(jsonStr); + Query query = parser.parseAnnotationQueryMetric(jsonStr, isGrafana); QueryExecutor executor = new QueryExecutor(query); QueryResult result = executor.execute(false); - String entity = parser.parseResultToGrafanaAnnotationJson(result); + String entity = parser.parseResultToAnnotationJson(result, isGrafana); return setHeaders(Response.status(Status.OK).entity(entity + "\n")).build(); } catch (Exception e) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java index 5f3230ba2..2991b7f4d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java @@ -1,17 +1,11 @@ package cn.edu.tsinghua.iginx.rest.query; -import cn.edu.tsinghua.iginx.rest.insert.DataPointsParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; - +import java.util.regex.Pattern; import java.util.ArrayList; import java.util.List; -import java.util.logging.Logger; - -import static java.lang.Thread.sleep; public class Annotation { @@ -72,4 +66,15 @@ public boolean isEqual(Annotation p) } return true; } + + boolean match(AnnotationLimit annotationLimit) + { + if (!Pattern.matches(annotationLimit.getText(), text)) return false; + if (!Pattern.matches(annotationLimit.getTitle(), title)) return false; + for (String tag: tags) + { + if (Pattern.matches(annotationLimit.getTag(), tag)) return true; + } + return false; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/AnnotationLimit.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/AnnotationLimit.java new file mode 100644 index 000000000..7ba1e66f1 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/AnnotationLimit.java @@ -0,0 +1,38 @@ +package cn.edu.tsinghua.iginx.rest.query; + +public class AnnotationLimit +{ + private String tag = ".*"; + private String text = ".*"; + private String title = ".*"; + + public void setTag(String tag) + { + this.tag = tag; + } + + public void setText(String text) + { + this.text = text; + } + + public void setTitle(String title) + { + this.title = title; + } + + public String getTag() + { + return tag; + } + + public String getText() + { + return text; + } + + public String getTitle() + { + return title; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryMetric.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryMetric.java index 59b75cb92..be93af362 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryMetric.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryMetric.java @@ -31,6 +31,7 @@ public class QueryMetric { private Map> tags = new TreeMap(); private List aggregators = new ArrayList<>(); private Boolean annotation = false; + private AnnotationLimit annotationLimit; public String getName() { return name; @@ -85,4 +86,14 @@ public Boolean getAnnotation() { return annotation; } + + public AnnotationLimit getAnnotationLimit() + { + return annotationLimit; + } + + public void setAnnotationLimit(AnnotationLimit annotationLimit) + { + this.annotationLimit = annotationLimit; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java index c453074dd..e2118345a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java @@ -85,11 +85,11 @@ public Query parseQueryMetric(String json) throws Exception { return ret; } - public Query parseAnnotationQueryMetric(String json) throws Exception { + public Query parseAnnotationQueryMetric(String json, boolean isGrafana) throws Exception { Query ret; try { JsonNode node = mapper.readTree(json); - ret = getAnnotationQuery(node); + ret = getAnnotationQuery(node, isGrafana); } catch (Exception e) { LOGGER.error("Error occurred during parsing query ", e); throw e; @@ -283,48 +283,180 @@ else if (start_absolute != null && end_absolute != null) { return ret; } - private Query getAnnotationQuery(JsonNode node) throws JsonProcessingException + private Query getAnnotationQuery(JsonNode node, boolean isGrafana) throws JsonProcessingException { Query ret = new Query(); - JsonNode range = node.get("range"); - if (range == null) - return null; - JsonNode start_absolute = range.get("from"); - JsonNode end_absolute = range.get("to"); - if (start_absolute == null || end_absolute == null) - return null; - else + if (isGrafana) { - Long start = dealDateFormat(start_absolute.asText()); - Long end = dealDateFormat(end_absolute.asText()); - ret.setStartAbsolute(start); - ret.setEndAbsolute(end); - } + JsonNode range = node.get("range"); + if (range == null) + return null; + JsonNode start_absolute = range.get("from"); + JsonNode end_absolute = range.get("to"); + if (start_absolute == null || end_absolute == null) + return null; + else + { + Long start = dealDateFormat(start_absolute.asText()); + Long end = dealDateFormat(end_absolute.asText()); + ret.setStartAbsolute(start); + ret.setEndAbsolute(end); + } - JsonNode metric = node.get("annotation"); - if (metric == null) - return null; - QueryMetric ins = new QueryMetric(); - JsonNode name = metric.get("name"); - if (name != null) - ins.setName(name.asText()); - JsonNode tags = metric.get("query"); - if (tags != null) + JsonNode metric = node.get("annotation"); + if (metric == null) + return null; + QueryMetric ins = new QueryMetric(); + JsonNode name = metric.get("name"); + if (name != null) + ins.setName(name.asText()); + JsonNode tags = metric.get("query"); + if (tags != null) + { + if (tags.get("tags") == null) + tags = mapper.readTree(tags.asText()); + tags = tags.get("tags"); + Iterator fieldNames = tags.fieldNames(); + while (fieldNames.hasNext()) + { + String key = fieldNames.next(); + JsonNode valuenode = tags.get(key); + ins.addTag(key, valuenode.asText()); + } + } + ins.setAnnotation(true); + ret.addQueryMetrics(ins); + } + else { - if (tags.get("tags") == null) - tags = mapper.readTree(tags.asText()); - tags = tags.get("tags"); - Iterator fieldNames = tags.fieldNames(); - while (fieldNames.hasNext()) + JsonNode start_absolute = node.get("start_absolute"); + JsonNode end_absolute = node.get("end_absolute"); + Long now = System.currentTimeMillis(); + if (start_absolute == null && end_absolute == null) { - String key = fieldNames.next(); - JsonNode valuenode = tags.get(key); - ins.addTag(key, valuenode.asText()); + ret.setStartAbsolute(0l); + ret.setEndAbsolute(now); + } + else if (start_absolute != null && end_absolute != null) { + ret.setStartAbsolute(start_absolute.asLong()); + ret.setEndAbsolute(end_absolute.asLong()); + } else if (start_absolute != null) { + ret.setStartAbsolute(start_absolute.asLong()); + JsonNode end_relative = node.get("end_relative"); + if (end_relative == null) + ret.setEndAbsolute(now); + else { + JsonNode value = end_relative.get("value"); + if (value == null) return null; + long v = value.asLong(); + JsonNode unit = end_relative.get("unit"); + if (unit == null) return null; + String u = unit.asText(); + switch (u) { + case "millis": + ret.setEndAbsolute(now - v * 1L); + break; + case "seconds": + ret.setEndAbsolute(now - v * 1000L); + break; + case "minutes": + ret.setEndAbsolute(now - v * 60000L); + break; + case "hours": + ret.setEndAbsolute(now - v * 3600000L); + break; + case "days": + ret.setEndAbsolute(now - v * 86400000L); + break; + case "weeks": + ret.setEndAbsolute(now - v * 604800000L); + break; + case "months": + ret.setEndAbsolute(now - v * 2419200000L); + break; + case "years": + ret.setEndAbsolute(now - v * 29030400000L); + break; + default: + ret.setEndAbsolute(now); + break; + } + } + } else { + ret.setEndAbsolute(end_absolute.asLong()); + JsonNode start_relative = node.get("start_relative"); + if (start_relative == null) + ret.setStartAbsolute(0l); + else { + JsonNode value = start_relative.get("value"); + if (value == null) return null; + long v = value.asLong(); + JsonNode unit = start_relative.get("unit"); + if (unit == null) return null; + String u = value.asText(); + switch (u) { + case "millis": + ret.setStartAbsolute(now - v * 1L); + break; + case "seconds": + ret.setStartAbsolute(now - v * 1000L); + break; + case "minutes": + ret.setStartAbsolute(now - v * 60000L); + break; + case "hours": + ret.setStartAbsolute(now - v * 3600000L); + break; + case "days": + ret.setStartAbsolute(now - v * 86400000L); + break; + case "weeks": + ret.setStartAbsolute(now - v * 604800000L); + break; + case "months": + ret.setStartAbsolute(now - v * 2419200000L); + break; + case "years": + ret.setStartAbsolute(now - v * 29030400000L); + break; + default: + ret.setStartAbsolute(now); + break; + } + } + } + JsonNode metrics = node.get("metrics"); + if (metrics != null && metrics.isArray()) { + for (JsonNode dpnode : metrics) { + QueryMetric ins = new QueryMetric(); + JsonNode name = dpnode.get("name"); + if (name != null) + ins.setName(name.asText()); + JsonNode tags = dpnode.get("tags"); + if (tags != null) { + Iterator fieldNames = tags.fieldNames(); + Iterator elements = tags.elements(); + while (elements.hasNext() && fieldNames.hasNext()) { + String key = fieldNames.next(); + for (JsonNode valuenode : elements.next()) + ins.addTag(key, valuenode.asText()); + } + } + AnnotationLimit annotationLimit = new AnnotationLimit(); + JsonNode category = dpnode.get("category"); + if (category != null) + annotationLimit.setTag(category.asText()); + JsonNode text = dpnode.get("text"); + if (text != null) + annotationLimit.setText(text.asText()); + JsonNode description = dpnode.get("description"); + if (description != null) + annotationLimit.setTitle(description.asText()); + ins.setAnnotationLimit(annotationLimit); + ret.addQueryMetrics(ins); + } } } - ins.setAnnotation(true); - ret.addQueryMetrics(ins); - return ret; } @@ -548,10 +680,10 @@ public String parseResultToJson(QueryResult result, boolean isDelete) { return ret.toString(); } - public String parseResultToGrafanaAnnotationJson(QueryResult result) + public String parseResultToAnnotationJson(QueryResult result, boolean isGrafana) { StringBuilder ret = new StringBuilder("["); - ret.append(result.toAnnotationResultString()); + ret.append(result.toAnnotationResultString(isGrafana)); ret.append("]"); return ret.toString(); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java index 619012afe..79138f51f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java @@ -109,49 +109,83 @@ public String toResultString(int num) { - /* -String entity = "[{\"text\": \"text shown in body\",\"title\": \"Annotation Title\",\"isRegion\": true,\"time\": \"1400000013000\",\"timeEnd\": \"1400000015000\",\"tags\": [\"tag1\"]}]"; -*/ - - public String toAnnotationResultString() { + public String toAnnotationResultString(boolean isGrafana) { StringBuilder ret = new StringBuilder(""); List values = new ArrayList<>(); - int siz = queryResultDatasets.get(0).getValues().size(); - for (int i = 0; i < siz; i++) - values.add(new Annotation(new String((byte[]) queryResultDatasets.get(0).getValues().get(i)), - queryResultDatasets.get(0).getTimestamps().get(i))); + int oldsiz = queryResultDatasets.get(0).getValues().size(), siz = 0; + for (int i = 0; i < oldsiz; i++) + { + Annotation ins = new Annotation(new String((byte[]) queryResultDatasets.get(0).getValues().get(i)), queryResultDatasets.get(0).getTimestamps().get(i)); + if (isGrafana || ins.match(queryMetrics.get(0).getAnnotationLimit())) + { + values.add(ins); + siz++; + } + } int now = 0; if (siz == 0) return "{}"; - for (int i = 1; i < siz; i++) - if (!values.get(i).isEqual(values.get(i-1))) + if (isGrafana) { + for (int i = 1; i < siz; i++) + if (!values.get(i).isEqual(values.get(i - 1))) + { + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(i - 1).text)); + ret.append(String.format("\"title\": \"%s\",", values.get(i - 1).title)); + ret.append("\"isRegion\": true,"); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(i - 1).timestamp)); + ret.append("\"tags\": ["); + for (String tag : values.get(i - 1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]},"); + now = i; + } ret.append("{"); - ret.append(String.format("\"text\": \"%s\",", values.get(i-1).text)); - ret.append(String.format("\"title\": \"%s\",", values.get(i-1).title)); + ret.append(String.format("\"text\": \"%s\",", values.get(siz - 1).text)); + ret.append(String.format("\"title\": \"%s\",", values.get(siz - 1).title)); ret.append("\"isRegion\": true,"); ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); - ret.append(String.format("\"timeEnd\": \"%d\",", values.get(i-1).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(siz - 1).timestamp)); ret.append("\"tags\": ["); - for (String tag : values.get(i-1).tags) + for (String tag : values.get(siz - 1).tags) ret.append(String.format("\"%s\",", tag)); if (ret.charAt(ret.length() - 1) == ',') ret.deleteCharAt(ret.length() - 1); - ret.append("]},"); - now = i; + ret.append("]}"); + } + else { + for (int i = 1; i < siz; i++) + if (!values.get(i).isEqual(values.get(i - 1))) + { + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(i - 1).text)); + ret.append(String.format("\"description\": \"%s\",", values.get(i - 1).title)); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(i - 1).timestamp)); + ret.append("\"category\": ["); + for (String tag : values.get(i - 1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]},"); + now = i; + } + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(siz - 1).text)); + ret.append(String.format("\"description\": \"%s\",", values.get(siz - 1).title)); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(siz - 1).timestamp)); + ret.append("\"category\": ["); + for (String tag : values.get(siz - 1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]}"); } - ret.append("{"); - ret.append(String.format("\"text\": \"%s\",", values.get(siz-1).text)); - ret.append(String.format("\"title\": \"%s\",", values.get(siz-1).title)); - ret.append("\"isRegion\": true,"); - ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); - ret.append(String.format("\"timeEnd\": \"%d\",", values.get(siz-1).timestamp)); - ret.append("\"tags\": ["); - for (String tag : values.get(siz-1).tags) - ret.append(String.format("\"%s\",", tag)); - if (ret.charAt(ret.length() - 1) == ',') - ret.deleteCharAt(ret.length() - 1); - ret.append("]}"); return ret.toString(); } From 380d3468b2df8b2bf5d5179a110ed2cba4762735 Mon Sep 17 00:00:00 2001 From: humanfy Date: Fri, 4 Jun 2021 07:50:19 +0800 Subject: [PATCH 8/8] fix bug --- .../tsinghua/iginx/rest/query/Annotation.java | 2 +- .../iginx/rest/query/QueryParser.java | 34 ++++-- .../iginx/rest/query/QueryResult.java | 112 ++++++++++-------- 3 files changed, 88 insertions(+), 60 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java index 2991b7f4d..530b3c5f4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/Annotation.java @@ -67,7 +67,7 @@ public boolean isEqual(Annotation p) return true; } - boolean match(AnnotationLimit annotationLimit) + boolean match(AnnotationLimit annotationLimit) throws Exception { if (!Pattern.matches(annotationLimit.getText(), text)) return false; if (!Pattern.matches(annotationLimit.getTitle(), title)) return false; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java index e2118345a..e1643a6e9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java @@ -310,20 +310,35 @@ private Query getAnnotationQuery(JsonNode node, boolean isGrafana) throws JsonPr JsonNode name = metric.get("name"); if (name != null) ins.setName(name.asText()); - JsonNode tags = metric.get("query"); + JsonNode query = metric.get("query"); + if (query.get("tags") == null) + query = mapper.readTree(query.asText()); + JsonNode tags = query.get("tags"); if (tags != null) { - if (tags.get("tags") == null) - tags = mapper.readTree(tags.asText()); tags = tags.get("tags"); - Iterator fieldNames = tags.fieldNames(); - while (fieldNames.hasNext()) + if (tags != null) { - String key = fieldNames.next(); - JsonNode valuenode = tags.get(key); - ins.addTag(key, valuenode.asText()); + Iterator fieldNames = tags.fieldNames(); + while (fieldNames.hasNext()) + { + String key = fieldNames.next(); + JsonNode valuenode = tags.get(key); + ins.addTag(key, valuenode.asText()); + } } } + AnnotationLimit annotationLimit = new AnnotationLimit(); + JsonNode category = query.get("category"); + if (category != null) + annotationLimit.setTag(category.asText()); + JsonNode text = query.get("text"); + if (text != null) + annotationLimit.setText(text.asText()); + JsonNode description = query.get("description"); + if (description != null) + annotationLimit.setTitle(description.asText()); + ins.setAnnotationLimit(annotationLimit); ins.setAnnotation(true); ret.addQueryMetrics(ins); } @@ -453,6 +468,7 @@ else if (start_absolute != null && end_absolute != null) { if (description != null) annotationLimit.setTitle(description.asText()); ins.setAnnotationLimit(annotationLimit); + ins.setAnnotation(true); ret.addQueryMetrics(ins); } } @@ -680,7 +696,7 @@ public String parseResultToJson(QueryResult result, boolean isDelete) { return ret.toString(); } - public String parseResultToAnnotationJson(QueryResult result, boolean isGrafana) + public String parseResultToAnnotationJson(QueryResult result, boolean isGrafana) throws Exception { StringBuilder ret = new StringBuilder("["); ret.append(result.toAnnotationResultString(isGrafana)); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java index 79138f51f..d7ad97046 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryResult.java @@ -109,18 +109,14 @@ public String toResultString(int num) { - public String toAnnotationResultString(boolean isGrafana) { + public String toAnnotationResultString(boolean isGrafana) throws Exception{ StringBuilder ret = new StringBuilder(""); List values = new ArrayList<>(); - int oldsiz = queryResultDatasets.get(0).getValues().size(), siz = 0; - for (int i = 0; i < oldsiz; i++) + int siz = queryResultDatasets.get(0).getValues().size(); + for (int i = 0; i < siz; i++) { Annotation ins = new Annotation(new String((byte[]) queryResultDatasets.get(0).getValues().get(i)), queryResultDatasets.get(0).getTimestamps().get(i)); - if (isGrafana || ins.match(queryMetrics.get(0).getAnnotationLimit())) - { - values.add(ins); - siz++; - } + values.add(ins); } int now = 0; if (siz == 0) @@ -130,61 +126,77 @@ public String toAnnotationResultString(boolean isGrafana) { for (int i = 1; i < siz; i++) if (!values.get(i).isEqual(values.get(i - 1))) { - ret.append("{"); - ret.append(String.format("\"text\": \"%s\",", values.get(i - 1).text)); - ret.append(String.format("\"title\": \"%s\",", values.get(i - 1).title)); - ret.append("\"isRegion\": true,"); - ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); - ret.append(String.format("\"timeEnd\": \"%d\",", values.get(i - 1).timestamp)); - ret.append("\"tags\": ["); - for (String tag : values.get(i - 1).tags) - ret.append(String.format("\"%s\",", tag)); - if (ret.charAt(ret.length() - 1) == ',') - ret.deleteCharAt(ret.length() - 1); - ret.append("]},"); + if (values.get(i-1).match(queryMetrics.get(0).getAnnotationLimit())) + { + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(i - 1).text)); + ret.append(String.format("\"title\": \"%s\",", values.get(i - 1).title)); + ret.append("\"isRegion\": true,"); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(i - 1).timestamp)); + ret.append("\"tags\": ["); + for (String tag : values.get(i - 1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]},"); + } now = i; } - ret.append("{"); - ret.append(String.format("\"text\": \"%s\",", values.get(siz - 1).text)); - ret.append(String.format("\"title\": \"%s\",", values.get(siz - 1).title)); - ret.append("\"isRegion\": true,"); - ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); - ret.append(String.format("\"timeEnd\": \"%d\",", values.get(siz - 1).timestamp)); - ret.append("\"tags\": ["); - for (String tag : values.get(siz - 1).tags) - ret.append(String.format("\"%s\",", tag)); + if (values.get(siz - 1).match(queryMetrics.get(0).getAnnotationLimit())) + { + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(siz - 1).text)); + ret.append(String.format("\"title\": \"%s\",", values.get(siz - 1).title)); + ret.append("\"isRegion\": true,"); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(siz - 1).timestamp)); + ret.append("\"tags\": ["); + for (String tag : values.get(siz - 1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]}"); + } if (ret.charAt(ret.length() - 1) == ',') ret.deleteCharAt(ret.length() - 1); - ret.append("]}"); } else { for (int i = 1; i < siz; i++) if (!values.get(i).isEqual(values.get(i - 1))) { - ret.append("{"); - ret.append(String.format("\"text\": \"%s\",", values.get(i - 1).text)); - ret.append(String.format("\"description\": \"%s\",", values.get(i - 1).title)); - ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); - ret.append(String.format("\"timeEnd\": \"%d\",", values.get(i - 1).timestamp)); - ret.append("\"category\": ["); - for (String tag : values.get(i - 1).tags) - ret.append(String.format("\"%s\",", tag)); - if (ret.charAt(ret.length() - 1) == ',') - ret.deleteCharAt(ret.length() - 1); - ret.append("]},"); + if (values.get(i-1).match(queryMetrics.get(0).getAnnotationLimit())) + { + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(i - 1).text)); + ret.append(String.format("\"description\": \"%s\",", values.get(i - 1).title)); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(i - 1).timestamp)); + ret.append("\"category\": ["); + for (String tag : values.get(i - 1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]},"); + } now = i; } - ret.append("{"); - ret.append(String.format("\"text\": \"%s\",", values.get(siz - 1).text)); - ret.append(String.format("\"description\": \"%s\",", values.get(siz - 1).title)); - ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); - ret.append(String.format("\"timeEnd\": \"%d\",", values.get(siz - 1).timestamp)); - ret.append("\"category\": ["); - for (String tag : values.get(siz - 1).tags) - ret.append(String.format("\"%s\",", tag)); + if (values.get(siz - 1).match(queryMetrics.get(0).getAnnotationLimit())) + { + ret.append("{"); + ret.append(String.format("\"text\": \"%s\",", values.get(siz - 1).text)); + ret.append(String.format("\"description\": \"%s\",", values.get(siz - 1).title)); + ret.append(String.format("\"time\": \"%d\",", values.get(now).timestamp)); + ret.append(String.format("\"timeEnd\": \"%d\",", values.get(siz - 1).timestamp)); + ret.append("\"category\": ["); + for (String tag : values.get(siz - 1).tags) + ret.append(String.format("\"%s\",", tag)); + if (ret.charAt(ret.length() - 1) == ',') + ret.deleteCharAt(ret.length() - 1); + ret.append("]}"); + } if (ret.charAt(ret.length() - 1) == ',') ret.deleteCharAt(ret.length() - 1); - ret.append("]}"); } return ret.toString(); }