diff --git a/CHANGELOG.md b/CHANGELOG.md index 90fa92816b..51b245bfd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.9.2 (April 2016) + +- Bugfix: Fixed examples + ## 0.9.1 (April 2016) - Bugfix: Fixed documentation links diff --git a/examples/policies/fragments/IWebSocket-OCassandra.json b/examples/policies/fragments/IWebSocket-OCassandra.json index da4587d8a7..2796909df2 100644 --- a/examples/policies/fragments/IWebSocket-OCassandra.json +++ b/examples/policies/fragments/IWebSocket-OCassandra.json @@ -1,39 +1,23 @@ { + "storageLevel": "MEMORY_AND_DISK_SER", "name": "Policy-WebSocket-Cassandra", "description": "Input WebSocket, output Cassandra", - "sparkStreamingWindow": 2000, + "sparkStreamingWindow": "2s", + "checkpointPath": "checkpoint", "rawData": { "enabled": "false", "path": "myTestParquetPath" }, - "checkpointPath": "checkpoint", - "fragments": [ - { - "id": "_input_id_", - "name": "websocket", - "fragmentType": "input", - "description": "description", - "shortDescription": "shortDescription", - "icon": "icon", - "element": null - }, - { - "id": "_output_id_", - "name": "Cassandra", - "fragmentType": "output", - "description": "description", - "shortDescription": "shortDescription", - "icon": "icon", - "element": null - } - ], "transformations": [ { - "name": "morphline", "type": "Morphlines", "order": 0, + "inputField": "_attachment_body", "outputFields": [ - {"name":"response"} + { + "name": "response", + "type": "string" + } ], "configuration": { "morphline": { @@ -57,43 +41,106 @@ } }, { - "name": "timestamp", - "order": 1, "type": "DateTime", + "order": 1, + "inputField": "_attachment_body", "outputFields": [ - {"name":"timestamp"} + { + "name": "timestamp", + "type": "datetime" + } ], "configuration": { "inputFormat": "autoGenerated" } } ], + "streamTriggers": [], "cubes": [ { "name": "testCube", - "writer": { - "outputs" : ["Cassandra"] - }, "dimensions": [ { + "name": "response", "field": "response", - "name": "response" + "precision": "identity", + "type": "Default" }, { "name": "minute", "field": "timestamp", + "precision": "1m", "type": "DateTime", - "precision": "minute", "computeLast": "10m" } ], "operators": [ { - "name": "countoperator", "type": "Count", + "name": "count1", "configuration": {} } - ] + ], + "writer": { + "outputs": [ + "cassandra" + ], + "dateType": "timestamp", + "isAutoCalculatedId": false + }, + "triggers": [] } - ] -} \ No newline at end of file + ], + "input": { + "name": "websocket", + "type": "WebSocket", + "configuration": { + "url": "ws://stream.meetup.com/2/rsvps" + } + }, + "outputs": [ + { + "name": "cassandra", + "type": "Cassandra", + "configuration": { + "connectionHost": "ip-172-31-17-8.eu-west-1.compute.internal", + "cluster": "Stratio Cluster", + "keyspace": "sparta" + } + } + ], + "fragments": [ + { + "id": "_input_id_", + "fragmentType": "input", + "name": "websocket", + "description": "websocket input", + "shortDescription": "websocket input", + "element": { + "name": "in-websocket", + "type": "WebSocket", + "configuration": { + "url": "ws://stream.meetup.com/2/rsvps" + } + } + }, + { + "id": "_output_id_", + "fragmentType": "output", + "name": "cassandra", + "description": "Cassandra output", + "shortDescription": "Cassandra output", + "element": { + "name": "out-cassandra", + "type": "Cassandra", + "configuration": { + "connectionHost": "ip-172-31-17-8.eu-west-1.compute.internal", + "cluster": "Stratio Cluster", + "keyspace": "sparta" + } + } + } + ], + "userPluginsJars": [] +} + diff --git a/examples/policies/fragments/IWebSocket-OElasticsearch.json b/examples/policies/fragments/IWebSocket-OElasticsearch.json index 13dd3cb752..6d6627ccf8 100644 --- a/examples/policies/fragments/IWebSocket-OElasticsearch.json +++ b/examples/policies/fragments/IWebSocket-OElasticsearch.json @@ -1,40 +1,23 @@ { - "name": "Policy-WebSocket-ElasticSearch", + "storageLevel": "MEMORY_AND_DISK_SER", + "name": "policy-websocket-elasticsearch", "description": "Input WebSocket, output ElasticSearch", - "sparkStreamingWindow": 2000, + "sparkStreamingWindow": "2s", + "checkpointPath": "/tmp/checkpoint", "rawData": { "enabled": "false", - "path": "myTestParquetPath" + "path": "default" }, - "checkpointPath": "checkpoint", - "fragments": [ - { - "id": "_input_id_", - "name": "WebSocket", - "fragmentType": "input", - "description": "description", - "shortDescription": "shortDescription", - "icon": "icon", - "element": null - }, - { - "id": "_output_id_", - "name": "Elasticsearch", - "fragmentType": "output", - "description": "description", - "shortDescription": "shortDescription", - "icon": "icon", - "element": null - } - ], - "transformations": [ { - "name": "morphline", "type": "Morphlines", "order": 0, + "inputField": "_attachment_body", "outputFields": [ - {"name":"response"} + { + "name": "response", + "type": "string" + } ], "configuration": { "morphline": { @@ -58,45 +41,120 @@ } }, { - "name": "timestamp", - "order": 1, "type": "DateTime", + "order": 1, + "inputField": "_attachment_body", "outputFields": [ - {"name":"timestamp"} + { + "name": "timestamp", + "type": "datetime" + } ], "configuration": { "inputFormat": "autoGenerated" } } ], + "streamTriggers": [], "cubes": [ { - "name": "testCube", - "writer": { - "outputs" : ["Elasticsearch"], - "dateType": "long", - "isAutoCalculatedId": true - }, + "name": "cube1", "dimensions": [ { + "name": "response", "field": "response", - "name": "response" + "precision": "identity", + "type": "Default" }, { "name": "minute", "field": "timestamp", + "precision": "1m", "type": "DateTime", - "precision": "minute", "computeLast": "10m" } ], "operators": [ { - "name": "countoperator", "type": "Count", + "name": "count1", "configuration": {} } - ] + ], + "writer": { + "outputs": [ + "elasticsearch" + ], + "dateType": "timestamp", + "isAutoCalculatedId": false + }, + "triggers": [] + } + ], + "input": { + "name": "websocket", + "type": "WebSocket", + "configuration": { + "url": "ws://stream.meetup.com/2/rsvps" + } + }, + "outputs": [ + { + "name": "elasticsearch", + "type": "ElasticSearch", + "configuration": { + "nodes": [ + { + "node": "localhost", + "tcpPort": "9300", + "httpPort": "9200" + } + ], + "clusterName": "elasticsearch", + "idField": "", + "indexMapping": "sparta" + } + } + ], + "fragments": [ + { + "id": "_input_id_", + "fragmentType": "input", + "name": "websocket", + "description": "websocket input", + "shortDescription": "websocket input", + "element": { + "name": "in-websocket", + "type": "WebSocket", + "configuration": { + "url": "ws://stream.meetup.com/2/rsvps" + } + } + }, + { + "id": "_output_id_", + "fragmentType": "output", + "name": "elasticsearch", + "description": "Elasticsearch is a search server based on Lucene. It provides a distributed, multitenant-capable full-text search engine with a RESTful web interface and schema-free JSON documents.", + "shortDescription": "The Elasticsearch output uses the generic implementation with DataFrames.", + "element": { + "name": "in-ElasticSearch", + "type": "ElasticSearch", + "configuration": { + "nodes": [ + { + "node": "localhost", + "tcpPort": "9300", + "httpPort": "9200" + } + ], + "clusterName": "elasticsearch", + "idField": "", + "indexMapping": "sparta" + } + } } - ] -} \ No newline at end of file + ], + "userPluginsJars": [] +} + diff --git a/examples/policies/fragments/IWebSocket-OMongo.json b/examples/policies/fragments/IWebSocket-OMongo.json index 2df9a3104b..343ebdf63b 100644 --- a/examples/policies/fragments/IWebSocket-OMongo.json +++ b/examples/policies/fragments/IWebSocket-OMongo.json @@ -1,40 +1,23 @@ { - "name": "Policy-WebSocket-MongoDb", + "storageLevel": "MEMORY_AND_DISK_SER", + "name": "policy-websocket-mongodb", "description": "Input WebSocket, output MongoDb", - "sparkStreamingWindow": 2000, + "sparkStreamingWindow": "2s", + "checkpointPath": "checkpoint", "rawData": { "enabled": "false", - "path": "myTestParquetPath" + "path": "default" }, - "checkpointPath": "checkpoint", - "fragments": [ - { - "id": "_input_id_", - "name": "WebSocket", - "fragmentType": "input", - "description": "description", - "shortDescription": "shortDescription", - "icon": "icon", - "element": null - }, - { - "id": "_output_id_", - "name": "MongoDB", - "fragmentType": "output", - "description": "description", - "shortDescription": "shortDescription", - "icon": "icon", - "element": null - } - ], - "transformations": [ { - "name": "morphline", "type": "Morphlines", "order": 0, + "inputField": "_attachment_body", "outputFields": [ - {"name":"response"} + { + "name": "response", + "type": "string" + } ], "configuration": { "morphline": { @@ -58,44 +41,121 @@ } }, { - "name": "timestamp", - "order": 1, "type": "DateTime", + "order": 1, + "inputField": "_attachment_body", "outputFields": [ - {"name":"timestamp"} + { + "name": "timestamp", + "type": "datetime" + } ], "configuration": { "inputFormat": "autoGenerated" } } ], + "streamTriggers": [], "cubes": [ { - "name": "testCube", - "writer": { - "outputs" : ["MongoDB"], - "isAutoCalculatedId": true - }, + "name": "cube1", "dimensions": [ { + "name": "response", "field": "response", - "name": "response" + "precision": "identity", + "type": "Default" }, { "name": "minute", "field": "timestamp", + "precision": "1m", "type": "DateTime", - "precision": "minute", "computeLast": "10m" } ], "operators": [ { - "name": "countoperator", "type": "Count", + "name": "count1", "configuration": {} } - ] + ], + "writer": { + "outputs": [ + "mongodb" + ], + "dateType": "timestamp", + "isAutoCalculatedId": false + }, + "triggers": [] + } + ], + "input": { + "name": "websocket", + "type": "WebSocket", + "configuration": { + "url": "ws://stream.meetup.com/2/rsvps" + } + }, + "outputs": [ + { + "name": "mongodb", + "type": "MongoDb", + "configuration": { + "connectionsPerHost": "5", + "dbName": "sparta", + "threadsAllowedToBlock": "10", + "language": "", + "hosts": [ + { + "host": "localhost", + "port": "27017" + } + ], + "retrySleep": "1000" + } + } + ], + "fragments": [ + { + "id": "_input_id_", + "fragmentType": "input", + "name": "websocket", + "description": "websocket input", + "shortDescription": "websocket input", + "element": { + "name": "in-websocket", + "type": "WebSocket", + "configuration": { + "url": "ws://stream.meetup.com/2/rsvps" + } + } + }, + { + "id": "_output_id_", + "fragmentType": "output", + "name": "mongodb", + "description": "MongoDB is an open-source document database that provides high performance, high availability, and automatic scaling.", + "shortDescription": "MongoDB is an open-source document database, and the leading NoSQL database.", + "element": { + "name": "in-MongoDb", + "type": "MongoDb", + "configuration": { + "connectionsPerHost": "5", + "dbName": "sparta", + "threadsAllowedToBlock": "10", + "language": "", + "hosts": [ + { + "host": "localhost", + "port": "27017" + } + ], + "retrySleep": "1000" + } + } } - ] -} \ No newline at end of file + ], + "userPluginsJars": [] +}