From 77031c4a23124ad7785ec09ac4dab4f31aa3ab1a Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 2 Jun 2016 11:36:41 +0200
Subject: [PATCH 01/58] Created dataset to run the tests.

---
 dbod/tests/db_test.sql | 151 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 151 insertions(+)
 create mode 100644 dbod/tests/db_test.sql

diff --git a/dbod/tests/db_test.sql b/dbod/tests/db_test.sql
new file mode 100644
index 0000000..9f60591
--- /dev/null
+++ b/dbod/tests/db_test.sql
@@ -0,0 +1,151 @@
+-- Copyright (C) 2015, CERN
+-- This software is distributed under the terms of the GNU General Public
+-- Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+-- In applying this license, CERN does not waive the privileges and immunities
+-- granted to it by virtue of its status as Intergovernmental Organization
+-- or submit itself to any jurisdiction.
+
+------------------------------------------------
+-- Create the structure for the test database --
+------------------------------------------------
+
+-- DOD_COMMAND_DEFINITION
+CREATE TABLE dod_command_definition (
+    command_name varchar(64) NOT NULL,
+    type varchar(64) NOT NULL,
+    exec varchar(2048),
+    category varchar(20),
+    PRIMARY KEY (command_name, type, category)
+);
+
+-- DOD_COMMAND_PARAMS
+CREATE TABLE dod_command_params (
+    username varchar(32) NOT NULL,
+    db_name varchar(128) NOT NULL,
+    command_name varchar(64) NOT NULL,
+    type varchar(64) NOT NULL,
+    creation_date date NOT NULL,
+    name varchar(64) NOT NULL,
+    value text,
+    category varchar(20),
+    PRIMARY KEY (db_name)
+);
+
+-- DOD_INSTANCE_CHANGES
+CREATE TABLE dod_instance_changes (
+    username varchar(32) NOT NULL,
+    db_name varchar(128) NOT NULL,
+    attribute varchar(32) NOT NULL,
+    change_date date NOT NULL,
+    requester varchar(32) NOT NULL,
+    old_value varchar(1024),
+    new_value varchar(1024),
+    PRIMARY KEY (db_name)
+);
+
+-- DOD_INSTANCES
+CREATE TABLE dod_instances (
+    username varchar(32) NOT NULL,
+    db_name varchar(128) NOT NULL,
+    e_group varchar(256),
+    category varchar(32) NOT NULL,
+    creation_date date NOT NULL,
+    expiry_date date,
+    db_type varchar(32) NOT NULL,
+    db_size int NOT NULL,
+    no_connections int,
+    project varchar(128),
+    description varchar(1024),
+    version varchar(128),
+    master varchar(32),
+    slave varchar(32),
+    host varchar(128),
+    state varchar(32),
+    status varchar(32),
+    id int,
+    PRIMARY KEY (id)
+);
+
+-- DOD_JOBS
+CREATE TABLE dod_jobs (
+    username varchar(32) NOT NULL,
+    db_name varchar(128) NOT NULL,
+    command_name varchar(64) NOT NULL,
+    type varchar(64) NOT NULL,
+    creation_date date NOT NULL,
+    completion_date date,
+    requester varchar(32) NOT NULL,
+    admin_action int NOT NULL,
+    state varchar(32) NOT NULL,
+    log text,
+    result varchar(2048),
+    email_sent date,
+    category varchar(20),
+    PRIMARY KEY (username, db_name, command_name, type, creation_date)
+);
+
+-- DOD_UPGRADES
+CREATE TABLE dod_upgrades (
+    db_type varchar(32) NOT NULL,
+    category varchar(32) NOT NULL,
+    version_from varchar(128) NOT NULL,
+    version_to varchar(128) NOT NULL,
+    PRIMARY KEY (db_type, category, version_from)
+);
+
+-- VOLUME
+CREATE TABLE volume (
+    id serial,
+    instance_id integer NOT NULL,
+    file_mode char(4) NOT NULL,
+    owner varchar(32) NOT NULL,
+    vgroup varchar(32) NOT NULL,
+    server varchar(63) NOT NULL,
+    mount_options varchar(256) NOT NULL,
+    mounting_path varchar(256) NOT NULL
+);
+
+-- HOST
+CREATE TABLE host (
+    id serial,
+    name varchar(63) NOT NULL,
+    memory integer NOT NULL
+);
+
+-- ATTRIBUTE
+CREATE TABLE attribute (
+    id serial,
+    instance_id integer NOT NULL,
+    name varchar(32) NOT NULL,
+    value varchar(250) NOT NULL
+);
+
+-- Insert test data for instances
+INSERT INTO dod_instances (username, db_name, e_group, category, creation_date, expiry_date, db_type, db_size, no_connections, project, description, version, master, slave, host, state, status, id)
+VALUES ('user01', 'dbod01', 'testgroupA', 'TEST', now(), NULL, 'MYSQL', 100, 100, 'API', 'Test instance 1', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1, 1),
+       ('user01', 'dbod02', 'testgroupB', 'PROD', now(), NULL, 'PG', 50, 500, 'API', 'Test instance 2', '9.4.4', NULL, NULL, 'host03', 'RUNNING', 1, 2),
+       ('user02', 'dbod03', 'testgroupB', 'TEST', now(), NULL, 'MYSQL', 100, 200, 'WEB', 'Expired instance 1', '5.5', NULL, NULL, 'host01', 'RUNNING', 0, 3),
+       ('user03', 'dbod04', 'testgroupA', 'PROD', now(), NULL, 'PG', 250, 10, 'LCC', 'Test instance 3', '9.4.5', NULL, NULL, 'host01', 'RUNNING', 1, 4),
+       ('user04', 'dbod05', 'testgroupC', 'TEST', now(), NULL, 'MYSQL', 300, 200, 'WEB', 'Test instance 4', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1, 5);
+       
+-- Insert test data for volumes
+INSERT INTO volume (instance_id, file_mode, owner, vgroup, server, mount_options, mounting_path)
+VALUES (1, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data1'),
+       (2, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data2'),
+       (4, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard,tcp', '/MNT/data4'),
+       (5, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data5');
+
+-- Insert test data for attributes
+INSERT INTO attribute (instance_id, name, value)
+VALUES (1, 'port', '5501'),
+       (2, 'port', '6603'),
+       (3, 'port', '5510'),
+       (4, 'port', '6601'),
+       (5, 'port', '5500');
+        
+-- Insert test data for hosts
+INSERT INTO host (name, memory)
+VALUES ('host01', 12),
+       ('host02', 24),
+       ('host03', 64),
+       ('host04', 256);

From 1a484ac77eabadd158d0ad8151cb7d1971ce5509 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 2 Jun 2016 16:55:56 +0200
Subject: [PATCH 02/58] Added postgREST and SQL imports to Travis.

---
 .travis.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 73435c3..2113a6b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,9 +12,15 @@ before_install:
     - sudo mkdir /var/log/dbod
     - sudo chown travis /var/log/dbod
     - sudo cp static/api.cfg /etc/dbod
+    - wget https://github.com/begriffs/postgrest/releases/download/v0.3.1.1/postgrest-0.3.1.1-ubuntu.tar.xz
+    - tar xf postgrest-0.3.1.1-ubuntu.tar.xz
 install:
     - pip install -r requirements.txt
     - pip install coveralls
+before_script:
+    - psql -c 'CREATE DATABASE dbod;' -U postgres
+    - psql -f dbod/tests/db_test.sql -U postgres
+    - postgrest postgres://postgres@localhost/dbod -a postgres
 script:
     - nosetests --with-coverage --cover-package=dbod --cover-html
 after_success:

From de2fc9d82f937dd029ea4e7ce21a0a860b51cbfc Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 2 Jun 2016 17:59:36 +0200
Subject: [PATCH 03/58] Start postgREST in background.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 2113a6b..fd6107e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,7 @@ install:
 before_script:
     - psql -c 'CREATE DATABASE dbod;' -U postgres
     - psql -f dbod/tests/db_test.sql -U postgres
-    - postgrest postgres://postgres@localhost/dbod -a postgres
+    - ./postgrest postgres://postgres@localhost/dbod -a postgres &
 script:
     - nosetests --with-coverage --cover-package=dbod --cover-html
 after_success:

From 3901b7bcbaf15161b31ebdaa467fbb550c44ed5e Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 3 Jun 2016 21:06:41 +0200
Subject: [PATCH 04/58] Very basic first test.

---
 dbod/tests/db_tests.py | 7 ++++++-
 requirements.txt       | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/dbod/tests/db_tests.py b/dbod/tests/db_tests.py
index 4d3b452..52fc071 100644
--- a/dbod/tests/db_tests.py
+++ b/dbod/tests/db_tests.py
@@ -13,12 +13,17 @@
 import json
 import logging
 import sys
+import requests
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 
-from dbod.api.dbops import *
+#from dbod.api.dbops import *
 
 def empty():
     assert(True)
+    
+def test_connection():
+    response = requests.get("http://localhost:3000")
+    assert(response.status_code == 200)
 
 if __name__ == "__main__":
     pass
diff --git a/requirements.txt b/requirements.txt
index ca4de0f..7889b8b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 configparser
 nose
 psycopg2
-tornado
+tornado==4.2
 virtualenv
 requests

From 484fdd3c52d5d6877cf50a1455187de51e7a6b9f Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Mon, 6 Jun 2016 15:57:03 +0200
Subject: [PATCH 05/58] Added functions to the test dataset. Added schema to
 all the objects.

---
 dbod/tests/db_test.sql | 113 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 100 insertions(+), 13 deletions(-)

diff --git a/dbod/tests/db_test.sql b/dbod/tests/db_test.sql
index 9f60591..a6bb7ea 100644
--- a/dbod/tests/db_test.sql
+++ b/dbod/tests/db_test.sql
@@ -10,7 +10,7 @@
 ------------------------------------------------
 
 -- DOD_COMMAND_DEFINITION
-CREATE TABLE dod_command_definition (
+CREATE TABLE public.dod_command_definition (
     command_name varchar(64) NOT NULL,
     type varchar(64) NOT NULL,
     exec varchar(2048),
@@ -19,7 +19,7 @@ CREATE TABLE dod_command_definition (
 );
 
 -- DOD_COMMAND_PARAMS
-CREATE TABLE dod_command_params (
+CREATE TABLE public.dod_command_params (
     username varchar(32) NOT NULL,
     db_name varchar(128) NOT NULL,
     command_name varchar(64) NOT NULL,
@@ -32,7 +32,7 @@ CREATE TABLE dod_command_params (
 );
 
 -- DOD_INSTANCE_CHANGES
-CREATE TABLE dod_instance_changes (
+CREATE TABLE public.dod_instance_changes (
     username varchar(32) NOT NULL,
     db_name varchar(128) NOT NULL,
     attribute varchar(32) NOT NULL,
@@ -44,7 +44,7 @@ CREATE TABLE dod_instance_changes (
 );
 
 -- DOD_INSTANCES
-CREATE TABLE dod_instances (
+CREATE TABLE public.dod_instances (
     username varchar(32) NOT NULL,
     db_name varchar(128) NOT NULL,
     e_group varchar(256),
@@ -67,7 +67,7 @@ CREATE TABLE dod_instances (
 );
 
 -- DOD_JOBS
-CREATE TABLE dod_jobs (
+CREATE TABLE public.dod_jobs (
     username varchar(32) NOT NULL,
     db_name varchar(128) NOT NULL,
     command_name varchar(64) NOT NULL,
@@ -85,7 +85,7 @@ CREATE TABLE dod_jobs (
 );
 
 -- DOD_UPGRADES
-CREATE TABLE dod_upgrades (
+CREATE TABLE public.dod_upgrades (
     db_type varchar(32) NOT NULL,
     category varchar(32) NOT NULL,
     version_from varchar(128) NOT NULL,
@@ -94,7 +94,7 @@ CREATE TABLE dod_upgrades (
 );
 
 -- VOLUME
-CREATE TABLE volume (
+CREATE TABLE public.volume (
     id serial,
     instance_id integer NOT NULL,
     file_mode char(4) NOT NULL,
@@ -106,22 +106,31 @@ CREATE TABLE volume (
 );
 
 -- HOST
-CREATE TABLE host (
+CREATE TABLE public.host (
     id serial,
     name varchar(63) NOT NULL,
     memory integer NOT NULL
 );
 
 -- ATTRIBUTE
-CREATE TABLE attribute (
+CREATE TABLE public.attribute (
     id serial,
     instance_id integer NOT NULL,
     name varchar(32) NOT NULL,
     value varchar(250) NOT NULL
 );
 
+-- FUNCTIONAL ALIASES
+CREATE TABLE public.functional_aliases
+(
+    dns_name character varying(256) NOT NULL,
+    db_name character varying(8),
+    alias character varying(256),
+    CONSTRAINT functional_aliases_pkey PRIMARY KEY (dns_name)
+);
+
 -- Insert test data for instances
-INSERT INTO dod_instances (username, db_name, e_group, category, creation_date, expiry_date, db_type, db_size, no_connections, project, description, version, master, slave, host, state, status, id)
+INSERT INTO public.dod_instances (username, db_name, e_group, category, creation_date, expiry_date, db_type, db_size, no_connections, project, description, version, master, slave, host, state, status, id)
 VALUES ('user01', 'dbod01', 'testgroupA', 'TEST', now(), NULL, 'MYSQL', 100, 100, 'API', 'Test instance 1', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1, 1),
        ('user01', 'dbod02', 'testgroupB', 'PROD', now(), NULL, 'PG', 50, 500, 'API', 'Test instance 2', '9.4.4', NULL, NULL, 'host03', 'RUNNING', 1, 2),
        ('user02', 'dbod03', 'testgroupB', 'TEST', now(), NULL, 'MYSQL', 100, 200, 'WEB', 'Expired instance 1', '5.5', NULL, NULL, 'host01', 'RUNNING', 0, 3),
@@ -129,14 +138,14 @@ VALUES ('user01', 'dbod01', 'testgroupA', 'TEST', now(), NULL, 'MYSQL', 100, 100
        ('user04', 'dbod05', 'testgroupC', 'TEST', now(), NULL, 'MYSQL', 300, 200, 'WEB', 'Test instance 4', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1, 5);
        
 -- Insert test data for volumes
-INSERT INTO volume (instance_id, file_mode, owner, vgroup, server, mount_options, mounting_path)
+INSERT INTO public.volume (instance_id, file_mode, owner, vgroup, server, mount_options, mounting_path)
 VALUES (1, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data1'),
        (2, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data2'),
        (4, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard,tcp', '/MNT/data4'),
        (5, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data5');
 
 -- Insert test data for attributes
-INSERT INTO attribute (instance_id, name, value)
+INSERT INTO public.attribute (instance_id, name, value)
 VALUES (1, 'port', '5501'),
        (2, 'port', '6603'),
        (3, 'port', '5510'),
@@ -144,8 +153,86 @@ VALUES (1, 'port', '5501'),
        (5, 'port', '5500');
         
 -- Insert test data for hosts
-INSERT INTO host (name, memory)
+INSERT INTO public.host (name, memory)
 VALUES ('host01', 12),
        ('host02', 24),
        ('host03', 64),
        ('host04', 256);
+       
+-- Job stats view
+CREATE OR REPLACE VIEW api.job_stats AS 
+SELECT db_name, command_name, COUNT(*) as COUNT, ROUND(AVG(completion_date - creation_date) * 24*60*60) AS mean_duration
+FROM dod_jobs GROUP BY command_name, db_name;
+
+-- Command stats view
+CREATE OR REPLACE VIEW api.command_stats AS
+SELECT command_name, COUNT(*) AS COUNT, ROUND(AVG(completion_date - creation_date) * 24*60*60) AS mean_duration
+FROM dod_jobs GROUP BY command_name;
+       
+-- Get hosts function
+CREATE OR REPLACE FUNCTION get_hosts(host_ids INTEGER[])
+RETURNS VARCHAR[] AS $$
+DECLARE
+  hosts VARCHAR := '';
+BEGIN
+  SELECT ARRAY (SELECT name FROM host WHERE id = ANY(host_ids)) INTO hosts;
+  RETURN hosts;
+END
+$$ LANGUAGE plpgsql;
+
+-- Get volumes function
+CREATE OR REPLACE FUNCTION get_volumes(pid INTEGER)
+RETURNS JSON[] AS $$
+DECLARE
+  volumes JSON[];
+BEGIN
+  SELECT ARRAY (SELECT row_to_json(t) FROM (SELECT * FROM public.volume WHERE instance_id = pid) t) INTO volumes;
+  return volumes;
+END
+$$ LANGUAGE plpgsql;
+
+-- Get port function
+CREATE OR REPLACE FUNCTION get_attribute(name VARCHAR, instance_id INTEGER)
+RETURNS VARCHAR AS $$
+DECLARE
+  res VARCHAR;
+BEGIN
+  SELECT value FROM public.attribute A WHERE A.instance_id = instance_id AND A.name = name INTO res;
+  return res;
+END
+$$ LANGUAGE plpgsql;
+
+
+-- Get directories function
+CREATE OR REPLACE FUNCTION get_directories(inst_name VARCHAR, type VARCHAR, version VARCHAR, port VARCHAR)
+RETURNS TABLE (basedir VARCHAR, bindir VARCHAR, datadir VARCHAR, logdir VARCHAR, socket VARCHAR) AS $$
+BEGIN
+  IF type = 'MYSQL' THEN
+    RETURN QUERY SELECT 
+      ('/usr/local/mysql/mysql-' || version)::VARCHAR basedir, 
+      ('/usr/local/mysql/mysql-' || version || '/bin')::VARCHAR bindir, 
+      ('/ORA/dbs03/' || upper(inst_name) || '/mysql')::VARCHAR datadir, 
+      ('/ORA/dbs02/' || upper(inst_name) || '/mysql')::VARCHAR logdir, 
+      ('/var/lib/mysql/mysql.sock.' || lower(inst_name) || '.' || port)::VARCHAR socket;
+  ELSIF type = 'PG' THEN
+    RETURN QUERY SELECT 
+      ('/usr/local/pgsql/pgsql-' || version)::VARCHAR basedir, 
+      ('/usr/local/mysql/mysql-' || version || '/bin')::VARCHAR bindir, 
+      ('/ORA/dbs03/' || upper(inst_name) || '/data')::VARCHAR datadir, 
+      ('/ORA/dbs02/' || upper(inst_name) || '/pg_xlog')::VARCHAR logdir, 
+      ('/var/lib/pgsql/')::VARCHAR socket;
+  END IF;
+END
+$$ LANGUAGE plpgsql;
+       
+CREATE VIEW api.instance AS
+SELECT * FROM public.dod_instances;
+
+CREATE VIEW api.volume AS
+SELECT * FROM public.volume;
+
+CREATE VIEW api.attribute AS
+SELECT * FROM public.attribute;
+
+CREATE VIEW api.host AS
+SELECT * FROM public.host;

From 946bb2470d7933609cf355b60bd217620de7bf3f Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 7 Jun 2016 09:40:11 +0200
Subject: [PATCH 06/58] Added views and some problems solved with the SQL
 Script.

---
 dbod/tests/db_test.sql | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/dbod/tests/db_test.sql b/dbod/tests/db_test.sql
index a6bb7ea..e64f6ce 100644
--- a/dbod/tests/db_test.sql
+++ b/dbod/tests/db_test.sql
@@ -159,6 +159,9 @@ VALUES ('host01', 12),
        ('host03', 64),
        ('host04', 256);
        
+-- Schema API
+CREATE SCHEMA api;
+       
 -- Job stats view
 CREATE OR REPLACE VIEW api.job_stats AS 
 SELECT db_name, command_name, COUNT(*) as COUNT, ROUND(AVG(completion_date - creation_date) * 24*60*60) AS mean_duration
@@ -192,12 +195,12 @@ END
 $$ LANGUAGE plpgsql;
 
 -- Get port function
-CREATE OR REPLACE FUNCTION get_attribute(name VARCHAR, instance_id INTEGER)
+CREATE OR REPLACE FUNCTION get_attribute(attr_name VARCHAR, inst_id INTEGER)
 RETURNS VARCHAR AS $$
 DECLARE
   res VARCHAR;
 BEGIN
-  SELECT value FROM public.attribute A WHERE A.instance_id = instance_id AND A.name = name INTO res;
+  SELECT value FROM public.attribute A WHERE A.instance_id = inst_id AND A.name = attr_name INTO res;
   return res;
 END
 $$ LANGUAGE plpgsql;
@@ -236,3 +239,26 @@ SELECT * FROM public.attribute;
 
 CREATE VIEW api.host AS
 SELECT * FROM public.host;
+
+-- Metadata View
+CREATE OR REPLACE VIEW api.metadata AS
+SELECT id, username, db_name, category, db_type, version, host, get_attribute('port', id) port, get_volumes volumes, d.*
+FROM dod_instances, get_volumes(id), get_directories(db_name, db_type, version, get_attribute('port', id)) d;
+
+-- Rundeck instances View
+CREATE OR REPLACE VIEW api.rundeck_instances AS
+SELECT public.dod_instances.db_name, 
+       public.functional_aliases.alias hostname,
+       public.get_attribute('port', public.dod_instances.id) port,
+       'dbod' username,
+       public.dod_instances.db_type db_type,
+       public.dod_instances.category category,
+       db_type || ',' || category tags
+FROM public.dod_instances JOIN public.functional_aliases ON
+public.dod_instances.db_name = public.functional_aliases.db_name;
+
+-- Host aliases View
+CREATE OR REPLACE VIEW api.host_aliases AS
+SELECT host, string_agg('dbod-' || db_name || 'domain', E',') aliases 
+FROM dod_instances 
+GROUP BY host;

From f9575a3c668abd4a90e5b88f6473307a8f2b611b Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 7 Jun 2016 17:47:03 +0200
Subject: [PATCH 07/58] First tests for metadata.

---
 dbod/tests/db_tests.py | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/dbod/tests/db_tests.py b/dbod/tests/db_tests.py
index 52fc071..485fa1e 100644
--- a/dbod/tests/db_tests.py
+++ b/dbod/tests/db_tests.py
@@ -8,7 +8,7 @@
 # granted to it by virtue of its status as Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
-import nose
+import unittest
 from types import *
 import json
 import logging
@@ -16,14 +16,34 @@
 import requests
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 
-#from dbod.api.dbops import *
+class Test(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        pass
 
-def empty():
-    assert(True)
-    
-def test_connection():
-    response = requests.get("http://localhost:3000")
-    assert(response.status_code == 200)
+    @classmethod
+    def tearDownClass(self):
+        pass
+        
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+        
+    def test_connection(self):
+        response = requests.get("http://localhost:3000")
+        self.assertEquals(response.status_code, 200)
+        
+    def test_metadata(self):
+        response = requests.get("http://localhost:3000/metadata")
+        self.assertEquals(response.status_code, 200)
+        
+    def test_metadata_has_5_instances(self):
+        response = requests.get("http://localhost:3000/metadata")
+        data = response.json()
+        self.assertEquals(len(data), 5)
+        
 
 if __name__ == "__main__":
-    pass
+    unittest.main()

From a0070733c5417fd259cf9e7c7819832d2d428ff4 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 7 Jun 2016 18:11:35 +0200
Subject: [PATCH 08/58] Execute PostgREST to the API Schema.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index fd6107e..a5bb7c1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,7 @@ install:
 before_script:
     - psql -c 'CREATE DATABASE dbod;' -U postgres
     - psql -f dbod/tests/db_test.sql -U postgres
-    - ./postgrest postgres://postgres@localhost/dbod -a postgres &
+    - ./postgrest postgres://postgres@localhost/dbod -a postgres -s api &
 script:
     - nosetests --with-coverage --cover-package=dbod --cover-html
 after_success:

From f06cb746cf1445f82156a1e08a3e714907dc1a9a Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 8 Jun 2016 10:17:48 +0200
Subject: [PATCH 09/58] Added some temporal prints to see the error in Travis.

---
 .travis.yml            | 2 +-
 dbod/tests/db_tests.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index a5bb7c1..f099bee 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,6 +22,6 @@ before_script:
     - psql -f dbod/tests/db_test.sql -U postgres
     - ./postgrest postgres://postgres@localhost/dbod -a postgres -s api &
 script:
-    - nosetests --with-coverage --cover-package=dbod --cover-html
+    - nosetests --with-coverage --cover-package=dbod --cover-html -s
 after_success:
     coveralls 
diff --git a/dbod/tests/db_tests.py b/dbod/tests/db_tests.py
index 485fa1e..d17718e 100644
--- a/dbod/tests/db_tests.py
+++ b/dbod/tests/db_tests.py
@@ -33,14 +33,17 @@ def tearDown(self):
         
     def test_connection(self):
         response = requests.get("http://localhost:3000")
+        print response.json()
         self.assertEquals(response.status_code, 200)
         
     def test_metadata(self):
         response = requests.get("http://localhost:3000/metadata")
+        print response.json()
         self.assertEquals(response.status_code, 200)
         
     def test_metadata_has_5_instances(self):
         response = requests.get("http://localhost:3000/metadata")
+        print response.json()
         data = response.json()
         self.assertEquals(len(data), 5)
         

From 0594c21501bc841242ffce30cfcfbcdfa8429e6b Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 8 Jun 2016 10:24:32 +0200
Subject: [PATCH 10/58] Set DBOD database properly. Added curl for debugging
 purposes. Added sleep to make sure the server is started.

---
 .travis.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f099bee..c84630a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,9 +19,11 @@ install:
     - pip install coveralls
 before_script:
     - psql -c 'CREATE DATABASE dbod;' -U postgres
-    - psql -f dbod/tests/db_test.sql -U postgres
+    - psql -d dbod -f dbod/tests/db_test.sql -U postgres
     - ./postgrest postgres://postgres@localhost/dbod -a postgres -s api &
+    - sleep 5
 script:
+    - curl -g http://localhost:3000
     - nosetests --with-coverage --cover-package=dbod --cover-html -s
 after_success:
     coveralls 

From c6ba5ef24a368a000bffbb92652331f61fe4ee3e Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 10 Jun 2016 10:34:10 +0200
Subject: [PATCH 11/58] CONFIG: Added new options to config file.

- SSL can be now disabled just removing the section.
- Logging options are now configurable in the config file.
---
 bin/dbod-api   | 22 +++++++++++++---------
 static/api.cfg | 11 +++++++++++
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/bin/dbod-api b/bin/dbod-api
index 59019a9..312d9af 100755
--- a/bin/dbod-api
+++ b/bin/dbod-api
@@ -26,9 +26,9 @@ def main():
     """ Main body """
 
     # Set up log file and level.
-    options.log_file_prefix = config.get('logging','path')
-    options.logging = 'debug'
-    options.log_to_stderr = False
+    options.log_file_prefix = config.get('logging', 'path')
+    options.logging = config.get('logging', 'level')
+    options.log_to_stderr = config.getboolean('logging', 'stderr')
 
     # Parse server command line and set up logging defaults, if necessary
     parse_command_line()
@@ -42,14 +42,18 @@ def main():
             (r"/api/v1/host/aliases/([^/]+)", HostAliases),
             (r"/api/v1/metadata/(?P<class>[^\/]+)/?(?P<name>[^\/]+)?", Metadata),
             (r"/api/v1/rundeck/resources.xml", RundeckResources),
-            ], debug=True)
+            ], debug=config.getboolean('tornado', 'debug'))
 
     logging.info("Configuring HTTP server")
-    http_server = HTTPServer(application,
-            ssl_options = {
-                "certfile" : config.get('ssl', 'hostcert') ,
-                "keyfile" : config.get('ssl', 'hostkey'),
-                })
+    if (config.has_section('ssl')):
+        http_server = HTTPServer(application,
+                ssl_options = {
+                    "certfile" : config.get('ssl', 'hostcert') ,
+                    "keyfile" : config.get('ssl', 'hostkey'),
+                    })
+    else:
+        http_server = HTTPServer(application)
+        logging.info("Host certificate undefined, SSL is DISABLED")
 
     http_server.listen(config.get('server', 'port'))
 
diff --git a/static/api.cfg b/static/api.cfg
index f36194b..fe734dc 100644
--- a/static/api.cfg
+++ b/static/api.cfg
@@ -1,21 +1,32 @@
 [server]
 port=5443
+
 [database]
 user=travis
 host=localhost
 port=5432
 database=travis
 password=travis
+
 [cache]
 path=/etc/dbod/cache/metadata.json
+
 [ssl]
 hostcert=/etc/dbod/hostcert.pem
 hostkey=/etc/dbod/hostkey.pem
+
 [logging]
 path=/var/log/dbod/api.log
+level=debug
+stderr=true
+
+[tornado]
+debug=true
+
 [api]
 user=api-user
 password=api-password
+
 [postgrest]
 rundeck_resources_url=http://localhost:3000/rundeck_instances
 host_aliases_url=http://localhost:3000/host_aliases

From 0566f8afcf3a66b5f4e84669db3839e2119d0637 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 10 Jun 2016 16:06:24 +0200
Subject: [PATCH 12/58] HANDLERS: Removed deprecated code that was using
 connections to PostgreSQL.

---
 bin/dbod-api         |   6 +-
 dbod/api/dbops.py    | 186 -------------------------------------------
 dbod/api/handlers.py | 119 +--------------------------
 3 files changed, 4 insertions(+), 307 deletions(-)
 delete mode 100644 dbod/api/dbops.py

diff --git a/bin/dbod-api b/bin/dbod-api
index 312d9af..b7f5e67 100755
--- a/bin/dbod-api
+++ b/bin/dbod-api
@@ -36,9 +36,9 @@ def main():
     logging.info("Defining application (url, handler) pairs")
     application = tornado.web.Application([
             (r"/", DocHandler),
-            (r"/api/v1/entity/([^/]+)", EntityHandler),
-            (r"/api/v1/entity/alias/([^/]+)", FunctionalAliasHandler),
-            (r"/api/v1/host/([^/]+)", HostHandler),
+            # (r"/api/v1/entity/([^/]+)", EntityHandler),
+            # (r"/api/v1/entity/alias/([^/]+)", FunctionalAliasHandler),
+            # (r"/api/v1/host/([^/]+)", HostHandler),
             (r"/api/v1/host/aliases/([^/]+)", HostAliases),
             (r"/api/v1/metadata/(?P<class>[^\/]+)/?(?P<name>[^\/]+)?", Metadata),
             (r"/api/v1/rundeck/resources.xml", RundeckResources),
diff --git a/dbod/api/dbops.py b/dbod/api/dbops.py
deleted file mode 100644
index 111a369..0000000
--- a/dbod/api/dbops.py
+++ /dev/null
@@ -1,186 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# Copyright (C) 2015, CERN
-# This software is distributed under the terms of the GNU General Public
-# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
-# In applying this license, CERN does not waive the privileges and immunities
-# granted to it by virtue of its status as Intergovernmental Organization
-# or submit itself to any jurisdiction.
-
-"""
-This file contains all database related code
-"""
-
-from psycopg2 import connect, DatabaseError, pool, errorcodes
-import sys, traceback, logging
-
-from dbod.config import config
-
-try:
-    POOL = pool.ThreadedConnectionPool(
-            5,  # Min. # of connections
-            20, # Max. # of connections
-            database = config.get('database', 'database'),
-            user = config.get('database', 'user'),
-            host = config.get('database', 'host'),
-            port = config.get('database', 'port'),
-            password = config.get('database', 'password'))
-except DatabaseError as dberr:
-    logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode[:2]))
-    logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode))
-
-def get_metadata(entity):
-    """Returns a JSON object containing all the metadata for a certain entity"""
-    try:
-        entity = str(entity)
-        with POOL.getconn() as conn:
-            with conn.cursor() as curs:
-                curs.execute("""select data from metadata where db_name = %s""",
-                        (entity, ))
-                res = curs.fetchone()
-                return res[0] if res else None
-    except DatabaseError as dberr:
-        logging.error("PG Error: %s", dberr.pgerror)
-        logging.error("PG Error lookup: %s", errorcodes.lookup(dberr.pgcode))
-        return None
-    finally:
-        POOL.putconn(conn)
-
-
-def insert_metadata(entity, metadata):
-    """Creates an entry in the metadata table"""
-    try:
-        with POOL.getconn() as conn:
-            with conn.cursor() as curs:
-                logging.debug("Creating metadata entry for %s", entity)
-                logging.debug("Metadata: %s", metadata)
-                curs.execute("""insert into metadata(db_name, data) values(%s, %s)""",
-                        (entity, metadata, ))
-                logging.debug('DB query: %s', curs.query)
-                conn.commit()
-                return curs.rowcount == 1
-    except DatabaseError as dberr:
-        logging.error("PG Error: %s", dberr.pgerror)
-        logging.error("PG Error lookup: %s", errorcodes.lookup(dberr.pgcode))
-        return None
-    finally:
-        POOL.putconn(conn)
-
-def update_metadata(entity, metadata):
-    """Updates the JSON object containing all the metadata for an entity"""
-    try:
-        with POOL.getconn() as conn:
-            with conn.cursor() as curs:
-                logging.debug("Updating metadata entry for %s", entity)
-                logging.debug("Metadata: %s", metadata)
-                curs.execute("""update metadata set data =%s where db_name = %s""",
-                        (metadata, entity,))
-                logging.debug('DB query: %s', curs.query)
-                conn.commit()
-                return curs.rowcount == 1
-    except DatabaseError as dberr:
-        logging.error("PG Error: %s", dberr.pgerror)
-        logging.error("PG Error lookup: %s", errorcodes.lookup(dberr.pgcode))
-        return None
-    finally:
-        POOL.putconn(conn)
-
-def delete_metadata(entity):
-    """Deletes the metadata entry for an entity"""
-    try:
-        with POOL.getconn() as conn:
-            with conn.cursor() as curs:
-                curs.execute("""delete from metadata where db_name = %s""",
-                        (entity, ))
-                logging.debug('DB query: %s', curs.query)
-                conn.commit()
-                return curs.rowcount == 1
-    except DatabaseError as dberr:
-        logging.error("PG Error: %s", dberr.pgerror)
-        logging.error("PG Error lookup: %s", errorcodes.lookup(dberr.pgcode))
-        return None
-    finally:
-        POOL.putconn(conn)
-
-def host_metadata(host):
-    """Returns a JSON object containing the metadata for all the entities
-        residing on a host"""
-    try:
-        with POOL.getconn() as conn:
-            with conn.cursor() as curs:
-                curs.execute("""select db_name, data
-                from (
-                    select db_name, json_array_elements(data->'hosts') host, data
-                        from metadata)
-                    as foo
-                    where trim(foo.host::text, '"') = %s""", (host, ))
-                res = curs.fetchall() # Either a list of tuples or empty list
-                return res if res else None
-    except DatabaseError as dberr:
-        logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode[:2]))
-        logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode))
-        return None
-    finally:
-        POOL.putconn(conn)
-
-# Functional aliases related methods
-#   The assumption for the first implementation is that the database
-#   table contains empty entries for pre-created dnsnames that are
-#   considered valid
-
-def next_dnsname():
-    """Returns the next dnsname which can be used for a newly created
-        instance, if any."""
-    try:
-        with POOL.getconn() as conn:
-            with conn.cursor() as curs:
-                curs.execute("""select dns_name
-                from functional_aliases
-                where db_name is NULL order by dns_name limit 1""")
-                logging.debug('DB query: %s', curs.query)
-                return curs.fetchone() # First unused dnsname or None
-    except DatabaseError as dberr:
-        logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode[:2]))
-        logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode))
-        return None
-    finally:
-        POOL.putconn(conn)
-
-def update_functional_alias(dnsname, db_name, alias):
-    """Updates a dnsname record with its db_name and alias"""
-    try:
-        with POOL.getconn() as conn:
-            with conn.cursor() as curs:
-                logging.debug("Updating functional alias record (%s, %s, %s)",
-                    dnsname, db_name, alias)
-                curs.execute("""update functional_aliases
-                set db_name = %s, alias = %s where dns_name = %s""",
-                    (db_name, alias, dnsname,))
-                logging.debug('DB query: %s', curs.query)
-                conn.commit()
-                # Return True if the record was updated succesfully
-                return curs.rowcount == 1
-    except DatabaseError as dberr:
-        logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode[:2]))
-        logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode))
-        return None
-    finally:
-        POOL.putconn(conn)
-
-def get_functional_alias(db_name):
-    """Returns the funcional alias and dnsname for a certain database"""
-    try:
-        with POOL.getconn() as conn:
-            with conn.cursor() as curs:
-                curs.execute("""select dns_name, alias
-                from functional_aliases
-                where db_name = %s""", (db_name,))
-                logging.debug('DB query: %s', curs.query)
-                return curs.fetchone()
-    except DatabaseError as dberr:
-        logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode[:2]))
-        logging.error("PG Error: %s", errorcodes.lookup(dberr.pgcode))
-        return None
-    finally:
-        POOL.putconn(conn)
diff --git a/dbod/api/handlers.py b/dbod/api/handlers.py
index 07041d9..8c32092 100644
--- a/dbod/api/handlers.py
+++ b/dbod/api/handlers.py
@@ -20,6 +20,7 @@
 import functools
 import logging
 import json
+import requests
 
 
 # HTTP API status codes
@@ -81,126 +82,10 @@ def get(self):
             <p>http://hostname:port/api/v1/host/HOSTNAME</p>"""
         self.write(response)
 
-class EntityHandler(tornado.web.RequestHandler):
-
-    def get(self, entity):
-        """Returns metadata for a certain entity"""
-        response = get_metadata(entity)
-        if response:
-            logging.debug(response)
-            self.write(response)
-        else:
-            logging.warning("Entity not found: %s", entity)
-            raise tornado.web.HTTPError(NOT_FOUND)
-
-    @http_basic_auth
-    def post(self, entity):
-        """Returns metadata for a certain entity"""
-        try:
-            metadata = self.get_argument('metadata')
-            response = insert_metadata(entity, metadata)
-            if response:
-                logging.debug("Metadata successfully created for %s: %s",
-                        entity, metadata)
-                self.set_status(CREATED)
-                self.finish()
-        except tornado.web.MissingArgumentError as err:
-            logging.error("Missing 'metadata' argument in request!")
-            raise tornado.web.MissingArgumentError()
-    
-    @http_basic_auth
-    def put(self, entity):
-        """Returns metadata for a certain entity"""
-        try:
-            metadata = self.get_argument('metadata')
-            response = update_metadata(entity, metadata)
-            if response:
-                logging.debug("Metadata successfully updated for %s: %s",
-                        entity, metadata)
-                self.set_status(CREATED)
-                self.finish()
-        except tornado.web.MissingArgumentError as err:
-            logging.error("Missing 'metadata' argument in request!")
-            raise tornado.web.MissingArgumentError()
-
-    @http_basic_auth
-    def delete(self, entity):
-        """Returns metadata for a certain entity"""
-        response = delete_metadata(entity)
-        if response:
-            self.set_status(NO_CONTENT)
-            self.finish()
-        else:
-            logging.warning("Entity not found: %s", entity)
-            raise tornado.web.HTTPError(NOT_FOUND)
-
-class HostHandler(tornado.web.RequestHandler):
-    def get(self, host):
-        """Returns an object containing the metadata for all the entities
-            on a certain host"""
-        response = host_metadata(host)
-        if response:
-            logging.debug(response)
-            self.write(json.dumps(response))
-        else:
-            logging.warning("Metadata not found for host:  %s", host)
-            raise tornado.web.HTTPError(NOT_FOUND)
-
-class FunctionalAliasHandler(tornado.web.RequestHandler):
-    def get(self, entity):
-        """Returns the functional alias association for an entity"""
-        response = get_functional_alias(entity)
-        if response:
-            logging.debug(response)
-            self.write(json.dumps(response))
-        else:
-            logging.error("Functional alias not found for entity: %s", entity)
-            raise tornado.web.HTTPError(NOT_FOUND)
-
-    @http_basic_auth 
-    def post(self, entity):
-        """Creates a functional alias association for an entity"""
-        dnsname = next_dnsname()
-        if dnsname:
-            try:
-                alias = self.get_argument('alias')
-                response = update_functional_alias(dnsname[0], entity, alias)
-                if response:
-                    logging.debug("Functional alias (%s) successfully added for %s",
-                            alias, entity)
-                    self.set_status(CREATED)
-                    self.write(json.dumps(dnsname))
-            except tornado.web.MissingArgumentError as err:
-                logging.error("Missing 'alias' argument in request!")
-                raise tornado.web.MissingArgumentError()
-        else:
-            logging.error("No available dnsnames found!")
-            raise tornado.web.HTTPError(NOT_FOUND)
-
-    @http_basic_auth 
-    def delete(self, entity):
-        """Removes the functional alias association for an entity.
-            If the functional alias doesn't exist it doesn't do anything"""
-        dnsname_full = get_functional_alias(entity)
-        if dnsname_full:
-            dnsname = dnsname_full[0]
-            response = update_functional_alias(dnsname, None, None)
-            if response:
-                logging.debug("Functional alias successfully removed for %s",
-                        entity)
-                self.set_status(NO_CONTENT)
-                self.finish()
-        else:
-            logging.error("Functional alias not found for entity: %s", entity)
-            raise tornado.web.HTTPError(NOT_FOUND)
-
-
 class RundeckResources(tornado.web.RequestHandler):
     def get(self):
         """Returns an valid resources.xml file to import target entities in 
             Rundeck"""
-
-        import requests
         url = config.get('postgrest', 'rundeck_resources_url')
         if url:
             response = requests.get(url)
@@ -236,7 +121,6 @@ def get(self):
 class HostAliases(tornado.web.RequestHandler):
     def get(self, host):
         """list of ip-aliases registered in a host"""
-        import requests
         url = config.get('postgrest', 'host_aliases_url')
         if url:
             composed_url = url + '?host=eq.' + host
@@ -255,7 +139,6 @@ def get(self, host):
 class Metadata(tornado.web.RequestHandler):
     def get(self, **args):
         """Returns entity metadata"""
-        import requests
         url = config.get('postgrest', 'entity_metadata_url')
         name = args.get('name')
         etype = args.get('class')

From 0cfbd4b40e5eb7cc55eea3b049982c24e468f59e Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 10 Jun 2016 16:32:58 +0200
Subject: [PATCH 13/58] CONFIG: Removed unneeded 'database' section.

---
 static/api.cfg | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/static/api.cfg b/static/api.cfg
index fe734dc..8a69057 100644
--- a/static/api.cfg
+++ b/static/api.cfg
@@ -1,13 +1,6 @@
 [server]
 port=5443
 
-[database]
-user=travis
-host=localhost
-port=5432
-database=travis
-password=travis
-
 [cache]
 path=/etc/dbod/cache/metadata.json
 

From 5b8843fb1fde1c0862bbc9081d72ba0b1833cae6 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 10 Jun 2016 16:40:08 +0200
Subject: [PATCH 14/58] CONFIG: Fixed api.pass option in config file.

---
 static/api.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/static/api.cfg b/static/api.cfg
index 8a69057..7aac7f3 100644
--- a/static/api.cfg
+++ b/static/api.cfg
@@ -18,7 +18,7 @@ debug=true
 
 [api]
 user=api-user
-password=api-password
+pass=api-password
 
 [postgrest]
 rundeck_resources_url=http://localhost:3000/rundeck_instances

From da947370ace555140c3eab6d811c7f3a93c57c46 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 10 Jun 2016 17:10:39 +0200
Subject: [PATCH 15/58] HANDLERS: More work in the structure of handlers.

- New file 'base.py' to include all the common methods.
- Moved some handlers to runder.py (work in progress).
- Deleted handlers.py.
---
 bin/dbod-api                         |  5 +-
 dbod/api/base.py                     | 79 ++++++++++++++++++++++++++++
 dbod/api/{handlers.py => rundeck.py} | 62 +---------------------
 3 files changed, 84 insertions(+), 62 deletions(-)
 create mode 100644 dbod/api/base.py
 rename dbod/api/{handlers.py => rundeck.py} (64%)

diff --git a/bin/dbod-api b/bin/dbod-api
index b7f5e67..f24c0f6 100755
--- a/bin/dbod-api
+++ b/bin/dbod-api
@@ -14,10 +14,13 @@ DB On Demand metadata REST API server
 
 import ConfigParser
 import sys, traceback
+
 from tornado.options import parse_command_line, options
 from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
-from dbod.api.handlers import *
+
+from dbod.api.base import DocHandler
+from dbod.api.rundeck import *
 from dbod.config import config
 
 import logging
diff --git a/dbod/api/base.py b/dbod/api/base.py
new file mode 100644
index 0000000..174c358
--- /dev/null
+++ b/dbod/api/base.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+"""
+REST API Server for the DB On Demand System
+"""
+
+import tornado.web
+import base64
+import functools
+import logging
+
+from dbod.config import config
+
+# HTTP API status codes
+OK = 200
+CREATED = 201 # Request fulfilled resulting in creation of new resource
+NO_CONTENT = 204 # Succesfull delete
+NOT_FOUND = 404
+UNAUTHORIZED = 401
+
+# Basic HTTP Authentication decorator
+def http_basic_auth(fun):
+    """Decorator for extracting HTTP basic authentication user/password pairs
+    from the request headers and matching them to configurated credentials.
+    It will generate an HTTP UNAUTHORIZED (Error code 401) if the request is 
+    not using HTTP basic authentication.
+
+    Example:
+        @http_basic_auth
+        def get(self, user, pwd)
+    """
+    @functools.wraps(fun)
+    def wrapper(*args, **kwargs):
+        """ Decorator wrapper """
+        self = args[0]
+        try:
+            # Try
+            auth = self.request.headers.get('Authorization')
+            scheme, _, token = auth.partition(' ')
+            if scheme.lower() == 'basic':
+                # Decode user and password
+                user, _, pwd = base64.decodestring(token).partition(':')
+                if user == config.get('api','user') and pwd == config.get('api','pass'):
+                    return fun(*args, **kwargs)
+                else:
+                    # Raise UNAUTHORIZED HTTP Error (401) 
+                    logging.error("Unauthorized access from: %s",
+                        self.request.headers)
+                    raise tornado.web.HTTPError(UNAUTHORIZED)
+
+            else:
+                # We only support basic authentication
+                logging.error("Authentication scheme not recognized")
+                return "Authentication scheme not recognized"
+        except AttributeError:
+            # Raise UNAUTHORIZED HTTP Error (401) if the request is not
+            # using autentication (auth will be None and partition() will fail
+            logging.error("Unauthorized access from: %s",
+                self.request.headers)
+            raise tornado.web.HTTPError(UNAUTHORIZED)
+
+    return wrapper
+    
+class DocHandler(tornado.web.RequestHandler):
+    """Generates the API endpoint documentation"""
+    def get(self):
+        logging.info("Generating API endpoints doc")
+        response = """Please use :
+            <p>http://hostname:port/api/v1/entity/NAME</p>
+            <p>http://hostname:port/api/v1/host/HOSTNAME</p>"""
+        self.write(response)
diff --git a/dbod/api/handlers.py b/dbod/api/rundeck.py
similarity index 64%
rename from dbod/api/handlers.py
rename to dbod/api/rundeck.py
index 8c32092..e6166d5 100644
--- a/dbod/api/handlers.py
+++ b/dbod/api/rundeck.py
@@ -12,8 +12,6 @@
 REST API Server for the DB On Demand System
 """
 
-from dbod.api.dbops import *
-from dbod.config import config
 import tornado.web
 import tornado.log
 import base64
@@ -22,65 +20,7 @@
 import json
 import requests
 
-
-# HTTP API status codes
-OK = 200
-CREATED = 201 # Request fulfilled resulting in creation of new resource
-NO_CONTENT = 204 # Succesfull delete
-NOT_FOUND = 404
-UNAUTHORIZED = 401
-
-# Basic HTTP Authentication decorator
-def http_basic_auth(fun):
-    """Decorator for extracting HTTP basic authentication user/password pairs
-    from the request headers and matching them to configurated credentials.
-    It will generate an HTTP UNAUTHORIZED (Error code 401) if the request is 
-    not using HTTP basic authentication.
-
-    Example:
-        @http_basic_auth
-        def get(self, user, pwd)
-    """
-    @functools.wraps(fun)
-    def wrapper(*args, **kwargs):
-        """ Decorator wrapper """
-        self = args[0]
-        try:
-            # Try
-            auth = self.request.headers.get('Authorization')
-            scheme, _, token = auth.partition(' ')
-            if scheme.lower() == 'basic':
-                # Decode user and password
-                user, _, pwd = base64.decodestring(token).partition(':')
-                if user == config.get('api','user') and pwd == config.get('api','pass'):
-                    return fun(*args, **kwargs)
-                else:
-                    # Raise UNAUTHORIZED HTTP Error (401) 
-                    logging.error("Unauthorized access from: %s",
-                        self.request.headers)
-                    raise tornado.web.HTTPError(UNAUTHORIZED)
-
-            else:
-                # We only support basic authentication
-                logging.error("Authentication scheme not recognized")
-                return "Authentication scheme not recognized"
-        except AttributeError:
-            # Raise UNAUTHORIZED HTTP Error (401) if the request is not
-            # using autentication (auth will be None and partition() will fail
-            logging.error("Unauthorized access from: %s",
-                self.request.headers)
-            raise tornado.web.HTTPError(UNAUTHORIZED)
-
-    return wrapper
-
-class DocHandler(tornado.web.RequestHandler):
-    """Generates the API endpoint documentation"""
-    def get(self):
-        logging.info("Generating API endpoints doc")
-        response = """Please use :
-            <p>http://hostname:port/api/v1/entity/NAME</p>
-            <p>http://hostname:port/api/v1/host/HOSTNAME</p>"""
-        self.write(response)
+from dbod.config import config
 
 class RundeckResources(tornado.web.RequestHandler):
     def get(self):

From 605f26ffff256ed2aa40cb44afdfe1d53b941b77 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 10 Jun 2016 17:55:21 +0200
Subject: [PATCH 16/58] HANDLERS: Split in different modules (rundeck,
 hostaliases, metadata).

---
 bin/dbod-api            |  5 +++-
 dbod/api/hostaliases.py | 38 ++++++++++++++++++++++++++++++
 dbod/api/metadata.py    | 50 +++++++++++++++++++++++++++++++++++++++
 dbod/api/rundeck.py     | 52 -----------------------------------------
 4 files changed, 92 insertions(+), 53 deletions(-)
 create mode 100644 dbod/api/hostaliases.py
 create mode 100644 dbod/api/metadata.py

diff --git a/bin/dbod-api b/bin/dbod-api
index f24c0f6..7b59d22 100755
--- a/bin/dbod-api
+++ b/bin/dbod-api
@@ -14,13 +14,16 @@ DB On Demand metadata REST API server
 
 import ConfigParser
 import sys, traceback
+import tornado.web
 
 from tornado.options import parse_command_line, options
 from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
 
 from dbod.api.base import DocHandler
-from dbod.api.rundeck import *
+from dbod.api.rundeck import RundeckResources
+from dbod.api.metadata import Metadata
+from dbod.api.hostaliases import HostAliases
 from dbod.config import config
 
 import logging
diff --git a/dbod/api/hostaliases.py b/dbod/api/hostaliases.py
new file mode 100644
index 0000000..17c8b82
--- /dev/null
+++ b/dbod/api/hostaliases.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+"""
+REST API Server for the DB On Demand System
+"""
+
+import tornado.web
+import logging
+import json
+import requests
+
+from dbod.config import config
+
+class HostAliases(tornado.web.RequestHandler):
+    def get(self, host):
+        """list of ip-aliases registered in a host"""
+        url = config.get('postgrest', 'host_aliases_url')
+        if url:
+            composed_url = url + '?host=eq.' + host
+            logging.debug('Requesting ' + composed_url )
+            response = requests.get(composed_url)
+            if response.ok:
+                data = json.loads(response.text)
+                d = data.pop()
+                self.write(d.get('aliases'))
+            else: 
+                logging.error("Error fetching aliases in host: " + host)
+                raise tornado.web.HTTPError(NOT_FOUND)
+        else:
+            logging.error("Internal host aliases endpoint not configured")
diff --git a/dbod/api/metadata.py b/dbod/api/metadata.py
new file mode 100644
index 0000000..1486247
--- /dev/null
+++ b/dbod/api/metadata.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+"""
+REST API Server for the DB On Demand System
+"""
+
+import tornado.web
+import logging
+import json
+import requests
+
+from dbod.config import config
+
+class Metadata(tornado.web.RequestHandler):
+    def get(self, **args):
+        """Returns entity metadata"""
+        url = config.get('postgrest', 'entity_metadata_url')
+        name = args.get('name')
+        etype = args.get('class')
+        if url:
+            if etype == u'entity':
+                composed_url = url + '?db_name=eq.' + name
+            else:
+                composed_url = url + '?host=eq.' + name
+            logging.debug('Requesting ' + composed_url )
+            response = requests.get(composed_url)
+            if response.ok:
+                data = json.loads(response.text)
+                if data != []:
+                    if etype == u'entity':
+                        d = data.pop()
+                        self.write(d)
+                    else:
+                        self.write(json.dumps(data))
+                else: 
+                    logging.error("Entity metadata not found: " + name)
+                    raise tornado.web.HTTPError(NOT_FOUND)
+            else: 
+                logging.error("Error fetching entity metadata: " + name)
+                raise tornado.web.HTTPError(response.status_code)
+        else:
+            logging.error("Internal entity metadata endpoint not configured")
diff --git a/dbod/api/rundeck.py b/dbod/api/rundeck.py
index e6166d5..11ea225 100644
--- a/dbod/api/rundeck.py
+++ b/dbod/api/rundeck.py
@@ -13,9 +13,6 @@
 """
 
 import tornado.web
-import tornado.log
-import base64
-import functools
 import logging
 import json
 import requests
@@ -56,52 +53,3 @@ def get(self):
                 raise tornado.web.HTTPError(NOT_FOUND)
         else:
             logging.error("Internal Rundeck resources endpoint not configured")
-            
-
-class HostAliases(tornado.web.RequestHandler):
-    def get(self, host):
-        """list of ip-aliases registered in a host"""
-        url = config.get('postgrest', 'host_aliases_url')
-        if url:
-            composed_url = url + '?host=eq.' + host
-            logging.debug('Requesting ' + composed_url )
-            response = requests.get(composed_url)
-            if response.ok:
-                data = json.loads(response.text)
-                d = data.pop()
-                self.write(d.get('aliases'))
-            else: 
-                logging.error("Error fetching aliases in host: " + host)
-                raise tornado.web.HTTPError(NOT_FOUND)
-        else:
-            logging.error("Internal host aliases endpoint not configured")
-            
-class Metadata(tornado.web.RequestHandler):
-    def get(self, **args):
-        """Returns entity metadata"""
-        url = config.get('postgrest', 'entity_metadata_url')
-        name = args.get('name')
-        etype = args.get('class')
-        if url:
-            if etype == u'entity':
-                composed_url = url + '?db_name=eq.' + name
-            else:
-                composed_url = url + '?host=eq.' + name
-            logging.debug('Requesting ' + composed_url )
-            response = requests.get(composed_url)
-            if response.ok:
-                data = json.loads(response.text)
-                if data != []:
-                    if etype == u'entity':
-                        d = data.pop()
-                        self.write(d)
-                    else:
-                        self.write(json.dumps(data))
-                else: 
-                    logging.error("Entity metadata not found: " + name)
-                    raise tornado.web.HTTPError(NOT_FOUND)
-            else: 
-                logging.error("Error fetching entity metadata: " + name)
-                raise tornado.web.HTTPError(response.status_code)
-        else:
-            logging.error("Internal entity metadata endpoint not configured")

From 6e354de2cb33bc5a39953266b596136f983838a4 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Mon, 20 Jun 2016 10:24:23 +0200
Subject: [PATCH 17/58] Set content to Json to ensure the correct type.

---
 dbod/api/metadata.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dbod/api/metadata.py b/dbod/api/metadata.py
index 1486247..94bf9f5 100644
--- a/dbod/api/metadata.py
+++ b/dbod/api/metadata.py
@@ -21,6 +21,7 @@
 
 class Metadata(tornado.web.RequestHandler):
     def get(self, **args):
+        self.set_header("Content-Type", 'application/json')
         """Returns entity metadata"""
         url = config.get('postgrest', 'entity_metadata_url')
         name = args.get('name')

From b673e21da3dcca08e40aa3fc273b6f938d08ea70 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 21 Jun 2016 14:33:52 +0200
Subject: [PATCH 18/58] Cleanup

---
 dbod/tests/db_tests.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/dbod/tests/db_tests.py b/dbod/tests/db_tests.py
index d17718e..bf7714a 100644
--- a/dbod/tests/db_tests.py
+++ b/dbod/tests/db_tests.py
@@ -16,7 +16,9 @@
 import requests
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 
-class Test(unittest.TestCase):
+from dbod.api.dbops import *
+
+class TestMetadata(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         pass
@@ -33,20 +35,39 @@ def tearDown(self):
         
     def test_connection(self):
         response = requests.get("http://localhost:3000")
-        print response.json()
         self.assertEquals(response.status_code, 200)
         
     def test_metadata(self):
         response = requests.get("http://localhost:3000/metadata")
-        print response.json()
         self.assertEquals(response.status_code, 200)
         
     def test_metadata_has_5_instances(self):
         response = requests.get("http://localhost:3000/metadata")
-        print response.json()
         data = response.json()
         self.assertEquals(len(data), 5)
         
+    # def test_metadata_get_entity_dbod01(self):
+        # response = requests.get("http://localhost:3000/metadata")
+        
+class TestAPI(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        pass
+
+    @classmethod
+    def tearDownClass(self):
+        pass
+        
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+        
+    def test_connection(self):
+        response = requests.get("https://localhost:5432/", verify=False)
+        self.assertEquals(response.status_code, 200)
+        
 
 if __name__ == "__main__":
     unittest.main()

From eaea1ecfa7421ee444dbda4132c6cca655755298 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 29 Jun 2016 10:56:52 +0200
Subject: [PATCH 19/58] TEST: Separated classes in different files.

---
 dbod/tests/{db_tests.py => metadata.py} | 33 ++---------------------
 dbod/tests/rundeck.py                   | 35 +++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 31 deletions(-)
 rename dbod/tests/{db_tests.py => metadata.py} (63%)
 create mode 100644 dbod/tests/rundeck.py

diff --git a/dbod/tests/db_tests.py b/dbod/tests/metadata.py
similarity index 63%
rename from dbod/tests/db_tests.py
rename to dbod/tests/metadata.py
index bf7714a..c544422 100644
--- a/dbod/tests/db_tests.py
+++ b/dbod/tests/metadata.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2015, CERN
 # This software is distributed under the terms of the GNU General Public
@@ -16,9 +14,7 @@
 import requests
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 
-from dbod.api.dbops import *
-
-class TestMetadata(unittest.TestCase):
+class Metadata(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         pass
@@ -45,29 +41,4 @@ def test_metadata_has_5_instances(self):
         response = requests.get("http://localhost:3000/metadata")
         data = response.json()
         self.assertEquals(len(data), 5)
-        
-    # def test_metadata_get_entity_dbod01(self):
-        # response = requests.get("http://localhost:3000/metadata")
-        
-class TestAPI(unittest.TestCase):
-    @classmethod
-    def setUpClass(self):
-        pass
-
-    @classmethod
-    def tearDownClass(self):
-        pass
-        
-    def setUp(self):
-        pass
-
-    def tearDown(self):
-        pass
-        
-    def test_connection(self):
-        response = requests.get("https://localhost:5432/", verify=False)
-        self.assertEquals(response.status_code, 200)
-        
-
-if __name__ == "__main__":
-    unittest.main()
+        
\ No newline at end of file
diff --git a/dbod/tests/rundeck.py b/dbod/tests/rundeck.py
new file mode 100644
index 0000000..695e557
--- /dev/null
+++ b/dbod/tests/rundeck.py
@@ -0,0 +1,35 @@
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+import unittest
+from types import *
+import json
+import logging
+import sys
+import requests
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+class Rundeck(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        pass
+
+    @classmethod
+    def tearDownClass(self):
+        pass
+        
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+        
+    def test_connection(self):
+        response = requests.get("https://localhost:5432/", verify=False)
+        self.assertEquals(response.status_code, 200)
+        
\ No newline at end of file

From c2b3e16d6dcfd93afc7e4dea351662ad74095162 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 29 Jun 2016 11:01:55 +0200
Subject: [PATCH 20/58] TRAVIS: Added API install and execution.

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index c84630a..f8712ae 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,10 +17,12 @@ before_install:
 install:
     - pip install -r requirements.txt
     - pip install coveralls
+    - python setup.py install
 before_script:
     - psql -c 'CREATE DATABASE dbod;' -U postgres
     - psql -d dbod -f dbod/tests/db_test.sql -U postgres
     - ./postgrest postgres://postgres@localhost/dbod -a postgres -s api &
+    - bin/dbod-api
     - sleep 5
 script:
     - curl -g http://localhost:3000

From e660258cb3a91ac0f12afca978221941b5f9ee2b Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 30 Jun 2016 09:28:35 +0200
Subject: [PATCH 21/58] TEST: Fixed tests, now nose will be able to find them.

---
 dbod/tests/{metadata.py => metadata_test.py} | 6 +++---
 dbod/tests/{rundeck.py => rundeck_test.py}   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename dbod/tests/{metadata.py => metadata_test.py} (92%)
 rename dbod/tests/{rundeck.py => rundeck_test.py} (93%)

diff --git a/dbod/tests/metadata.py b/dbod/tests/metadata_test.py
similarity index 92%
rename from dbod/tests/metadata.py
rename to dbod/tests/metadata_test.py
index c544422..617cc34 100644
--- a/dbod/tests/metadata.py
+++ b/dbod/tests/metadata_test.py
@@ -30,15 +30,15 @@ def tearDown(self):
         pass
         
     def test_connection(self):
-        response = requests.get("http://localhost:3000")
+        response = requests.get("http://localhost:3000", verify=False)
         self.assertEquals(response.status_code, 200)
         
     def test_metadata(self):
-        response = requests.get("http://localhost:3000/metadata")
+        response = requests.get("http://localhost:3000/metadata", verify=False)
         self.assertEquals(response.status_code, 200)
         
     def test_metadata_has_5_instances(self):
-        response = requests.get("http://localhost:3000/metadata")
+        response = requests.get("http://localhost:3000/metadata", verify=False)
         data = response.json()
         self.assertEquals(len(data), 5)
         
\ No newline at end of file
diff --git a/dbod/tests/rundeck.py b/dbod/tests/rundeck_test.py
similarity index 93%
rename from dbod/tests/rundeck.py
rename to dbod/tests/rundeck_test.py
index 695e557..08c73f4 100644
--- a/dbod/tests/rundeck.py
+++ b/dbod/tests/rundeck_test.py
@@ -30,6 +30,6 @@ def tearDown(self):
         pass
         
     def test_connection(self):
-        response = requests.get("https://localhost:5432/", verify=False)
+        response = requests.get("https://localhost:5443/", verify=False)
         self.assertEquals(response.status_code, 200)
         
\ No newline at end of file

From e454af2ef70ab851ccbfd3f55405b3167434a7ef Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 30 Jun 2016 15:09:54 +0200
Subject: [PATCH 22/58] CONFIG: Added rundeck and removed ssl sections.

---
 static/api.cfg | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/static/api.cfg b/static/api.cfg
index 7aac7f3..353ea56 100644
--- a/static/api.cfg
+++ b/static/api.cfg
@@ -4,10 +4,6 @@ port=5443
 [cache]
 path=/etc/dbod/cache/metadata.json
 
-[ssl]
-hostcert=/etc/dbod/hostcert.pem
-hostkey=/etc/dbod/hostkey.pem
-
 [logging]
 path=/var/log/dbod/api.log
 level=debug
@@ -24,3 +20,11 @@ pass=api-password
 rundeck_resources_url=http://localhost:3000/rundeck_instances
 host_aliases_url=http://localhost:3000/host_aliases
 entity_metadata_url=http://localhost:3000/metadata
+
+[rundeck]
+api_run_job = https://rundeck/api/14/job/{0}/run
+api_job_output = https://rundeck/api/14/execution/{0}/output
+
+[rundeck-jobs]
+get-snapshots = d4072a88-b7fc-4e0b-bd0a-06e2d97e16dd
+

From e0ebb3ea4dc16a72c40ea8a28b4e9f66f39023fa Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 30 Jun 2016 15:23:31 +0200
Subject: [PATCH 23/58] TRAVIS: Start dbod-api in background.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f8712ae..3cb365e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,7 +22,7 @@ before_script:
     - psql -c 'CREATE DATABASE dbod;' -U postgres
     - psql -d dbod -f dbod/tests/db_test.sql -U postgres
     - ./postgrest postgres://postgres@localhost/dbod -a postgres -s api &
-    - bin/dbod-api
+    - bin/dbod-api &
     - sleep 5
 script:
     - curl -g http://localhost:3000

From bd3557e248126f2275880b9314e1ae804ff9468f Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 1 Jul 2016 11:02:36 +0200
Subject: [PATCH 24/58] HANDLERS: Responses are sent in a 'response' object.

---
 dbod/api/metadata.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/dbod/api/metadata.py b/dbod/api/metadata.py
index 94bf9f5..56cbf9e 100644
--- a/dbod/api/metadata.py
+++ b/dbod/api/metadata.py
@@ -14,9 +14,9 @@
 
 import tornado.web
 import logging
-import json
 import requests
 
+from dbod.api.base import *
 from dbod.config import config
 
 class Metadata(tornado.web.RequestHandler):
@@ -26,21 +26,20 @@ def get(self, **args):
         url = config.get('postgrest', 'entity_metadata_url')
         name = args.get('name')
         etype = args.get('class')
-        if url:
+        if url and name:
             if etype == u'entity':
                 composed_url = url + '?db_name=eq.' + name
-            else:
+            elif etype == u'host':
                 composed_url = url + '?host=eq.' + name
-            logging.debug('Requesting ' + composed_url )
-            response = requests.get(composed_url)
+            else:
+                logging.error("Unsupported endpoint")
+                raise tornado.web.HTTPError(NOT_FOUND)
+            logging.debug('Requesting ' + composed_url)
+            response = requests.get(composed_url, verify=False)
             if response.ok:
-                data = json.loads(response.text)
-                if data != []:
-                    if etype == u'entity':
-                        d = data.pop()
-                        self.write(d)
-                    else:
-                        self.write(json.dumps(data))
+                data = response.json()
+                if data:
+                    self.write({'response' : data})
                 else: 
                     logging.error("Entity metadata not found: " + name)
                     raise tornado.web.HTTPError(NOT_FOUND)
@@ -49,3 +48,4 @@ def get(self, **args):
                 raise tornado.web.HTTPError(response.status_code)
         else:
             logging.error("Internal entity metadata endpoint not configured")
+            raise tornado.web.HTTPError(NOT_FOUND)

From 75f8e714eed50e256eb2e22cc63d87c89d6cb3f0 Mon Sep 17 00:00:00 2001
From: Ignacio Coterillo <ignacio.coterillo@cern.ch>
Date: Fri, 1 Jul 2016 11:29:59 +0200
Subject: [PATCH 25/58] HANDLERS: HostAliases response packaged as object

---
 dbod/api/hostaliases.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dbod/api/hostaliases.py b/dbod/api/hostaliases.py
index 17c8b82..abb0276 100644
--- a/dbod/api/hostaliases.py
+++ b/dbod/api/hostaliases.py
@@ -28,9 +28,8 @@ def get(self, host):
             logging.debug('Requesting ' + composed_url )
             response = requests.get(composed_url)
             if response.ok:
-                data = json.loads(response.text)
-                d = data.pop()
-                self.write(d.get('aliases'))
+                data = response.json()
+                self.write({'response' : data})
             else: 
                 logging.error("Error fetching aliases in host: " + host)
                 raise tornado.web.HTTPError(NOT_FOUND)

From 4e091f71c54ec99b0301f7740fc6585f924079bd Mon Sep 17 00:00:00 2001
From: Ignacio Coterillo <ignacio.coterillo@cern.ch>
Date: Fri, 1 Jul 2016 11:31:37 +0200
Subject: [PATCH 26/58] SQL: Modified host_aliases to return a list of aliases

---
 sql/views.sql | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 sql/views.sql

diff --git a/sql/views.sql b/sql/views.sql
new file mode 100644
index 0000000..18fcb38
--- /dev/null
+++ b/sql/views.sql
@@ -0,0 +1,96 @@
+
+-- Drop view and function to be able to insert it again
+DROP VIEW api.test_metadata;
+DROP FUNCTION get_hosts(INTEGER[]);
+DROP FUNCTION get_volumes(INTEGER);
+
+-- Job stats view
+CREATE OR REPLACE VIEW job_stats AS 
+SELECT db_name, command_name, COUNT(*) as COUNT, ROUND(AVG(completion_date - creation_date) * 24*60*60) AS mean_duration
+FROM dod_jobs GROUP BY command_name, db_name;
+
+-- Command stats view
+CREATE OR REPLACE VIEW command_stats AS
+SELECT command_name, COUNT(*) AS COUNT, ROUND(AVG(completion_date - creation_date) * 24*60*60) AS mean_duration
+FROM dod_jobs GROUP BY command_name;
+
+-- Get hosts function
+CREATE OR REPLACE FUNCTION get_hosts(host_ids INTEGER[])
+RETURNS VARCHAR[] AS $$
+DECLARE
+  hosts VARCHAR := '';
+BEGIN
+  SELECT ARRAY (SELECT name FROM host WHERE id = ANY(host_ids)) INTO hosts;
+  RETURN hosts;
+END
+$$ LANGUAGE plpgsql;
+
+-- Get volumes function
+CREATE OR REPLACE FUNCTION get_volumes(pid INTEGER)
+RETURNS JSON[] AS $$
+DECLARE
+  volumes JSON[];
+BEGIN
+  SELECT ARRAY (SELECT row_to_json(t) FROM (SELECT * FROM public.volume WHERE instance_id = pid) t) INTO volumes;
+  return volumes;
+END
+$$ LANGUAGE plpgsql;
+
+-- Get port function
+CREATE OR REPLACE FUNCTION get_attribute(name VARCHAR, instance_id INTEGER)
+RETURNS VARCHAR AS $$
+DECLARE
+  res VARCHAR;
+BEGIN
+  SELECT value FROM public.attribute A WHERE A.instance_id = instance_id AND A.name = name INTO res;
+  return res;
+END
+$$ LANGUAGE plpgsql;
+
+
+-- Get directories function
+CREATE OR REPLACE FUNCTION get_directories(inst_name VARCHAR, type VARCHAR, version VARCHAR, port VARCHAR)
+RETURNS TABLE (basedir VARCHAR, bindir VARCHAR, datadir VARCHAR, logdir VARCHAR, socket VARCHAR) AS $$
+BEGIN
+  IF type = 'MYSQL' THEN
+    RETURN QUERY SELECT 
+      ('/usr/local/mysql/mysql-' || version)::VARCHAR basedir, 
+      ('/usr/local/mysql/mysql-' || version || '/bin')::VARCHAR bindir, 
+      ('/ORA/dbs03/' || upper(inst_name) || '/mysql')::VARCHAR datadir, 
+      ('/ORA/dbs02/' || upper(inst_name) || '/mysql')::VARCHAR logdir, 
+      ('/var/lib/mysql/mysql.sock.' || lower(inst_name) || '.' || port)::VARCHAR socket;
+  ELSIF type = 'PG' THEN
+    RETURN QUERY SELECT 
+      ('/usr/local/pgsql/pgsql-' || version)::VARCHAR basedir, 
+      ('/usr/local/mysql/mysql-' || version || '/bin')::VARCHAR bindir, 
+      ('/ORA/dbs03/' || upper(inst_name) || '/data')::VARCHAR datadir, 
+      ('/ORA/dbs02/' || upper(inst_name) || '/pg_xlog')::VARCHAR logdir, 
+      ('/var/lib/pgsql/')::VARCHAR socket;
+  END IF;
+END
+$$ LANGUAGE plpgsql;
+
+
+-- Metadata View
+CREATE OR REPLACE VIEW api.metadata AS
+SELECT id, username, db_name, category, db_type, version, host, get_attribute('port', id) port, get_volumes volumes, d.*
+FROM fo_dod_instances, get_volumes(id), get_directories(db_name, db_type, version, get_attribute('port', id)) d;
+
+-- Rundeck instances View
+CREATE OR REPLACE VIEW api.rundeck_instances AS
+SELECT public.dod_instances.db_name, 
+       public.functional_aliases.alias hostname,
+       public.get_attribute('port', public.dod_instances.id) port,
+       'dbod' username,
+       public.dod_instances.db_type db_type,
+       public.dod_instances.category category,
+       db_type || ',' || category tags
+FROM public.dod_instances JOIN public.functional_aliases ON
+public.dod_instances.db_name = public.functional_aliases.db_name;
+
+-- Host aliases View
+CREATE OR REPLACE VIEW api.host_aliases AS
+SELECT host, array_agg('dbod-' || db_name || '.cern.ch') aliases 
+FROM dod_instances 
+GROUP BY host;
+

From 904989ea749728129e3bfcc239f69819b57485b9 Mon Sep 17 00:00:00 2001
From: Ignacio Coterillo <ignacio.coterillo@cern.ch>
Date: Fri, 1 Jul 2016 11:32:52 +0200
Subject: [PATCH 27/58] GITIGNORE: Added vim .swp files

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index ed1fbcc..07c7194 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@ cover/
 __pycache__/
 *.py[cod]
 
+*.swp
+
 # C extensions
 *.so
 

From ed1ab677e0f0b13ce508f76326986d9e09caf74a Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 1 Jul 2016 11:32:55 +0200
Subject: [PATCH 28/58] TEST: Added 5 metadata tests.

---
 dbod/tests/metadata_test.py | 43 ++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/dbod/tests/metadata_test.py b/dbod/tests/metadata_test.py
index 617cc34..3c1fc4e 100644
--- a/dbod/tests/metadata_test.py
+++ b/dbod/tests/metadata_test.py
@@ -29,16 +29,43 @@ def setUp(self):
     def tearDown(self):
         pass
         
-    def test_connection(self):
-        response = requests.get("http://localhost:3000", verify=False)
+    def test_single_instance_by_name(self):
+        response = requests.get("http://localhost:5443/api/v1/metadata/entity/dbod01", verify=False)
+        data = response.json()["response"]
         self.assertEquals(response.status_code, 200)
+        self.assertEquals(len(data), 1)
+        self.assertEquals(data[0]["db_name"], "dbod01")
+        self.assertTrue(data[0]["volumes"] != None)
+        self.assertTrue(data[0]["host"] != None)
         
-    def test_metadata(self):
-        response = requests.get("http://localhost:3000/metadata", verify=False)
+    def test_no_instance_by_name(self):
+        response = requests.get("http://localhost:5443/api/v1/metadata/entity/invalid", verify=False)
+        self.assertEquals(response.status_code, 404)
+        
+    def test_single_instance_by_host(self):
+        response = requests.get("http://localhost:5443/api/v1/metadata/host/host03", verify=False)
+        data = response.json()["response"]
+        self.assertEquals(response.status_code, 200)
+        self.assertEquals(len(data), 1)
+        self.assertTrue(data[0]["volumes"] != None)
+        self.assertEquals(data[0]["host"], "host03")
+        
+    def test_multiple_instances_by_host(self):
+        response = requests.get("http://localhost:5443/api/v1/metadata/host/host01", verify=False)
+        data = response.json()["response"]
         self.assertEquals(response.status_code, 200)
+        self.assertEquals(len(data), 4)
+        list = []
+        for i in range(4):
+            self.assertEquals(data[i]["host"], "host01")
+            self.assertTrue(data[i]["volumes"] != None)
+            self.assertNotIn(data[i]["db_name"], list)
+            list.append(data[i]["db_name"])
+        self.assertEquals(len(list), 4)
+        
+    def test_no_instance_by_host(self):
+        response = requests.get("http://localhost:5443/api/v1/metadata/host/invalid", verify=False)
+        self.assertEquals(response.status_code, 404)
+    
         
-    def test_metadata_has_5_instances(self):
-        response = requests.get("http://localhost:3000/metadata", verify=False)
-        data = response.json()
-        self.assertEquals(len(data), 5)
         
\ No newline at end of file

From 5546e578f872b8d77fbf32818068495e459732b7 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 1 Jul 2016 11:57:40 +0200
Subject: [PATCH 29/58] TEST: Removed useless test in rundeck_test.py

---
 dbod/tests/rundeck_test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/dbod/tests/rundeck_test.py b/dbod/tests/rundeck_test.py
index 08c73f4..752afa1 100644
--- a/dbod/tests/rundeck_test.py
+++ b/dbod/tests/rundeck_test.py
@@ -29,7 +29,4 @@ def setUp(self):
     def tearDown(self):
         pass
         
-    def test_connection(self):
-        response = requests.get("https://localhost:5443/", verify=False)
-        self.assertEquals(response.status_code, 200)
         
\ No newline at end of file

From 6afecd34fd6abc25e33000a3f32ef4516e5aa9fe Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 1 Jul 2016 15:07:25 +0200
Subject: [PATCH 30/58] TEST: Added timeout to tests.

---
 dbod/tests/metadata_test.py | 28 +++++++++++++++-------------
 requirements.txt            |  1 +
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/dbod/tests/metadata_test.py b/dbod/tests/metadata_test.py
index 3c1fc4e..477e6b5 100644
--- a/dbod/tests/metadata_test.py
+++ b/dbod/tests/metadata_test.py
@@ -7,28 +7,26 @@
 # or submit itself to any jurisdiction.
 
 import unittest
-from types import *
-import json
-import logging
-import sys
 import requests
-logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+from timeout_decorator import timeout
 
 class Metadata(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         pass
-
+    
     @classmethod
     def tearDownClass(self):
         pass
-        
+    
     def setUp(self):
         pass
-
+    
     def tearDown(self):
         pass
-        
+    
+    @timeout(5)
     def test_single_instance_by_name(self):
         response = requests.get("http://localhost:5443/api/v1/metadata/entity/dbod01", verify=False)
         data = response.json()["response"]
@@ -37,11 +35,13 @@ def test_single_instance_by_name(self):
         self.assertEquals(data[0]["db_name"], "dbod01")
         self.assertTrue(data[0]["volumes"] != None)
         self.assertTrue(data[0]["host"] != None)
-        
+    
+    @timeout(5)
     def test_no_instance_by_name(self):
         response = requests.get("http://localhost:5443/api/v1/metadata/entity/invalid", verify=False)
         self.assertEquals(response.status_code, 404)
-        
+    
+    @timeout(5)
     def test_single_instance_by_host(self):
         response = requests.get("http://localhost:5443/api/v1/metadata/host/host03", verify=False)
         data = response.json()["response"]
@@ -49,7 +49,8 @@ def test_single_instance_by_host(self):
         self.assertEquals(len(data), 1)
         self.assertTrue(data[0]["volumes"] != None)
         self.assertEquals(data[0]["host"], "host03")
-        
+    
+    @timeout(5)
     def test_multiple_instances_by_host(self):
         response = requests.get("http://localhost:5443/api/v1/metadata/host/host01", verify=False)
         data = response.json()["response"]
@@ -62,7 +63,8 @@ def test_multiple_instances_by_host(self):
             self.assertNotIn(data[i]["db_name"], list)
             list.append(data[i]["db_name"])
         self.assertEquals(len(list), 4)
-        
+    
+    @timeout(5)
     def test_no_instance_by_host(self):
         response = requests.get("http://localhost:5443/api/v1/metadata/host/invalid", verify=False)
         self.assertEquals(response.status_code, 404)
diff --git a/requirements.txt b/requirements.txt
index 7889b8b..842c9e5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@ psycopg2
 tornado==4.2
 virtualenv
 requests
+timeout-decorator
\ No newline at end of file

From 86491d9cd28071248aaf5067fd0f55fcb19f05fe Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 5 Jul 2016 15:07:48 +0200
Subject: [PATCH 31/58] STRUCTURE: Changed the structure of the application to
 make the tests properly.

---
 bin/dbod-api                | 57 ++----------------------------
 dbod/api/api.py             | 69 +++++++++++++++++++++++++++++++++++++
 dbod/tests/metadata_test.py | 57 ++++++++++++++----------------
 3 files changed, 97 insertions(+), 86 deletions(-)
 create mode 100644 dbod/api/api.py

diff --git a/bin/dbod-api b/bin/dbod-api
index 7b59d22..d79e0f2 100755
--- a/bin/dbod-api
+++ b/bin/dbod-api
@@ -12,59 +12,8 @@
 DB On Demand metadata REST API server
 """
 
-import ConfigParser
-import sys, traceback
-import tornado.web
-
-from tornado.options import parse_command_line, options
-from tornado.httpserver import HTTPServer
-from tornado.ioloop import IOLoop
-
-from dbod.api.base import DocHandler
-from dbod.api.rundeck import RundeckResources
-from dbod.api.metadata import Metadata
-from dbod.api.hostaliases import HostAliases
-from dbod.config import config
-
-import logging
-
-def main():
-    """ Main body """
-
-    # Set up log file and level.
-    options.log_file_prefix = config.get('logging', 'path')
-    options.logging = config.get('logging', 'level')
-    options.log_to_stderr = config.getboolean('logging', 'stderr')
-
-    # Parse server command line and set up logging defaults, if necessary
-    parse_command_line()
-
-    logging.info("Defining application (url, handler) pairs")
-    application = tornado.web.Application([
-            (r"/", DocHandler),
-            # (r"/api/v1/entity/([^/]+)", EntityHandler),
-            # (r"/api/v1/entity/alias/([^/]+)", FunctionalAliasHandler),
-            # (r"/api/v1/host/([^/]+)", HostHandler),
-            (r"/api/v1/host/aliases/([^/]+)", HostAliases),
-            (r"/api/v1/metadata/(?P<class>[^\/]+)/?(?P<name>[^\/]+)?", Metadata),
-            (r"/api/v1/rundeck/resources.xml", RundeckResources),
-            ], debug=config.getboolean('tornado', 'debug'))
-
-    logging.info("Configuring HTTP server")
-    if (config.has_section('ssl')):
-        http_server = HTTPServer(application,
-                ssl_options = {
-                    "certfile" : config.get('ssl', 'hostcert') ,
-                    "keyfile" : config.get('ssl', 'hostkey'),
-                    })
-    else:
-        http_server = HTTPServer(application)
-        logging.info("Host certificate undefined, SSL is DISABLED")
-
-    http_server.listen(config.get('server', 'port'))
-
-    logging.info("Starting application")
-    IOLoop.instance().start()
+from dbod.api.api import Application
 
 if __name__ == "__main__":
-    main()
+    Application()
+
diff --git a/dbod/api/api.py b/dbod/api/api.py
new file mode 100644
index 0000000..6339a29
--- /dev/null
+++ b/dbod/api/api.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+import ConfigParser
+import sys, traceback
+import tornado.web
+import logging
+
+from tornado.options import parse_command_line, options, define
+from tornado.httpserver import HTTPServer
+from tornado.ioloop import IOLoop
+
+from dbod.api.base import DocHandler
+from dbod.api.rundeck import RundeckResources
+from dbod.api.metadata import Metadata
+from dbod.api.hostaliases import HostAliases
+from dbod.config import config
+
+handlers = [
+    (r"/", DocHandler),
+    # (r"/api/v1/entity/([^/]+)", EntityHandler),
+    # (r"/api/v1/entity/alias/([^/]+)", FunctionalAliasHandler),
+    # (r"/api/v1/host/([^/]+)", HostHandler),
+    (r"/api/v1/host/aliases/([^/]+)", HostAliases),
+    (r"/api/v1/metadata/(?P<class>[^\/]+)/?(?P<name>[^\/]+)?", Metadata),
+    (r"/api/v1/rundeck/resources.xml", RundeckResources),
+    ]
+
+class Application():
+    def __init__(self):
+        # Set up log file and level.
+        options.log_file_prefix = config.get('logging', 'path')
+        options.logging = config.get('logging', 'level')
+        options.log_to_stderr = config.getboolean('logging', 'stderr')
+        
+        # Port and arguments
+        port = config.get('server', 'port')
+        define('port', default=port, help='Port to be used')
+        parse_command_line()
+        
+        # Defining handlers
+        logging.info("Defining application (url, handler) pairs")
+        application = tornado.web.Application(handlers, debug=config.getboolean('tornado', 'debug'))
+        
+        # Configuring server and SSL
+        logging.info("Configuring HTTP server")
+        if (config.has_section('ssl')):
+            http_server = HTTPServer(application,
+                    ssl_options = {
+                        "certfile" : config.get('ssl', 'hostcert') ,
+                        "keyfile" : config.get('ssl', 'hostkey'),
+                        })
+        else:
+            http_server = HTTPServer(application)
+            logging.info("Host certificate undefined, SSL is DISABLED")
+            
+        # Listening port
+        http_server.listen(options.port)
+        
+        # Starting
+        logging.info("Starting application")
+        tornado.ioloop.IOLoop.instance().start()
diff --git a/dbod/tests/metadata_test.py b/dbod/tests/metadata_test.py
index 477e6b5..3bdd15b 100644
--- a/dbod/tests/metadata_test.py
+++ b/dbod/tests/metadata_test.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
 # Copyright (C) 2015, CERN
 # This software is distributed under the terms of the GNU General Public
@@ -6,31 +8,24 @@
 # granted to it by virtue of its status as Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
-import unittest
-import requests
+import tornado.web
+import json
 
+from tornado.testing import AsyncHTTPTestCase
+from tornado.testing import get_unused_port
 from timeout_decorator import timeout
 
-class Metadata(unittest.TestCase):
-    @classmethod
-    def setUpClass(self):
-        pass
-    
-    @classmethod
-    def tearDownClass(self):
-        pass
-    
-    def setUp(self):
-        pass
-    
-    def tearDown(self):
-        pass
-    
+from dbod.api.api import *
+
+class MetadataTest(AsyncHTTPTestCase):
+    def get_app(self):
+        return tornado.web.Application(handlers)
+
     @timeout(5)
     def test_single_instance_by_name(self):
-        response = requests.get("http://localhost:5443/api/v1/metadata/entity/dbod01", verify=False)
-        data = response.json()["response"]
-        self.assertEquals(response.status_code, 200)
+        response = self.fetch("/api/v1/metadata/entity/dbod01")
+        data = json.loads(response.body)["response"]
+        self.assertEquals(response.code, 200)
         self.assertEquals(len(data), 1)
         self.assertEquals(data[0]["db_name"], "dbod01")
         self.assertTrue(data[0]["volumes"] != None)
@@ -38,23 +33,23 @@ def test_single_instance_by_name(self):
     
     @timeout(5)
     def test_no_instance_by_name(self):
-        response = requests.get("http://localhost:5443/api/v1/metadata/entity/invalid", verify=False)
-        self.assertEquals(response.status_code, 404)
+        response = self.fetch("/api/v1/metadata/entity/invalid")
+        self.assertEquals(response.code, 404)
     
     @timeout(5)
     def test_single_instance_by_host(self):
-        response = requests.get("http://localhost:5443/api/v1/metadata/host/host03", verify=False)
-        data = response.json()["response"]
-        self.assertEquals(response.status_code, 200)
+        response = self.fetch("/api/v1/metadata/host/host03")
+        data = json.loads(response.body)["response"]
+        self.assertEquals(response.code, 200)
         self.assertEquals(len(data), 1)
         self.assertTrue(data[0]["volumes"] != None)
         self.assertEquals(data[0]["host"], "host03")
     
     @timeout(5)
     def test_multiple_instances_by_host(self):
-        response = requests.get("http://localhost:5443/api/v1/metadata/host/host01", verify=False)
-        data = response.json()["response"]
-        self.assertEquals(response.status_code, 200)
+        response = self.fetch("/api/v1/metadata/host/host01")
+        data = json.loads(response.body)["response"]
+        self.assertEquals(response.code, 200)
         self.assertEquals(len(data), 4)
         list = []
         for i in range(4):
@@ -66,8 +61,6 @@ def test_multiple_instances_by_host(self):
     
     @timeout(5)
     def test_no_instance_by_host(self):
-        response = requests.get("http://localhost:5443/api/v1/metadata/host/invalid", verify=False)
-        self.assertEquals(response.status_code, 404)
+        response = self.fetch("/api/v1/metadata/host/invalid")
+        self.assertEquals(response.code, 404)
     
-        
-        
\ No newline at end of file

From 2efd733c2adbc655e7b45252ac6dbf6e229337a2 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 5 Jul 2016 15:23:23 +0200
Subject: [PATCH 32/58] FIX: Temporal solution to solve the problem with
 multiple 'localhost' in Travis.

+Info: https://github.com/travis-ci/travis-ci/issues/4978
---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 3cb365e..6bb596b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,7 @@ addons:
 services:
     - postgresql
 before_install:
+    - sudo [ $(ip addr show | grep "inet6 ::1" | wc -l) -lt "1" ] && sudo sed -i '/^::1/d' /etc/hosts && sudo sed -i '/^127.0.1.1/d' /etc/hosts
     - sudo mkdir /etc/dbod
     - sudo mkdir /var/log/dbod
     - sudo chown travis /var/log/dbod

From 928299b39e3313cd2ca9ccc42b4973cc3926d084 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 5 Jul 2016 16:52:32 +0200
Subject: [PATCH 33/58] TRAVIS: The API is not needed in Travis any more.

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6bb596b..4b65559 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,6 @@ before_script:
     - psql -c 'CREATE DATABASE dbod;' -U postgres
     - psql -d dbod -f dbod/tests/db_test.sql -U postgres
     - ./postgrest postgres://postgres@localhost/dbod -a postgres -s api &
-    - bin/dbod-api &
     - sleep 5
 script:
     - curl -g http://localhost:3000

From 7f6a4a5c422160f1b664c161a6548dc9c66bd69b Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 6 Jul 2016 10:12:28 +0200
Subject: [PATCH 34/58] CONFIG: Changes in config file to use the json format
 and use authorization.

---
 static/api.cfg | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/static/api.cfg b/static/api.cfg
index 353ea56..642c348 100644
--- a/static/api.cfg
+++ b/static/api.cfg
@@ -22,8 +22,9 @@ host_aliases_url=http://localhost:3000/host_aliases
 entity_metadata_url=http://localhost:3000/metadata
 
 [rundeck]
-api_run_job = https://rundeck/api/14/job/{0}/run
-api_job_output = https://rundeck/api/14/execution/{0}/output
+api_run_job = https://rundeck/api/14/job/{0}/run?format=json
+api_job_output = https://rundeck/api/14/execution/{0}/output?format=json
+api_authorization = Basic abcdefghijklm
 
 [rundeck-jobs]
 get-snapshots = d4072a88-b7fc-4e0b-bd0a-06e2d97e16dd

From fc24ceabc6e6ef3294c451fa1257e34ce97346a7 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 6 Jul 2016 18:44:52 +0200
Subject: [PATCH 35/58] TEST: Added test for invalid class in metadata.

---
 dbod/tests/metadata_test.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dbod/tests/metadata_test.py b/dbod/tests/metadata_test.py
index 3bdd15b..0b30b79 100644
--- a/dbod/tests/metadata_test.py
+++ b/dbod/tests/metadata_test.py
@@ -63,4 +63,9 @@ def test_multiple_instances_by_host(self):
     def test_no_instance_by_host(self):
         response = self.fetch("/api/v1/metadata/host/invalid")
         self.assertEquals(response.code, 404)
+        
+    @timeout(5)
+    def test_invalid_class(self):
+        response = self.fetch("/api/v1/metadata/invalid/invalid")
+        self.assertEquals(response.code, 404)
     

From 4a7ee62f1e5d38c960991242cb5486d0dc55b01b Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 7 Jul 2016 16:32:39 +0200
Subject: [PATCH 36/58] HANDLERS: Added handlers for Rundeck jobs.

---
 dbod/api/api.py     |  3 ++-
 dbod/api/rundeck.py | 62 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/dbod/api/api.py b/dbod/api/api.py
index 6339a29..a7a369b 100644
--- a/dbod/api/api.py
+++ b/dbod/api/api.py
@@ -18,7 +18,7 @@
 from tornado.ioloop import IOLoop
 
 from dbod.api.base import DocHandler
-from dbod.api.rundeck import RundeckResources
+from dbod.api.rundeck import RundeckResources, RundeckJobs
 from dbod.api.metadata import Metadata
 from dbod.api.hostaliases import HostAliases
 from dbod.config import config
@@ -31,6 +31,7 @@
     (r"/api/v1/host/aliases/([^/]+)", HostAliases),
     (r"/api/v1/metadata/(?P<class>[^\/]+)/?(?P<name>[^\/]+)?", Metadata),
     (r"/api/v1/rundeck/resources.xml", RundeckResources),
+    (r"/api/v1/rundeck/job/(?P<job>[^\/]+)/?(?P<entity>[^\/]+)?", RundeckJobs),
     ]
 
 class Application():
diff --git a/dbod/api/rundeck.py b/dbod/api/rundeck.py
index 11ea225..edeb7f1 100644
--- a/dbod/api/rundeck.py
+++ b/dbod/api/rundeck.py
@@ -16,7 +16,9 @@
 import logging
 import json
 import requests
+import time
 
+from dbod.api.base import *
 from dbod.config import config
 
 class RundeckResources(tornado.web.RequestHandler):
@@ -53,3 +55,63 @@ def get(self):
                 raise tornado.web.HTTPError(NOT_FOUND)
         else:
             logging.error("Internal Rundeck resources endpoint not configured")
+            
+class RundeckJobs(tornado.web.RequestHandler):
+    def get(self, **args):
+        """Returns the output of a job execution"""
+        job = args.get('job')
+        response = self.__get_output__(job)
+        if response.ok:
+            self.set_header("Content-Type", 'application/json')
+            self.write({'response' : json.loads(response.text)})
+        else:
+            logging.error("Error reading the job: " + job)
+            raise tornado.web.HTTPError(response.status_code)
+
+    def post(self, **args):
+        """Executes a new Rundeck job and returns the output"""
+        job = args.get('job')
+        entity = args.get('entity')
+        response_run = self.__run_job__(job, entity)
+        if response_run.ok:
+            data = json.loads(response_run.text)
+            exid = str(data["id"])
+            timeout = 20
+            while timeout > 0:
+                response_output = self.__get_output__(exid)
+                if response_output.ok:
+                    output = json.loads(response_output.text)
+                    logging.debug(output)
+                    if output["execState"] != "running":
+                        if output["execState"] == "succeeded":
+                            self.set_header("Content-Type", 'application/json')
+                            self.write({'response' : json.loads(response_output.text)})
+                            timeout = 0
+                        else:
+                            logging.error("The job completed with errors: " + exid)
+                            raise tornado.web.HTTPError(NOT_FOUND)
+                    else:
+                        timeout -= 1
+                        time.sleep(0.500)
+                else:
+                    logging.error("Error reading the job: " + exid)
+                    raise tornado.web.HTTPError(response_output.status_code)
+        else:
+            logging.error("Error running the job: " + jobid)
+            raise tornado.web.HTTPError(response_run.status_code)
+        
+    def __get_output__(self, execution):
+        """Returns the output of a job execution"""
+        api_job_output = config.get('rundeck', 'api_job_output').format(execution)
+        return requests.get(api_job_output, headers={'Authorization': config.get('rundeck', 'api_authorization')}, verify=False)
+            
+    def __run_job__(self, job, entity):
+        """Executes a new Rundeck job and returns the output"""
+        jobid = config.get('rundeck-jobs', job)
+        if jobid:
+            run_job_url = config.get('rundeck', 'api_run_job').format(jobid)
+            return requests.post(run_job_url, headers={'Authorization': config.get('rundeck', 'api_authorization')}, verify=False, data = {'filter':'name: ' + entity})
+        
+        
+        
+        

From c10aa09e9738c920336a9e2cc9335f09242d4a2d Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 19 Jul 2016 15:18:29 +0200
Subject: [PATCH 37/58] Removed unneeded 'psycopg2' dependency.

---
 requirements.txt | 1 -
 setup.py         | 1 -
 2 files changed, 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 842c9e5..e3e6942 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
 configparser
 nose
-psycopg2
 tornado==4.2
 virtualenv
 requests
diff --git a/setup.py b/setup.py
index 3c4df63..95907d6 100644
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,6 @@
       requires=[
           'ConfigParser',
           'tornado',
-          'psycopg2',
           'nose',
           'mock',
           'requests',

From 6db4b370b73cb68baab661bebb4920c6149e6bf3 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 20 Jul 2016 18:44:24 +0200
Subject: [PATCH 38/58] SQL: Added instances with several volumes and some PKs.

---
 dbod/tests/db_test.sql | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/dbod/tests/db_test.sql b/dbod/tests/db_test.sql
index e64f6ce..bfb30fd 100644
--- a/dbod/tests/db_test.sql
+++ b/dbod/tests/db_test.sql
@@ -9,6 +9,8 @@
 -- Create the structure for the test database --
 ------------------------------------------------
 
+CREATE SCHEMA public;
+
 -- DOD_COMMAND_DEFINITION
 CREATE TABLE public.dod_command_definition (
     command_name varchar(64) NOT NULL,
@@ -28,7 +30,7 @@ CREATE TABLE public.dod_command_params (
     name varchar(64) NOT NULL,
     value text,
     category varchar(20),
-    PRIMARY KEY (db_name)
+    PRIMARY KEY (username, db_name, command_name, type, creation_date, name)
 );
 
 -- DOD_INSTANCE_CHANGES
@@ -40,11 +42,12 @@ CREATE TABLE public.dod_instance_changes (
     requester varchar(32) NOT NULL,
     old_value varchar(1024),
     new_value varchar(1024),
-    PRIMARY KEY (db_name)
+    PRIMARY KEY (username, db_name, attribute, change_date)
 );
 
 -- DOD_INSTANCES
 CREATE TABLE public.dod_instances (
+    id serial,
     username varchar(32) NOT NULL,
     db_name varchar(128) NOT NULL,
     e_group varchar(256),
@@ -52,7 +55,7 @@ CREATE TABLE public.dod_instances (
     creation_date date NOT NULL,
     expiry_date date,
     db_type varchar(32) NOT NULL,
-    db_size int NOT NULL,
+    db_size int,
     no_connections int,
     project varchar(128),
     description varchar(1024),
@@ -62,7 +65,6 @@ CREATE TABLE public.dod_instances (
     host varchar(128),
     state varchar(32),
     status varchar(32),
-    id int,
     PRIMARY KEY (id)
 );
 
@@ -102,14 +104,16 @@ CREATE TABLE public.volume (
     vgroup varchar(32) NOT NULL,
     server varchar(63) NOT NULL,
     mount_options varchar(256) NOT NULL,
-    mounting_path varchar(256) NOT NULL
+    mounting_path varchar(256) NOT NULL,
+    PRIMARY KEY (id)
 );
 
 -- HOST
 CREATE TABLE public.host (
     id serial,
     name varchar(63) NOT NULL,
-    memory integer NOT NULL
+    memory integer NOT NULL,
+    PRIMARY KEY (id)
 );
 
 -- ATTRIBUTE
@@ -117,12 +121,12 @@ CREATE TABLE public.attribute (
     id serial,
     instance_id integer NOT NULL,
     name varchar(32) NOT NULL,
-    value varchar(250) NOT NULL
+    value varchar(250) NOT NULL,
+    PRIMARY KEY (id)
 );
 
 -- FUNCTIONAL ALIASES
-CREATE TABLE public.functional_aliases
-(
+CREATE TABLE public.functional_aliases (
     dns_name character varying(256) NOT NULL,
     db_name character varying(8),
     alias character varying(256),
@@ -140,9 +144,11 @@ VALUES ('user01', 'dbod01', 'testgroupA', 'TEST', now(), NULL, 'MYSQL', 100, 100
 -- Insert test data for volumes
 INSERT INTO public.volume (instance_id, file_mode, owner, vgroup, server, mount_options, mounting_path)
 VALUES (1, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data1'),
+       (1, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw', '/MNT/bin'),
        (2, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data2'),
        (4, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard,tcp', '/MNT/data4'),
-       (5, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data5');
+       (5, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data5'),
+       (5, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw', '/MNT/bin');
 
 -- Insert test data for attributes
 INSERT INTO public.attribute (instance_id, name, value)

From a4e105f7bc86245e0adc96dff7df02dde67c2af0 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 22 Jul 2016 16:02:19 +0200
Subject: [PATCH 39/58] CONFIG: Added instance PostgREST url.

---
 static/api.cfg | 1 +
 1 file changed, 1 insertion(+)

diff --git a/static/api.cfg b/static/api.cfg
index 642c348..b86e988 100644
--- a/static/api.cfg
+++ b/static/api.cfg
@@ -20,6 +20,7 @@ pass=api-password
 rundeck_resources_url=http://localhost:3000/rundeck_instances
 host_aliases_url=http://localhost:3000/host_aliases
 entity_metadata_url=http://localhost:3000/metadata
+instance_url=http://localhost:3000/instance
 
 [rundeck]
 api_run_job = https://rundeck/api/14/job/{0}/run?format=json

From 88a3bb7c0f43262858553e885063da1083859302 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Mon, 25 Jul 2016 10:43:15 +0200
Subject: [PATCH 40/58] SQL: Added constraints and more test data to the test
 sql file.

---
 dbod/tests/db_test.sql | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/dbod/tests/db_test.sql b/dbod/tests/db_test.sql
index bfb30fd..6fdc782 100644
--- a/dbod/tests/db_test.sql
+++ b/dbod/tests/db_test.sql
@@ -65,7 +65,8 @@ CREATE TABLE public.dod_instances (
     host varchar(128),
     state varchar(32),
     status varchar(32),
-    PRIMARY KEY (id)
+    CONSTRAINT dod_instances_pkey PRIMARY KEY (id),
+    CONSTRAINT dod_instances_dbname UNIQUE (db_name)
 );
 
 -- DOD_JOBS
@@ -134,12 +135,12 @@ CREATE TABLE public.functional_aliases (
 );
 
 -- Insert test data for instances
-INSERT INTO public.dod_instances (username, db_name, e_group, category, creation_date, expiry_date, db_type, db_size, no_connections, project, description, version, master, slave, host, state, status, id)
-VALUES ('user01', 'dbod01', 'testgroupA', 'TEST', now(), NULL, 'MYSQL', 100, 100, 'API', 'Test instance 1', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1, 1),
-       ('user01', 'dbod02', 'testgroupB', 'PROD', now(), NULL, 'PG', 50, 500, 'API', 'Test instance 2', '9.4.4', NULL, NULL, 'host03', 'RUNNING', 1, 2),
-       ('user02', 'dbod03', 'testgroupB', 'TEST', now(), NULL, 'MYSQL', 100, 200, 'WEB', 'Expired instance 1', '5.5', NULL, NULL, 'host01', 'RUNNING', 0, 3),
-       ('user03', 'dbod04', 'testgroupA', 'PROD', now(), NULL, 'PG', 250, 10, 'LCC', 'Test instance 3', '9.4.5', NULL, NULL, 'host01', 'RUNNING', 1, 4),
-       ('user04', 'dbod05', 'testgroupC', 'TEST', now(), NULL, 'MYSQL', 300, 200, 'WEB', 'Test instance 4', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1, 5);
+INSERT INTO public.dod_instances (username, db_name, e_group, category, creation_date, expiry_date, db_type, db_size, no_connections, project, description, version, master, slave, host, state, status)
+VALUES ('user01', 'dbod01', 'testgroupA', 'TEST', now(), NULL, 'MYSQL', 100, 100, 'API', 'Test instance 1', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1),
+       ('user01', 'dbod02', 'testgroupB', 'PROD', now(), NULL, 'PG', 50, 500, 'API', 'Test instance 2', '9.4.4', NULL, NULL, 'host03', 'RUNNING', 1),
+       ('user02', 'dbod03', 'testgroupB', 'TEST', now(), NULL, 'MYSQL', 100, 200, 'WEB', 'Expired instance 1', '5.5', NULL, NULL, 'host01', 'RUNNING', 0),
+       ('user03', 'dbod04', 'testgroupA', 'PROD', now(), NULL, 'PG', 250, 10, 'LCC', 'Test instance 3', '9.4.5', NULL, NULL, 'host01', 'RUNNING', 1),
+       ('user04', 'dbod05', 'testgroupC', 'TEST', now(), NULL, 'MYSQL', 300, 200, 'WEB', 'Test instance 4', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1);
        
 -- Insert test data for volumes
 INSERT INTO public.volume (instance_id, file_mode, owner, vgroup, server, mount_options, mounting_path)
@@ -167,6 +168,11 @@ VALUES ('host01', 12),
        
 -- Schema API
 CREATE SCHEMA api;
+
+-- Dod_instances view
+CREATE OR REPLACE VIEW api.dod_instances AS
+SELECT id, username, db_name, e_group, category, creation_date, expiry_date, db_type, db_size, no_connections, project, description, version, master, slave, host, state, status
+FROM dod_instances;
        
 -- Job stats view
 CREATE OR REPLACE VIEW api.job_stats AS 

From 280addcb5c7ab88700bd2577db2b86b3c9f94e7d Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 27 Jul 2016 18:31:02 +0200
Subject: [PATCH 41/58] HANDLERS: Added entity handler.

---
 dbod/api/api.py    |  4 +-
 dbod/api/entity.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+), 2 deletions(-)
 create mode 100644 dbod/api/entity.py

diff --git a/dbod/api/api.py b/dbod/api/api.py
index a7a369b..9f248bc 100644
--- a/dbod/api/api.py
+++ b/dbod/api/api.py
@@ -21,13 +21,13 @@
 from dbod.api.rundeck import RundeckResources, RundeckJobs
 from dbod.api.metadata import Metadata
 from dbod.api.hostaliases import HostAliases
+from dbod.api.entity import Entity
 from dbod.config import config
 
 handlers = [
     (r"/", DocHandler),
-    # (r"/api/v1/entity/([^/]+)", EntityHandler),
     # (r"/api/v1/entity/alias/([^/]+)", FunctionalAliasHandler),
-    # (r"/api/v1/host/([^/]+)", HostHandler),
+    (r"/api/v1/entity/([^/]+)", Entity),
     (r"/api/v1/host/aliases/([^/]+)", HostAliases),
     (r"/api/v1/metadata/(?P<class>[^\/]+)/?(?P<name>[^\/]+)?", Metadata),
     (r"/api/v1/rundeck/resources.xml", RundeckResources),
diff --git a/dbod/api/entity.py b/dbod/api/entity.py
new file mode 100644
index 0000000..de01980
--- /dev/null
+++ b/dbod/api/entity.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+"""
+REST API Server for the DB On Demand System
+"""
+
+import tornado.web
+import logging
+import requests
+import json
+
+from dbod.api.base import *
+from dbod.config import config
+
+class Entity(tornado.web.RequestHandler):
+    def post(self, instance):
+        """Inserts a new instance in the database"""
+        entity = json.loads(self.request.body)
+        
+        # Get the port
+        port = entity["port"]
+        del entity["port"]
+        
+        # Get the volumes
+        volumes = entity["volumes"]
+        del entity["volumes"]
+        
+        # Insert the entity in database using PostREST
+        response = requests.post("http://localhost:3000/instance", json=entity, headers={'Prefer': 'return=representation'})
+        if response.ok:
+            entid = json.loads(response.text)["id"]
+            logging.debug("Created entity with id: " + str(entid))
+        
+            # Add entity id to volumes
+            for volume in volumes:
+                volume["instance_id"] = entid
+
+            # Insert the volumes in database using PostREST
+            response = requests.post("http://localhost:3000/volume", json=volumes)
+            if response.ok:
+                response = requests.post("http://localhost:3000/attribute", json={'instance_id': entid, 'name': 'port', 'value': port})
+                if response.ok:
+                    self.set_status(CREATED)
+                else:
+                    logging.error("Error inserting the port attribute: " + response.text)
+                    self.__delete_instance__(entid)
+                    raise tornado.web.HTTPError(response.status_code)
+            else:
+                logging.error("Error creating the volumes: " + response.text)
+                self.__delete_instance__(entid)
+                raise tornado.web.HTTPError(response.status_code)
+        else:
+            logging.error("Error creating the entity: " + response.text)
+            raise tornado.web.HTTPError(response.status_code)
+            
+    def put(self, instance):
+        """Updates an instance"""
+        entity = json.loads(self.request.body)
+        logging.debug(entity)
+        response = requests.patch("http://localhost:3000/instance?db_name=eq." + instance, json=entity)
+        if response.ok:
+            self.set_status(response.status_code)
+        else:
+            logging.error("Instance not found: " + instance)
+            raise tornado.web.HTTPError(response.status_code)
+            
+    def delete(self, instance):
+        """Deletes an instance by name"""
+        response = requests.get("http://localhost:3000/instance?db_name=eq." + instance)
+        if response.ok:
+            entid = json.loads(response.text)[0]["id"]
+            logging.debug("Deleting instance id: " + str(entid))
+            __delete_instance__(entid)
+            self.set_status(response.status_code)
+        else:
+            logging.error("Instance not found: " + instance)
+            raise tornado.web.HTTPError(response.status_code)
+            
+    def __delete_instance__(self, inst_id):
+        requests.delete("http://localhost:3000/attribute?instance_id=eq." + str(inst_id))
+        requests.delete("http://localhost:3000/volume?instance_id=eq." + str(inst_id))
+        requests.delete("http://localhost:3000/instance?id=eq." + str(inst_id))
+        
+

From 52e8e8ae581d0d1e5b3107779ec62f4340cbb0f3 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 27 Jul 2016 18:49:20 +0200
Subject: [PATCH 42/58] HANDLERS: Fixed call to delete instance.

---
 dbod/api/entity.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dbod/api/entity.py b/dbod/api/entity.py
index de01980..385cee2 100644
--- a/dbod/api/entity.py
+++ b/dbod/api/entity.py
@@ -78,8 +78,8 @@ def delete(self, instance):
         if response.ok:
             entid = json.loads(response.text)[0]["id"]
             logging.debug("Deleting instance id: " + str(entid))
-            __delete_instance__(entid)
-            self.set_status(response.status_code)
+            self.__delete_instance__(entid)
+            self.set_status(204)
         else:
             logging.error("Instance not found: " + instance)
             raise tornado.web.HTTPError(response.status_code)

From f9ad7bd4838f99204cbace265da9ffe6cf2231cf Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 28 Jul 2016 09:50:10 +0200
Subject: [PATCH 43/58] TESTS: Instance creation test.

---
 dbod/tests/entity_test.py | 63 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 dbod/tests/entity_test.py

diff --git a/dbod/tests/entity_test.py b/dbod/tests/entity_test.py
new file mode 100644
index 0000000..08d5d9b
--- /dev/null
+++ b/dbod/tests/entity_test.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+import tornado.web
+import json
+import urllib
+import logging
+
+from tornado.testing import AsyncHTTPTestCase
+from tornado.testing import get_unused_port
+from timeout_decorator import timeout
+
+from dbod.api.api import *
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+class EntityTest(AsyncHTTPTestCase):
+    def get_app(self):
+        return tornado.web.Application(handlers, debug=True)
+
+    @timeout(5)
+    def test_create_entity(self):
+        response = self.fetch("/api/v1/entity/testdb", method='DELETE')
+        
+        entity = """{
+        "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
+        "version": "5.6.17", "db_type": "MYSQL", "port": "5505", "host": "testhost", "db_name": "testdb", 
+        "volumes": [
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/data1"}, 
+            {"vgroup": "ownergroup", "file_mode": "0755", 
+            "server": "NAS-server", "mount_options": "rw,bg,hard", "owner": "TSM", "mounting_path": "/MNT/bin"}
+        ]}"""
+        
+        # Create the instance
+        response = self.fetch("/api/v1/entity/create", method='POST', body=entity)
+        self.assertEquals(response.code, 201)
+        
+        # Check the metadata for this new instance
+        response = self.fetch("/api/v1/metadata/entity/testdb")
+        self.assertEquals(response.code, 200)
+        data = json.loads(response.body)["response"]
+        self.assertEquals(data[0]["db_name"], "testdb")
+        self.assertEquals(len(data[0]["volumes"]), 2)
+        self.assertEquals(data[0]["port"], "5505")  # Reminder: the port is saved as a String in DB
+        
+        # Delete the created instance
+        response = self.fetch("/api/v1/entity/testdb", method='DELETE')
+        self.assertEquals(response.code, 204)
+        
+        # Check again, the metadata should be empty
+        response = self.fetch("/api/v1/metadata/entity/testdb")
+        self.assertEquals(response.code, 404)
+    
+    
+    

From 0823efabcf13b02200a754cff65f51c23f48c280 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Thu, 28 Jul 2016 13:55:54 +0200
Subject: [PATCH 44/58] HANDLERS: Properly modification of port when the field
 was set in the requet.

---
 dbod/api/entity.py | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/dbod/api/entity.py b/dbod/api/entity.py
index 385cee2..e06c1f3 100644
--- a/dbod/api/entity.py
+++ b/dbod/api/entity.py
@@ -65,18 +65,39 @@ def put(self, instance):
         """Updates an instance"""
         entity = json.loads(self.request.body)
         logging.debug(entity)
-        response = requests.patch("http://localhost:3000/instance?db_name=eq." + instance, json=entity)
-        if response.ok:
-            self.set_status(response.status_code)
+        entid = self.__get_instance_id__(instance)
+        
+        # Check if the port is changed
+        if "port" in entity:
+            port = {"value":entity["port"]}
+            del entity["port"]
+            response = requests.patch("http://localhost:3000/attribute?instance_id=eq." + str(entid) + "&name=eq.port", json=port)
+            if response.ok:
+                self.set_status(response.status_code)
+            else:
+                logging.error("Error updating port on instance: " + instance)
+                raise tornado.web.HTTPError(response.status_code)
+        
+        # Check if the volumes are changed
+        if "volumes" in entity:
+            volumes = entity["volumes"]
+            del entity["volumes"]
+            logging.debug("Cambiar volumes a: " + str(volumes))
+        
+        if entity:
+            response = requests.patch("http://localhost:3000/instance?db_name=eq." + instance, json=entity)
+            if response.ok:
+                self.set_status(response.status_code)
+            else:
+                logging.error("Instance not found: " + instance)
+                raise tornado.web.HTTPError(response.status_code)
         else:
-            logging.error("Instance not found: " + instance)
-            raise tornado.web.HTTPError(response.status_code)
+            self.set_status(NO_CONTENT)
             
     def delete(self, instance):
         """Deletes an instance by name"""
-        response = requests.get("http://localhost:3000/instance?db_name=eq." + instance)
-        if response.ok:
-            entid = json.loads(response.text)[0]["id"]
+        entid = self.__get_instance_id__(instance)
+        if entid:
             logging.debug("Deleting instance id: " + str(entid))
             self.__delete_instance__(entid)
             self.set_status(204)
@@ -84,6 +105,13 @@ def delete(self, instance):
             logging.error("Instance not found: " + instance)
             raise tornado.web.HTTPError(response.status_code)
             
+    def __get_instance_id__(self, instance):
+        response = requests.get("http://localhost:3000/instance?db_name=eq." + instance)
+        if response.ok:
+            return json.loads(response.text)[0]["id"]
+        else:
+            return None
+            
     def __delete_instance__(self, inst_id):
         requests.delete("http://localhost:3000/attribute?instance_id=eq." + str(inst_id))
         requests.delete("http://localhost:3000/volume?instance_id=eq." + str(inst_id))

From 3694d8eaddcccdb18d6883db91f9e612e56b8261 Mon Sep 17 00:00:00 2001
From: Yiannis Sotiropoulos <yianni.sotiro@cern.ch>
Date: Mon, 1 Aug 2016 15:13:13 +0200
Subject: [PATCH 45/58] HANDLERS: Added handler and tests for functional
 aliases

---
 dbod/api/api.py                    |   6 +-
 dbod/api/base.py                   |   1 +
 dbod/api/functionalalias.py        | 188 +++++++++++++++++++
 dbod/tests/db_test_isotirop.sql    | 280 +++++++++++++++++++++++++++++
 dbod/tests/functionalalias_test.py | 137 ++++++++++++++
 5 files changed, 610 insertions(+), 2 deletions(-)
 create mode 100644 dbod/api/functionalalias.py
 create mode 100644 dbod/tests/db_test_isotirop.sql
 create mode 100644 dbod/tests/functionalalias_test.py

diff --git a/dbod/api/api.py b/dbod/api/api.py
index a7a369b..ad8a537 100644
--- a/dbod/api/api.py
+++ b/dbod/api/api.py
@@ -21,12 +21,14 @@
 from dbod.api.rundeck import RundeckResources, RundeckJobs
 from dbod.api.metadata import Metadata
 from dbod.api.hostaliases import HostAliases
+from dbod.api.entity import Entity
+from dbod.api.functionalalias import FunctionalAliasHandler
 from dbod.config import config
 
 handlers = [
     (r"/", DocHandler),
-    # (r"/api/v1/entity/([^/]+)", EntityHandler),
-    # (r"/api/v1/entity/alias/([^/]+)", FunctionalAliasHandler),
+    #(r"/api/v1/entity/([^/]+)", Entity),
+    (r"/api/v1/entity/alias/([^/]*)", FunctionalAliasHandler),
     # (r"/api/v1/host/([^/]+)", HostHandler),
     (r"/api/v1/host/aliases/([^/]+)", HostAliases),
     (r"/api/v1/metadata/(?P<class>[^\/]+)/?(?P<name>[^\/]+)?", Metadata),
diff --git a/dbod/api/base.py b/dbod/api/base.py
index 174c358..126564b 100644
--- a/dbod/api/base.py
+++ b/dbod/api/base.py
@@ -25,6 +25,7 @@
 NO_CONTENT = 204 # Succesfull delete
 NOT_FOUND = 404
 UNAUTHORIZED = 401
+BAD_REQUEST = 400
 
 # Basic HTTP Authentication decorator
 def http_basic_auth(fun):
diff --git a/dbod/api/functionalalias.py b/dbod/api/functionalalias.py
new file mode 100644
index 0000000..eb21694
--- /dev/null
+++ b/dbod/api/functionalalias.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+"""
+REST API Server for the DB On Demand System
+"""
+
+import logging
+import json
+from ast import literal_eval
+from sys import exc_info
+import requests
+import tornado.web
+import tornado.escape
+from dbod.api.base import NOT_FOUND, BAD_REQUEST
+from dbod.config import config
+
+class FunctionalAliasHandler(tornado.web.RequestHandler):
+    '''The handler for the entity/alias/<endpoint>'''
+    url = config.get('postgrest', 'functional_alias_url')
+    if not url:
+        logging.error("Internal entity/alias endpoint not configured")
+        raise tornado.web.HTTPError(NOT_FOUND)
+
+    def data_received(self, *arg, **kwargs):
+        '''Abstract method which handles streamed request data.'''
+        #No need for implementation
+        pass
+
+    def get(self, db_name, *args):
+        '''Returns db_name's alias and dns name'''
+        #self.set_header('Content-Type', 'application/json')
+        
+        logging.debug(args)
+        logging.debug('Arguments:' + str(self.request.arguments))
+        composed_url = self.url + '?db_name=eq.' + db_name + '&select=dns_name,alias'
+        logging.debug('Requesting ' + composed_url)
+        response = requests.get(composed_url)
+        if response.ok:
+            data = response.json()
+            if data:
+                self.write({'response' : data})
+                self.set_status(response.status_code)
+            else:
+                logging.error("Functional alias not found")
+                raise tornado.web.HTTPError(NOT_FOUND)
+        else:
+            logging.error("Error fetching functional alias of: " + db_name)
+            raise tornado.web.HTTPError(response.status_code)
+
+
+    def post(self, *args):
+        '''Updates a row with db_name and the alias. The dns_name is already there.'''
+
+        def next_dnsname():
+            '''Returns the next dnsname which can be used for a newly created
+        instance, if any'''
+            
+            #self.set_header('Content-Type', 'application/json')
+            #LIMIT is not working in postgrest but it uses some headers for that as well
+            headers = {'Range-Unit': 'items', 'Range': '0-0'}
+            # select the next available dns_name with db_name and alias assigned to NULL
+            query_select = '?select=dns_name&order=dns_name.asc&'
+            query_filter = 'db_name=is.null&alias=is.null&dns_name=isnot.null'
+            composed_url = self.url + query_select + query_filter
+            try:
+                response_dns = requests.get(composed_url, headers=headers)
+                response_dns_dict = literal_eval(response_dns.text)[0]
+                if response_dns.ok:
+                    return response_dns_dict['dns_name']
+                else:
+                    return None
+            except:
+                error_msg = exc_info()[0]
+                logging.error(error_msg)
+                return None
+            
+
+        def check_if_exists(column, value):
+            '''It checks if the inserted data already exists in the functional_aliases table'''
+            query = '?%s=eq.%s&select=%s' %(column, value, column)
+            composed_url = self.url + query
+            response = requests.get(composed_url)
+            if response.ok and literal_eval(response.text):
+                logging.error("A(n) %s with the value of %s already exists in the functional_aliases table" %(column,value))
+                return True
+            else:
+                
+                return False
+        #self.set_header("Content-Type", 'application/json')
+        self.set_header('Prefer', 'return=representation')
+        logging.debug(args)
+        logging.debug('Arguments:' + str(self.request.arguments))
+        try:
+            functional_alias = json.loads(self.get_argument('functional_alias'))
+            logging.debug(str(len(functional_alias)) + " Argument(s) given:")
+            logging.debug(functional_alias)
+        except:
+            logging.error("Argument not recognized or not defined.")
+            logging.error("Try adding header 'Content-Type:application/x-www-form-urlencoded'")
+            logging.error("The right format should be: functional_alias={'<db_name>':'<alias>'}")
+            raise tornado.web.HTTPError(NOT_FOUND)
+
+        dns_name = next_dnsname()
+        logging.info("dns_name picked: " + str(dns_name))
+
+        if dns_name:
+            logging.debug("dns_name picked: " + str(dns_name))
+            headers = {'Prefer': 'return=representation'}
+            db_name = str(functional_alias.keys()[0])
+            alias = str(functional_alias.values()[0])
+            insert_data = {"db_name": db_name, 
+                           "alias": alias}
+            logging.debug("Data to insert: " + str(insert_data))
+
+            #if not check_if_exists('db_name', db_name) and  not check_if_exists('alias', alias):
+            composed_url = self.url + '?dns_name=eq.' + dns_name
+            logging.debug('Requesting insertion: ' + composed_url)
+            
+            response = requests.patch(composed_url, json=insert_data, headers=headers)
+        
+            if response.ok:
+                data = response.json()
+                #if data:
+                logging.info('Success. Data inserted in the functional_aliases table:')
+                logging.info(data)
+                self.set_status(response.status_code)
+                #else:
+                #    logging.error("Empty data")
+                #    raise tornado.web.HTTPError(response.status_code)  
+            else:
+                logging.error("Unsuccessful insertion")
+                self.set_status(response.status_code)
+                    
+        else:
+            logging.error("No dns_name available in the functional_aliases table")
+            raise tornado.web.HTTPError(BAD_REQUEST)
+
+    def delete(self, db_name, *args):
+        '''Deletes or else asssigns to NULL the db_name and alias fields.'''
+        """Removes the functional alias association for an entity.
+           If the functional alias doesn't exist it doesn't do anything"""
+
+        def get_dns(db_name):
+            '''Get the dns_name given the db_name'''
+            composed_url = self.url + '?db_name=eq.' + db_name + '&select=dns_name'
+            response = requests.get(composed_url)
+            if response.ok:
+                try:
+                    dns_name_dict = literal_eval(response.text)[0]
+                    return dns_name_dict['dns_name']
+                except IndexError:
+                    return None
+            else: 
+                return None
+
+
+        logging.debug(args)
+        logging.debug('Arguments:' + str(self.request.arguments))
+
+        dns_name = get_dns(db_name)
+        logging.debug(dns_name)
+        if dns_name:
+            headers = {'Prefer': 'return=representation', 'Content-Type': 'application/json'}
+            composed_url = self.url + '?dns_name=eq.' + dns_name
+            logging.debug('Requesting deletion: ' + composed_url)
+            delete_data = '{"db_name": null, "alias": null}'
+            logging.debug("dns_name to be remained: " + dns_name)
+            response = requests.patch(composed_url, json=json.loads(delete_data), headers=headers)
+
+            if response.ok:
+                data = response.json()
+                logging.info("Delete success of: " + str(data))
+                self.set_status(response.status_code)
+            else:
+                logging.error("Unsuccessful deletion")
+                raise tornado.web.HTTPError(response.status_code)
+
+        else:
+            logging.info("db_name not found. Nothing to do")
+            self.set_status(BAD_REQUEST)
diff --git a/dbod/tests/db_test_isotirop.sql b/dbod/tests/db_test_isotirop.sql
new file mode 100644
index 0000000..8ec0643
--- /dev/null
+++ b/dbod/tests/db_test_isotirop.sql
@@ -0,0 +1,280 @@
+-- Copyright (C) 2015, CERN
+-- This software is distributed under the terms of the GNU General Public
+-- Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+-- In applying this license, CERN does not waive the privileges and immunities
+-- granted to it by virtue of its status as Intergovernmental Organization
+-- or submit itself to any jurisdiction.
+
+------------------------------------------------
+-- Create the structure for the test database --
+------------------------------------------------
+
+-- DOD_COMMAND_DEFINITION
+CREATE TABLE public.dod_command_definition (
+    command_name varchar(64) NOT NULL,
+    type varchar(64) NOT NULL,
+    exec varchar(2048),
+    category varchar(20),
+    PRIMARY KEY (command_name, type, category)
+);
+
+-- DOD_COMMAND_PARAMS
+CREATE TABLE public.dod_command_params (
+    username varchar(32) NOT NULL,
+    db_name varchar(128) NOT NULL,
+    command_name varchar(64) NOT NULL,
+    type varchar(64) NOT NULL,
+    creation_date date NOT NULL,
+    name varchar(64) NOT NULL,
+    value text,
+    category varchar(20),
+    PRIMARY KEY (db_name)
+);
+
+-- DOD_INSTANCE_CHANGES
+CREATE TABLE public.dod_instance_changes (
+    username varchar(32) NOT NULL,
+    db_name varchar(128) NOT NULL,
+    attribute varchar(32) NOT NULL,
+    change_date date NOT NULL,
+    requester varchar(32) NOT NULL,
+    old_value varchar(1024),
+    new_value varchar(1024),
+    PRIMARY KEY (db_name)
+);
+
+-- DOD_INSTANCES
+CREATE TABLE public.dod_instances (
+    username varchar(32) NOT NULL,
+    db_name varchar(128) NOT NULL,
+    e_group varchar(256),
+    category varchar(32) NOT NULL,
+    creation_date date NOT NULL,
+    expiry_date date,
+    db_type varchar(32) NOT NULL,
+    db_size int NOT NULL,
+    no_connections int,
+    project varchar(128),
+    description varchar(1024),
+    version varchar(128),
+    master varchar(32),
+    slave varchar(32),
+    host varchar(128),
+    state varchar(32),
+    status varchar(32),
+    id int,
+    PRIMARY KEY (id)
+);
+
+-- DOD_JOBS
+CREATE TABLE public.dod_jobs (
+    username varchar(32) NOT NULL,
+    db_name varchar(128) NOT NULL,
+    command_name varchar(64) NOT NULL,
+    type varchar(64) NOT NULL,
+    creation_date date NOT NULL,
+    completion_date date,
+    requester varchar(32) NOT NULL,
+    admin_action int NOT NULL,
+    state varchar(32) NOT NULL,
+    log text,
+    result varchar(2048),
+    email_sent date,
+    category varchar(20),
+    PRIMARY KEY (username, db_name, command_name, type, creation_date)
+);
+
+-- DOD_UPGRADES
+CREATE TABLE public.dod_upgrades (
+    db_type varchar(32) NOT NULL,
+    category varchar(32) NOT NULL,
+    version_from varchar(128) NOT NULL,
+    version_to varchar(128) NOT NULL,
+    PRIMARY KEY (db_type, category, version_from)
+);
+
+-- VOLUME
+CREATE TABLE public.volume (
+    id serial,
+    instance_id integer NOT NULL,
+    file_mode char(4) NOT NULL,
+    owner varchar(32) NOT NULL,
+    vgroup varchar(32) NOT NULL,
+    server varchar(63) NOT NULL,
+    mount_options varchar(256) NOT NULL,
+    mounting_path varchar(256) NOT NULL
+);
+
+-- HOST
+CREATE TABLE public.host (
+    id serial,
+    name varchar(63) NOT NULL,
+    memory integer NOT NULL
+);
+
+-- ATTRIBUTE
+CREATE TABLE public.attribute (
+    id serial,
+    instance_id integer NOT NULL,
+    name varchar(32) NOT NULL,
+    value varchar(250) NOT NULL
+);
+
+-- FUNCTIONAL ALIASES
+CREATE TABLE public.functional_aliases
+(
+    dns_name character varying(256) NOT NULL,
+    db_name character varying(8),
+    alias character varying(256),
+    CONSTRAINT functional_aliases_pkey PRIMARY KEY (dns_name)
+    CONSTRAINT db_name_con UNIQUE (db_name)
+);
+
+-- Insert test data for instances
+INSERT INTO public.dod_instances (username, db_name, e_group, category, creation_date, expiry_date, db_type, db_size, no_connections, project, description, version, master, slave, host, state, status, id)
+VALUES ('user01', 'dbod01', 'testgroupA', 'TEST', now(), NULL, 'MYSQL', 100, 100, 'API', 'Test instance 1', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1, 1),
+       ('user01', 'dbod02', 'testgroupB', 'PROD', now(), NULL, 'PG', 50, 500, 'API', 'Test instance 2', '9.4.4', NULL, NULL, 'host03', 'RUNNING', 1, 2),
+       ('user02', 'dbod03', 'testgroupB', 'TEST', now(), NULL, 'MYSQL', 100, 200, 'WEB', 'Expired instance 1', '5.5', NULL, NULL, 'host01', 'RUNNING', 0, 3),
+       ('user03', 'dbod04', 'testgroupA', 'PROD', now(), NULL, 'PG', 250, 10, 'LCC', 'Test instance 3', '9.4.5', NULL, NULL, 'host01', 'RUNNING', 1, 4),
+       ('user04', 'dbod05', 'testgroupC', 'TEST', now(), NULL, 'MYSQL', 300, 200, 'WEB', 'Test instance 4', '5.6.17', NULL, NULL, 'host01', 'RUNNING', 1, 5);
+       
+-- Insert test data for volumes
+INSERT INTO public.volume (instance_id, file_mode, owner, vgroup, server, mount_options, mounting_path)
+VALUES (1, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data1'),
+       (2, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data2'),
+       (4, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard,tcp', '/MNT/data4'),
+       (5, '0755', 'TSM', 'ownergroup', 'NAS-server', 'rw,bg,hard', '/MNT/data5');
+
+-- Insert test data for attributes
+INSERT INTO public.attribute (instance_id, name, value)
+VALUES (1, 'port', '5501'),
+       (2, 'port', '6603'),
+       (3, 'port', '5510'),
+       (4, 'port', '6601'),
+       (5, 'port', '5500');
+        
+-- Insert test data for hosts
+INSERT INTO public.host (name, memory)
+VALUES ('host01', 12),
+       ('host02', 24),
+       ('host03', 64),
+       ('host04', 256);
+       
+INSERT INTO public.functional_aliases (dns_name, db_name, alias)
+VALUES ('db-dbod-dns01','dbod_01','dbod-dbod-01.cern.ch'),
+       ('db-dbod-dns02','dbod_02','dbod-dbod-02.cern.ch'),
+       ('db-dbod-dns03','dbod_03','dbod-dbod-03.cern.ch'),
+       ('db-dbod-dns04','dbod_04','dbod-dbod-04.cern.ch'),
+       ('db-dbod-dns05', NULL, NULL),
+       ('db-dbod-dns06', NULL, NULL),
+       ('db-dbod-dns07', NULL, NULL),
+       ('db-dbod-dns08', NULL, NULL);
+
+-- Schema API
+CREATE SCHEMA api;
+       
+-- Job stats view
+CREATE OR REPLACE VIEW api.job_stats AS 
+SELECT db_name, command_name, COUNT(*) as COUNT, ROUND(AVG(completion_date - creation_date) * 24*60*60) AS mean_duration
+FROM dod_jobs GROUP BY command_name, db_name;
+
+-- Command stats view
+CREATE OR REPLACE VIEW api.command_stats AS
+SELECT command_name, COUNT(*) AS COUNT, ROUND(AVG(completion_date - creation_date) * 24*60*60) AS mean_duration
+FROM dod_jobs GROUP BY command_name;
+       
+-- Get hosts function
+CREATE OR REPLACE FUNCTION get_hosts(host_ids INTEGER[])
+RETURNS VARCHAR[] AS $$
+DECLARE
+  hosts VARCHAR := '';
+BEGIN
+  SELECT ARRAY (SELECT name FROM host WHERE id = ANY(host_ids)) INTO hosts;
+  RETURN hosts;
+END
+$$ LANGUAGE plpgsql;
+
+-- Get volumes function
+CREATE OR REPLACE FUNCTION get_volumes(pid INTEGER)
+RETURNS JSON[] AS $$
+DECLARE
+  volumes JSON[];
+BEGIN
+  SELECT ARRAY (SELECT row_to_json(t) FROM (SELECT * FROM public.volume WHERE instance_id = pid) t) INTO volumes;
+  return volumes;
+END
+$$ LANGUAGE plpgsql;
+
+-- Get port function
+CREATE OR REPLACE FUNCTION get_attribute(attr_name VARCHAR, inst_id INTEGER)
+RETURNS VARCHAR AS $$
+DECLARE
+  res VARCHAR;
+BEGIN
+  SELECT value FROM public.attribute A WHERE A.instance_id = inst_id AND A.name = attr_name INTO res;
+  return res;
+END
+$$ LANGUAGE plpgsql;
+
+
+-- Get directories function
+CREATE OR REPLACE FUNCTION get_directories(inst_name VARCHAR, type VARCHAR, version VARCHAR, port VARCHAR)
+RETURNS TABLE (basedir VARCHAR, bindir VARCHAR, datadir VARCHAR, logdir VARCHAR, socket VARCHAR) AS $$
+BEGIN
+  IF type = 'MYSQL' THEN
+    RETURN QUERY SELECT 
+      ('/usr/local/mysql/mysql-' || version)::VARCHAR basedir, 
+      ('/usr/local/mysql/mysql-' || version || '/bin')::VARCHAR bindir, 
+      ('/ORA/dbs03/' || upper(inst_name) || '/mysql')::VARCHAR datadir, 
+      ('/ORA/dbs02/' || upper(inst_name) || '/mysql')::VARCHAR logdir, 
+      ('/var/lib/mysql/mysql.sock.' || lower(inst_name) || '.' || port)::VARCHAR socket;
+  ELSIF type = 'PG' THEN
+    RETURN QUERY SELECT 
+      ('/usr/local/pgsql/pgsql-' || version)::VARCHAR basedir, 
+      ('/usr/local/mysql/mysql-' || version || '/bin')::VARCHAR bindir, 
+      ('/ORA/dbs03/' || upper(inst_name) || '/data')::VARCHAR datadir, 
+      ('/ORA/dbs02/' || upper(inst_name) || '/pg_xlog')::VARCHAR logdir, 
+      ('/var/lib/pgsql/')::VARCHAR socket;
+  END IF;
+END
+$$ LANGUAGE plpgsql;
+       
+CREATE VIEW api.instance AS
+SELECT * FROM public.dod_instances;
+
+CREATE VIEW api.volume AS
+SELECT * FROM public.volume;
+
+CREATE VIEW api.attribute AS
+SELECT * FROM public.attribute;
+
+CREATE VIEW api.host AS
+SELECT * FROM public.host;
+
+-- Metadata View
+CREATE OR REPLACE VIEW api.metadata AS
+SELECT id, username, db_name, category, db_type, version, host, get_attribute('port', id) port, get_volumes volumes, d.*
+FROM dod_instances, get_volumes(id), get_directories(db_name, db_type, version, get_attribute('port', id)) d;
+
+-- Rundeck instances View
+CREATE OR REPLACE VIEW api.rundeck_instances AS
+SELECT public.dod_instances.db_name, 
+       public.functional_aliases.alias hostname,
+       public.get_attribute('port', public.dod_instances.id) port,
+       'dbod' username,
+       public.dod_instances.db_type db_type,
+       public.dod_instances.category category,
+       db_type || ',' || category tags
+FROM public.dod_instances JOIN public.functional_aliases ON
+public.dod_instances.db_name = public.functional_aliases.db_name;
+
+-- Host aliases View
+CREATE OR REPLACE VIEW api.host_aliases AS
+SELECT host, string_agg('dbod-' || db_name || 'domain', E',') aliases 
+FROM dod_instances 
+GROUP BY host;
+
+-- functional aliases view
+CREATE OR REPLACE VIEW api.functional_aliases AS
+SELECT * 
+FROM functional_aliases;
diff --git a/dbod/tests/functionalalias_test.py b/dbod/tests/functionalalias_test.py
new file mode 100644
index 0000000..a19baf4
--- /dev/null
+++ b/dbod/tests/functionalalias_test.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+'''Testing functional alias endpoint'''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015, CERN
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+import json
+from timeout_decorator import timeout
+import tornado.web
+
+from tornado.testing import AsyncHTTPTestCase
+
+from dbod.api.api import handlers
+
+class FunctionalAliasTest(AsyncHTTPTestCase):
+    '''Class for testing functional alias with nosetest'''
+    headers = {'Content-Type': 'application/x-www-form-urlencoded', 
+               'Prefer': 'return=representation',
+               'Accept': 'text/json'}
+
+    def get_app(self):
+        return tornado.web.Application(handlers)
+
+    @timeout(5)
+    def test_get_single_alias_by_name(self):
+        '''test for getting the right data'''
+        db_name_test = 'dbod_01'
+        response = self.fetch("/api/v1/entity/alias/%s" %(db_name_test))
+        data = json.loads(response.body)["response"]
+        self.assertEquals(response.code, 200)
+        self.assertEquals(len(data), 1)
+        self.assertEquals(data[0]["alias"], "dbod-dbod-01.cern.ch")
+        self.assertTrue(data[0]["dns_name"] != None)
+
+    @timeout(5)
+    def test_novalid_db(self):
+        '''test when the given db does not exist'''
+        response = self.fetch("/api/v1/entity/alias/some_db")
+        self.assertEquals(response.code, 404)
+
+    @timeout(5)
+    def test_invalid_endpoint(self):
+        '''test when the given endpoint does not exist'''
+        response = self.fetch("/api/v1/entity/something/some_db")
+        self.assertEquals(response.code, 404)
+
+
+    @timeout(5)
+    def test_post_valid_request(self):
+        '''test when the arguments are valid and dns_name is available'''
+        body = 'functional_alias={"dbod_42": "dbod-dbod-42.cern.ch"}'
+        response = self.fetch("/api/v1/entity/alias/", 
+                              method="POST", 
+                              body=body, 
+                              headers=self.headers)
+        self.assertEquals(response.code, 200)
+        self.fetch("/api/v1/entity/alias/dbod_42", 
+                   method="DELETE")
+
+    @timeout(5)
+    def test_post_duplicate(self):
+        '''test when there is a request to insert a db_name which already exists'''
+        body = 'functional_alias={"dbod_01": "dbod-dbod-01.cern.ch"}'
+        response = self.fetch("/api/v1/entity/alias/", 
+                              method="POST", 
+                              body=body, 
+                              headers=self.headers)
+        self.assertEquals(response.code, 409)    
+
+    @timeout(5)
+    def test_post_no_dns(self):
+        '''test when there are no any dns available'''
+        body = 'functional_alias={"dbod_42": "dbod-dbod-42.cern.ch"}'
+        self.fetch("/api/v1/entity/alias/", 
+                   method="POST", 
+                   body=body, 
+                   headers=self.headers)
+        body = 'functional_alias={"dbod_42": "dbod-dbod-42.cern.ch"}'
+        response = self.fetch("/api/v1/entity/alias/", 
+                              method="POST", 
+                              body=body, 
+                              headers=self.headers)
+        self.assertEquals(response.code, 400)
+        self.fetch("/api/v1/entity/alias/dbod_42", 
+                   method="DELETE")
+
+    @timeout(5)
+    def test_post_no_valid_argument(self):
+        ''' test if the provided argument is valid'''
+        body = 'something={"dbod_42": "dbod-dbod-42.cern.ch"}'
+        response = self.fetch("/api/v1/entity/alias/", 
+                              method="POST", 
+                              body=body, 
+                              headers=self.headers)
+        self.assertEquals(response.code, 404)
+
+    @timeout(5)
+    def test_post_no_valid_headers(self):
+        ''' test if the provided argument is valid'''
+        body = 'something={"dbod_42": "dbod-dbod-42.cern.ch"}'
+        response = self.fetch("/api/v1/entity/alias/", 
+                              method="POST", 
+                              body=body)
+        self.assertEquals(response.code, 404)
+
+    def test_post_bad_argument(self):
+        ''' test if the provided argument is valid'''
+        body = 'functional_alias={dbod_42: dbod-dbod-42.cern.ch}'
+        response = self.fetch("/api/v1/entity/alias/", 
+                              method="POST", 
+                              body=body, 
+                              headers=self.headers)
+        self.assertEquals(response.code, 404)
+
+    @timeout(5)
+    def test_delete_valid_request(self):
+        '''test when there is a valid request to delete a previous inserted db_name'''
+        body = 'functional_alias={"dbod_42": "dbod-dbod-42.cern.ch"}'
+        response = self.fetch("/api/v1/entity/alias/", 
+                              method="POST", 
+                              body=body, 
+                              headers=self.headers)
+        response = self.fetch("/api/v1/entity/alias/dbod_42", 
+                              method="DELETE")
+        self.assertEquals(response.code, 200)
+
+
+    def test_delete_invalid_dbname(self):
+        '''test when the given db_name does not exist'''
+        response = self.fetch("/api/v1/entity/alias/dbod_42", 
+                              method="DELETE")
+        self.assertEquals(response.code, 400)

From b486c497233e0b1d99d44c2551109adf10312565 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 29 Jul 2016 11:51:36 +0200
Subject: [PATCH 46/58] HANDLERS: Remove old volumes and replace by new ones in
 put handler.

---
 dbod/api/entity.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/dbod/api/entity.py b/dbod/api/entity.py
index e06c1f3..ff9834c 100644
--- a/dbod/api/entity.py
+++ b/dbod/api/entity.py
@@ -16,6 +16,7 @@
 import logging
 import requests
 import json
+import urllib
 
 from dbod.api.base import *
 from dbod.config import config
@@ -64,7 +65,6 @@ def post(self, instance):
     def put(self, instance):
         """Updates an instance"""
         entity = json.loads(self.request.body)
-        logging.debug(entity)
         entid = self.__get_instance_id__(instance)
         
         # Check if the port is changed
@@ -82,7 +82,18 @@ def put(self, instance):
         if "volumes" in entity:
             volumes = entity["volumes"]
             del entity["volumes"]
-            logging.debug("Cambiar volumes a: " + str(volumes))
+            # Delete current volumes
+            response = requests.delete("http://localhost:3000/volume?instance_id=eq." + str(entid))
+            if response.ok:
+                response = requests.post("http://localhost:3000/volume", json=volumes)
+                if response.ok:
+                    self.set_status(response.status_code)
+                else:
+                    logging.error("Error adding volumes for instance: " + str(entid))
+                    raise tornado.web.HTTPError(response.status_code)
+            else:
+                logging.error("Error deleting old volumes for instance: " + (entid))
+                raise tornado.web.HTTPError(response.status_code)
         
         if entity:
             response = requests.patch("http://localhost:3000/instance?db_name=eq." + instance, json=entity)

From 3ebd0c1c0a33ef655de252d984d4ec7ccffef48c Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 29 Jul 2016 13:49:45 +0200
Subject: [PATCH 47/58] HANDLERS: Check for missing parameters in the creation
 of new entities.

---
 dbod/api/base.py   | 1 +
 dbod/api/entity.py | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/dbod/api/base.py b/dbod/api/base.py
index 174c358..86d499f 100644
--- a/dbod/api/base.py
+++ b/dbod/api/base.py
@@ -25,6 +25,7 @@
 NO_CONTENT = 204 # Succesfull delete
 NOT_FOUND = 404
 UNAUTHORIZED = 401
+INVALID_REQUEST = 400
 
 # Basic HTTP Authentication decorator
 def http_basic_auth(fun):
diff --git a/dbod/api/entity.py b/dbod/api/entity.py
index ff9834c..8f818cd 100644
--- a/dbod/api/entity.py
+++ b/dbod/api/entity.py
@@ -26,6 +26,10 @@ def post(self, instance):
         """Inserts a new instance in the database"""
         entity = json.loads(self.request.body)
         
+        if not "port" in entity or not "volumes" in entity:
+            logging.error("Port or volumes not defined for entity: " + instance)
+            raise tornado.web.HTTPError(INVALID_REQUEST)
+        
         # Get the port
         port = entity["port"]
         del entity["port"]

From 2fbb40987750b241309ee83dc1c64ae168dd7f14 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Fri, 29 Jul 2016 18:54:16 +0200
Subject: [PATCH 48/58] TESTS: Added more tests for entities.

---
 dbod/tests/entity_test.py | 137 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 136 insertions(+), 1 deletion(-)

diff --git a/dbod/tests/entity_test.py b/dbod/tests/entity_test.py
index 08d5d9b..833b340 100644
--- a/dbod/tests/entity_test.py
+++ b/dbod/tests/entity_test.py
@@ -58,6 +58,141 @@ def test_create_entity(self):
         # Check again, the metadata should be empty
         response = self.fetch("/api/v1/metadata/entity/testdb")
         self.assertEquals(response.code, 404)
+        
+    @timeout(5)
+    def test_create_existing_entity(self):
+        entity = """{
+        "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
+        "version": "5.6.17", "db_type": "MYSQL", "port": "5505", "host": "testhost", "db_name": "dbod01", 
+        "volumes": [
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/data1"}, 
+            {"vgroup": "ownergroup", "file_mode": "0755", 
+            "server": "NAS-server", "mount_options": "rw,bg,hard", "owner": "TSM", "mounting_path": "/MNT/bin"}
+        ]}"""
+        
+        # Create the instance
+        response = self.fetch("/api/v1/entity/create", method='POST', body=entity)
+        self.assertEquals(response.code, 409)
     
+    @timeout(5)
+    def test_create_entity_invalid_fields(self):
+        entity = """{
+        "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
+        "version": "5.6.17", "port": "5505", "host": "testhost", "db_name": "very_long_name", 
+        "volumes": [
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/data1"}, 
+            {"vgroup": "ownergroup", "file_mode": "0755", 
+            "server": "NAS-server", "mount_options": "rw,bg,hard", "owner": "TSM", "mounting_path": "/MNT/bin"}
+        ]}"""
+        
+        # Create the instance
+        response = self.fetch("/api/v1/entity/create", method='POST', body=entity)
+        self.assertEquals(response.code, 400)
+        
+    @timeout(5)
+    def test_create_entity_no_port(self):
+        entity = """{
+        "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
+        "version": "5.6.17", "db_type": "MYSQL", "host": "testhost", "db_name": "very_long_name", 
+        "volumes": [
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/data1"}, 
+            {"vgroup": "ownergroup", "file_mode": "0755", 
+            "server": "NAS-server", "mount_options": "rw,bg,hard", "owner": "TSM", "mounting_path": "/MNT/bin"}
+        ]}"""
+        
+        # Create the instance
+        response = self.fetch("/api/v1/entity/create", method='POST', body=entity)
+        self.assertEquals(response.code, 400)
     
-    
+    @timeout(5)
+    def test_create_entity_no_volumes(self):
+        entity = """{
+        "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
+        "version": "5.6.17", "db_type": "MYSQL", "port": "5505", "host": "testhost", "db_name": "very_long_name"}"""
+        
+        # Create the instance
+        response = self.fetch("/api/v1/entity/create", method='POST', body=entity)
+        self.assertEquals(response.code, 400)
+        
+    @timeout(5)
+    def test_edit_entity_username(self):
+        entity = """{"username": "newuser"}"""
+        restore = """{"username": "user01"}"""
+        
+        # Edit the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=entity)
+        self.assertEquals(response.code, 204)
+        
+        # Check the metadata for this instance
+        response = self.fetch("/api/v1/metadata/entity/dbod01")
+        self.assertEquals(response.code, 200)
+        data = json.loads(response.body)["response"]
+        self.assertEquals(data[0]["username"], "newuser")
+        
+        # Restore the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=restore)
+        self.assertEquals(response.code, 204)
+        
+    @timeout(5)
+    def test_edit_entity_dbname(self):
+        entity = """{"db_name": "newdb01"}"""
+        restore = """{"db_name": "dbod01"}"""
+        
+        # Edit the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=entity)
+        self.assertEquals(response.code, 204)
+        
+        # Check the metadata for this instance
+        response = self.fetch("/api/v1/metadata/entity/newdb01")
+        self.assertEquals(response.code, 200)
+        data = json.loads(response.body)["response"]
+        self.assertEquals(data[0]["db_name"], "newdb01")
+        
+        # Restore the instance
+        response = self.fetch("/api/v1/entity/newdb01", method='PUT', body=restore)
+        self.assertEquals(response.code, 204)
+        
+    @timeout(5)
+    def test_edit_entity_port(self):
+        entity = """{"port": "3005"}"""
+        restore = """{"port": "5501"}"""
+        
+        # Edit the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=entity)
+        self.assertEquals(response.code, 204)
+        
+        # Check the metadata for this instance
+        response = self.fetch("/api/v1/metadata/entity/dbod01")
+        self.assertEquals(response.code, 200)
+        data = json.loads(response.body)["response"]
+        self.assertEquals(data[0]["port"], "3005")
+        
+        # Restore the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=restore)
+        self.assertEquals(response.code, 204)
+        
+    @timeout(5)
+    def test_edit_entity_port_and_host(self):
+        entity = """{"port": "3005", "host": "newhost"}"""
+        restore = """{"port": "5501", "host": "host01"}"""
+
+        # Edit the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=entity)
+        self.assertEquals(response.code, 204)
+        
+        # Check the metadata for this instance
+        response = self.fetch("/api/v1/metadata/entity/dbod01")
+        self.assertEquals(response.code, 200)
+        data = json.loads(response.body)["response"]
+        self.assertEquals(data[0]["port"], "3005")
+        self.assertEquals(data[0]["host"], "newhost")
+        
+        # Restore the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=restore)
+        self.assertEquals(response.code, 204)
+        
+
+        
\ No newline at end of file

From 6cc5f60f040288962158f1b2da4218559ea41255 Mon Sep 17 00:00:00 2001
From: Yiannis Sotiropoulos <yianni.sotiro@cern.ch>
Date: Mon, 1 Aug 2016 18:23:27 +0200
Subject: [PATCH 49/58] SQL:Add functional_alias table and view and insert data

---
 dbod/tests/db_test.sql | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/dbod/tests/db_test.sql b/dbod/tests/db_test.sql
index 6fdc782..5f2a795 100644
--- a/dbod/tests/db_test.sql
+++ b/dbod/tests/db_test.sql
@@ -132,6 +132,7 @@ CREATE TABLE public.functional_aliases (
     db_name character varying(8),
     alias character varying(256),
     CONSTRAINT functional_aliases_pkey PRIMARY KEY (dns_name)
+    CONSTRAINT db_name_con UNIQUE (db_name)
 );
 
 -- Insert test data for instances
@@ -166,6 +167,14 @@ VALUES ('host01', 12),
        ('host03', 64),
        ('host04', 256);
        
+-- Insert test data for database aliases
+INSERT INTO public.functional_aliases (dns_name, db_name, alias)
+VALUES ('db-dbod-dns01','dbod_01','dbod-dbod-01.cern.ch'),
+       ('db-dbod-dns02','dbod_02','dbod-dbod-02.cern.ch'),
+       ('db-dbod-dns03','dbod_03','dbod-dbod-03.cern.ch'),
+       ('db-dbod-dns04','dbod_04','dbod-dbod-04.cern.ch'),
+       ('db-dbod-dns05', NULL, NULL);
+
 -- Schema API
 CREATE SCHEMA api;
 
@@ -274,3 +283,8 @@ CREATE OR REPLACE VIEW api.host_aliases AS
 SELECT host, string_agg('dbod-' || db_name || 'domain', E',') aliases 
 FROM dod_instances 
 GROUP BY host;
+
+-- Functional aliases view
+CREATE OR REPLACE VIEW api.functional_aliases AS
+SELECT * 
+FROM functional_aliases;
\ No newline at end of file

From 8a63bdd343aec4255bb7ab58f3e11406c021c3c9 Mon Sep 17 00:00:00 2001
From: Yiannis Sotiropoulos <yianni.sotiro@cern.ch>
Date: Mon, 1 Aug 2016 18:30:59 +0200
Subject: [PATCH 50/58] CONFIG:Added functional alias PostgREST url.

---
 static/api.cfg | 1 +
 1 file changed, 1 insertion(+)

diff --git a/static/api.cfg b/static/api.cfg
index b86e988..2c6b0d1 100644
--- a/static/api.cfg
+++ b/static/api.cfg
@@ -21,6 +21,7 @@ rundeck_resources_url=http://localhost:3000/rundeck_instances
 host_aliases_url=http://localhost:3000/host_aliases
 entity_metadata_url=http://localhost:3000/metadata
 instance_url=http://localhost:3000/instance
+functional_alias_url=http://localhost:3000/functional_aliases
 
 [rundeck]
 api_run_job = https://rundeck/api/14/job/{0}/run?format=json

From 1bf48a69cdcfef1592a9cce4339aece36bd887f4 Mon Sep 17 00:00:00 2001
From: Yiannis Sotiropoulos <yianni.sotiro@cern.ch>
Date: Tue, 2 Aug 2016 10:16:57 +0200
Subject: [PATCH 51/58] HANDLERS:Adding missing import for functional alias

---
 dbod/api/api.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dbod/api/api.py b/dbod/api/api.py
index 9f248bc..aa70511 100644
--- a/dbod/api/api.py
+++ b/dbod/api/api.py
@@ -20,6 +20,7 @@
 from dbod.api.base import DocHandler
 from dbod.api.rundeck import RundeckResources, RundeckJobs
 from dbod.api.metadata import Metadata
+from dbod.api.functionalalias import FunctionalAliasHandler
 from dbod.api.hostaliases import HostAliases
 from dbod.api.entity import Entity
 from dbod.config import config
@@ -29,6 +30,7 @@
     # (r"/api/v1/entity/alias/([^/]+)", FunctionalAliasHandler),
     (r"/api/v1/entity/([^/]+)", Entity),
     (r"/api/v1/host/aliases/([^/]+)", HostAliases),
+    (r"/api/v1/entity/alias/([^/]*)", FunctionalAliasHandler),
     (r"/api/v1/metadata/(?P<class>[^\/]+)/?(?P<name>[^\/]+)?", Metadata),
     (r"/api/v1/rundeck/resources.xml", RundeckResources),
     (r"/api/v1/rundeck/job/(?P<job>[^\/]+)/?(?P<entity>[^\/]+)?", RundeckJobs),

From 3ed412ffdd350d7c380a559aa7c6932825727b65 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 2 Aug 2016 10:28:25 +0200
Subject: [PATCH 52/58] HANDLERS: Changed invalid_request to bad_request.

---
 dbod/api/entity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbod/api/entity.py b/dbod/api/entity.py
index 8f818cd..50a18f5 100644
--- a/dbod/api/entity.py
+++ b/dbod/api/entity.py
@@ -28,7 +28,7 @@ def post(self, instance):
         
         if not "port" in entity or not "volumes" in entity:
             logging.error("Port or volumes not defined for entity: " + instance)
-            raise tornado.web.HTTPError(INVALID_REQUEST)
+            raise tornado.web.HTTPError(BAD_REQUEST)
         
         # Get the port
         port = entity["port"]

From f6c3388246b4e4b0b09907aa6cc4e15633559879 Mon Sep 17 00:00:00 2001
From: Yiannis Sotiropoulos <yianni.sotiro@cern.ch>
Date: Tue, 2 Aug 2016 10:53:48 +0200
Subject: [PATCH 53/58] SQL:Fixing functional aliases table

---
 dbod/tests/db_test.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dbod/tests/db_test.sql b/dbod/tests/db_test.sql
index 5f2a795..e07913d 100644
--- a/dbod/tests/db_test.sql
+++ b/dbod/tests/db_test.sql
@@ -131,7 +131,7 @@ CREATE TABLE public.functional_aliases (
     dns_name character varying(256) NOT NULL,
     db_name character varying(8),
     alias character varying(256),
-    CONSTRAINT functional_aliases_pkey PRIMARY KEY (dns_name)
+    CONSTRAINT functional_aliases_pkey PRIMARY KEY (dns_name),
     CONSTRAINT db_name_con UNIQUE (db_name)
 );
 
@@ -287,4 +287,4 @@ GROUP BY host;
 -- Functional aliases view
 CREATE OR REPLACE VIEW api.functional_aliases AS
 SELECT * 
-FROM functional_aliases;
\ No newline at end of file
+FROM functional_aliases;

From b43e7e45660ee12771f6d88872639502b8e25519 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 2 Aug 2016 11:34:54 +0200
Subject: [PATCH 54/58] TESTS: Added comments to tests, and new test to edit
 volumes.

---
 dbod/tests/entity_test.py | 62 +++++++++++++++++++++++++++++++++------
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/dbod/tests/entity_test.py b/dbod/tests/entity_test.py
index 833b340..24633f9 100644
--- a/dbod/tests/entity_test.py
+++ b/dbod/tests/entity_test.py
@@ -27,6 +27,7 @@ def get_app(self):
 
     @timeout(5)
     def test_create_entity(self):
+        ### Creation of a new instance in a correct way.
         response = self.fetch("/api/v1/entity/testdb", method='DELETE')
         
         entity = """{
@@ -35,8 +36,8 @@ def test_create_entity(self):
         "volumes": [
             {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
             "owner": "TSM", "mounting_path": "/MNT/data1"}, 
-            {"vgroup": "ownergroup", "file_mode": "0755", 
-            "server": "NAS-server", "mount_options": "rw,bg,hard", "owner": "TSM", "mounting_path": "/MNT/bin"}
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/bin"}
         ]}"""
         
         # Create the instance
@@ -61,14 +62,15 @@ def test_create_entity(self):
         
     @timeout(5)
     def test_create_existing_entity(self):
+        ### Creation of an entity that already exists
         entity = """{
         "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
         "version": "5.6.17", "db_type": "MYSQL", "port": "5505", "host": "testhost", "db_name": "dbod01", 
         "volumes": [
             {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
             "owner": "TSM", "mounting_path": "/MNT/data1"}, 
-            {"vgroup": "ownergroup", "file_mode": "0755", 
-            "server": "NAS-server", "mount_options": "rw,bg,hard", "owner": "TSM", "mounting_path": "/MNT/bin"}
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard",
+            "owner": "TSM", "mounting_path": "/MNT/bin"}
         ]}"""
         
         # Create the instance
@@ -77,14 +79,15 @@ def test_create_existing_entity(self):
     
     @timeout(5)
     def test_create_entity_invalid_fields(self):
+        ### Creation of an entity with an undefined required field (db_type)
         entity = """{
         "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
         "version": "5.6.17", "port": "5505", "host": "testhost", "db_name": "very_long_name", 
         "volumes": [
             {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
             "owner": "TSM", "mounting_path": "/MNT/data1"}, 
-            {"vgroup": "ownergroup", "file_mode": "0755", 
-            "server": "NAS-server", "mount_options": "rw,bg,hard", "owner": "TSM", "mounting_path": "/MNT/bin"}
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/bin"}
         ]}"""
         
         # Create the instance
@@ -93,14 +96,15 @@ def test_create_entity_invalid_fields(self):
         
     @timeout(5)
     def test_create_entity_no_port(self):
+        ### Creation of an entity without port
         entity = """{
         "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
         "version": "5.6.17", "db_type": "MYSQL", "host": "testhost", "db_name": "very_long_name", 
         "volumes": [
             {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
             "owner": "TSM", "mounting_path": "/MNT/data1"}, 
-            {"vgroup": "ownergroup", "file_mode": "0755", 
-            "server": "NAS-server", "mount_options": "rw,bg,hard", "owner": "TSM", "mounting_path": "/MNT/bin"}
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/bin"}
         ]}"""
         
         # Create the instance
@@ -109,6 +113,7 @@ def test_create_entity_no_port(self):
     
     @timeout(5)
     def test_create_entity_no_volumes(self):
+        ### Creation of an entity without volumes
         entity = """{
         "username": "testuser", "category": "TEST", "creation_date":"2016-07-20", 
         "version": "5.6.17", "db_type": "MYSQL", "port": "5505", "host": "testhost", "db_name": "very_long_name"}"""
@@ -119,6 +124,7 @@ def test_create_entity_no_volumes(self):
         
     @timeout(5)
     def test_edit_entity_username(self):
+        ### Edit the username correctly
         entity = """{"username": "newuser"}"""
         restore = """{"username": "user01"}"""
         
@@ -138,6 +144,7 @@ def test_edit_entity_username(self):
         
     @timeout(5)
     def test_edit_entity_dbname(self):
+        ### Edit the dbname correctly
         entity = """{"db_name": "newdb01"}"""
         restore = """{"db_name": "dbod01"}"""
         
@@ -157,6 +164,7 @@ def test_edit_entity_dbname(self):
         
     @timeout(5)
     def test_edit_entity_port(self):
+        ### Edit the port correctly
         entity = """{"port": "3005"}"""
         restore = """{"port": "5501"}"""
         
@@ -176,6 +184,7 @@ def test_edit_entity_port(self):
         
     @timeout(5)
     def test_edit_entity_port_and_host(self):
+        ### Edit the host and port correctly
         entity = """{"port": "3005", "host": "newhost"}"""
         restore = """{"port": "5501", "host": "host01"}"""
 
@@ -194,5 +203,40 @@ def test_edit_entity_port_and_host(self):
         response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=restore)
         self.assertEquals(response.code, 204)
         
-
+    @timeout(5)
+    def test_edit_entity_volumes(self):
+        ### Edit volumes correctly
+        entity = """{"volumes": [
+            {"vgroup": "testgroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/data1"}, 
+            {"vgroup": "testgroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/test"},
+            {"vgroup": "testgroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/bin"}
+        ]}"""
+        restore = """{"volumes": [
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw,bg,hard", 
+            "owner": "TSM", "mounting_path": "/MNT/data1"}, 
+            {"vgroup": "ownergroup", "file_mode": "0755", "server": "NAS-server", "mount_options": "rw", 
+            "owner": "TSM", "mounting_path": "/MNT/bin"}
+        ]}"""
+        
+        # Edit the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=entity)
+        self.assertEquals(response.code, 204)
+        
+        # Check the metadata for this instance
+        response = self.fetch("/api/v1/metadata/entity/dbod01")
+        self.assertEquals(response.code, 200)
+        data = json.loads(response.body)["response"]
+        self.assertEquals(len(data[0]["volumes"]), 3)
+        self.assertEquals(data[0]["volumes"][0]["vgroup"], "testgroup")
+        self.assertEquals(data[0]["volumes"][1]["vgroup"], "testgroup")
+        self.assertEquals(data[0]["volumes"][2]["vgroup"], "testgroup")
+        self.assertEquals(data[0]["volumes"][1]["mounting_path"], "/MNT/test")
+        
+        # Restore the instance
+        response = self.fetch("/api/v1/entity/dbod01", method='PUT', body=restore)
+        self.assertEquals(response.code, 204)
+        
         
\ No newline at end of file

From b0789c92f36f13498ae41e5f45375b88a6094a39 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 2 Aug 2016 11:52:18 +0200
Subject: [PATCH 55/58] HANDLERS: Fixed error codes in PUT entity. Add
 instance_id to volumes.

---
 dbod/api/entity.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/dbod/api/entity.py b/dbod/api/entity.py
index 50a18f5..da676f4 100644
--- a/dbod/api/entity.py
+++ b/dbod/api/entity.py
@@ -85,18 +85,21 @@ def put(self, instance):
         # Check if the volumes are changed
         if "volumes" in entity:
             volumes = entity["volumes"]
+            for volume in volumes:
+                volume["instance_id"] = entid
             del entity["volumes"]
+            
             # Delete current volumes
             response = requests.delete("http://localhost:3000/volume?instance_id=eq." + str(entid))
-            if response.ok:
+            if response.status_code == 204:
                 response = requests.post("http://localhost:3000/volume", json=volumes)
-                if response.ok:
+                if response.status_code == 201:
                     self.set_status(response.status_code)
                 else:
                     logging.error("Error adding volumes for instance: " + str(entid))
                     raise tornado.web.HTTPError(response.status_code)
             else:
-                logging.error("Error deleting old volumes for instance: " + (entid))
+                logging.error("Error deleting old volumes for instance: " + str(entid))
                 raise tornado.web.HTTPError(response.status_code)
         
         if entity:

From 1fd2ab774814a2fb3bb080951917225458bb21d3 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 2 Aug 2016 15:12:34 +0200
Subject: [PATCH 56/58] HANDLERS: Removed commented out handler.

---
 dbod/api/api.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dbod/api/api.py b/dbod/api/api.py
index aa70511..82df5ca 100644
--- a/dbod/api/api.py
+++ b/dbod/api/api.py
@@ -27,7 +27,6 @@
 
 handlers = [
     (r"/", DocHandler),
-    # (r"/api/v1/entity/alias/([^/]+)", FunctionalAliasHandler),
     (r"/api/v1/entity/([^/]+)", Entity),
     (r"/api/v1/host/aliases/([^/]+)", HostAliases),
     (r"/api/v1/entity/alias/([^/]*)", FunctionalAliasHandler),

From 6bd4498281cd23fa46bb073f0c678b9f8a6f5c97 Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Tue, 2 Aug 2016 18:33:15 +0200
Subject: [PATCH 57/58] HANDLERS: Added get handler to entity.

---
 dbod/api/entity.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/dbod/api/entity.py b/dbod/api/entity.py
index da676f4..50a0da8 100644
--- a/dbod/api/entity.py
+++ b/dbod/api/entity.py
@@ -22,6 +22,18 @@
 from dbod.config import config
 
 class Entity(tornado.web.RequestHandler):
+    def get(self, name):
+        """Returns an instance by db_name"""
+        response = requests.get("http://localhost:3000/instance?db_name=eq." + name)
+        if response.ok:
+            data = response.json()
+            if data:
+                self.write({'response' : data})
+                self.set_status(OK)
+            else: 
+                logging.error("Entity metadata not found: " + name)
+                raise tornado.web.HTTPError(NOT_FOUND)
+
     def post(self, instance):
         """Inserts a new instance in the database"""
         entity = json.loads(self.request.body)
@@ -91,9 +103,9 @@ def put(self, instance):
             
             # Delete current volumes
             response = requests.delete("http://localhost:3000/volume?instance_id=eq." + str(entid))
-            if response.status_code == 204:
+            if response.ok:
                 response = requests.post("http://localhost:3000/volume", json=volumes)
-                if response.status_code == 201:
+                if response.ok:
                     self.set_status(response.status_code)
                 else:
                     logging.error("Error adding volumes for instance: " + str(entid))
@@ -126,7 +138,11 @@ def delete(self, instance):
     def __get_instance_id__(self, instance):
         response = requests.get("http://localhost:3000/instance?db_name=eq." + instance)
         if response.ok:
-            return json.loads(response.text)[0]["id"]
+            data = response.json()
+            if data:
+                return data[0]["id"]
+            else:
+                return None
         else:
             return None
             

From 2f70536e2af6c309d854f84c169c384a2ba846cc Mon Sep 17 00:00:00 2001
From: jocorder <j.cordero@cern.ch>
Date: Wed, 3 Aug 2016 10:32:07 +0200
Subject: [PATCH 58/58] CONFIG: Removed hardcoded urls and added to config
 file.

---
 dbod/api/entity.py | 24 ++++++++++++------------
 static/api.cfg     |  2 ++
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/dbod/api/entity.py b/dbod/api/entity.py
index 50a0da8..37f2cd9 100644
--- a/dbod/api/entity.py
+++ b/dbod/api/entity.py
@@ -24,7 +24,7 @@
 class Entity(tornado.web.RequestHandler):
     def get(self, name):
         """Returns an instance by db_name"""
-        response = requests.get("http://localhost:3000/instance?db_name=eq." + name)
+        response = requests.get(config.get('postgrest', 'instance_url') + "?db_name=eq." + name)
         if response.ok:
             data = response.json()
             if data:
@@ -51,7 +51,7 @@ def post(self, instance):
         del entity["volumes"]
         
         # Insert the entity in database using PostREST
-        response = requests.post("http://localhost:3000/instance", json=entity, headers={'Prefer': 'return=representation'})
+        response = requests.post(config.get('postgrest', 'instance_url'), json=entity, headers={'Prefer': 'return=representation'})
         if response.ok:
             entid = json.loads(response.text)["id"]
             logging.debug("Created entity with id: " + str(entid))
@@ -61,9 +61,9 @@ def post(self, instance):
                 volume["instance_id"] = entid
 
             # Insert the volumes in database using PostREST
-            response = requests.post("http://localhost:3000/volume", json=volumes)
+            response = requests.post(config.get('postgrest', 'volume_url'), json=volumes)
             if response.ok:
-                response = requests.post("http://localhost:3000/attribute", json={'instance_id': entid, 'name': 'port', 'value': port})
+                response = requests.post(config.get('postgrest', 'attribute_url'), json={'instance_id': entid, 'name': 'port', 'value': port})
                 if response.ok:
                     self.set_status(CREATED)
                 else:
@@ -87,7 +87,7 @@ def put(self, instance):
         if "port" in entity:
             port = {"value":entity["port"]}
             del entity["port"]
-            response = requests.patch("http://localhost:3000/attribute?instance_id=eq." + str(entid) + "&name=eq.port", json=port)
+            response = requests.patch(config.get('postgrest', 'attribute_url') + "?instance_id=eq." + str(entid) + "&name=eq.port", json=port)
             if response.ok:
                 self.set_status(response.status_code)
             else:
@@ -102,9 +102,9 @@ def put(self, instance):
             del entity["volumes"]
             
             # Delete current volumes
-            response = requests.delete("http://localhost:3000/volume?instance_id=eq." + str(entid))
+            response = requests.delete(config.get('postgrest', 'volume_url') + "?instance_id=eq." + str(entid))
             if response.ok:
-                response = requests.post("http://localhost:3000/volume", json=volumes)
+                response = requests.post(config.get('postgrest', 'volume_url'), json=volumes)
                 if response.ok:
                     self.set_status(response.status_code)
                 else:
@@ -115,7 +115,7 @@ def put(self, instance):
                 raise tornado.web.HTTPError(response.status_code)
         
         if entity:
-            response = requests.patch("http://localhost:3000/instance?db_name=eq." + instance, json=entity)
+            response = requests.patch(config.get('postgrest', 'instance_url') + "?db_name=eq." + instance, json=entity)
             if response.ok:
                 self.set_status(response.status_code)
             else:
@@ -136,7 +136,7 @@ def delete(self, instance):
             raise tornado.web.HTTPError(response.status_code)
             
     def __get_instance_id__(self, instance):
-        response = requests.get("http://localhost:3000/instance?db_name=eq." + instance)
+        response = requests.get(config.get('postgrest', 'instance_url') + "?db_name=eq." + instance)
         if response.ok:
             data = response.json()
             if data:
@@ -147,8 +147,8 @@ def __get_instance_id__(self, instance):
             return None
             
     def __delete_instance__(self, inst_id):
-        requests.delete("http://localhost:3000/attribute?instance_id=eq." + str(inst_id))
-        requests.delete("http://localhost:3000/volume?instance_id=eq." + str(inst_id))
-        requests.delete("http://localhost:3000/instance?id=eq." + str(inst_id))
+        requests.delete(config.get('postgrest', 'attribute_url') + "?instance_id=eq." + str(inst_id))
+        requests.delete(config.get('postgrest', 'volume_url') + "?instance_id=eq." + str(inst_id))
+        requests.delete(config.get('postgrest', 'instance_url') + "?id=eq." + str(inst_id))
         
 
diff --git a/static/api.cfg b/static/api.cfg
index 2c6b0d1..785d0e3 100644
--- a/static/api.cfg
+++ b/static/api.cfg
@@ -21,6 +21,8 @@ rundeck_resources_url=http://localhost:3000/rundeck_instances
 host_aliases_url=http://localhost:3000/host_aliases
 entity_metadata_url=http://localhost:3000/metadata
 instance_url=http://localhost:3000/instance
+volume_url=http://localhost:3000/volume
+attribute_url=http://localhost:3000/attribute
 functional_alias_url=http://localhost:3000/functional_aliases
 
 [rundeck]