Skip to content

Commit

Permalink
chore(code): refactor code
Browse files Browse the repository at this point in the history
  • Loading branch information
@giangbui
giangbui committed Mar 26, 2018
1 parent d60fc10 commit 6907e65
Show file tree
Hide file tree
Showing 4 changed files with 195 additions and 280 deletions.
79 changes: 40 additions & 39 deletions peregrine/resources/submission/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
from peregrine.utils import jsonify_check_errors



def get_open_project_ids():
"""
List project ids corresponding to projects with ``availability_type ==
Expand Down Expand Up @@ -70,7 +69,7 @@ def set_read_access_projects():
open_project_ids = get_open_project_ids()
flask.g.read_access_projects.extend(open_project_ids)


@peregrine.blueprints.blueprint.route('/graphql', methods=['POST'])
@peregrine.auth.set_global_user
def root_graphql_query():
Expand Down Expand Up @@ -99,16 +98,17 @@ def root_graphql_query():
if export_format == 'bdbag':
data, code = return_data
if code == 200:
if peregrine.utils.contain_node_with_category(data.json,'data_file') == False:
return flask.jsonify({ 'errors': 'No data_file node'}), 400
res = peregrine.utils.json2tbl(json.loads(data.data),'', "_" )
if peregrine.utils.contain_node_with_category(data.json, 'data_file') == False:
return flask.jsonify({'errors': 'No data_file node'}), 400
res = peregrine.utils.json2tbl(json.loads(data.data), '', "_")
tsv = peregrine.utils.dicts2tsv(res)
return flask.Response(tsv, mimetype='text/tab-separated-values'), code
else:
return data, code
else:
return return_data
#return flask.jsonify({'data': 'Format not supported !!!'}), 400
# return flask.jsonify({'data': 'Format not supported !!!'}), 400


def get_introspection_query():
"""
Expand Down Expand Up @@ -137,37 +137,38 @@ def root_graphql_schema_query():
)
)

@peregrine.blueprints.blueprint.route('/export', methods=['POST'])
def get_manifest():
"""
Creates and returns a manifest based on the filters pased on
to this endpoint
parameters:
- name: filters
in: graphql result in json format
description: Filters to be applied when generating the manifest
:return: A manifest that the user can use to download the files in there
"""
payload = peregrine.utils.parse_request_json()
export_data = payload.get('export_data')
bag_path = payload.get('bag_path')

if(bag_path is None):
return flask.jsonify({'bag_path': None, 'errors': 'bag_path is required!!!'}), 400

if peregrine.utils.contain_node_with_category(export_data,'data_file') == False:
return flask.jsonify({ 'errors': 'No data_file node'}), 400

res = peregrine.utils.json2tbl(export_data,'', "_" )
tsv = peregrine.utils.dicts2tsv(res)

bag_info = {'organization': 'CDIS',
'data_type': 'TOPMed',
'date_created': datetime.date.today().isoformat()}
args = dict(
bag_path=bag_path,
bag_info=bag_info,
payload=res)
peregrine.utils.create_bdbag(**args) # bag is a compressed file

return flask.jsonify({'data': res}), 200
# @peregrine.blueprints.blueprint.route('/export', methods=['POST'])
# def get_manifest():
# """
# Creates and returns a manifest based on the filters pased on
# to this endpoint
# parameters:
# - name: filters
# in: graphql result in json format
# description: Filters to be applied when generating the manifest
# :return: A manifest that the user can use to download the files in there
# """
# payload = peregrine.utils.parse_request_json()
# export_data = payload.get('export_data')
# bag_path = payload.get('bag_path')

# if(bag_path is None):
# return flask.jsonify({'bag_path': None, 'errors': 'bag_path is required!!!'}), 400

# if peregrine.utils.contain_node_with_category(export_data, 'data_file') == False:
# return flask.jsonify({'errors': 'No data_file node'}), 400

# res = peregrine.utils.json2tbl(export_data, '', "_")
# tsv = peregrine.utils.dicts2tsv(res)

# bag_info = {'organization': 'CDIS',
# 'data_type': 'TOPMed',
# 'date_created': datetime.date.today().isoformat()}
# args = dict(
# bag_path=bag_path,
# bag_info=bag_info,
# payload=res)
# peregrine.utils.create_bdbag(**args) # bag is a compressed file

# return flask.jsonify({'data': res}), 200
44 changes: 24 additions & 20 deletions peregrine/utils/json2csv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,26 +59,29 @@ def dicts2tsv(dict_list):
output string
"""
tsv = ""
row = []
for k in sorted(dict_list[0]):
k = k.replace('_data_','')
tsv = tsv + "{}\t".format(k)
tsv = tsv[:-1] + "\n"

header_set = set()

for dict_row in dict_list:
header_set.update(dict_row.keys())

for h in sorted(header_set):
h = h.replace('_data_', '')
tsv = tsv + "{}\t".format(h)

nrow = 0
for dict_row in dict_list:
row=[]
for k in sorted(dict_row):
if dict_row[k]:
tsv = tsv + "{}\t".format(dict_row[k])
row = []
for h in sorted(header_set):
if dict_row[h]:
tsv = tsv + "{}\t".format(dict_row[h])
else:
tsv = tsv + "None\t"
tsv = tsv[:-1] + "\n"
nrow = nrow + 1
if nrow >= 1000:
break

return tsv
tsv = tsv + "None\t"
tsv = tsv[:-1] + "\n"
nrow = nrow + 1
if nrow >= 1000:
break
return tsv

def join(table_list, L, index, row):
'''
Expand All @@ -100,7 +103,8 @@ def join(table_list, L, index, row):
newrow.update(item)
join(table_list, L, index + 1, newrow)

def json2tbl(json,prefix,delem):

def json2tbl(json, prefix, delem):
'''
Args:
json: graphQL output JSON
Expand All @@ -113,16 +117,16 @@ def json2tbl(json,prefix,delem):
L = []
if isinstance(json, list) and json != []:
for l in json:
L += (json2tbl(l,prefix,delem))
L += (json2tbl(l, prefix, delem))
return L
if isinstance(json, dict):
#handle dictionary
# handle dictionary
table_list = []
for k in json.keys():
table = json2tbl(json[k], prefix + delem + k, delem)
table_list.append(table)

join(table_list,L,0,{})
join(table_list, L, 0, {})
else:
L.append({prefix: json})
return L
11 changes: 8 additions & 3 deletions peregrine/utils/payload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

from peregrine.resources.submission.graphql.node import get_fields


def get_external_proxies():
"""Get any custom proxies set in the config.
Expand Down Expand Up @@ -216,13 +217,15 @@ def get_introspection_query():
f = open(os.path.join(cur_dir, 'graphql', 'introspection_query.txt'), 'r')
return f.read()


def json_dumps_formatted(data):
"""Return json string with standard format."""
dump = json.dumps(
data, indent=2, separators=(', ', ': '), ensure_ascii=False
)
return dump.encode('utf-8')


def jsonify_check_errors(data_and_errors, error_code=400):
"""
TODO
Expand All @@ -247,6 +250,7 @@ def get_variables(payload):
errors = ['Unable to parse variables', str(e)]
return variables, errors


def contain_node_with_category(json, category):
'''
Check if JSON object contain `category` keys or not
Expand All @@ -256,12 +260,12 @@ def contain_node_with_category(json, category):
True: if JSON object contains data_file key
False: otherwise
'''
keys_list=[]
keys_list = []
get_keys(json, keys_list)
ns_field = get_fields()

category_map = {}
for (k,v) in ns_field.iteritems():
for (k, v) in ns_field.iteritems():
category_map[v] = k._dictionary['category']

for key in keys_list:
Expand All @@ -272,6 +276,7 @@ def contain_node_with_category(json, category):
pass
return False


def get_keys(payload, keys_list):
'''
Get all keys of JSON object and update to the keys_list
Expand All @@ -280,4 +285,4 @@ def get_keys(payload, keys_list):
keys_list += payload.keys()
map(lambda x: get_keys(x, keys_list), payload.values())
elif isinstance(payload, list):
map(lambda x: get_keys(x, keys_list), payload)
map(lambda x: get_keys(x, keys_list), payload)
Loading

0 comments on commit 6907e65

Please sign in to comment.