From 30d7f5e9961b060ec528af5716b14be11c2f008b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoshiki=20V=C3=A1zquez=20Baeza?= Date: Tue, 12 Jan 2021 10:40:59 -0800 Subject: [PATCH] Dev updates 2021 (#3063) * rm key * use test_ssh_key * flake8 * add chmod * ssh_key -> ssh_key_fp * fix #3057 * adding JS for download * minor improvements * addressing @ElDeveloper comment * fix #3049 * fix #3060 * flake8 * addressing @ElDeveloper comments * Delete 81.sql * No * Yes * No * Yes * Replace qiita-spots to qiita Co-authored-by: Antonio Gonzalez --- README.rst | 16 +++---- .../base_metadata_template.py | 27 +++++++----- qiita_db/metadata_template/constants.py | 1 + qiita_db/metadata_template/sample_template.py | 6 ++- .../test/test_sample_template.py | 13 ++++-- qiita_db/processing_job.py | 4 +- .../artifact_handlers/base_handlers.py | 2 +- .../tests/test_base_handlers.py | 16 +++++++ qiita_pet/handlers/download.py | 30 +++++++++++-- qiita_pet/handlers/upload.py | 3 +- .../templates/study_ajax/prep_summary.html | 1 + .../study_ajax/sample_prep_summary.html | 42 ++++++++++++++++++ qiita_pet/test/test_download.py | 21 +++++++++ qiita_pet/webserver.py | 4 +- qiita_ware/commands.py | 22 +++++++--- qiita_ware/test/test_commands.py | 23 +++++++--- scripts/qiita-recover-jobs | 44 ++++++++++++------- 17 files changed, 215 insertions(+), 60 deletions(-) diff --git a/README.rst b/README.rst index f3acc203a..918bba626 100644 --- a/README.rst +++ b/README.rst @@ -19,12 +19,12 @@ typically limiting for researchers studying microbial ecology. Qiita is currently in beta status. We are very open to community contributions and feedback. If you're interested in contributing to Qiita, -see `CONTRIBUTING.md `__. +see `CONTRIBUTING.md `__. If you'd like to report bugs or request features, you can do that in the -`Qiita issue tracker `__. +`Qiita issue tracker `__. To install and configure your own Qiita server, see -`INSTALL.md `__. +`INSTALL.md `__. For more specific details about qiita visit `the Qiita main site tutorial `__. @@ -71,9 +71,7 @@ future. existing studies by submitting a fix proposals to the authors of the study. -.. |Build Status| image:: https://travis-ci.org/biocore/qiita.png?branch=master - :target: https://travis-ci.org/biocore/qiita -.. |Coverage Status| image:: https://codecov.io/gh/biocore/qiita/branch/master/graph/badge.svg - :target: https://codecov.io/gh/biocore/qiita -.. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg - :target: https://gitter.im/biocore/qiita?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +.. |Build Status| image:: https://travis-ci.org/qiita-spots/qiita.png?branch=master + :target: https://travis-ci.org/qiita-spots/qiita +.. |Coverage Status| image:: https://codecov.io/gh/qiita-spots/qiita/branch/master/graph/badge.svg + :target: https://codecov.io/gh/qiita-spots/qiita diff --git a/qiita_db/metadata_template/base_metadata_template.py b/qiita_db/metadata_template/base_metadata_template.py index 7e5f7aac7..f83609f88 100644 --- a/qiita_db/metadata_template/base_metadata_template.py +++ b/qiita_db/metadata_template/base_metadata_template.py @@ -558,20 +558,20 @@ def _clean_validate_template(cls, md_template, study_id, error = [] if pgsql_reserved: error.append( - "The following column names in the template contain PgSQL " - "reserved words: %s." % ", ".join(pgsql_reserved)) + "These column names are PgSQL reserved words, replace them: " + "~~ %s ~~." % ", ".join(pgsql_reserved)) if invalid: error.append( - "The following column names in the template contain invalid " - "chars: %s." % ", ".join(invalid)) + "These column names contain invalid chars, remove or replace " + "them: ~~ %s ~~." % ", ".join(invalid)) if forbidden: error.append( - "The following column names in the template contain invalid " - "values: %s." % ", ".join(forbidden)) + "These column names are not valid in this information file, " + "remove them: ~~ %s ~~." % ", ".join(forbidden)) if qiime2_reserved: error.append( - "The following column names in the template contain QIIME2 " - "reserved words: %s." % ", ".join(pgsql_reserved)) + "These columns are QIIME2 reserved words, replace them: " + " ~~ %s ~~." % ", ".join(pgsql_reserved)) if error: raise qdb.exceptions.QiitaDBColumnError( @@ -1160,13 +1160,15 @@ def to_file(self, fp, samples=None): df.to_csv(fp, index_label='sample_name', na_rep="", sep='\t', encoding='utf-8') - def _common_to_dataframe_steps(self): + def _common_to_dataframe_steps(self, samples=None): """Perform the common to_dataframe steps Returns ------- pandas DataFrame The metadata in the template,indexed on sample id + samples list of string, optional + A list of the sample names we actually want to retrieve """ with qdb.sql_connection.TRN: # Retrieve all the information from the database @@ -1175,7 +1177,12 @@ def _common_to_dataframe_steps(self): FROM qiita.{0} WHERE sample_id != '{1}'""".format( self._table_name(self._id), QIITA_COLUMN_NAME) - qdb.sql_connection.TRN.add(sql) + if samples is None: + qdb.sql_connection.TRN.add(sql) + else: + sql += ' AND sample_id IN %s' + qdb.sql_connection.TRN.add(sql, [tuple(samples)]) + # this query is going to return a tuple # (sample_id, dict of columns/values); however it's important to # notice that we can't assure that all column/values pairs are the diff --git a/qiita_db/metadata_template/constants.py b/qiita_db/metadata_template/constants.py index 9416bd0e8..9fbeb7ddf 100644 --- a/qiita_db/metadata_template/constants.py +++ b/qiita_db/metadata_template/constants.py @@ -17,6 +17,7 @@ 'EBI': Restriction(columns={'collection_timestamp': datetime, 'physical_specimen_location': str, 'taxon_id': int, + 'description': str, 'scientific_name': str}, error_msg="EBI submission disabled"), # The following columns are required for the official main QIITA site diff --git a/qiita_db/metadata_template/sample_template.py b/qiita_db/metadata_template/sample_template.py index 1e209ce98..14f9ee9d6 100644 --- a/qiita_db/metadata_template/sample_template.py +++ b/qiita_db/metadata_template/sample_template.py @@ -333,15 +333,17 @@ def biosample_accessions(self, value): """ self._update_accession_numbers('biosample_accession', value) - def to_dataframe(self, add_ebi_accessions=False): + def to_dataframe(self, add_ebi_accessions=False, samples=None): """Returns the metadata template as a dataframe Parameters ---------- add_ebi_accessions : bool, optional If this should add the ebi accessions + samples list of string, optional + A list of the sample names we actually want to retrieve """ - df = self._common_to_dataframe_steps() + df = self._common_to_dataframe_steps(samples=samples) if add_ebi_accessions: accessions = self.ebi_sample_accessions diff --git a/qiita_db/metadata_template/test/test_sample_template.py b/qiita_db/metadata_template/test/test_sample_template.py index 501c5e38e..5c877873b 100644 --- a/qiita_db/metadata_template/test/test_sample_template.py +++ b/qiita_db/metadata_template/test/test_sample_template.py @@ -1963,8 +1963,7 @@ def test_to_dataframe(self): '1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187', '1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192'} self.assertEqual(set(obs.index), exp) - - self.assertEqual(set(obs.columns), { + exp_columns = { 'physical_specimen_location', 'physical_specimen_remaining', 'dna_extracted', 'sample_type', 'collection_timestamp', 'host_subject_id', 'description', 'latitude', 'longitude', @@ -1973,7 +1972,15 @@ def test_to_dataframe(self): 'water_content_soil', 'elevation', 'temp', 'tot_nitro', 'samp_salinity', 'altitude', 'env_biome', 'country', 'ph', 'anonymized_name', 'tot_org_carb', 'description_duplicate', - 'env_feature', 'scientific_name', 'qiita_study_id'}) + 'env_feature', 'scientific_name', 'qiita_study_id'} + self.assertEqual(set(obs.columns), exp_columns) + + # test limiting samples produced + exp_samples = set(['1.SKD4.640185', '1.SKD5.640186']) + obs = self.tester.to_dataframe(samples=exp_samples) + self.assertEqual(len(obs), 2) + self.assertEqual(set(obs.index), exp_samples) + self.assertEqual(set(obs.columns), exp_columns) # test with add_ebi_accessions as True obs = self.tester.to_dataframe(True) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index d1e51b9b8..6fde2731b 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1763,8 +1763,8 @@ def shape(self): if self.command.name == 'Validate': # Validate only has two options to calculate it's size: template (a # job that has a preparation linked) or analysis (is from an - # analysis). - if 'template' in parameters: + # analysis). However, 'template' can be present and be None + if 'template' in parameters and parameters['template'] is not None: try: pt = qdb.metadata_template.prep_template.PrepTemplate( parameters['template']) diff --git a/qiita_pet/handlers/artifact_handlers/base_handlers.py b/qiita_pet/handlers/artifact_handlers/base_handlers.py index 832c66694..8cc52d1af 100644 --- a/qiita_pet/handlers/artifact_handlers/base_handlers.py +++ b/qiita_pet/handlers/artifact_handlers/base_handlers.py @@ -160,7 +160,7 @@ def artifact_summary_get_request(user, artifact_id): buttons.append(btn_base % ('revert to sandbox', 'sandbox', 'Revert to sandbox')) - if user.level == 'admin': + if user.level == 'admin' and not study.autoloaded: if artifact.can_be_submitted_to_ebi: buttons.append( 'Make public' + ' ' + private_download_button % 2) + obs = artifact_summary_get_request(User('admin@foo.bar'), 2) + self.assertEqual(obs, exp) + study.autoloaded = False + # analysis artifact obs = artifact_summary_get_request(user, 8) exp = {'name': 'noname', diff --git a/qiita_pet/handlers/download.py b/qiita_pet/handlers/download.py index 6e31caac7..61fba9457 100644 --- a/qiita_pet/handlers/download.py +++ b/qiita_pet/handlers/download.py @@ -46,10 +46,7 @@ def _check_permissions(self, sid): study_info['message'], self.current_user.email, sid)) return Study(sid) - def _generate_files(self, header_name, accessions, filename): - text = "sample_name\t%s\n%s" % (header_name, '\n'.join( - ["%s\t%s" % (k, v) for k, v in accessions.items()])) - + def _finish_generate_files(self, filename, text): self.set_header('Content-Description', 'text/csv') self.set_header('Expires', '0') self.set_header('Cache-Control', 'no-cache') @@ -58,6 +55,12 @@ def _generate_files(self, header_name, accessions, filename): self.write(text) self.finish() + def _generate_files(self, header_name, accessions, filename): + text = "sample_name\t%s\n%s" % (header_name, '\n'.join( + ["%s\t%s" % (k, v) for k, v in accessions.items()])) + + self._finish_generate_files(filename, text) + def _list_dir_files_nginx(self, dirpath): """Generates a nginx list of files in the given dirpath for nginx @@ -327,6 +330,25 @@ def get(self, prep_template_id): 'ebi_experiment_accessions_study_%s_prep_%s.tsv' % (sid, pid)) +class DownloadSampleInfoPerPrep(BaseHandlerDownload): + @authenticated + @coroutine + @execute_as_transaction + def get(self, prep_template_id): + pid = int(prep_template_id) + pt = PrepTemplate(pid) + sid = pt.study_id + + self._check_permissions(sid) + + st = SampleTemplate(sid) + + text = st.to_dataframe(samples=list(pt)).to_csv(None, sep='\t') + + self._finish_generate_files( + 'sample_information_from_prep_%s.tsv' % pid, text) + + class DownloadUpload(BaseHandlerDownload): @authenticated @coroutine diff --git a/qiita_pet/handlers/upload.py b/qiita_pet/handlers/upload.py index d9542ee4b..a3595fa6d 100644 --- a/qiita_pet/handlers/upload.py +++ b/qiita_pet/handlers/upload.py @@ -9,7 +9,7 @@ from tornado.web import authenticated, HTTPError from os.path import join, exists -from os import remove +from os import remove, chmod from json import loads, dumps from collections import defaultdict @@ -142,6 +142,7 @@ def post(self, study_id): with open(ssh_key_fp, 'wb') as f: f.write(ssh_key) + chmod(ssh_key_fp, 0o600) qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') if method == 'list': diff --git a/qiita_pet/templates/study_ajax/prep_summary.html b/qiita_pet/templates/study_ajax/prep_summary.html index b104d8bbd..d4d7947c7 100644 --- a/qiita_pet/templates/study_ajax/prep_summary.html +++ b/qiita_pet/templates/study_ajax/prep_summary.html @@ -429,6 +429,7 @@

{{name}} - ID {{prep_id}} ({{data_type}}) Edit name Prep info + Sample info (only this prep) {% if is_submitted_to_ebi %} EBI experiment accessions {% end %} diff --git a/qiita_pet/templates/study_ajax/sample_prep_summary.html b/qiita_pet/templates/study_ajax/sample_prep_summary.html index 06661b3fc..af1672613 100644 --- a/qiita_pet/templates/study_ajax/sample_prep_summary.html +++ b/qiita_pet/templates/study_ajax/sample_prep_summary.html @@ -52,6 +52,44 @@ sampleTemplatePage.$refs.stElem.deleteSamples(to_delete); } } + + /** + * + * Function to download the data displayed in the page + * + */ + function download_summary(){ + var information = []; + var prep_names = {% raw columns %}; + + // adding header + var header = ['sample_id'] + $.each(prep_colums, function(i, prep){ + header.push(prep_names[prep]); + }); + information.push(header.join('\t')) + + // adding data + $.each(rows, function (i, row){ + var line = [row['sample']] + $.each(prep_colums, function(i, prep){ + line.push(row[prep]); + }); + information.push(line.join('\t')); + }); + + information = information.join('\n'); + + // downloading file + var element = document.createElement('a'); + element.setAttribute('href', 'data:text/plain;charset=utf-8,' + encodeURIComponent(information)); + element.setAttribute('download', 'study_{{study_id}}_sample_preparations_summary.tsv'); + element.style.display = 'none'; + document.body.appendChild(element); + element.click(); + document.body.removeChild(element); + } + var columns = [ { 'name': '', 'field': 'sample-delete', 'id': 'sample-delete', width: 69, sortable: false, formatter: linkFormatter }, @@ -271,6 +309,9 @@

Sample Summary


+ @@ -280,6 +321,7 @@

Sample Summary


Add sample column information to table +