From 049b0b2d73f28994960f04c24441e6f126275fe1 Mon Sep 17 00:00:00 2001 From: Yuwei Sun Date: Sat, 1 Jun 2024 11:04:22 -0400 Subject: [PATCH 01/10] add cleanup.sh and change email --- cleanup.py | 68 ------------------------------------------------------ cleanup.sh | 29 +++++++++++++++++++++++ job.py | 6 ++--- 3 files changed, 32 insertions(+), 71 deletions(-) delete mode 100644 cleanup.py create mode 100644 cleanup.sh diff --git a/cleanup.py b/cleanup.py deleted file mode 100644 index 785454e..0000000 --- a/cleanup.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import logging -from datetime import datetime, timedelta - -''' -This script is designed to manage and clean up old directories and files within the ./downloads folder. -Its primary purpose is to delete directories and their contents that are older than 72 hours to free up disk space -and maintain a clean file system. This script will iterate through all the files in the ./downloads folder -and delete all sub-folders and files if the folder was created more than 72 hours ago. If there are files in this folder, -the files will also be deleted one by one. -''' - -# Configure logging -logging.basicConfig( - level=logging.INFO, # INFO, WARNING, ERROR, and CRITICAL - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler() - ] -) - -# Define the path to the downloads directory -download_folder = "./downloads" -# cutoff time = 72 hours ago -cutoff = datetime.now() - timedelta(hours=72) - -# Function to echo messages with timestamp -def echo_message(message): - timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - os.system(f"echo '{message} - {timestamp}'") - - -# Traverse the download folder -# directory path, a list of sub-directories inside the current directory, and filenames -for root, dirs, files in os.walk(download_folder): - for dir_name in dirs: - dirpath = os.path.join(root, dir_name) - # Get modification time - dir_modified_time = datetime.fromtimestamp(os.path.getmtime(dirpath)) - # Check if the directory is older than 72 hours - if dir_modified_time < cutoff: - try: - # Attempt to remove the directory - os.rmdir(dirpath) - echo_message(f"Removed old directory: {dirpath}") # echo: remove folders - except OSError as e: - # If directory not empty, remove files inside first - for root_dir, sub_dirs, sub_files in os.walk(dirpath, topdown=False): - for name in sub_files: - filepath = os.path.join(root_dir, name) - try: - os.remove(filepath) - echo_message(f"Removed file: {filepath}") # echo: remove files - except Exception as e: - echo_message(f"Error removing file {filepath}: {e}") - for name in sub_dirs: - sub_dirpath = os.path.join(root_dir, name) - try: - os.rmdir(sub_dirpath) - echo_message(f"Removed directory: {sub_dirpath}") - except Exception as e: - echo_message(f"Error removing directory {sub_dirpath}: {e}") - # Finally, remove the main directory - try: - os.rmdir(dirpath) - echo_message(f"Removed old directory: {dirpath}") - except Exception as e: - echo_message(f"Error removing directory {dirpath}: {e}") \ No newline at end of file diff --git a/cleanup.sh b/cleanup.sh new file mode 100644 index 0000000..b6458e2 --- /dev/null +++ b/cleanup.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# This script is designed to manage and clean up old directories and files within the ./downloads folder. +# Its primary purpose is to delete directories and their contents that are older than 1 week (168 hours) to free up disk space +# and maintain a clean file system. This script will iterate through all the files in the ./downloads folder +# and delete all sub-folders and files if the folder was created more than 1 week ago. If there are files in this folder, +# the files will also be deleted one by one. + +# path to the downloads directory +download_folder="./downloads/" +# cutoffMins = 1 week ago = 168 hours ago (in seconds) +cutoffMins=$((168 * 60)) + +# Traverse the download folder and remove directories older than cutoff time +# -mindepth 1: Excludes the top-level directory, includes only subdirectories +find "$download_folder" -mindepth 1 -type d -mmin +$((cutoffMins)) | while read -r dirpath; do + echo "$dirpath" + if [ -d "$dirpath" ]; then + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + rm -rf "$dirpath" + result=$? # capture successful or failure + if [ $result -eq 0 ]; then + echo "Removed old directory: $dirpath - $timestamp" + else + echo "Error removing directory: $dirpath - $timestamp" + fi + fi +done + diff --git a/job.py b/job.py index 7739b8f..d67deaf 100644 --- a/job.py +++ b/job.py @@ -174,8 +174,8 @@ def runReform(target_dir, ref_fasta, ref_gff, timestamp, position, chrom, in_fas def send_email(email, timestamp): - # calculate 72h DDL - deadline = datetime.now() + timedelta(hours=72) + # calculate 168h DDL + deadline = datetime.now() + timedelta(hours=168) deadline_str = deadline.strftime('%Y-%m-%d %H:%M:%S') with j.app_context(): @@ -183,7 +183,7 @@ def send_email(email, timestamp): msg = Message(subject, sender='reform@nyu.edu', recipients=[email]) msg.html = f"""Reform job complete. Click here to download results. - The file will be available for the next 72 hours. + The file will be available for the next 1 week (168 hours). The deadline to download the file is {deadline_str}. If you do not download the file before this time, it will be deleted.""" mail.send(msg) From 35a0469f8d2a33d0ff9495a1ded9e8f662945354 Mon Sep 17 00:00:00 2001 From: Yuwei Sun Date: Sat, 1 Jun 2024 11:13:56 -0400 Subject: [PATCH 02/10] update corn job Setup --- INSTALL/install.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/INSTALL/install.sh b/INSTALL/install.sh index 5e84973..2c07ded 100644 --- a/INSTALL/install.sh +++ b/INSTALL/install.sh @@ -124,3 +124,9 @@ mkdir -p /var/log/reform cp ./conf/supervisor*ini /etc/supervisord.d/ systemctl start supervisord systemctl enable supervisord + + +# Create the following cron job +crontab -e +# add following command to the files, and save it. +0 * * * * /home/reform/venv/bin/python /home/reform/reformWeb/cleanup.sh From 97f0914eb3a373219498315f3c962ab5400fe717 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 4 Jun 2024 13:35:30 -0400 Subject: [PATCH 03/10] Update cron time Instead of cleaning up every hour it cleans once daily --- INSTALL/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL/install.sh b/INSTALL/install.sh index 2c07ded..1312d8b 100644 --- a/INSTALL/install.sh +++ b/INSTALL/install.sh @@ -129,4 +129,4 @@ systemctl enable supervisord # Create the following cron job crontab -e # add following command to the files, and save it. -0 * * * * /home/reform/venv/bin/python /home/reform/reformWeb/cleanup.sh +0 1 * * * /home/reform/venv/bin/python /home/reform/reformWeb/cleanup.sh From 54ba08369aebcf788c001419b5acc0bf4a980beb Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 4 Jun 2024 13:36:18 -0400 Subject: [PATCH 04/10] python to bash call for cron --- INSTALL/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL/install.sh b/INSTALL/install.sh index 1312d8b..496c0d6 100644 --- a/INSTALL/install.sh +++ b/INSTALL/install.sh @@ -129,4 +129,4 @@ systemctl enable supervisord # Create the following cron job crontab -e # add following command to the files, and save it. -0 1 * * * /home/reform/venv/bin/python /home/reform/reformWeb/cleanup.sh +0 1 * * * /bin/bash /home/reform/reformWeb/cleanup.sh From 4805210d919d2a5c33705c008dc675d051b5ff60 Mon Sep 17 00:00:00 2001 From: Yuwei Sun Date: Wed, 5 Jun 2024 15:09:47 -0400 Subject: [PATCH 05/10] update email formatting --- cleanup.sh | 4 ++-- job.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cleanup.sh b/cleanup.sh index b6458e2..d1ba6bd 100644 --- a/cleanup.sh +++ b/cleanup.sh @@ -9,12 +9,12 @@ # path to the downloads directory download_folder="./downloads/" # cutoffMins = 1 week ago = 168 hours ago (in seconds) -cutoffMins=$((168 * 60)) +cutoffMins=$((169 * 60)) # add 1 hour to handle edge case, which is job finished at 1:00 am. # Traverse the download folder and remove directories older than cutoff time # -mindepth 1: Excludes the top-level directory, includes only subdirectories find "$download_folder" -mindepth 1 -type d -mmin +$((cutoffMins)) | while read -r dirpath; do - echo "$dirpath" + # echo "$dirpath" if [ -d "$dirpath" ]; then timestamp=$(date '+%Y-%m-%d %H:%M:%S') rm -rf "$dirpath" diff --git a/job.py b/job.py index d67deaf..d338e75 100644 --- a/job.py +++ b/job.py @@ -176,14 +176,14 @@ def runReform(target_dir, ref_fasta, ref_gff, timestamp, position, chrom, in_fas def send_email(email, timestamp): # calculate 168h DDL deadline = datetime.now() + timedelta(hours=168) - deadline_str = deadline.strftime('%Y-%m-%d %H:%M:%S') + deadline_str = deadline.strftime('%B %d, %Y') with j.app_context(): subject = f"Reform Results - Download Deadline: {deadline_str}" msg = Message(subject, sender='reform@nyu.edu', recipients=[email]) msg.html = f"""Reform job complete. Click here to download results. - The file will be available for the next 1 week (168 hours). + The file will be available for the next 7 days. The deadline to download the file is {deadline_str}. If you do not download the file before this time, it will be deleted.""" mail.send(msg) From 0c7c0074d2f0274f80efb2b5091723b5d6290f83 Mon Sep 17 00:00:00 2001 From: Yuwei Sun Date: Wed, 5 Jun 2024 15:16:28 -0400 Subject: [PATCH 06/10] resolve conflicts --- INSTALL/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL/install.sh b/INSTALL/install.sh index 2c07ded..1312d8b 100644 --- a/INSTALL/install.sh +++ b/INSTALL/install.sh @@ -129,4 +129,4 @@ systemctl enable supervisord # Create the following cron job crontab -e # add following command to the files, and save it. -0 * * * * /home/reform/venv/bin/python /home/reform/reformWeb/cleanup.sh +0 1 * * * /home/reform/venv/bin/python /home/reform/reformWeb/cleanup.sh From 9c3a4140f787dc82aac4de2b06201b1f591c8cdb Mon Sep 17 00:00:00 2001 From: Yuwei Sun Date: Fri, 7 Jun 2024 13:31:28 -0400 Subject: [PATCH 07/10] add comments --- app.py | 42 ++++++++++++++++++++++-------------------- cleanup.sh | 5 +++-- forms.py | 4 ++-- job.py | 25 ++++++++++++++----------- 4 files changed, 41 insertions(+), 35 deletions(-) diff --git a/app.py b/app.py index b4f4c85..252064c 100644 --- a/app.py +++ b/app.py @@ -14,13 +14,14 @@ app.secret_key = 'development key' UPLOAD_FOLDER = './uploads' +# The type of file that needs to be uploaded to the server by user. UPLOAD_FILES = ['in_fasta', 'in_gff'] -DOWNLOAD_FILES = ['ref_fasta', 'ref_gff'] - +# Route for submitting data on the production site. @app.route('/', methods=['GET', 'POST']) def submit(): form = SubmitJob(request.form) + # Validate user input according to the validation rules defined in forms.py. if request.method == 'POST' and form.validate(): if (request.files['downstream_fasta'].filename or request.files['upstream_fasta'].filename) and request.form[ 'position']: @@ -62,9 +63,12 @@ def submit(): upload(target_dir, 'upstream_fasta') upload(target_dir, 'downstream_fasta') + # (4) Send the job to the backend + # Connect to the Redis server and intial a queue redis_conn = Redis() q = Queue(connection=redis_conn, default_timeout=3000) + # Push job function and parameters into RQ job = q.enqueue(redisjob, args=(target_dir, timestamp, request.form['email'], @@ -79,16 +83,17 @@ def submit(): result_ttl=-1, job_timeout=3000 ) + # (5) Update record in the database and flush message on the user front-end db_update(timestamp, "jobID", job.get_id()) flash(Markup('JOB ID: ' + job.get_id() + '
' + "You'll receive an e-mail when job is done with download link"), 'info') return render_template('form.html', form=form) -# test site +# Route for submitting data on the test site @app.route('/test', methods=['GET', 'POST']) def submit_test(): - # Default in_fasta and in_gff + # Path for local files DEFAULT_FILES = { 'ref_fasta': './staticData/ref/Mus_musculus.GRCm38.dna.toplevel.fa', 'ref_gff': './staticData/ref/Mus_musculus.GRCm38.88.gff3', @@ -97,8 +102,8 @@ def submit_test(): 'upstream_fasta': './staticData/up-down-seq/test-up.fa', 'downstream_fasta': './staticData/up-down-seq/test-down.fa' } - - form = Testjob(request.form) # test job + # Validate user input based on test site rule + form = Testjob(request.form) if request.method == 'POST' and form.validate(): if (request.files['downstream_fasta'].filename or request.files['upstream_fasta'].filename) and request.form[ 'position']: @@ -108,11 +113,6 @@ def submit_test(): if not (request.files['downstream_fasta'].filename and request.files['upstream_fasta'].filename): flash("Error: Must enter both upstream and downstream", 'error') return redirect(url_for('submit')) - # # comment out the condition check, since we allowed default up/down stream fasta - # if not (request.files['downstream_fasta'].filename or request.files['upstream_fasta'].filename) and not \ - # request.form['position']: - # flash("Error: You must provide either the position, or the upstream and downstream sequences.", 'error') - # return redirect(url_for('submit')) else: # User Submits Job # # (1) Create unique ID for each submission @@ -131,23 +131,22 @@ def submit_test(): if not verified: return redirect(url_for('submit')) - # Upload Files to UPLOAD_DIR/timestamp/ and save the name into uploaded_files or use local files + # Choose to upload new files or use local files if verified: - # Storing all files that will be passed to run.sh uploaded_files = {} - for file_key in UPLOAD_FILES: # upload inserted files + for file_key in UPLOAD_FILES: uploaded_files[file_key] = upload_test(target_dir, file_key, DEFAULT_FILES) - # set defualt None to up/down stream fasta + # Set defualt None to up/down stream fasta for file_key in ['upstream_fasta', 'downstream_fasta']: uploaded_files['upstream_fasta'] = None uploaded_files['downstream_fasta'] = None - + + # Uploaded upstream/downstream files when position is not provided if not request.form['position']: - # Handle case where position is not provided and upstream/downstream files are required for file_key in ['upstream_fasta', 'downstream_fasta']: uploaded_files[file_key] = upload_test(target_dir, file_key, DEFAULT_FILES) - # Replace Ref Sequence files with local file realpath + # Replace Ref Sequence with local path if example ftp detected if request.form['ref_fasta'] == 'ftp://ftp.ensembl.org/pub/release-88/fasta/mus_musculus/dna/Mus_musculus.GRCm38.dna.toplevel.fa.gz': uploaded_files['ref_fasta'] = DEFAULT_FILES['ref_fasta'] else: @@ -157,8 +156,9 @@ def submit_test(): else: uploaded_files['ref_gff'] = request.form['ref_gff'] - # Use same Redis for production site and test site - redis_conn = Redis() # initializes a connection to the default Redis server running on localhost + # (4) Send job to the backend + # Use the redis queue as same as production site + redis_conn = Redis() q = Queue(connection=redis_conn, default_timeout=3000) job = q.enqueue(redisjob, args=(target_dir, @@ -176,11 +176,13 @@ def submit_test(): result_ttl=-1, job_timeout=3000 ) + # (5) Update record in the database and flush message on the user front-end db_update(timestamp, "jobID", job.get_id()) flash(Markup('JOB ID: ' + job.get_id() + '
' + "You'll receive an e-mail when job is done with download link"), 'info') return render_template('form.html', form=form) +# Route for downloading result @app.route('/download/') def downloadFile(timestamp): try: diff --git a/cleanup.sh b/cleanup.sh index d1ba6bd..95d3272 100644 --- a/cleanup.sh +++ b/cleanup.sh @@ -6,7 +6,7 @@ # and delete all sub-folders and files if the folder was created more than 1 week ago. If there are files in this folder, # the files will also be deleted one by one. -# path to the downloads directory +# Define path to the downloads directory download_folder="./downloads/" # cutoffMins = 1 week ago = 168 hours ago (in seconds) cutoffMins=$((169 * 60)) # add 1 hour to handle edge case, which is job finished at 1:00 am. @@ -18,7 +18,8 @@ find "$download_folder" -mindepth 1 -type d -mmin +$((cutoffMins)) | while read if [ -d "$dirpath" ]; then timestamp=$(date '+%Y-%m-%d %H:%M:%S') rm -rf "$dirpath" - result=$? # capture successful or failure + # Captures the success of the rm command + result=$? if [ $result -eq 0 ]; then echo "Removed old directory: $dirpath - $timestamp" else diff --git a/forms.py b/forms.py index be092e2..be4783c 100644 --- a/forms.py +++ b/forms.py @@ -4,7 +4,7 @@ ALLOWED_EXTENSIONS = {'fa', 'gff', 'gff3', 'gtf', 'fasta', 'fna', 'tar', 'gz'} - +# Use it for production site class SubmitJob(Form): email = StringField('Email Address', description="When job is complete this e-mail will receive the download links", @@ -74,7 +74,7 @@ class SubmitJob(Form): InputRequired() ]) -# form for test job +# Use it for test site class Testjob(Form): email = StringField('Email Address', description="When job is complete this e-mail will receive the download links", diff --git a/job.py b/job.py index d338e75..5209489 100644 --- a/job.py +++ b/job.py @@ -75,11 +75,11 @@ def redisjob1(target_dir, timestamp, email, chrom, upstream_fasta, downstream_fa print("ERROR: ") db_update(timestamp, "status", "failed running reform") - +# Determine if this file is accepted by checking the file extension def allowed_file(filename): return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS - +# Verify that the file is uploaded to the form and it is allowed def verify_uploads(file): fileObj = request.files[file] @@ -92,10 +92,10 @@ def verify_uploads(file): flash('Invalid File Type for ' + file, 'error') return False -# verify upload files for test site +# verify_uploads() for test site def verify_test_uploads(file): fileObj = request.files[file] - + # Not require user to upload files if fileObj.filename == '': # flash('No ' + file + ' file selected for uploading', 'error') # return False @@ -107,7 +107,7 @@ def verify_test_uploads(file): return False - +# Upload files to uploads/timestamp def upload(target_dir, file): fileObj = request.files[file] # make the directory based on timestamp @@ -115,12 +115,14 @@ def upload(target_dir, file): # save the file fileObj.save(os.path.join(target_dir, secure_filename(fileObj.filename))) -# upload file function for test site + +# upload() for test site def upload_test(target_dir, file_key, default_files): # if file is empty (indicated use default file), fileObj set to None fileObj = request.files[file_key] if file_key in request.files else None os.makedirs(target_dir, exist_ok=True) # dirs for upload files + # User provided new files in form if fileObj: # save the uploaded file filename = secure_filename(fileObj.filename) @@ -128,11 +130,12 @@ def upload_test(target_dir, file_key, default_files): fileObj.save(file_path) return fileObj.filename else: - # Use the default file if no file was uploaded, pass realpath + # Use default file if no file was uploaded src = os.path.abspath(default_files[file_key]) - dst = os.path.join(target_dir, os.path.basename(src)) # link name in target_dir - if not os.path.exists(dst): # Only create the symlink if it doesn't already exist - os.symlink(src, dst) # Create a soft link + dst = os.path.join(target_dir, os.path.basename(src)) + # Create soft link in uploads/timestamp + if not os.path.exists(dst): + os.symlink(src, dst) return os.path.basename(src) @@ -174,7 +177,7 @@ def runReform(target_dir, ref_fasta, ref_gff, timestamp, position, chrom, in_fas def send_email(email, timestamp): - # calculate 168h DDL + # Set DDL to 1 week later (168hrs) deadline = datetime.now() + timedelta(hours=168) deadline_str = deadline.strftime('%B %d, %Y') From a71d5f74b04b5f1b90042097d9abaae8e8eafd68 Mon Sep 17 00:00:00 2001 From: Yuwei Sun Date: Fri, 7 Jun 2024 13:32:25 -0400 Subject: [PATCH 08/10] add staticData setup in install.sh --- INSTALL/install.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/INSTALL/install.sh b/INSTALL/install.sh index 496c0d6..d2d7271 100644 --- a/INSTALL/install.sh +++ b/INSTALL/install.sh @@ -130,3 +130,18 @@ systemctl enable supervisord crontab -e # add following command to the files, and save it. 0 1 * * * /bin/bash /home/reform/reformWeb/cleanup.sh + +# Create local File folder for test site +mkdir -p /home/reform/reformWeb/staticData +cd /home/reform/reformWeb/staticData +# Create directory for reference sequences and upload Exampl Ref Sequences +# for example: +mkdir ref +cd ref +wget --no-check-certificate -nv ftp://ftp.ensembl.org/pub/release-88/fasta/mus_musculus/dna/Mus_musculus.GRCm38.dna.toplevel.fa.gz +wget --no-check-certificate -nv ftp://ftp.ensembl.org/pub/release-88/gff3/mus_musculus/Mus_musculus.GRCm38.88.gff3.gz +# Create directory for inserted and up-down-seq +mkdir ../inserted +mkdir ../up-down-seq +# Please upload files to inserted and up-down-seq. +# And change the local files path in jobs.py line 97 \ No newline at end of file From 76e55e5c0b2930e2e80f4a81eaf66961c6e89616 Mon Sep 17 00:00:00 2001 From: Yuwei Sun Date: Fri, 7 Jun 2024 14:43:39 -0400 Subject: [PATCH 09/10] update troubleshooting --- README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/README.md b/README.md index 5293487..cca72c3 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,20 @@ Installation script found at [`./INSTALL/install.sh`](https://github.com/gencore - After submission, the data and files will be gathered and submitted to a message queue to run [*ref*orm](https://github.com/gencorefacility/reform) - If sucessful or failure, an e-mail will be sent to the e-mail address provided +## Using the Test Site +Test site is designed to help developers and researchers test updates more easily and quickly by using local files and default populated addresses. Files used for testing need to be uploaded in advance according to the requirements in INSTALL/install.sh. + +- Fill the form with the required parameters + * `email` Use the default test email address. + * `chrom` Default is 1. + * `position` User-provided. + * `upstream_fasta` If no file is uploaded, use the local upstream_fasta file. + * `downstream_fasta` If no file is uploaded, use the local downstream_fasta file. + * `in_fasta` If no file is uploaded, use the local in_fasta file. + * `in_gff` If no file is uploaded, use the local in_gff file. + * `ref_fasta` Use the example FTP link or the local ref_fasta file. + * `ref_gff` Use the example FTP link or the local ref_gff file. + ## Troubleshooting ### Error or Unexepected behavior when submitting form @@ -49,6 +63,30 @@ reform: started ### How to monitor logs Logs are written to `/var/log/reform`. Most of the echo outs are controlled in `run.sh`. Edit as needed for debugging. +* `reform.err.log` Contains errors from Flask (e.g., errors in app.py) and ERROR, INFO logs from Gunicorn. +* `reform.out.log` Contains standard output from Gunicorn. +* `worker.out.log` Contains echo outputs from run.sh, indicating the current task of the worker. +* `worker.err.log` Contains errors from reform.py and FTP download records. + +### Receive an Email with a Zip of an Empty Download Folder +An empty download folder means `reform` didn't finish, resulting in empty `result` and `download` folders. To debug this error, please check `/var/log/reform` for more information. Here are some common issues: + +**In `worker.err.log`:** + +1. **`OSError: [Errno 28] No space left on device`**: + - There is no free space on the server. Please clean up files in `/data/downloads/`, `/data/results/`, and `/data/uploads/`. +2. **`FileNotFoundError: [Errno 2] No such file or directory`**: + - Please check the corresponding folder, especially when using the test site. + +**In `worker.err.log`:** + +1. **`./run.sh: line XX: syntax error near unexpected token`**: + - An invalid input has been passed into `run.sh`, usually an invalid FTP link. + +### Get Error Status Code when access reform web +Error status code usually represent the web service has not run as expectly. +1. `502 Bad Gateway`: This error typically occurs due to communication problems between servers. It might be helpful to check `app.py` for any unhandled routes. +2. `500 Internal Server Error`: This error indicates that the server encountered an unexpected condition that prevented it from fulfilling the request. Check `/var/log/reform/reform.err.log` for detailed information. ### Cannot SSH SSH is only accessible on NYU VPN From e788d1ce38cc5e66c14919d99e5ee62cb9af8f1a Mon Sep 17 00:00:00 2001 From: Yuwei Sun Date: Mon, 10 Jun 2024 14:38:33 -0400 Subject: [PATCH 10/10] add gitignore for staticData --- .gitignore | 3 +++ INSTALL/install.sh | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 56cf2b3..a4d7f20 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,6 @@ venv.bak/ # mypy .mypy_cache/ /database.db + +# staticData +/staticData/ \ No newline at end of file diff --git a/INSTALL/install.sh b/INSTALL/install.sh index d2d7271..64f0c1a 100644 --- a/INSTALL/install.sh +++ b/INSTALL/install.sh @@ -143,5 +143,5 @@ wget --no-check-certificate -nv ftp://ftp.ensembl.org/pub/release-88/gff3/mus_mu # Create directory for inserted and up-down-seq mkdir ../inserted mkdir ../up-down-seq -# Please upload files to inserted and up-down-seq. -# And change the local files path in jobs.py line 97 \ No newline at end of file +# Please upload files to inserted and up-down-seq folders, and relevent test files can be found in https://github.com/gencorefacility/reform/tree/master/test_data +# After upload the inserted Seq and Up&Down Seq, please change the files path in app.py (https://github.com/gencorefacility/reformWeb/blob/7083d7fe682db149775f021f61a3bfdfdb57dc83/app.py#L92C1-L92C5) \ No newline at end of file