Merge pull request #420 from atlas-bi/dev

migrations/versions/6b8978d637c5_.py

@@ -0,0 +1,35 @@
+"""empty message
+
+Revision ID: 6b8978d637c5
+Revises: 0fe33867bdbf
+Create Date: 2023-08-03 12:53:03.650434
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '6b8978d637c5'
+down_revision = '0fe33867bdbf'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('connection_sftp', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('key_password', sa.Text(), nullable=True))
+
+    # ### end Alembic commands ###
+
+    # move passwords on connections that have a key into the key_password field and clear the
+    # password field.
+    op.execute('update connection_sftp set key_password=password where key is not null')
+    op.execute('update connection_sftp set password=null where key is not null and key_password=password')
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('connection_sftp', schema=None) as batch_op:
+        batch_op.drop_column('key_password')
+
+    # ### end Alembic commands ###

package.json

@@ -46,7 +46,7 @@
     "gulp-purgecss": "5.0.0",
     "gulp-rename": "2.0.0",
     "gulp-uglify": "3.0.2",
-    "prettier": "2.8.8",
+    "prettier": "3.0.0",
     "semantic-release": "21.0.5"
   },
   "homepage": "https://atlas-hub.atlas.bi",
@@ -80,5 +80,5 @@
     "test:lint": "poetry run tox -e lint",
     "test:prettier": "prettier --check --config .prettierrc \"web/**/*.{ts,css,less,scss,js,json,md,yaml,html}\""
   },
-  "version": "2.10.3-alpha.1"
+  "version": "2.10.5"
 }

pyproject.toml

@@ -120,7 +120,7 @@ cryptography = "^41.0.0"
 flask-assets = {git = "https://github.com/christopherpickering/flask-assets.git"}
 flask-redis = "^0.4.0"
 flask-session2 = "^1.2.0"
-gevent = "^22.0.0"
+gevent = "^23.0.0"
 gunicorn = "^20.1.0"
 is-safe-url = "^1.0"
 paramiko = "^3.0.0"

runner/model.py

@@ -374,6 +374,7 @@ class ConnectionSftp(db.Model):
     username = db.Column(db.String(120), nullable=True)
     key = db.Column(db.String(8000), nullable=True)
     password = db.Column(db.Text, nullable=True)
+    key_password = db.Column(db.Text, nullable=True)
     task = db.relationship(
         "Task",
         backref="destination_sftp_conn",

runner/scripts/em_messages.py

@@ -110,6 +110,7 @@ def __init__(
                         success=0,
                         date=date,
                         logs=logs,
+                        org=app.config["ORG_NAME"],
                     ),
                     short_message=task.email_error_message
                     or f"Atlas Hub task {task} failed.",

runner/scripts/em_python.py

@@ -65,8 +65,8 @@ def run(self) -> Optional[List[Path]]:
 
         # if output is not a file list, then swallow it.
 
-        if isinstance(self.output, List):
-            return self.output
+        if isinstance(eval(self.output), List):
+            return [Path(x) for x in eval(self.output)]
         return None
 
     def __build_env(self) -> None:

runner/scripts/em_sftp.py

@@ -39,8 +39,8 @@ def connection_key(connection: ConnectionSftp) -> Optional[paramiko.pkey.PKey]:
 
             key = paramiko.RSAKey.from_private_key_file(
                 key_file.name,
-                password=em_decrypt(connection.password, app.config["PASS_KEY"])
-                if connection.password
+                password=em_decrypt(connection.key_password, app.config["PASS_KEY"])
+                if connection.key_password
                 else None,
             )
 
@@ -54,6 +54,7 @@ def connection_json(connection: ConnectionSftp) -> Dict:
         "port": connection.port or 22,
         "key": em_decrypt(connection.key, app.config["PASS_KEY"]),
         "password": em_decrypt(connection.password, app.config["PASS_KEY"]),
+        "key_password": em_decrypt(connection.key_password, app.config["PASS_KEY"]),
         "username": str(connection.username),
     }
 
@@ -74,15 +75,28 @@ def connect(connection: ConnectionSftp) -> Tuple[Transport, SFTPClient]:
 
                 key = connection_key(connection)
 
-                transport.connect(
-                    username=str(connection.username),
-                    password=(
+                if key and connection.password and connection.username:
+                    transport.start_client(event=None, timeout=15)
+                    transport.get_remote_server_key()
+                    transport.auth_publickey(connection.username, key, event=None)
+                    transport.auth_password(
+                        connection.username,
                         em_decrypt(connection.password, app.config["PASS_KEY"])
-                        if key is None
-                        else ""
-                    ),
-                    pkey=key,
-                )
+                        if connection.password
+                        else "",
+                        event=None,
+                    )
+
+                else:
+                    transport.connect(
+                        username=str(connection.username),
+                        password=(
+                            em_decrypt(connection.password, app.config["PASS_KEY"])
+                            if connection.password
+                            else ""
+                        ),
+                        pkey=key,
+                    )
 
                 conn = paramiko.SFTPClient.from_transport(transport)
                 if conn is None:

runner/scripts/task_runner.py

@@ -746,7 +746,7 @@ def __send_email(self) -> None:
             .all()
         )
 
-        date = str(datetime.datetime.now())
+        date = datetime.datetime.now()
 
         # pylint: disable=broad-except
         try:
@@ -820,6 +820,7 @@ def __send_email(self) -> None:
                         logs=logs,
                         output=output,
                         host=app.config["WEB_HOST"],
+                        org=app.config["ORG_NAME"],
                     ),
                     short_message=self.task.email_completion_message
                     or f"Atlas Hub job {self.task} completed successfully.",

runner/templates/email/email.html.j2

@@ -23,22 +23,22 @@
                    font-size: 35px;
                    margin:0;
                    font-weight:lighter">Atlas / Hub</h2>
-        <div style="color: #6c7378;font-size: 15px;">&#169; 2020 Riverside Healthcare</div>
+        <div style="color: #6c7378;font-size: 15px;">&#169;{{ date.year }} {{ org }}</div>
     </div>
     <!-- task title -->
     <h3>
         Project:
         <a style="color: #6c7378;
                   text-decoration: none"
-           href="{{ host }}/project/{{ task.projectId }}">{{ task.project.name }}</a>
+           href="{{ host }}/project/{{ task.project.id }}">{{ task.project.name }}</a>
         / Task:
         <a style="color: #6c7378;
                   text-decoration: none"
            href="{{ host }}/task/{{ task.id }}">{{ task.name }}</a>
         / Run:
         <a style="color: #6c7378;
                   text-decoration: none"
-           href="{{ host }}/task/{{ task.id }}/log/{{ task.lastRunJobId }}">{{ task.last_run_job_id }}</a>
+           href="{{ host }}/task/{{ task.id }}#tasklog">{{ task.last_run_job_id }}</a>
     </h3>
     <!-- message body -->
     {% if success == 1 %}

runner/web/web.py

@@ -221,7 +221,7 @@ def send_email(run_id: int, file_id: int) -> dict:
 
         # send the file
 
-        date = str(datetime.datetime.now())
+        date = datetime.datetime.now()
 
         template = env.get_template("email/email.html.j2")
 
@@ -235,10 +235,7 @@ def send_email(run_id: int, file_id: int) -> dict:
             + " / Task: "
             + task.name,
             message=template.render(
-                task=task,
-                success=1,
-                date=date,
-                logs=[],
+                task=task, success=1, date=date, logs=[], org=app.config["ORG_NAME"]
             ),
             attachments=[x.name for x in downloaded_files],
         )

scheduler/model.py

@@ -374,6 +374,7 @@ class ConnectionSftp(db.Model):
     username = db.Column(db.String(120), nullable=True)
     key = db.Column(db.String(8000), nullable=True)
     password = db.Column(db.Text, nullable=True)
+    key_password = db.Column(db.Text, nullable=True)
     task = db.relationship(
         "Task",
         backref="destination_sftp_conn",

web/model.py

@@ -374,6 +374,7 @@ class ConnectionSftp(db.Model):
     username = db.Column(db.String(120), nullable=True)
     key = db.Column(db.String(8000), nullable=True)
     password = db.Column(db.Text, nullable=True)
+    key_password = db.Column(db.Text, nullable=True)
     task = db.relationship(
         "Task",
         backref="destination_sftp_conn",

web/static/assets/base.scss

@@ -25,9 +25,20 @@
 
 $title-family: 'Rasa', 'Georgia', 'Times New Roman', serif;
 
-$family-sans-serif: 'Inter', -apple-system, 'BlinkMacSystemFont', 'Segoe UI',
-  'Roboto', 'Helvetica Neue', 'Lucida Sans Unicode', 'Lucida Grande', 'Arial',
-  sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
+$family-sans-serif:
+  'Inter',
+  -apple-system,
+  'BlinkMacSystemFont',
+  'Segoe UI',
+  'Roboto',
+  'Helvetica Neue',
+  'Lucida Sans Unicode',
+  'Lucida Grande',
+  'Arial',
+  sans-serif,
+  'Apple Color Emoji',
+  'Segoe UI Emoji',
+  'Segoe UI Symbol';
 
 @import '../../../node_modules/bulma-checkradio/src/sass';
 @import '../../../node_modules/bulma/bulma';