Merge pull request #120 from JVT038/threading-attempt-two

Add some threading to the download process

README.md

@@ -1,10 +1,3 @@
-<!-- <div align="center" id="top"> 
-  <img src="./.github/app.gif" alt="metatube" />
-
-  &#xa0;
-
-  <a href="https://metatube.netlify.app">Demo</a>
-</div> -->
 <h1 align="center">MetaTube</h1>
 
 <p align="center">
@@ -265,8 +258,9 @@ Made with :heart: by <a href="https://github.com/JVT038" target="_blank">JVT038<
 - [X] Dark mode support
 - [X] Fix error `Synchronous XMLHttpRequest on the main thread is deprecated because of its detrimental effects to the end user’s experience. For more help http://xhr.spec.whatwg.org/` in overview
 - [X] Make sure the search for downloaded song field works
+- [x] Make sure the progress bar works properly in a Docker container, because it doesn't work properly rn
 
-### Not finished (I'll never finish this)
+### Not finished (I'll probably never finish this lol)
 
 - [ ] Add it to the PyPi library
 - [ ] Add support for sites other than YouTube
@@ -289,8 +283,10 @@ Made with :heart: by <a href="https://github.com/JVT038" target="_blank">JVT038<
 - [ ] Have a proper versioning system, because it's impossible to keep track of versions rn
 - [ ] Cache and store the segments and other video data, so next time of loading a video will be faster
 - [ ] Send websocket requests to one specific device / client only, to prevent duplicate websocket requests
-- [ ] Make sure the progress bar works properly in a Docker container, because it doesn't work properly rn.
-- [ ] Use proper queues and threading during download instead of the weird ping-pong system between the client and the server.
+- [ ] Use proper queues and threading during download instead of the weird ping-pong system between the client and the server.*
+- [ ] Add unit tests for the download, metadata logic, template / database stuff, config detection, automatic migrations
+
+* in progress
 &#xa0;
 
 ## Disclaimer

cli_to_api.py

@@ -0,0 +1,51 @@
+'''
+Source: https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py
+Allow direct execution
+'''
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import yt_dlp
+import yt_dlp.options
+
+create_parser = yt_dlp.options.create_parser
+
+
+def parse_patched_options(opts):
+    patched_parser = create_parser()
+    patched_parser.defaults.update({
+        'ignoreerrors': False,
+        'retries': 0,
+        'fragment_retries': 0,
+        'extract_flat': False,
+        'concat_playlist': 'never',
+    })
+    yt_dlp.options.create_parser = lambda: patched_parser
+    try:
+        return yt_dlp.parse_options(opts)
+    finally:
+        yt_dlp.options.create_parser = create_parser
+
+
+default_opts = parse_patched_options([]).ydl_opts
+
+
+def cli_to_api(opts, cli_defaults=False):
+    opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts
+
+    diff = {k: v for k, v in opts.items() if default_opts[k] != v}
+    if 'postprocessors' in diff:
+        diff['postprocessors'] = [pp for pp in diff['postprocessors']
+                                  if pp not in default_opts['postprocessors']]
+    return diff
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+
+    print('\nThe arguments passed translate to:\n')
+    pprint(cli_to_api(sys.argv[1:]))
+    print('\nCombining these with the CLI defaults gives:\n')
+    pprint(cli_to_api(sys.argv[1:], True))

metatube.py

@@ -14,6 +14,6 @@
     get_hub().NOT_ERROR += (KeyboardInterrupt,)
     try:
         print(u'Starting the webserver on http://%s:%s...'%(host, port))
-        socketio.run(app, str(host), int(port), log_output=strtobool(str(log_output)))
+        socketio.run(app, str(host), int(port), log_output=strtobool(str(log_output))) # type: ignore
     except KeyboardInterrupt:
         print('Stopped server because of KeyboardInterrupt')

metatube/__init__.py

@@ -1,3 +1,5 @@
+from gevent import monkey
+monkey.patch_all()
 from flask import Flask, json
 from flask.logging import default_handler
 from flask_sqlalchemy import SQLAlchemy
@@ -34,12 +36,12 @@ def create_app(config_class=Config):
     )
     app.register_error_handler(Exception, error)
     app.logger.removeHandler(default_handler)
-    app.logger.addHandler(logger)
+    app.logger.addHandler(console)
     console.setLevel(int(app.config["LOG_LEVEL"]))
     socket_log = logger if strtobool(str(app.config["SOCKET_LOG"])) == 1 else False
     db.init_app(app)
     migrate.init_app(app, db, compare_type=True, ping_interval=60)
-    socketio.init_app(app, json=json, engineio_logger=socket_log, logger=socket_log)
+    socketio.init_app(app, json=json, engineio_logger=socket_log, logger=socket_log, async_mode='gevent')
     app.register_blueprint(bp_overview)
     app.register_blueprint(bp_settings)
     if app.config.get('INIT_DB') == True:

metatube/database.py

@@ -19,9 +19,11 @@ def ffmpeg(self, ffmpeg_path):
         db.session.commit()
         logger.info('Set FFmpeg path to %s', ffmpeg_path)
 
+    @staticmethod
     def get_ffmpeg():
         return Config.query.get(1).ffmpeg_directory
 
+    @staticmethod
     def get_hwt():
         return Config.query.get(1).hardware_transcoding
 
@@ -32,7 +34,6 @@ def set_amount(self, amount):
 
     def set_spotify(self, spotify):
         self.spotify_api = spotify
-        print(spotify)
         db.session.commit()
         logger.info('Changed the Spotify API settings')
 
@@ -51,15 +52,19 @@ def set_hwtranscoding(self, hw_transcoding):
         db.session.commit()
         logger.info('Set hardware transcoding to %s', hw_transcoding)
 
+    @staticmethod
     def get_metadata_sources():
         return Config.query.get(1).metadata_sources
 
+    @staticmethod
     def get_spotify():
         return Config.query.get(1).spotify_api
 
+    @staticmethod
     def get_genius():
         return Config.query.get(1).genius_api
 
+    @staticmethod
     def get_max():
         return Config.query.get(1).amount
 
@@ -80,12 +85,15 @@ class Templates(db.Model):
     proxy_address = db.Column(db.String(128))
     proxy_port = db.Column(db.Integer)
 
+    @staticmethod
     def check_existing(value):
         return True if Templates.query.filter_by(name = value).count() > 0 else False
 
+    @staticmethod
     def counttemplates():
         return Templates.query.count()
 
+    @staticmethod
     def add(data):
         row = Templates(
             name = data["name"],
@@ -101,23 +109,26 @@ def add(data):
             proxy_password = data["proxy"]["password"],
             proxy_address = data["proxy"]["address"],
             proxy_port = data["proxy"]["port"]
-        )
+        ) # type: ignore
         db.session.add(row)
         db.session.commit()
         logger.info('Added template %s', data["name"])
         return row.id
 
+    @staticmethod
     def fetchtemplate(input_id):
         return Templates.query.filter_by(id = input_id).first()
 
+    @staticmethod
     def fetchalltemplates():
         return Templates.query.all()
 
     def delete(self):
         logger.info('Deleting template %s', self.name)
         db.session.delete(self)
         db.session.commit()
-
+
+    @staticmethod
     def searchdefault():
         return Templates.query.filter_by(default = True).first()
 
@@ -159,30 +170,38 @@ class Database(db.Model):
     audio_id = db.Column(db.String(128))
     youtube_id = db.Column(db.String(16), unique=True)
 
+    @staticmethod
     def searchrecords(query):
         return Database.query.filter(Database.name.like(query + "%")).all()
 
+    @staticmethod
     def itemtodict(item):
         dict = {}
         for column in item.__table__.columns:
                 dict[column.name] = str(getattr(item, column.name))
         return dict
 
+    @staticmethod
     def getrecords():
         return Database.query.all()
 
+    @staticmethod
     def fetchitem(input_id):
         return Database.query.filter_by(id = input_id).first()
 
+    @staticmethod
     def checkfile(filepath_input):
         return Database.query.filter_by(filepath = filepath_input).first()
 
+    @staticmethod
     def checkyt(youtube_id_input):
         return Database.query.filter_by(youtube_id = youtube_id_input).first()
 
+    @staticmethod
     def checktrackid(release_id_input):
         return Database.query.filter_by(audio_id = release_id_input).first()
 
+    @staticmethod
     def insert(data):
         row = Database(
             filepath = data["filepath"],
@@ -193,7 +212,7 @@ def insert(data):
             cover = data["image"],
             audio_id = data["track_id"],
             youtube_id = data["ytid"]
-        )
+        ) # type: ignore
         db.session.add(row)
         db.session.commit()
         logger.info('Inserted item %s into database', data["name"])
@@ -213,7 +232,7 @@ def update(self, data):
         logger.info('Updated item %s', data["name"])
         data["date"] = data["date"].strftime('%d-%m-%Y')
         sockets.overview({'msg': 'changed_metadata_db', 'data': data})
-        
+
     def updatefilepath(self, filepath):
         self.filepath = filepath
         db.session.commit()