lrsjng · glubsy · Nov 10, 2020 · Nov 11, 2020 · Nov 14, 2020 · Nov 14, 2020
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,19 @@
 # Changelog
 
 
+## v0.31.0 - *2020-02-07*
+
+* update all languages
+* drastically reduce disk usage of thumbnails
+* fix thumbnails for video files under 10 seconds
+* add thumbnail support for most common archive formats (requires Zip, Rar PHP modules)
+* fix broken thumbnails and previews for files with incorrect file extensions
+* add file type detection based on MIME (requires Fileinfo PHP module)
+* remove client's ability to request custom samples and thumbnails
+
+
+## v0.30.0 - *2020-01-24*
+
 * now require PHP 7.0.0+
 * fix archive-single-item problem
 * add header/footer search stop condition

diff --git a/README.md b/README.md
@@ -29,6 +29,18 @@ configurations).
 ~~~
 
 
+## Optional Dependencies
+
+* FFmpeg/FFprobe or AVconv/AVprobe
+* gm (GraphicsMagick) or convert (ImageMagick)
+* PHP FileInfo module
+* PHP Sqlite3 module
+* PHP Zip module
+* PHP [Rar](https://pecl.php.net/package/rar) module
+* du
+* tar
+* zip
+
 ## License
 
 The MIT License (MIT)

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "h5ai",
-  "version": "0.30.0",
+  "version": "0.31.0",
   "description": "Modern HTTP web server index.",
   "homepage": "https://larsjung.de/h5ai/",
   "author": "Lars Jung <[email protected]> (https://larsjung.de)",

diff --git a/src/_h5ai/private/cache/README.md b/src/_h5ai/private/cache/README.md
@@ -2,8 +2,9 @@
 
 Private cache.
 
-This directory is used for server side caching. To use caching make this
-directory writable for your webserver.
+This directory is used for server side caching.
 
-There is no critical data in here. You can savely remove any content. This
+To use caching make this directory writable by your webserver.
+
+There is no critical data in here. You can safely remove any content. This
 will clear the cache.
diff --git a/src/_h5ai/private/conf/options.json b/src/_h5ai/private/conf/options.json
@@ -234,11 +234,9 @@
     Show an image preview on click.
 
     - types: array of strings
-    - size: number, sample size, or false for original size
     */
     "preview-img": {
         "enabled": true,
-        "size": false,
         "types": ["img", "img-bmp", "img-gif", "img-ico", "img-jpg", "img-png", "img-raw", "img-svg"]
     },
 
@@ -292,7 +290,7 @@
     "preview-vid": {
         "enabled": true,
         "autoplay": true,
-        "types": ["vid-avi", "vid-flv", "vid-mkv", "vid-mov", "vid-mp4", "vid-mpg", "vid-webm"]
+        "types": ["vid-mkv", "vid-mov", "vid-mp4", "vid-mpg", "vid-webm"]
     },
 
     /*
@@ -349,25 +347,32 @@
 
     /*
     Show thumbnails for image files. Needs the "/_h5ai/public/cache" folder to be
-    writable for the web Server.
+    writable for the web Server. Removing a default value from an array of strings
+    results in adding it to the blocklist implicitly.
 
     - img: array of strings
     - mov: array of strings
     - doc: array of strings
+    - ar : array of strings
     - delay: number, delay in milliseconds after "dom-ready" before thumb-requesting starts
-    - size: number, size in pixel of the generated thumbnails
+    - size: number, height in pixel of the generated thumbnails
+    - seek: number, percentage of total video duration to seek into
     - exif: boolean, use included EXIF thumbs if possible
     - chunksize: int, number of thumbs per request
+    - blocklist: array of strings, do not generate thumbnails for these types
     */
     "thumbnails": {
         "enabled": true,
-        "img": ["img-bmp", "img-gif", "img-ico", "img-jpg", "img-png"],
-        "mov": ["vid-avi", "vid-flv", "vid-mkv", "vid-mov", "vid-mp4", "vid-mpg", "vid-webm"],
+        "img": ["img-bmp", "img-gif", "img-ico", "img-jpg", "img-png", "img-svg", "img-tiff"],
+        "mov": ["vid-avi", "vid-mkv", "vid-flv", "vid-swf", "vid-mov", "vid-mp4", "vid-mpg", "vid-webm", "vid-wmv", "vid-ts"],
         "doc": ["x-pdf", "x-ps"],
+        "ar": ["ar-zip", "ar-rar"],
         "delay": 1,
         "size": 240,
+        "seek": 50,
         "exif": false,
-        "chunksize": 20
+        "chunksize": 20,
+        "blocklist": []
     },
 
     /*

diff --git a/src/_h5ai/private/conf/types.json b/src/_h5ai/private/conf/types.json
@@ -1,75 +1,77 @@
 {
-    "ar":               ["*.tar.bz2", "*.crx"],
-    "ar-apk":           ["*.apk"],
-    "ar-deb":           ["*.deb"],
-    "ar-gz":            ["*.gz", "*.tar.gz", "*.tgz"],
-    "ar-rar":           ["*.rar"],
-    "ar-rpm":           ["*.rpm"],
-    "ar-tar":           ["*.tar"],
-    "ar-zip":           ["*.7z", "*.bz2", "*.jar", "*.lzma", "*.war", "*.z", "*.Z", "*.zip"],
-    "aud":              ["*.aif", "*.aiff", "*.flac", "*.m4a", "*.mid", "*.mp3", "*.mpa", "*.ra", "*.ogg", "*.wav", "*.wma"],
-    "aud-pls":          ["*.m3u", "*.m3u8", "*.pls"],
-    "bin":              ["*.class", "*.o", "*.so"],
-    "bin-exe":          ["*.bat", "*.cmd", "*.exe"],
-    "img":              ["*.xpm"],
-    "img-bmp":          ["*.bmp"],
-    "img-gif":          ["*.gif"],
-    "img-ico":          ["*.ico"],
-    "img-jpg":          ["*.jpg", "*.jpeg"],
-    "img-png":          ["*.png"],
-    "img-raw":          ["*.cr2", "*.nef"],
-    "img-svg":          ["*.svg"],
-    "img-tiff":         ["*.tiff"],
-    "txt":              ["*.text", "*.txt"],
-    "txt-build":        ["*.pom", "build.xml", "pom.xml"],
-    "txt-c":            ["*.c"],
-    "txt-cpp":          ["*.cpp"],
-    "txt-css":          ["*.css"],
-    "txt-diff":         ["*.diff", "*.patch"],
-    "txt-go":           ["*.go"],
-    "txt-h":            ["*.h"],
-    "txt-html":         ["*.htm", "*.html", "*.shtml", "*.xhtml"],
-    "txt-hpp":          ["*.hpp"],
-    "txt-java":         ["*.java"],
-    "txt-scala":        ["*.scala"],
-    "txt-js":           ["*.js"],
-    "txt-json":         ["*.json"],
-    "txt-less":         ["*.less"],
-    "txt-log":          ["*.log", "changelog*"],
-    "txt-md":           ["*.markdown", "*.md"],
-    "txt-php":          ["*.php"],
-    "txt-py":           ["*.py"],
-    "txt-rb":           ["*.rb"],
-    "txt-rss":          ["*.rss"],
-    "txt-rtf":          ["*.rtf"],
-    "txt-rust":         ["*.rs", "*.rlib"],
-    "txt-script":       ["*.conf", "*.bsh", "*.csh", "*.ini", "*.ksh", "*.sh", "*.shar", "*.tcl", "*.zsh"],
-    "txt-source":       [],
-    "txt-tex":          ["*.tex"],
-    "txt-vcal":         ["*.vcal"],
-    "txt-xml":          ["*.xml"],
-    "vid":              [],
-    "vid-avi":          ["*.avi"],
-    "vid-flv":          ["*.flv"],
-    "vid-mkv":          ["*.mkv"],
-    "vid-mov":          ["*.mov"],
-    "vid-mp4":          ["*.mp4", "*.m4v"],
-    "vid-mpg":          ["*.mpg"],
-    "vid-rm":           ["*.rm"],
-    "vid-swf":          ["*.swf"],
-    "vid-ts":           ["*.ts"],
-    "vid-vob":          ["*.vob"],
-    "vid-webm":         ["*.webm"],
-    "vid-wmv":          ["*.wmv"],
-    "x":                [],
-    "x-bak":            ["*.bak", "*~"],
-    "x-calc":           ["*.ods", "*.ots", "*.xlr", "*.xls", "*.xlsx"],
-    "x-disc":           ["*.cue", "*.iso"],
-    "x-doc":            ["*.doc", "*.docx", "*.odm", "*.odt", "*.ott"],
-    "x-draw":           ["*.drw"],
-    "x-eps":            ["*.eps"],
-    "x-pdf":            ["*.pdf"],
-    "x-pres":           ["*.odp", "*.otp", "*.pps", "*.ppt", "*.pptx"],
-    "x-ps":             ["*.ps"],
-    "x-psd":            ["*.psd"]
+    "ar":               { "glob": ["*.tar.bz2", "*.crx"]},
+    "ar-apk":           { "glob": ["*.apk"] },
+    "ar-deb":           { "glob": ["*.deb"] },
+    "ar-cbr":           { "glob": ["*.cbr", "*.cbz"], "mime": ["application/cbz", "application/cbr"] },
+    "ar-gz":            { "glob": ["*.gz", "*.tar.gz", "*.tgz"] },
+    "ar-rar":           { "glob": ["*.rar"], "mime": ["rar"] },
+    "ar-rpm":           { "glob": ["*.rpm"] },
+    "ar-tar":           { "glob": ["*.tar"] },
+    "ar-7z":            { "glob": ["*.7z"] },
+    "ar-zip":           { "glob": ["*.bz2", "*.jar", "*.lzma", "*.war", "*.z", "*.Z", "*.zip"], "mime": ["zip", "lzma"] },
+    "aud":              { "glob": ["*.aif", "*.aiff", "*.flac", "*.m4a", "*.mid", "*.mp3", "*.mpa", "*.ra", "*.ogg", "*.wav", "*.wma"] },
+    "aud-pls":          { "glob": ["*.m3u", "*.m3u8", "*.pls"] },
+    "bin":              { "glob": ["*.class", "*.o", "*.so"] },
+    "bin-exe":          { "glob": ["*.bat", "*.cmd", "*.exe"] },
+    "img":              { "glob": ["*.xpm"] },
+    "img-bmp":          { "glob": ["*.bmp"], "mime": ["image/bmp", "image/x-xbitmap", "image/x-portable-bitmap"] },
+    "img-gif":          { "glob": ["*.gif"], "mime": ["image/gif"] },
+    "img-ico":          { "glob": ["*.ico"], "mime": ["image/vnd.microsoft.icon"] },
+    "img-jpg":          { "glob": ["*.jpg", "*.jpeg"], "mime": ["jpeg"] },
+    "img-png":          { "glob": ["*.png"], "mime": ["png"] },
+    "img-raw":          { "glob": ["*.cr2", "*.nef"], "mime": ["cr2"] },
+    "img-svg":          { "glob": ["*.svg"], "mime": ["svg"] },
+    "img-tiff":         { "glob": ["*.tiff"], "mime": ["tiff"] },
+    "txt":              { "glob": ["*.text", "*.txt"] },
+    "txt-build":        { "glob": ["*.pom", "build.xml", "pom.xml"] },
+    "txt-c":            { "glob": ["*.c"] },
+    "txt-cpp":          { "glob": ["*.cpp"] },
+    "txt-css":          { "glob": ["*.css"] },
+    "txt-diff":         { "glob": ["*.diff", "*.patch"] },
+    "txt-go":           { "glob": ["*.go"] },
+    "txt-h":            { "glob": ["*.h"] },
+    "txt-html":         { "glob": ["*.htm", "*.html", "*.shtml", "*.xhtml"] },
+    "txt-hpp":          { "glob": ["*.hpp"] },
+    "txt-java":         { "glob": ["*.java"] },
+    "txt-scala":        { "glob": ["*.scala"] },
+    "txt-js":           { "glob": ["*.js"] },
+    "txt-json":         { "glob": ["*.json"] },
+    "txt-less":         { "glob": ["*.less"] },
+    "txt-log":          { "glob": ["*.log", "changelog*"] },
+    "txt-md":           { "glob": ["*.markdown", "*.md"] },
+    "txt-php":          { "glob": ["*.php"] },
+    "txt-py":           { "glob": ["*.py"] },
+    "txt-rb":           { "glob": ["*.rb"] },
+    "txt-rss":          { "glob": ["*.rss"] },
+    "txt-rtf":          { "glob": ["*.rtf"] },
+    "txt-rust":         { "glob": ["*.rs", "*.rlib"] },
+    "txt-script":       { "glob": ["*.conf", "*.bsh", "*.csh", "*.ini", "*.ksh", "*.sh", "*.shar", "*.tcl", "*.zsh"] },
+    "txt-source":       { "glob": [] },
+    "txt-tex":          { "glob": ["*.tex"] },
+    "txt-vcal":         { "glob": ["*.vcal"] },
+    "txt-xml":          { "glob": ["*.xml"] },
+    "vid":              { "glob": [] },
+    "vid-avi":          { "glob": ["*.avi"], "mime": ["x-msvideo"] },
+    "vid-flv":          { "glob": ["*.flv"], "mime": ["x-flv"] },
+    "vid-mkv":          { "glob": ["*.mkv"], "mime": ["video/x-matroska"] },
+    "vid-mov":          { "glob": ["*.mov"], "mime": ["mov"] },
+    "vid-mp4":          { "glob": ["*.mp4", "*.m4v"], "mime": ["video/mp4"] },
+    "vid-mpg":          { "glob": ["*.mpg"], "mime": ["video/mpeg"] },
+    "vid-rm":           { "glob": ["*.rm"], "mime": ["video/vnd.rn-realvideo", "application/vnd.rn-realmedia"] },
+    "vid-swf":          { "glob": ["*.swf"], "mime": ["application/x-shockwave-flash"] },
+    "vid-ts":           { "glob": ["*.ts"] },
+    "vid-vob":          { "glob": ["*.vob"] },
+    "vid-webm":         { "glob": ["*.webm"], "mime": ["video/webm"] },
+    "vid-wmv":          { "glob": ["*.wmv"], "mime": ["video/x-mx-video"] },
+    "x":                { "glob": [] },
+    "x-bak":            { "glob": ["*.bak", "*~"] },
+    "x-calc":           { "glob": ["*.ods", "*.ots", "*.xlr", "*.xls", "*.xlsx"] },
+    "x-disc":           { "glob": ["*.cue", "*.iso"] },
+    "x-doc":            { "glob": ["*.doc", "*.docx", "*.odm", "*.odt", "*.ott"] },
+    "x-draw":           { "glob": ["*.drw"] },
+    "x-eps":            { "glob": ["*.eps"] },
+    "x-pdf":            { "glob": ["*.pdf", "*.gs"], "mime": ["pdf", "ghostscript"] },
+    "x-pres":           { "glob": ["*.odp", "*.otp", "*.pps", "*.ppt", "*.pptx"] },
+    "x-ps":             { "glob": ["*.ps"] },
+    "x-psd":            { "glob": ["*.psd"] }
 }
diff --git a/src/_h5ai/private/php/core/class-api.php b/src/_h5ai/private/php/core/class-api.php
@@ -93,7 +93,7 @@ private function on_get() {
             Util::json_fail(Util::ERR_DISABLED, 'thumbnails disabled', !$this->context->query_option('thumbnails.enabled', false));
             Util::json_fail(Util::ERR_UNSUPPORTED, 'thumbnails not supported', !$this->setup->get('HAS_PHP_JPEG'));
             $thumbs = $this->request->query_array('thumbs');
-            $response['thumbs'] = $this->context->get_thumbs($thumbs);
+            [$response['thumbs'], $response['filetypes']] = $this->context->get_thumbs($thumbs);
         }
 
         Util::json_exit($response);

diff --git a/src/_h5ai/private/php/core/class-context.php b/src/_h5ai/private/php/core/class-context.php
@@ -15,6 +15,7 @@ class Context {
     private $setup;
     private $options;
     private $passhash;
+    private $types;
 
     public function __construct($session, $request, $setup) {
         $this->session = $session;
@@ -49,7 +50,10 @@ public function query_option($keypath = '', $default = null) {
     }
 
     public function get_types() {
-        return Json::load($this->setup->get('CONF_PATH') . '/types.json');
+        if (!isset($this->types)) {
+            $this->types = Json::load($this->setup->get('CONF_PATH') . '/types.json');
+        }
+        return $this->types;
     }
 
     public function login_admin($pass) {
@@ -188,15 +192,15 @@ public function get_items($href, $what) {
         $cache = [];
         $folder = Item::get($this, $this->to_path($href), $cache);
 
-        // add content of subfolders
+        // Add content of subfolders.
         if ($what >= 2 && $folder !== null) {
             foreach ($folder->get_content($cache) as $item) {
                 $item->get_content($cache);
             }
             $folder = $folder->get_parent($cache);
         }
 
-        // add content of this folder and all parent folders
+        // Add content of this folder and all parent folders.
         while ($what >= 1 && $folder !== null) {
             $folder->get_content($cache);
             $folder = $folder->get_parent($cache);
@@ -247,13 +251,42 @@ public function get_l10n($iso_codes) {
 
     public function get_thumbs($requests) {
         $hrefs = [];
+        $thumbs = [];
+        $filetypes = [];
+
+        $height = $this->options['thumbnails']['size'] ?? 240;
+        $width = floor($height * (4 / 3));
+
+        $db = new CacheDB($this->setup);
 
         foreach ($requests as $req) {
-            $thumb = new Thumb($this);
-            $hrefs[] = $thumb->thumb($req['type'], $req['href'], $req['width'], $req['height']);
-        }
+            if ($req['type'] === 'blocked') {
+                // FIXME the client is still free to choose what is blocked or not.
+                $hrefs[] = null;
+                $filetypes[] = null;
+                continue;
+            }
+            $path = $this->to_path($req['href']);
+            if (!array_key_exists($path, $thumbs)) {
+                $thumbs[$path] = new Thumb($this, $path, $req['type'], $db);
+            }
+            else if ($thumbs[$path]->type->name === 'file') {
+                // File has already been mime tested and cannot have a thumbnail
+                // Only applies if we request the same path again in the same request (security measure)
+                $hrefs[] = null;
+                $filetypes[] = 'file';
+                continue;
+            }
 
-        return $hrefs;
+            $hrefs[] = $thumbs[$path]->thumb($width, $height);
+
+            if ($thumbs[$path]->type->was_wrong()) {
+                $filetypes[] = $thumbs[$path]->type->name;
+            } else {
+                $filetypes[] = null; // No client-side update needed.
+            }
+        }
+        return [$hrefs, $filetypes];
     }
 
     private function prefix_x_head_href($href) {

diff --git a/src/_h5ai/private/php/core/class-setup.php b/src/_h5ai/private/php/core/class-setup.php
@@ -66,6 +66,11 @@ private function add_php_checks() {
             $has_php_jpeg = array_key_exists('JPEG Support', $infos) && $infos['JPEG Support'];
         }
         $this->set('HAS_PHP_JPEG', $has_php_jpeg);
+
+        $this->set('HAS_PHP_FILEINFO', extension_loaded('fileinfo'));
+
+        $this->set('HAS_PHP_ZIP', extension_loaded('zip'));
+        $this->set('HAS_PHP_RAR', extension_loaded('rar'));
     }
 
     private function add_app_metadata() {
@@ -133,7 +138,7 @@ private function add_sys_cmd_checks() {
                 $cmd = 'where';
             }
 
-            foreach (['avconv', 'convert', 'du', 'ffmpeg', 'gm', 'tar', 'zip'] as $c) {
+            foreach (['avconv', 'avprobe', 'convert', 'du', 'ffmpeg', 'ffprobe', 'gm', 'tar', 'zip'] as $c) {
                 $cmds[$c] = ($cmd !== false) && (Util::exec_0($cmd . ' ' . $c) || Util::exec_0($cmd . ' ' . $c . '.exe'));
             }
 
@@ -159,6 +164,9 @@ public function to_jsono($as_admin = false) {
                 'PHP_ARCH',
                 'HAS_PHP_EXIF',
                 'HAS_PHP_JPEG',
+                'HAS_PHP_FILEINFO',
+                'HAS_PHP_ZIP',
+                'HAS_PHP_RAR',
 
                 'SERVER_NAME',
                 'SERVER_VERSION',