Bring back Mercury (#569)

* Get images working again * Re-add custom extractors
jocmp · Dec 2, 2024 · 97a99f0 · 97a99f0
1 parent 5f08dbc
commit 97a99f0
Show file tree

Hide file tree

Showing 16 changed files with 179 additions and 25 deletions.
diff --git a/app/mercury.web.js b/app/mercury.web.js
diff --git a/app/src/main/java/com/capyreader/app/ui/articles/ArticleScreenViewModel.kt b/app/src/main/java/com/capyreader/app/ui/articles/ArticleScreenViewModel.kt
@@ -17,7 +17,6 @@ import com.jocmp.capy.ArticleStatus
 import com.jocmp.capy.Feed
 import com.jocmp.capy.Folder
 import com.jocmp.capy.MarkRead
-import com.jocmp.capy.articles.parseHtml
 import com.jocmp.capy.buildArticlePager
 import com.jocmp.capy.common.UnauthorizedError
 import com.jocmp.capy.common.launchIO
@@ -408,7 +407,7 @@ class ArticleScreenViewModel(
             onSuccess = { value ->
                 if (_article?.id == article.id) {
                     _article = article.copy(
-                        content = parseHtml(article, value),
+                        content = value,
                         fullContent = Article.FullContentState.LOADED
                     )
                 }

diff --git a/app/src/main/java/com/capyreader/app/ui/components/WebView.kt b/app/src/main/java/com/capyreader/app/ui/components/WebView.kt
@@ -168,7 +168,7 @@ class WebViewState(
 
                 withContext(Dispatchers.Main) {
                     webView.loadDataWithBaseURL(
-                        ASSET_BASE_URL,
+                        article.url?.toString(),
                         html,
                         null,
                         "UTF-8",
@@ -211,7 +211,6 @@ fun rememberWebViewState(
             settings.apply {
                 javaScriptEnabled = true
                 mediaPlaybackRequiresUserGesture = false
-                offscreenPreRaster = true
             }
             isVerticalScrollBarEnabled = false
             isHorizontalScrollBarEnabled = false

diff --git a/article_forge/Makefile b/article_forge/Makefile
@@ -9,7 +9,7 @@ SHELL:=/usr/bin/env bash
 
 .PHONY: forge clean
 
-build: $(TEMPLATE_DEST) $(STYLE_DEST) $(MEDIA_LISTENERS_JS_DEST)
+build: $(TEMPLATE_DEST) $(STYLE_DEST) $(MEDIA_LISTENERS_JS_DEST) $(CUSTOM_EXTRACTOR_JS_DEST)
 
 .PHONY: forge
 forge:

diff --git a/article_forge/public/assets/custom-extractors.js b/article_forge/public/assets/custom-extractors.js
diff --git a/article_forge/script/generate-android-template b/article_forge/script/generate-android-template
@@ -8,6 +8,7 @@ def main
   file_content = File.read("./views/#{filename}.liquid")
   file_content = file_content.gsub("{{body}}", "")
   file_content = file_content.gsub("    {{debug_script}}\n", "")
+  file_content = file_content.gsub("/assets/", 'https://appassets.androidplatform.net/assets/')
 
   assert_clean(file_content)
 

diff --git a/article_forge/views/template.liquid b/article_forge/views/template.liquid
@@ -21,6 +21,8 @@
     {{font_preload}}
     <link rel="stylesheet" href="/assets/stylesheet.css">
     <script type="text/javascript" src="/assets/media.js"></script>
+    <script type="text/javascript" src="/assets/mercury-parser.js"></script>
+    <script type="text/javascript" src="/assets/custom-extractors.js"></script>
   </head>
   <body>
     <article>

diff --git a/capy/src/main/assets/custom-extractors.js b/capy/src/main/assets/custom-extractors.js
diff --git a/capy/src/main/assets/mercury-parser.js b/capy/src/main/assets/mercury-parser.js
diff --git a/capy/src/main/java/com/jocmp/capy/Article.kt b/capy/src/main/java/com/jocmp/capy/Article.kt
@@ -27,6 +27,8 @@ data class Article(
 ) {
     val defaultContent = contentHTML.ifBlank { summary }
 
+    val parseFullContent = fullContent == FullContentState.LOADED && extractedContentURL == null
+
     enum class FullContentState {
         NONE,
         LOADING,

diff --git a/capy/src/main/java/com/jocmp/capy/articles/ArticleRenderer.kt b/capy/src/main/java/com/jocmp/capy/articles/ArticleRenderer.kt
@@ -47,14 +47,17 @@ class ArticleRenderer(
             article.siteURL?.let { setBaseUri(it) }
         }
 
-        document.getElementById("article-body-content")?.append(article.content)
+        if (article.parseFullContent) {
+            val contentHTML = Jsoup.parse(article.content)
 
-        cleanStyles(document)
-        cleanLinks(document)
-        if (hideImages) {
-            removeImages(document)
+            HtmlPostProcessor.clean(contentHTML, hideImages = hideImages)
+
+            document.getElementById("article-body-content")?.append(parseHtml(article, contentHTML.html()))
+        } else {
+            document.getElementById("article-body-content")?.append(article.content)
+
+            HtmlPostProcessor.clean(document, hideImages = hideImages)
         }
-        wrapTables(document)
 
         return document.html()
     }

diff --git a/capy/src/main/java/com/jocmp/capy/articles/HtmlPostProcessor.kt b/capy/src/main/java/com/jocmp/capy/articles/HtmlPostProcessor.kt
@@ -0,0 +1,14 @@
+package com.jocmp.capy.articles
+
+import org.jsoup.nodes.Document
+
+object HtmlPostProcessor {
+    fun clean(document: Document, hideImages: Boolean) {
+        cleanStyles(document)
+        cleanLinks(document)
+        if (hideImages) {
+            removeImages(document)
+        }
+        wrapTables(document)
+    }
+}
diff --git a/capy/src/main/java/com/jocmp/capy/articles/ParseHTML.kt b/capy/src/main/java/com/jocmp/capy/articles/ParseHTML.kt
@@ -1,20 +1,33 @@
 package com.jocmp.capy.articles
 
 import com.jocmp.capy.Article
-import net.dankito.readability4j.Readability4J
+import org.json.JSONObject
 
 fun parseHtml(article: Article, html: String): String {
-    try {
-        val uri = (article.feedURL ?: article.url).toString()
-        val readability4J = Readability4J(uri, html)
-        val content = readability4J.parse().articleContent ?: return ""
+    return """
+      <script>
+        (async () => {
+          let downloaded = ${JSONObject(mapOf("value" to html))};
 
-        content.getElementsByClass("readability-styled").forEach { element ->
-            element.append("&nbsp;")
-        }
+          Mercury.parse("${article.url?.toString()}", { html: downloaded.value }).then(article => {
+            let extracted = document.createElement("div");
 
-        return content.html()
-    } catch (ex: Throwable) {
-        return ""
-    }
+            extracted.id = "article-body-content"
+            extracted.innerHTML = article.content;
+
+            let shouldAddImage = article.lead_image_url &&
+                ![...extracted.querySelectorAll("img")].some(img => img.src.includes(article.lead_image_url));
+
+            if (shouldAddImage) {
+              let leadImage = document.createElement("img");
+              leadImage.src = article.lead_image_url;
+              extracted.prepend(leadImage);
+            }
+
+            let content = document.getElementById("article-body-content");
+            content.replaceWith(extracted);
+          });
+        })();
+      </script>
+    """.trimIndent()
 }
diff --git a/capy/src/main/res/raw/template.html b/capy/src/main/res/raw/template.html
diff --git a/technotes/2024-08 Media.md → technotes/Scratch/2024-08 Media.md b/technotes/2024-08 Media.md → technotes/Scratch/2024-08 Media.md
diff --git a/technotes/Scratch/2024-11 Mercury Parser.md b/technotes/Scratch/2024-11 Mercury Parser.md
@@ -0,0 +1,13 @@
+```kotlin
+webView.evaluateJavascript(
+  """(function test() {
+        return "hello";
+      })();
+  """.trimIndent()) {  
+  it
+}
+```
+
+1. Load placeholder text
+2. Fetch full content
+3. evaluateJavascript -> Parser(content)