Cui cloud publishing & GitHub republishing (mamoe#177)

* Add CuiCloud tasks

* Add CuiCloud workflow

* Fix shadowJar

* Fix `reply` function prohibition in MessageSelectBuilder

* Fix doc

* Update CI

* Update CI

* Test CI

* Fix file

* Increase jvm memory limitation

* Add comments

* Remove unnecessary experimental api use

* Increase upload task timeout

* Fix sha

* Increase timeout

* Fix github

* Trigger publishing on release

.github/workflows/cui.yml

@@ -0,0 +1,48 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CuiCloud Publish
+
+# Controls when the action will run. Triggers the workflow on push or pull request 
+# events but only for the master branch
+on:
+  release:
+    types:
+      - created
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+      - name: Gradle clean
+        run: ./gradlew clean
+      - name: Gradle build
+        run: ./gradlew build # if test's failed, don't publish
+      - name: Gradle :mirai-core:cuiCloudUpload
+        run: ./gradlew :mirai-core:cuiCloudUpload -Dcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Pcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Dcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} -Pcui_cloud_url=${{ secrets.CUI_CLOUD_URL }}
+      - name: Gradle :mirai-core-qqandroid:cuiCloudUpload
+        run: ./gradlew :mirai-core-qqandroid:cuiCloudUpload -Dcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Pcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Dcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} -Pcui_cloud_url=${{ secrets.CUI_CLOUD_URL }}
+
+
+#      - name: Upload artifact
+#        uses: actions/[email protected]
+#        with:
+#          # Artifact name
+#          name: mirai-core
+#          # Directory containing files to upload
+#          path: "mirai-core/build/libs/mirai-core-*-all.jar"
+#      - name: Upload artifact
+#        uses: actions/[email protected]
+#        with:
+#          # Artifact name
+#          name: mirai-core-qqandroid-all
+#          # Directory containing files to upload
+#          path: "mirai-core-qqandroid/build/libs/mirai-core-qqandroid-*-all.jar"

.github/workflows/shadow.yml

@@ -1,6 +1,6 @@
 # This is a basic workflow to help you get started with Actions
 
-name: Shadow Publish
+name: mirai-repo Publish
 
 # Controls when the action will run. Triggers the workflow on push or pull request 
 # events but only for the master branch
@@ -25,7 +25,7 @@ jobs:
       - name: Gradle clean
         run: ./gradlew clean
       - name: Gradle build
-        run: ./gradlew build
+        run: ./gradlew build # if test's failed, don't publish
       - name: Gradle :mirai-core:githubUpload
         run: ./gradlew :mirai-core:githubUpload -Dgithub_token=${{ secrets.MAMOE_TOKEN }} -Pgithub_token=${{ secrets.MAMOE_TOKEN }}
       - name: Gradle :mirai-core-qqandroid:githubUpload

build.gradle.kts

@@ -84,36 +84,64 @@ subprojects {
             dependsOn(shadowJvmJar)
 
             doFirst {
-                timeout.set(Duration.ofMinutes(10))
-                File(projectDir, "build/libs").walk()
-                    .filter { it.isFile }
-                    .onEach { println("all files=$it") }
-                    .filter { it.name.matches(Regex("""${project.name}-([0-9]|\.)*\.jar""")) }
-                    .onEach { println("matched file: ${it.name}") }
-                    .associateBy { it.nameWithoutExtension.substringAfterLast('-') }
-                    .onEach { println("versions: $it") }
-                    .maxBy {
-                        it.key.split('.').foldRightIndexed(0) { index: Int, s: String, acc: Int ->
-                            acc + 100.0.pow(2 - index).toInt() * (s.toIntOrNull() ?: 0)
-                        }
-                    }?.let { (_, file) ->
-                        val filename = file.name
-                        println("Uploading file $filename")
-                        runCatching {
-                            upload.GitHub.upload(
-                                file,
-                                "https://api.github.com/repos/mamoe/mirai-repo/contents/shadow/${project.name}/$filename",
-                                project
-                            )
-                        }.exceptionOrNull()?.let {
-                            System.err.println("Upload failed")
-                            it.printStackTrace() // force show stacktrace
-                            throw it
-                        }
+                timeout.set(Duration.ofHours(3))
+                findLatestFile()?.let { (_, file) ->
+                    val filename = file.name
+                    println("Uploading file $filename")
+                    runCatching {
+                        upload.GitHub.upload(
+                            file,
+                            "https://api.github.com/repos/mamoe/mirai-repo/contents/shadow/${project.name}/$filename",
+                            project
+                        )
+                    }.exceptionOrNull()?.let {
+                        System.err.println("GitHub Upload failed")
+                        it.printStackTrace() // force show stacktrace
+                        throw it
                     }
+                }
+            }
+        }
+
+        val cuiCloudUpload by tasks.creating {
+            group = "mirai"
+            dependsOn(shadowJvmJar)
+
+            doFirst {
+                timeout.set(Duration.ofHours(3))
+                findLatestFile()?.let { (_, file) ->
+                    val filename = file.name
+                    println("Uploading file $filename")
+                    runCatching {
+                        upload.CuiCloud.upload(
+                            file,
+                            project
+                        )
+                    }.exceptionOrNull()?.let {
+                        System.err.println("CuiCloud Upload failed")
+                        it.printStackTrace() // force show stacktrace
+                        throw it
+                    }
+                }
             }
 
         }
     }
 
+}
+
+
+fun Project.findLatestFile(): Map.Entry<String, File>? {
+    return File(projectDir, "build/libs").walk()
+        .filter { it.isFile }
+        .onEach { println("all files=$it") }
+        .filter { it.name.matches(Regex("""${project.name}-([0-9]|\.)*\.jar""")) }
+        .onEach { println("matched file: ${it.name}") }
+        .associateBy { it.nameWithoutExtension.substringAfterLast('-') }
+        .onEach { println("versions: $it") }
+        .maxBy {
+            it.key.split('.').foldRightIndexed(0) { index: Int, s: String, acc: Int ->
+                acc + 100.0.pow(2 - index).toInt() * (s.toIntOrNull() ?: 0)
+            }
+        }
 }

buildSrc/build.gradle.kts

@@ -6,10 +6,20 @@ repositories {
     jcenter()
 }
 
+kotlin {
+    sourceSets.all {
+        languageSettings.useExperimentalAnnotation("kotlin.Experimental")
+        languageSettings.useExperimentalAnnotation("kotlin.RequiresOptIn")
+    }
+}
+
 dependencies {
     fun kotlinx(id: String, version: String) = "org.jetbrains.kotlinx:kotlinx-$id:$version"
     fun ktor(id: String, version: String) = "io.ktor:ktor-$id:$version"
 
+    api("org.jsoup:jsoup:1.12.1")
+
+    api("com.google.code.gson:gson:2.8.6")
     api(kotlinx("coroutines-core", "1.3.3"))
     api(ktor("client-core", "1.3.2"))
     api(ktor("client-cio", "1.3.2"))

buildSrc/src/main/kotlin/upload/CuiCloud.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+package upload
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
+import org.gradle.api.Project
+import org.gradle.kotlin.dsl.provideDelegate
+import org.jsoup.Connection
+import org.jsoup.Jsoup
+import java.io.File
+import java.util.*
+
+object CuiCloud {
+    private fun getUrl(project: Project): String {
+        kotlin.runCatching {
+            @Suppress("UNUSED_VARIABLE", "LocalVariableName")
+            val cui_cloud_url: String by project
+            return cui_cloud_url
+        }
+
+        System.getProperty("cui_cloud_url", null)?.let {
+            return it.trim()
+        }
+        error("cannot find url for CuiCloud")
+    }
+
+    private fun getKey(project: Project): String {
+        kotlin.runCatching {
+            @Suppress("UNUSED_VARIABLE", "LocalVariableName")
+            val cui_cloud_key: String by project
+            return cui_cloud_key
+        }
+
+        System.getProperty("cui_cloud_key", null)?.let {
+            return it.trim()
+        }
+        error("cannot find key for CuiCloud")
+    }
+
+    fun upload(file: File, project: Project) {
+        val cuiCloudUrl = getUrl(project)
+        val key = getKey(project)
+
+        runBlocking {
+            uploadToCuiCloud(
+                cuiCloudUrl,
+                key,
+                "/mirai/${project.name}/${file.nameWithoutExtension}.mp4",
+                file.readBytes()
+            )
+        }
+    }
+
+    private suspend fun uploadToCuiCloud(
+        cuiCloudUrl: String,
+        cuiToken: String,
+        filePath: String,
+        content: ByteArray
+    ) {
+        val response = withContext(Dispatchers.IO) {
+            Jsoup.connect(cuiCloudUrl).method(Connection.Method.POST)
+                .data("base64", Base64.getEncoder().encodeToString(content))
+                .data("filePath", filePath)
+                .data("key", cuiToken)
+                .timeout(Int.MAX_VALUE)
+                .execute()
+        }
+        if (response.statusCode() != 200) {
+            println(response.body())
+            error("Cui Cloud Does Not Return 200")
+        }
+    }
+}

buildSrc/src/main/kotlin/upload/GitHub.kt

@@ -2,13 +2,19 @@
 
 package upload
 
+import com.google.gson.JsonObject
+import com.google.gson.JsonParser
 import io.ktor.client.HttpClient
 import io.ktor.client.engine.cio.CIO
 import io.ktor.client.features.HttpTimeout
 import io.ktor.client.request.put
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
 import org.gradle.api.Project
 import org.gradle.kotlin.dsl.provideDelegate
+import org.jsoup.Connection
+import org.jsoup.Jsoup
 import java.io.File
 import java.util.*
 
@@ -57,16 +63,114 @@ object GitHub {
                 connectTimeoutMillis = 600_000
             }
         }.put<String>("$url?access_token=$token") {
-            //header("token", token)
+            val sha = getGithubSha("mirai-repo", "shadow/${project.name}/${file.name}", "master", project)
+            println("sha=$sha")
             val content = String(Base64.getEncoder().encode(file.readBytes()))
             body = """
                     {
                       "message": "automatically upload on release",
                       "content": "$content"
+                      ${if (sha == null) "" else """, "sha": "$sha" """}
                     }
                 """.trimIndent()
         }.let {
             println("Upload response: $it")
         }
     }
+
+
+    private suspend fun getGithubSha(
+        repo: String,
+        filePath: String,
+        branch: String,
+        project: Project
+    ): String? {
+        fun String.asJson(): JsonObject {
+            return JsonParser.parseString(this).asJsonObject
+        }
+
+        /*
+        * 只能获取1M以内/branch为master的sha
+        * */
+        class TargetTooLargeException() : Exception("Target TOO Large")
+
+        suspend fun getShaSmart(repo: String, filePath: String, project: Project): String? {
+            return withContext(Dispatchers.IO) {
+                val response = Jsoup
+                    .connect(
+                        "https://api.github.com/repos/mamoe/$repo/contents/$filePath?access_token=" + getGithubToken(
+                            project
+                        )
+                    )
+                    .ignoreContentType(true)
+                    .ignoreHttpErrors(true)
+                    .method(Connection.Method.GET)
+                    .execute()
+                if (response.statusCode() == 404) {
+                    null
+                } else {
+                    val p = response.body().asJson()
+                    if (p.has("message") && p["message"].asString == "This API returns blobs up to 1 MB in size. The requested blob is too large to fetch via the API, but you can use the Git Data API to request blobs up to 100 MB in size.") {
+                        throw TargetTooLargeException()
+                    }
+                    p.get("sha").asString
+                }
+            }
+        }
+
+        suspend fun getShaStupid(
+            repo: String,
+            filePath: String,
+            branch: String,
+            project: Project
+        ): String? {
+            val resp = withContext(Dispatchers.IO) {
+                Jsoup
+                    .connect(
+                        "https://api.github.com/repos/mamoe/$repo/git/ref/heads/$branch?access_token=" + getGithubToken(
+                            project
+                        )
+                    )
+                    .ignoreContentType(true)
+                    .ignoreHttpErrors(true)
+                    .method(Connection.Method.GET)
+                    .execute()
+            }
+            if (resp.statusCode() == 404) {
+                println("Branch Not Found")
+                return null
+            }
+            val info = resp.body().asJson().get("object").asJsonObject.get("url").asString
+            var parentNode = withContext(Dispatchers.IO) {
+                Jsoup.connect(info + "?access_token=" + getGithubToken(project)).ignoreContentType(true)
+                    .method(Connection.Method.GET)
+                    .execute().body().asJson().get("tree").asJsonObject.get("url").asString
+            }
+            filePath.split("/").forEach { subPath ->
+                withContext(Dispatchers.IO) {
+                    Jsoup.connect(parentNode + "?access_token=" + getGithubToken(project)).ignoreContentType(true)
+                        .method(Connection.Method.GET).execute().body().asJson().get("tree").asJsonArray
+                }.forEach list@{
+                    with(it.asJsonObject) {
+                        if (this.get("path").asString == subPath) {
+                            parentNode = this.get("url").asString
+                            return@list
+                        }
+                    }
+                }
+            }
+            check(parentNode.contains("/blobs/"))
+            return parentNode.substringAfterLast("/")
+        }
+
+        return if (branch == "master") {
+            try {
+                getShaSmart(repo, filePath, project)
+            } catch (e: TargetTooLargeException) {
+                getShaStupid(repo, filePath, branch, project)
+            }
+        } else {
+            getShaStupid(repo, filePath, branch, project)
+        }
+    }
 }