Redact access tokens in attachments (#709)

* Redact access tokens in attachments

Redact access tokens in attachments (e.g. JVM crash reports) by deleting the
original attachment and uploading a redacted version. Additionally add a
comment to let the user know that their attachment has been redacted.

* Log where PrivacyModule found sensitive data

This hopefully makes it easier to understand why an issue has been made
private and allows improving or removing imprecise regex patterns.

config/local.template.yml

@@ -7,6 +7,6 @@ arisa:
     # These configs are optional and can be omitted
     dandelionToken: <Dandelion token>
     discordLogWebhook: <Discord log webhook>
-    discordErrorWebhook: <Discord error webhook>
+    discordErrorLogWebhook: <Discord error webhook>
   debug:
     # You can add some debug options here, see `ArisaConfig.kt` for details

docs/Modules.md

@@ -229,20 +229,22 @@ Resolves tickets about pirated games as `Invalid`.
 - Any of the description, environment, and/or summary contains any of the `piracySignatures` defined in the [config](../config/config.yml).
 
 ## Privacy
-| Entry | Value                                                                      |
-| ----- | -------------------------------------------------------------------------- |
-| Name  | `Privacy`                                                                  |
-| Class | [Link](../src/main/kotlin/io/github/mojira/arisa/modules/PrivacyModule.kt) |
+| Entry | Value                                                                              |
+| ----- | ---------------------------------------------------------------------------------- |
+| Name  | `Privacy`                                                                          |
+| Class | [Link](../src/main/kotlin/io/github/mojira/arisa/modules/privacy/PrivacyModule.kt) |
 
-Hides privacy information like Email addresses in tickets or comments.
+Makes tickets and comments, which contain sensitive data like Email addresses, private.
+Additionally in some cases it redacts sensitive data from attachments by deleting the original attachment,
+reuploading it with redacted content and its name prefixed with `redacted_` and adding a comment informing the uploader.
 
 ### Checks
 - The ticket is not set to private.
 #### For Setting Tickets to Private
 - Any of the following matches one of the regex patterns specified by `sensitiveTextRegexes` in the
   [config](../config/config.yml), or contains an email address which does not match one of the `allowedEmailRegexes`:
   - summary, environment, description (if ticket was created after last run)
-  - text attachment (if it was created after last run)
+  - text attachment (if it was created after last run), and the module was unable to redact the sensitive data
   - changelog entry string value (if it was created after last run)
 - Or any of the attachments created after the last run has a name which matches one of `sensitiveFileNameRegexes` in
   the [config](../config/config.yml).

src/main/kotlin/io/github/mojira/arisa/ModuleExecutor.kt

@@ -62,6 +62,6 @@ class ModuleExecutor(
         return allIssues
             .filter { it.project.key in projects }
             .filter { it.status.toLowerCase() !in excludedStatuses }
-            .filter { it.resolution?.toLowerCase() ?: "unresolved" in resolutions }
+            .filter { (it.resolution?.toLowerCase() ?: "unresolved") in resolutions }
     }
 }

src/main/kotlin/io/github/mojira/arisa/domain/Attachment.kt

@@ -12,4 +12,10 @@ data class Attachment(
     val openContentStream: () -> InputStream,
     val getContent: () -> ByteArray,
     val uploader: User?
-)
+) {
+    /** Returns whether the type of the content is text */
+    fun hasTextContent() = mimeType.startsWith("text/")
+
+    /** Decodes the content as UTF-8 String */
+    fun getTextContent() = String(getContent())
+}

src/main/kotlin/io/github/mojira/arisa/domain/ChangeLogItem.kt

@@ -3,6 +3,10 @@ package io.github.mojira.arisa.domain
 import java.time.Instant
 
 data class ChangeLogItem(
+    /** ID of the enclosing change log entry */
+    val entryId: String,
+    /** 0-based index of this change log item within the enclosing change log entry */
+    val itemIndex: Int,
     val created: Instant,
     val field: String,
     val changedFrom: String?,

src/main/kotlin/io/github/mojira/arisa/domain/Issue.kt

@@ -50,6 +50,7 @@ data class Issue(
     val addRestrictedComment: (options: CommentOptions) -> Unit,
     val addNotEnglishComment: (language: String) -> Unit,
     val addRawRestrictedComment: (body: String, restriction: String) -> Unit,
+    val addRawBotComment: (rawMessage: String) -> Unit,
     val markAsFixedWithSpecificVersion: (fixVersionName: String) -> Unit,
     val changeReporter: (reporter: String) -> Unit,
     val addAttachmentFromFile: (file: File, cleanupCallback: () -> Unit) -> Unit,

src/main/kotlin/io/github/mojira/arisa/domain/User.kt

@@ -4,5 +4,6 @@ data class User(
     val name: String?,
     val displayName: String?,
     val getGroups: () -> List<String>?,
-    val isNewUser: () -> Boolean
+    val isNewUser: () -> Boolean,
+    val isBotUser: () -> Boolean
 )

src/main/kotlin/io/github/mojira/arisa/infrastructure/AttachmentUtils.kt

@@ -13,8 +13,7 @@ fun getDeobfName(name: String): String = "deobf_$name"
 
 class AttachmentUtils(
     private val crashReportExtensions: List<String>,
-    private val crashReader: CrashReader,
-    private val botUserName: String
+    private val crashReader: CrashReader
 ) {
     private val mappingsDir by lazy {
         val file = File("mc-mappings")
@@ -37,7 +36,7 @@ class AttachmentUtils(
         // Get crashes from issue attachments
         val textDocuments = attachments
             // Ignore attachments from Arisa (e.g. deobfuscated crash reports)
-            .filterNot { it.uploader?.name == botUserName }
+            .filterNot { it.uploader?.isBotUser?.invoke() == true }
 
             // Only check attachments with allowed extensions
             .filter { isCrashAttachment(it.name) }
@@ -61,10 +60,7 @@ class AttachmentUtils(
         crashReportExtensions.any { it == fileName.substring(fileName.lastIndexOf(".") + 1) }
 
     fun fetchAttachment(attachment: Attachment): TextDocument {
-        val getText = {
-            val data = attachment.getContent()
-            String(data)
-        }
+        val getText = attachment::getTextContent
 
         return TextDocument(getText, attachment.created, attachment.name)
     }

src/main/kotlin/io/github/mojira/arisa/infrastructure/HelperMessages.kt

@@ -1,6 +1,7 @@
 package io.github.mojira.arisa.infrastructure
 
 import arrow.core.Either
+import arrow.core.getOrElse
 import arrow.core.left
 import arrow.core.right
 import arrow.core.rightIfNotNull
@@ -119,12 +120,19 @@ object HelperMessageService {
         }
     }
 
+    private const val BOT_SIGNATURE_KEY = "i-am-a-bot"
     fun getMessageWithBotSignature(project: String, key: String, filledText: String? = null, lang: String = "en") =
-        getMessage(project, listOf(key, "i-am-a-bot"), listOf(filledText), lang)
+        getMessage(project, listOf(key, BOT_SIGNATURE_KEY), listOf(filledText), lang)
 
     fun getMessageWithDupeBotSignature(project: String, key: String, filledText: String? = null, lang: String = "en") =
         getMessage(project, listOf(key, "i-am-a-bot-dupe"), listOf(filledText), lang)
 
+    fun getRawMessageWithBotSignature(rawMessage: String): String {
+        // Note: Project does not matter, message is (currently) the same for all projects
+        val botSignature = getSingleMessage("MC", BOT_SIGNATURE_KEY).getOrElse { "" }
+        return "$rawMessage\n$botSignature"
+    }
+
     fun setHelperMessages(json: String) = data.fromJSON(json)
         ?: throw IOException("Couldn't deserialize helper messages from setHelperMessages()")
 

src/main/kotlin/io/github/mojira/arisa/infrastructure/jira/Mappers.kt

@@ -39,19 +39,20 @@ import net.rcarz.jiraclient.Project as JiraProject
 import net.rcarz.jiraclient.User as JiraUser
 import net.rcarz.jiraclient.Version as JiraVersion
 
-fun JiraAttachment.toDomain(jiraClient: JiraClient, issue: JiraIssue) = Attachment(
+fun JiraAttachment.toDomain(jiraClient: JiraClient, issue: JiraIssue, config: Config) = Attachment(
     id,
     fileName,
     getCreationDate(issue, id, issue.createdDate.toInstant()),
     mimeType,
     ::deleteAttachment.partially1(issue.getUpdateContext(jiraClient)).partially1(this),
     { openAttachmentStream(jiraClient, this) },
-    this::download,
-    author?.toDomain(jiraClient)
+    // Cache attachment content once it has been downloaded
+    lazy { this.download() }::value,
+    author?.toDomain(jiraClient, config)
 )
 
 fun getCreationDate(issue: JiraIssue, id: String, default: Instant) = issue.changeLog.entries
-    .filter { it.items.any { it.field == "Attachment" && it.to == id } }
+    .filter { changeLogEntry -> changeLogEntry.items.any { it.field == "Attachment" && it.to == id } }
     .maxByOrNull { it.created }
     ?.created
     ?.toInstant() ?: default
@@ -93,7 +94,7 @@ fun JiraIssue.toDomain(
         description,
         getEnvironment(),
         security?.id,
-        reporter?.toDomain(jiraClient),
+        reporter?.toDomain(jiraClient, config),
         resolution?.name,
         createdDate.toInstant(),
         updatedDate.toInstant(),
@@ -108,10 +109,10 @@ fun JiraIssue.toDomain(
         getDungeonsPlatform(config),
         mapVersions(),
         mapFixVersions(),
-        mapAttachments(jiraClient),
-        mapComments(jiraClient),
+        mapAttachments(jiraClient, config),
+        mapComments(jiraClient, config),
         mapLinks(jiraClient, config),
-        getChangeLogEntries(jiraClient),
+        getChangeLogEntries(jiraClient, config),
         ::reopen.partially1(context),
         ::resolveAs.partially1(context).partially1("Awaiting Response"),
         ::resolveAs.partially1(context).partially1("Invalid"),
@@ -155,12 +156,19 @@ fun JiraIssue.toDomain(
         },
         addNotEnglishComment = { language ->
             createComment(
-                context, HelperMessageService.getMessageWithBotSignature(
+                context,
+                HelperMessageService.getMessageWithBotSignature(
                     project.key, config[Arisa.Modules.Language.message], lang = language
                 )
             )
         },
         addRawRestrictedComment = ::addRestrictedComment.partially1(context),
+        addRawBotComment = { rawMessage ->
+            createComment(
+                context,
+                HelperMessageService.getRawMessageWithBotSignature(rawMessage)
+            )
+        },
         ::markAsFixedWithSpecificVersion.partially1(context),
         ::changeReporter.partially1(context),
         addAttachmentFromFile,
@@ -191,13 +199,14 @@ fun JiraProject.toDomain(
 
 fun JiraComment.toDomain(
     jiraClient: JiraClient,
-    issue: JiraIssue
+    issue: JiraIssue,
+    config: Config
 ): Comment {
     val context = issue.getUpdateContext(jiraClient)
     return Comment(
         id,
         body,
-        author.toDomain(jiraClient),
+        author.toDomain(jiraClient, config),
         { getGroups(jiraClient, author.name).fold({ null }, { it }) },
         createdDate.toInstant(),
         updatedDate.toInstant(),
@@ -209,11 +218,15 @@ fun JiraComment.toDomain(
     )
 }
 
-fun JiraUser.toDomain(jiraClient: JiraClient) = User(
+fun JiraUser.toDomain(jiraClient: JiraClient, config: Config) = User(
     name, displayName,
     ::getUserGroups.partially1(jiraClient).partially1(name),
     ::isNewUser.partially1(jiraClient).partially1(name)
-)
+) {
+    // Check case insensitively because it apparently does not matter when logging in, so `username` might have
+    // incorrect capitalization
+    name.equals(config[Arisa.Credentials.username], ignoreCase = true)
+}
 
 private fun getUserGroups(jiraClient: JiraClient, username: String) = getGroups(
     jiraClient,
@@ -265,14 +278,21 @@ fun JiraIssueLink.toDomain(
     ::deleteLink.partially1(issue.getUpdateContext(jiraClient)).partially1(this)
 )
 
-fun JiraChangeLogItem.toDomain(jiraClient: JiraClient, entry: JiraChangeLogEntry) = ChangeLogItem(
+fun JiraChangeLogItem.toDomain(
+    jiraClient: JiraClient,
+    entry: JiraChangeLogEntry,
+    itemIndex: Int,
+    config: Config
+) = ChangeLogItem(
+    entry.id,
+    itemIndex,
     entry.created.toInstant(),
     field,
     from,
     fromString,
     to,
     toString,
-    entry.author.toDomain(jiraClient),
+    entry.author.toDomain(jiraClient, config),
     ::getUserGroups.partially1(jiraClient).partially1(entry.author.name)
 )
 
@@ -284,22 +304,22 @@ private fun JiraIssue.mapLinks(
     it.toDomain(jiraClient, this, config)
 }
 
-private fun JiraIssue.mapComments(jiraClient: JiraClient) =
-    comments.map { it.toDomain(jiraClient, this) }
+private fun JiraIssue.mapComments(jiraClient: JiraClient, config: Config) =
+    comments.map { it.toDomain(jiraClient, this, config) }
 
-private fun JiraIssue.mapAttachments(jiraClient: JiraClient) =
-    attachments.map { it.toDomain(jiraClient, this) }
+private fun JiraIssue.mapAttachments(jiraClient: JiraClient, config: Config) =
+    attachments.map { it.toDomain(jiraClient, this, config) }
 
 private fun JiraIssue.mapVersions() =
     versions.map { it.toDomain() }
 
 private fun JiraIssue.mapFixVersions() =
     fixVersions.map { it.toDomain() }
 
-private fun JiraIssue.getChangeLogEntries(jiraClient: JiraClient) =
+private fun JiraIssue.getChangeLogEntries(jiraClient: JiraClient, config: Config) =
     changeLog.entries.flatMap { e ->
-        e.items.map { i ->
-            i.toDomain(jiraClient, e)
+        e.items.mapIndexed { index, item ->
+            item.toDomain(jiraClient, e, index, config)
         }
     }
 

src/main/kotlin/io/github/mojira/arisa/modules/AttachmentModule.kt

@@ -7,8 +7,11 @@ import io.github.mojira.arisa.domain.Attachment
 import io.github.mojira.arisa.domain.CommentOptions
 import io.github.mojira.arisa.domain.Issue
 import io.github.mojira.arisa.infrastructure.jira.sanitizeCommentArg
+import org.slf4j.LoggerFactory
 import java.time.Instant
 
+private val log = LoggerFactory.getLogger("AttachmentModule")
+
 class AttachmentModule(
     private val extensionBlackList: List<String>,
     private val attachmentRemovedMessage: String
@@ -17,13 +20,16 @@ class AttachmentModule(
     override fun invoke(issue: Issue, lastRun: Instant): Either<ModuleError, ModuleResponse> = with(issue) {
         Either.fx {
             val endsWithBlacklistedExtensionAdapter = ::endsWithBlacklistedExtensions.partially1(extensionBlackList)
-            val functions = attachments
+            val attachmentsToDelete = attachments
                 .filter { endsWithBlacklistedExtensionAdapter(it.name) }
-            assertNotEmpty(functions).bind()
-            val commentInfo = functions.getCommentInfo()
-            functions
-                .map { it.remove }
-                .forEach { it.invoke() }
+            assertNotEmpty(attachmentsToDelete).bind()
+            val commentInfo = attachmentsToDelete.getCommentInfo()
+
+            val attachmentsString = attachmentsToDelete.joinToString(separator = ", ", transform = Attachment::id)
+            log.info("Deleting attachments of issue $key because they have forbidden extensions: $attachmentsString")
+            attachmentsToDelete
+                .forEach { it.remove() }
+
             addComment(CommentOptions(attachmentRemovedMessage))
             addRawRestrictedComment("Removed attachments:\n$commentInfo", "helper")
         }

src/main/kotlin/io/github/mojira/arisa/modules/Helpers.kt

@@ -126,10 +126,7 @@ fun String?.getOrDefault(default: String) =
         this
 
 fun String?.getOrDefaultNull(default: String) =
-    if (this == null)
-        default
-    else
-        this
+    this ?: default
 
 fun MutableList<String>.splitElemsByCommas() {
     val newList = this.flatMap { s ->

src/main/kotlin/io/github/mojira/arisa/modules/KeepPlatformModule.kt

@@ -64,11 +64,7 @@ class KeepPlatformModule(
             val userChange = firstOrNull {
                 it.created.isAfter(markedTime)
             }
-            if (userChange != null) {
-                userChange.changedFromString.getOrDefault("None")
-            } else {
-                null
-            }
+            userChange?.changedFromString?.getOrDefault("None")
         }
     }
 }