From cfeffa09ef16af94c64280f8ad0d0fb3e51363e0 Mon Sep 17 00:00:00 2001 From: troiganto Date: Tue, 19 Nov 2024 14:36:49 +0100 Subject: [PATCH] feat(attach): add `OrgAttach:attach_url()` --- docs/configuration.org | 30 ++++ lua/orgmode/attach/core.lua | 27 ++- lua/orgmode/attach/fileops.lua | 40 +++++ lua/orgmode/attach/init.lua | 56 +++++- lua/orgmode/config/_meta.lua | 2 + lua/orgmode/config/defaults.lua | 2 + lua/orgmode/objects/remote_resource.lua | 169 ++++++++++++++++++ lua/orgmode/state/state.lua | 2 +- lua/orgmode/utils/fs.lua | 1 + tests/plenary/object/remote_resource_spec.lua | 109 +++++++++++ 10 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 lua/orgmode/objects/remote_resource.lua create mode 100644 tests/plenary/object/remote_resource_spec.lua diff --git a/docs/configuration.org b/docs/configuration.org index f9734a175..f545d7d94 100644 --- a/docs/configuration.org +++ b/docs/configuration.org @@ -556,6 +556,36 @@ See [[https://orgmode.org/manual/Property-Inheritance.html][Property Inheritance - Default: ~{ [':tangle'] = 'no', [':noweb'] = no }~ Default header args for extracting source code. See [[#extract-source-code-tangle][Extract source code (tangle)]] for more details. +*** org_resource_download_policy +:PROPERTIES: +:CUSTOM_ID: org_resource_download_policy +:END: +- Type: ='always' | 'prompt' | 'safe' | 'never'= +- Default: ='prompt'= +Policy applied to requests to obtain remote resources. + +- =always= - Always download remote resources (dangerous!) +- =prompt= - Prompt before downloading an unsafe resource +- =safe= - Only download resources allowed by [[#org_safe_remote_resources][org_safe_remote_resources]] +- =never= - Never download any resources + +In Emacs Orgmode, this affects keywords like =#+setupfile= and =#+include= +on export, =org-persist-write:url=; and =org-attach-url= in non-interactive +sessions. Nvim Orgmode currently does not use this option, but defines it +for future use. + +*** org_safe_remote_resources +:PROPERTIES: +:CUSTOM_ID: org_safe_remote_resources +:END: +- Type: =string[]= +- Default: ={}= + +List of regex patterns matching safe URIs. URI regexps are applied to both +URLs and Org files requesting remote resources. The test uses +=vim.regex()=, so the regexes are always interpreted as magic and +case-sensitive. + *** calendar_week_start_day :PROPERTIES: :CUSTOM_ID: calendar_week_start_day diff --git a/lua/orgmode/attach/core.lua b/lua/orgmode/attach/core.lua index 6b838ce86..8f7519448 100644 --- a/lua/orgmode/attach/core.lua +++ b/lua/orgmode/attach/core.lua @@ -438,11 +438,36 @@ function AttachCore:attach(node, file, opts) end) end ----@class orgmode.attach.core.attach_buffer.opts +---@class orgmode.attach.core.attach_url.opts ---@inlinedoc ---@field set_dir_method fun(): OrgPromise ---@field new_dir fun(): OrgPromise +---Download a file from a URL and attach it to the current outline node. +--- +---@param node OrgAttachNode +---@param url string URL to the file to attach +---@param opts orgmode.attach.core.attach_url.opts +---@return OrgPromise attachment_name +function AttachCore:attach_url(node, url, opts) + local basename = basename_safe(url) + return self:get_dir_or_create(node, opts.set_dir_method, opts.new_dir):next(function(attach_dir) + local attach_file = vim.fs.joinpath(attach_dir, basename) + return fileops.download_file(url, attach_file, { exist_ok = false }):next(function(success) + if not success then + return nil + end + EventManager.dispatch(EventManager.event.AttachChanged:new(node, attach_dir)) + node:add_auto_tag() + local link = self.links:store_link_to_attachment({ attach_dir = attach_dir, original = url }) + vim.fn.setreg(vim.v.register, link) + return basename + end) + end) +end + +---@alias orgmode.attach.core.attach_buffer.opts orgmode.attach.core.attach_url.opts + ---Attach buffer's contents to current outline node. --- ---Throws a file-exists error if it would overwrite an existing filename. diff --git a/lua/orgmode/attach/fileops.lua b/lua/orgmode/attach/fileops.lua index c3661603e..c68be8f7c 100644 --- a/lua/orgmode/attach/fileops.lua +++ b/lua/orgmode/attach/fileops.lua @@ -339,4 +339,44 @@ function M.remove_directory(path, opts) end) end +--[[ +-- Scary hacks 💀 +--]] + +---Helper function to `download_file`. +---This uses NetRW to download a file and returns the download location. +---@param url string +---@return OrgPromise tmpfile +local function netrw_read(url) + return Promise.new(function(resolve, reject) + if not vim.g.loaded_netrwPlugin then + return reject('Netrw plugin must be loaded in order to download urls.') + end + vim.schedule(function() + local ok, err = pcall(vim.fn['netrw#NetRead'], 3, url) + if ok then + resolve(vim.b.netrw_tmpfile) + else + reject(err) + end + end) + end) +end + +---Download a file via NetRW. +---The file is first downloaded to a temporary location (no matter the value of +---`exist_ok`) and only then copied over to `dest`. The copy operation uses the +---`exist_ok` flag exactly like `copy_file`. +---@param url string +---@param dest string +---@param opts? {exist_ok: boolean?} +---@return OrgPromise success +function M.download_file(url, dest, opts) + opts = opts or {} + local exist_ok = opts.exist_ok or false + return netrw_read(url):next(function(source) + return M.copy_file(source, dest, { excl = not exist_ok, ficlone = true, ficlone_force = false }) + end) +end + return M diff --git a/lua/orgmode/attach/init.lua b/lua/orgmode/attach/init.lua index 1156001be..abecebeff 100644 --- a/lua/orgmode/attach/init.lua +++ b/lua/orgmode/attach/init.lua @@ -4,6 +4,7 @@ local Input = require('orgmode.ui.input') local Menu = require('orgmode.ui.menu') local Promise = require('orgmode.utils.promise') local config = require('orgmode.config') +local remote_resource = require('orgmode.objects.remote_resource') local ui = require('orgmode.attach.ui') local utils = require('orgmode.utils') @@ -65,6 +66,13 @@ function Attach:_build_menu() return self:attach_lns() end, }) + menu:add_option({ + label = 'Attach a file by download from URL.', + key = 'u', + action = function() + return self:attach_url() + end, + }) menu:add_option({ label = "Attach a buffer's contents.", key = 'b', @@ -376,18 +384,62 @@ function Attach:attach(file, opts) :wait(MAX_TIMEOUT) end ----@class orgmode.attach.attach_buffer.Options +---@class orgmode.attach.attach_url.Options ---@inlinedoc ---@field visit_dir? boolean if true, visit the directory subsequently using --- `org_attach_visit_command` ---@field node? OrgAttachNode +---Download a URL. +--- +---@param url? string +---@param opts? orgmode.attach.attach_url.Options +---@return string|nil attachment_name +function Attach:attach_url(url, opts) + local node = opts and opts.node or self.core:get_current_node() + local visit_dir = opts and opts.visit_dir or false + return Promise + .resolve() + :next(function() + if not url then + return Input.open('URL of the file to attach: ') + end + return remote_resource.should_fetch(url):next(function(ok) + if not ok then + error(("remote resource %s is unsafe, won't download"):format(url)) + end + return url + end) + end) + ---@param chosen_url? string + :next(function(chosen_url) + if not chosen_url then + return nil + end + return self.core:attach_url(node, chosen_url, { + set_dir_method = get_set_dir_method(), + new_dir = ui.ask_attach_dir_property, + }) + end) + :next(function(attachment_name) + if attachment_name then + utils.echo_info(('File %s is now an attachment'):format(attachment_name)) + if visit_dir then + local attach_dir = self.core:get_dir(node) + self.core:reveal_nvim(attach_dir) + end + end + return attachment_name + end) + :wait(MAX_TIMEOUT) +end + ---Attach buffer's contents to current outline node. --- ---Throws a file-exists error if it would overwrite an existing filename. --- ---@param buffer? string | integer A buffer number or name. ----@param opts? orgmode.attach.attach_buffer.Options +---@param opts? orgmode.attach.attach_url.Options ---@return string|nil attachment_name function Attach:attach_buffer(buffer, opts) local node = opts and opts.node or self.core:get_current_node() diff --git a/lua/orgmode/config/_meta.lua b/lua/orgmode/config/_meta.lua index e4e89ae7d..e39db7be2 100644 --- a/lua/orgmode/config/_meta.lua +++ b/lua/orgmode/config/_meta.lua @@ -249,6 +249,8 @@ ---@field org_attach_archive_delete 'always' | 'ask' | 'never' Determines whether to delete a headline's attachments when it is archived. Default: 'never' ---@field org_attach_id_to_path_function_list (string | fun(id: string): (string|nil))[] List of functions used to derive the attachments directory from an ID property. ---@field org_attach_sync_delete_empty_dir 'always' | 'ask' | 'never' Determines whether to delete empty directories when using `org.attach.sync()`. Default: 'ask' +---@field org_resource_download_policy 'always' | 'prompt' | 'safe' | 'never' Policy for downloading files from the Internet. Default: 'prompt' +---@field org_safe_remote_resources string[] List of regex patterns for URIs considered always safe to download from. Default: {} ---@field win_split_mode? 'horizontal' | 'vertical' | 'auto' | 'float' | string[] How to open agenda and capture windows. Default: 'horizontal' ---@field win_border? 'none' | 'single' | 'double' | 'rounded' | 'solid' | 'shadow' | string[] Border configuration for `win_split_mode = 'float'`. Default: 'single' ---@field notifications? OrgNotificationsConfig Notification settings diff --git a/lua/orgmode/config/defaults.lua b/lua/orgmode/config/defaults.lua index 8cb2f85ae..afe827f80 100644 --- a/lua/orgmode/config/defaults.lua +++ b/lua/orgmode/config/defaults.lua @@ -86,6 +86,8 @@ local DefaultConfig = { 'fallback_folder_format', }, org_attach_sync_delete_empty_dir = 'ask', + org_resource_download_policy = 'prompt', + org_safe_remote_resources = {}, win_split_mode = 'horizontal', win_border = 'single', notifications = { diff --git a/lua/orgmode/objects/remote_resource.lua b/lua/orgmode/objects/remote_resource.lua new file mode 100644 index 000000000..79680d0b3 --- /dev/null +++ b/lua/orgmode/objects/remote_resource.lua @@ -0,0 +1,169 @@ +local config = require('orgmode.config') +local fs = require('orgmode.utils.fs') +local utils = require('orgmode.utils') +local Menu = require('orgmode.ui.menu') +local State = require('orgmode.state.state') +local Promise = require('orgmode.utils.promise') + +local M = {} + +---Return true if the URI should be fetched. +---@param uri string +---@return OrgPromise safe +function M.should_fetch(uri) + local policy = config.org_resource_download_policy + return Promise.resolve(policy == 'always' or M.is_uri_safe(uri)):next(function(safe) + if safe then + return true + end + if policy == 'prompt' then + return M.confirm_safe(uri) + end + return false + end) +end + +---@param resource_uri string +---@param file_uri string | false +---@param patterns string[] +---@return boolean matches +local function check_patterns(resource_uri, file_uri, patterns) + for _, pattern in ipairs(patterns) do + local re = vim.regex(pattern) + if re:match_str(resource_uri) or (file_uri and re:match_str(file_uri)) then + return true + end + end + return false +end + +---Check the uri matches any of the (configured or cached) safe patterns. +---@param uri string +---@return OrgPromise safe +function M.is_uri_safe(uri) + local current_file = fs.get_real_path(utils.current_file_path()) + ---@type string | false # deduced type is `string | boolean` + local file_uri = current_file and vim.uri_from_fname(current_file) or false + local uri_patterns = {} + if config.org_safe_remote_resources then + vim.list_extend(uri_patterns, config.org_safe_remote_resources) + end + return State:load():next(function(state) + local cached = state['org_safe_remote_resources'] + if cached then + vim.list_extend(uri_patterns, cached) + end + return check_patterns(uri, file_uri, uri_patterns) + end) +end + +---@param uri string +---@return string escaped +local function uri_to_pattern(uri) + -- Escape backslashes, disable magic characters, anchor front and back of the + -- pattern. + return string.format([[\V\^%s\$]], uri:gsub([[\]], [[\\]])) +end + +---@param filename string +---@return string escaped +local function filename_to_pattern(filename) + return uri_to_pattern(vim.uri_from_fname(filename)) +end + +---@param domain string +---@return string escaped +local function domain_to_pattern(domain) + -- We construct the following regex: + -- 1. http or https protocol; + -- 2. followed by userinfo (`name:password@`), + -- 3. followed by potentially `www.` (for convenience), + -- 4. followed by the domain (in very-nomagic mode) + -- 5. followed by either a slash or nothing at all. + return string.format( + [[\v^https?://([^@/?#]*\@)?(www\.)?(\V%s\v)($|/)]], + -- `domain` here includes the host name and port. If it doesn't contain + -- characters illegal in a host or port, this encoding should do nothing. + -- If it contains illegal characters, the domain is broken in a safe way. + vim.uri_encode(domain) + ) +end + +---@param pattern string +---@return OrgPromise +local function cache_safe_pattern(pattern) + ---@param state OrgState + return State:load():next(function(state) + -- We manipulate `cached` in a strange way here to ensure that `state` gets + -- marked as dirty. + local patterns = { pattern } + local cached = state['org_safe_remote_resources'] + if cached then + vim.list_extend(patterns, cached) + end + state['org_safe_remote_resources'] = patterns + end) +end + +---Ask the user if URI should be considered safe. +---@param uri string +---@return OrgPromise safe +function M.confirm_safe(uri) + ---@type OrgMenu + return Promise.new(function(resolve) + local menu = Menu:new({ + title = string.format('An org-mode document would like to download %s, which is not considered safe.', uri), + prompt = 'Do you want to download this?', + }) + menu:add_option({ + key = '!', + label = 'Yes, and mark it as safe.', + action = function() + cache_safe_pattern(uri_to_pattern(uri)) + return true + end, + }) + local authority = uri:match('^https?://([^/?#]*)') + -- `domain` here includes the host name and port. + local domain = authority and authority:match('^[^@]*@(.*)$') or authority + if domain then + menu:add_option({ + key = 'd', + label = string.format('Yes, and mark the domain as safe. (%s)', domain), + action = function() + cache_safe_pattern(domain_to_pattern(domain)) + return true + end, + }) + end + local filename = fs.get_real_path(utils.current_file_path()) + if filename then + menu:add_option({ + key = 'f', + label = string.format('Yes, and mark the org file as safe. (%s)', filename), + action = function() + cache_safe_pattern(filename_to_pattern(filename)) + return true + end, + }) + end + menu:add_option({ + key = 'y', + label = 'Yes, just this once.', + action = function() + return true + end, + }) + menu:add_option({ + key = 'n', + label = 'No, skip this resource.', + action = function() + return false + end, + }) + menu:add_separator({ icon = ' ', length = 1 }) + resolve(menu:open()) + end) +end + +return M diff --git a/lua/orgmode/state/state.lua b/lua/orgmode/state/state.lua index 280a43555..b63d64ff4 100644 --- a/lua/orgmode/state/state.lua +++ b/lua/orgmode/state/state.lua @@ -172,7 +172,7 @@ function OrgState:wipe(overwrite) self._ctx.saved = false self._ctx.dirty = true if overwrite then - state:save_sync() + self:save_sync() end end diff --git a/lua/orgmode/utils/fs.lua b/lua/orgmode/utils/fs.lua index ed277679a..90d0fc346 100644 --- a/lua/orgmode/utils/fs.lua +++ b/lua/orgmode/utils/fs.lua @@ -21,6 +21,7 @@ function M.substitute_path(path_str) end ---@param filepath string +---@return string | false function M.get_real_path(filepath) if not filepath then return false diff --git a/tests/plenary/object/remote_resource_spec.lua b/tests/plenary/object/remote_resource_spec.lua new file mode 100644 index 000000000..bac25d9b9 --- /dev/null +++ b/tests/plenary/object/remote_resource_spec.lua @@ -0,0 +1,109 @@ +local State = require('orgmode.state.state') +local config = require('orgmode.config') +local remote = require('orgmode.objects.remote_resource') + +local config_backup = vim.deepcopy(config.opts) + +describe('Remote resource', function() + local SAFE_URL = 'https://example.com/' + local UNSAFE_URL = 'http://bad.example.org/' + + after_each(function() + config:extend(config_backup) + end) + + describe('with policy "always"', function() + before_each(function() + config:extend({ org_resource_download_policy = 'always' }) + end) + it('accepts everything', function() + assert.is.True(remote.should_fetch(UNSAFE_URL):wait()) + end) + end) + + describe('with policy "safe"', function() + before_each(function() + config:extend({ + org_resource_download_policy = 'safe', + -- This implicitly tests that we use actual regexes and not Lua + -- patterns (which would use `%` as escape character, not `\`). + org_safe_remote_resources = { '^https://.*\\.com/\\?$' }, + }) + end) + it('accepts safe URLs', function() + assert.is.True(remote.should_fetch(SAFE_URL):wait()) + end) + it('rejects unsafe URLs', function() + assert.is.False(remote.should_fetch(UNSAFE_URL):wait()) + end) + end) + + describe('with policy "prompt"', function() + before_each(function() + config:extend({ org_resource_download_policy = 'prompt' }) + end) + it('opens a prompt', function() + vim.api.nvim_input('') + assert.is.Nil(remote.should_fetch(SAFE_URL):wait()) + end) + it('accepts on "y"', function() + vim.api.nvim_input('y') + assert.is.True(remote.should_fetch(SAFE_URL):wait()) + end) + it('rejects on "n"', function() + vim.api.nvim_input('n') + assert.is.False(remote.should_fetch(SAFE_URL):wait()) + end) + describe('and saving decisions', function() + after_each(function() + State:wipe() + end) + it('accepts forever with "!"', function() + vim.api.nvim_input('!') + assert.is.True(remote.should_fetch(SAFE_URL):wait()) + config:extend({ org_resource_download_policy = 'safe' }) + assert.is.True(remote.should_fetch(SAFE_URL):wait()) + assert.is.False(remote.should_fetch(SAFE_URL .. '/more'):wait()) + assert.is.False(remote.should_fetch(UNSAFE_URL):wait()) + end) + it('accepts forever with "d"', function() + vim.api.nvim_input('d') + assert.is.True(remote.should_fetch(SAFE_URL):wait()) + config:extend({ org_resource_download_policy = 'safe' }) + assert.is.True(remote.should_fetch(SAFE_URL):wait()) + assert.is.True(remote.should_fetch(SAFE_URL .. '/more'):wait()) + assert.is.False(remote.should_fetch(UNSAFE_URL):wait()) + end) + it('accepts forever with "f"', function() + local todo_file = vim.fn.getcwd() .. '/tests/plenary/fixtures/todo.org' + vim.cmd.edit(todo_file) + vim.api.nvim_input('f') + assert.is.True(remote.should_fetch(SAFE_URL):wait()) + config:extend({ org_resource_download_policy = 'safe' }) + assert.is.True(remote.should_fetch(SAFE_URL):wait()) + assert.is.True(remote.should_fetch(UNSAFE_URL):wait()) + vim.api.nvim_buf_set_name(0, '') + assert.is.False(remote.should_fetch(SAFE_URL):wait()) + end) + end) + end) + + describe('with policy "never"', function() + before_each(function() + config:extend({ org_resource_download_policy = 'never' }) + end) + it('rejects everything', function() + assert.is.False(remote.should_fetch(SAFE_URL):wait()) + end) + end) + + describe('default config', function() + it('prompts', function() + assert.are.equal('prompt', config.org_resource_download_policy) + end) + + it('has no safe patterns', function() + assert.are.same({}, config.org_safe_remote_resources) + end) + end) +end)