Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ability to request downloads using telegram (new) #1031

Merged
merged 5 commits into from
Nov 30, 2023
Merged

Add ability to request downloads using telegram (new) #1031

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8a588cf
config_api now broadcasts when a config item has changed
Tzahi12345 Nov 29, 2023
3151200
Updated findChangedConfigItems function
Tzahi12345 Nov 29, 2023
2c97403
Added ability to request video downloads through telegram bots
Tzahi12345 Nov 29, 2023
91c2fdc
Failed telegram request now sends a response to telegram
Tzahi12345 Nov 30, 2023
8f246d9
Added ability to set a webhook proxy for telegram requests
Tzahi12345 Nov 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion backend/app.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ const twitch_api = require('./twitch');
const youtubedl_api = require('./youtube-dl');
const archive_api = require('./archive');
const files_api = require('./files');
const notifications_api = require('./notifications');

var app = express();

Expand Down Expand Up @@ -685,7 +686,7 @@ app.use(function(req, res, next) {
next();
} else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
next();
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss') || req.path.includes('/api/telegramRequest')) {
next();
} else {
logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
Expand Down Expand Up @@ -1784,6 +1785,10 @@ app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
app.post('/api/getTasks', optionalJwt, async (req, res) => {
const tasks = await db_api.getRecords('tasks');
for (let task of tasks) {
if (!tasks_api.TASKS[task['key']]) {
logger.verbose(`Task ${task['key']} does not exist!`);
continue;
}
if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task['key']]['job'].nextInvocation().getTime();
}
res.send({tasks: tasks});
Expand Down Expand Up @@ -2092,6 +2097,25 @@ app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
res.send({success: success});
});

app.post('/api/telegramRequest', async (req, res) => {
if (!req.body.message && !req.body.message.text) {
logger.error('Invalid Telegram request received!');
res.sendStatus(400);
return;
}
const text = req.body.message.text;
const regex_exp = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)?/gi;
const url_regex = new RegExp(regex_exp);
if (text.match(url_regex)) {
downloader_api.createDownload(text, 'video', {}, req.query.user_uid ? req.query.user_uid : null);
res.sendStatus(200);
} else {
logger.error('Invalid Telegram request received! Make sure you only send a valid URL.');
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if you're looking for feedback, but would it make sense to send this error in telegram so that feedback is immediate?

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Always looking for feedback! I think that's a great idea, will implement.

notifications_api.sendTelegramNotification({title: 'Invalid Telegram Request', body: 'Make sure you only send a valid URL.', url: text});
res.sendStatus(400);
}
});

// rss feed

app.get('/api/rss', async function (req, res) {
Expand Down
1 change: 1 addition & 0 deletions backend/appdata/default.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
"use_telegram_API": false,
"telegram_bot_token": "",
"telegram_chat_id": "",
"telegram_webhook_proxy": "",
"webhook_URL": "",
"discord_webhook_URL": "",
"slack_webhook_URL": ""
Expand Down
99 changes: 60 additions & 39 deletions backend/config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,26 @@
const logger = require('./logger');

const fs = require('fs');
const { BehaviorSubject } = require('rxjs');

exports.CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
exports.descriptors = {}; // to get rid of file locks when needed, TODO: move to youtube-dl.js

let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
const debugMode = process.env.YTDL_MODE === 'debug';

let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
exports.config_updated = new BehaviorSubject();

function initialize() {
exports.initialize = () => {
ensureConfigFileExists();
ensureConfigItemsExist();
}

function ensureConfigItemsExist() {
const config_keys = Object.keys(CONFIG_ITEMS);
const config_keys = Object.keys(exports.CONFIG_ITEMS);
for (let i = 0; i < config_keys.length; i++) {
const config_key = config_keys[i];
getConfigItem(config_key);
exports.getConfigItem(config_key);
}
}

Expand Down Expand Up @@ -57,17 +61,17 @@ function getElementNameInConfig(path) {
/**
* Check if config exists. If not, write default config to config path
*/
function configExistsCheck() {
exports.configExistsCheck = () => {
let exists = fs.existsSync(configPath);
if (!exists) {
setConfigFile(DEFAULT_CONFIG);
exports.setConfigFile(DEFAULT_CONFIG);
}
}

/*
* Gets config file and returns as a json
*/
function getConfigFile() {
exports.getConfigFile = () => {
try {
let raw_data = fs.readFileSync(configPath);
let parsed_data = JSON.parse(raw_data);
Expand All @@ -78,35 +82,40 @@ function getConfigFile() {
}
}

function setConfigFile(config) {
exports.setConfigFile = (config) => {
try {
const old_config = exports.getConfigFile();
fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
const changes = exports.findChangedConfigItems(old_config, config);
if (changes.length > 0) {
for (const change of changes) exports.config_updated.next(change);
}
return true;
} catch(e) {
return false;
}
}

function getConfigItem(key) {
let config_json = getConfigFile();
if (!CONFIG_ITEMS[key]) {
exports.getConfigItem = (key) => {
let config_json = exports.getConfigFile();
if (!exports.CONFIG_ITEMS[key]) {
logger.error(`Config item with key '${key}' is not recognized.`);
return null;
}
let path = CONFIG_ITEMS[key]['path'];
let path = exports.CONFIG_ITEMS[key]['path'];
const val = Object.byString(config_json, path);
if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
exports.setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
return Object.byString(DEFAULT_CONFIG, path);
}
return Object.byString(config_json, path);
}

function setConfigItem(key, value) {
exports.setConfigItem = (key, value) => {
let success = false;
let config_json = getConfigFile();
let path = CONFIG_ITEMS[key]['path'];
let config_json = exports.getConfigFile();
let path = exports.CONFIG_ITEMS[key]['path'];
let element_name = getElementNameInConfig(path);
let parent_path = getParentPath(path);
let parent_object = Object.byString(config_json, parent_path);
Expand All @@ -118,20 +127,18 @@ function setConfigItem(key, value) {
parent_parent_object[parent_parent_single_key] = {};
parent_object = Object.byString(config_json, parent_path);
}
if (value === 'false') value = false;
if (value === 'true') value = true;
parent_object[element_name] = value;

if (value === 'false' || value === 'true') {
parent_object[element_name] = (value === 'true');
} else {
parent_object[element_name] = value;
}
success = setConfigFile(config_json);
success = exports.setConfigFile(config_json);

return success;
}

function setConfigItems(items) {
exports.setConfigItems = (items) => {
let success = false;
let config_json = getConfigFile();
let config_json = exports.getConfigFile();
for (let i = 0; i < items.length; i++) {
let key = items[i].key;
let value = items[i].value;
Expand All @@ -141,36 +148,49 @@ function setConfigItems(items) {
value = (value === 'true');
}

let item_path = CONFIG_ITEMS[key]['path'];
let item_path = exports.CONFIG_ITEMS[key]['path'];
let item_parent_path = getParentPath(item_path);
let item_element_name = getElementNameInConfig(item_path);

let item_parent_object = Object.byString(config_json, item_parent_path);
item_parent_object[item_element_name] = value;
}

success = setConfigFile(config_json);
success = exports.setConfigFile(config_json);
return success;
}

function globalArgsRequiresSafeDownload() {
const globalArgs = getConfigItem('ytdl_custom_args').split(',,');
exports.globalArgsRequiresSafeDownload = () => {
const globalArgs = exports.getConfigItem('ytdl_custom_args').split(',,');
const argsThatRequireSafeDownload = ['--write-sub', '--write-srt', '--proxy'];
const failedArgs = globalArgs.filter(arg => argsThatRequireSafeDownload.includes(arg));
return failedArgs && failedArgs.length > 0;
}

module.exports = {
getConfigItem: getConfigItem,
setConfigItem: setConfigItem,
setConfigItems: setConfigItems,
getConfigFile: getConfigFile,
setConfigFile: setConfigFile,
configExistsCheck: configExistsCheck,
CONFIG_ITEMS: CONFIG_ITEMS,
initialize: initialize,
descriptors: {},
globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
exports.findChangedConfigItems = (old_config, new_config, path = '', changedConfigItems = [], depth = 0) => {
if (typeof old_config === 'object' && typeof new_config === 'object' && depth < 3) {
for (const key in old_config) {
if (Object.prototype.hasOwnProperty.call(new_config, key)) {
exports.findChangedConfigItems(old_config[key], new_config[key], `${path}${path ? '.' : ''}${key}`, changedConfigItems, depth + 1);
}
}
} else {
if (JSON.stringify(old_config) !== JSON.stringify(new_config)) {
const key = getConfigItemKeyByPath(path);
changedConfigItems.push({
key: key ? key : path.split('.')[path.split('.').length - 1], // return key in CONFIG_ITEMS or the object key
old_value: JSON.parse(JSON.stringify(old_config)),
new_value: JSON.parse(JSON.stringify(new_config))
});
}
}
return changedConfigItems;
}

function getConfigItemKeyByPath(path) {
const found_item = Object.values(exports.CONFIG_ITEMS).find(item => item.path === path);
if (found_item) return found_item['key'];
else return null;
}

const DEFAULT_CONFIG = {
Expand Down Expand Up @@ -219,6 +239,7 @@ const DEFAULT_CONFIG = {
"use_telegram_API": false,
"telegram_bot_token": "",
"telegram_chat_id": "",
"telegram_webhook_proxy": "",
"webhook_URL": "",
"discord_webhook_URL": "",
"slack_webhook_URL": "",
Expand Down
4 changes: 4 additions & 0 deletions backend/consts.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,10 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_telegram_chat_id',
'path': 'YoutubeDLMaterial.API.telegram_chat_id'
},
'ytdl_telegram_webhook_proxy': {
'key': 'ytdl_telegram_webhook_proxy',
'path': 'YoutubeDLMaterial.API.telegram_webhook_proxy'
},
'ytdl_webhook_url': {
'key': 'ytdl_webhook_url',
'path': 'YoutubeDLMaterial.API.webhook_URL'
Expand Down
58 changes: 51 additions & 7 deletions backend/notifications.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ const { uuid } = require('uuidv4');

const fetch = require('node-fetch');
const { gotify } = require("gotify");
const TelegramBot = require('node-telegram-bot-api');
const TelegramBotAPI = require('node-telegram-bot-api');
let telegram_bot = null;
const REST = require('@discordjs/rest').REST;
const API = require('@discordjs/core').API;
const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
Expand Down Expand Up @@ -56,7 +57,7 @@ exports.sendNotification = async (notification) => {
sendGotifyNotification(data);
}
if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
sendTelegramNotification(data);
exports.sendTelegramNotification(data);
}
if (config_api.getConfigItem('ytdl_webhook_url')) {
sendGenericNotification(data);
Expand Down Expand Up @@ -113,6 +114,8 @@ function notificationEnabled(type) {
return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
}

// ntfy

function sendNtfyNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to ntfy');
fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
Expand All @@ -127,6 +130,8 @@ function sendNtfyNotification({body, title, type, url, thumbnail}) {
});
}

// Gotify

async function sendGotifyNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to gotify');
await gotify({
Expand All @@ -145,15 +150,50 @@ async function sendGotifyNotification({body, title, type, url, thumbnail}) {
});
}

async function sendTelegramNotification({body, title, type, url, thumbnail}) {
logger.verbose('Sending notification to Telegram');
// Telegram

setupTelegramBot();
config_api.config_updated.subscribe(change => {
const use_telegram_api = config_api.getConfigItem('ytdl_use_telegram_API');
const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
if (!use_telegram_api || !bot_token) return;
if (!change) return;
if (change['key'] === 'ytdl_use_telegram_API' || change['key'] === 'ytdl_telegram_bot_token' || change['key'] === 'ytdl_telegram_webhook_proxy') {
logger.debug('Telegram bot setting up');
setupTelegramBot();
}
});

async function setupTelegramBot() {
const use_telegram_api = config_api.getConfigItem('ytdl_use_telegram_API');
const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
if (!use_telegram_api || !bot_token) return;

telegram_bot = new TelegramBotAPI(bot_token);
const webhook_proxy = config_api.getConfigItem('ytdl_telegram_webhook_proxy');
const webhook_url = webhook_proxy ? webhook_proxy : `${utils.getBaseURL()}/api/telegramRequest`;
telegram_bot.setWebHook(webhook_url);
}

exports.sendTelegramNotification = async ({body, title, type, url, thumbnail}) => {
if (!telegram_bot){
logger.error('Telegram bot not found!');
return;
}

const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
const bot = new TelegramBot(bot_token);
if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
if (!chat_id){
logger.error('Telegram chat ID required!');
return;
}

logger.verbose('Sending notification to Telegram');
if (thumbnail) await telegram_bot.sendPhoto(chat_id, thumbnail);
telegram_bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
}

// Discord

async function sendDiscordNotification({body, title, type, url, thumbnail}) {
const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
const url_split = discord_webhook_url.split('webhooks/');
Expand All @@ -177,6 +217,8 @@ async function sendDiscordNotification({body, title, type, url, thumbnail}) {
return result;
}

// Slack

function sendSlackNotification({body, title, type, url, thumbnail}) {
const slack_webhook_url = config_api.getConfigItem('ytdl_slack_webhook_url');
logger.verbose(`Sending slack notification to ${slack_webhook_url}`);
Expand Down Expand Up @@ -236,6 +278,8 @@ function sendSlackNotification({body, title, type, url, thumbnail}) {
});
}

// Generic

function sendGenericNotification(data) {
const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
logger.verbose(`Sending generic notification to ${webhook_url}`);
Expand Down
Loading