From 8272dfbc70445d63cd8403cee5485f2a181b68a3 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Wed, 15 Jan 2025 17:39:02 +0100 Subject: [PATCH 01/31] First version of the storageService that communicates with the file system --- lib/Service/StorageService.php | 150 +++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 lib/Service/StorageService.php diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php new file mode 100644 index 00000000..fd1e3955 --- /dev/null +++ b/lib/Service/StorageService.php @@ -0,0 +1,150 @@ +cache = $cacheFactory->createDistributed(self::CACHE_KEY); + } + + private function checkPreconditions(int $size, bool $checkMetadata = false): bool + { + if($size < $this->config->getValueInt('openconnector', 'upload-size', 2000000)) { + return false; + } + + if($this->cache instanceof Redis === false && $this->cache instanceof Memcached === false) { + return false; + } + if($this->uploadFolder->getStorage()->instanceOfStorage(IChunkedFileWrite::class) === false) { + return false; + } + if($this->uploadFolder->getStorage()->instanceOfStorage(ObjectStoreStorage::class) === false + && $this->uploadFolder->getStorage()->getObjectStore() instanceof IObjectStoreMultiPartUpload === false + ) { + return false; + } + if ($checkMetadata === true && ($this->uploadId === null || $this->uploadPath === null)) { + return false; + } + + return true; + } + + private function getFile(string $path, bool $createIfNotExists = false): File + { + try{ + $file = $this->rootFolder->get($path); + if($file instanceof File && $this->uploadFolder->getStorage()->getId() === $file->getStorage()->getId()) { + return $file; + } + } catch (NotFoundException $e) { + + } + + if ($createIfNotExists === true + && $this->uploadFolder instanceof Folder === true + ) { + $file = $this->uploadFolder->newFile(self::TEMP_TARGET); + } + + return $file; + } + + /** + * @return array [IStorage, string] + */ + private function getUploadStorage(string $targetPath): array { + $storage = $this->uploadFolder->getStorage(); + $targetFile = $this->getFile(path: $targetPath); + return [$storage, $targetFile->getInternalPath()]; + } + + public function createUpload(string $path, string $fileName, int $size): int + { + $this->uploadFolder = $this->rootFolder->get($path); + + if ($this->checkPreconditions($size) === false + ) { + return 0; + } + $this->uploadPath = $path.$fileName; + + $targetFile = $this->getFile(path: $path.$fileName, createIfNotExists: true); + [$storage, $storagePath] = $this->getUploadStorage($this->uploadPath); + $this->uploadId = $storage->startChunkedWrite($storagePath); + + $this->cache->set($this->uploadFolder->getName(), [ + self::UPLOAD_ID => $this->uploadId, + self::UPLOAD_TARGET_PATH => $this->uploadPath, + self::UPLOAD_TARGET_ID => $targetFile->getId(), + ], 86400); + + return ceil($size / $this->config->getValueInt('openconnector', 'part-size', 500000)); + } + + public function writePart(int $partId, string $data, string $path, $numParts): bool + { + try { + $this->uploadFolder = $this->rootFolder->get($path); + + $uploadMetadata = $this->cache->get($this->uploadFolder->getName()); + $this->uploadId = $uploadMetadata[self::UPLOAD_ID] ?? null; + $this->uploadPath = $uploadMetadata[self::UPLOAD_TARGET_PATH] ?? null; + + [$storage, $storagePath] = $this->getUploadStorage($this->uploadPath); + + $file = $this->getFile(path: $this->uploadPath); + if($this->uploadFolder instanceof Folder) { + $tempTarget = $this->uploadFolder->get(self::TEMP_TARGET); + } + + $storage->putChunkedWritePart($storagePath, $this->uploadId, (string)$partId, $data, strlen($data)); + + $storage->getCache()->update($file->getId(), ['size' => $file->getSize() + strlen($data)]); + if ($tempTarget) { + $storage->getPropagator()->propagateChange($tempTarget->getInternalPath(), time(), strlen($data)); + } + + if(count($storage->getObjectStore()->getMultipartUploads($storage->getUrn($file->getId()), $this->uploadId)) === $numParts) { + $storage->completeChunkedWrite($path, $this->uploadId); + } + } catch(Exception $e) { + throw $e; + } + + return true; + } +} From aba5f8edd4a69f102aaa3da523bb4fd5a02c7b2b Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Tue, 21 Jan 2025 09:33:21 +0100 Subject: [PATCH 02/31] First steps in connecting endpoints with storage --- lib/Service/EndpointService.php | 68 +++++++++++++++++++++++++++++++-- lib/Service/StorageService.php | 23 +++++++++-- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/lib/Service/EndpointService.php b/lib/Service/EndpointService.php index f34dc3a1..110a55e4 100644 --- a/lib/Service/EndpointService.php +++ b/lib/Service/EndpointService.php @@ -66,6 +66,7 @@ public function __construct( private readonly EndpointMapper $endpointMapper, private readonly RuleMapper $ruleMapper, private readonly IConfig $config, + private readonly StorageService $storageService ) { } @@ -101,7 +102,12 @@ public function handleRequest(Endpoint $endpoint, IRequest $request, string $pat ]; // Process rules before handling the request - $ruleResult = $this->processRules($endpoint, $request, $data); + $ruleResult = $this->processRules( + endpoint: $endpoint, + request: $request, + data: $data, + timing: 'before' + ); if ($ruleResult instanceof JSONResponse) { return $ruleResult; } @@ -112,7 +118,14 @@ public function handleRequest(Endpoint $endpoint, IRequest $request, string $pat // Check if endpoint connects to a schema if ($endpoint->getTargetType() === 'register/schema') { // Handle CRUD operations via ObjectService - return $this->handleSchemaRequest($endpoint, $request, $path); + $result = $this->handleSchemaRequest($endpoint, $request, $path); + $ruleResult = $this->processRules( + endpoint: $endpoint, + request: $request, + data: $data, + timing: 'after', + objectId: $result->getData()['id'] ?? null + ); } // Check if endpoint connects to a source @@ -539,7 +552,7 @@ public function generateEndpointUrl(string $id, ?int $register = null, ?int $sch * * @return array|JSONResponse Returns modified data or error response if rule fails */ - private function processRules(Endpoint $endpoint, IRequest $request, array $data): array|JSONResponse + private function processRules(Endpoint $endpoint, IRequest $request, array $data, string $timing, ?string $objectId = null): array|JSONResponse { $rules = $endpoint->getRules(); if (empty($rules) === true) { @@ -566,7 +579,7 @@ private function processRules(Endpoint $endpoint, IRequest $request, array $data } // Check rule conditions - if ($this->checkRuleConditions($rule, $data) === false) { + if ($this->checkRuleConditions($rule, $data) === false && $rule->getTiming() === $timing) { continue; } @@ -576,6 +589,7 @@ private function processRules(Endpoint $endpoint, IRequest $request, array $data 'mapping' => $this->processMappingRule($rule, $data), 'synchronization' => $this->processSyncRule($rule, $data), 'javascript' => $this->processJavaScriptRule($rule, $data), + 'fileParts' => $this->processFilePartRule($rule, $data, $objectId), default => throw new Exception('Unsupported rule type: ' . $rule->getType()), }; @@ -666,6 +680,52 @@ private function processSyncRule(Rule $rule, array $data): array return $data; } + private function processFilePartRule(Rule $rule, array $data, ?string $objectId = null): array + { + if($objectId === null) { + throw new Exception('Filepart rules can only be applied after the object has been created'); + } + + + + $config = $rule->getConfiguration(); + + $sizeLocation = $config['sizeLocation']; + $filenameLocation = $config['filenameLocation']; + $schemaId = $config['schemaId']; + $registerId = $config['registerId']; + $mapping = $this->mappingService->getMapping($config['mappingId']); + $filePartLocation = $config['filePartLocation']; + + $openRegister = $this->objectService->getOpenRegisters(); + $openRegister->setRegister($registerId); + $openRegister->setSchema($schemaId); + + $object = $openRegister->find(id: $objectId); + $location = $object->getFolder(); + + $dataDot = new Dot($data); + $size = $dataDot[$sizeLocation]; + $filename = $dataDot[$filenameLocation]; + + $fileParts = $this->storageService->createUpload($location, $filename, $size); + + + $fileParts = array_map(function ($filePart) use ($mapping, $registerId, $schemaId) { + $formatted = $this->mappingService->executeMapping(mapping: $mapping, input: $filePart); + return $this->objectService->getOpenRegisters()->saveObject( + register: $registerId, + schema: $schemaId, + object: $formatted + )->jsonSerialize(); + }, $fileParts); + + + $dataDot[$filePartLocation] = $fileParts; + + return $dataDot->jsonSerialize(); + } + /** * Processes a JavaScript rule * diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php index fd1e3955..5d54b2c4 100644 --- a/lib/Service/StorageService.php +++ b/lib/Service/StorageService.php @@ -1,6 +1,6 @@ getInternalPath()]; } - public function createUpload(string $path, string $fileName, int $size): int + public function createUpload(string $path, string $fileName, int $size): array { $this->uploadFolder = $this->rootFolder->get($path); if ($this->checkPreconditions($size) === false ) { - return 0; + return []; } $this->uploadPath = $path.$fileName; @@ -112,7 +112,22 @@ public function createUpload(string $path, string $fileName, int $size): int self::UPLOAD_TARGET_ID => $targetFile->getId(), ], 86400); - return ceil($size / $this->config->getValueInt('openconnector', 'part-size', 500000)); + $partSize = $this->config->getValueInt('openconnector', 'part-size', 500000); + + $numParts = ceil($size / $partSize); + + $remainingSize = $size; + $parts = []; + + for ($i = 0; $i < $numParts; $i++) { + $parts[] = [ + 'size' => $partSize < $remainingSize ? $partSize : $remainingSize, + 'order' => $i+1, + ]; + $remainingSize -= $partSize; + } + + return $parts; } public function writePart(int $partId, string $data, string $path, $numParts): bool From 6a026fd4bb8ff643b67cae7d212209feae1d3225 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Tue, 21 Jan 2025 16:54:44 +0100 Subject: [PATCH 03/31] Add a writeFile function (move to registers) and some docblocks --- lib/Service/StorageService.php | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php index 5d54b2c4..1627d5bb 100644 --- a/lib/Service/StorageService.php +++ b/lib/Service/StorageService.php @@ -92,6 +92,16 @@ private function getUploadStorage(string $targetPath): array { return [$storage, $targetFile->getInternalPath()]; } + /** + * Based upon the webDAV partial files plugin + * + * @param string $path + * @param string $fileName + * @param int $size + * @return array + * @throws NotFoundException + * @throws \OCP\Files\InvalidPathException + */ public function createUpload(string $path, string $fileName, int $size): array { $this->uploadFolder = $this->rootFolder->get($path); @@ -102,7 +112,7 @@ public function createUpload(string $path, string $fileName, int $size): array } $this->uploadPath = $path.$fileName; - $targetFile = $this->getFile(path: $path.$fileName, createIfNotExists: true); + $targetFile = $this->getFile(path: $path.'/'.$fileName, createIfNotExists: true); [$storage, $storagePath] = $this->getUploadStorage($this->uploadPath); $this->uploadId = $storage->startChunkedWrite($storagePath); @@ -130,6 +140,27 @@ public function createUpload(string $path, string $fileName, int $size): array return $parts; } + public function writeFile(string $path, string $fileName, string $content): File + { + $uploadFolder = $this->rootFolder->get($path); + + $target = $this->getFile(path: $path.'/'.$fileName, createIfNotExists: true); + $target->putContent($content); + + return $target; + } + + /** + * Based upon the webDAV partial files plugin + * + * @param int $partId + * @param string $data + * @param string $path + * @param $numParts + * @return bool + * @throws NotFoundException + * @throws \OCP\Files\InvalidPathException + */ public function writePart(int $partId, string $data, string $path, $numParts): bool { try { From 36f376833ee4af2ac554f4ead2a36d0f300e0432 Mon Sep 17 00:00:00 2001 From: Thijn Date: Thu, 23 Jan 2025 13:27:12 +0100 Subject: [PATCH 04/31] fetch logs and contracts after test and run --- src/modals/Synchronization/RunSynchronization.vue | 3 +++ src/modals/Synchronization/TestSynchronization.vue | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/modals/Synchronization/RunSynchronization.vue b/src/modals/Synchronization/RunSynchronization.vue index 81f728b5..b0304abb 100644 --- a/src/modals/Synchronization/RunSynchronization.vue +++ b/src/modals/Synchronization/RunSynchronization.vue @@ -143,6 +143,9 @@ export default { this.responseBody = data this.responseBodyString = JSON.stringify(data, null, 2) this.success = response.ok + + synchronizationStore.refreshSynchronizationLogs() + synchronizationStore.refreshSynchronizationContracts() }).catch((error) => { this.success = false this.error = error.message || 'An error occurred while running the synchronization' diff --git a/src/modals/Synchronization/TestSynchronization.vue b/src/modals/Synchronization/TestSynchronization.vue index e3c8e40e..d4efe981 100644 --- a/src/modals/Synchronization/TestSynchronization.vue +++ b/src/modals/Synchronization/TestSynchronization.vue @@ -127,6 +127,9 @@ export default { this.response = response this.responseBody = JSON.stringify({ ...data }, null, 2) this.success = response.ok + + synchronizationStore.refreshSynchronizationLogs() + synchronizationStore.refreshSynchronizationContracts() }).catch((error) => { this.success = false this.error = error.message || 'An error occurred while testing the synchronization' From 40efee05f5c46c67a273f9f15d5f5dc24a4ee892 Mon Sep 17 00:00:00 2001 From: Remko Date: Thu, 23 Jan 2025 13:57:03 +0100 Subject: [PATCH 05/31] Added multi file upload --- css/main.css | 3 - src/composables/UseFileSelection.js | 31 ++-- src/modals/Import/ImportFile.vue | 2 +- src/navigation/MainMenu.vue | 10 ++ src/store/modules/importExport.js | 36 +++- src/views/Import/ImportIndex.vue | 250 ++++++++++++++++++++++++++++ src/views/Views.vue | 2 + 7 files changed, 316 insertions(+), 18 deletions(-) create mode 100644 src/views/Import/ImportIndex.vue diff --git a/css/main.css b/css/main.css index 4151c9a9..5c187009 100644 --- a/css/main.css +++ b/css/main.css @@ -24,9 +24,6 @@ --OC-color-status-emergency: #ffffff; } - - - /* Pages */ .pageHeader { margin-block-start: var(--app-navigation-padding); diff --git a/src/composables/UseFileSelection.js b/src/composables/UseFileSelection.js index a3e35f35..a67808f6 100644 --- a/src/composables/UseFileSelection.js +++ b/src/composables/UseFileSelection.js @@ -23,11 +23,11 @@ export function useFileSelection(options) { // Data types computed ref const dataTypes = computed(() => { - if (allowedFileTypes?.value) { - if (!Array.isArray(allowedFileTypes.value)) { - return [allowedFileTypes.value] + if (allowedFileTypes) { + if (!Array.isArray(allowedFileTypes)) { + return [allowedFileTypes] } - return allowedFileTypes.value + return allowedFileTypes } return null }) @@ -48,16 +48,25 @@ export function useFileSelection(options) { if (files instanceof FileList) { files = Array.from(files) } - if (files.length > 1 && !allowMultiple.value) { + if (files.length > 1 && !allowMultiple) { files = [files[0]] } + if (filesList.value?.length > 0 && allowMultiple) { + const filteredFiles = files.filter(file => !filesList.value.some(f => f.name === file.name)) + files = [...filesList.value, ...filteredFiles] + } + filesList.value = files onFileDrop && onFileDrop() onFileSelect && onFileSelect() } - const reset = () => { - filesList.value = null + const reset = (name = null) => { + if (name) { + filesList.value = filesList.value.filter(file => file.name !== name).length > 0 ? filesList.value.filter(file => file.name !== name) : null + } else { + filesList.value = null + } } const setFiles = (files) => { @@ -65,16 +74,18 @@ export function useFileSelection(options) { } // Setup dropzone and file dialog composables - const { isOverDropZone } = useDropZone(dropzone, { dataTypes, onDrop }) + const { isOverDropZone } = useDropZone(dropzone, { dataTypes: '*', onDrop }) const { onChange, open } = useFileDialog({ accept: accept.value, - multiple: allowMultiple?.value, + multiple: allowMultiple, }) const filesList = ref(null) // Use onChange handler - onChange(fileList => onDrop(fileList)) + onChange(fileList => { + onDrop(fileList) + }) // Expose interface return { diff --git a/src/modals/Import/ImportFile.vue b/src/modals/Import/ImportFile.vue index fcc864fa..329c19cd 100644 --- a/src/modals/Import/ImportFile.vue +++ b/src/modals/Import/ImportFile.vue @@ -87,7 +87,7 @@ import TrayArrowDown from 'vue-material-design-icons/TrayArrowDown.vue' import FileImportOutline from 'vue-material-design-icons/FileImportOutline.vue' const dropZoneRef = ref() -const { openFileUpload, files, reset, setFiles } = useFileSelection({ allowMultiple: false, dropzone: dropZoneRef }) +const { openFileUpload, files, reset, setFiles } = useFileSelection({ allowMultiple: false, dropzone: dropZoneRef, allowedFileTypes: ['.json', '.yaml', '.yml'] }) export default { name: 'ImportFile', diff --git a/src/navigation/MainMenu.vue b/src/navigation/MainMenu.vue index 2346b4f1..3c408b16 100644 --- a/src/navigation/MainMenu.vue +++ b/src/navigation/MainMenu.vue @@ -65,6 +65,13 @@ import { navigationStore } from '../store/store.js' + + + + + @@ -73,6 +80,7 @@ import { NcAppNavigation, NcAppNavigationList, NcAppNavigationItem, + NcAppNavigationSettings, } from '@nextcloud/vue' // Icons @@ -85,6 +93,7 @@ import Update from 'vue-material-design-icons/Update.vue' import VectorPolylinePlus from 'vue-material-design-icons/VectorPolylinePlus.vue' import CloudUploadOutline from 'vue-material-design-icons/CloudUploadOutline.vue' import MessageTextFastOutline from 'vue-material-design-icons/MessageTextFastOutline.vue' +import FileImportOutline from 'vue-material-design-icons/FileImportOutline.vue' export default { name: 'MainMenu', @@ -103,6 +112,7 @@ export default { VectorPolylinePlus, CloudUploadOutline, MessageTextFastOutline, + FileImportOutline, }, methods: { openLink(url, type = '') { diff --git a/src/store/modules/importExport.js b/src/store/modules/importExport.js index c2e294ee..605cb0fe 100644 --- a/src/store/modules/importExport.js +++ b/src/store/modules/importExport.js @@ -46,16 +46,16 @@ export const useImportExportStore = defineStore( return { response, blob, download } }, - importFile(files, reset) { - if (!files) { - throw Error('No files to import') + importFile(file, reset) { + if (!file) { + throw Error('No file to import') } if (!reset) { throw Error('No reset function to call') } return axios.post('/index.php/apps/openconnector/api/import', { - file: files.value ? files.value[0] : '', + file: file.value ? file.value[0] : '', }, { headers: { 'Content-Type': 'multipart/form-data', @@ -110,6 +110,7 @@ export const useImportExportStore = defineStore( }) ) } + reset() } return setItem() // Wait for the user to read the feedback then close the model @@ -120,6 +121,33 @@ export const useImportExportStore = defineStore( }) }, + importFiles(files, reset) { + if (!files) { + throw Error('No files to import') + } + if (!reset) { + throw Error('No reset function to call') + } + + return axios.post('/index.php/apps/openconnector/api/import', { + files: files.value, + }, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }) + .then((response) => { + + console.info('Importing files:', response.data) + + // Wait for the user to read the feedback then close the model + }) + .catch((err) => { + console.error('Error importing files:', err) + throw err + }) + + }, }, }, ) diff --git a/src/views/Import/ImportIndex.vue b/src/views/Import/ImportIndex.vue new file mode 100644 index 00000000..2718ba64 --- /dev/null +++ b/src/views/Import/ImportIndex.vue @@ -0,0 +1,250 @@ + + + + + + diff --git a/src/views/Views.vue b/src/views/Views.vue index 8523d8e1..f6a54280 100644 --- a/src/views/Views.vue +++ b/src/views/Views.vue @@ -16,6 +16,7 @@ import { navigationStore } from '../store/store.js' + @@ -32,6 +33,7 @@ import MappingsIndex from './Mapping/MappingsIndex.vue' import SynchronizationsIndex from './Synchronization/SynchronizationsIndex.vue' import EventsIndex from './event/EventIndex.vue' import RulesIndex from './rule/RuleIndex.vue' +import ImportIndex from './Import/ImportIndex.vue' export default { name: 'Views', From a75d3f3544aa323dbf2e487082e9f4d8da8f703b Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Thu, 23 Jan 2025 15:11:27 +0100 Subject: [PATCH 06/31] Sync files from source - When fetching a file (fetch_file), set in rule config the filePath property in the rule configuration to find the property in the object that is synced, and source (value = id) to set the source the file can be downloaded from - When writing the file (write_file), set the fileNamePath in the rule configuration the property in the object that is synced and filePath as for fetch_file. fetch_file will set the base64 of the file in the url property, write_file will write this into the file and overwrite the value with the path of the file --- appinfo/info.xml | 2 +- lib/Db/Synchronization.php | 3 + lib/Migration/Version1Date20250123100521.php | 60 +++++ lib/Service/StorageService.php | 16 +- lib/Service/SynchronizationService.php | 226 ++++++++++++++++++- 5 files changed, 299 insertions(+), 8 deletions(-) create mode 100644 lib/Migration/Version1Date20250123100521.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 72953d5b..d121247f 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -13,7 +13,7 @@ The OpenConnector Nextcloud app provides a ESB-framework to work together in an - 🆓 Map and translate API calls ]]> - 0.1.29 + 0.1.31 agpl integration Conduction diff --git a/lib/Db/Synchronization.php b/lib/Db/Synchronization.php index 53a334f1..af81f8f7 100644 --- a/lib/Db/Synchronization.php +++ b/lib/Db/Synchronization.php @@ -39,6 +39,7 @@ class Synchronization extends Entity implements JsonSerializable protected array $conditions = []; protected array $followUps = []; + protected array $actions = []; public function __construct() { @@ -69,6 +70,7 @@ public function __construct() { $this->addType('updated', 'datetime'); $this->addType(fieldName:'conditions', type: 'json'); $this->addType(fieldName:'followUps', type: 'json'); + $this->addType(fieldName: 'actions', type: 'json'); } /** @@ -147,6 +149,7 @@ public function jsonSerialize(): array 'updated' => isset($this->updated) === true ? $this->updated->format('c') : null, 'conditions' => $this->conditions, 'followUps' => $this->followUps, + 'actions' => $this->actions, ]; } } diff --git a/lib/Migration/Version1Date20250123100521.php b/lib/Migration/Version1Date20250123100521.php new file mode 100644 index 00000000..61c86b38 --- /dev/null +++ b/lib/Migration/Version1Date20250123100521.php @@ -0,0 +1,60 @@ +hasTable(tableName: 'openconnector_synchronizations') === true) { + $table = $schema->getTable(tableName: 'openconnector_synchronizations'); + if($table->hasColumn('actions') === false){ + $table->addColumn(name: 'actions', typeName: Types::JSON)->setNotnull(false)->setDefault('[]'); + } + } + + return $schema; + } + + /** + * @param IOutput $output + * @param Closure(): ISchemaWrapper $schemaClosure + * @param array $options + */ + public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void { + } +} diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php index 1627d5bb..96888db9 100644 --- a/lib/Service/StorageService.php +++ b/lib/Service/StorageService.php @@ -18,6 +18,7 @@ use OCP\ICache; use OCP\ICacheFactory; use OCP\IConfig; +use OCP\IUserSession; class StorageService { @@ -34,7 +35,8 @@ class StorageService public function __construct( private readonly IRootFolder $rootFolder, private readonly IAppConfig $config, - ICacheFactory $cacheFactory + ICacheFactory $cacheFactory, + private readonly IUserSession $userSession, ){ $this->cache = $cacheFactory->createDistributed(self::CACHE_KEY); } @@ -142,10 +144,16 @@ public function createUpload(string $path, string $fileName, int $size): array public function writeFile(string $path, string $fileName, string $content): File { - $uploadFolder = $this->rootFolder->get($path); + $currentUser = $this->userSession->getUser(); + $userFolder = $this->rootFolder->getUserFolder(userId: $currentUser ? $currentUser->getUID() : 'Guest'); - $target = $this->getFile(path: $path.'/'.$fileName, createIfNotExists: true); - $target->putContent($content); + $uploadFolder = $userFolder->get($path); + + try{ + $target = $uploadFolder->get($fileName); + } catch (NotFoundException $e) { + $target = $uploadFolder->newFile($fileName, $content); + } return $target; } diff --git a/lib/Service/SynchronizationService.php b/lib/Service/SynchronizationService.php index e6c6195c..0ebf0b37 100644 --- a/lib/Service/SynchronizationService.php +++ b/lib/Service/SynchronizationService.php @@ -6,7 +6,10 @@ use GuzzleHttp\Exception\GuzzleException; use JWadhams\JsonLogic; use OCA\OpenConnector\Db\CallLog; +use OCA\OpenConnector\Db\Endpoint; use OCA\OpenConnector\Db\Mapping; +use OCA\OpenConnector\Db\Rule; +use OCA\OpenConnector\Db\RuleMapper; use OCA\OpenConnector\Db\Source; use OCA\OpenConnector\Db\SourceMapper; use OCA\OpenConnector\Db\MappignMapper; @@ -20,6 +23,8 @@ use OCA\OpenConnector\Service\MappingService; use OCA\OpenRegister\Db\ObjectEntity; use OCP\AppFramework\Db\MultipleObjectsReturnedException; +use OCP\AppFramework\Http\JSONResponse; +use OCP\IRequest; use Psr\Container\ContainerExceptionInterface; use Psr\Container\NotFoundExceptionInterface; use Symfony\Component\HttpKernel\Exception\TooManyRequestsHttpException; @@ -66,6 +71,8 @@ public function __construct( SynchronizationContractMapper $synchronizationContractMapper, SynchronizationContractLogMapper $synchronizationContractLogMapper, private readonly ObjectService $objectService, + private readonly StorageService $storageService, + private readonly RuleMapper $ruleMapper, ) { $this->callService = $callService; @@ -115,6 +122,7 @@ public function synchronize(Synchronization $synchronization, ?bool $isTest = fa } $synchronizedTargetIds = []; + foreach ($objectList as $key => $object) { // Check if object adheres to conditions. @@ -540,9 +548,14 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont $targetObject = $object; } - // set the target hash - $targetHash = md5(serialize($targetObject)); - $synchronizationContract->setTargetHash($targetHash); + if($synchronization->getActions() !== []) { + $targetObject = $this->processRules(synchronization: $synchronization, data: $targetObject, timing: 'before'); + } + + // set the target hash + $targetHash = md5(serialize($targetObject)); + + $synchronizationContract->setTargetHash($targetHash); $synchronizationContract->setTargetLastChanged(new DateTime()); $synchronizationContract->setTargetLastSynced(new DateTime()); $synchronizationContract->setSourceLastSynced(new DateTime()); @@ -571,6 +584,11 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont targetObject: $targetObject ); + if($synchronization->getTargetType() === 'register/schema') { + [$registerId, $schemaId] = explode(separator: '/', string: $synchronization->getTargetId()); + $this->processRules(synchronization: $synchronization, data: $targetObject, timing: 'after', objectId: $synchronizationContract->getTargetId(), registerId: $registerId, schemaId: $schemaId); + } + // Create log entry for the synchronization $this->synchronizationContractLogMapper->createFromArray($log); @@ -1113,4 +1131,206 @@ public function synchronizeToTarget(ObjectEntity $object, ?SynchronizationContra return [$synchronizationContract]; } + + + + /** + * Processes rules for an endpoint request + * + * @param Endpoint $synchronization The endpoint being processed + * @param IRequest $request The incoming request + * @param array $data Current request data + * + * @return array|JSONResponse Returns modified data or error response if rule fails + */ + private function processRules(Synchronization $synchronization, array $data, string $timing, ?string $objectId = null, ?int $registerId = null, ?int $schemaId = null): array|JSONResponse + { + $rules = $synchronization->getActions(); + if (empty($rules) === true) { + return $data; + } + + try { + // Get all rules at once and sort by order + $ruleEntities = array_filter( + array_map( + fn($ruleId) => $this->getRuleById($ruleId), + $rules + ) + ); + + // Sort rules by order + usort($ruleEntities, fn($a, $b) => $a->getOrder() - $b->getOrder()); + + // Process each rule in order + foreach ($ruleEntities as $rule) { + // Check rule conditions + if ($this->checkRuleConditions($rule, $data) === false || $rule->getTiming() !== $timing) { + continue; + } + + // Process rule based on type + $result = match ($rule->getType()) { + 'error' => $this->processErrorRule($rule), + 'mapping' => $this->processMappingRule($rule, $data), + 'synchronization' => $this->processSyncRule($rule, $data), + 'fetch_file' => $this->processFetchFileRule($rule, $data), + 'write_file' => $this->processWriteFileRule($rule, $data, $objectId, $registerId, $schemaId), + default => throw new Exception('Unsupported rule type: ' . $rule->getType()), + }; + + // If result is JSONResponse, return error immediately + if ($result instanceof JSONResponse) { + return $result; + } + + // Update data with rule result + $data = $result; + } + + return $data; + } catch (Exception $e) { +// $this->logger->error('Error processing rules: ' . $e->getMessage()); + return new JSONResponse(['error' => 'Rule processing failed: ' . $e->getMessage()], 500); + } + } + + /** + * Get a rule by its ID using RuleMapper + * + * @param string $id The unique identifier of the rule + * + * @return Rule|null The rule object if found, or null if not found + */ + private function getRuleById(string $id): ?Rule + { + try { + return $this->ruleMapper->find((int)$id); + } catch (Exception $e) { +// $this->logger->error('Error fetching rule: ' . $e->getMessage()); + return null; + } + } + + private function processFetchFileRule(Rule $rule, array $data): array + { + $config = $rule->getConfiguration(); + + $source = $this->sourceMapper->find($config['source']); + $dataDot = new Dot($data); + $endpoint = $dataDot[$config['filePath']]; + $endpoint = str_contains(haystack: $endpoint, needle: $source->getLocation()) === true ? substr(string: $endpoint, offset: strlen(string: $source->getLocation())) : $endpoint; + + $response = $this->callService->call( + source: $source, + endpoint: $endpoint, + method: $config['method'] ?? 'GET', + config: $config['sourceConfiguration'] ?? [] + )->getResponse(); + + $dataDot[$config['filePath']] = base64_encode($response['body']); + + return $dataDot->jsonSerialize(); + } + + private function processWriteFileRule(Rule $rule, array $data, string $objectId, int $registerId, int $schemaId): array + { + $config = $rule->getConfiguration(); + $dataDot = new Dot($data); + $content = base64_decode($dataDot[$config['filePath']]); + $fileName = $dataDot[$config['fileNamePath']]; + $openRegisters = $this->objectService->getOpenRegisters(); + $openRegisters->setRegister($registerId); + $openRegisters->setSchema($schemaId); + + $object = $openRegisters->find($objectId); + + try { + $file = $this->storageService->writeFile( + path: $object->getFolder(), + fileName: $fileName, + content: $content + ); + } catch (Exception $exception) { + } + + $dataDot[$config['filePath']] = $file->getPath(); + + return $dataDot->jsonSerialize(); + } + + + + /** + * Processes an error rule + * + * @param Rule $rule The rule object containing error details + * + * @return JSONResponse Response containing error details and HTTP status code + */ + private function processErrorRule(Rule $rule): JSONResponse + { + $config = $rule->getConfiguration(); + return new JSONResponse( + [ + 'error' => $config['error']['name'], + 'message' => $config['error']['message'] + ], + $config['error']['code'] + ); + } + + /** + * Processes a mapping rule + * + * @param Rule $rule The rule object containing mapping details + * @param array $data The data to be processed through the mapping rule + * + * @return array The processed data after applying the mapping rule + * @throws DoesNotExistException When the mapping configuration does not exist + * @throws MultipleObjectsReturnedException When multiple mapping objects are returned unexpectedly + * @throws LoaderError When there is an error loading the mapping + * @throws SyntaxError When there is a syntax error in the mapping configuration + */ + private function processMappingRule(Rule $rule, array $data): array + { + $config = $rule->getConfiguration(); + $mapping = $this->mappingService->getMapping($config['mapping']); + return $this->mappingService->executeMapping($mapping, $data); + } + + /** + * Processes a synchronization rule + * + * @param Rule $rule The rule object containing synchronization details + * @param array $data The data to be synchronized + * + * @return array The data after synchronization processing + */ + private function processSyncRule(Rule $rule, array $data): array + { + $config = $rule->getConfiguration(); + // Here you would implement the synchronization logic + // For now, just return the data unchanged + return $data; + } + + /** + * Checks if rule conditions are met + * + * @param Rule $rule The rule object containing conditions to be checked + * @param array $data The input data against which the conditions are evaluated + * + * @return bool True if conditions are met, false otherwise + * @throws Exception + */ + private function checkRuleConditions(Rule $rule, array $data): bool + { + $conditions = $rule->getConditions(); + if (empty($conditions) === true) { + return true; + } + + return JsonLogic::apply($conditions, $data) === true; + } } From f96b7a3b6fc7c4e4a5fc6805565149ac6b9e1f16 Mon Sep 17 00:00:00 2001 From: Thijn Date: Thu, 23 Jan 2025 15:19:39 +0100 Subject: [PATCH 07/31] added error reset before upload --- src/views/Import/ImportIndex.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/views/Import/ImportIndex.vue b/src/views/Import/ImportIndex.vue index 2718ba64..7d4d9deb 100644 --- a/src/views/Import/ImportIndex.vue +++ b/src/views/Import/ImportIndex.vue @@ -161,6 +161,8 @@ export default { importFiles() { this.loading = true + this.error = null + importExportStore.importFiles(files, reset) .then((response) => { this.success = true From 41953a3d29db06f44fbc87430342fcf7f8a52c51 Mon Sep 17 00:00:00 2001 From: Thijn Date: Thu, 23 Jan 2025 16:17:22 +0100 Subject: [PATCH 08/31] prettified json --- src/modals/TestSource/TestSource.vue | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/modals/TestSource/TestSource.vue b/src/modals/TestSource/TestSource.vue index a819157a..81527e46 100644 --- a/src/modals/TestSource/TestSource.vue +++ b/src/modals/TestSource/TestSource.vue @@ -59,7 +59,9 @@ import { sourceStore, navigationStore } from '../../store/store.js'

Size: {{ sourceStore.sourceTest.response.size }} (Bytes)

Remote IP: {{ sourceStore.sourceTest.response.remoteIp }}

Headers: {{ sourceStore.sourceTest.response.headers }}

-

Body: {{ sourceStore.sourceTest.response.body }}

+
+ Body: 
{{ prettifyJson(sourceStore.sourceTest.response.body) }}
+
@@ -153,6 +155,14 @@ export default { sourceStore.setSourceTest(false) } }, + prettifyJson(json) { + if (!json) return '' + try { + return JSON.stringify(JSON.parse(json), null, 2) + } catch (error) { + return json + } + }, }, } @@ -163,3 +173,9 @@ export default { gap: 5px; } + + From 4365b179be5d672c48f3344925d571179136b268 Mon Sep 17 00:00:00 2001 From: Thijn Date: Thu, 23 Jan 2025 16:28:13 +0100 Subject: [PATCH 09/31] removed endpoint tab --- src/views/Source/SourceDetails.vue | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/views/Source/SourceDetails.vue b/src/views/Source/SourceDetails.vue index e89e2f47..0bab144c 100644 --- a/src/views/Source/SourceDetails.vue +++ b/src/views/Source/SourceDetails.vue @@ -146,9 +146,6 @@ import { sourceStore, navigationStore, logStore } from '../../store/store.js' No authentications found - - No endpoints found - No synchronizations found From 8a4764d0e97d69ad439237ce33d2a78c83d143b6 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Thu, 23 Jan 2025 16:46:58 +0100 Subject: [PATCH 10/31] Document fetching and writing files in actions --- docs/synchronization.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 docs/synchronization.md diff --git a/docs/synchronization.md b/docs/synchronization.md new file mode 100644 index 00000000..00e6664c --- /dev/null +++ b/docs/synchronization.md @@ -0,0 +1,39 @@ +# Synchronization Actions + +## Concept + +Just as with Endpoints, synchronizations can trigger additional actions (at this moment called rules). +These can change the content of an object, but also trigger additional synchronizations. + +These actions can be created by creating a rule (an action) and adding it by id to the property +actions in the Synchronization. + +## Synchronizing files +In order to fetch a file from an external source and store it in the Nextcloud Filesystem in a way that OpenRegister can +connect it to an object, there are two predefined actions: + +- `fetch_file`: This action downloads the file and substitutes the base encoded content into the variable that contained the file url +- `write_file`: This action takes a file's content in base encoding and the filename (otherwise it will use a default filename), and writes it to the filesystem. + +### Fetch file + +This action should be run on timing `after` (when the object has been stored). +The action takes the following parameters in the `configuration` property: + +- `source` (required): The id of the source where the file can be downloaded +- `filePath` (required): The dot path of the location in the input object that contains the file url or file path. +- `method` (optional): The HTTP method that should be used to fetch the file. Defaults to GET +- `sourceConfiguration` (optional): Additional configuration for the source that only holds for fetching files. + +When properly configured this action will download the file from the given source and substitute the base64 encoded content in the returned object. +It is preferred to run this action in combination with `write_file` immediately after, so the file contents are properly stored in the Nextcloud file system instead of written to a database. + +### Write file + +This action should be run on timing `after`, and if combined with `fetch_file` it should be run in order after `fetch_file`. +The action takes the followin parameters in the `configuration` property: + +- `filePath` (required): The dot path of the location in the input object that contains the base64 encoded content of the file. +- `fileNamePath` (required): The dot path of the location in the input object that contains the filename + +This will write the file to the nextcloud filesystem in the folder that belongs to the written object, and substitutes the file content in the returned data with the path of the object in the Nextcloud File System. From d462a7216495ed07a837de79354aded7c1350fc4 Mon Sep 17 00:00:00 2001 From: Thijn Date: Fri, 24 Jan 2025 15:43:02 +0100 Subject: [PATCH 11/31] show list of synchronizations in synchronization tab on source --- src/views/Source/SourceDetails.vue | 51 ++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/src/views/Source/SourceDetails.vue b/src/views/Source/SourceDetails.vue index 0bab144c..75ed5164 100644 --- a/src/views/Source/SourceDetails.vue +++ b/src/views/Source/SourceDetails.vue @@ -1,5 +1,5 @@ + + + + + +