From dda85d3a332ebd07358c4446abe80d1e12ccdd2b Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Mon, 6 Jan 2025 17:50:16 +0100 Subject: [PATCH 01/13] Added upload & download controllers & services --- appinfo/routes.php | 4 + lib/Controller/DownloadController.php | 52 +++++++ lib/Controller/UploadController.php | 43 ++++++ lib/Db/ConsumerMapper.php | 10 +- lib/Db/EndpointMapper.php | 10 +- lib/Db/JobMapper.php | 10 +- lib/Db/MappingMapper.php | 6 +- lib/Db/SourceMapper.php | 10 +- lib/Db/SynchronizationMapper.php | 10 +- lib/Service/DownloadService.php | 110 +++++++++++++++ lib/Service/EndpointService.php | 1 + lib/Service/ObjectService.php | 38 ++++- lib/Service/UploadService.php | 192 ++++++++++++++++++++++++++ 13 files changed, 467 insertions(+), 29 deletions(-) create mode 100644 lib/Controller/DownloadController.php create mode 100644 lib/Controller/UploadController.php create mode 100644 lib/Service/DownloadService.php create mode 100644 lib/Service/UploadService.php diff --git a/appinfo/routes.php b/appinfo/routes.php index 26fb4da9..c54b6537 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -30,5 +30,9 @@ // ['name' => 'endpoints#handlePath', 'url' => '/api/endpoint/{path}', 'verb' => 'PUT', 'requirements' => ['path' => '.+']], // ['name' => 'endpoints#handlePath', 'url' => '/api/endpoint/{path}', 'verb' => 'POST', 'requirements' => ['path' => '.+']], // ['name' => 'endpoints#handlePath', 'url' => '/api/endpoint/{path}', 'verb' => 'DELETE', 'requirements' => ['path' => '.+']], + // Upload & Download + ['name' => 'upload#upload', 'url' => '/api/upload', 'verb' => 'POST'], +// ['name' => 'upload#uploadUpdate', 'url' => '/api/upload/{id}', 'verb' => 'PUT'], + ['name' => 'download#download', 'url' => '/api/download/{type}/{id}', 'verb' => 'GET'], ], ]; diff --git a/lib/Controller/DownloadController.php b/lib/Controller/DownloadController.php new file mode 100644 index 00000000..f1072f9c --- /dev/null +++ b/lib/Controller/DownloadController.php @@ -0,0 +1,52 @@ +request->getHeader('Accept'); + + if (empty($accept) === true) { + return new JSONResponse(data: ['error' => 'Request is missing header Accept'], statusCode: 400); + } + + return $this->downloadService->download(objectType: $type, id: $id, accept: $accept); + } +} diff --git a/lib/Controller/UploadController.php b/lib/Controller/UploadController.php new file mode 100644 index 00000000..efe7ff30 --- /dev/null +++ b/lib/Controller/UploadController.php @@ -0,0 +1,43 @@ +uploadService->upload($this->request->getParams()); + } +} diff --git a/lib/Db/ConsumerMapper.php b/lib/Db/ConsumerMapper.php index ff73ed8e..b14fdd01 100644 --- a/lib/Db/ConsumerMapper.php +++ b/lib/Db/ConsumerMapper.php @@ -116,10 +116,12 @@ public function updateFromArray(int $id, array $object): Consumer $obj = $this->find($id); $obj->hydrate($object); - // Set or update the version - // $version = explode('.', $obj->getVersion()); - // $version[2] = (int)$version[2] + 1; - // $obj->setVersion(implode('.', $version)); + if (isset($object['version']) === false) { + // Set or update the version + $version = explode('.', $obj->getVersion()); + $version[2] = (int)$version[2] + 1; + $obj->setVersion(implode('.', $version)); + } return $this->update($obj); } diff --git a/lib/Db/EndpointMapper.php b/lib/Db/EndpointMapper.php index a7727cbe..3fe37a40 100644 --- a/lib/Db/EndpointMapper.php +++ b/lib/Db/EndpointMapper.php @@ -85,10 +85,12 @@ public function updateFromArray(int $id, array $object): Endpoint $obj = $this->find($id); $obj->hydrate($object); - // Set or update the version - $version = explode('.', $obj->getVersion()); - $version[2] = (int)$version[2] + 1; - $obj->setVersion(implode('.', $version)); + if (isset($object['version']) === false) { + // Set or update the version + $version = explode('.', $obj->getVersion()); + $version[2] = (int)$version[2] + 1; + $obj->setVersion(implode('.', $version)); + } $obj->setEndpointRegex($this->createEndpointRegex($obj->getEndpoint())); $obj->setEndpointArray(explode('/', $obj->getEndpoint())); diff --git a/lib/Db/JobMapper.php b/lib/Db/JobMapper.php index c6461bed..605b2d8e 100644 --- a/lib/Db/JobMapper.php +++ b/lib/Db/JobMapper.php @@ -74,10 +74,12 @@ public function updateFromArray(int $id, array $object): Job $obj = $this->find($id); $obj->hydrate($object); - // Set or update the version - $version = explode('.', $obj->getVersion()); - $version[2] = (int)$version[2] + 1; - $obj->setVersion(implode('.', $version)); + if (isset($object['version']) === false) { + // Set or update the version + $version = explode('.', $obj->getVersion()); + $version[2] = (int)$version[2] + 1; + $obj->setVersion(implode('.', $version)); + } return $this->update($obj); } diff --git a/lib/Db/MappingMapper.php b/lib/Db/MappingMapper.php index 9b5871c2..e6840c6b 100644 --- a/lib/Db/MappingMapper.php +++ b/lib/Db/MappingMapper.php @@ -73,9 +73,9 @@ public function updateFromArray(int $id, array $object): Mapping { $obj = $this->find($id); $obj->hydrate($object); - - // Set or update the version - if ($obj->getVersion() !== null) { + + if ($obj->getVersion() !== null && isset($object['version']) === false) { + // Set or update the version $version = explode('.', $obj->getVersion()); if (isset($version[2]) === true) { $version[2] = (int) $version[2] + 1; diff --git a/lib/Db/SourceMapper.php b/lib/Db/SourceMapper.php index 46d22fe0..2ef1338f 100644 --- a/lib/Db/SourceMapper.php +++ b/lib/Db/SourceMapper.php @@ -74,10 +74,12 @@ public function updateFromArray(int $id, array $object): Source $obj = $this->find($id); $obj->hydrate($object); - // Set or update the version - $version = explode('.', $obj->getVersion()); - $version[2] = (int)$version[2] + 1; - $obj->setVersion(implode('.', $version)); + if (isset($object['version']) === false) { + // Set or update the version + $version = explode('.', $obj->getVersion()); + $version[2] = (int)$version[2] + 1; + $obj->setVersion(implode('.', $version)); + } return $this->update($obj); } diff --git a/lib/Db/SynchronizationMapper.php b/lib/Db/SynchronizationMapper.php index e09ac7c8..4e8076c1 100644 --- a/lib/Db/SynchronizationMapper.php +++ b/lib/Db/SynchronizationMapper.php @@ -74,10 +74,12 @@ public function updateFromArray(int $id, array $object): Synchronization $obj = $this->find($id); $obj->hydrate($object); - // Set or update the version - $version = explode('.', $obj->getVersion()); - $version[2] = (int)$version[2] + 1; - $obj->setVersion(implode('.', $version)); + if (isset($object['version']) === false) { + // Set or update the version + $version = explode('.', $obj->getVersion()); + $version[2] = (int)$version[2] + 1; + $obj->setVersion(implode('.', $version)); + } return $this->update($obj); } diff --git a/lib/Service/DownloadService.php b/lib/Service/DownloadService.php new file mode 100644 index 00000000..eb7eb576 --- /dev/null +++ b/lib/Service/DownloadService.php @@ -0,0 +1,110 @@ +objectService->getMapper(objectType: $objectType); + } catch (InvalidArgumentException|NotFoundExceptionInterface|ContainerExceptionInterface $e) { + return new JSONResponse(data: ['error' => "Could not find a mapper for this {type}: " . $objectType], statusCode: 400); + } + + try { + $object = $mapper->find($id); + } catch (Exception $exception) { + return new JSONResponse(data: ['error' => "Could not find an object with this {id}: ".$id], statusCode: 400); + } + + $objectArray = $object->jsonSerialize(); + $filename = $objectArray['name'].ucfirst($objectType).'-v'.$objectArray['version']; + + if (str_contains(haystack: $accept, needle: 'application/json') === true) { + $url = $this->urlGenerator->getAbsoluteURL($this->urlGenerator->linkToRoute('openconnector.'.ucfirst($objectType).'s.show', ['id' => $object->getId()])); + if (isset($objectArray['reference']) === true) { + $url = $objectArray['reference']; + } + + $objArray['@context'] = "http://schema.org"; + $objArray['@type'] = $objectType; + $objArray['@id'] = $url; + unset($objectArray['id'], $objectArray['uuid']); + $objArray = array_merge($objArray, $objectArray); + + // Convert the object data to JSON + $jsonData = json_encode($objArray, JSON_PRETTY_PRINT); + + $this->downloadJson($jsonData, $filename); + } + + return new JSONResponse(data: ['error' => "The Accept type $accept is not supported."], statusCode: 400); + } + + /** + * Generate a downloadable json file response. + * + * @param string $jsonData The json data to create a json file with. + * @param string $filename The filename, .json will be added after this filename in this function. + * + * @return void + */ + #[NoReturn] private function downloadJson(string $jsonData, string $filename): void + { + // Define the file name and path for the temporary JSON file + $fileName = $filename.'.json'; + $filePath = sys_get_temp_dir() . DIRECTORY_SEPARATOR . $fileName; + + // Create and write the JSON data to the file + file_put_contents($filePath, $jsonData); + + // Set headers to download the file + header('Content-Type: application/json'); + header('Content-Disposition: attachment; filename="' . $fileName . '"'); + header('Content-Length: ' . filesize($filePath)); + + // Output the file contents + readfile($filePath); + + // Clean up: delete the temporary file + unlink($filePath); + exit; // Ensure no further script execution + } +} diff --git a/lib/Service/EndpointService.php b/lib/Service/EndpointService.php index ea778512..bfef5ac0 100644 --- a/lib/Service/EndpointService.php +++ b/lib/Service/EndpointService.php @@ -228,6 +228,7 @@ private function handleSchemaRequest(Endpoint $endpoint, IRequest $request, stri $register = $target[0]; $schema = $target[1]; + // @todo: shouldn't this just get OpenRegister ObjectService here? In this case we should use: $this->getOpenRegisters() $mapper = $this->objectService->getMapper(schema: $schema, register: $register); $parameters = $request->getParams(); diff --git a/lib/Service/ObjectService.php b/lib/Service/ObjectService.php index c1d8e6a5..2b7c6619 100644 --- a/lib/Service/ObjectService.php +++ b/lib/Service/ObjectService.php @@ -7,6 +7,13 @@ use GuzzleHttp\Client; use GuzzleHttp\Exception\ClientException; use GuzzleHttp\Exception\GuzzleException; +use InvalidArgumentException; +use OCA\OpenConnector\Db\ConsumerMapper; +use OCA\OpenConnector\Db\EndpointMapper; +use OCA\OpenConnector\Db\JobMapper; +use OCA\OpenConnector\Db\MappingMapper; +use OCA\OpenConnector\Db\SourceMapper; +use OCA\OpenConnector\Db\SynchronizationMapper; use OCP\App\IAppManager; use OCP\AppFramework\Db\QBMapper; use Psr\Container\ContainerExceptionInterface; @@ -25,6 +32,12 @@ class ObjectService public function __construct( private readonly IAppManager $appManager, private readonly ContainerInterface $container, + private readonly ConsumerMapper $consumerMapper, + private readonly EndpointMapper $endpointMapper, + private readonly JobMapper $jobMapper, + private readonly MappingMapper $mappingMapper, + private readonly SourceMapper $sourceMapper, + private readonly SynchronizationMapper $synchronizationMapper, ) { @@ -253,23 +266,36 @@ public function getOpenRegisters(): ?\OCA\OpenRegister\Service\ObjectService } /** - * Get the mapper for the given objecttype (usually the proper instantiation of the objectService of OpenRegister. + * Gets the appropriate mapper based on the object type. + * (This can be a objectType for OpenRegister, by using an instantiation of the objectService of OpenRegister). * - * @param string|null $objecttype The objecttype as string + * @param string|null $objectType The objectType as string * @param int|null $schema The openregister schema * @param int|null $register The openregister register * - * @return QBMapper|\OCA\OpenRegister\Service\ObjectService|null The resulting mapper + * @return mixed The appropriate mapper. * @throws ContainerExceptionInterface * @throws NotFoundExceptionInterface + * @throws InvalidArgumentException If an unknown object type is provided. */ - public function getMapper(?string $objecttype = null, ?int $schema = null, ?int $register = null): QBMapper|\OCA\OpenRegister\Service\ObjectService|null + public function getMapper(?string $objectType = null, ?int $schema = null, ?int $register = null): mixed { - if ($register !== null && $schema !== null && $objecttype === null) { + if ($register !== null && $schema !== null && $objectType === null) { return $this->getOpenRegisters()->getMapper(register: $register, schema: $schema); } - return null; + $objectTypeLower = strtolower($objectType); + + // If the source is internal, return the appropriate mapper based on the object type + return match ($objectTypeLower) { + 'consumer' => $this->consumerMapper, + 'endpoint' => $this->endpointMapper, + 'job' => $this->jobMapper, + 'mapping' => $this->mappingMapper, + 'source' => $this->sourceMapper, + 'synchronization' => $this->synchronizationMapper, + default => throw new InvalidArgumentException("Unknown object type: $objectType"), + }; } diff --git a/lib/Service/UploadService.php b/lib/Service/UploadService.php new file mode 100644 index 00000000..d01368d4 --- /dev/null +++ b/lib/Service/UploadService.php @@ -0,0 +1,192 @@ +client = new Client([]); + } + + /** + * Handles an upload api-call to create a new object or update an existing one. + * + * @param array $data The data from the request body to use in creating/updating an object. + * + * @return JSONResponse The JSONResponse response. + * @throws GuzzleException + */ + public function upload(array $data): JSONResponse + { + // @todo: 1. support file upload instead of taking the json body or url + // @todo: 2. (To refine) We should create NextCloud files for uploads through OpenCatalogi (If url is posted we should just be able to download and copy the file) + + foreach ($data as $key => $value) { + if (str_starts_with($key, '_')) { + unset($data[$key]); + } + } + + // Define the allowed keys + $allowedKeys = ['file', 'url', 'json']; + + // Find which of the allowed keys are in the array + $matchingKeys = array_intersect_key($data, array_flip($allowedKeys)); + + // Check if there is exactly one matching key + if (count($matchingKeys) === 0) { + return new JSONResponse(data: ['error' => 'Missing one of these keys in your POST body: file, url or json.'], statusCode: 400); + } + + if (empty($data['file']) === false) { + // @todo use .json file content from POST as $phpArray + return $this->getJSONfromFile(); + } + + if (empty($data['url']) === false && isset($phpArray) === false) { + return $this->getJSONfromURL($data['url']); + } + + $phpArray = $data['json']; + + // @todo: ? +// if (is_string($phpArray) === true) { +// $phpArray = json_decode($phpArray, associative: true); +// } +// +// if ($phpArray === null || $phpArray === false) { +// return new JSONResponse(data: ['error' => 'Failed to decode JSON input'], statusCode: 400); +// } + + return $this->saveObject($phpArray); + } + + /** + * Creates or updates an object using the given array as input. + * + * @param array $phpArray The input php array. + * + * @return Entity|JSONResponse + */ + private function saveObject(array $phpArray): Entity|JSONResponse + { + try { + $mapper = $this->objectService->getMapper(objectType: $phpArray['@type']); + } catch (InvalidArgumentException|NotFoundExceptionInterface|ContainerExceptionInterface $e) { + return new JSONResponse(data: ['error' => "Could not find a mapper for this @type: ".$phpArray['@type']], statusCode: 400); + } + + // Check if object already exists + if (isset($phpArray['@id']) === true) { + // @todo: find by using the full url @id instead? ($reference?) + $explodedId = explode('/', $phpArray['@id']); + $id = end($explodedId); + try { + $mapper->find($id); + } catch (Exception $exception) { + // @todo: should we just create a new object in this case? + return new JSONResponse(data: ['error' => "Could not find an object with this @id: ".$phpArray['@id']], statusCode: 400); + } + + // @todo: +// $phpArray['reference'] = $phpArray['@id']; + } + + unset($phpArray['@context'], $phpArray['@type'], $phpArray['@id']); + + if (isset($id) === true) { + $object = $mapper->updateFromArray($id, $phpArray); + return new JSONResponse(data: ['message' => "Upload successful, updated", 'object' => $object->jsonSerialize()]); + } + + $object = $mapper->createFromArray($phpArray); + return new JSONResponse(data: ['message' => "Upload successful, created", 'object' => $object->jsonSerialize()]); + } + + /** + * Gets uploaded file form request and returns it as PHP array to use for creating/updating an object. + * + * @return array|JSONResponse The file content converted to a PHP array or JSONResponse in case of an error. + */ + private function getJSONfromFile(): array|JSONResponse + { + // @todo +// return $this->saveObject(phpArray: $phpArray); + + return new JSONResponse(data: ['error' => 'Not yet implemented'], statusCode: 501); + } + + /** + * Uses Guzzle to call the given URL and returns response as PHP array. + * + * @param string $url The URL to call. + * + * @return array|JSONResponse The response from the call converted to PHP array or JSONResponse in case of an error. + * @throws GuzzleException + */ + private function getJSONfromURL(string $url): array|JSONResponse + { + try { + $response = $this->client->request('GET', $url); + } catch (GuzzleHttp\Exception\BadResponseException $e) { + return new JSONResponse(data: ['error' => 'Failed to do a GET api-call on url: '.$url.' '.$e->getMessage()], statusCode: 400); + } + + $responseBody = $response->getBody()->getContents(); + + // Use Content-Type header to determine the format + $contentType = $response->getHeaderLine('Content-Type'); + switch ($contentType) { + case 'application/json': + $phpArray = json_decode(json: $responseBody, associative: true); + break; + case 'application/yaml': + $phpArray = Yaml::parse(input: $responseBody); + break; + default: + // If Content-Type is not specified or not recognized, try to parse as JSON first, then YAML + $phpArray = json_decode(json: $responseBody, associative: true); + if ($phpArray === null) { + $phpArray = Yaml::parse(input: $responseBody); + } + break; + } + + if ($phpArray === null || $phpArray === false) { + return new JSONResponse(data: ['error' => 'Failed to parse response body as JSON or YAML'], statusCode: 400); + } + + // @todo: + // Set reference, might be overwritten if $phpArray has @id set. +// $phpArray['reference'] = $url; + + return $this->saveObject(phpArray: $phpArray); + } +} From 43d4d991c67e3a36a1fc7d517d21e2cdaf91f887 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Mon, 6 Jan 2025 17:52:11 +0100 Subject: [PATCH 02/13] A todo note --- lib/Service/UploadService.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Service/UploadService.php b/lib/Service/UploadService.php index d01368d4..05a73f88 100644 --- a/lib/Service/UploadService.php +++ b/lib/Service/UploadService.php @@ -122,6 +122,7 @@ private function saveObject(array $phpArray): Entity|JSONResponse unset($phpArray['@context'], $phpArray['@type'], $phpArray['@id']); if (isset($id) === true) { + // @todo: maybe we should do kind of hash comparison here as well? $object = $mapper->updateFromArray($id, $phpArray); return new JSONResponse(data: ['message' => "Upload successful, updated", 'object' => $object->jsonSerialize()]); } From b652a5a62e7759170bb55f5fb8e36eed9aa2b23f Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Tue, 7 Jan 2025 17:36:15 +0100 Subject: [PATCH 03/13] Added upload for file(s) and check correctly by reference --- lib/Controller/DownloadController.php | 2 +- lib/Controller/UploadController.php | 29 ++- lib/Db/Consumer.php | 3 + lib/Db/ConsumerMapper.php | 27 ++- lib/Db/EndpointMapper.php | 13 + lib/Db/Job.php | 5 +- lib/Db/JobMapper.php | 13 + lib/Db/MappingMapper.php | 13 + lib/Db/SourceMapper.php | 13 + lib/Db/Synchronization.php | 5 +- lib/Db/SynchronizationMapper.php | 13 + lib/Migration/Version1Date20250107163601.php | 69 ++++++ lib/Service/DownloadService.php | 24 +- lib/Service/UploadService.php | 240 +++++++++++++------ 14 files changed, 380 insertions(+), 89 deletions(-) create mode 100644 lib/Migration/Version1Date20250107163601.php diff --git a/lib/Controller/DownloadController.php b/lib/Controller/DownloadController.php index f1072f9c..40d200ad 100644 --- a/lib/Controller/DownloadController.php +++ b/lib/Controller/DownloadController.php @@ -41,7 +41,7 @@ public function __construct( */ public function download(string $type, string $id): JSONResponse { - $accept = $this->request->getHeader('Accept'); + $accept = $this->request->getHeader(name: 'Accept'); if (empty($accept) === true) { return new JSONResponse(data: ['error' => 'Request is missing header Accept'], statusCode: 400); diff --git a/lib/Controller/UploadController.php b/lib/Controller/UploadController.php index efe7ff30..bfa394e1 100644 --- a/lib/Controller/UploadController.php +++ b/lib/Controller/UploadController.php @@ -2,6 +2,7 @@ namespace OCA\OpenConnector\Controller; +use GuzzleHttp\Exception\GuzzleException; use OCA\OpenConnector\Service\UploadService; use OCP\AppFramework\Controller; use OCP\AppFramework\Http\JSONResponse; @@ -35,9 +36,35 @@ public function __construct( * @NoCSRFRequired * * @return JSONResponse + * @throws GuzzleException */ public function upload(): JSONResponse { - return $this->uploadService->upload($this->request->getParams()); + $data = $this->request->getParams(); + $uploadedFiles = []; + + // Check if multiple files have been uploaded. + $files = $_FILES['files'] ?? null; + + if (empty($files) === false) { + // Loop through each file using the count of 'name' + for ($i = 0; $i < count($files['name']); $i++) { + $uploadedFiles[] = [ + 'name' => $files['name'][$i], + 'type' => $files['type'][$i], + 'tmp_name' => $files['tmp_name'][$i], + 'error' => $files['error'][$i], + 'size' => $files['size'][$i] + ]; + } + } + + // Get the uploaded file from the request if a single file hase been uploaded. + $uploadedFile = $this->request->getUploadedFile(key: 'file'); + if (empty($uploadedFile) === false) { + $uploadedFiles[] = $uploadedFile; + } + + return $this->uploadService->upload(data: $data, uploadedFiles: $uploadedFiles); } } diff --git a/lib/Db/Consumer.php b/lib/Db/Consumer.php index 2fcd8c7b..552a00e1 100644 --- a/lib/Db/Consumer.php +++ b/lib/Db/Consumer.php @@ -20,6 +20,7 @@ class Consumer extends Entity implements JsonSerializable protected ?string $uuid = null; protected ?string $name = null; // The name of the consumer protected ?string $description = null; // The description of the consumer + protected ?string $reference = null; // The reference of the consumer protected ?array $domains = []; // The domains the consumer is allowed to run from protected ?array $ips = []; // The ips the consumer is allowed to run from protected ?string $authorizationType = null; // The authorization type of the consumer, should be one of the following: 'none', 'basic', 'bearer', 'apiKey', 'oauth2', 'jwt'. Keep in mind that the consumer needs to be able to handle the authorization type. @@ -36,6 +37,7 @@ public function __construct() { $this->addType('uuid', 'string'); $this->addType('name', 'string'); $this->addType('description', 'string'); + $this->addType(fieldName:'reference', type: 'string'); $this->addType('domains', 'json'); $this->addType('ips', 'json'); $this->addType('authorizationType', 'string'); @@ -98,6 +100,7 @@ public function jsonSerialize(): array 'uuid' => $this->uuid, 'name' => $this->name, 'description' => $this->description, + 'reference' => $this->reference, 'domains' => $this->domains, 'ips' => $this->ips, 'authorizationType' => $this->authorizationType, diff --git a/lib/Db/ConsumerMapper.php b/lib/Db/ConsumerMapper.php index b14fdd01..9ece7fe0 100644 --- a/lib/Db/ConsumerMapper.php +++ b/lib/Db/ConsumerMapper.php @@ -30,7 +30,7 @@ public function __construct(IDBConnection $db) } /** - * Find a Consumer by its ID + * Find a Consumer by its ID. * * @param int $id The ID of the Consumer * @return Consumer The found Consumer entity @@ -49,7 +49,26 @@ public function find(int $id): Consumer } /** - * Find all Consumers with optional filtering and pagination + * Find a Consumer by its Reference. + * + * @param string $reference + * @return Endpoint + */ + public function findByRef(string $reference): Consumer + { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from('openconnector_consumers') + ->where( + $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) + ); + + return $this->findEntity(query: $qb); + } + + /** + * Find all Consumers with optional filtering and pagination. * * @param int|null $limit Maximum number of results to return * @param int|null $offset Number of results to skip @@ -88,7 +107,7 @@ public function findAll(?int $limit = null, ?int $offset = null, ?array $filters } /** - * Create a new Consumer from an array of data + * Create a new Consumer from an array of data. * * @param array $object An array of Consumer data * @return Consumer The newly created Consumer entity @@ -105,7 +124,7 @@ public function createFromArray(array $object): Consumer } /** - * Update an existing Consumer from an array of data + * Update an existing Consumer from an array of data. * * @param int $id The ID of the Consumer to update * @param array $object An array of updated Consumer data diff --git a/lib/Db/EndpointMapper.php b/lib/Db/EndpointMapper.php index 3fe37a40..dd92dc78 100644 --- a/lib/Db/EndpointMapper.php +++ b/lib/Db/EndpointMapper.php @@ -32,6 +32,19 @@ public function find(int $id): Endpoint return $this->findEntity(query: $qb); } + public function findByRef(string $reference): Endpoint + { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from('openconnector_endpoints') + ->where( + $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) + ); + + return $this->findEntity(query: $qb); + } + public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array { $qb = $this->db->getQueryBuilder(); diff --git a/lib/Db/Job.php b/lib/Db/Job.php index 463e75b6..5e2d7eca 100644 --- a/lib/Db/Job.php +++ b/lib/Db/Job.php @@ -11,7 +11,8 @@ class Job extends Entity implements JsonSerializable protected ?string $uuid = null; protected ?string $name = null; protected ?string $description = null; - protected ?string $version = '0.0.0'; // The version of the endpoint + protected ?string $reference = null; // The reference of the Job + protected ?string $version = '0.0.0'; // The version of the Job protected ?string $jobClass = 'OCA\OpenConnector\Action\PingAction'; protected ?array $arguments = null; protected ?int $interval = 3600; // seconds in an hour @@ -34,6 +35,7 @@ public function __construct() { $this->addType('uuid', 'string'); $this->addType('name', 'string'); $this->addType('description', 'string'); + $this->addType(fieldName:'reference', type: 'string'); $this->addType('version', 'string'); $this->addType('jobClass', 'string'); $this->addType('arguments', 'json'); @@ -91,6 +93,7 @@ public function jsonSerialize(): array 'uuid' => $this->uuid, 'name' => $this->name, 'description' => $this->description, + 'reference' => $this->reference, 'version' => $this->version, 'jobClass' => $this->jobClass, 'arguments' => $this->arguments, diff --git a/lib/Db/JobMapper.php b/lib/Db/JobMapper.php index 605b2d8e..20c3e97b 100644 --- a/lib/Db/JobMapper.php +++ b/lib/Db/JobMapper.php @@ -29,6 +29,19 @@ public function find(int $id): Job return $this->findEntity(query: $qb); } + public function findByRef(string $reference): Job + { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from('openconnector_endpoints') + ->where( + $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) + ); + + return $this->findEntity(query: $qb); + } + public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array { $qb = $this->db->getQueryBuilder(); diff --git a/lib/Db/MappingMapper.php b/lib/Db/MappingMapper.php index e6840c6b..fde57f5f 100644 --- a/lib/Db/MappingMapper.php +++ b/lib/Db/MappingMapper.php @@ -29,6 +29,19 @@ public function find(int $id): Mapping return $this->findEntity(query: $qb); } + public function findByRef(string $reference): Mapping + { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from('openconnector_mappings') + ->where( + $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) + ); + + return $this->findEntity(query: $qb); + } + public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array { $qb = $this->db->getQueryBuilder(); diff --git a/lib/Db/SourceMapper.php b/lib/Db/SourceMapper.php index 2ef1338f..1b454b07 100644 --- a/lib/Db/SourceMapper.php +++ b/lib/Db/SourceMapper.php @@ -29,6 +29,19 @@ public function find(int $id): Source return $this->findEntity(query: $qb); } + public function findByRef(string $reference): Source + { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from('openconnector_sources') + ->where( + $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) + ); + + return $this->findEntity(query: $qb); + } + public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array { $qb = $this->db->getQueryBuilder(); diff --git a/lib/Db/Synchronization.php b/lib/Db/Synchronization.php index 54ccbc37..ffbc4670 100644 --- a/lib/Db/Synchronization.php +++ b/lib/Db/Synchronization.php @@ -11,6 +11,7 @@ class Synchronization extends Entity implements JsonSerializable protected ?string $uuid = null; protected ?string $name = null; // The name of the synchronization protected ?string $description = null; // The description of the synchronization + protected ?string $reference = null; // The reference of the endpoint protected ?string $version = null; // The version of the synchronization // Source protected ?string $sourceId = null; // The id of the source object @@ -44,6 +45,7 @@ public function __construct() { $this->addType('uuid', 'string'); $this->addType('name', 'string'); $this->addType('description', 'string'); + $this->addType(fieldName:'reference', type: 'string'); $this->addType('version', 'string'); $this->addType('sourceId', 'string'); $this->addType('sourceType', 'string'); @@ -71,7 +73,7 @@ public function __construct() { /** * Checks through sourceConfig if the source of this sync uses pagination - * + * * @return bool true if its uses pagination */ public function usesPagination(): bool @@ -121,6 +123,7 @@ public function jsonSerialize(): array 'uuid' => $this->uuid, 'name' => $this->name, 'description' => $this->description, + 'reference' => $this->reference, 'version' => $this->version, 'sourceId' => $this->sourceId, 'sourceType' => $this->sourceType, diff --git a/lib/Db/SynchronizationMapper.php b/lib/Db/SynchronizationMapper.php index 4e8076c1..81bfeef8 100644 --- a/lib/Db/SynchronizationMapper.php +++ b/lib/Db/SynchronizationMapper.php @@ -29,6 +29,19 @@ public function find(int $id): Synchronization return $this->findEntity(query: $qb); } + public function findByRef(string $reference): Synchronization + { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from('openconnector_synchronizations') + ->where( + $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) + ); + + return $this->findEntity(query: $qb); + } + public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array { $qb = $this->db->getQueryBuilder(); diff --git a/lib/Migration/Version1Date20250107163601.php b/lib/Migration/Version1Date20250107163601.php new file mode 100644 index 00000000..40275808 --- /dev/null +++ b/lib/Migration/Version1Date20250107163601.php @@ -0,0 +1,69 @@ +hasTable(tableName: 'openconnector_consumers') === true) { + $table = $schema->getTable(tableName: 'openconnector_consumers'); + $table->addColumn('reference', Types::STRING, ['notnull' => false, 'length' => 255]); + } + if ($schema->hasTable(tableName: 'openconnector_jobs') === true) { + $table = $schema->getTable(tableName: 'openconnector_jobs'); + $table->addColumn('reference', Types::STRING, ['notnull' => false, 'length' => 255]); + } + if ($schema->hasTable(tableName: 'openconnector_synchronizations') === true) { + $table = $schema->getTable(tableName: 'openconnector_synchronizations'); + $table->addColumn('reference', Types::STRING, ['notnull' => false, 'length' => 255]); + } + + return $schema; + } + + /** + * @param IOutput $output + * @param Closure(): ISchemaWrapper $schemaClosure + * @param array $options + */ + public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void { + } +} diff --git a/lib/Service/DownloadService.php b/lib/Service/DownloadService.php index eb7eb576..b09b375e 100644 --- a/lib/Service/DownloadService.php +++ b/lib/Service/DownloadService.php @@ -42,6 +42,8 @@ public function __construct( */ public function download(string $objectType, string $id, string $accept): JSONResponse { + // @todo: remove backslash for / in urls like @id and reference + try { $mapper = $this->objectService->getMapper(objectType: $objectType); } catch (InvalidArgumentException|NotFoundExceptionInterface|ContainerExceptionInterface $e) { @@ -58,21 +60,27 @@ public function download(string $objectType, string $id, string $accept): JSONRe $filename = $objectArray['name'].ucfirst($objectType).'-v'.$objectArray['version']; if (str_contains(haystack: $accept, needle: 'application/json') === true) { - $url = $this->urlGenerator->getAbsoluteURL($this->urlGenerator->linkToRoute('openconnector.'.ucfirst($objectType).'s.show', ['id' => $object->getId()])); if (isset($objectArray['reference']) === true) { $url = $objectArray['reference']; + } else { + $url = $objectArray['reference'] = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( + routeName: 'openconnector.'.ucfirst($objectType).'s.show', + arguments: ['id' => $object->getId()]) + ); + unset($objectArray['id'], $objectArray['uuid'], $objectArray['created'], $objectArray['updated'], + $objectArray['dateCreated'], $objectArray['dateModified']); + $mapper->updateFromArray(id: $id, object: $objectArray); } $objArray['@context'] = "http://schema.org"; $objArray['@type'] = $objectType; $objArray['@id'] = $url; - unset($objectArray['id'], $objectArray['uuid']); $objArray = array_merge($objArray, $objectArray); // Convert the object data to JSON - $jsonData = json_encode($objArray, JSON_PRETTY_PRINT); + $jsonData = json_encode(value: $objArray, flags: JSON_PRETTY_PRINT); - $this->downloadJson($jsonData, $filename); + $this->downloadJson(jsonData: $jsonData, filename: $filename); } return new JSONResponse(data: ['error' => "The Accept type $accept is not supported."], statusCode: 400); @@ -93,18 +101,18 @@ public function download(string $objectType, string $id, string $accept): JSONRe $filePath = sys_get_temp_dir() . DIRECTORY_SEPARATOR . $fileName; // Create and write the JSON data to the file - file_put_contents($filePath, $jsonData); + file_put_contents(filename: $filePath, data: $jsonData); // Set headers to download the file header('Content-Type: application/json'); header('Content-Disposition: attachment; filename="' . $fileName . '"'); - header('Content-Length: ' . filesize($filePath)); + header('Content-Length: ' . filesize(filename: $filePath)); // Output the file contents - readfile($filePath); + readfile(filename: $filePath); // Clean up: delete the temporary file - unlink($filePath); + unlink(filename: $filePath); exit; // Ensure no further script execution } } diff --git a/lib/Service/UploadService.php b/lib/Service/UploadService.php index 05a73f88..f74bbdae 100644 --- a/lib/Service/UploadService.php +++ b/lib/Service/UploadService.php @@ -14,6 +14,7 @@ use OCA\OpenRegister\Service\GuzzleHttp; use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Http\JSONResponse; +use OCP\IURLGenerator; use Psr\Container\ContainerExceptionInterface; use Psr\Container\NotFoundExceptionInterface; use Symfony\Component\Yaml\Yaml; @@ -30,6 +31,7 @@ class UploadService { public function __construct( private Client $client, + private readonly IURLGenerator $urlGenerator, private readonly ObjectService $objectService ) { $this->client = new Client([]); @@ -37,55 +39,55 @@ public function __construct( /** * Handles an upload api-call to create a new object or update an existing one. + * In case of multiple uploaded files this will create/update multipel objects. + * @todo: (To refine) We should create NextCloud files for uploads through OpenCatalogi (If url is posted we should just be able to download and copy the file) * - * @param array $data The data from the request body to use in creating/updating an object. + * @param array $data The data from the request body to use in creating/updating a single object. + * @param array|null $uploadedFiles The uploaded files or null. * - * @return JSONResponse The JSONResponse response. + * @return JSONResponse The JSONResponse response with a message and the created or updated object(s) or an error message. * @throws GuzzleException */ - public function upload(array $data): JSONResponse + public function upload(array $data, ?array $uploadedFiles): JSONResponse { - // @todo: 1. support file upload instead of taking the json body or url - // @todo: 2. (To refine) We should create NextCloud files for uploads through OpenCatalogi (If url is posted we should just be able to download and copy the file) - - foreach ($data as $key => $value) { - if (str_starts_with($key, '_')) { - unset($data[$key]); - } - } + // @todo: remove backslash for / in urls like @id and reference // Define the allowed keys - $allowedKeys = ['file', 'url', 'json']; + $allowedKeys = ['url', 'json']; // Find which of the allowed keys are in the array $matchingKeys = array_intersect_key($data, array_flip($allowedKeys)); - // Check if there is exactly one matching key - if (count($matchingKeys) === 0) { - return new JSONResponse(data: ['error' => 'Missing one of these keys in your POST body: file, url or json.'], statusCode: 400); + // Check if there is no matching key / no input. + if (count($matchingKeys) === 0 && empty($uploadedFiles) === true) { + return new JSONResponse(data: ['error' => 'Missing one of these keys in your POST body: url or json. Or the key file or files[] in form-data.'], statusCode: 400); } - if (empty($data['file']) === false) { - // @todo use .json file content from POST as $phpArray - return $this->getJSONfromFile(); - } + // [if] Check if we need to create or update object(s) using uploaded file(s). + if (empty($uploadedFiles) === false) { + if (count($uploadedFiles) === 1) { + return $this->getJSONfromFile(uploadedFile: $uploadedFiles[array_key_first($uploadedFiles)]); + } - if (empty($data['url']) === false && isset($phpArray) === false) { - return $this->getJSONfromURL($data['url']); + $responses = []; + foreach ($uploadedFiles as $i => $uploadedFile) { + $response = $this->getJSONfromFile(uploadedFile: $uploadedFile); + $responses[] = [ + 'filename' => "($i) {$uploadedFile['name']}", + 'statusCode' => $response->getStatus(), + 'response' => $response->getData() + ]; + } + return new JSONResponse(data: ['message' => 'Files processed', 'details' => $responses], statusCode: 200); } - $phpArray = $data['json']; - - // @todo: ? -// if (is_string($phpArray) === true) { -// $phpArray = json_decode($phpArray, associative: true); -// } -// -// if ($phpArray === null || $phpArray === false) { -// return new JSONResponse(data: ['error' => 'Failed to decode JSON input'], statusCode: 400); -// } + // [elseif] Check if we need to create or update object using given url from the post body. + if (empty($data['url']) === false) { + return $this->getJSONfromURL(url: $data['url']); + } - return $this->saveObject($phpArray); + // [else] Create or update object using given json blob from the post body. + return $this->getJSONfromBody($data['json']); } /** @@ -93,9 +95,9 @@ public function upload(array $data): JSONResponse * * @param array $phpArray The input php array. * - * @return Entity|JSONResponse + * @return JSONResponse A JSON response with a message and the created or updated object or an error message. */ - private function saveObject(array $phpArray): Entity|JSONResponse + private function saveObject(array $phpArray): JSONResponse { try { $mapper = $this->objectService->getMapper(objectType: $phpArray['@type']); @@ -103,56 +105,102 @@ private function saveObject(array $phpArray): Entity|JSONResponse return new JSONResponse(data: ['error' => "Could not find a mapper for this @type: ".$phpArray['@type']], statusCode: 400); } - // Check if object already exists + // Check if object already exists. if (isset($phpArray['@id']) === true) { - // @todo: find by using the full url @id instead? ($reference?) - $explodedId = explode('/', $phpArray['@id']); - $id = end($explodedId); - try { - $mapper->find($id); - } catch (Exception $exception) { - // @todo: should we just create a new object in this case? - return new JSONResponse(data: ['error' => "Could not find an object with this @id: ".$phpArray['@id']], statusCode: 400); - } - - // @todo: -// $phpArray['reference'] = $phpArray['@id']; + $phpArray['reference'] = $phpArray['@id']; + $object = $this->checkIfExists(mapper: $mapper, phpArray: $phpArray); } - unset($phpArray['@context'], $phpArray['@type'], $phpArray['@id']); + unset($phpArray['@context'], $phpArray['@type'], $phpArray['@id'], $phpArray['id'], $phpArray['uuid'], + $phpArray['created'], $phpArray['updated'], $phpArray['dateCreated'], $phpArray['dateModified']); - if (isset($id) === true) { + if (isset($object) === true && isset($id) === true) { // @todo: maybe we should do kind of hash comparison here as well? - $object = $mapper->updateFromArray($id, $phpArray); - return new JSONResponse(data: ['message' => "Upload successful, updated", 'object' => $object->jsonSerialize()]); + $updatedObject = $mapper->updateFromArray(id: $id, object: $phpArray); + return new JSONResponse(data: ['message' => "Upload successful, updated", 'object' => $updatedObject->jsonSerialize()]); + } + + $newObject = $mapper->createFromArray(object: $phpArray); + if (isset($phpArray['reference']) === false) { + $phpArray['reference'] = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( + routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', + arguments: ['id' => $newObject->getId()]) + ); + $newObject = $mapper->updateFromArray(object: $phpArray); + } + return new JSONResponse(data: ['message' => "Upload successful, created", 'object' => $newObject->jsonSerialize()], statusCode: 201); + } + + /** + * Check if the object already exists by using the @id from the phpArray data. + * + * @param mixed $mapper The mapper of the correct object type to look for. + * @param array $phpArray The array data we want to use to create/update an object. + * + * @return Entity|null The already existing object or null. + */ + private function checkIfExists(mixed $mapper, array $phpArray): ?Entity + { + // Check reference + if (method_exists($mapper, 'findByRef')) { + try { + return $mapper->findByRef($phpArray['@id']); + } catch (Exception $exception) {} } - $object = $mapper->createFromArray($phpArray); - return new JSONResponse(data: ['message' => "Upload successful, created", 'object' => $object->jsonSerialize()]); + // Check if @id matches an object of that type in OpenConnector. 'failsafe' + $explodedId = explode(separator: '/', string: $phpArray['@id']); + $id = end(array: $explodedId); + $url = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( + routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', + arguments: ['id' => $id]) + ); + if ($phpArray['@id'] === $url) { + try { + return $mapper->find($id); + } catch (Exception $exception) {} + } + + return null; } /** - * Gets uploaded file form request and returns it as PHP array to use for creating/updating an object. + * Gets uploaded file content from a file in the api request as PHP array and use it for creating/updating an object. * - * @return array|JSONResponse The file content converted to a PHP array or JSONResponse in case of an error. + * @param array $uploadedFile The uploaded file. + * + * @return JSONResponse A JSON response with a message and the created or updated object or an error message. */ - private function getJSONfromFile(): array|JSONResponse + private function getJSONfromFile(array $uploadedFile): JSONResponse { - // @todo -// return $this->saveObject(phpArray: $phpArray); + // Check for upload errors + if ($uploadedFile['error'] !== UPLOAD_ERR_OK) { + return new JSONResponse(data: ['error' => 'File upload error: '.$uploadedFile['error']], statusCode: 400); + } + + $fileExtension = pathinfo(path: $uploadedFile['name'], flags: PATHINFO_EXTENSION); + $fileContent = file_get_contents(filename: $uploadedFile['tmp_name']); + + $phpArray = $this->decode(data: $fileContent, type: $fileExtension); + if ($phpArray === null) { + return new JSONResponse( + data: ['error' => 'Failed to decode file content as JSON or YAML', 'MIME-type' => $fileExtension], + statusCode: 400 + ); + } - return new JSONResponse(data: ['error' => 'Not yet implemented'], statusCode: 501); + return $this->saveObject(phpArray: $phpArray); } /** - * Uses Guzzle to call the given URL and returns response as PHP array. + * Uses Guzzle to call the given URL and use the response data as PHP array for creating/updating an object. * * @param string $url The URL to call. * - * @return array|JSONResponse The response from the call converted to PHP array or JSONResponse in case of an error. + * @return JSONResponse A JSON response with a message and the created or updated object or an error message. * @throws GuzzleException */ - private function getJSONfromURL(string $url): array|JSONResponse + private function getJSONfromURL(string $url): JSONResponse { try { $response = $this->client->request('GET', $url); @@ -164,30 +212,76 @@ private function getJSONfromURL(string $url): array|JSONResponse // Use Content-Type header to determine the format $contentType = $response->getHeaderLine('Content-Type'); - switch ($contentType) { + $phpArray = $this->decode(data: $responseBody, type: $contentType); + + if ($phpArray === null) { + return new JSONResponse( + data: ['error' => 'Failed to parse response body as JSON or YAML', 'Content-Type' => $contentType], + statusCode: 400 + ); + } + + // Set reference, might be overwritten if $phpArray has @id set. + $phpArray['reference'] = $url; + + return $this->saveObject(phpArray: $phpArray); + } + + /** + * Uses the given string or array as PHP array for creating/updating an object. + * + * @param array|string $phpArray An array or string containing a json blob of data. + * + * @return JSONResponse A JSON response with a message and the created or updated object or an error message. + */ + private function getJSONfromBody(array|string $phpArray): JSONResponse + { + if (is_string($phpArray) === true) { + $phpArray = json_decode($phpArray, associative: true); + } + + if ($phpArray === null || $phpArray === false) { + return new JSONResponse(data: ['error' => 'Failed to decode JSON input'], statusCode: 400); + } + + return $this->saveObject(phpArray: $phpArray); + } + + /** + * A function used to decode file content or the response of an url get call. + * Before the data can be used to create or update an object. + * + * @param string $data The file content or the response body content. + * @param string|null $type The file MIME type or the response Content-Type header. + * + * @return array|null The decoded data or null. + */ + private function decode(string $data, ?string $type): ?array + { + switch ($type) { case 'application/json': - $phpArray = json_decode(json: $responseBody, associative: true); + $phpArray = json_decode(json: $data, associative: true); break; case 'application/yaml': - $phpArray = Yaml::parse(input: $responseBody); + $phpArray = Yaml::parse(input: $data); break; default: // If Content-Type is not specified or not recognized, try to parse as JSON first, then YAML - $phpArray = json_decode(json: $responseBody, associative: true); + $phpArray = json_decode(json: $data, associative: true); if ($phpArray === null) { - $phpArray = Yaml::parse(input: $responseBody); + try { + $phpArray = Yaml::parse(input: $data); + } catch (Exception $exception) { + $phpArray = null; + } } break; } if ($phpArray === null || $phpArray === false) { - return new JSONResponse(data: ['error' => 'Failed to parse response body as JSON or YAML'], statusCode: 400); + return null; } - // @todo: - // Set reference, might be overwritten if $phpArray has @id set. -// $phpArray['reference'] = $url; - - return $this->saveObject(phpArray: $phpArray); + return $phpArray; } } From 6c9bf4e507a184d83dfc5128cf581777b188b406 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Thu, 9 Jan 2025 14:41:56 +0100 Subject: [PATCH 04/13] Rename upload/download to import/export --- appinfo/routes.php | 6 +++--- ...loadController.php => ExportController.php} | 18 +++++++++--------- ...loadController.php => ImportController.php} | 14 +++++++------- lib/Db/Consumer.php | 3 --- lib/Migration/Version1Date20250107163601.php | 4 ---- .../{DownloadService.php => ExportService.php} | 14 +++++++------- .../{UploadService.php => ImportService.php} | 18 +++++++++--------- lib/Service/ObjectService.php | 1 - 8 files changed, 35 insertions(+), 43 deletions(-) rename lib/Controller/{DownloadController.php => ExportController.php} (65%) rename lib/Controller/{UploadController.php => ImportController.php} (81%) rename lib/Service/{DownloadService.php => ExportService.php} (89%) rename lib/Service/{UploadService.php => ImportService.php} (94%) diff --git a/appinfo/routes.php b/appinfo/routes.php index c54b6537..5eea342d 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -31,8 +31,8 @@ // ['name' => 'endpoints#handlePath', 'url' => '/api/endpoint/{path}', 'verb' => 'POST', 'requirements' => ['path' => '.+']], // ['name' => 'endpoints#handlePath', 'url' => '/api/endpoint/{path}', 'verb' => 'DELETE', 'requirements' => ['path' => '.+']], // Upload & Download - ['name' => 'upload#upload', 'url' => '/api/upload', 'verb' => 'POST'], -// ['name' => 'upload#uploadUpdate', 'url' => '/api/upload/{id}', 'verb' => 'PUT'], - ['name' => 'download#download', 'url' => '/api/download/{type}/{id}', 'verb' => 'GET'], + ['name' => 'import#import', 'url' => '/api/import', 'verb' => 'POST'], +// ['name' => 'import#importUpdate', 'url' => '/api/import/{id}', 'verb' => 'PUT'], + ['name' => 'export#export', 'url' => '/api/export/{type}/{id}', 'verb' => 'GET'], ], ]; diff --git a/lib/Controller/DownloadController.php b/lib/Controller/ExportController.php similarity index 65% rename from lib/Controller/DownloadController.php rename to lib/Controller/ExportController.php index 40d200ad..10915b81 100644 --- a/lib/Controller/DownloadController.php +++ b/lib/Controller/ExportController.php @@ -2,27 +2,27 @@ namespace OCA\OpenConnector\Controller; -use OCA\OpenConnector\Service\DownloadService; +use OCA\OpenConnector\Service\ExportService; use OCP\AppFramework\Controller; use OCP\AppFramework\Http\JSONResponse; use OCP\IAppConfig; use OCP\IRequest; -class DownloadController extends Controller +class ExportController extends Controller { /** - * Constructor for the DownloadController + * Constructor for the ExportController * * @param string $appName The name of the app * @param IRequest $request The request object * @param IAppConfig $config The app configuration object - * @param DownloadService $downloadService The Download Service. + * @param ExportService $exportService The Export Service. */ public function __construct( $appName, IRequest $request, private IAppConfig $config, - private readonly DownloadService $downloadService + private readonly ExportService $exportService ) { parent::__construct($appName, $request); @@ -34,12 +34,12 @@ public function __construct( * @NoAdminRequired * @NoCSRFRequired * - * @param string $type The object type we want to download an object for. - * @param string $id The id used to find an existing object to download. + * @param string $type The object type we want to export an object for. + * @param string $id The id used to find an existing object to export. * * @return JSONResponse */ - public function download(string $type, string $id): JSONResponse + public function export(string $type, string $id): JSONResponse { $accept = $this->request->getHeader(name: 'Accept'); @@ -47,6 +47,6 @@ public function download(string $type, string $id): JSONResponse return new JSONResponse(data: ['error' => 'Request is missing header Accept'], statusCode: 400); } - return $this->downloadService->download(objectType: $type, id: $id, accept: $accept); + return $this->exportService->export(objectType: $type, id: $id, accept: $accept); } } diff --git a/lib/Controller/UploadController.php b/lib/Controller/ImportController.php similarity index 81% rename from lib/Controller/UploadController.php rename to lib/Controller/ImportController.php index bfa394e1..a17ef838 100644 --- a/lib/Controller/UploadController.php +++ b/lib/Controller/ImportController.php @@ -3,27 +3,27 @@ namespace OCA\OpenConnector\Controller; use GuzzleHttp\Exception\GuzzleException; -use OCA\OpenConnector\Service\UploadService; +use OCA\OpenConnector\Service\ImportService; use OCP\AppFramework\Controller; use OCP\AppFramework\Http\JSONResponse; use OCP\IAppConfig; use OCP\IRequest; -class UploadController extends Controller +class ImportController extends Controller { /** - * Constructor for the UploadController + * Constructor for the ImportController * * @param string $appName The name of the app * @param IRequest $request The request object * @param IAppConfig $config The app configuration object - * @param UploadService $uploadService The Upload Service. + * @param ImportService $importService The Import Service. */ public function __construct( $appName, IRequest $request, private IAppConfig $config, - private readonly UploadService $uploadService + private readonly ImportService $importService ) { parent::__construct($appName, $request); @@ -38,7 +38,7 @@ public function __construct( * @return JSONResponse * @throws GuzzleException */ - public function upload(): JSONResponse + public function import(): JSONResponse { $data = $this->request->getParams(); $uploadedFiles = []; @@ -65,6 +65,6 @@ public function upload(): JSONResponse $uploadedFiles[] = $uploadedFile; } - return $this->uploadService->upload(data: $data, uploadedFiles: $uploadedFiles); + return $this->importService->import(data: $data, uploadedFiles: $uploadedFiles); } } diff --git a/lib/Db/Consumer.php b/lib/Db/Consumer.php index 552a00e1..2fcd8c7b 100644 --- a/lib/Db/Consumer.php +++ b/lib/Db/Consumer.php @@ -20,7 +20,6 @@ class Consumer extends Entity implements JsonSerializable protected ?string $uuid = null; protected ?string $name = null; // The name of the consumer protected ?string $description = null; // The description of the consumer - protected ?string $reference = null; // The reference of the consumer protected ?array $domains = []; // The domains the consumer is allowed to run from protected ?array $ips = []; // The ips the consumer is allowed to run from protected ?string $authorizationType = null; // The authorization type of the consumer, should be one of the following: 'none', 'basic', 'bearer', 'apiKey', 'oauth2', 'jwt'. Keep in mind that the consumer needs to be able to handle the authorization type. @@ -37,7 +36,6 @@ public function __construct() { $this->addType('uuid', 'string'); $this->addType('name', 'string'); $this->addType('description', 'string'); - $this->addType(fieldName:'reference', type: 'string'); $this->addType('domains', 'json'); $this->addType('ips', 'json'); $this->addType('authorizationType', 'string'); @@ -100,7 +98,6 @@ public function jsonSerialize(): array 'uuid' => $this->uuid, 'name' => $this->name, 'description' => $this->description, - 'reference' => $this->reference, 'domains' => $this->domains, 'ips' => $this->ips, 'authorizationType' => $this->authorizationType, diff --git a/lib/Migration/Version1Date20250107163601.php b/lib/Migration/Version1Date20250107163601.php index 40275808..7fe3a3a8 100644 --- a/lib/Migration/Version1Date20250107163601.php +++ b/lib/Migration/Version1Date20250107163601.php @@ -43,10 +43,6 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt */ $schema = $schemaClosure(); - if ($schema->hasTable(tableName: 'openconnector_consumers') === true) { - $table = $schema->getTable(tableName: 'openconnector_consumers'); - $table->addColumn('reference', Types::STRING, ['notnull' => false, 'length' => 255]); - } if ($schema->hasTable(tableName: 'openconnector_jobs') === true) { $table = $schema->getTable(tableName: 'openconnector_jobs'); $table->addColumn('reference', Types::STRING, ['notnull' => false, 'length' => 255]); diff --git a/lib/Service/DownloadService.php b/lib/Service/ExportService.php similarity index 89% rename from lib/Service/DownloadService.php rename to lib/Service/ExportService.php index b09b375e..bcdabde6 100644 --- a/lib/Service/DownloadService.php +++ b/lib/Service/ExportService.php @@ -18,13 +18,13 @@ use Psr\Container\NotFoundExceptionInterface; /** - * Service for handling download requests for database entities. + * Service for handling export requests for database entities. * - * This service enables downloading database entities as files in various formats, + * This service enables exporting database entities as files in various formats, * determined by the `Accept` header of the request. It retrieves the appropriate * data from mappers and generates responses or downloadable files. */ -class DownloadService +class ExportService { public function __construct( private readonly IURLGenerator $urlGenerator, @@ -34,13 +34,13 @@ public function __construct( /** * Handles an upload api-call to create a new object or update an existing one. * - * @param string $objectType The type of object to download. - * @param string $id The id of the object to download. - * @param string $accept The Accept-header from the download request. + * @param string $objectType The type of object to export. + * @param string $id The id of the object to export. + * @param string $accept The Accept-header from the export request. * * @return JSONResponse The JSONResponse response. */ - public function download(string $objectType, string $id, string $accept): JSONResponse + public function export(string $objectType, string $id, string $accept): JSONResponse { // @todo: remove backslash for / in urls like @id and reference diff --git a/lib/Service/UploadService.php b/lib/Service/ImportService.php similarity index 94% rename from lib/Service/UploadService.php rename to lib/Service/ImportService.php index f74bbdae..eeb3b0f4 100644 --- a/lib/Service/UploadService.php +++ b/lib/Service/ImportService.php @@ -20,14 +20,14 @@ use Symfony\Component\Yaml\Yaml; /** - * Service for handling file and JSON uploads. + * Service for handling file and JSON imports. * - * This service processes uploaded JSON data, either directly via a POST body, + * This service processes imported JSON data, either directly via a POST body, * from a provided URL, or from an uploaded file. It supports multiple data * formats (e.g., JSON, YAML) and integrates with consumers, endpoints, jobs, * mappings, sources and synchronizations for database updates. */ -class UploadService +class ImportService { public function __construct( private Client $client, @@ -38,9 +38,9 @@ public function __construct( } /** - * Handles an upload api-call to create a new object or update an existing one. - * In case of multiple uploaded files this will create/update multipel objects. - * @todo: (To refine) We should create NextCloud files for uploads through OpenCatalogi (If url is posted we should just be able to download and copy the file) + * Handles an import api-call to create a new object or update an existing one. + * In case of multiple uploaded files this will create/update multiple objects. + * @todo: (To refine) We should create NextCloud files for imports through OpenCatalogi (If url is posted we should just be able to download and copy the file) * * @param array $data The data from the request body to use in creating/updating a single object. * @param array|null $uploadedFiles The uploaded files or null. @@ -48,7 +48,7 @@ public function __construct( * @return JSONResponse The JSONResponse response with a message and the created or updated object(s) or an error message. * @throws GuzzleException */ - public function upload(array $data, ?array $uploadedFiles): JSONResponse + public function import(array $data, ?array $uploadedFiles): JSONResponse { // @todo: remove backslash for / in urls like @id and reference @@ -117,7 +117,7 @@ private function saveObject(array $phpArray): JSONResponse if (isset($object) === true && isset($id) === true) { // @todo: maybe we should do kind of hash comparison here as well? $updatedObject = $mapper->updateFromArray(id: $id, object: $phpArray); - return new JSONResponse(data: ['message' => "Upload successful, updated", 'object' => $updatedObject->jsonSerialize()]); + return new JSONResponse(data: ['message' => "Import successful, updated", 'object' => $updatedObject->jsonSerialize()]); } $newObject = $mapper->createFromArray(object: $phpArray); @@ -128,7 +128,7 @@ private function saveObject(array $phpArray): JSONResponse ); $newObject = $mapper->updateFromArray(object: $phpArray); } - return new JSONResponse(data: ['message' => "Upload successful, created", 'object' => $newObject->jsonSerialize()], statusCode: 201); + return new JSONResponse(data: ['message' => "Import successful, created", 'object' => $newObject->jsonSerialize()], statusCode: 201); } /** diff --git a/lib/Service/ObjectService.php b/lib/Service/ObjectService.php index 2b7c6619..bab05e54 100644 --- a/lib/Service/ObjectService.php +++ b/lib/Service/ObjectService.php @@ -288,7 +288,6 @@ public function getMapper(?string $objectType = null, ?int $schema = null, ?int // If the source is internal, return the appropriate mapper based on the object type return match ($objectTypeLower) { - 'consumer' => $this->consumerMapper, 'endpoint' => $this->endpointMapper, 'job' => $this->jobMapper, 'mapping' => $this->mappingMapper, From 5270884f760e3f03950d9769cade9401f3802f54 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Thu, 9 Jan 2025 17:06:34 +0100 Subject: [PATCH 05/13] Remove findByRef function in Consumermapper --- lib/Db/ConsumerMapper.php | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lib/Db/ConsumerMapper.php b/lib/Db/ConsumerMapper.php index 9ece7fe0..87d08a1e 100644 --- a/lib/Db/ConsumerMapper.php +++ b/lib/Db/ConsumerMapper.php @@ -48,25 +48,6 @@ public function find(int $id): Consumer return $this->findEntity(query: $qb); } - /** - * Find a Consumer by its Reference. - * - * @param string $reference - * @return Endpoint - */ - public function findByRef(string $reference): Consumer - { - $qb = $this->db->getQueryBuilder(); - - $qb->select('*') - ->from('openconnector_consumers') - ->where( - $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) - ); - - return $this->findEntity(query: $qb); - } - /** * Find all Consumers with optional filtering and pagination. * From 8376ebe0e25933b2a3fedc95622581504f16fc04 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Thu, 9 Jan 2025 17:19:55 +0100 Subject: [PATCH 06/13] Fixes for updating objects when importing --- lib/Db/EndpointMapper.php | 4 ++-- lib/Db/JobMapper.php | 4 ++-- lib/Db/MappingMapper.php | 4 ++-- lib/Db/SourceMapper.php | 4 ++-- lib/Db/SynchronizationMapper.php | 4 ++-- lib/Service/ImportService.php | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/Db/EndpointMapper.php b/lib/Db/EndpointMapper.php index dd92dc78..07f5815d 100644 --- a/lib/Db/EndpointMapper.php +++ b/lib/Db/EndpointMapper.php @@ -32,7 +32,7 @@ public function find(int $id): Endpoint return $this->findEntity(query: $qb); } - public function findByRef(string $reference): Endpoint + public function findByRef(string $reference): array { $qb = $this->db->getQueryBuilder(); @@ -42,7 +42,7 @@ public function findByRef(string $reference): Endpoint $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) ); - return $this->findEntity(query: $qb); + return $this->findEntities(query: $qb); } public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array diff --git a/lib/Db/JobMapper.php b/lib/Db/JobMapper.php index 20c3e97b..ed9174f4 100644 --- a/lib/Db/JobMapper.php +++ b/lib/Db/JobMapper.php @@ -29,7 +29,7 @@ public function find(int $id): Job return $this->findEntity(query: $qb); } - public function findByRef(string $reference): Job + public function findByRef(string $reference): array { $qb = $this->db->getQueryBuilder(); @@ -39,7 +39,7 @@ public function findByRef(string $reference): Job $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) ); - return $this->findEntity(query: $qb); + return $this->findEntities(query: $qb); } public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array diff --git a/lib/Db/MappingMapper.php b/lib/Db/MappingMapper.php index fde57f5f..d16c69fb 100644 --- a/lib/Db/MappingMapper.php +++ b/lib/Db/MappingMapper.php @@ -29,7 +29,7 @@ public function find(int $id): Mapping return $this->findEntity(query: $qb); } - public function findByRef(string $reference): Mapping + public function findByRef(string $reference): array { $qb = $this->db->getQueryBuilder(); @@ -39,7 +39,7 @@ public function findByRef(string $reference): Mapping $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) ); - return $this->findEntity(query: $qb); + return $this->findEntities(query: $qb); } public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array diff --git a/lib/Db/SourceMapper.php b/lib/Db/SourceMapper.php index 1b454b07..d20538ce 100644 --- a/lib/Db/SourceMapper.php +++ b/lib/Db/SourceMapper.php @@ -29,7 +29,7 @@ public function find(int $id): Source return $this->findEntity(query: $qb); } - public function findByRef(string $reference): Source + public function findByRef(string $reference): array { $qb = $this->db->getQueryBuilder(); @@ -39,7 +39,7 @@ public function findByRef(string $reference): Source $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) ); - return $this->findEntity(query: $qb); + return $this->findEntities(query: $qb); } public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array diff --git a/lib/Db/SynchronizationMapper.php b/lib/Db/SynchronizationMapper.php index 81bfeef8..dadc7822 100644 --- a/lib/Db/SynchronizationMapper.php +++ b/lib/Db/SynchronizationMapper.php @@ -29,7 +29,7 @@ public function find(int $id): Synchronization return $this->findEntity(query: $qb); } - public function findByRef(string $reference): Synchronization + public function findByRef(string $reference): array { $qb = $this->db->getQueryBuilder(); @@ -39,7 +39,7 @@ public function findByRef(string $reference): Synchronization $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) ); - return $this->findEntity(query: $qb); + return $this->findEntities(query: $qb); } public function findAll(?int $limit = null, ?int $offset = null, ?array $filters = [], ?array $searchConditions = [], ?array $searchParams = []): array diff --git a/lib/Service/ImportService.php b/lib/Service/ImportService.php index eeb3b0f4..270ca107 100644 --- a/lib/Service/ImportService.php +++ b/lib/Service/ImportService.php @@ -114,9 +114,9 @@ private function saveObject(array $phpArray): JSONResponse unset($phpArray['@context'], $phpArray['@type'], $phpArray['@id'], $phpArray['id'], $phpArray['uuid'], $phpArray['created'], $phpArray['updated'], $phpArray['dateCreated'], $phpArray['dateModified']); - if (isset($object) === true && isset($id) === true) { + if (isset($object) === true) { // @todo: maybe we should do kind of hash comparison here as well? - $updatedObject = $mapper->updateFromArray(id: $id, object: $phpArray); + $updatedObject = $mapper->updateFromArray(id: $object->getId(), object: $phpArray); return new JSONResponse(data: ['message' => "Import successful, updated", 'object' => $updatedObject->jsonSerialize()]); } @@ -144,7 +144,7 @@ private function checkIfExists(mixed $mapper, array $phpArray): ?Entity // Check reference if (method_exists($mapper, 'findByRef')) { try { - return $mapper->findByRef($phpArray['@id']); + return $mapper->findByRef($phpArray['@id'])[0]; } catch (Exception $exception) {} } From 39936a75b6a8ce74f2c548140f6fbad42b8d7840 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Thu, 9 Jan 2025 23:29:12 +0100 Subject: [PATCH 07/13] Reference bug fix & @type import response & check type when importing --- lib/Service/ExportService.php | 4 +- lib/Service/ImportService.php | 85 +++++++++++++++++++++++------------ 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/lib/Service/ExportService.php b/lib/Service/ExportService.php index bcdabde6..ae4a2bf3 100644 --- a/lib/Service/ExportService.php +++ b/lib/Service/ExportService.php @@ -42,8 +42,6 @@ public function __construct( */ public function export(string $objectType, string $id, string $accept): JSONResponse { - // @todo: remove backslash for / in urls like @id and reference - try { $mapper = $this->objectService->getMapper(objectType: $objectType); } catch (InvalidArgumentException|NotFoundExceptionInterface|ContainerExceptionInterface $e) { @@ -60,7 +58,7 @@ public function export(string $objectType, string $id, string $accept): JSONResp $filename = $objectArray['name'].ucfirst($objectType).'-v'.$objectArray['version']; if (str_contains(haystack: $accept, needle: 'application/json') === true) { - if (isset($objectArray['reference']) === true) { + if (empty($objectArray['reference']) === false) { $url = $objectArray['reference']; } else { $url = $objectArray['reference'] = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( diff --git a/lib/Service/ImportService.php b/lib/Service/ImportService.php index 270ca107..0ce2ce17 100644 --- a/lib/Service/ImportService.php +++ b/lib/Service/ImportService.php @@ -50,8 +50,6 @@ public function __construct( */ public function import(array $data, ?array $uploadedFiles): JSONResponse { - // @todo: remove backslash for / in urls like @id and reference - // Define the allowed keys $allowedKeys = ['url', 'json']; @@ -63,15 +61,21 @@ public function import(array $data, ?array $uploadedFiles): JSONResponse return new JSONResponse(data: ['error' => 'Missing one of these keys in your POST body: url or json. Or the key file or files[] in form-data.'], statusCode: 400); } + // If type=x has been added to post body. + $allowedType = null; + if (empty($data['type']) === false) { + $allowedType = $data['type']; + } + // [if] Check if we need to create or update object(s) using uploaded file(s). if (empty($uploadedFiles) === false) { if (count($uploadedFiles) === 1) { - return $this->getJSONfromFile(uploadedFile: $uploadedFiles[array_key_first($uploadedFiles)]); + return $this->getJSONfromFile(uploadedFile: $uploadedFiles[array_key_first($uploadedFiles)], type: $allowedType); } $responses = []; foreach ($uploadedFiles as $i => $uploadedFile) { - $response = $this->getJSONfromFile(uploadedFile: $uploadedFile); + $response = $this->getJSONfromFile(uploadedFile: $uploadedFile, type: $allowedType); $responses[] = [ 'filename' => "($i) {$uploadedFile['name']}", 'statusCode' => $response->getStatus(), @@ -83,22 +87,30 @@ public function import(array $data, ?array $uploadedFiles): JSONResponse // [elseif] Check if we need to create or update object using given url from the post body. if (empty($data['url']) === false) { - return $this->getJSONfromURL(url: $data['url']); + return $this->getJSONfromURL(url: $data['url'], type: $allowedType); } // [else] Create or update object using given json blob from the post body. - return $this->getJSONfromBody($data['json']); + return $this->getJSONfromBody($data['json'], type: $allowedType); } /** * Creates or updates an object using the given array as input. * * @param array $phpArray The input php array. + * @param string|null $type If the object should be a specific type of object. * * @return JSONResponse A JSON response with a message and the created or updated object or an error message. */ - private function saveObject(array $phpArray): JSONResponse + private function saveObject(array $phpArray, ?string $type): JSONResponse { + if (empty($type) === false && $phpArray['@type'] !== strtolower($type)) { + return new JSONResponse( + data: ['error' => "The object you are trying to import is not a $type object", '@type' => $phpArray['@type']], + statusCode: 400 + ); + } + try { $mapper = $this->objectService->getMapper(objectType: $phpArray['@type']); } catch (InvalidArgumentException|NotFoundExceptionInterface|ContainerExceptionInterface $e) { @@ -111,24 +123,36 @@ private function saveObject(array $phpArray): JSONResponse $object = $this->checkIfExists(mapper: $mapper, phpArray: $phpArray); } + $type = $phpArray['@type']; unset($phpArray['@context'], $phpArray['@type'], $phpArray['@id'], $phpArray['id'], $phpArray['uuid'], $phpArray['created'], $phpArray['updated'], $phpArray['dateCreated'], $phpArray['dateModified']); if (isset($object) === true) { // @todo: maybe we should do kind of hash comparison here as well? - $updatedObject = $mapper->updateFromArray(id: $object->getId(), object: $phpArray); - return new JSONResponse(data: ['message' => "Import successful, updated", 'object' => $updatedObject->jsonSerialize()]); + $updatedObject = $mapper->updateFromArray(id: $object->getId(), object: $phpArray)->jsonSerialize(); + return new JSONResponse( + data: [ + 'message' => "Import successful, updated", + 'object' => array_merge(['@type' => $type], $updatedObject) + ] + ); } $newObject = $mapper->createFromArray(object: $phpArray); - if (isset($phpArray['reference']) === false) { + if (empty($phpArray['reference']) === true) { $phpArray['reference'] = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', arguments: ['id' => $newObject->getId()]) ); - $newObject = $mapper->updateFromArray(object: $phpArray); + $newObject = $mapper->updateFromArray(id: $newObject->getId(), object: $phpArray)->jsonSerialize(); } - return new JSONResponse(data: ['message' => "Import successful, created", 'object' => $newObject->jsonSerialize()], statusCode: 201); + return new JSONResponse( + data: [ + 'message' => "Import successful, created", + 'object' => array_merge(['@type' => $type], $newObject) + ], + statusCode: 201 + ); } /** @@ -148,18 +172,18 @@ private function checkIfExists(mixed $mapper, array $phpArray): ?Entity } catch (Exception $exception) {} } - // Check if @id matches an object of that type in OpenConnector. 'failsafe' - $explodedId = explode(separator: '/', string: $phpArray['@id']); - $id = end(array: $explodedId); - $url = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( - routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', - arguments: ['id' => $id]) - ); - if ($phpArray['@id'] === $url) { - try { + // Check if @id matches an object of that type in OpenConnector. 'failsafe' / backup + try { + $explodedId = explode(separator: '/', string: $phpArray['@id']); + $id = end(array: $explodedId); + $url = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( + routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', + arguments: ['id' => $id]) + ); + if ($phpArray['@id'] === $url) { return $mapper->find($id); - } catch (Exception $exception) {} - } + } + } catch (Exception $exception) {} return null; } @@ -168,10 +192,11 @@ private function checkIfExists(mixed $mapper, array $phpArray): ?Entity * Gets uploaded file content from a file in the api request as PHP array and use it for creating/updating an object. * * @param array $uploadedFile The uploaded file. + * @param string|null $type If the uploaded file should be a specific type of object. * * @return JSONResponse A JSON response with a message and the created or updated object or an error message. */ - private function getJSONfromFile(array $uploadedFile): JSONResponse + private function getJSONfromFile(array $uploadedFile, ?string $type = null): JSONResponse { // Check for upload errors if ($uploadedFile['error'] !== UPLOAD_ERR_OK) { @@ -189,18 +214,19 @@ private function getJSONfromFile(array $uploadedFile): JSONResponse ); } - return $this->saveObject(phpArray: $phpArray); + return $this->saveObject(phpArray: $phpArray, type: $type); } /** * Uses Guzzle to call the given URL and use the response data as PHP array for creating/updating an object. * * @param string $url The URL to call. + * @param string|null $type If the object should be a specific type of object. * * @return JSONResponse A JSON response with a message and the created or updated object or an error message. * @throws GuzzleException */ - private function getJSONfromURL(string $url): JSONResponse + private function getJSONfromURL(string $url, ?string $type = null): JSONResponse { try { $response = $this->client->request('GET', $url); @@ -224,17 +250,18 @@ private function getJSONfromURL(string $url): JSONResponse // Set reference, might be overwritten if $phpArray has @id set. $phpArray['reference'] = $url; - return $this->saveObject(phpArray: $phpArray); + return $this->saveObject(phpArray: $phpArray, type: $type); } /** * Uses the given string or array as PHP array for creating/updating an object. * * @param array|string $phpArray An array or string containing a json blob of data. + * @param string|null $type If the object should be a specific type of object. * * @return JSONResponse A JSON response with a message and the created or updated object or an error message. */ - private function getJSONfromBody(array|string $phpArray): JSONResponse + private function getJSONfromBody(array|string $phpArray, ?string $type = null): JSONResponse { if (is_string($phpArray) === true) { $phpArray = json_decode($phpArray, associative: true); @@ -244,7 +271,7 @@ private function getJSONfromBody(array|string $phpArray): JSONResponse return new JSONResponse(data: ['error' => 'Failed to decode JSON input'], statusCode: 400); } - return $this->saveObject(phpArray: $phpArray); + return $this->saveObject(phpArray: $phpArray, type: $type); } /** From 866c53e80fe056ea2489bb2ba3601c46be1546e5 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Fri, 10 Jan 2025 10:39:52 +0100 Subject: [PATCH 08/13] Small fix, Consumer doesn't have a field $version --- lib/Db/ConsumerMapper.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/Db/ConsumerMapper.php b/lib/Db/ConsumerMapper.php index 87d08a1e..61e3dbd7 100644 --- a/lib/Db/ConsumerMapper.php +++ b/lib/Db/ConsumerMapper.php @@ -116,12 +116,13 @@ public function updateFromArray(int $id, array $object): Consumer $obj = $this->find($id); $obj->hydrate($object); - if (isset($object['version']) === false) { - // Set or update the version - $version = explode('.', $obj->getVersion()); - $version[2] = (int)$version[2] + 1; - $obj->setVersion(implode('.', $version)); - } + // @todo: does Consumer need a version? $version field does currently not exist. +// if (isset($object['version']) === false) { +// // Set or update the version +// $version = explode('.', $obj->getVersion()); +// $version[2] = (int)$version[2] + 1; +// $obj->setVersion(implode('.', $version)); +// } return $this->update($obj); } From 66630f47f042f8b0b65c30acf74f8ab6850c3721 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Fri, 10 Jan 2025 12:51:12 +0100 Subject: [PATCH 09/13] Small fix and code cleanup --- lib/Service/ExportService.php | 4 +- lib/Service/ImportService.php | 300 +++++++++++++++++++++------------- 2 files changed, 189 insertions(+), 115 deletions(-) diff --git a/lib/Service/ExportService.php b/lib/Service/ExportService.php index ae4a2bf3..543db654 100644 --- a/lib/Service/ExportService.php +++ b/lib/Service/ExportService.php @@ -55,7 +55,7 @@ public function export(string $objectType, string $id, string $accept): JSONResp } $objectArray = $object->jsonSerialize(); - $filename = $objectArray['name'].ucfirst($objectType).'-v'.$objectArray['version']; + $filename = ucfirst($objectType).'-'.$objectArray['name'].'-v'.$objectArray['version']; if (str_contains(haystack: $accept, needle: 'application/json') === true) { if (empty($objectArray['reference']) === false) { @@ -70,7 +70,7 @@ public function export(string $objectType, string $id, string $accept): JSONResp $mapper->updateFromArray(id: $id, object: $objectArray); } - $objArray['@context'] = "http://schema.org"; + $objArray['@context'] = ["schema" => "http://schema.org", "register" => "501"]; $objArray['@type'] = $objectType; $objArray['@id'] = $url; $objArray = array_merge($objArray, $objectArray); diff --git a/lib/Service/ImportService.php b/lib/Service/ImportService.php index 0ce2ce17..d5873e43 100644 --- a/lib/Service/ImportService.php +++ b/lib/Service/ImportService.php @@ -95,97 +95,41 @@ public function import(array $data, ?array $uploadedFiles): JSONResponse } /** - * Creates or updates an object using the given array as input. + * A function used to decode file content or the response of an url get call. + * Before the data can be used to create or update an object. * - * @param array $phpArray The input php array. - * @param string|null $type If the object should be a specific type of object. + * @param string $data The file content or the response body content. + * @param string|null $type The file MIME type or the response Content-Type header. * - * @return JSONResponse A JSON response with a message and the created or updated object or an error message. + * @return array|null The decoded data or null. */ - private function saveObject(array $phpArray, ?string $type): JSONResponse + private function decode(string $data, ?string $type): ?array { - if (empty($type) === false && $phpArray['@type'] !== strtolower($type)) { - return new JSONResponse( - data: ['error' => "The object you are trying to import is not a $type object", '@type' => $phpArray['@type']], - statusCode: 400 - ); - } - - try { - $mapper = $this->objectService->getMapper(objectType: $phpArray['@type']); - } catch (InvalidArgumentException|NotFoundExceptionInterface|ContainerExceptionInterface $e) { - return new JSONResponse(data: ['error' => "Could not find a mapper for this @type: ".$phpArray['@type']], statusCode: 400); - } - - // Check if object already exists. - if (isset($phpArray['@id']) === true) { - $phpArray['reference'] = $phpArray['@id']; - $object = $this->checkIfExists(mapper: $mapper, phpArray: $phpArray); - } - - $type = $phpArray['@type']; - unset($phpArray['@context'], $phpArray['@type'], $phpArray['@id'], $phpArray['id'], $phpArray['uuid'], - $phpArray['created'], $phpArray['updated'], $phpArray['dateCreated'], $phpArray['dateModified']); - - if (isset($object) === true) { - // @todo: maybe we should do kind of hash comparison here as well? - $updatedObject = $mapper->updateFromArray(id: $object->getId(), object: $phpArray)->jsonSerialize(); - return new JSONResponse( - data: [ - 'message' => "Import successful, updated", - 'object' => array_merge(['@type' => $type], $updatedObject) - ] - ); + switch ($type) { + case 'application/json': + $phpArray = json_decode(json: $data, associative: true); + break; + case 'application/yaml': + $phpArray = Yaml::parse(input: $data); + break; + default: + // If Content-Type is not specified or not recognized, try to parse as JSON first, then YAML + $phpArray = json_decode(json: $data, associative: true); + if ($phpArray === null) { + try { + $phpArray = Yaml::parse(input: $data); + } catch (Exception $exception) { + $phpArray = null; + } + } + break; } - $newObject = $mapper->createFromArray(object: $phpArray); - if (empty($phpArray['reference']) === true) { - $phpArray['reference'] = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( - routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', - arguments: ['id' => $newObject->getId()]) - ); - $newObject = $mapper->updateFromArray(id: $newObject->getId(), object: $phpArray)->jsonSerialize(); - } - return new JSONResponse( - data: [ - 'message' => "Import successful, created", - 'object' => array_merge(['@type' => $type], $newObject) - ], - statusCode: 201 - ); - } - - /** - * Check if the object already exists by using the @id from the phpArray data. - * - * @param mixed $mapper The mapper of the correct object type to look for. - * @param array $phpArray The array data we want to use to create/update an object. - * - * @return Entity|null The already existing object or null. - */ - private function checkIfExists(mixed $mapper, array $phpArray): ?Entity - { - // Check reference - if (method_exists($mapper, 'findByRef')) { - try { - return $mapper->findByRef($phpArray['@id'])[0]; - } catch (Exception $exception) {} + if ($phpArray === null || $phpArray === false) { + return null; } - // Check if @id matches an object of that type in OpenConnector. 'failsafe' / backup - try { - $explodedId = explode(separator: '/', string: $phpArray['@id']); - $id = end(array: $explodedId); - $url = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( - routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', - arguments: ['id' => $id]) - ); - if ($phpArray['@id'] === $url) { - return $mapper->find($id); - } - } catch (Exception $exception) {} - - return null; + return $phpArray; } /** @@ -214,7 +158,7 @@ private function getJSONfromFile(array $uploadedFile, ?string $type = null): JSO ); } - return $this->saveObject(phpArray: $phpArray, type: $type); + return $this->saveObject(objectArray: $phpArray, type: $type); } /** @@ -250,7 +194,7 @@ private function getJSONfromURL(string $url, ?string $type = null): JSONResponse // Set reference, might be overwritten if $phpArray has @id set. $phpArray['reference'] = $url; - return $this->saveObject(phpArray: $phpArray, type: $type); + return $this->saveObject(objectArray: $phpArray, type: $type); } /** @@ -271,44 +215,174 @@ private function getJSONfromBody(array|string $phpArray, ?string $type = null): return new JSONResponse(data: ['error' => 'Failed to decode JSON input'], statusCode: 400); } - return $this->saveObject(phpArray: $phpArray, type: $type); + return $this->saveObject(objectArray: $phpArray, type: $type); } /** - * A function used to decode file content or the response of an url get call. - * Before the data can be used to create or update an object. + * Creates or updates an object using the given array as input. * - * @param string $data The file content or the response body content. - * @param string|null $type The file MIME type or the response Content-Type header. + * @param array $objectArray The input php array we use to create or update an object. + * @param string|null $type If the object should be a specific type of object. * - * @return array|null The decoded data or null. + * @return JSONResponse A JSON response with a message and the created or updated object or an error message. */ - private function decode(string $data, ?string $type): ?array + private function saveObject(array $objectArray, ?string $type): JSONResponse { - switch ($type) { - case 'application/json': - $phpArray = json_decode(json: $data, associative: true); - break; - case 'application/yaml': - $phpArray = Yaml::parse(input: $data); - break; - default: - // If Content-Type is not specified or not recognized, try to parse as JSON first, then YAML - $phpArray = json_decode(json: $data, associative: true); - if ($phpArray === null) { - try { - $phpArray = Yaml::parse(input: $data); - } catch (Exception $exception) { - $phpArray = null; - } - } - break; + if (empty($type) === false && $objectArray['@type'] !== strtolower($type)) { + return new JSONResponse( + data: ['error' => "The object you are trying to import is not a $type object", '@type' => $objectArray['@type']], + statusCode: 400 + ); } - if ($phpArray === null || $phpArray === false) { - return null; + try { + $mapper = $this->objectService->getMapper(objectType: $objectArray['@type']); + } catch (InvalidArgumentException|NotFoundExceptionInterface|ContainerExceptionInterface $e) { + return new JSONResponse(data: ['error' => "Could not find a mapper for this @type: ".$objectArray['@type']], statusCode: 400); } - return $phpArray; + // Check if object already exists. + if (isset($objectArray['@id']) === true) { + $objectArray['reference'] = $objectArray['@id']; + $object = $this->checkIfExists(mapper: $mapper, phpArray: $objectArray); + } + + if (isset($object) === true && $object instanceof Entity) { + return $this->updateObject(mapper: $mapper, objectArray: $objectArray, object: $object); + } + + return $this->createObject(mapper: $mapper, objectArray: $objectArray); + } + + /** + * Check if the object already exists by using the @id from the phpArray data. + * + * @param mixed $mapper The mapper of the correct object type to look for. + * @param array $phpArray The array data we want to use to create/update an object. + * + * @return Entity|null The already existing object or null. + */ + private function checkIfExists(mixed $mapper, array $phpArray): ?Entity + { + // Check reference + if (method_exists($mapper, 'findByRef')) { + try { + return $mapper->findByRef($phpArray['@id'])[0]; + } catch (Exception $exception) {} + } + + // Check if @id matches an object of that type in OpenConnector. 'failsafe' / backup + try { + $explodedId = explode(separator: '/', string: $phpArray['@id']); + $id = end(array: $explodedId); + $url = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( + routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', + arguments: ['id' => $id]) + ); + if ($phpArray['@id'] === $url) { + return $mapper->find($id); + } + } catch (Exception $exception) {} + + return null; + } + + /** + * Prepare the input array before creating or updating an object. + * Will prepare and return some default values for the object to save. + * Will also unset some fields from the array used to create or update the object. + * + * @param array $objectArray + * + * @return array + */ + private function prepareObject(array &$objectArray): array + { + $jsonLdDefault = [ + '@context' => ["schema" => "http://schema.org", "register" => "501"], + '@type' => $objectArray['@type'], + '@id' => $objectArray['reference'] ?? null + ]; + + unset( + $objectArray['@context'], + $objectArray['@type'], + $objectArray['@id'], + $objectArray['id'], + $objectArray['uuid'], + $objectArray['created'], + $objectArray['updated'], + $objectArray['dateCreated'], + $objectArray['dateModified'] + ); + + return $jsonLdDefault; + } + + /** + * Updates an object using the given array as input. + * + * @param mixed $mapper The mapper of the object type we want to update an object for. + * @param array $objectArray + * @param Entity $object + * + * @return JSONResponse + */ + private function updateObject(mixed $mapper, array $objectArray, Entity $object): JSONResponse + { + $jsonLdDefault = $this->prepareObject($objectArray); + + if ($object->getVersion() !== null && empty($objectArray['version']) === false + && version_compare(version1: $object->getVersion(), version2: $objectArray['version'], operator: '>=') + ) { + return new JSONResponse( + data: [ + 'message' => "Import ok, but nothing changed (currentVersion >= importVersion)", + 'currentVersion' => $object->getVersion(), + 'importVersion' => $objectArray['version'], + 'object' => array_merge($jsonLdDefault, $object->jsonSerialize()) + ] + ); + } + // @todo: maybe we should do kind of hash comparison here as well? + $updatedObject = $mapper->updateFromArray(id: $object->getId(), object: $objectArray); + return new JSONResponse( + data: [ + 'message' => "Import successful, updated", + 'object' => array_merge($jsonLdDefault, $updatedObject->jsonSerialize()) + ] + ); + } + + /** + * Creates an object using the given array as input. + * + * @param mixed $mapper The mapper of the object type we want to create an object for. + * @param array $objectArray + * + * @return JSONResponse + */ + private function createObject(mixed $mapper, array $objectArray): JSONResponse + { + $jsonLdDefault = $this->prepareObject($objectArray); + + $newObject = $mapper->createFromArray(object: $objectArray); + + // Make sure we set the reference when creating an imported object. + if (empty($objectArray['reference']) === true) { + $objectArray['reference'] = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( + routeName: 'openconnector.'.ucfirst($objectArray['@type']).'s.show', + arguments: ['id' => $newObject->getId()]) + ); + $newObject = $mapper->updateFromArray(id: $newObject->getId(), object: $objectArray); + } + + return new JSONResponse( + data: [ + 'message' => "Import successful, created", + 'object' => array_merge($jsonLdDefault, ['@id' => $objectArray['reference']], $newObject->jsonSerialize()) + ], + statusCode: 201 + ); } } From 277147e01ab6409d3e59438f2315b08e6ed3de54 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Fri, 10 Jan 2025 13:47:32 +0100 Subject: [PATCH 10/13] Finish code cleanup in the ImportService --- lib/Service/ImportService.php | 42 ++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/lib/Service/ImportService.php b/lib/Service/ImportService.php index d5873e43..6f8b25c6 100644 --- a/lib/Service/ImportService.php +++ b/lib/Service/ImportService.php @@ -212,7 +212,10 @@ private function getJSONfromBody(array|string $phpArray, ?string $type = null): } if ($phpArray === null || $phpArray === false) { - return new JSONResponse(data: ['error' => 'Failed to decode JSON input'], statusCode: 400); + return new JSONResponse( + data: ['error' => 'Failed to decode JSON input'], + statusCode: 400 + ); } return $this->saveObject(objectArray: $phpArray, type: $type); @@ -244,7 +247,7 @@ private function saveObject(array $objectArray, ?string $type): JSONResponse // Check if object already exists. if (isset($objectArray['@id']) === true) { $objectArray['reference'] = $objectArray['@id']; - $object = $this->checkIfExists(mapper: $mapper, phpArray: $objectArray); + $object = $this->checkIfExists(mapper: $mapper, objectArray: $objectArray); } if (isset($object) === true && $object instanceof Entity) { @@ -258,28 +261,28 @@ private function saveObject(array $objectArray, ?string $type): JSONResponse * Check if the object already exists by using the @id from the phpArray data. * * @param mixed $mapper The mapper of the correct object type to look for. - * @param array $phpArray The array data we want to use to create/update an object. + * @param array $objectArray The array data we want to use to create/update an object. * * @return Entity|null The already existing object or null. */ - private function checkIfExists(mixed $mapper, array $phpArray): ?Entity + private function checkIfExists(mixed $mapper, array $objectArray): ?Entity { // Check reference if (method_exists($mapper, 'findByRef')) { try { - return $mapper->findByRef($phpArray['@id'])[0]; + return $mapper->findByRef($objectArray['@id'])[0]; } catch (Exception $exception) {} } // Check if @id matches an object of that type in OpenConnector. 'failsafe' / backup try { - $explodedId = explode(separator: '/', string: $phpArray['@id']); + $explodedId = explode(separator: '/', string: $objectArray['@id']); $id = end(array: $explodedId); $url = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( - routeName: 'openconnector.'.ucfirst($phpArray['@type']).'s.show', + routeName: 'openconnector.'.ucfirst($objectArray['@type']).'s.show', arguments: ['id' => $id]) ); - if ($phpArray['@id'] === $url) { + if ($objectArray['@id'] === $url) { return $mapper->find($id); } } catch (Exception $exception) {} @@ -292,18 +295,23 @@ private function checkIfExists(mixed $mapper, array $phpArray): ?Entity * Will prepare and return some default values for the object to save. * Will also unset some fields from the array used to create or update the object. * - * @param array $objectArray + * @param array $objectArray The array data we want to use to create/update an object. * - * @return array + * @return array The Json-LD default properties for any object created or updated. */ private function prepareObject(array &$objectArray): array { + // Prepare Json-LD default properties before unsetting fields from the $objectArray. $jsonLdDefault = [ - '@context' => ["schema" => "http://schema.org", "register" => "501"], + '@context' => [ + "schema" => "http://schema.org", + "register" => "501" + ], '@type' => $objectArray['@type'], '@id' => $objectArray['reference'] ?? null ]; + // Remove all fields from ObjectArray that we should not copy or update when importing an object. unset( $objectArray['@context'], $objectArray['@type'], @@ -323,15 +331,16 @@ private function prepareObject(array &$objectArray): array * Updates an object using the given array as input. * * @param mixed $mapper The mapper of the object type we want to update an object for. - * @param array $objectArray + * @param array $objectArray The array data we want to use to update an object. * @param Entity $object * - * @return JSONResponse + * @return JSONResponse A JSON response with a message and the updated object or an error message. */ private function updateObject(mixed $mapper, array $objectArray, Entity $object): JSONResponse { $jsonLdDefault = $this->prepareObject($objectArray); + // @todo: maybe we should do some kind of hash comparison here as well? if ($object->getVersion() !== null && empty($objectArray['version']) === false && version_compare(version1: $object->getVersion(), version2: $objectArray['version'], operator: '>=') ) { @@ -344,8 +353,9 @@ private function updateObject(mixed $mapper, array $objectArray, Entity $object) ] ); } - // @todo: maybe we should do kind of hash comparison here as well? + $updatedObject = $mapper->updateFromArray(id: $object->getId(), object: $objectArray); + return new JSONResponse( data: [ 'message' => "Import successful, updated", @@ -358,9 +368,9 @@ private function updateObject(mixed $mapper, array $objectArray, Entity $object) * Creates an object using the given array as input. * * @param mixed $mapper The mapper of the object type we want to create an object for. - * @param array $objectArray + * @param array $objectArray The array data we want to use to create an object. * - * @return JSONResponse + * @return JSONResponse A JSON response with a message and the created object or an error message. */ private function createObject(mixed $mapper, array $objectArray): JSONResponse { From 899516ba4d7ca3be2f94d39781e9d83a3e432983 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Fri, 10 Jan 2025 14:47:35 +0100 Subject: [PATCH 11/13] Fix for jobs --- lib/Db/JobMapper.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Db/JobMapper.php b/lib/Db/JobMapper.php index ed9174f4..b68eafd3 100644 --- a/lib/Db/JobMapper.php +++ b/lib/Db/JobMapper.php @@ -34,7 +34,7 @@ public function findByRef(string $reference): array $qb = $this->db->getQueryBuilder(); $qb->select('*') - ->from('openconnector_endpoints') + ->from('openconnector_jobs') ->where( $qb->expr()->eq('reference', $qb->createNamedParameter($reference)) ); From 007f298984a22f38756b968ed6c49714a98a7027 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Fri, 10 Jan 2025 15:03:04 +0100 Subject: [PATCH 12/13] Cleanup ExportService --- lib/Db/Mapping.php | 2 +- lib/Db/MappingMapper.php | 1 - lib/Db/Source.php | 2 +- lib/Db/Synchronization.php | 2 +- lib/Service/ExportService.php | 118 +++++++++++++++++++++++++--------- 5 files changed, 90 insertions(+), 35 deletions(-) diff --git a/lib/Db/Mapping.php b/lib/Db/Mapping.php index a6bf62b7..58cbfb3c 100644 --- a/lib/Db/Mapping.php +++ b/lib/Db/Mapping.php @@ -10,7 +10,7 @@ class Mapping extends Entity implements JsonSerializable { protected ?string $uuid = null; protected ?string $reference = null; - protected ?string $version = null; + protected ?string $version = '0.0.0'; protected ?string $name = null; protected ?string $description = null; protected ?array $mapping = null; diff --git a/lib/Db/MappingMapper.php b/lib/Db/MappingMapper.php index d16c69fb..8078fe98 100644 --- a/lib/Db/MappingMapper.php +++ b/lib/Db/MappingMapper.php @@ -96,7 +96,6 @@ public function updateFromArray(int $id, array $object): Mapping } } - return $this->update($obj); } diff --git a/lib/Db/Source.php b/lib/Db/Source.php index cb35906d..9ae7bfda 100644 --- a/lib/Db/Source.php +++ b/lib/Db/Source.php @@ -12,7 +12,7 @@ class Source extends Entity implements JsonSerializable protected ?string $name = null; protected ?string $description = null; protected ?string $reference = null; - protected ?string $version = null; + protected ?string $version = '0.0.0'; protected ?string $location = null; protected ?bool $isEnabled = null; protected ?string $type = null; diff --git a/lib/Db/Synchronization.php b/lib/Db/Synchronization.php index ffbc4670..53a334f1 100644 --- a/lib/Db/Synchronization.php +++ b/lib/Db/Synchronization.php @@ -12,7 +12,7 @@ class Synchronization extends Entity implements JsonSerializable protected ?string $name = null; // The name of the synchronization protected ?string $description = null; // The description of the synchronization protected ?string $reference = null; // The reference of the endpoint - protected ?string $version = null; // The version of the synchronization + protected ?string $version = '0.0.0'; // The version of the synchronization // Source protected ?string $sourceId = null; // The id of the source object protected ?string $sourceType = null; // The type of the source object (e.g. api, database, register/schema.) diff --git a/lib/Service/ExportService.php b/lib/Service/ExportService.php index 543db654..f382c7f6 100644 --- a/lib/Service/ExportService.php +++ b/lib/Service/ExportService.php @@ -12,10 +12,12 @@ use OCA\OpenRegister\Db\SchemaMapper; use OCA\OpenRegister\Service\DoesNotExistException; use OCA\OpenRegister\Service\GuzzleHttp; +use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Http\JSONResponse; use OCP\IURLGenerator; use Psr\Container\ContainerExceptionInterface; use Psr\Container\NotFoundExceptionInterface; +use Symfony\Component\Yaml\Yaml; /** * Service for handling export requests for database entities. @@ -42,6 +44,16 @@ public function __construct( */ public function export(string $objectType, string $id, string $accept): JSONResponse { + $type = match (true) { + str_contains(haystack: $accept, needle: 'application/json') => 'json', + $accept === 'application/yaml' => 'yaml', + default => null + }; + + if ($type === null) { + return new JSONResponse(data: ['error' => "The Accept type $accept is not supported."], statusCode: 400); + } + try { $mapper = $this->objectService->getMapper(objectType: $objectType); } catch (InvalidArgumentException|NotFoundExceptionInterface|ContainerExceptionInterface $e) { @@ -54,55 +66,99 @@ public function export(string $objectType, string $id, string $accept): JSONResp return new JSONResponse(data: ['error' => "Could not find an object with this {id}: ".$id], statusCode: 400); } - $objectArray = $object->jsonSerialize(); + $objectArray = $this->prepareObject($objectType, $mapper, $object); + $filename = ucfirst($objectType).'-'.$objectArray['name'].'-v'.$objectArray['version']; - if (str_contains(haystack: $accept, needle: 'application/json') === true) { - if (empty($objectArray['reference']) === false) { - $url = $objectArray['reference']; - } else { - $url = $objectArray['reference'] = $this->urlGenerator->getAbsoluteURL(url: $this->urlGenerator->linkToRoute( + $dataString = $this->encode(objectArray: $objectArray, type: $type); + + $this->download(dataString: $dataString, filename: $filename, type: $type); + } + + /** + * Prepares a PHP array with all data of the object we want to end up in the downloadable file. + * + * @param string $objectType The type of object to export. + * @param mixed $mapper The mapper of the correct object type to export. + * @param Entity $object The object we want to export. + * + * @return array The object array data. + */ + private function prepareObject(string $objectType, mixed $mapper, Entity $object): array + { + $objectArray = $object->jsonSerialize(); + + if (empty($objectArray['reference']) === false) { + $url = $objectArray['reference']; + } else { + $url = $objectArray['reference'] = $this->urlGenerator->getAbsoluteURL( + url: $this->urlGenerator->linkToRoute( routeName: 'openconnector.'.ucfirst($objectType).'s.show', - arguments: ['id' => $object->getId()]) - ); - unset($objectArray['id'], $objectArray['uuid'], $objectArray['created'], $objectArray['updated'], - $objectArray['dateCreated'], $objectArray['dateModified']); - $mapper->updateFromArray(id: $id, object: $objectArray); - } - - $objArray['@context'] = ["schema" => "http://schema.org", "register" => "501"]; - $objArray['@type'] = $objectType; - $objArray['@id'] = $url; - $objArray = array_merge($objArray, $objectArray); - - // Convert the object data to JSON - $jsonData = json_encode(value: $objArray, flags: JSON_PRETTY_PRINT); - - $this->downloadJson(jsonData: $jsonData, filename: $filename); + arguments: ['id' => $object->getId()] + ) + ); + + unset($objectArray['id'], $objectArray['uuid'], $objectArray['created'], $objectArray['updated'], + $objectArray['dateCreated'], $objectArray['dateModified']); + + // Make sure we update the reference of this object if it wasn't set yet. + $mapper->updateFromArray(id: $object->getId(), object: $objectArray); } - return new JSONResponse(data: ['error' => "The Accept type $accept is not supported."], statusCode: 400); + // Prepare Json-LD default properties. + $jsonLdDefault = [ + '@context' => [ + "schema" => "http://schema.org", + "register" => "501" + ], + '@type' => $objectType, + '@id' => $url + ]; + + return array_merge($jsonLdDefault, $objectArray); + } + + /** + * A function used to encode object array to a data string. + * So it can be used to create a downloadable file. + * + * @param array $objectArray The object array data. + * @param string|null $type The type from the accept header. + * + * @return string|null The encoded data string or null. + */ + private function encode(array $objectArray, ?string $type): ?string + { + switch ($type) { + case 'application/json': + return json_encode(value: $objectArray, flags: JSON_PRETTY_PRINT); + case 'application/yaml': + return Yaml::dump(input: $objectArray); + default: + return null; + } } /** - * Generate a downloadable json file response. + * Generate a downloadable file response. * - * @param string $jsonData The json data to create a json file with. - * @param string $filename The filename, .json will be added after this filename in this function. + * @param string $dataString The data to create a file with of the given $type. + * @param string $filename The filename, .[$type] will be added after this filename in this function. + * @param string $type The type of file to create and download. Default = json. * * @return void */ - #[NoReturn] private function downloadJson(string $jsonData, string $filename): void + #[NoReturn] private function download(string $dataString, string $filename, string $type = 'json'): void { // Define the file name and path for the temporary JSON file - $fileName = $filename.'.json'; + $fileName = "$filename.$type"; $filePath = sys_get_temp_dir() . DIRECTORY_SEPARATOR . $fileName; - // Create and write the JSON data to the file - file_put_contents(filename: $filePath, data: $jsonData); + // Create and write the (JSON) data to the file + file_put_contents(filename: $filePath, data: $dataString); // Set headers to download the file - header('Content-Type: application/json'); + header("Content-Type: application/$type"); header('Content-Disposition: attachment; filename="' . $fileName . '"'); header('Content-Length: ' . filesize(filename: $filePath)); From 121188e32f27140fda5ffdecba7e953a2b6df9f0 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Fri, 10 Jan 2025 15:15:07 +0100 Subject: [PATCH 13/13] Small fix & final cleanup ExportService --- lib/Service/ExportService.php | 29 +++++++++++++++++++++++------ lib/Service/ImportService.php | 2 +- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/Service/ExportService.php b/lib/Service/ExportService.php index f382c7f6..3aa8f0ed 100644 --- a/lib/Service/ExportService.php +++ b/lib/Service/ExportService.php @@ -66,11 +66,11 @@ public function export(string $objectType, string $id, string $accept): JSONResp return new JSONResponse(data: ['error' => "Could not find an object with this {id}: ".$id], statusCode: 400); } - $objectArray = $this->prepareObject($objectType, $mapper, $object); - + $objectArray = $this->prepareObject(objectType: $objectType, mapper: $mapper, object: $object); + $filename = ucfirst($objectType).'-'.$objectArray['name'].'-v'.$objectArray['version']; - $dataString = $this->encode(objectArray: $objectArray, type: $type); + $dataString = $this->encode(objectArray: $objectArray, type: $accept); $this->download(dataString: $dataString, filename: $filename, type: $type); } @@ -131,12 +131,29 @@ private function encode(array $objectArray, ?string $type): ?string { switch ($type) { case 'application/json': - return json_encode(value: $objectArray, flags: JSON_PRETTY_PRINT); + $dataString = json_encode(value: $objectArray, flags: JSON_PRETTY_PRINT); + break; case 'application/yaml': - return Yaml::dump(input: $objectArray); + $dataString = Yaml::dump(input: $objectArray); + break; default: - return null; + // If type is not specified or not recognized, try to encode as JSON first, then YAML + $dataString = json_encode(value: $objectArray, flags: JSON_PRETTY_PRINT); + if ($dataString === false) { + try { + $dataString = Yaml::dump(input: $objectArray); + } catch (Exception $exception) { + $dataString = null; + } + } + break; + } + + if ($dataString === null || $dataString === false) { + return null; } + + return $dataString; } /** diff --git a/lib/Service/ImportService.php b/lib/Service/ImportService.php index 6f8b25c6..9a7570f6 100644 --- a/lib/Service/ImportService.php +++ b/lib/Service/ImportService.php @@ -115,7 +115,7 @@ private function decode(string $data, ?string $type): ?array default: // If Content-Type is not specified or not recognized, try to parse as JSON first, then YAML $phpArray = json_decode(json: $data, associative: true); - if ($phpArray === null) { + if ($phpArray === null || $phpArray === false) { try { $phpArray = Yaml::parse(input: $data); } catch (Exception $exception) {