feat(keystone): DOMA-10804 FileStorageText field is added in keystone

packages/keystone/KSv5v6/v5/registerSchema.js

@@ -26,6 +26,7 @@ const {
     SignedDecimal,
     Text,
     EncryptedText,
+    FileStorageText,
 } = require('../../fields')
 const { HiddenRelationship } = require('../../plugins/utils/HiddenRelationship')
 const { AuthedRelationship, Relationship } = require('../../plugins/utils/Relationship')
@@ -85,6 +86,7 @@ function convertStringToTypes (schema) {
         SignedDecimal,
         Text,
         EncryptedText,
+        FileStorageText,
     }
     const allTypesForPrint = Object.keys(mapping).map(item => `"${item}"`).join(', ')
 

packages/keystone/fields/FileStorageText/Implementation.js

@@ -0,0 +1,34 @@
+const { Text } = require('@keystonejs/fields')
+
+class FileStorageTextImplementation extends Text.implementation {
+    constructor (path, {
+        adapter,
+    }) {
+        super(...arguments)
+        this.fileAdapter = adapter
+        this.isMultiline = true
+        this.isOrderable = false
+    }
+    equalityInputFields () {
+        return []
+    }
+    equalityInputFieldsInsensitive () {
+        return []
+    }
+    inInputFields () {
+        return []
+    }
+    orderingInputFields () {
+        return []
+    }
+    stringInputFields () {
+        return []
+    }
+    stringInputFieldsInsensitive () {
+        return []
+    }
+}
+
+module.exports = {
+    FileStorageTextImplementation,
+}

packages/keystone/fields/FileStorageText/README.md

@@ -0,0 +1,64 @@
+# FileStorageText
+
+The `FileStorageText` field simplifies the storage of large text data by leveraging file storage solutions, reducing database load, and ensuring efficient handling of large datasets.
+
+## Overview
+The `FileStorageText` field is designed to store large text data in an external file storage system instead of directly in the database. This helps avoid database size limitations and improves overall system performance.
+
+You can use this field in your code just like a standard `Text` field, but instead of storing the actual text in the database, only a reference (link) to the file is saved.
+
+## Basic Usage
+
+Add the `FileStorageText` field type to your schema along with a file adapter, and your text data will be stored in a file storage system. Below is an example:
+
+```js
+const fileAdapter = new FileAdapter('LOG_FILE_NAME')
+
+keystone.createList('Log', {
+  fields: {
+    xmlLog: { 
+        type: 'FileStorageText',
+        adapter: fileAdapter,
+    },
+  },
+});
+```
+
+## GraphQL Behavior
+
+The `FileStorageText` field behaves like a standard string (`Text` field) in GraphQL operations:
+
+- **Create/Update:** The input value is saved to the configured file storage, and the database stores only a reference to the saved file.
+- **Read:** The saved file is fetched from the file storage, and its contents are returned as a string.
+
+However, the `FileStorageText` field does not support sorting, ordering, filtering, or searching in GraphQL.
+
+## Storage
+
+- **File Storage:** The actual text data is saved as a file in the configured file storage.
+- **Database:** Only a reference (link) to the file in the file storage is saved in the database.
+
+This approach keeps the database lightweight and optimized for operations.
+
+## Configuration
+
+You must configure a file adapter to use the `FileStorageText` field. Below are some examples:
+
+```js
+const fileAdapter = new FileAdapter('LOG_FILE_NAME')
+```
+
+or
+
+```js
+const fileAdapter = new FileAdapter('LOG_FILE_NAME', false, false, { bucket: 'BUCKET_FOR_LOGS'})
+```
+
+Make sure your file storage system is correctly configured and accessible.
+
+## Notes
+
+- The `FileStorageText` field is ideal for storing large logs, parsed data, or other text values that exceed typical database field size limits.
+- Since the text is stored externally as a file, the database remains lean, and operations are more efficient.
+- This field does not support sorting, ordering, filtering, or searching in GraphQL. It is designed solely for storing and retrieving large text values.
+- Be sure to test your file storage setup to ensure data accessibility and integrity.

packages/keystone/fields/FileStorageText/adapters/index.js

@@ -0,0 +1,77 @@
+const { Text } = require('@keystonejs/fields')
+const cuid = require('cuid')
+
+const { FILE_STORAGE_TEXT_MIMETYPE, FILE_STORAGE_TEXT_ENCODING } = require('@open-condo/keystone/fields/FileStorageText/constants')
+const { bufferToStream, readFileFromStream, getObjectStream } = require('@open-condo/keystone/file')
+
+
+const CommonInterface = superclass => class extends superclass {
+
+    constructor () {
+        super(...arguments)
+        if (!this.config.adapter) {
+            throw new Error('FileStorageText field cannot be used without a file adapter')
+        }
+        this.fileAdapter = this.config.adapter
+    }
+
+    setupHooks ({ addPreSaveHook, addPostReadHook }) {
+        addPreSaveHook(async (item) => {
+            if (item[this.path]) {
+                item[this.path] = await this._saveFileToAdapter(item[this.path])
+            }
+            return item
+        })
+
+        addPostReadHook(async (item) => {
+            if (item[this.path]) {
+                item[this.path] = await this._readFileFromAdapter(item[this.path])
+            }
+            return item
+        })
+    }
+
+    async _saveFileToAdapter (content) {
+        const stream = bufferToStream(content)
+        const originalFilename = cuid()
+
+        const { id, filename, _meta } = await this.fileAdapter.save({
+            stream,
+            filename: originalFilename,
+            mimetype: FILE_STORAGE_TEXT_MIMETYPE,
+            encoding: FILE_STORAGE_TEXT_ENCODING,
+            id: cuid(),
+        })
+
+        return {
+            id,
+            filename,
+            originalFilename,
+            mimetype: FILE_STORAGE_TEXT_MIMETYPE,
+            encoding: FILE_STORAGE_TEXT_ENCODING,
+            _meta,
+        }
+    }
+
+    async _readFileFromAdapter (fileMetadata) {
+        const fileStream = await getObjectStream(fileMetadata, this.fileAdapter)
+        const fileContent = await readFileFromStream(fileStream)
+        return fileContent.toString()
+    }
+
+    addToTableSchema (table) {
+        const column = table.jsonb(this.path)
+        if (this.isNotNullable) column.notNullable()
+        if (this.defaultTo) column.defaultTo(this.defaultTo)
+    }
+}
+
+class FileStorageTextKnexFieldAdapter extends CommonInterface(Text.adapters.knex) {}
+class FileStorageTextMongooseFieldAdapter extends CommonInterface(Text.adapters.mongoose) {}
+class FileStorageTextPrismaFieldAdapter extends CommonInterface(Text.adapters.prisma) {}
+
+module.exports = {
+    mongoose: FileStorageTextKnexFieldAdapter,
+    knex: FileStorageTextMongooseFieldAdapter,
+    prisma: FileStorageTextPrismaFieldAdapter,
+}

packages/keystone/fields/FileStorageText/constants.js

@@ -0,0 +1,7 @@
+const FILE_STORAGE_TEXT_MIMETYPE = 'text/plain'
+const FILE_STORAGE_TEXT_ENCODING = 'utf8'
+
+module.exports = {
+    FILE_STORAGE_TEXT_MIMETYPE,
+    FILE_STORAGE_TEXT_ENCODING,
+}

packages/keystone/fields/FileStorageText/index.js

@@ -0,0 +1,14 @@
+const { Text } = require('@keystonejs/fields')
+
+const FileStorageTextAdapters = require('./adapters')
+const { FileStorageTextImplementation } = require('./Implementation')
+
+module.exports = {
+    type: 'FileStorageText',
+    implementation: FileStorageTextImplementation,
+    views: {
+        ...Text.views,
+        Controller: require.resolve('./views/Controller'),
+    },
+    adapters: FileStorageTextAdapters,
+}

packages/keystone/fields/FileStorageText/views/Controller.js

@@ -0,0 +1,5 @@
+import FieldController from '@keystonejs/fields/Controller'
+
+export default class FileStorageTextController extends FieldController {
+    getFilterTypes = () => []
+}

packages/keystone/fields/index.js

@@ -2,6 +2,7 @@ const AddressPartWithType = require('./AddressPartWithType')
 const AutoIncrementInteger = require('./AutoIncrementInteger')
 const DateInterval = require('./DateInterval')
 const EncryptedText = require('./EncryptedText')
+const FileStorageText = require('./FileStorageText')
 const FileWithUTF8Name = require('./FileWithUTF8Name')
 const Json = require('./Json')
 const LocalizedText = require('./LocalizedText')
@@ -24,4 +25,5 @@ module.exports = {
     FileWithUTF8Name,
     Text,
     EncryptedText,
+    FileStorageText,
 }

packages/keystone/fileAdapter/fileAdapter.js

@@ -49,7 +49,7 @@ class LocalFilesMiddleware {
 }
 
 class FileAdapter {
-    constructor (folder, isPublic = false, saveFileName = false) {
+    constructor (folder, isPublic = false, saveFileName = false, customConfig = {}) {
         const type = conf.FILE_FIELD_ADAPTER || DEFAULT_FILE_ADAPTER
         this.folder = folder
         this.type = type
@@ -61,7 +61,7 @@ class FileAdapter {
                 Adapter = this.createLocalFileApapter()
                 break
             case 'sbercloud':
-                Adapter = this.createSbercloudFileApapter()
+                Adapter = this.createSbercloudFileApapter(customConfig)
                 break
         }
         if (!Adapter) {
@@ -110,17 +110,35 @@ class FileAdapter {
         return true
     }
 
-    createSbercloudFileApapter () {
-        const config = this.getEnvConfig('SBERCLOUD_OBS_CONFIG', [
+    createSbercloudFileApapter (customConfig) {
+        let config = this.getEnvConfig('SBERCLOUD_OBS_CONFIG', [
             'bucket',
             's3Options.server',
             's3Options.access_key_id',
             's3Options.secret_access_key',
         ])
+
         if (!config) {
             return null
         }
-        return new SberCloudFileAdapter({ ...config, folder: this.folder, isPublic: this.isPublic, saveFileName: this.saveFileName })
+
+        config = { ...config, ...customConfig }
+
+        if (!this.isConfigValid(config, [
+            'bucket',
+            's3Options.server',
+            's3Options.access_key_id',
+            's3Options.secret_access_key',
+        ])) {
+            return null
+        }
+
+        return new SberCloudFileAdapter({
+            ...config,
+            folder: this.folder,
+            isPublic: this.isPublic,
+            saveFileName: this.saveFileName,
+        })
     }
 
     // TODO(pahaz): DOMA-1569 it's better to create just a function. But we already use FileAdapter in many places. I just want to save a backward compatibility