diff --git a/packages/keystone/KSv5v6/v5/registerSchema.js b/packages/keystone/KSv5v6/v5/registerSchema.js index 7ccc566cdb5..560a95b6798 100644 --- a/packages/keystone/KSv5v6/v5/registerSchema.js +++ b/packages/keystone/KSv5v6/v5/registerSchema.js @@ -26,6 +26,7 @@ const { SignedDecimal, Text, EncryptedText, + LargeText, } = require('../../fields') const { HiddenRelationship } = require('../../plugins/utils/HiddenRelationship') const { AuthedRelationship, Relationship } = require('../../plugins/utils/Relationship') @@ -85,6 +86,7 @@ function convertStringToTypes (schema) { SignedDecimal, Text, EncryptedText, + LargeText, } const allTypesForPrint = Object.keys(mapping).map(item => `"${item}"`).join(', ') diff --git a/packages/keystone/fields/LargeText/Implementation.js b/packages/keystone/fields/LargeText/Implementation.js new file mode 100644 index 00000000000..d2edd51b39b --- /dev/null +++ b/packages/keystone/fields/LargeText/Implementation.js @@ -0,0 +1,14 @@ +const { Text } = require('@keystonejs/fields') + +class LargeTextImplementation extends Text.implementation { + constructor (path, { + adapter, + }) { + super(...arguments) + this.fileAdapter = adapter + } +} + +module.exports = { + LargeTextImplementation, +} diff --git a/packages/keystone/fields/LargeText/README.md b/packages/keystone/fields/LargeText/README.md new file mode 100644 index 00000000000..07ace26a6c3 --- /dev/null +++ b/packages/keystone/fields/LargeText/README.md @@ -0,0 +1,49 @@ +# LargeText + +The `LargeText` field simplifies the storage of large text data by abstracting the implementation of cloud storage usage. + +## Basic Usage + +Simply add the `LargeText` field type and a file adapter, and your text data will be stored in cloud storage. This helps avoid storing large datasets directly in the database. You can use it just like a common `Text` field in your code. + +```js +const fileAdapter = new FileAdapter('LOG_FILE_NAME') + +keystone.createList('Log', { + fields: { + xmlLog: { + type: 'LargeText', + adapter: fileAdapter, + }, + }, +}); +``` + +# GraphQL + +The `LargeText` field behaves like a standard string (`Text` field). During create/update operations, the input value is saved to cloud storage, and only a reference to the saved file is stored in the database. For read operations, the saved file is downloaded from cloud storage, and its contents are provided as a string. + +# Storage + +The text value is stored as a file in the cloud, while the database only holds a reference (link) to the source file. + +# Configuration + +You will need to configure a file adapter to work with cloud storage. Example: + +```js +const fileAdapter = new FileAdapter('LOG_FILE_NAME') +``` + +or + +```js +const fileAdapter = new FileAdapter('LOG_FILE_NAME', false, false, { bucket: 'BUCKET_FOR_LOGS'}) +``` + +Ensure your cloud storage configuration is properly set up and accessible for file storage. + +# Notes + +- The `LargeText` field is ideal for storing large logs, parsed values, or other text data that may exceed the typical size limits of database fields. +- The text is stored as a file in cloud storage, reducing the load on the database while still providing easy access to the data. \ No newline at end of file diff --git a/packages/keystone/fields/LargeText/adapters/index.js b/packages/keystone/fields/LargeText/adapters/index.js new file mode 100644 index 00000000000..5a1b9ba27bc --- /dev/null +++ b/packages/keystone/fields/LargeText/adapters/index.js @@ -0,0 +1,76 @@ +const { Text } = require('@keystonejs/fields') +const cuid = require('cuid') +const isNil = require('lodash/isNil') + +const { bufferToStream, readFileFromStream, getObjectStream } = require('@open-condo/keystone/file') + +const CommonInterface = superclass => class extends superclass { + + constructor () { + super(...arguments) + if (!this.config.adapter) { + throw new Error('LargeText field cannot be used without a file adapter') + } + this.fileAdapter = this.config.adapter + } + + setupHooks ({ addPreSaveHook, addPostReadHook }) { + + addPreSaveHook(async item => { + const fieldIsDefined = !isNil(item) && !isNil(item[this.path]) + if (fieldIsDefined) { + const stream = bufferToStream(item[this.path]) + //TODO: need to think about name + const originalFilename = `${new Date().toISOString()}` + const mimetype = 'text/plain' + const encoding = 'utf8' + + const { + id, + filename, + _meta, + } = await this.fileAdapter.save({ + stream, + filename: originalFilename, + mimetype, + encoding, + id: cuid(), + }) + item[this.path] = JSON.stringify( + { + id, + filename, + originalFilename, + mimetype, + encoding, + _meta, + } + ) + } + return item + }) + + addPostReadHook(async item => { + if (item[this.path]) { + item[this.path] = (await readFileFromStream(await getObjectStream(item[this.path], this.fileAdapter))).toString() + } + + return item + }) + } + + addToTableSchema (table) { + const column = table.jsonb(this.path) + column.isMultiline = true + } +} + +class LargeTextKnexFieldAdapter extends CommonInterface(Text.adapters.knex) {} +class LargeTextMongooseFieldAdapter extends CommonInterface(Text.adapters.mongoose) {} +class LargeTextPrismaFieldAdapter extends CommonInterface(Text.adapters.prisma) {} + +module.exports = { + mongoose: LargeTextKnexFieldAdapter, + knex: LargeTextMongooseFieldAdapter, + prisma: LargeTextPrismaFieldAdapter, +} \ No newline at end of file diff --git a/packages/keystone/fields/LargeText/index.js b/packages/keystone/fields/LargeText/index.js new file mode 100644 index 00000000000..69cfdebed5e --- /dev/null +++ b/packages/keystone/fields/LargeText/index.js @@ -0,0 +1,14 @@ +const { Text } = require('@keystonejs/fields') + +const LargeTextAdapters = require('./adapters') +const { LargeTextImplementation } = require('./Implementation') + +module.exports = { + type: 'LargeText', + implementation: LargeTextImplementation, + views: { + ...Text.views, + Controller: require.resolve('./views/Controller'), + }, + adapters: LargeTextAdapters, +} \ No newline at end of file diff --git a/packages/keystone/fields/LargeText/views/Controller.js b/packages/keystone/fields/LargeText/views/Controller.js new file mode 100644 index 00000000000..cc32d0eb237 --- /dev/null +++ b/packages/keystone/fields/LargeText/views/Controller.js @@ -0,0 +1,5 @@ +import FieldController from '@keystonejs/fields/Controller' + +export default class LargeTextController extends FieldController { + getFilterTypes = () => [] +} diff --git a/packages/keystone/fields/index.js b/packages/keystone/fields/index.js index 5a680f3042c..b2205c6c7c4 100644 --- a/packages/keystone/fields/index.js +++ b/packages/keystone/fields/index.js @@ -4,6 +4,7 @@ const DateInterval = require('./DateInterval') const EncryptedText = require('./EncryptedText') const FileWithUTF8Name = require('./FileWithUTF8Name') const Json = require('./Json') +const LargeText = require('./LargeText') const LocalizedText = require('./LocalizedText') const Options = require('./Options') const Select = require('./Select') @@ -24,4 +25,5 @@ module.exports = { FileWithUTF8Name, Text, EncryptedText, + LargeText, } diff --git a/packages/keystone/fileAdapter/fileAdapter.js b/packages/keystone/fileAdapter/fileAdapter.js index 30bab737fe1..f2212a2b50c 100644 --- a/packages/keystone/fileAdapter/fileAdapter.js +++ b/packages/keystone/fileAdapter/fileAdapter.js @@ -48,8 +48,9 @@ class LocalFilesMiddleware { } } +//Needs to thing how to make configurable file adapter class FileAdapter { - constructor (folder, isPublic = false, saveFileName = false) { + constructor (folder, isPublic = false, saveFileName = false, customConfig = {}) { const type = conf.FILE_FIELD_ADAPTER || DEFAULT_FILE_ADAPTER this.folder = folder this.type = type @@ -61,7 +62,7 @@ class FileAdapter { Adapter = this.createLocalFileApapter() break case 'sbercloud': - Adapter = this.createSbercloudFileApapter() + Adapter = this.createSbercloudFileApapter(customConfig) break } if (!Adapter) { @@ -110,13 +111,14 @@ class FileAdapter { return true } - createSbercloudFileApapter () { - const config = this.getEnvConfig('SBERCLOUD_OBS_CONFIG', [ + createSbercloudFileApapter (customConfig) { + let config = this.getEnvConfig('SBERCLOUD_OBS_CONFIG', [ 'bucket', 's3Options.server', 's3Options.access_key_id', 's3Options.secret_access_key', ]) + config = { ...config, ...customConfig } if (!config) { return null }