diff --git a/README.md b/README.md index 130d906b8..43dc0d21f 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,7 @@ All operation use es7 async/await to implement. All api is async function. - [.getObjectTagging(name, [, options])](#getObjectTaggingname-options) - [.putObjectTagging(name, tag[, options])](#putObjectTaggingname-tag-options) - [.deleteObjectTagging(name, [, options])](#deleteObjectTaggingname-options) + - [.selectObject(name, query, grammar[, options])](#selectObjectname-query-grammar-options) - [RTMP Operations](#rtmp-operations) - [.putChannel(id, conf[, options])](#putchannelid-conf-options) - [.getChannel(id[, options])](#getchannelid-options) @@ -3619,6 +3620,62 @@ object: - status {Number} response status - res {Object} response info +### .selectObject(name, query, grammar[, options]) + +SelectObject is used to execute SQL statements on the target file and return the execution result. + +parameters: + +- name {String} the object name +- query {Object} query parameters eg. `'select * from my_table'` +- grammar {String} The request syntax is divided into two formats, CSV and JSON. +- [options] {Object} optional args + - [InputSerialization] {Object} InputSerialization + - [CompressionType] {String} The type of compression used to compress the object. + - [CSV] {Object} CSV + - [FileHeaderInfo] {String} The file header information, which is optional. + - [RecordDelimiter] {String} The record delimiter, which is optional. + - [FieldDelimiter] {String} The field delimiter, which is optional. + - [QuoteCharacter] {String} The quote character, which is optional. + - [CommentCharacter] {String} The comment character, which is optional. + - [Range] {String} The range, which is optional. + - [AllowQuotedRecordDelimiter] {Boolean} Whether to allow quoted record delimiter, which is optional. + - [JSON] {Object} JSON + - [Type] {String} The type, which is optional. + - [Range] {String} The range, which is optional. + - [ParseJsonNumberAsString] {Boolean} The parse json number as string, which is optional. + - [OutputSerialization] {Object} OutputSerialization + - [CSV] {Object} CSV + - [RecordDelimiter] {String} The record delimiter, which is optional. + - [FieldDelimiter] {String} The field delimiter, which is optional. + - [JSON] {Object} JSON + - [RecordDelimiter] {String} The record delimiter, which is optional. + - [KeepAllColumns] {Boolean} Whether to keep all columns, which is optional. + - [OutputRawData] {Boolean} Whether to output raw data, which is optional. + - [EnablePayloadCrc] {Boolean} Whether to enable payload crc, which is optional. + - [OutputHeader] {Boolean} Whether to output header, which is optional. + - [Other] {Object} Other option + - [SkipPartialDataRecord] {Boolean} Whether to skip partial data record, which is optional. + - [MaxSkippedRecordsAllowed] {String} The max skipped records allowed, which is optional. + +[More code info](https://help.aliyun.com/document_detail/74054.html#section-v2y-dzp-o5k) + +Succeed will return the channel information. + +object: + +- res {Object} response info +- data {Object} data info + +```javascript +// example CSV +store.selectObject('example.csv', 'select * from ossobject limit 10', 'csv').then(r => console.log(r)); + +// example JSON +store.selectObject('example.json', 'select address from ossobject.contacts[*]', 'json') + .then(r => console.log(r)); +``` + ### .processObjectSave(sourceObject, targetObject, process[, targetBucket]) Persistency indicates that images are asynchronously stored in the specified Bucket diff --git a/lib/common/object/index.d.ts b/lib/common/object/index.d.ts index b800cb674..bf6bf3d5b 100644 --- a/lib/common/object/index.d.ts +++ b/lib/common/object/index.d.ts @@ -23,6 +23,7 @@ import { putObjectTagging } from './putObjectTagging'; import { putSymlink } from './putSymlink'; import { restore } from './restore'; import { signatureUrl } from './signatureUrl'; +import { selectObject } from './selectObject'; declare const _default: { append: typeof append; calculatePostSignature: typeof calculatePostSignature; @@ -51,5 +52,6 @@ declare const _default: { putSymlink: typeof putSymlink; restore: typeof restore; signatureUrl: typeof signatureUrl; + selectObject: typeof selectObject; }; export default _default; diff --git a/lib/common/object/index.js b/lib/common/object/index.js index fe71e843d..91497dcec 100644 --- a/lib/common/object/index.js +++ b/lib/common/object/index.js @@ -25,6 +25,7 @@ const putObjectTagging_1 = require("./putObjectTagging"); const putSymlink_1 = require("./putSymlink"); const restore_1 = require("./restore"); const signatureUrl_1 = require("./signatureUrl"); +const selectObject_1 = require("./selectObject"); exports.default = { append: append_1.append, calculatePostSignature: calculatePostSignature_1.calculatePostSignature, @@ -55,4 +56,5 @@ exports.default = { putSymlink: putSymlink_1.putSymlink, restore: restore_1.restore, signatureUrl: signatureUrl_1.signatureUrl, + selectObject: selectObject_1.selectObject }; diff --git a/lib/common/object/selectObject.d.ts b/lib/common/object/selectObject.d.ts new file mode 100644 index 000000000..94e073572 --- /dev/null +++ b/lib/common/object/selectObject.d.ts @@ -0,0 +1,4 @@ +export declare function selectObject(this: any, name: string, expression: string, grammar: string, options: any): Promise<{ + res: any; + data: any; +}>; diff --git a/lib/common/object/selectObject.js b/lib/common/object/selectObject.js new file mode 100644 index 000000000..015692873 --- /dev/null +++ b/lib/common/object/selectObject.js @@ -0,0 +1,95 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.selectObject = void 0; +const obj2xml_1 = require("../utils/obj2xml"); +const js_base64_1 = require("js-base64"); +const unpackFrame_1 = __importDefault(require("../utils/unpackFrame")); +const needToEncode = ['RecordDelimiter', 'FieldDelimiter', 'QuoteCharacter', 'CommentCharacter']; +async function selectObject(name, expression, grammar, options) { + if (!['json', 'csv'].includes(grammar.toLocaleLowerCase())) { + throw new Error('grammar must be json or csv'); + } + const opts = Object.assign({}, options); + opts.subres = Object.assign({ 'x-oss-process': `${grammar}/select` }); + let InputSerialization; + let OutputSerialization; + let Options; + const paramsXMLObj = { + SelectRequest: { + Expression: js_base64_1.Base64.encode(expression) + } + }; + // CompressionType + if (opts.InputSerialization) { + opts.InputSerialization.CompressionType = opts.InputSerialization.CompressionType + ? opts.InputSerialization.CompressionType + : 'None'; + } + // CSV + if (grammar.toLocaleLowerCase() === 'csv') { + // inputSerialization + if (opts.InputSerialization && opts.InputSerialization.CSV) { + Object.keys(opts.InputSerialization.CSV).forEach(i => { + if (needToEncode.includes(i)) { + opts.InputSerialization.CSV[i] = js_base64_1.Base64.encode(opts.InputSerialization.CSV[i]); + } + }); + } + InputSerialization = Object.assign({}, opts.InputSerialization); + paramsXMLObj.SelectRequest.InputSerialization = InputSerialization; + // OutputSerialization + if (opts.OutputSerialization && opts.OutputSerialization.CSV) { + Object.keys(opts.OutputSerialization.CSV).forEach(i => { + if (needToEncode.includes(i)) { + opts.OutputSerialization.CSV[i] = js_base64_1.Base64.encode(opts.OutputSerialization.CSV[i]); + } + }); + } + OutputSerialization = Object.assign({}, opts.OutputSerialization); + paramsXMLObj.SelectRequest.OutputSerialization = OutputSerialization; + } + // JSON + if (grammar.toLowerCase() === 'json') { + // InputSerialization + if (opts.InputSerialization && opts.InputSerialization.JSON) { + opts.InputSerialization.JSON.Type = opts.InputSerialization.JSON.Type + ? opts.InputSerialization.JSON.Type.toUpperCase() + : 'DOCUMENT'; + opts.InputSerialization.JSON = Object.assign({}, opts.InputSerialization.JSON); + } + InputSerialization = Object.assign({}, opts.InputSerialization); + paramsXMLObj.SelectRequest.InputSerialization = InputSerialization; + // OutputSerialization + if (opts.OutputSerialization && opts.OutputSerialization.JSON) { + if (opts.OutputSerialization.JSON.RecordDelimiter) { + opts.OutputSerialization.JSON.RecordDelimiter = js_base64_1.Base64.encode(opts.OutputSerialization.JSON.RecordDelimiter); + } + } + OutputSerialization = Object.assign({}, opts.OutputSerialization); + paramsXMLObj.SelectRequest.OutputSerialization = OutputSerialization; + } + // Options + if (opts.Other) { + Options = Object.assign({}, opts.Other); + paramsXMLObj.SelectRequest.Options = Options; + } + const params = this._objectRequestParams('POST', name, opts); + params.content = obj2xml_1.obj2xml(paramsXMLObj); + params.mime = 'xml'; + params.successStatuses = [206]; + const result = await this.request(params); + if (result.res.headers['x-oss-select-output-raw'] !== 'true') { + result.data = unpackFrame_1.default(result.data); + } + else { + result.data = result.data.toString(); + } + return { + res: result.res, + data: result.data + }; +} +exports.selectObject = selectObject; diff --git a/lib/common/utils/unpackFrame.d.ts b/lib/common/utils/unpackFrame.d.ts new file mode 100644 index 000000000..f78282b27 --- /dev/null +++ b/lib/common/utils/unpackFrame.d.ts @@ -0,0 +1,19 @@ +/// +declare type frameData = { + payload: payloadType; + version: number; + frameType: number; + payloadLength: number; + headerCheckSum: number; + payloadCheckSum: number; +}; +declare type payloadType = { + offset: number; + frameType: string; + data?: string; + totalScannedBytes?: number; + httpStatusCode?: number; + errorMessage?: string; +}; +declare const _default: (data: Buffer) => frameData; +export default _default; diff --git a/lib/common/utils/unpackFrame.js b/lib/common/utils/unpackFrame.js new file mode 100644 index 000000000..b5d043d2a --- /dev/null +++ b/lib/common/utils/unpackFrame.js @@ -0,0 +1,40 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +// all integers in Frame are encoded in big endian and Version is currently 1. +exports.default = (data) => { + const payload = { frameType: '', offset: 0 }; + const version = data.readUIntBE(0, 1); + const frameType = data.readUIntBE(1, 3); + const payloadLength = data.readUIntBE(4, 4); + const headerCheckSum = data.readUIntBE(8, 4); + const payloadCheckSum = data.readUIntBE(data.length - 4, 4); + // payload offset + payload.offset = parseInt(data.slice(12, 20).toString('hex'), 16); + /** + * Frame Type + * 8388609 - Data Frame + * 8388612 - Continuous Frame + * 8388613 - End Frame + */ + if (frameType === 8388609) { + payload.frameType = 'DataFrame'; + payload.data = data.slice(20, 12 + payloadLength).toString(); + } + else if (frameType === 8388612) { + payload.frameType = 'ContinuousFrame'; + } + else if (frameType === 8388613) { + payload.frameType = 'End Frame'; + payload.totalScannedBytes = data.readUIntBE(20, 8); + payload.httpStatusCode = data.readUIntBE(28, 4); + payload.errorMessage = data.slice(32, data.length - 4).toString(); + } + return { + payload, + version, + frameType, + payloadCheckSum, + payloadLength, + headerCheckSum + }; +}; diff --git a/package.json b/package.json index bdee3381e..955f8821b 100644 --- a/package.json +++ b/package.json @@ -152,7 +152,6 @@ "lint-staged": { "**/!(dist)/*": [ "npm run detect-secrets --" - ], - "*.js": "eslint --cache --fix" + ] } -} +} \ No newline at end of file diff --git a/src/common/object/index.ts b/src/common/object/index.ts index 61c6bce2d..18f3bd2ea 100644 --- a/src/common/object/index.ts +++ b/src/common/object/index.ts @@ -23,6 +23,7 @@ import { putObjectTagging } from './putObjectTagging'; import { putSymlink } from './putSymlink'; import { restore } from './restore'; import { signatureUrl } from './signatureUrl'; +import { selectObject } from './selectObject'; export default { append, @@ -52,4 +53,5 @@ export default { putSymlink, restore, signatureUrl, + selectObject }; diff --git a/src/common/object/putSymlink.ts b/src/common/object/putSymlink.ts index 8e3649216..3ad1b709c 100644 --- a/src/common/object/putSymlink.ts +++ b/src/common/object/putSymlink.ts @@ -1,4 +1,3 @@ - import { objectName } from '../utils/objectName'; import { convertMetaToHeaders } from '../utils/convertMetaToHeaders'; import { escapeName } from '../utils/escapeName'; @@ -12,7 +11,12 @@ import { putSymlinkOptions } from '../../types/params'; * @param {{res}} */ -export async function putSymlink(this: any, name: string, targetName: string, options: putSymlinkOptions = {}) { +export async function putSymlink( + this: any, + name: string, + targetName: string, + options: putSymlinkOptions = {} +) { options.headers = options.headers || {}; targetName = escapeName(objectName(targetName)); convertMetaToHeaders(options.meta, options.headers); diff --git a/src/common/object/selectObject.ts b/src/common/object/selectObject.ts new file mode 100644 index 000000000..bb954677b --- /dev/null +++ b/src/common/object/selectObject.ts @@ -0,0 +1,157 @@ +import { obj2xml } from '../utils/obj2xml'; +import { Base64 } from 'js-base64'; +import unpackFrame from '../utils/unpackFrame'; + +type InputCSV = { + FileHeaderInfo?: string; + FieldDelimiter?: string; + RecordDelimiter?: string; + QuoteCharacter?: string; + CommentCharacter?: string; + Range?: string; + AllowQuotedRecordDelimiter?: boolean; +}; +type OutputCSV = { + RecordDelimiter?: string; + FieldDelimiter?: string; +}; + +type InputJSON = { + Type?: string; + Range?: string; + ParseJsonNumberAsString?: boolean; +}; + +type OutputJSON = { + RecordDelimiter?: string; +}; + +type InputSerialization = { + CompressionType?: string; + CSV?: InputCSV; + JSON?: InputJSON; +}; + +type OutputSerialization = { + CSV?: OutputCSV; + JSON?: OutputJSON; + OutputRawData?: boolean; + KeepAllColumns?: boolean; + EnablePayloadCrc?: boolean; + OutputHeader?: boolean; +}; + +type Other = { + SkipPartialDataRecord?: boolean; + MaxSkippedRecordsAllowed?: number; +}; + +type SelectRequest = { + Expression?: string; + InputSerialization?: InputSerialization; + OutputSerialization?: OutputSerialization; + Options?: Other; +}; + +type ParamsXMLObj = { + SelectRequest: SelectRequest; +}; + +const needToEncode: string[] = ['RecordDelimiter', 'FieldDelimiter', 'QuoteCharacter', 'CommentCharacter']; + +export async function selectObject(this: any, name: string, expression: string, grammar: string, options: any) { + if (!['json', 'csv'].includes(grammar.toLocaleLowerCase())) { + throw new Error('grammar must be json or csv'); + } + const opts = Object.assign({}, options); + + opts.subres = Object.assign({ 'x-oss-process': `${grammar}/select` }); + + let InputSerialization: InputSerialization; + let OutputSerialization: OutputSerialization; + let Options: Other; + + const paramsXMLObj: ParamsXMLObj = { + SelectRequest: { + Expression: Base64.encode(expression) + } + }; + + // CompressionType + if (opts.InputSerialization) { + opts.InputSerialization.CompressionType = opts.InputSerialization.CompressionType + ? opts.InputSerialization.CompressionType + : 'None'; + } + + // CSV + if (grammar.toLocaleLowerCase() === 'csv') { + // inputSerialization + if (opts.InputSerialization && opts.InputSerialization.CSV) { + Object.keys(opts.InputSerialization.CSV).forEach(i => { + if (needToEncode.includes(i)) { + opts.InputSerialization.CSV[i] = Base64.encode(opts.InputSerialization.CSV[i]); + } + }); + } + InputSerialization = Object.assign({}, opts.InputSerialization); + paramsXMLObj.SelectRequest.InputSerialization = InputSerialization; + + // OutputSerialization + if (opts.OutputSerialization && opts.OutputSerialization.CSV) { + Object.keys(opts.OutputSerialization.CSV).forEach(i => { + if (needToEncode.includes(i)) { + opts.OutputSerialization.CSV[i] = Base64.encode(opts.OutputSerialization.CSV[i]); + } + }); + } + OutputSerialization = Object.assign({}, opts.OutputSerialization); + paramsXMLObj.SelectRequest.OutputSerialization = OutputSerialization; + } + + // JSON + if (grammar.toLowerCase() === 'json') { + // InputSerialization + if (opts.InputSerialization && opts.InputSerialization.JSON) { + opts.InputSerialization.JSON.Type = opts.InputSerialization.JSON.Type + ? opts.InputSerialization.JSON.Type.toUpperCase() + : 'DOCUMENT'; + + opts.InputSerialization.JSON = Object.assign({}, opts.InputSerialization.JSON); + } + InputSerialization = Object.assign({}, opts.InputSerialization); + paramsXMLObj.SelectRequest.InputSerialization = InputSerialization; + + // OutputSerialization + if (opts.OutputSerialization && opts.OutputSerialization.JSON) { + if (opts.OutputSerialization.JSON.RecordDelimiter) { + opts.OutputSerialization.JSON.RecordDelimiter = Base64.encode(opts.OutputSerialization.JSON.RecordDelimiter); + } + } + OutputSerialization = Object.assign({}, opts.OutputSerialization); + paramsXMLObj.SelectRequest.OutputSerialization = OutputSerialization; + } + + // Options + if (opts.Other) { + Options = Object.assign({}, opts.Other); + paramsXMLObj.SelectRequest.Options = Options; + } + + const params = this._objectRequestParams('POST', name, opts); + params.content = obj2xml(paramsXMLObj); + params.mime = 'xml'; + params.successStatuses = [206]; + + const result = await this.request(params); + if (result.res.headers['x-oss-select-output-raw'] !== 'true') { + result.data = unpackFrame(result.data); + } else { + result.data = result.data.toString(); + } + + return { + res: result.res, + data: result.data + }; +} diff --git a/src/common/utils/unpackFrame.ts b/src/common/utils/unpackFrame.ts new file mode 100644 index 000000000..8d917c6db --- /dev/null +++ b/src/common/utils/unpackFrame.ts @@ -0,0 +1,58 @@ +type frameData = { + payload: payloadType; + version: number; + frameType: number; + payloadLength: number; + headerCheckSum: number; + payloadCheckSum: number; +}; + +type payloadType = { + offset: number; + frameType: string; + data?: string; + totalScannedBytes?: number; + httpStatusCode?: number; + errorMessage?: string; +}; + +// all integers in Frame are encoded in big endian and Version is currently 1. +export default (data: Buffer): frameData => { + const payload: payloadType = { frameType: '', offset: 0 }; + + const version = data.readUIntBE(0, 1); + const frameType = data.readUIntBE(1, 3); + const payloadLength = data.readUIntBE(4, 4); + const headerCheckSum = data.readUIntBE(8, 4); + const payloadCheckSum = data.readUIntBE(data.length - 4, 4); + + // payload offset + payload.offset = parseInt(data.slice(12, 20).toString('hex'), 16); + + /** + * Frame Type + * 8388609 - Data Frame + * 8388612 - Continuous Frame + * 8388613 - End Frame + */ + if (frameType === 8388609) { + payload.frameType = 'DataFrame'; + payload.data = data.slice(20, 12 + payloadLength).toString(); + } else if (frameType === 8388612) { + payload.frameType = 'ContinuousFrame'; + } else if (frameType === 8388613) { + payload.frameType = 'End Frame'; + payload.totalScannedBytes = data.readUIntBE(20, 8); + payload.httpStatusCode = data.readUIntBE(28, 4); + payload.errorMessage = data.slice(32, data.length - 4).toString(); + } + + return { + payload, + version, + frameType, + payloadCheckSum, + payloadLength, + headerCheckSum + }; +}; diff --git a/test/browser/browser.test.js b/test/browser/browser.test.js index 803eee1f5..7766031e7 100644 --- a/test/browser/browser.test.js +++ b/test/browser/browser.test.js @@ -1088,7 +1088,7 @@ describe('browser', () => { assert.equal(url.indexOf('http://www.aliyun.com/'), 0); }); - it.only('signatureUrl will should use refreshSTSToken', async () => { + it('signatureUrl will should use refreshSTSToken', async () => { const stsService = () => { return new Promise((resolve, reject) => { resolve({ diff --git a/test/exampleFile/example.csv b/test/exampleFile/example.csv new file mode 100644 index 000000000..e84e82c4b --- /dev/null +++ b/test/exampleFile/example.csv @@ -0,0 +1,102 @@ +Year,Industry_aggregation_NZSIOC,Industry_code_NZSIOC,Industry_name_NZSIOC,Units,Variable_code,Variable_name,Variable_category,Value +2020,Level 1,99999,All industries,Dollars (millions),H01,Total income,Financial performance,"733,258" +2020,Level 1,99999,All industries,Dollars (millions),H04,"Sales, government funding, grants and subsidies",Financial performance,"660,630" +2020,Level 1,99999,All industries,Dollars (millions),H05,"Interest, dividends and donations",Financial performance,"54,342" +2020,Level 1,99999,All industries,Dollars (millions),H07,Non-operating income,Financial performance,"18,285" +2020,Level 1,99999,All industries,Dollars (millions),H08,Total expenditure,Financial performance,"654,872" +2020,Level 1,99999,All industries,Dollars (millions),H09,Interest and donations,Financial performance,"32,730" +2020,Level 1,99999,All industries,Dollars (millions),H10,Indirect taxes,Financial performance,"7,509" +2020,Level 1,99999,All industries,Dollars (millions),H11,Depreciation,Financial performance,"26,821" +2020,Level 1,99999,All industries,Dollars (millions),H12,Salaries and wages paid,Financial performance,"119,387" +2020,Level 1,99999,All industries,Dollars (millions),H13,Redundancy and severance,Financial performance,305 +2020,Level 1,99999,All industries,Dollars (millions),H14,Salaries and wages to self employed commission agents,Financial performance,"1,756" +2020,Level 1,99999,All industries,Dollars (millions),H19,Purchases and other operating expenses,Financial performance,"450,394" +2020,Level 1,99999,All industries,Dollars (millions),H20,Non-operating expenses,Financial performance,"16,274" +2020,Level 1,99999,All industries,Dollars (millions),H21,Opening stocks,Financial performance,"68,109" +2020,Level 1,99999,All industries,Dollars (millions),H22,Closing stocks,Financial performance,"69,123" +2020,Level 1,99999,All industries,Dollars (millions),H23,Surplus before income tax,Financial performance,"79,400" +2020,Level 1,99999,All industries,Dollars (millions),H24,Total assets,Financial position,"2,396,143" +2020,Level 1,99999,All industries,Dollars (millions),H25,Current assets,Financial position,"665,643" +2020,Level 1,99999,All industries,Dollars (millions),H26,Fixed tangible assets,Financial position,"562,646" +2020,Level 1,99999,All industries,Dollars (millions),H29,Other assets,Financial position,"1,167,854" +2020,Level 1,99999,All industries,Dollars (millions),H30,Total equity and liabilities,Financial position,"2,396,143" +2020,Level 1,99999,All industries,Dollars (millions),H31,Shareholders funds or owners equity,Financial position,"775,466" +2020,Level 1,99999,All industries,Dollars (millions),H32,Current liabilities,Financial position,"866,332" +2020,Level 1,99999,All industries,Dollars (millions),H33,Other liabilities,Financial position,"754,345" +2020,Level 1,99999,All industries,Dollars,H34,Total income per employee count,Financial ratios,"385,100" +2020,Level 1,99999,All industries,Dollars,H35,Surplus per employee count,Financial ratios,"41,700" +2020,Level 1,99999,All industries,Percentage,H36,Current ratio,Financial ratios,77 +2020,Level 1,99999,All industries,Percentage,H37,Quick ratio,Financial ratios,69 +2020,Level 1,99999,All industries,Percentage,H39,Return on equity,Financial ratios,10 +2020,Level 1,99999,All industries,Percentage,H40,Return on total assets,Financial ratios,3 +2020,Level 1,99999,All industries,Percentage,H41,Liabilities structure,Financial ratios,32 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H01,Total income,Financial performance,"5,099" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H04,"Sales, government funding, grants and subsidies",Financial performance,"4,721" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H05,"Interest, dividends and donations",Financial performance,164 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H07,Non-operating income,Financial performance,215 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H08,Total expenditure,Financial performance,"4,165" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H09,Interest and donations,Financial performance,207 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H10,Indirect taxes,Financial performance,36 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H11,Depreciation,Financial performance,264 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H12,Salaries and wages paid,Financial performance,836 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H13,Redundancy and severance,Financial performance,0 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H19,Purchases and other operating expenses,Financial performance,"2,800" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H20,Non-operating expenses,Financial performance,23 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H21,Opening stocks,Financial performance,374 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H22,Closing stocks,Financial performance,365 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H23,Surplus before income tax,Financial performance,924 +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H24,Total assets,Financial position,"16,005" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H25,Current assets,Financial position,"2,798" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H26,Fixed tangible assets,Financial position,"10,532" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H29,Other assets,Financial position,"2,675" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H30,Total equity and liabilities,Financial position,"16,005" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H31,Shareholders funds or owners equity,Financial position,"7,653" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H32,Current liabilities,Financial position,"3,480" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars (millions),H33,Other liabilities,Financial position,"4,872" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars,H34,Total income per employee count,Financial ratios,"195,800" +2020,Level 4,AA111,Horticulture and Fruit Growing,Dollars,H35,Surplus per employee count,Financial ratios,"35,500" +2020,Level 4,AA111,Horticulture and Fruit Growing,Percentage,H36,Current ratio,Financial ratios,80 +2020,Level 4,AA111,Horticulture and Fruit Growing,Percentage,H37,Quick ratio,Financial ratios,70 +2020,Level 4,AA111,Horticulture and Fruit Growing,Percentage,H39,Return on equity,Financial ratios,12 +2020,Level 4,AA111,Horticulture and Fruit Growing,Percentage,H40,Return on total assets,Financial ratios,6 +2020,Level 4,AA111,Horticulture and Fruit Growing,Percentage,H41,Liabilities structure,Financial ratios,48 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H01,Total income,Financial performance,"5,099" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H04,"Sales, government funding, grants and subsidies",Financial performance,"4,721" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H05,"Interest, dividends and donations",Financial performance,164 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H07,Non-operating income,Financial performance,215 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H08,Total expenditure,Financial performance,"4,165" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H09,Interest and donations,Financial performance,207 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H10,Indirect taxes,Financial performance,36 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H11,Depreciation,Financial performance,264 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H12,Salaries and wages paid,Financial performance,836 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H13,Redundancy and severance,Financial performance,0 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H19,Purchases and other operating expenses,Financial performance,"2,800" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H20,Non-operating expenses,Financial performance,23 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H21,Opening stocks,Financial performance,374 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H22,Closing stocks,Financial performance,365 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H23,Surplus before income tax,Financial performance,924 +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H24,Total assets,Financial position,"16,005" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H25,Current assets,Financial position,"2,798" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H26,Fixed tangible assets,Financial position,"10,532" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H29,Other assets,Financial position,"2,675" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H30,Total equity and liabilities,Financial position,"16,005" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H31,Shareholders funds or owners equity,Financial position,"7,653" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H32,Current liabilities,Financial position,"3,480" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars (millions),H33,Other liabilities,Financial position,"4,872" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars,H34,Total income per employee count,Financial ratios,"195,800" +2020,Level 3,AA11,Horticulture and Fruit Growing,Dollars,H35,Surplus per employee count,Financial ratios,"35,500" +2020,Level 3,AA11,Horticulture and Fruit Growing,Percentage,H36,Current ratio,Financial ratios,80 +2020,Level 3,AA11,Horticulture and Fruit Growing,Percentage,H37,Quick ratio,Financial ratios,70 +2020,Level 3,AA11,Horticulture and Fruit Growing,Percentage,H39,Return on equity,Financial ratios,12 +2020,Level 3,AA11,Horticulture and Fruit Growing,Percentage,H40,Return on total assets,Financial ratios,6 +2020,Level 3,AA11,Horticulture and Fruit Growing,Percentage,H41,Liabilities structure,Financial ratios,48 +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H01,Total income,Financial performance,"12,174" +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H04,"Sales, government funding, grants and subsidies",Financial performance,"11,437" +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H05,"Interest, dividends and donations",Financial performance,139 +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H07,Non-operating income,Financial performance,598 +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H08,Total expenditure,Financial performance,"10,449" +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H09,Interest and donations,Financial performance,684 +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H10,Indirect taxes,Financial performance,188 +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H11,Depreciation,Financial performance,582 +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H12,Salaries and wages paid,Financial performance,"1,088" +2020,Level 4,AA121,"Sheep, Beef Cattle and Grain Farming",Dollars (millions),H13,Redundancy and severance,Financial performance,0 \ No newline at end of file diff --git a/test/exampleFile/example.json b/test/exampleFile/example.json new file mode 100644 index 000000000..86d15877d --- /dev/null +++ b/test/exampleFile/example.json @@ -0,0 +1,28 @@ +{ + "contacts": [ + { + "firstName": "John", + "lastName": "Smith", + "address": { + "streetAddress": "21 2nd Street", + "city": "New York", + "state": "NY", + "postalCode": "10021-3100" + }, + "phoneNumbers": [ + { + "type": "home", + "number": "212 555-1234" + }, + { + "type": "office", + "number": "646 555-4567" + }, + { + "type": "mobile", + "number": "123 456-7890" + } + ] + } + ] +} diff --git a/test/node/cleanAllBucket.js b/test/node/cleanAllBucket.js index 9995227b2..354511c0f 100644 --- a/test/node/cleanAllBucket.js +++ b/test/node/cleanAllBucket.js @@ -2,7 +2,7 @@ const utils = require('./utils'); const config = require('../config').oss; const OSS = require('../..'); -const store = OSS(config); +const store = new OSS(config); const interval = new Date().getTime() - 24 * 60 * 60 * 7 * 1000; store.listBuckets().then(r => { diff --git a/test/node/object.test.js b/test/node/object.test.js index 33af55b2d..0e3768091 100644 --- a/test/node/object.test.js +++ b/test/node/object.test.js @@ -2618,4 +2618,72 @@ describe('test/object.test.js', () => { assert.equal(info.meta.b, latin1_content); }); }); + + describe('selectObject()', () => { + before(async () => { + await store.put('example.csv', path.join(__dirname, '../exampleFile/example.csv')); + await store.put('example.json', path.join(__dirname, '../exampleFile/example.json')); + }); + + it('csv should return 206', async () => { + const result = await store.selectObject('example.csv', 'select * from ossobject limit 2', 'csv'); + assert.equal( + result.data.payload.data.toString(), + 'Year,Industry_aggregation_NZSIOC,Industry_code_NZSIOC,Industry_name_NZSIOC,Units,Variable_code,Variable_name,Variable_category,Value\r\n' + ); + assert.equal(result.res.status, 206); + }); + + it('json should return 206', async () => { + const result = await store.selectObject('example.json', 'select address from ossobject.contacts[*]', 'json'); + const jsonData = JSON.parse(result.data.payload.data); + assert.equal(typeof jsonData, 'object'); + assert.equal(jsonData.address.city, 'New York'); + assert.equal(result.res.status, 206); + }); + + it('json add options', async () => { + const result = await store.selectObject('example.json', 'select address from ossobject.contacts[*]', 'json', { + InputSerialization: { + CompressionType: 'NONE', + JSON: { + Type: 'LINES', + ParseJsonNumberAsString: true + } + }, + OutputSerialization: { + JSON: { + OutputRawData: false, + EnablePayloadCrc: true + } + } + }); + assert.equal( + result.data.payload.data, + '{"address":{"streetAddress":"21 2nd Street","city":"New York","state":"NY","postalCode":"10021-3100"}}\n' + ); + assert.equal(result.res.status, 206); + }); + + it('csv add options', async () => { + const result = await store.selectObject('example.csv', 'select * from ossobject limit 2', 'csv', { + InputSerialization: { + CompressionType: 'NONE', + CSV: { + FileHeaderInfo: 'NONE', + AllowQuotedRecordDelimiter: false + } + }, + OutputSerialization: { + KeepAllColumns: true, + OutputRawData: false + } + }); + assert.equal( + result.data.payload.data.toString(), + 'Year,Industry_aggregation_NZSIOC,Industry_code_NZSIOC,Industry_name_NZSIOC,Units,Variable_code,Variable_name,Variable_category,Value\r\n' + ); + assert.equal(result.res.status, 206); + }); + }); });