From 123137d54b78acb7f20bb17cade0de90b5651791 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 4 May 2023 17:12:29 -0700 Subject: [PATCH] feat: get provisions working implement a test that represents my current best understanding of how provisions should work for now. --- package-lock.json | 31 +++++++-- upload-api/package.json | 1 + upload-api/tables/provisions.js | 29 +++++--- upload-api/test/service/provisions.test.js | 80 +++++++++------------- 4 files changed, 82 insertions(+), 59 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1c524d08..cfca0913 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7855,7 +7855,8 @@ "node_modules/@balena/dockerignore": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", - "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true }, "node_modules/@es-joy/jsdoccomment": { "version": "0.36.1", @@ -9854,6 +9855,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", + "dev": true, "engines": { "node": ">= 4.0.0" } @@ -10375,7 +10377,8 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true }, "node_modules/base64-js": { "version": "1.5.1", @@ -10542,6 +10545,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -11192,7 +11196,8 @@ "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true }, "node_modules/concordance": { "version": "5.0.4", @@ -13504,6 +13509,7 @@ "version": "9.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, "dependencies": { "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", @@ -13848,7 +13854,8 @@ "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true }, "node_modules/grapheme-splitter": { "version": "1.0.4", @@ -14079,6 +14086,7 @@ "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", + "dev": true, "engines": { "node": ">= 4" } @@ -14772,6 +14780,7 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, "dependencies": { "universalify": "^2.0.0" }, @@ -15715,6 +15724,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -16999,6 +17009,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dev": true, "engines": { "node": ">=6" } @@ -17550,6 +17561,7 @@ "version": "7.4.0", "resolved": "https://registry.npmjs.org/semver/-/semver-7.4.0.tgz", "integrity": "sha512-RgOxM8Mw+7Zus0+zcLEUn8+JfoLpj/huFTItQy2hsM4khuC1HYRDp0cU482Ewn/Fcy6bCjufD8vAj7voC66KQw==", + "dev": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -17942,6 +17954,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/streaming-iterables": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/streaming-iterables/-/streaming-iterables-7.1.0.tgz", + "integrity": "sha512-t2KmiLVhqafTRqGefD98s5XAMskfkfprr/BTzPIZz0kWB23iyR7XUkY03yjUf4aZpAuuV2/2SUOVri3LgKuOKw==", + "engines": { + "node": ">=14" + } + }, "node_modules/streamroller": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-3.1.5.tgz", @@ -18728,6 +18748,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true, "engines": { "node": ">= 10.0.0" } @@ -19094,6 +19115,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==", + "dev": true, "engines": { "node": ">= 14" } @@ -19426,6 +19448,7 @@ "@web3-storage/w3infra-ucan-invocation": "*", "multiformats": "^11.0.1", "prom-client": "^14.2.0", + "streaming-iterables": "^7.1.0", "uint8arrays": "^4.0.2" }, "devDependencies": { diff --git a/upload-api/package.json b/upload-api/package.json index bbd44496..78631015 100644 --- a/upload-api/package.json +++ b/upload-api/package.json @@ -29,6 +29,7 @@ "@web3-storage/w3infra-ucan-invocation": "*", "multiformats": "^11.0.1", "prom-client": "^14.2.0", + "streaming-iterables": "^7.1.0", "uint8arrays": "^4.0.2" }, "devDependencies": { diff --git a/upload-api/tables/provisions.js b/upload-api/tables/provisions.js index a9980222..fcfb6592 100644 --- a/upload-api/tables/provisions.js +++ b/upload-api/tables/provisions.js @@ -80,16 +80,27 @@ export function useProvisionsTable (dynamoDb, tableName, services) { provider: item.provider, sponsor: item.account, } - const hasProvider = await hasStorageProvider(dynamoDb, tableName, row.consumer) - if (hasProvider) { - return new ConflictError({ - message: `Space ${row.consumer} cannot be provisioned with ${row.provider}: it already has a provider` - }) + try { + await dynamoDb.send(new PutItemCommand({ + TableName: tableName, + Item: marshall(row), + ConditionExpression: `attribute_not_exists(consumer) OR ((cid = :cid) AND (consumer = :consumer) AND (provider = :provider) AND (sponsor = :sponsor))`, + ExpressionAttributeValues: { + ':cid': { 'S': row.cid }, + ':consumer': { 'S': row.consumer }, + ':provider': { 'S': row.provider }, + ':sponsor': { 'S': row.sponsor } + } + })) + } catch (error) { + if (error instanceof Error && error.message === 'The conditional request failed') { + return new ConflictError({ + message: `Space ${row.consumer} cannot be provisioned with ${row.provider}: it already has a provider` + }) + } else { + throw error + } } - await dynamoDb.send(new PutItemCommand({ - TableName: tableName, - Item: marshall(row) - })) return {} }, diff --git a/upload-api/test/service/provisions.test.js b/upload-api/test/service/provisions.test.js index a4fcb9cb..29dd93f6 100644 --- a/upload-api/test/service/provisions.test.js +++ b/upload-api/test/service/provisions.test.js @@ -7,7 +7,7 @@ import { } from '../helpers/resources.js' import { useProvisionsTable } from '../../tables/provisions.js' import * as principal from '@ucanto/principal' -import {Signer} from '@ucanto/principal/ed25519' +import { Signer } from '@ucanto/principal/ed25519' import { Provider } from '@web3-storage/capabilities' import { CID } from 'multiformats' import { provisionsTableProps } from '../../tables/index.js' @@ -23,7 +23,6 @@ test.before(async (t) => { }) /** - * TODO: migrate back to test in w3up access-api/test/provisions.test.js */ test('should persist provisions', async (t) => { const { dynamo, service } = t.context @@ -32,37 +31,33 @@ test('should persist provisions', async (t) => { await createTable(dynamo, provisionsTableProps), [service.did()] ) - const count = 2 + Math.round(Math.random() * 3) const spaceA = await principal.ed25519.generate() - const [firstProvision, ...lastProvisions] = await Promise.all( - Array.from({ length: count }).map(async () => { - const issuerKey = await principal.ed25519.generate() - const issuer = issuerKey.withDID('did:mailto:example.com:foo') - const invocation = await Provider.add - .invoke({ - issuer, - audience: issuer, - with: issuer.did(), - nb: { - consumer: spaceA.did(), - provider: 'did:web:web3.storage:providers:w3up-alpha', - }, - }) - .delegate() - /** @type {import('../../access-types').Provision<'did:web:web3.storage:providers:w3up-alpha'>} */ - const provision = { - invocation, - space: spaceA.did(), + const issuerKey = await principal.ed25519.generate() + const issuer = issuerKey.withDID('did:mailto:example.com:foo') + const invocation = await Provider.add + .invoke({ + issuer, + audience: issuer, + with: issuer.did(), + nb: { + consumer: spaceA.did(), provider: 'did:web:web3.storage:providers:w3up-alpha', - account: issuer.did(), - } - return provision + }, }) - ) + .delegate() + /** @type {import('../../access-types').Provision<'did:web:web3.storage:providers:w3up-alpha'>} */ + const provision = { + invocation, + space: spaceA.did(), + provider: 'did:web:web3.storage:providers:w3up-alpha', + account: issuer.did(), + } + + t.deepEqual(await storage.count(), BigInt(0)) - // TODO: I think this should fail because all of the provisions in lastProvisions have the same space and provider?! - await Promise.all(lastProvisions.map((p) => storage.put(p))) - t.deepEqual(await storage.count(), BigInt(lastProvisions.length)) + const result = await storage.put(provision) + t.falsy(result.error, 'adding a provision failed') + t.deepEqual(await storage.count(), BigInt(1)) const spaceHasStorageProvider = await storage.hasStorageProvider( spaceA.did() @@ -70,27 +65,20 @@ test('should persist provisions', async (t) => { t.deepEqual(spaceHasStorageProvider, true) // ensure no error if we try to store same provision twice - // all of lastProvisions are duplicate, but firstProvision is new so that should be added - await storage.put(lastProvisions[0]) - await storage.put(firstProvision) - t.deepEqual(await storage.count(), BigInt(count)) + const dupeResult = await storage.put(provision) + t.falsy(dupeResult.error, 'putting the same provision twice did not succeed') + t.deepEqual(await storage.count(), BigInt(1)) - // but if we try to store the same provision (same `cid`) with different - // fields derived from invocation, it should error - const modifiedFirstProvision = { - ...firstProvision, - space: /** @type {const} */ ('did:key:foo'), - account: /** @type {const} */ ('did:mailto:example.com:foo'), - // note this type assertion is wrong, but useful to set up the test + const modifiedProvision = { + ...provision, provider: /** @type {import('@ucanto/interface').DID<'web'>} */ ( 'did:provider:foo' ), } - const result = await storage.put(modifiedFirstProvision) - t.is( - result.error && result.name, - 'ConflictError', - 'cannot put with same cid but different derived fields' - ) + + // ensure no error if we try to store a provision for a consumer that already has a provider + const modifiedResult = await storage.put(modifiedProvision) + t.truthy(modifiedResult.error, 'provisioning for a consumer who already has a provider succeeded and should not have!') + t.deepEqual(await storage.count(), BigInt(1)) })