From 92acaf3a60459165f27de34f978c26864bc86932 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 27 Mar 2023 16:56:39 +0100 Subject: [PATCH 1/3] feat: add local ipfs pinning service api adaptor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `w3 ps` command to run an http server on localhost to allow you to use the IPFS pinning service api with w3up and get ucan auth in pinning service mode. ```sh $ w3 ps ⁂ IPFS Pinning Service on http://127.0.0.1:1337 $ ipfs pin remote service add w3 'http://127.0.0.1:1337' 'did:key:z6MkvqaczHouddZ91gWNsuy2QFm419WFodXbVjBJykvzw1eK' $ ipfs pin remote add --service w3 ``` License: MIT Signed-off-by: Oli Evans --- bin.js | 10 +++++ pin.js | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 pin.js diff --git a/bin.js b/bin.js index b0b0d35..c24c94c 100755 --- a/bin.js +++ b/bin.js @@ -26,6 +26,9 @@ import { uploadAdd, uploadList } from './can.js' +import { + startPinService +} from './pin.js' const cli = sade('w3') @@ -147,6 +150,13 @@ cli.command('can upload ls') .option('--pre', 'If true, return the page of results preceding the cursor') .action(uploadList) +cli.command('ps') + .describe('Start IPFS pinning service server') + .option('--port', 'Override the default port to listen on (:1337)', 1337) + .option('--host', 'Override the default host to listen on (127.0.0.1)', '127.0.0.1') + .option('--key', 'Override the default bearer token for api ') + .action(startPinService) + // show help text if no command provided cli.command('help [cmd]', 'Show help text', { default: true }) .action(cmd => { diff --git a/pin.js b/pin.js new file mode 100644 index 0000000..2f35084 --- /dev/null +++ b/pin.js @@ -0,0 +1,121 @@ +import { CID } from 'multiformats' +import http from 'node:http' +import { getPkg, getClient } from './lib.js' + +/** + * a pinning service api on your localhost + * + * ## Example + * w3 ps --port 1337 + */ +export async function startPinService ({ port, host = '127.0.0.1', key }) { + const pkg = getPkg() + const pinCache = new Map() + const client = await getClient() + const whoami = client.agent().did() + const token = key ?? whoami + const api = http.createServer(async (req, res) => { + if (req.headers.authorization !== `Bearer ${token}`) { + return send({ res, status: 401, body: { error: { reason: 'Unauthorized; access token is missing or invalid' } } }) + } + const { pathname } = new URL(req.url, `http://${req.headers.host}`) + if (pathname === '/' || pathname === '') { + return send({ res, body: { service: 'w3', version: pkg.version } }) + } + if (req.method === 'POST' && pathname === '/pins') { + const body = await getJsonBody(req) + const pinStatus = await addPin({ ...body, client }) + pinCache.set(pinStatus.requestid, pinStatus) + return send({ res, body: pinStatus }) + } + if (req.method === 'GET' && pathname.startsWith('/pins/')) { + const requestid = pathname.split('/').at(2) + const pinStatus = pinCache.get(requestid) + if (pinStatus) { + return send({ res, body: pinStatus }) + } + return send({ res, status: 404, body: { error: { reason: 'Not Found', details: requestid } } }) + } + return send({ res, status: 501, body: { error: { reason: 'Not Implmented', details: `${req.method} ${pathname}` } } }) + }) + api.listen(port, host, () => { + console.log(`⁂ IPFS Pinning Service on http://127.0.0.1:1337 + +## Add w3 as a remote +$ ipfs pin remote service add w3 'http://${host}:${port}' '${token}' + +## Pin to w3 +$ ipfs pin remote add --service w3 + +## Waiting for requests`) + }) +} + +/** + * @param {object} config + * @param {import('@web3-storage/w3up-client').Client} confg.client + * @param {string} config.cid + * @param {string} [config.ipfsGatewayUrl] + * @param {AbortSignal} [config.signal] + */ +export async function addPin ({ client, cid, ipfsGatewayUrl = 'http://127.0.0.1:8080', signal }) { + const rootCID = CID.parse(cid) + const ipfsUrl = new URL(`/ipfs/${cid}?format=car`, ipfsGatewayUrl, { signal }) + const res = await fetch(ipfsUrl) + const storedCID = await client.uploadCAR({ stream: () => res.body }, { + onShardStored: (car) => console.log(`${new Date().toISOString()} ${car.cid} shard stored`), + rootCID, + signal + + }) + console.log(`${new Date().toISOString()} ${storedCID} uploaded`) + return { + // we use cid as requestid to avoid needing to track extra state + requestid: storedCID.toString(), + status: 'pinned', + created: new Date().toISOString(), + pin: { + cid: storedCID.toString() + }, + delgates: [] + } +} + +/** + * @param {object} config + * @param {http.OutgoingMessage} config.res + * @param {object} config.body + * @param {number} [config.status] + * @param {string} [config.contentType] + */ +function send ({ res, body, status = 200, contentType = 'application/json' }) { + res.setHeader('Content-Type', 'application/json') + res.writeHead(status) + const str = contentType === 'application/json' ? JSON.stringify(body) : body + res.end(str) +} + +/** + * @param {http.IncomingMessage} req + */ +export async function getJsonBody (req) { + const contentlength = parseInt(req.headers['content-length'] || 0, 10) + if (contentlength > 100 * 1024) { + throw new Error('Request body too large') + } + const contentType = req.headers['content-type'] + if (contentType !== 'application/json') { + throw new Error('Request body must be be content-type: application/json') + } + let body = '' + for await (const chonk of req) { + body += chonk + if (Buffer.byteLength(body, 'utf-8') > contentlength) { + throw new Error('Request body size exceeds specfied content-length') + } + } + if (Buffer.byteLength(body, 'utf-8') !== contentlength) { + throw new Error('Request body size does not match specified content-length') + } + return JSON.parse(body) +} From 761c8d1014db0e56c33216c5b630ff2bdd76aa6d Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Wed, 13 Sep 2023 15:39:18 +0100 Subject: [PATCH 2/3] fix: respond with error dont throw License: MIT Signed-off-by: Oli Evans --- bin.js | 2 +- package-lock.json | 14 +++++ package.json | 1 + pin.js | 133 ++++++++++++++++++++++++++++++++++++---------- 4 files changed, 120 insertions(+), 30 deletions(-) diff --git a/bin.js b/bin.js index cc5c76a..f7d2764 100755 --- a/bin.js +++ b/bin.js @@ -160,7 +160,7 @@ cli.command('can upload ls') cli.command('ps') .describe('Start IPFS pinning service server') - .option('--port', 'Override the default port to listen on (:1337)', 1337) + .option('--port', 'Override the default port to listen on (:1337)', '1337') .option('--host', 'Override the default host to listen on (127.0.0.1)', '127.0.0.1') .option('--key', 'Override the default bearer token for api ') .action(startPinService) diff --git a/package-lock.json b/package-lock.json index 0bda1bb..fc50225 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@web3-storage/w3up-client": "^8.0.2", "chalk": "^5.3.0", "files-from-path": "^1.0.0", + "mnemonist": "^0.39.5", "open": "^8.4.0", "ora": "^6.1.2", "pretty-tree": "^1.0.0", @@ -4419,6 +4420,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mnemonist": { + "version": "0.39.5", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.5.tgz", + "integrity": "sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ==", + "dependencies": { + "obliterator": "^2.0.1" + } + }, "node_modules/mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -4861,6 +4870,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/obliterator": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.4.tgz", + "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ==" + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", diff --git a/package.json b/package.json index 6f61711..ed867a0 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "@web3-storage/w3up-client": "^8.0.2", "chalk": "^5.3.0", "files-from-path": "^1.0.0", + "mnemonist": "^0.39.5", "open": "^8.4.0", "ora": "^6.1.2", "pretty-tree": "^1.0.0", diff --git a/pin.js b/pin.js index 2f35084..a3b2ba4 100644 --- a/pin.js +++ b/pin.js @@ -1,3 +1,4 @@ +import LRUMap from 'mnemonist/lru-map' import { CID } from 'multiformats' import http from 'node:http' import { getPkg, getClient } from './lib.js' @@ -7,10 +8,16 @@ import { getPkg, getClient } from './lib.js' * * ## Example * w3 ps --port 1337 + * + * @param {object} config + * @param {string} config.port + * @param {string} config.host + * @param {string} config.key */ -export async function startPinService ({ port, host = '127.0.0.1', key }) { +export async function startPinService ({ port = '1337', host = '127.0.0.1', key }) { const pkg = getPkg() - const pinCache = new Map() + /** @type LRUMap */ + const pinCache = new LRUMap(100_000) const client = await getClient() const whoami = client.agent().did() const token = key ?? whoami @@ -18,27 +25,37 @@ export async function startPinService ({ port, host = '127.0.0.1', key }) { if (req.headers.authorization !== `Bearer ${token}`) { return send({ res, status: 401, body: { error: { reason: 'Unauthorized; access token is missing or invalid' } } }) } - const { pathname } = new URL(req.url, `http://${req.headers.host}`) + const { pathname } = new URL(req.url ?? '', `http://${req.headers.host}`) if (pathname === '/' || pathname === '') { return send({ res, body: { service: 'w3', version: pkg.version } }) } if (req.method === 'POST' && pathname === '/pins') { - const body = await getJsonBody(req) - const pinStatus = await addPin({ ...body, client }) + const reqBody = await getJsonBody(req) + if (reqBody.error) { + return send({ status: 400, res, body: reqBody }) + } + const { cid } = reqBody + const pinStatus = await addPin({ cid, client }) + if (pinStatus.error) { + return send({ status: 400, res, body: pinStatus }) + } pinCache.set(pinStatus.requestid, pinStatus) return send({ res, body: pinStatus }) } if (req.method === 'GET' && pathname.startsWith('/pins/')) { const requestid = pathname.split('/').at(2) + if (!requestid) { + return send({ res, status: 404, body: { error: { reason: 'Not Found', details: requestid } } }) + } const pinStatus = pinCache.get(requestid) if (pinStatus) { return send({ res, body: pinStatus }) } return send({ res, status: 404, body: { error: { reason: 'Not Found', details: requestid } } }) } - return send({ res, status: 501, body: { error: { reason: 'Not Implmented', details: `${req.method} ${pathname}` } } }) + return send({ res, status: 501, body: { error: { reason: 'Not Implemented', details: `${req.method} ${pathname}` } } }) }) - api.listen(port, host, () => { + api.listen(parseInt(port, 10), host, () => { console.log(`⁂ IPFS Pinning Service on http://127.0.0.1:1337 ## Add w3 as a remote @@ -53,31 +70,85 @@ $ ipfs pin remote add --service w3 /** * @param {object} config - * @param {import('@web3-storage/w3up-client').Client} confg.client + * @param {import('@web3-storage/w3up-client').Client} config.client * @param {string} config.cid * @param {string} [config.ipfsGatewayUrl] * @param {AbortSignal} [config.signal] + * @returns {Promise} */ export async function addPin ({ client, cid, ipfsGatewayUrl = 'http://127.0.0.1:8080', signal }) { - const rootCID = CID.parse(cid) - const ipfsUrl = new URL(`/ipfs/${cid}?format=car`, ipfsGatewayUrl, { signal }) - const res = await fetch(ipfsUrl) - const storedCID = await client.uploadCAR({ stream: () => res.body }, { - onShardStored: (car) => console.log(`${new Date().toISOString()} ${car.cid} shard stored`), + let rootCID + let ipfsUrl + /** @type Response | undefined */ + let res + + try { + rootCID = CID.parse(cid) + } catch (err) { + return errorResponse(`Failed to parse ${cid} as a CID`) + } + + try { + ipfsUrl = new URL(`/ipfs/${cid}?format=car`, ipfsGatewayUrl) + } catch (err) { + return errorResponse(`Failed to parse ${ipfsGatewayUrl} /ipfs/${cid}?format=car`) + } + + try { + res = await fetch(ipfsUrl, { signal }) + } catch (err) { + return errorResponse(`Error fetching CAR from IPFS ${ipfsUrl}`, err.message ?? err) + } + + if (!res.ok) { + return errorResponse(`http status ${res.status} fetching CAR from IPFS ${ipfsUrl}`) + } + + let shardCount = 0 + let byteCount = 0 + + await client.uploadCAR({ stream: () => res.body }, { + onShardStored: (meta) => { shardCount++; byteCount += meta.size }, rootCID, signal - }) - console.log(`${new Date().toISOString()} ${storedCID} uploaded`) + + console.log(`${new Date().toISOString()} uploaded ${cid} (shards: ${shardCount}, total bytes sent: ${byteCount} )`) + return pinResponse(cid, 'pinned') +} + +/** + * @typedef {{requestid: string, status: 'pinned' | 'failed', created: string, pin: { cid: string }, delegates: [], error?: undefined }} PinStatus + * + * @param {string} cidStr + * @param {'pinned' | 'failed'} status + * @returns {PinStatus} + */ +function pinResponse (cidStr, status = 'pinned') { return { - // we use cid as requestid to avoid needing to track extra state - requestid: storedCID.toString(), - status: 'pinned', + requestid: cidStr, + status, created: new Date().toISOString(), pin: { - cid: storedCID.toString() + cid: cidStr }, - delgates: [] + delegates: [] + } +} + +/** + * @typedef {{error: { reason: string, details: string }}} ErrorStatus + * + * @param {string} details + * @returns {ErrorStatus} + */ +function errorResponse (details) { + console.error(`${new Date().toISOString()} Error: ${details}`) + return { + error: { + reason: 'BAD_REQUEST', + details + } } } @@ -99,23 +170,27 @@ function send ({ res, body, status = 200, contentType = 'application/json' }) { * @param {http.IncomingMessage} req */ export async function getJsonBody (req) { - const contentlength = parseInt(req.headers['content-length'] || 0, 10) - if (contentlength > 100 * 1024) { - throw new Error('Request body too large') + const contentLength = parseInt(req.headers['content-length'] || '0', 10) + if (contentLength > 100 * 1024) { + return errorResponse('Request body too large') } const contentType = req.headers['content-type'] if (contentType !== 'application/json') { - throw new Error('Request body must be be content-type: application/json') + return errorResponse('Request body must be be content-type: application/json') } let body = '' for await (const chonk of req) { body += chonk - if (Buffer.byteLength(body, 'utf-8') > contentlength) { - throw new Error('Request body size exceeds specfied content-length') + if (Buffer.byteLength(body, 'utf-8') > contentLength) { + return errorResponse('Request body size exceeds specfied content-length') } } - if (Buffer.byteLength(body, 'utf-8') !== contentlength) { - throw new Error('Request body size does not match specified content-length') + if (Buffer.byteLength(body, 'utf-8') !== contentLength) { + return errorResponse('Request body size does not match specified content-length') + } + try { + return JSON.parse(body) + } catch (err) { + return errorResponse('Request body is not valid json', err.message ?? err) } - return JSON.parse(body) } From 0aad8144c9830490566ad222a533baf29044cbee Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Wed, 13 Sep 2023 15:41:17 +0100 Subject: [PATCH 3/3] fix: bad import License: MIT Signed-off-by: Oli Evans --- pin.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pin.js b/pin.js index a3b2ba4..972256a 100644 --- a/pin.js +++ b/pin.js @@ -1,4 +1,4 @@ -import LRUMap from 'mnemonist/lru-map' +import LRUMap from 'mnemonist/lru-map.js' import { CID } from 'multiformats' import http from 'node:http' import { getPkg, getClient } from './lib.js'