From cb8c2c943a31d6e8bc171e7fe83b1db967fac349 Mon Sep 17 00:00:00 2001 From: stobiewan Date: Sat, 16 Apr 2022 19:46:25 +1200 Subject: [PATCH 1/2] handle ipfs connection in view fallback to embedded if local doesn't work add buffer webpack plugin --- dist/index.html | 3 --- dmap.js | 25 +++++-------------------- package.json | 7 +++++-- view/app.js | 34 +++++++++++++++++++++++++--------- webpack.config.js | 7 +++++++ 5 files changed, 42 insertions(+), 34 deletions(-) diff --git a/dist/index.html b/dist/index.html index cd3dc8d..884dbbc 100644 --- a/dist/index.html +++ b/dist/index.html @@ -24,9 +24,6 @@

dmap

- - - diff --git a/dmap.js b/dmap.js index 69f9fa7..8462126 100644 --- a/dmap.js +++ b/dmap.js @@ -1,9 +1,6 @@ const ebnf = require('ebnf') -const multiformats = require('multiformats') - const ethers = require('ethers') -const { b32 } = require('minihat'); - +const multiformats = require('multiformats') const abi = require('./artifacts/core/dmap.sol/Dmap.json').abi const dmap_i = new ethers.utils.Interface(abi) @@ -91,11 +88,11 @@ lib.walk = async (dmap, path) => { if (zone === '0x' + '00'.repeat(20)) { fail(`zero register`) } - const fullname = '0x' + lib._strToHex(step.name) + '00'.repeat(32-step.name.length); + const fullname = '0x' + Buffer.from(step.name).toString('hex') + '00'.repeat(32-step.name.length); [meta, data] = await lib.get(dmap, zone, fullname) if (step.locked) { need(ctx.locked, `Encountered ':' in unlocked subpath`) - need((lib._hexToArrayBuffer(meta)[0] & lib.FLAG_LOCK) !== 0, `Entry is not locked`) + need((Buffer.from(meta.slice(2), 'hex')[0] & lib.FLAG_LOCK) !== 0, `Entry is not locked`) ctx.locked = true } ctx.locked = step.locked @@ -118,8 +115,8 @@ lib.prepareCID = (cidStr, lock) => { } lib.unpackCID = (metaStr, dataStr) => { - const meta = lib._hexToArrayBuffer(metaStr) - const data = lib._hexToArrayBuffer(dataStr) + const meta = Buffer.from(metaStr.slice(2), 'hex') + const data = Buffer.from(dataStr.slice(2), 'hex') const prefixLen = meta[prefLenIndex] const specs = multiformats.CID.inspectBytes(meta.slice(-prefixLen)) const hashLen = specs.digestSize @@ -135,15 +132,3 @@ lib.readCID = async (dmap, path) => { const packed = await lib.walk(dmap, path) return lib.unpackCID(packed.meta, packed.data) } - -lib._hexToArrayBuffer = hex => { - const bytes = [] - for (let c = 2; c < hex.length; c += 2) - bytes.push(parseInt(hex.slice(c, c + 2), 16)) - return new Uint8Array(bytes) -} - -lib._strToHex = str => { - let codes = str.split('').map(c => c.charCodeAt(0)) - return codes.map(c => c.toString(16)).join('') -} diff --git a/package.json b/package.json index 1ec1dcd..23ced1d 100644 --- a/package.json +++ b/package.json @@ -11,13 +11,16 @@ "@nomiclabs/hardhat-ethers": "^2.0.5", "@types/mocha": "^9.0.0", "hardhat": "^2.8.3", + "ipfs-core": "^0.14.2", + "ipfs-http-client": "^56.0.2", + "ipfs-provider": "^2.1.0", "minihat": "^0.0.5", - "v8": "^0.1.0", "web3-utils": "^1.7.1", "webpack": "^5.70.0", "webpack-cli": "^4.9.2" }, "dependencies": { - "ebnf": "^1.9.0" + "ebnf": "^1.9.0", + "node-polyfill-webpack-plugin": "^1.1.4" } } diff --git a/view/app.js b/view/app.js index 2947660..600ac70 100644 --- a/view/app.js +++ b/view/app.js @@ -1,14 +1,26 @@ import { ethers } from 'ethers' -const dpack = require('@etherpacks/dpack') +const { getIpfs, providers } = require('ipfs-provider') +const { httpClient, jsIpfs } = providers const dmap = require('../dmap.js') const dmapAddress = '0x7fA88e1014B0640833a03ACfEC71F242b5fBDC85' const dmapArtifact = require('../artifacts/core/dmap.sol/Dmap.json') -window.onload =()=> { +window.onload = async() => { const $ = document.querySelector.bind(document); - const result = $('#result') - const line =s=> { $('#result').textContent += s + '\n' } + const node = await getIpfs({ + providers: [ + // attempt to use local node, if unsuccessful fallback to running embedded core js-ipfs in-page + httpClient({ + loadHttpClientModule: () => require('ipfs-http-client'), + apiAddress: '/ip4/127.0.0.1/tcp/5001' + }), + jsIpfs({ + loadJsIpfsModule: () => require('ipfs-core'), + options: { } + }) + ] + }) $('#btnGet').addEventListener('click', async () => { const dpath = $('#dpath').value; @@ -32,16 +44,20 @@ window.onload =()=> { } try { - // display json content from a CID if we can + // display ipfs content from a CID if we can, otherwise display as text const cidResult = dmap.unpackCID(walkResult.meta, walkResult.data) line(`ipfs: ${cidResult}`) - const ipfsResult = await dpack.getIpfsJson(cidResult) - line(JSON.stringify(ipfsResult, null, 4)) + const ipfsResult = await node.ipfs.cat(cidResult) + let s = '' + let utf8decoder = new TextDecoder() + for await (const chunk of ipfsResult) { + s += utf8decoder.decode(chunk) + } + line(s) } catch(e){ - // otherwise show text let utf8decoder = new TextDecoder() - const bytes = dmap._hexToArrayBuffer(walkResult.data) + const bytes = Buffer.from(walkResult.data.slice(2), 'hex') let i for (i = 0; i < bytes.length; i++) { if (bytes[bytes.length -1 - i] !== 0) { diff --git a/webpack.config.js b/webpack.config.js index f74ce27..64dcd38 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,4 +1,5 @@ const path = require('path'); +const webpack = require('webpack'); module.exports = { mode: 'production', @@ -9,6 +10,12 @@ module.exports = { path: path.resolve(__dirname, 'dist'), filename: '[name].js', }, + plugins: [ + new webpack.ProvidePlugin({ + Buffer: ['buffer', 'Buffer'], + process: 'process/browser' + }) + ], optimization: { minimize: false, splitChunks: { From 651faf78e764868c8542a9668b5ff828e099f9b1 Mon Sep 17 00:00:00 2001 From: stobiewan Date: Tue, 19 Apr 2022 22:58:20 +1200 Subject: [PATCH 2/2] fallback to fetching from gateways add verification of ipfs content can input local node address remove ipfs-core and ipfs-provider --- dist/index.html | 17 +++++++++--- package.json | 5 +--- view/app.js | 71 ++++++++++++++++++++++++++++++----------------- webpack.config.js | 1 - 4 files changed, 60 insertions(+), 34 deletions(-) diff --git a/dist/index.html b/dist/index.html index 884dbbc..0e5b96d 100644 --- a/dist/index.html +++ b/dist/index.html @@ -12,15 +12,24 @@ #result { white-space: pre; } - + #localNode { + width: 210px; + }

dmap

- - - +
+ + +
+ +
+ + + +
diff --git a/package.json b/package.json index 23ced1d..ffa6759 100644 --- a/package.json +++ b/package.json @@ -11,16 +11,13 @@ "@nomiclabs/hardhat-ethers": "^2.0.5", "@types/mocha": "^9.0.0", "hardhat": "^2.8.3", - "ipfs-core": "^0.14.2", "ipfs-http-client": "^56.0.2", - "ipfs-provider": "^2.1.0", "minihat": "^0.0.5", "web3-utils": "^1.7.1", "webpack": "^5.70.0", "webpack-cli": "^4.9.2" }, "dependencies": { - "ebnf": "^1.9.0", - "node-polyfill-webpack-plugin": "^1.1.4" + "ebnf": "^1.9.0" } } diff --git a/view/app.js b/view/app.js index 600ac70..7d9eb7f 100644 --- a/view/app.js +++ b/view/app.js @@ -1,26 +1,51 @@ import { ethers } from 'ethers' -const { getIpfs, providers } = require('ipfs-provider') -const { httpClient, jsIpfs } = providers +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' const dmap = require('../dmap.js') const dmapAddress = '0x7fA88e1014B0640833a03ACfEC71F242b5fBDC85' const dmapArtifact = require('../artifacts/core/dmap.sol/Dmap.json') +const IPFS = require('ipfs-http-client') + +const gateways = ['https://ipfs.fleek.co/ipfs/', + 'https://gateway.pinata.cloud/ipfs/', + 'https://cloudflare-ipfs.com/ipfs/', + 'https://storry.tv/ipfs/', + 'https://ipfs.io/ipfs/', + 'https://hub.textile.io/ipfs/'] + +const resolveCID = async (cid, targetDigest, nodeAddress) => { + const verify = async bytes => { + const hash = await sha256.digest(bytes) + const resultDigest = JSON.stringify(hash.digest) + return targetDigest === resultDigest + } + const node = IPFS.create(nodeAddress) + const catResponse = await node.cat(cid) + // initially handle only single chunk verification and sha256 + try { + const chunk = await catResponse.next() + if(await verify(chunk.value)) { + return chunk.value + } + } catch(e) {} + + for (const gateway of gateways) { + const url = gateway + cid + try { + const response = await fetch(url); + const reader = response.body.getReader(); + let readRes = await reader.read(); + if (await verify(readRes.value)) { + return readRes.value + } + } catch (e) {} + } + throw 'unable to resolve cid' +} window.onload = async() => { const $ = document.querySelector.bind(document); const line =s=> { $('#result').textContent += s + '\n' } - const node = await getIpfs({ - providers: [ - // attempt to use local node, if unsuccessful fallback to running embedded core js-ipfs in-page - httpClient({ - loadHttpClientModule: () => require('ipfs-http-client'), - apiAddress: '/ip4/127.0.0.1/tcp/5001' - }), - jsIpfs({ - loadJsIpfsModule: () => require('ipfs-core'), - options: { } - }) - ] - }) $('#btnGet').addEventListener('click', async () => { const dpath = $('#dpath').value; @@ -45,21 +70,17 @@ window.onload = async() => { try { // display ipfs content from a CID if we can, otherwise display as text - const cidResult = dmap.unpackCID(walkResult.meta, walkResult.data) - line(`ipfs: ${cidResult}`) - const ipfsResult = await node.ipfs.cat(cidResult) - let s = '' + const cid = dmap.unpackCID(walkResult.meta, walkResult.data) + line(`ipfs: ${cid}`) + const targetDigest = JSON.stringify(CID.parse(cid).multihash.digest) + const resolved = await resolveCID(cid, targetDigest, $('#localNode').value) let utf8decoder = new TextDecoder() - for await (const chunk of ipfsResult) { - s += utf8decoder.decode(chunk) - } - line(s) + line(utf8decoder.decode(resolved)) } catch(e){ let utf8decoder = new TextDecoder() const bytes = Buffer.from(walkResult.data.slice(2), 'hex') - let i - for (i = 0; i < bytes.length; i++) { + for (var i = 0; i < bytes.length; i++) { if (bytes[bytes.length -1 - i] !== 0) { break } diff --git a/webpack.config.js b/webpack.config.js index 64dcd38..427efd1 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -13,7 +13,6 @@ module.exports = { plugins: [ new webpack.ProvidePlugin({ Buffer: ['buffer', 'Buffer'], - process: 'process/browser' }) ], optimization: {